diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8a5a4b02..7e4708d5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,6 +54,15 @@ jobs: run: | docker exec spindlenode bash -c 'cd Spindle-build/testsuite && ./runTests' + - name: Configure host core_pattern for coredumps + if: ${{ !env.ACT }} + run: | + sudo sysctl -w kernel.core_pattern='core.%p' + + - name: Run spindle-serial-ubuntu crash tests + run: | + docker exec spindlenode bash -c 'cd Spindle-build/testsuite && ./run_crash_tests.sh --launcher=serial --scratch=/tmp/spindle_crash_test' + - name: Bring spindle-serial-ubuntu down id: serial-ubuntu-down if: ${{ always() }} @@ -61,7 +70,7 @@ jobs: run: | cd containers/spindle-serial-ubuntu docker compose down - + spindle-serial-ubuntu_arm64: name: Testsuite (Serial, Ubuntu, arm64) environment: Spindle CI @@ -98,6 +107,15 @@ jobs: run: | docker exec spindlenode bash -c 'cd Spindle-build/testsuite && ./runTests' + - name: Configure host core_pattern for coredumps + if: ${{ !env.ACT }} + run: | + sudo sysctl -w kernel.core_pattern='core.%p' + + - name: Run spindle-serial-ubuntu crash tests + run: | + docker exec spindlenode bash -c 'cd Spindle-build/testsuite && ./run_crash_tests.sh --launcher=serial --scratch=/tmp/spindle_crash_test' + - name: Bring spindle-serial-ubuntu down id: serial-ubuntu-down if: ${{ always() }} @@ -142,6 +160,15 @@ jobs: run: | docker exec node-1 bash -c 'cd Spindle-build/testsuite && flux alloc --nodes=${workers} ./runTests --nodes=${workers} --tasks-per-node=3' + - name: Configure host core_pattern for coredumps + if: ${{ !env.ACT }} + run: | + sudo sysctl -w kernel.core_pattern='core.%p' + + - name: Run spindle-flux-ubuntu crash tests + run: | + docker exec node-1 bash -c 'cd Spindle-build/testsuite && flux alloc --nodes=${workers} ./run_crash_tests.sh --launcher=flux --nodes=${workers} --scratch=/shared/spindle_crash_test' + - name: Bring spindle-flux-ubuntu down id: flux-ubuntu-down if: ${{ always() }} @@ -200,6 +227,15 @@ jobs: run: | docker exec slurm-head bash -c 'cd Spindle-build/testsuite && salloc -n${workers} -N${workers} ./runTests ${workers}' + - name: Configure host core_pattern for coredumps + if: ${{ !env.ACT }} + run: | + sudo sysctl -w kernel.core_pattern='core.%p' + + - name: Run spindle-slurm-ubuntu crash tests + run: | + docker exec slurm-head bash -c 'cd Spindle-build/testsuite && ./run_crash_tests.sh --launcher=slurm --nodes=${workers} --scratch=/shared/spindle_crash_test' + - name: Bring spindle-slurm-ubuntu down id: slurm-ubuntu-down if: ${{ always() }} @@ -321,6 +357,15 @@ jobs: run: | docker exec slurm-plugin-head bash -c 'cd Spindle-build/testsuite && salloc -n${workers} -N${workers} --spindle-session ./runTests ${workers}' + - name: Configure host core_pattern for coredumps + if: ${{ !env.ACT }} + run: | + sudo sysctl -w kernel.core_pattern='core.%p' + + - name: Run spindle-slurm-plugin-ubuntu crash tests + run: | + docker exec slurm-plugin-head bash -c 'cd Spindle-build/testsuite && ./run_crash_tests.sh --launcher=slurm-plugin --nodes=${workers} --scratch=/shared/spindle_crash_test' + - name: Bring spindle-slurm-plugin-ubuntu down id: slurm-ubuntu-down if: ${{ always() }} diff --git a/config.h.in b/config.h.in index 121c551f..00b83ef3 100644 --- a/config.h.in +++ b/config.h.in @@ -27,6 +27,9 @@ /* Define if were using sockets for client/server communication */ #undef COMM_SOCKET +/* Enable --crash-dedup by default */ +#undef CRASH_DEDUP_ENABLED_BY_DEFAULT + /* Defined if GLIBC has an allocation bug */ #undef DTV_ALLOCATION_BUG diff --git a/configure b/configure index ac8d1434..6b54926e 100755 --- a/configure +++ b/configure @@ -879,6 +879,7 @@ enable_sec_munge enable_sec_keydir enable_sec_launchmon enable_sec_none +enable_crash_dedup with_gcrypt_dir with_munge_dir with_glibc_be_dir @@ -1576,6 +1577,7 @@ Optional Features: --enable-sec-launchmon Enable LaunchmMON to propagate authentication key for connections --enable-sec-none Disable security authentication of connections + --enable-crash-dedup Enable --crash-dedup by default (default: disabled) --enable-remap-pageone Controls mmap behavior when replacing executables. Should be false on older linux systems --enable-testsuite Chooses whether to build the Spindle testsuite. @@ -17658,6 +17660,20 @@ if test "${enable_sec_none+set}" = set; then : fi +# Check whether --enable-crash-dedup was given. +if test "${enable_crash_dedup+set}" = set; then : + enableval=$enable_crash_dedup; enable_crash_dedup=$enableval +else + enable_crash_dedup=no +fi + + +if test "x$enable_crash_dedup" = xyes; then : + +$as_echo "#define CRASH_DEDUP_ENABLED_BY_DEFAULT 1" >>confdefs.h + +fi + # Check whether --with-gcrypt-dir was given. if test "${with_gcrypt_dir+set}" = set; then : diff --git a/configure.common.ac b/configure.common.ac index baa9c0a9..1b187f2e 100644 --- a/configure.common.ac +++ b/configure.common.ac @@ -427,6 +427,16 @@ AC_ARG_ENABLE(sec-none, [AS_HELP_STRING([--enable-sec-none],[Disable security authentication of connections])], [WANT_NOSEC="true";EXPLICIT_SEC="true"],) +AC_ARG_ENABLE([crash-dedup], + [AS_HELP_STRING([--enable-crash-dedup], + [Enable --crash-dedup by default (default: disabled)])], + [enable_crash_dedup=$enableval], + [enable_crash_dedup=no]) + +AS_IF([test "x$enable_crash_dedup" = xyes], + [AC_DEFINE([CRASH_DEDUP_ENABLED_BY_DEFAULT], [1], + [Enable --crash-dedup by default])]) + AC_ARG_WITH(gcrypt-dir, [AS_HELP_STRING([--with-gcrypt-dir=DIR],[Look in DIR for gcrypt installation])], [GCRYPT_CFLAGS="-I${withval}/include";GCRYPT_LIBS="-L${withval}/lib"],) diff --git a/containers/spindle-flux-ubuntu/Dockerfile b/containers/spindle-flux-ubuntu/Dockerfile index 57badbfc..01cd3625 100644 --- a/containers/spindle-flux-ubuntu/Dockerfile +++ b/containers/spindle-flux-ubuntu/Dockerfile @@ -7,6 +7,7 @@ ENV workers=${replicas} USER root ENV TMPDIR=/tmp RUN echo 'TMPDIR="/tmp"' >> /etc/environment +ENV SPINDLE_TEST_CONTAINER=1 RUN DEBIAN_FRONTEND="noninteractive" apt-get update \ && apt-get -qq install -y --no-install-recommends \ @@ -26,6 +27,7 @@ RUN DEBIAN_FRONTEND="noninteractive" apt-get update \ libgcrypt20 \ libgcrypt20-dev \ gdb \ + libc6-dbg \ software-properties-common ARG USER=fluxuser diff --git a/containers/spindle-flux-ubuntu/docker-compose.yml b/containers/spindle-flux-ubuntu/docker-compose.yml index f2aaf4b4..5903e778 100644 --- a/containers/spindle-flux-ubuntu/docker-compose.yml +++ b/containers/spindle-flux-ubuntu/docker-compose.yml @@ -31,12 +31,19 @@ networks: flux: driver: bridge +# Shared scratch volume so run_crash_tests.sh can count coredumps +# produced by any rank on any node from a single location. +volumes: + crash_scratch: + # Common parameters for all nodes. x-shared-node-parameters: &shared-node-parameters build: *shared-build-context networks: - flux environment: *shared-environment + volumes: + - crash_scratch:/shared cap_add: - SYS_NICE # Required for libnuma diff --git a/containers/spindle-flux-ubuntu/scripts/build_spindle.sh b/containers/spindle-flux-ubuntu/scripts/build_spindle.sh index a63b5d8d..276c59a6 100755 --- a/containers/spindle-flux-ubuntu/scripts/build_spindle.sh +++ b/containers/spindle-flux-ubuntu/scripts/build_spindle.sh @@ -4,7 +4,7 @@ set -euxo pipefail mkdir -p /home/${USER}/Spindle-build cd /home/${USER}/Spindle-build -/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=flux --enable-flux-plugin --with-cachepaths=/tmp/commpath/cachepath --with-commpath=/tmp/commpath CFLAGS="-O2 -g" CXXFLAGS="-O2 -g" +/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=flux --enable-flux-plugin --with-cachepaths=/tmp/commpath/cachepath --with-commpath=/tmp/commpath --enable-crash-dedup CFLAGS="-O2 -g" CXXFLAGS="-O2 -g" make -j$(nproc) make install diff --git a/containers/spindle-flux-ubuntu/scripts/entrypoint.sh b/containers/spindle-flux-ubuntu/scripts/entrypoint.sh index 8d3944ef..95efffdf 100755 --- a/containers/spindle-flux-ubuntu/scripts/entrypoint.sh +++ b/containers/spindle-flux-ubuntu/scripts/entrypoint.sh @@ -22,6 +22,11 @@ thisHost=${thisHost[0]} echo $thisHost export FLUX_FAKE_HOSTNAME=$thisHost +if [ -d /shared ]; then + sudo chown -R "$(id -un):$(id -gn)" /shared + sudo chmod 755 /shared +fi + # Start munged sudo -u munge /usr/sbin/munged diff --git a/containers/spindle-serial-ubuntu/Dockerfile b/containers/spindle-serial-ubuntu/Dockerfile index 62c0cbf5..3573f243 100644 --- a/containers/spindle-serial-ubuntu/Dockerfile +++ b/containers/spindle-serial-ubuntu/Dockerfile @@ -3,6 +3,7 @@ FROM ubuntu:${ubuntu_version} USER root ENV TMPDIR=/tmp RUN echo 'TMPDIR="/tmp"' >> /etc/environment +ENV SPINDLE_TEST_CONTAINER=1 RUN DEBIAN_FRONTEND="noninteractive" apt-get update \ # install latest pkg utils: @@ -26,6 +27,8 @@ RUN DEBIAN_FRONTEND="noninteractive" apt-get -qq install -y --no-install-recomme gfortran-13 \ gcc-13 \ g++-13 \ + gdb \ + libc6-dbg \ munge \ libmunge-dev \ libhwloc-dev \ diff --git a/containers/spindle-serial-ubuntu/scripts/build_spindle.sh b/containers/spindle-serial-ubuntu/scripts/build_spindle.sh index 1022ba14..b48e3b53 100755 --- a/containers/spindle-serial-ubuntu/scripts/build_spindle.sh +++ b/containers/spindle-serial-ubuntu/scripts/build_spindle.sh @@ -4,7 +4,7 @@ set -euxo pipefail mkdir -p /home/${USER}/Spindle-build cd /home/${USER}/Spindle-build -/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=serial --with-cachepaths=/tmp/commpath/cachepath --with-commpath=/tmp/commpath CFLAGS="-O2 -g" CXXFLAGS="-O2 -g" +/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=serial --with-cachepaths=/tmp/commpath/cachepath --with-commpath=/tmp/commpath --enable-crash-dedup CFLAGS="-O2 -g" CXXFLAGS="-O2 -g" make -j$(nproc) make install diff --git a/containers/spindle-slurm-ubuntu/base/Dockerfile b/containers/spindle-slurm-ubuntu/base/Dockerfile index d6eee9f2..b571235c 100644 --- a/containers/spindle-slurm-ubuntu/base/Dockerfile +++ b/containers/spindle-slurm-ubuntu/base/Dockerfile @@ -27,6 +27,7 @@ RUN apt-get update \ gfortran-13 \ gcc-13 \ g++-13 \ + gdb \ munge \ libmunge-dev \ libhwloc-dev \ diff --git a/containers/spindle-slurm-ubuntu/testing-plugin/Dockerfile b/containers/spindle-slurm-ubuntu/testing-plugin/Dockerfile index 951480f1..a5d95356 100644 --- a/containers/spindle-slurm-ubuntu/testing-plugin/Dockerfile +++ b/containers/spindle-slurm-ubuntu/testing-plugin/Dockerfile @@ -4,9 +4,15 @@ ARG replicas=4 ENV workers=${replicas} ENV TMPDIR=/tmp RUN echo 'TMPDIR="/tmp"' >> /etc/environment +ENV SPINDLE_TEST_CONTAINER=1 ARG BUILD_ROOT=containers/spindle-slurm-ubuntu/testing-plugin +RUN apt-get update \ + && DEBIAN_FRONTEND="noninteractive" apt-get -qq install -y --no-install-recommends \ + gdb \ + libc6-dbg + # Slurm daemons run as $SLURM_USER ARG SLURM_USER=slurm diff --git a/containers/spindle-slurm-ubuntu/testing-plugin/docker-compose.yml b/containers/spindle-slurm-ubuntu/testing-plugin/docker-compose.yml index 2751a71a..309e64b9 100644 --- a/containers/spindle-slurm-ubuntu/testing-plugin/docker-compose.yml +++ b/containers/spindle-slurm-ubuntu/testing-plugin/docker-compose.yml @@ -35,13 +35,21 @@ networks: slurm: driver: bridge +# Shared scratch volume so run_crash_tests.sh can count coredumps +# written by ranks on any node from the head node. +volumes: + crash_scratch: + # Common parameters for all nodes. x-shared-node-parameters: &shared-node-parameters build: *shared-build-context networks: - slurm + volumes: + - crash_scratch:/shared cap_add: - SYS_NICE # Required for libnuma + - SYS_RESOURCE # Allow changing ulimit x-healthcheck-parameters: &healthcheck-parameters start_period: 3s diff --git a/containers/spindle-slurm-ubuntu/testing-plugin/scripts/build_spindle.sh b/containers/spindle-slurm-ubuntu/testing-plugin/scripts/build_spindle.sh index 2b36be90..8b759415 100755 --- a/containers/spindle-slurm-ubuntu/testing-plugin/scripts/build_spindle.sh +++ b/containers/spindle-slurm-ubuntu/testing-plugin/scripts/build_spindle.sh @@ -3,7 +3,7 @@ set -euxo pipefail mkdir -p /home/${USER}/Spindle-build cd /home/${USER}/Spindle-build -/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=slurm-plugin --enable-slurm-plugin --with-cachepaths=/tmp/commpath/cachepath --with-commpath=/tmp/commpath CFLAGS="-O2 -g" CXXFLAGS="-O2 -g" +/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=slurm-plugin --enable-slurm-plugin --with-cachepaths=/tmp/commpath/cachepath --with-commpath=/tmp/commpath --enable-crash-dedup CFLAGS="-O2 -g" CXXFLAGS="-O2 -g" make -j$(nproc) make install diff --git a/containers/spindle-slurm-ubuntu/testing-plugin/scripts/entrypoint.sh b/containers/spindle-slurm-ubuntu/testing-plugin/scripts/entrypoint.sh index 54d40e48..5ba29ac7 100755 --- a/containers/spindle-slurm-ubuntu/testing-plugin/scripts/entrypoint.sh +++ b/containers/spindle-slurm-ubuntu/testing-plugin/scripts/entrypoint.sh @@ -3,9 +3,14 @@ echo "SLURM_ROLE: ${SLURM_ROLE}" echo "Starting sshd..." -sudo service ssh start +sudo bash -c 'ulimit -c unlimited; service ssh start' echo "Starting munged..." -sudo -u munge /usr/sbin/munged +sudo -u munge /usr/sbin/munged + +if [ -d /shared ]; then + sudo chown -R "$(id -un):$(id -gn)" /shared + sudo chmod 755 /shared +fi if [ "${SLURM_ROLE}" = "db" ]; then echo "Starting slurmdbd..." @@ -15,7 +20,7 @@ elif [ "${SLURM_ROLE}" = "ctl" ] ; then sudo -u slurm /usr/sbin/slurmctld -i -Dvvv elif [ "${SLURM_ROLE}" = "worker" ] ; then echo "Starting slurmd..." - sudo /usr/sbin/slurmd -Dvvv + sudo bash -c 'ulimit -c unlimited; exec env -u PWD /usr/sbin/slurmd -Dvvv' fi sleep inf diff --git a/containers/spindle-slurm-ubuntu/testing/Dockerfile b/containers/spindle-slurm-ubuntu/testing/Dockerfile index 99768535..2d0c6202 100644 --- a/containers/spindle-slurm-ubuntu/testing/Dockerfile +++ b/containers/spindle-slurm-ubuntu/testing/Dockerfile @@ -4,9 +4,15 @@ ARG replicas=4 ENV workers=${replicas} ENV TMPDIR=/tmp RUN echo 'TMPDIR="/tmp"' >> /etc/environment +ENV SPINDLE_TEST_CONTAINER=1 ARG BUILD_ROOT=containers/spindle-slurm-ubuntu/testing +RUN apt-get update \ + && DEBIAN_FRONTEND="noninteractive" apt-get -qq install -y --no-install-recommends \ + gdb \ + libc6-dbg + # Slurm daemons run as $SLURM_USER ARG SLURM_USER=slurm diff --git a/containers/spindle-slurm-ubuntu/testing/docker-compose.yml b/containers/spindle-slurm-ubuntu/testing/docker-compose.yml index 5d3deb74..6f9ed0dc 100644 --- a/containers/spindle-slurm-ubuntu/testing/docker-compose.yml +++ b/containers/spindle-slurm-ubuntu/testing/docker-compose.yml @@ -35,13 +35,21 @@ networks: slurm: driver: bridge +# Shared scratch volume so run_crash_tests.sh can count coredumps +# written by ranks on any node from the head node. +volumes: + crash_scratch: + # Common parameters for all nodes. x-shared-node-parameters: &shared-node-parameters build: *shared-build-context networks: - slurm + volumes: + - crash_scratch:/shared cap_add: - SYS_NICE # Required for libnuma + - SYS_RESOURCE # Allow changing ulimit x-healthcheck-parameters: &healthcheck-parameters start_period: 3s diff --git a/containers/spindle-slurm-ubuntu/testing/scripts/build_spindle.sh b/containers/spindle-slurm-ubuntu/testing/scripts/build_spindle.sh index 7fcb48e3..4a4c5eee 100755 --- a/containers/spindle-slurm-ubuntu/testing/scripts/build_spindle.sh +++ b/containers/spindle-slurm-ubuntu/testing/scripts/build_spindle.sh @@ -3,7 +3,7 @@ set -euxo pipefail mkdir -p /home/${USER}/Spindle-build cd /home/${USER}/Spindle-build -/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=slurm --with-rsh-launch --with-rsh-cmd=/usr/bin/ssh --with-cachepaths=/tmp/commpath/cachepath --with-commpath=/tmp/commpath CFLAGS="-O2 -g" CXXFLAGS="-O2 -g" +/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=slurm --with-rsh-launch --with-rsh-cmd=/usr/bin/ssh --with-cachepaths=/tmp/commpath/cachepath --with-commpath=/tmp/commpath --enable-crash-dedup CFLAGS="-O2 -g" CXXFLAGS="-O2 -g" make -j$(nproc) make install diff --git a/containers/spindle-slurm-ubuntu/testing/scripts/entrypoint.sh b/containers/spindle-slurm-ubuntu/testing/scripts/entrypoint.sh index 54d40e48..0e2f1a90 100755 --- a/containers/spindle-slurm-ubuntu/testing/scripts/entrypoint.sh +++ b/containers/spindle-slurm-ubuntu/testing/scripts/entrypoint.sh @@ -3,9 +3,14 @@ echo "SLURM_ROLE: ${SLURM_ROLE}" echo "Starting sshd..." -sudo service ssh start +sudo bash -c 'ulimit -c unlimited; service ssh start' echo "Starting munged..." -sudo -u munge /usr/sbin/munged +sudo -u munge /usr/sbin/munged + +if [ -d /shared ]; then + sudo chown -R "$(id -un):$(id -gn)" /shared + sudo chmod 755 /shared +fi if [ "${SLURM_ROLE}" = "db" ]; then echo "Starting slurmdbd..." @@ -15,7 +20,7 @@ elif [ "${SLURM_ROLE}" = "ctl" ] ; then sudo -u slurm /usr/sbin/slurmctld -i -Dvvv elif [ "${SLURM_ROLE}" = "worker" ] ; then echo "Starting slurmd..." - sudo /usr/sbin/slurmd -Dvvv + sudo bash -c 'ulimit -c unlimited; exec /usr/sbin/slurmd -Dvvv' fi sleep inf diff --git a/doc/spindle_launch_README.md b/doc/spindle_launch_README.md index 38f45eea..a30c8df8 100644 --- a/doc/spindle_launch_README.md +++ b/doc/spindle_launch_README.md @@ -122,6 +122,10 @@ we have to break ABI compatibility. `spindleRunBE()`. Whether the daemons are launched via ssh or rsh can be controlled through command arguments, configure, or an environment variable (SPINDLE_RSH). + - 'OPT_CRASH_HANDLER' - When the application crashes (e.g. SIGSEGV), + Spindle deduplicates the reports by crash site, selecting one + representative per unique site, permitting only those representatives + to write a coredump. - `typedef struct { ... } spindle_args_t` diff --git a/src/client/auditclient/Makefile.am b/src/client/auditclient/Makefile.am index 70dead4b..94305b76 100644 --- a/src/client/auditclient/Makefile.am +++ b/src/client/auditclient/Makefile.am @@ -12,7 +12,7 @@ endif AM_CFLAGS = -fvisibility=hidden -AM_CPPFLAGS = -I$(top_srcdir)/../logging -I$(top_srcdir)/../include -I$(top_srcdir)/client -I$(top_srcdir)/client_comlib +AM_CPPFLAGS = -I$(top_srcdir)/../logging -I$(top_srcdir)/../include -I$(top_srcdir)/client -I$(top_srcdir)/client_comlib -I$(top_srcdir)/crash_handler BASE_SRCS = auditclient.c auditclient_common.c patch_linkmap.c redirect.c bindgot.c writablegot.c patch_bad_dtv.c malloc_wrapper.c fixlocale.c if X86_64_BLD diff --git a/src/client/auditclient/Makefile.in b/src/client/auditclient/Makefile.in index a972661a..a7bfbe96 100644 --- a/src/client/auditclient/Makefile.in +++ b/src/client/auditclient/Makefile.in @@ -408,7 +408,7 @@ top_builddir = @top_builddir@ top_srcdir = @top_srcdir@ pkglib_LTLIBRARIES = $(am__append_1) $(am__append_2) $(am__append_3) AM_CFLAGS = -fvisibility=hidden -AM_CPPFLAGS = -I$(top_srcdir)/../logging -I$(top_srcdir)/../include -I$(top_srcdir)/client -I$(top_srcdir)/client_comlib +AM_CPPFLAGS = -I$(top_srcdir)/../logging -I$(top_srcdir)/../include -I$(top_srcdir)/client -I$(top_srcdir)/client_comlib -I$(top_srcdir)/crash_handler BASE_SRCS = auditclient.c auditclient_common.c patch_linkmap.c redirect.c bindgot.c writablegot.c patch_bad_dtv.c malloc_wrapper.c fixlocale.c @AARCH64_BLD_TRUE@ARCH_SRCS = auditclient_aarch64.c @PPC64LE_BLD_TRUE@ARCH_SRCS = auditclient_ppc64.c diff --git a/src/client/client/Makefile.am b/src/client/client/Makefile.am index b6f03700..47085b75 100644 --- a/src/client/client/Makefile.am +++ b/src/client/client/Makefile.am @@ -13,11 +13,20 @@ endif AM_CFLAGS = -fvisibility=hidden -AM_CPPFLAGS = -I$(top_srcdir)/../logging -I$(top_srcdir)/client_comlib -I$(top_srcdir)/../include -I$(top_srcdir)/shm_cache -I$(top_srcdir)/subaudit -I$(top_srcdir)/../utils -I$(top_srcdir)/auditclient +AM_CPPFLAGS = -I$(top_srcdir)/../logging -I$(top_srcdir)/client_comlib -I$(top_srcdir)/../include -I$(top_srcdir)/shm_cache -I$(top_srcdir)/subaudit -I$(top_srcdir)/../utils -I$(top_srcdir)/auditclient -I$(top_srcdir)/client -I$(top_srcdir)/crash_handler INTERCEPT_SRCS = intercept_open.c intercept_exec.c intercept_stat.c intercept_readlink.c intercept_spindleapi.c intercept.c spindle_regex.c patch_interception.c realpath.c $(top_srcdir)/../utils/fileutil.c -BASE_SRCS = client.c lookup.c should_intercept.c exec_util.c remap_exec.c lookup_libc.c $(top_srcdir)/../utils/parseloc.c $(top_srcdir)/../utils/getcpu.c adjust_argv.c +BASE_SRCS = client.c \ + lookup.c should_intercept.c exec_util.c remap_exec.c \ + lookup_libc.c adjust_argv.c $(top_srcdir)/../utils/parseloc.c \ + $(top_srcdir)/../utils/getcpu.c + +BASE_SRCS += $(top_srcdir)/crash_handler/crash_io.c \ + $(top_srcdir)/crash_handler/crash_fmt.c \ + $(top_srcdir)/crash_handler/crash_handler.c \ + $(top_srcdir)/crash_handler/crash_lib_offset.c \ + $(top_srcdir)/crash_handler/crash_sigchain.c libspindlec_socket_la_SOURCES = $(BASE_SRCS) libspindlec_socket_la_LIBADD = $(top_builddir)/client_comlib/libclient_socket.la $(top_builddir)/logging/libspindleclogc.la $(top_builddir)/shm_cache/libshmcache.la diff --git a/src/client/client/Makefile.in b/src/client/client/Makefile.in index d0241599..d45c5c95 100644 --- a/src/client/client/Makefile.in +++ b/src/client/client/Makefile.in @@ -135,9 +135,14 @@ libspindlec_biter_la_DEPENDENCIES = \ $(top_builddir)/logging/libspindleclogc.la \ $(top_builddir)/shm_cache/libshmcache.la am__objects_2 = client.lo lookup.lo should_intercept.lo exec_util.lo \ - remap_exec.lo lookup_libc.lo \ + remap_exec.lo lookup_libc.lo adjust_argv.lo \ $(top_builddir)/../utils/parseloc.lo \ - $(top_builddir)/../utils/getcpu.lo adjust_argv.lo + $(top_builddir)/../utils/getcpu.lo \ + $(top_builddir)/crash_handler/crash_io.lo \ + $(top_builddir)/crash_handler/crash_fmt.lo \ + $(top_builddir)/crash_handler/crash_handler.lo \ + $(top_builddir)/crash_handler/crash_lib_offset.lo \ + $(top_builddir)/crash_handler/crash_sigchain.lo am_libspindlec_biter_la_OBJECTS = $(am__objects_2) libspindlec_biter_la_OBJECTS = $(am_libspindlec_biter_la_OBJECTS) @BITER_TRUE@am_libspindlec_biter_la_rpath = @@ -173,6 +178,11 @@ am__maybe_remake_depfiles = depfiles am__depfiles_remade = $(top_builddir)/../utils/$(DEPDIR)/getcpu.Plo \ $(top_builddir)/../utils/$(DEPDIR)/libspindle_audit_la-fileutil.Plo \ $(top_builddir)/../utils/$(DEPDIR)/parseloc.Plo \ + $(top_builddir)/crash_handler/$(DEPDIR)/crash_fmt.Plo \ + $(top_builddir)/crash_handler/$(DEPDIR)/crash_handler.Plo \ + $(top_builddir)/crash_handler/$(DEPDIR)/crash_io.Plo \ + $(top_builddir)/crash_handler/$(DEPDIR)/crash_lib_offset.Plo \ + $(top_builddir)/crash_handler/$(DEPDIR)/crash_sigchain.Plo \ ./$(DEPDIR)/adjust_argv.Plo ./$(DEPDIR)/client.Plo \ ./$(DEPDIR)/exec_util.Plo \ ./$(DEPDIR)/libspindle_audit_la-intercept.Plo \ @@ -375,9 +385,17 @@ top_srcdir = @top_srcdir@ noinst_LTLIBRARIES = libspindle_audit.la $(am__append_1) \ $(am__append_2) $(am__append_3) AM_CFLAGS = -fvisibility=hidden -AM_CPPFLAGS = -I$(top_srcdir)/../logging -I$(top_srcdir)/client_comlib -I$(top_srcdir)/../include -I$(top_srcdir)/shm_cache -I$(top_srcdir)/subaudit -I$(top_srcdir)/../utils -I$(top_srcdir)/auditclient +AM_CPPFLAGS = -I$(top_srcdir)/../logging -I$(top_srcdir)/client_comlib -I$(top_srcdir)/../include -I$(top_srcdir)/shm_cache -I$(top_srcdir)/subaudit -I$(top_srcdir)/../utils -I$(top_srcdir)/auditclient -I$(top_srcdir)/client -I$(top_srcdir)/crash_handler INTERCEPT_SRCS = intercept_open.c intercept_exec.c intercept_stat.c intercept_readlink.c intercept_spindleapi.c intercept.c spindle_regex.c patch_interception.c realpath.c $(top_srcdir)/../utils/fileutil.c -BASE_SRCS = client.c lookup.c should_intercept.c exec_util.c remap_exec.c lookup_libc.c $(top_srcdir)/../utils/parseloc.c $(top_srcdir)/../utils/getcpu.c adjust_argv.c +BASE_SRCS = client.c lookup.c should_intercept.c exec_util.c \ + remap_exec.c lookup_libc.c adjust_argv.c \ + $(top_srcdir)/../utils/parseloc.c \ + $(top_srcdir)/../utils/getcpu.c \ + $(top_srcdir)/crash_handler/crash_io.c \ + $(top_srcdir)/crash_handler/crash_fmt.c \ + $(top_srcdir)/crash_handler/crash_handler.c \ + $(top_srcdir)/crash_handler/crash_lib_offset.c \ + $(top_srcdir)/crash_handler/crash_sigchain.c libspindlec_socket_la_SOURCES = $(BASE_SRCS) libspindlec_socket_la_LIBADD = $(top_builddir)/client_comlib/libclient_socket.la $(top_builddir)/logging/libspindleclogc.la $(top_builddir)/shm_cache/libshmcache.la libspindlec_pipe_la_SOURCES = $(BASE_SRCS) @@ -448,6 +466,27 @@ $(top_builddir)/../utils/parseloc.lo: \ $(top_builddir)/../utils/getcpu.lo: \ $(top_builddir)/../utils/$(am__dirstamp) \ $(top_builddir)/../utils/$(DEPDIR)/$(am__dirstamp) +$(top_builddir)/crash_handler/$(am__dirstamp): + @$(MKDIR_P) $(top_builddir)/crash_handler + @: > $(top_builddir)/crash_handler/$(am__dirstamp) +$(top_builddir)/crash_handler/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) $(top_builddir)/crash_handler/$(DEPDIR) + @: > $(top_builddir)/crash_handler/$(DEPDIR)/$(am__dirstamp) +$(top_builddir)/crash_handler/crash_io.lo: \ + $(top_builddir)/crash_handler/$(am__dirstamp) \ + $(top_builddir)/crash_handler/$(DEPDIR)/$(am__dirstamp) +$(top_builddir)/crash_handler/crash_fmt.lo: \ + $(top_builddir)/crash_handler/$(am__dirstamp) \ + $(top_builddir)/crash_handler/$(DEPDIR)/$(am__dirstamp) +$(top_builddir)/crash_handler/crash_handler.lo: \ + $(top_builddir)/crash_handler/$(am__dirstamp) \ + $(top_builddir)/crash_handler/$(DEPDIR)/$(am__dirstamp) +$(top_builddir)/crash_handler/crash_lib_offset.lo: \ + $(top_builddir)/crash_handler/$(am__dirstamp) \ + $(top_builddir)/crash_handler/$(DEPDIR)/$(am__dirstamp) +$(top_builddir)/crash_handler/crash_sigchain.lo: \ + $(top_builddir)/crash_handler/$(am__dirstamp) \ + $(top_builddir)/crash_handler/$(DEPDIR)/$(am__dirstamp) libspindlec_biter.la: $(libspindlec_biter_la_OBJECTS) $(libspindlec_biter_la_DEPENDENCIES) $(EXTRA_libspindlec_biter_la_DEPENDENCIES) $(AM_V_CCLD)$(LINK) $(am_libspindlec_biter_la_rpath) $(libspindlec_biter_la_OBJECTS) $(libspindlec_biter_la_LIBADD) $(LIBS) @@ -462,6 +501,8 @@ mostlyclean-compile: -rm -f *.$(OBJEXT) -rm -f $(top_builddir)/../utils/*.$(OBJEXT) -rm -f $(top_builddir)/../utils/*.lo + -rm -f $(top_builddir)/crash_handler/*.$(OBJEXT) + -rm -f $(top_builddir)/crash_handler/*.lo distclean-compile: -rm -f *.tab.c @@ -469,6 +510,11 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@$(top_builddir)/../utils/$(DEPDIR)/getcpu.Plo@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@$(top_builddir)/../utils/$(DEPDIR)/libspindle_audit_la-fileutil.Plo@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@$(top_builddir)/../utils/$(DEPDIR)/parseloc.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@$(top_builddir)/crash_handler/$(DEPDIR)/crash_fmt.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@$(top_builddir)/crash_handler/$(DEPDIR)/crash_handler.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@$(top_builddir)/crash_handler/$(DEPDIR)/crash_io.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@$(top_builddir)/crash_handler/$(DEPDIR)/crash_lib_offset.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@$(top_builddir)/crash_handler/$(DEPDIR)/crash_sigchain.Plo@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/adjust_argv.Plo@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/client.Plo@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/exec_util.Plo@am__quote@ # am--include-marker @@ -591,6 +637,7 @@ mostlyclean-libtool: clean-libtool: -rm -rf $(top_builddir)/../utils/.libs $(top_builddir)/../utils/_libs + -rm -rf $(top_builddir)/crash_handler/.libs $(top_builddir)/crash_handler/_libs -rm -rf .libs _libs ID: $(am__tagged_files) @@ -710,6 +757,8 @@ distclean-generic: -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES) -test -z "$(top_builddir)/../utils/$(DEPDIR)/$(am__dirstamp)" || rm -f $(top_builddir)/../utils/$(DEPDIR)/$(am__dirstamp) -test -z "$(top_builddir)/../utils/$(am__dirstamp)" || rm -f $(top_builddir)/../utils/$(am__dirstamp) + -test -z "$(top_builddir)/crash_handler/$(DEPDIR)/$(am__dirstamp)" || rm -f $(top_builddir)/crash_handler/$(DEPDIR)/$(am__dirstamp) + -test -z "$(top_builddir)/crash_handler/$(am__dirstamp)" || rm -f $(top_builddir)/crash_handler/$(am__dirstamp) maintainer-clean-generic: @echo "This command is intended for maintainers to use" @@ -723,6 +772,11 @@ distclean: distclean-am -rm -f $(top_builddir)/../utils/$(DEPDIR)/getcpu.Plo -rm -f $(top_builddir)/../utils/$(DEPDIR)/libspindle_audit_la-fileutil.Plo -rm -f $(top_builddir)/../utils/$(DEPDIR)/parseloc.Plo + -rm -f $(top_builddir)/crash_handler/$(DEPDIR)/crash_fmt.Plo + -rm -f $(top_builddir)/crash_handler/$(DEPDIR)/crash_handler.Plo + -rm -f $(top_builddir)/crash_handler/$(DEPDIR)/crash_io.Plo + -rm -f $(top_builddir)/crash_handler/$(DEPDIR)/crash_lib_offset.Plo + -rm -f $(top_builddir)/crash_handler/$(DEPDIR)/crash_sigchain.Plo -rm -f ./$(DEPDIR)/adjust_argv.Plo -rm -f ./$(DEPDIR)/client.Plo -rm -f ./$(DEPDIR)/exec_util.Plo @@ -787,6 +841,11 @@ maintainer-clean: maintainer-clean-am -rm -f $(top_builddir)/../utils/$(DEPDIR)/getcpu.Plo -rm -f $(top_builddir)/../utils/$(DEPDIR)/libspindle_audit_la-fileutil.Plo -rm -f $(top_builddir)/../utils/$(DEPDIR)/parseloc.Plo + -rm -f $(top_builddir)/crash_handler/$(DEPDIR)/crash_fmt.Plo + -rm -f $(top_builddir)/crash_handler/$(DEPDIR)/crash_handler.Plo + -rm -f $(top_builddir)/crash_handler/$(DEPDIR)/crash_io.Plo + -rm -f $(top_builddir)/crash_handler/$(DEPDIR)/crash_lib_offset.Plo + -rm -f $(top_builddir)/crash_handler/$(DEPDIR)/crash_sigchain.Plo -rm -f ./$(DEPDIR)/adjust_argv.Plo -rm -f ./$(DEPDIR)/client.Plo -rm -f ./$(DEPDIR)/exec_util.Plo diff --git a/src/client/client/client.c b/src/client/client/client.c index 1e2d5705..7e5a32a9 100644 --- a/src/client/client/client.c +++ b/src/client/client/client.c @@ -30,11 +30,12 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include #include -#include "ldcs_api.h" +#include "ldcs_api.h" #include "config.h" #include "client.h" #include "client_heap.h" #include "client_api.h" +#include "crash_handler.h" #include "spindle_launch.h" #include "shmcache.h" #include "ccwarns.h" @@ -364,6 +365,12 @@ int client_init() return -1; init_server_connection(); + + if (opts & OPT_CRASH_HANDLER) { + int global_rank = rankinfo[0] * rankinfo[3] + rankinfo[2]; + (void) crash_handler_install(global_rank, ldcsid); + } + intercept_open = (opts & OPT_RELOCPY) ? 1 : 0; intercept_stat = (opts & OPT_RELOCPY || !(opts & OPT_NOHIDE)) ? 1 : 0; intercept_exec = (opts & OPT_RELOCEXEC) ? 1 : 0; diff --git a/src/client/client/client.h b/src/client/client/client.h index 67f1842f..7004d577 100644 --- a/src/client/client/client.h +++ b/src/client/client/client.h @@ -107,6 +107,8 @@ extern void int_spindle_test_log_msg(char *buffer); typedef void* (*malloc_sig_t)(size_t); malloc_sig_t get_libc_malloc(); +void *get_libc_abort_msg(); + int lookup_libc_symbols(); /* ERRNO_NAME currently refers to a glibc internal symbol. */ diff --git a/src/client/client/intercept.c b/src/client/client/intercept.c index ea73253e..a6bb9d7a 100644 --- a/src/client/client/intercept.c +++ b/src/client/client/intercept.c @@ -20,6 +20,8 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include #include "intercept.h" #include "client.h" +#include "config.h" +#include "crash_sigchain.h" struct spindle_binding_t spindle_bindings[] = { { "", NULL, "", NULL }, @@ -45,6 +47,10 @@ struct spindle_binding_t spindle_bindings[] = { { "execvp", (void **) &orig_execvp, "execvp_wrapper", (void *) execvp_wrapper }, { "execvpe", (void **) &orig_execvpe, "execvpe_wrapper", (void *) execvpe_wrapper }, { "vfork", (void **) &orig_vfork, "vfork_wrapper", (void *) vfork_wrapper }, + { "sigaction", (void **) &orig_sigaction, "sigaction_wrapper", (void *) sigaction_wrapper }, + { "signal", (void **) &orig_signal, "signal_wrapper", (void *) signal_wrapper }, + { "bsd_signal", (void **) &orig_bsd_signal, "bsd_signal_wrapper", (void *) bsd_signal_wrapper }, + { "sysv_signal", (void **) &orig_sysv_signal, "sysv_signal_wrapper", (void *) sysv_signal_wrapper }, { "readlink", (void **) &orig_readlink, "readlink_wrapper", (void *) readlink_wrapper }, { "readlinkat", (void **) &orig_readlinkat, "readlinkat_wrapper", (void *) readlinkat_wrapper }, { "realpath", (void **) &orig_realpath, "spindle_realpath", (void *) spindle_realpath }, diff --git a/src/client/client/intercept.h b/src/client/client/intercept.h index 4ace2328..3442ef5d 100644 --- a/src/client/client/intercept.h +++ b/src/client/client/intercept.h @@ -54,6 +54,11 @@ extern int (*orig_unsetenv)(const char *name); extern char* (*orig_dlerror)(); extern char* (*orig_realpath)(const char *, char *); +extern void *orig_sigaction; +extern void *orig_signal; +extern void *orig_bsd_signal; +extern void *orig_sysv_signal; + extern int (*orig_spindle_open)(const char *pathname, int flags, ...); extern FILE* (*orig_spindle_fopen)(const char *path, const char *opts); extern int (*orig_spindle_stat)(const char *path, struct stat *buf); diff --git a/src/client/client/lookup_libc.c b/src/client/client/lookup_libc.c index f25497be..51344dfa 100644 --- a/src/client/client/lookup_libc.c +++ b/src/client/client/lookup_libc.c @@ -47,6 +47,8 @@ struct gnu_hash_header { static malloc_sig_t mallocfunc = NULL; +static void *volatile abort_msg_addr = NULL; + static signed long lookup_gnu_hash_symbol(const char *name, ElfW(Sym) *syms, char *symnames, struct gnu_hash_header *header) { uint32_t *buckets, *vals; @@ -214,7 +216,22 @@ int lookup_libc_symbols() mallocfunc = (malloc_sig_t) (symtab[result].st_value + libc->l_addr); debug_printf3("Bound mallocfunc to %p\n", mallocfunc); found++; - } + } + + result = -1; + if (gnu_hash) + result = lookup_gnu_hash_symbol("__abort_msg", symtab, strtab, (struct gnu_hash_header *) gnu_hash); + if (elf_hash && result == -1) + result = lookup_elf_hash_symbol("__abort_msg", symtab, strtab, (ElfW(Word) *) elf_hash); + if (result == -1) { + debug_printf3("Warning, Could not find symbol __abort_msg in libc\n"); + not_found++; + } + else { + abort_msg_addr = (void *) (symtab[result].st_value + libc->l_addr); + debug_printf3("Bound __abort_msg slot to %p\n", abort_msg_addr); + found++; + } } libdl_result = lookup_libdl_symbols(); @@ -282,3 +299,8 @@ malloc_sig_t get_libc_malloc() lookup_libc_symbols(); return mallocfunc; } + +void *get_libc_abort_msg() +{ + return abort_msg_addr; +} diff --git a/src/client/client_comlib/client_api.c b/src/client/client_comlib/client_api.c index cec31051..07735226 100644 --- a/src/client/client_comlib/client_api.c +++ b/src/client/client_comlib/client_api.c @@ -40,42 +40,31 @@ static struct lock_t comm_lock; int send_cachepath_query( int fd, char **chosen_realized_cachepath, char **chosen_parsed_cachepath){ - int retries = 0, max_retries = 1000, rc = 0; - struct timespec delay_between_retries = { .tv_sec = 0, .tv_nsec = 1000000 }; + int rc = 0; ldcs_message_t message; - char buffer[MAX_PATH_LEN+1]; + char buffer[2*(MAX_PATH_LEN+1)]; buffer[MAX_PATH_LEN] = '\0'; - do{ - message.header.type = LDCS_MSG_CHOSEN_CACHEPATH_REQUEST; - message.header.len = 0; - message.data = buffer; - - COMM_LOCK; - - debug_printf3("sending message of type: CHOSEN_CACHEPATH_REQUEST.\n" ); - rc = client_send_msg(fd, &message); - if( rc != 0 ){ - return rc; - } - rc = client_recv_msg_static(fd, &message, LDCS_READ_BLOCK); - if( rc != 0 ){ - return rc; - } - - COMM_UNLOCK; - - if( message.header.type == LDCS_MSG_NO_CACHEPATH_CONSENSUS_YET ){ - if( retries++ >= max_retries ){ - break; - } - nanosleep( &delay_between_retries, NULL ); - continue; - } - break; + message.header.type = LDCS_MSG_CHOSEN_CACHEPATH_REQUEST; + message.header.len = 0; + message.data = buffer; + + debug_printf3("sending message of type: CHOSEN_CACHEPATH_REQUEST.\n" ); + COMM_LOCK; - }while( 1 ); + rc = client_send_msg(fd, &message); + if( rc != 0 ){ + COMM_UNLOCK; + return rc; + } + rc = client_recv_msg_static(fd, &message, LDCS_READ_BLOCK); + if( rc != 0 ){ + COMM_UNLOCK; + return rc; + } + + COMM_UNLOCK; if (message.header.type != LDCS_MSG_CHOSEN_CACHEPATH || message.header.len > MAX_PATH_LEN) { err_printf("Got unexpected message of type %d\n", (int) message.header.type); diff --git a/src/client/client_comlib/client_api.h b/src/client/client_comlib/client_api.h index 3d7c41be..63f92c7a 100644 --- a/src/client/client_comlib/client_api.h +++ b/src/client/client_comlib/client_api.h @@ -54,6 +54,7 @@ char *client_get_connection_string(int fd); int client_send_msg(int connid, ldcs_message_t * msg); int client_recv_msg_static(int fd, ldcs_message_t *msg, ldcs_read_block_t block); int client_recv_msg_dynamic(int fd, ldcs_message_t *msg, ldcs_read_block_t block); +int client_get_raw_fds(int fd, int *read_fd, int *write_fd); int is_client_fd(int connfd, int fd); #endif diff --git a/src/client/client_comlib/client_api_pipe.c b/src/client/client_comlib/client_api_pipe.c index ea57efe5..7f3df457 100644 --- a/src/client/client_comlib/client_api_pipe.c +++ b/src/client/client_comlib/client_api_pipe.c @@ -381,6 +381,21 @@ int is_client_fd(int connfd, int fd) return (fdlist_pipe[connfd].in_fd == fd || fdlist_pipe[connfd].out_fd == fd); } +/* Get raw FDs for use in signal handlers. + We can't use the send/recv functions here in signal handlers + because they are not async-signal-safe. */ +int client_get_raw_fds_pipe(int fd, int *read_fd, int *write_fd) +{ + if (fd < 0 || fd >= MAX_FD) { + *read_fd = -1; + *write_fd = -1; + return -1; + } + *read_fd = fdlist_pipe[fd].in_fd; + *write_fd = fdlist_pipe[fd].out_fd; + return 0; +} + int client_close_connection_pipe(int fd) { int result; diff --git a/src/client/client_comlib/client_wrappers.c b/src/client/client_comlib/client_wrappers.c index 4b73718c..ff8488ec 100644 --- a/src/client/client_comlib/client_wrappers.c +++ b/src/client/client_comlib/client_wrappers.c @@ -43,6 +43,7 @@ extern char *RENAME(client_get_connection_string) (int fd); extern int RENAME(client_send_msg) (int connid, ldcs_message_t * msg); extern int RENAME(client_recv_msg_static) (int fd, ldcs_message_t *msg, ldcs_read_block_t block); extern int RENAME(client_recv_msg_dynamic) (int fd, ldcs_message_t *msg, ldcs_read_block_t block); +extern int RENAME(client_get_raw_fds) (int fd, int *read_fd, int *write_fd); int client_open_connection(char* location, number_t number) { @@ -79,3 +80,8 @@ int client_recv_msg_dynamic(int fd, ldcs_message_t *msg, ldcs_read_block_t block return RENAME(client_recv_msg_dynamic) (fd, msg, block); } +int client_get_raw_fds(int fd, int *read_fd, int *write_fd) +{ + return RENAME(client_get_raw_fds) (fd, read_fd, write_fd); +} + diff --git a/src/client/config.h.in b/src/client/config.h.in index 044ca9e1..872b9278 100644 --- a/src/client/config.h.in +++ b/src/client/config.h.in @@ -27,6 +27,9 @@ /* Define if were using sockets for client/server communication */ #undef COMM_SOCKET +/* Enable --crash-dedup by default */ +#undef CRASH_DEDUP_ENABLED_BY_DEFAULT + /* Defined if GLIBC has an allocation bug */ #undef DTV_ALLOCATION_BUG diff --git a/src/client/configure b/src/client/configure index eb84ee07..18f13fa5 100755 --- a/src/client/configure +++ b/src/client/configure @@ -842,6 +842,7 @@ enable_sec_munge enable_sec_keydir enable_sec_launchmon enable_sec_none +enable_crash_dedup with_gcrypt_dir with_munge_dir with_glibc_be_dir @@ -1520,6 +1521,7 @@ Optional Features: --enable-sec-launchmon Enable LaunchmMON to propagate authentication key for connections --enable-sec-none Disable security authentication of connections + --enable-crash-dedup Enable --crash-dedup by default (default: disabled) --enable-remap-pageone Controls mmap behavior when replacing executables. Should be false on older linux systems @@ -13706,6 +13708,20 @@ if test "${enable_sec_none+set}" = set; then : fi +# Check whether --enable-crash-dedup was given. +if test "${enable_crash_dedup+set}" = set; then : + enableval=$enable_crash_dedup; enable_crash_dedup=$enableval +else + enable_crash_dedup=no +fi + + +if test "x$enable_crash_dedup" = xyes; then : + +$as_echo "#define CRASH_DEDUP_ENABLED_BY_DEFAULT 1" >>confdefs.h + +fi + # Check whether --with-gcrypt-dir was given. if test "${with_gcrypt_dir+set}" = set; then : diff --git a/src/client/crash_handler/crash_arch.h b/src/client/crash_handler/crash_arch.h new file mode 100644 index 00000000..8334e75f --- /dev/null +++ b/src/client/crash_handler/crash_arch.h @@ -0,0 +1,85 @@ +/* +This file is part of Spindle. For copyright information see the COPYRIGHT +file in the top level directory, or at +https://github.com/hpc/Spindle/blob/master/COPYRIGHT + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU Lesser General Public License (as published by the Free Software +Foundation) version 2.1 dated February 1999. This program is distributed in the +hope that it will be useful, but WITHOUT ANY WARRANTY; without even the IMPLIED +WARRANTY OF MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms +and conditions of the GNU Lesser General Public License for more details. You should +have received a copy of the GNU Lesser General Public License along with this +program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA +*/ + +#if !defined(CRASH_ARCH_H_) +#define CRASH_ARCH_H_ + +#include "config.h" + +#include +#if defined(arch_aarch64) +#include +#endif + +static inline unsigned long extract_pc(void *uctx) +{ + ucontext_t *uc = (ucontext_t *) uctx; +#if defined(arch_x86_64) + return (unsigned long) uc->uc_mcontext.gregs[REG_RIP]; +#elif defined(arch_aarch64) + return (unsigned long) uc->uc_mcontext.pc; +#elif defined(arch_ppc64) || defined(arch_ppc64le) + return (unsigned long) uc->uc_mcontext.gp_regs[32]; +#else +#error "extract_pc: unsupported architecture" +#endif +} + +/* Unlike x86-64 and ppc64le, for aarch64, whether the fault was a read or a write + * is not found in a default register as a direct field of uc_mcontext. + * The relevant bit is found in the ESR (Exception Syndrome Register). + * The register is accessed by iterating over the extended context in __reserved + * to find the tag ESR_MAGIC; then the next entry is the esr_context. + * See https://blog.linuxplumbersconf.org/2017/ocw/system/presentations/4671/original/plumbers-dm-2017.pdf + * for info on the aarch64 extended context, and see + * https://developer.arm.com/documentation/ddi0595/2020-12/AArch64-Registers/ESR-EL1--Exception-Syndrome-Register--EL1- + * for info on the Exception Syndrome Register. */ +#if defined(arch_aarch64) +static inline const struct _aarch64_ctx *first_aarch64_ctx(const ucontext_t *uc) +{ + return (const struct _aarch64_ctx *) uc->uc_mcontext.__reserved; +} + +static inline const struct _aarch64_ctx *next_aarch64_ctx(const struct _aarch64_ctx *hdr) +{ + return (const struct _aarch64_ctx *) ((const char *) hdr + hdr->size); +} + +static inline int esr_is_write(unsigned long long esr) +{ + return ((esr >> 27) & 0x1f) == 0x12 && ((esr >> 6) & 1) != 0; +} +#endif /* arch_aarch64 */ + +static inline int pf_is_write(void *uctx) +{ + ucontext_t *uc = (ucontext_t *) uctx; +#if defined(arch_x86_64) + return (uc->uc_mcontext.gregs[REG_ERR] & 0x2UL) != 0; +#elif defined(arch_aarch64) + for (const struct _aarch64_ctx *hdr = first_aarch64_ctx(uc); hdr->magic; hdr = next_aarch64_ctx(hdr)) { + if (hdr->magic == ESR_MAGIC) + return esr_is_write(((const struct esr_context *) hdr)->esr); + } + return 0; +#elif defined(arch_ppc64) || defined(arch_ppc64le) + return (uc->uc_mcontext.gp_regs[42] & 0x02000000UL) != 0; +#else +#error "pf_is_write: unsupported architecture" +#endif +} + +#endif diff --git a/src/client/crash_handler/crash_fmt.c b/src/client/crash_handler/crash_fmt.c new file mode 100644 index 00000000..4e2b0ff9 --- /dev/null +++ b/src/client/crash_handler/crash_fmt.c @@ -0,0 +1,90 @@ +/* +This file is part of Spindle. For copyright information see the COPYRIGHT +file in the top level directory, or at +https://github.com/hpc/Spindle/blob/master/COPYRIGHT + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU Lesser General Public License (as published by the Free Software +Foundation) version 2.1 dated February 1999. This program is distributed in the +hope that it will be useful, but WITHOUT ANY WARRANTY; without even the IMPLIED +WARRANTY OF MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms +and conditions of the GNU Lesser General Public License for more details. You should +have received a copy of the GNU Lesser General Public License along with this +program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA +*/ + +#include "crash_fmt.h" + +#include + +/* This file contains async-signal-safe formatting + routines for use in the crash handler. The printf family + of functions are not async-signal-safe. */ + +static size_t crash_fmt_append_str(char *buf, size_t pos, size_t cap, + const char *s) +{ + size_t i = 0; + while (s[i]) { + if (pos + 1 >= cap) + return (size_t) -1; + buf[pos++] = s[i++]; + } + return pos; +} + +static size_t crash_fmt_append_hex(char *buf, size_t pos, size_t cap, + unsigned long v) +{ + char tmp[16]; + int i = 0; + + if (v == 0) { + tmp[i++] = '0'; + } else { + while (v && i < (int) sizeof(tmp)) { + unsigned d = (unsigned) (v & 0xFu); + tmp[i++] = (char) (d < 10 ? '0' + d : 'a' + (d - 10)); + v >>= 4; + } + } + + if (pos + (size_t) i + 1 > cap) + return (size_t) -1; + while (i > 0) + buf[pos++] = tmp[--i]; + return pos; +} + +/* Generates + crash site strings. */ +int crash_fmt_lib_offset(char *buf, size_t buflen, const char *lib, + unsigned long offset) +{ + size_t pos = 0; + + if (buflen == 0) + return -1; + + if (lib[0] != '\0') { + pos = crash_fmt_append_str(buf, pos, buflen, lib); + if (pos == (size_t) -1) + return -1; + if (pos + 1 >= buflen) + return -1; + buf[pos++] = '+'; + } + + pos = crash_fmt_append_str(buf, pos, buflen, "0x"); + if (pos == (size_t) -1) + return -1; + + pos = crash_fmt_append_hex(buf, pos, buflen, offset); + if (pos == (size_t) -1) + return -1; + + if (pos >= buflen) + return -1; + buf[pos] = '\0'; + return 0; +} diff --git a/src/client/crash_handler/crash_fmt.h b/src/client/crash_handler/crash_fmt.h new file mode 100644 index 00000000..7ae87a40 --- /dev/null +++ b/src/client/crash_handler/crash_fmt.h @@ -0,0 +1,26 @@ +/* +This file is part of Spindle. For copyright information see the COPYRIGHT +file in the top level directory, or at +https://github.com/hpc/Spindle/blob/master/COPYRIGHT + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU Lesser General Public License (as published by the Free Software +Foundation) version 2.1 dated February 1999. This program is distributed in the +hope that it will be useful, but WITHOUT ANY WARRANTY; without even the IMPLIED +WARRANTY OF MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms +and conditions of the GNU Lesser General Public License for more details. You should +have received a copy of the GNU Lesser General Public License along with this +program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA +*/ + +#if !defined(CRASH_FMT_H_) +#define CRASH_FMT_H_ + +#include + +/* Format a library+offset pair into buf as "+0x\0". */ +int crash_fmt_lib_offset(char *buf, size_t buflen, const char *lib, + unsigned long offset); + +#endif diff --git a/src/client/crash_handler/crash_handler.c b/src/client/crash_handler/crash_handler.c new file mode 100644 index 00000000..8482a30c --- /dev/null +++ b/src/client/crash_handler/crash_handler.c @@ -0,0 +1,325 @@ +/* +This file is part of Spindle. For copyright information see the COPYRIGHT +file in the top level directory, or at +https://github.com/hpc/Spindle/blob/master/COPYRIGHT + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU Lesser General Public License (as published by the Free Software +Foundation) version 2.1 dated February 1999. This program is distributed in the +hope that it will be useful, but WITHOUT ANY WARRANTY; without even the IMPLIED +WARRANTY OF MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms +and conditions of the GNU Lesser General Public License for more details. You should +have received a copy of the GNU Lesser General Public License along with this +program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA +*/ + +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "config.h" +#include "ldcs_api.h" +#include "client.h" +#include "client_api.h" +#include "crash_handler.h" +#include "crash_arch.h" +#include "crash_fmt.h" +#include "crash_io.h" +#include "crash_lib_offset.h" +#include "crash_sigchain.h" + +#define CRASH_ALTSTACK_SIZE 65536 +#define CRASH_SITE_BUF_SIZE (PATH_MAX + 32) +#define CRASH_REQ_BUF_SIZE \ + (sizeof(ldcs_message_header_t) + 2 * sizeof(int32_t) + CRASH_SITE_BUF_SIZE) +#define CRASH_ABORT_MSG_MAX (64u * 1024u) + +static int crash_global_rank = -1; +static int crash_read_fd = -1; +static int crash_write_fd = -1; +static int crash_installed = 0; + +static char *crash_altstack_buf = NULL; +static char crash_site_buf[CRASH_SITE_BUF_SIZE]; + +static volatile sig_atomic_t handler_active = 0; + +/* abort_msg_s is not defined in any glibc public header + so we provide our own here. If glibc changes this in + a future version, this will have to change here too. */ +struct abort_msg_s { + unsigned int size; + char msg[]; +}; + +/* Given the program counter pc, puts + + into buffer buf of size buflen */ +static void resolve_pc_to_crash_site(unsigned long pc, char *buf, size_t buflen) +{ + if (buflen == 0) return; + buf[0] = '\0'; + if (crash_lib_offset_get_signal_safe(pc, buf, buflen) == 0) + return; + (void) crash_fmt_lib_offset(buf, buflen, "", pc); +} + +/* Builds the crash report message */ +static size_t build_crash_report(char *buf, size_t buflen, int rank, + const char *site) +{ + size_t name_len = strlen(site) + 1; + size_t payload_len = 2 * sizeof(int32_t) + name_len; + size_t total_len = sizeof(ldcs_message_header_t) + payload_len; + if (total_len > buflen) + return 0; + + ldcs_message_header_t hdr; + hdr.type = LDCS_MSG_CRASH_REPORT; + hdr.len = payload_len; + memcpy(buf, &hdr, sizeof(hdr)); + + char *payload = buf + sizeof(hdr); + int32_t rank32 = (int32_t) rank; + int32_t nlen32 = (int32_t) name_len; + memcpy(payload, &rank32, sizeof(rank32)); + memcpy(payload + sizeof(int32_t), &nlen32, sizeof(nlen32)); + memcpy(payload + 2 * sizeof(int32_t), site, name_len); + return total_len; +} + +/* Attempts to read the value of the libc __abort_msg into + buffer buf of size buflen. */ +static size_t read_abort_msg(char *buf, size_t buflen) +{ + struct abort_msg_s *volatile *abort_msg_loc = (struct abort_msg_s *volatile *) get_libc_abort_msg(); + + if (buflen == 0) return 0; + if (abort_msg_loc == NULL) return 0; // We failed to find the symbol + + struct abort_msg_s *p = *abort_msg_loc; + if (p == NULL) return 0; + if (p->size <= sizeof(unsigned int)) return 0; // if not bigger than the header, there is no abort_msg + if (p->size > CRASH_ABORT_MSG_MAX) return 0; + + size_t msg_size = (size_t) p->size - sizeof(unsigned int); + size_t cap = msg_size < buflen - 1 ? msg_size : buflen - 1; + size_t n = 0; + while (n < cap && p->msg[n] != '\0') { + buf[n] = p->msg[n]; + n++; + } + buf[n] = '\0'; + return n; +} + +/* Builds the crash site string according to the signal type. + For SIGABRT, use the abort_msg; otherwise, +. */ +static void crash_build_site(int sig, unsigned long pc, + char *buf, size_t buflen) +{ + if (sig == SIGABRT) { + static const char prefix[] = "abort:"; + const size_t prefix_len = sizeof(prefix) - 1; + memcpy(buf, prefix, prefix_len); + size_t n = read_abort_msg(buf + prefix_len, buflen - prefix_len); + // if we failed to get the abort string, fall back to + + if (n == 0) + resolve_pc_to_crash_site(pc, buf, buflen); + } else { + resolve_pc_to_crash_site(pc, buf, buflen); + } +} + +/* Send the CRASH_REQUEST to the server and read back the CRASH_RESPONSE. + We can't use the normal send/recv here because we're in a signal handler, + so instead do raw read/write to pipe. */ +static int crash_query_server(const char *site, int rank, int32_t *winner) +{ + char req_buf[CRASH_REQ_BUF_SIZE]; + size_t req_len = build_crash_report(req_buf, sizeof req_buf, rank, site); + if (req_len == 0) + return -1; + + if (crash_raw_write(crash_write_fd, req_buf, req_len) != 0) + return -1; + + ldcs_message_header_t resp_hdr; + if (crash_raw_read_exact(crash_read_fd, &resp_hdr, sizeof resp_hdr) != 0) + return -1; + if (resp_hdr.type != LDCS_MSG_CRASH_RESPONSE || + resp_hdr.len != sizeof(int32_t)) + return -1; + + int32_t winning_rank = -1; + if (crash_raw_read_exact(crash_read_fd, &winning_rank, + sizeof winning_rank) != 0) + return -1; + + *winner = winning_rank; + return 0; +} + +/* This is the main crash handler. Chains to application handler, if + installed, and checks whether the application fixed the fault. + If not, we are going to crash, and we report the crash and determine + whether we are the winning rank that will write the coredump for + this specific crash site. */ +static void crash_handler_entry(int sig, siginfo_t *info, void *uctx) +{ + /* Reentrancy flag to detect if we crash again while handling the crash. */ + static __thread sig_atomic_t reentering = 0; + + /* We need to restore errno before returning. */ + int saved_errno = errno; + + /* Get the faulting address. */ + unsigned long pc_before = extract_pc(uctx); + + /* If the application registered its own signal handler, call it. */ + int chained = crash_sigchain_chain_to_app(sig, info, uctx); + + /* We got past the application's signal handler. + If we refault past this point, we crashed in the crash handler; + in that case, give up and exit. */ + if (reentering) { + _exit(128 + sig); + } + reentering = 1; + + /* Check whether the application's signal handler resolved the fault. + If it did, we return and let the instruction re-execute. */ + if (chained && crash_sigchain_fault_resolved(sig, info, uctx, pc_before)) { + reentering = 0; + errno = saved_errno; + return; + } + + /* If we reach this point, the application did NOT fix the issue, so we + know this is a real crash. Now we set the handler_active flag. + This ensures that only the first crash to make it here goes through the + deduplication process. (Since whether to dump is a process-wide decision, + we can't make a different decision for different threads.) + Faulting threads other than the first one pause forever until we're done + with this signal handler, at which point the process will terminate. */ + if (!__sync_bool_compare_and_swap(&handler_active, 0, 1)) { + for (;;) pause(); + } + + if (!crash_installed || crash_write_fd < 0 || crash_read_fd < 0) { + goto reraise; + } + + /* Now we do the actual deduplication part. We get the program counter, + resolve it to + or abort_msg, and pass that crashsite + to the server, which picks one winner per crashsite. */ + unsigned long pc = extract_pc(uctx); + crash_build_site(sig, pc, crash_site_buf, sizeof crash_site_buf); + + int32_t winning_rank = -1; + if (crash_query_server(crash_site_buf, crash_global_rank, + &winning_rank) != 0) + goto reraise; + + if ((int) winning_rank != crash_global_rank) { + /* If we are NOT the winner, we set our own core limit to zero, + preventing us from dumping. If we are the winner, do nothing, + preserving the existing core limit. */ + struct rlimit no_core = { 0, 0 }; + (void) setrlimit(RLIMIT_CORE, &no_core); + } + +reraise: + /* Finally, we restore the default signal handler and return, + terminating the process and producing a coredump if the limit allows.. */ + signal(sig, SIG_DFL); +} + +/* Performs setup and installs the signal handler. */ +int crash_handler_install(int global_rank, int ldcsid_in) +{ + if (crash_installed) + return 0; + + crash_sigchain_init(); + + crash_global_rank = global_rank; + + if (client_get_raw_fds(ldcsid_in, &crash_read_fd, &crash_write_fd) != 0 || + crash_read_fd < 0 || crash_write_fd < 0) { + err_printf("failed to get raw FDs for crash handler\n"); + crash_read_fd = -1; + crash_write_fd = -1; + return 0; + } + + crash_lib_offset_prime(); + + /* Perform a dummy resolution to populate the cache. */ + { + char tmp[256]; + resolve_pc_to_crash_site((unsigned long) &crash_handler_install, + tmp, sizeof tmp); + } + + /* Set up the altstack. + If the reason for a segfault is a stack overflow, the signal handler itself + will have no stack available. We handle this by registering an alternate stack + for the signal handler. However, note that this is per-thread, and currently + we do not register an alternate stack on any thread other than the main thread. + TODO: handle alternate stack on other threads */ + crash_altstack_buf = mmap(NULL, CRASH_ALTSTACK_SIZE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (crash_altstack_buf == MAP_FAILED) { + crash_altstack_buf = NULL; + } else { + stack_t ss; + memset(&ss, 0, sizeof ss); + ss.ss_sp = crash_altstack_buf; + ss.ss_size = CRASH_ALTSTACK_SIZE; + ss.ss_flags = 0; + if (sigaltstack(&ss, NULL) != 0) { + munmap(crash_altstack_buf, CRASH_ALTSTACK_SIZE); + crash_altstack_buf = NULL; + } + } + + /* Install the signal handler. */ + struct sigaction sa; + memset(&sa, 0, sizeof sa); + sa.sa_sigaction = crash_handler_entry; + sa.sa_flags = SA_SIGINFO | SA_RESTART; + if (crash_altstack_buf != NULL) + sa.sa_flags |= SA_ONSTACK; + sigemptyset(&sa.sa_mask); + sigaddset(&sa.sa_mask, SIGSEGV); + sigaddset(&sa.sa_mask, SIGBUS); + sigaddset(&sa.sa_mask, SIGFPE); + sigaddset(&sa.sa_mask, SIGILL); + sigaddset(&sa.sa_mask, SIGABRT); + + const int sigs[] = { SIGSEGV, SIGBUS, SIGFPE, SIGILL, SIGABRT }; + for (size_t i = 0; i < sizeof(sigs) / sizeof(sigs[0]); i++) { + /* If a signal handler was already installed by the time Spindle + registers its signal handler, we save it to use in + signal handler chaining. */ + struct sigaction handler_old; + memset(&handler_old, 0, sizeof handler_old); + if (sigaction(sigs[i], &sa, &handler_old) != 0) { + debug_printf("sigaction failed when installing crash handler for signal %d\n", sigs[i]); + continue; + } + crash_sigchain_register_existing_handler(sigs[i], &handler_old); + } + + crash_installed = 1; + return 0; +} diff --git a/src/client/crash_handler/crash_handler.h b/src/client/crash_handler/crash_handler.h new file mode 100644 index 00000000..c33c9762 --- /dev/null +++ b/src/client/crash_handler/crash_handler.h @@ -0,0 +1,23 @@ +/* +This file is part of Spindle. For copyright information see the COPYRIGHT +file in the top level directory, or at +https://github.com/hpc/Spindle/blob/master/COPYRIGHT + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU Lesser General Public License (as published by the Free Software +Foundation) version 2.1 dated February 1999. This program is distributed in the +hope that it will be useful, but WITHOUT ANY WARRANTY; without even the IMPLIED +WARRANTY OF MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms +and conditions of the GNU Lesser General Public License for more details. You should +have received a copy of the GNU Lesser General Public License along with this +program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA +*/ + +#if !defined(CRASH_HANDLER_H_) +#define CRASH_HANDLER_H_ + +/* Install Spindle's coredump-deduplicating signal handler. */ +int crash_handler_install(int global_rank, int ldcsid); + +#endif diff --git a/src/client/crash_handler/crash_io.c b/src/client/crash_handler/crash_io.c new file mode 100644 index 00000000..aa08cbe7 --- /dev/null +++ b/src/client/crash_handler/crash_io.c @@ -0,0 +1,69 @@ +/* +This file is part of Spindle. For copyright information see the COPYRIGHT +file in the top level directory, or at +https://github.com/hpc/Spindle/blob/master/COPYRIGHT + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU Lesser General Public License (as published by the Free Software +Foundation) version 2.1 dated February 1999. This program is distributed in the +hope that it will be useful, but WITHOUT ANY WARRANTY; without even the IMPLIED +WARRANTY OF MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms +and conditions of the GNU Lesser General Public License for more details. You should +have received a copy of the GNU Lesser General Public License along with this +program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA +*/ + +#include +#include +#include + +#include "crash_io.h" + +/* This file contains async-signal-safe functions for reading from + and writing to file descriptors. Used by the crash handler to + communicate with the server. */ + +int crash_raw_write(int fd, const void *buf, size_t len) +{ + const char *p = (const char *) buf; + size_t left = len; + while (left > 0) { + int saved_errno = errno; + ssize_t n = write(fd, p, left); + int write_errno = errno; + errno = saved_errno; + if (n < 0) { + if (write_errno == EINTR || write_errno == EAGAIN) + continue; + return -1; + } + if (n == 0) + return -1; + p += n; + left -= (size_t) n; + } + return 0; +} + +int crash_raw_read_exact(int fd, void *buf, size_t nbytes) +{ + char *p = (char *) buf; + size_t left = nbytes; + while (left > 0) { + int saved_errno = errno; + ssize_t n = read(fd, p, left); + int read_errno = errno; + errno = saved_errno; + if (n < 0) { + if (read_errno == EINTR || read_errno == EAGAIN) + continue; + return -1; + } + if (n == 0) + return -1; + p += n; + left -= (size_t) n; + } + return 0; +} diff --git a/src/client/crash_handler/crash_io.h b/src/client/crash_handler/crash_io.h new file mode 100644 index 00000000..0d192083 --- /dev/null +++ b/src/client/crash_handler/crash_io.h @@ -0,0 +1,25 @@ +/* +This file is part of Spindle. For copyright information see the COPYRIGHT +file in the top level directory, or at +https://github.com/hpc/Spindle/blob/master/COPYRIGHT + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU Lesser General Public License (as published by the Free Software +Foundation) version 2.1 dated February 1999. This program is distributed in the +hope that it will be useful, but WITHOUT ANY WARRANTY; without even the IMPLIED +WARRANTY OF MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms +and conditions of the GNU Lesser General Public License for more details. You should +have received a copy of the GNU Lesser General Public License along with this +program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA +*/ + +#if !defined(CRASH_IO_H_) +#define CRASH_IO_H_ + +#include + +int crash_raw_write(int fd, const void *buf, size_t len); +int crash_raw_read_exact(int fd, void *buf, size_t nbytes); + +#endif diff --git a/src/client/crash_handler/crash_lib_offset.c b/src/client/crash_handler/crash_lib_offset.c new file mode 100644 index 00000000..f6c45eac --- /dev/null +++ b/src/client/crash_handler/crash_lib_offset.c @@ -0,0 +1,163 @@ +/* +This file is part of Spindle. For copyright information see the COPYRIGHT +file in the top level directory, or at +https://github.com/hpc/Spindle/blob/master/COPYRIGHT + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU Lesser General Public License (as published by the Free Software +Foundation) version 2.1 dated February 1999. This program is distributed in the +hope that it will be useful, but WITHOUT ANY WARRANTY; without even the IMPLIED +WARRANTY OF MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms +and conditions of the GNU Lesser General Public License for more details. You should +have received a copy of the GNU Lesser General Public License along with this +program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA +*/ + +#define _GNU_SOURCE + +#include "crash_lib_offset.h" +#include "crash_fmt.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define MAX_PATH_LEN 4096 + +struct r_debug_ext_mirror { + struct r_debug base; + void *r_next; +}; + +/* It turns out that we can't get at the program headers via + dl_phdr_iterate in a signal handler, because dl_phdr_iterate + calls malloc, which is not async-signal-safe, and furthermore + dl_phdr_iterate only iterates the objects loaded in the current + namespace, so when called from the Spindle auditclient we only + see the auditclient and its own libc. Instead, we have to read + the program headers from the file. */ + +static const ElfW(Phdr) *exe_auxv_phdrs; +static unsigned long exe_auxv_phnum; +static char exe_path_cache[MAX_PATH_LEN + 1]; +static char *exe_path_cached; + +/* Get the path to the current executable. This is used for + the part of + when the address + in in the executable. */ +static char *get_executable_path(void) +{ + long r; + + if (exe_path_cached) + return exe_path_cached; + + r = syscall(SYS_readlinkat, AT_FDCWD, "/proc/self/exe", + exe_path_cache, (size_t) MAX_PATH_LEN); + if (r < 0 || r > MAX_PATH_LEN) + return exe_path_cached = (char *) "[EXECUTABLE]"; + exe_path_cache[r] = '\0'; + return exe_path_cached = exe_path_cache; +} + +/* Check the program headers of the file at path for the given address */ +static int pc_in_object_file(const char *path, unsigned long base, + unsigned long pc) +{ + ElfW(Ehdr) ehdr; + ElfW(Phdr) phdr; + long fd, n; + unsigned int i; + unsigned long off; + int found = 0; + + fd = syscall(SYS_openat, AT_FDCWD, path, O_RDONLY | O_CLOEXEC, 0); + if (fd < 0) + return 0; + + n = syscall(SYS_pread64, fd, &ehdr, sizeof(ehdr), (off_t) 0); + if (n != (long) sizeof(ehdr)) + goto done; + if (memcmp(ehdr.e_ident, ELFMAG, SELFMAG) != 0) + goto done; + if (ehdr.e_phnum == 0) + goto done; + + off = ehdr.e_phoff; + for (i = 0; i < ehdr.e_phnum; i++, off += ehdr.e_phentsize) { + n = syscall(SYS_pread64, fd, &phdr, sizeof(phdr), (off_t) off); + if (n != (long) sizeof(phdr)) + break; + if (phdr.p_type != PT_LOAD) + continue; + unsigned long start = base + phdr.p_vaddr; + if (pc >= start && pc < start + phdr.p_memsz) { + found = 1; + break; + } + } + +done: + syscall(SYS_close, fd); + return found; +} + +static int pc_in_exe(unsigned long base, unsigned long pc) +{ + unsigned long i, start; + + if (!exe_auxv_phdrs) + return 0; + for (i = 0; i < exe_auxv_phnum; i++) { + if (exe_auxv_phdrs[i].p_type != PT_LOAD) + continue; + start = base + exe_auxv_phdrs[i].p_vaddr; + if (pc >= start && pc < start + exe_auxv_phdrs[i].p_memsz) + return 1; + } + return 0; +} + +static int walk_link_map_list(struct link_map *cur, unsigned long pc, + char *buf, size_t buflen) +{ + for (; cur != NULL; cur = cur->l_next) { + int is_exe = !(cur->l_name && cur->l_name[0]); + const char *use_name = is_exe ? get_executable_path() : cur->l_name; + int hit = is_exe ? pc_in_exe(cur->l_addr, pc) + : pc_in_object_file(use_name, cur->l_addr, pc); + if (hit) + return crash_fmt_lib_offset(buf, buflen, use_name, pc - cur->l_addr); + } + return -1; +} + +int crash_lib_offset_get_signal_safe(unsigned long pc, char *buf, size_t buflen) +{ + const struct r_debug *rd = &_r_debug; + int extended = (_r_debug.r_version >= 2); + + while (rd != NULL) { + if (walk_link_map_list(rd->r_map, pc, buf, buflen) == 0) + return 0; + if (!extended) + break; + rd = (const struct r_debug *) + ((const struct r_debug_ext_mirror *) rd)->r_next; + } + return -1; +} + +void crash_lib_offset_prime(void) +{ + (void) get_executable_path(); + exe_auxv_phdrs = (const ElfW(Phdr) *) getauxval(AT_PHDR); + exe_auxv_phnum = getauxval(AT_PHNUM); +} diff --git a/src/client/crash_handler/crash_lib_offset.h b/src/client/crash_handler/crash_lib_offset.h new file mode 100644 index 00000000..a50bbe69 --- /dev/null +++ b/src/client/crash_handler/crash_lib_offset.h @@ -0,0 +1,27 @@ +/* +This file is part of Spindle. For copyright information see the COPYRIGHT +file in the top level directory, or at +https://github.com/hpc/Spindle/blob/master/COPYRIGHT + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU Lesser General Public License (as published by the Free Software +Foundation) version 2.1 dated February 1999. This program is distributed in the +hope that it will be useful, but WITHOUT ANY WARRANTY; without even the IMPLIED +WARRANTY OF MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms +and conditions of the GNU Lesser General Public License for more details. You should +have received a copy of the GNU Lesser General Public License along with this +program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA +*/ + +#if !defined(CRASH_LIB_OFFSET_H_) +#define CRASH_LIB_OFFSET_H_ + +#include + +void crash_lib_offset_prime(void); + +int crash_lib_offset_get_signal_safe(unsigned long pc, + char *buf, size_t buflen); + +#endif diff --git a/src/client/crash_handler/crash_sigchain.c b/src/client/crash_handler/crash_sigchain.c new file mode 100644 index 00000000..43d87b39 --- /dev/null +++ b/src/client/crash_handler/crash_sigchain.c @@ -0,0 +1,470 @@ +/* +This file is part of Spindle. For copyright information see the COPYRIGHT +file in the top level directory, or at +https://github.com/hpc/Spindle/blob/master/COPYRIGHT + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU Lesser General Public License (as published by the Free Software +Foundation) version 2.1 dated February 1999. This program is distributed in the +hope that it will be useful, but WITHOUT ANY WARRANTY; without even the IMPLIED +WARRANTY OF MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms +and conditions of the GNU Lesser General Public License for more details. You should +have received a copy of the GNU Lesser General Public License along with this +program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA +*/ + +#define _GNU_SOURCE + +#include "crash_sigchain.h" +#include "crash_arch.h" +#include "intercept.h" +#include "spindle_debug.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* This file deals with signal handler chaining. If the application + has registered its own signal handler, we want to run that one first, + before Spindle runs its own. This turns out to be rather complicated, + because the application might do something in its signal handler that + fixes the fault, making it no longer a crash that Spindle should handle. + As an example, the Java and Julia runtimes implement garbage collection + "safepoints" (points in the code where the GC is allowed to run) as reads + to PROT_NONE pages. The signal handler then waits for all threads to reach + a safepoint, runs the GC, then unprotects the page and returns, allowing the + faulting instruction to re-execute. If Spindle ran its own signal handler + after this, safepoints would crash. As another example, libsigsegv uses + a SIGSEGV handler to implement user-level paging. Much of the complexity in + this file comes from handling cases such as these. */ + +/* CAUTION: signal and sigaction are on the async-signal-safe list, so the wrappers + themselves must also be async-signal-safe! */ + +typedef void (*sighandler_t)(int); + +void *orig_sigaction; +void *orig_signal; +void *orig_bsd_signal; +void *orig_sysv_signal; + +struct app_disposition { + void (*handler)(int); + void (*sigaction_fn)(int, siginfo_t *, void *); + sigset_t mask; + int flags; + int has_sigaction; + int active; +}; + +static const int OWNED_SIGS[] = { SIGSEGV, SIGBUS, SIGFPE, SIGILL, SIGABRT }; + +int crash_sigchain_is_owned(int sig) +{ + for (size_t i = 0; i < sizeof(OWNED_SIGS) / sizeof(OWNED_SIGS[0]); i++) { + if (OWNED_SIGS[i] == sig) + return 1; + } + return 0; +} + +/* Storage for the application's registered signal handlers */ +static struct app_disposition disp_store[NSIG]; + +/* Spinlock to protect access to stored app signal dispositions.*/ +static volatile int disp_lock[NSIG]; + +static void crash_disp_lock(int sig) +{ + while (__sync_lock_test_and_set(&disp_lock[sig], 1)) { + sched_yield(); + } +} + +static void crash_disp_unlock(int sig) +{ + __sync_lock_release(&disp_lock[sig]); +} + +/* Caller must hold disp lock for signal */ +static void store_disposition(int sig, const struct sigaction *act) +{ + if (act == NULL) return; + struct app_disposition *d = &disp_store[sig]; + if (!(act->sa_flags & SA_SIGINFO) && + (act->sa_handler == SIG_DFL || act->sa_handler == SIG_IGN)) { + d->active = 0; + return; + } + d->has_sigaction = (act->sa_flags & SA_SIGINFO) ? 1 : 0; + if (d->has_sigaction) { + d->sigaction_fn = act->sa_sigaction; + d->handler = NULL; + } else { + d->handler = act->sa_handler; + d->sigaction_fn = NULL; + } + d->mask = act->sa_mask; + d->flags = act->sa_flags & ~(SA_SIGINFO | SA_ONSTACK | SA_RESTART); + d->active = 1; +} + +/* Caller must hold disp lock for signal */ +static void get_disposition(int sig, struct sigaction *out) +{ + struct app_disposition *d = &disp_store[sig]; + memset(out, 0, sizeof *out); + if (!d->active) { + out->sa_handler = SIG_DFL; + return; + } + if (d->has_sigaction) { + out->sa_sigaction = d->sigaction_fn; + out->sa_flags = d->flags | SA_SIGINFO; + } else { + out->sa_handler = d->handler; + out->sa_flags = d->flags; + } + out->sa_mask = d->mask; +} + +/* This is called during initialization to register a signal handler that + was already registered before Spindle's crash handler was registered */ +void crash_sigchain_register_existing_handler(int sig, const struct sigaction *handler_old) +{ + if (!crash_sigchain_is_owned(sig)) return; + crash_disp_lock(sig); + store_disposition(sig, handler_old); + crash_disp_unlock(sig); + /* This function is not called from a signal handler, so it's safe to log */ + debug_printf2("stored application signal handler for sig %d\n", sig); +} + +/* Run the application signal handler, if present. + Returns 0 if there was no handler to run, 1 if there was an application + handler and we successfully ran it and returned from it. */ +int crash_sigchain_chain_to_app(int sig, siginfo_t *info, void *ucontext) +{ + struct app_disposition local; + + crash_disp_lock(sig); + + /* If there's no application handler, nothing more for us to do */ + if (!disp_store[sig].active) { + crash_disp_unlock(sig); + return 0; + } + + /* Make a local copy of the signal disposition so we can release the lock + before calling the stored signal handler, as it might call sigaction itself */ + local = disp_store[sig]; + + /* If the app requested the handler be reset, do so */ + if (local.flags & SA_RESETHAND) { + disp_store[sig].active = 0; + } + crash_disp_unlock(sig); + + /* Build the mask we want active during the application's handler */ + sigset_t blockset = local.mask; + if (!(local.flags & SA_NODEFER)) sigaddset(&blockset, sig); + sigset_t saved; + (void) sigprocmask(SIG_BLOCK, &blockset, &saved); + + /* Call the application's signal handler */ + if (local.has_sigaction) { + local.sigaction_fn(sig, info, ucontext); + } else { + local.handler(sig); + } + + /* If we reach here, the application handler returned. + Restore the mask to its previous value. */ + (void) sigprocmask(SIG_SETMASK, &saved, NULL); + return 1; +} + +/* Fault resolution detection + This section contains functions used to detect whether the application's + signal handler fixed the fault or not. */ + +#ifdef PROCMAP_QUERY +/* Linux 6.11 and later let us query /proc/pid/maps without textual + parsing. This stores whether we can do that. */ +static int crash_maps_query_supported = 0; +#endif + +/* Probe whether addr is readable in this process by attempting a + 1-byte process_vm_readv from a local stack buffer. Returns + 1 if the read succeeded + 0 if the read faultedd + -1 if the read failed for some reason other than EFAULT + We use this to detect if the application has fixed the reason for + a read fault (e.g., for the Java/Julia safepoint pattern). + */ +static inline int crash_probe_byte_readable(const void *addr) +{ + char buf; + struct iovec local = { &buf, 1 }; + struct iovec remote = { (void *) addr, 1 }; + /* process_vm_readv is intended to let a process read the memory + of its ptrace target, but it can also be used to read one's own + memory. In the event of a fault, it returns EFAULT instead of + triggering a signal. */ + long n = syscall(SYS_process_vm_readv, + (long) getpid(), + &local, 1UL, + &remote, 1UL, + 0UL); + if (n == 1) return 1; // Successfully read 1 byte + if (n == -1 && errno == EFAULT) return 0; // Faulted + return -1; // Neither successfully read nor faulted +} + +/* Checking whether the application handler fixed a write fault is + much more annoying than the read case. For reads, we can safely + just retry the read and see if it succeeded. For writes, however, + we can't safely retry without actually writing into the application's + memory. So for the write case, we instead check /proc/self/maps to see + whether the address in question is writable after the application + handler returns. */ + +#ifdef PROCMAP_QUERY +/* On Linux 6.11 and later, we can read /proc/self/maps via ioctl + instead of having to parse it ourselves. */ +static inline int crash_maps_check_writable_via_ioctl(int fd, uintptr_t addr) +{ + struct procmap_query q; + memset(&q, 0, sizeof q); + q.size = sizeof q; + q.query_addr = (uint64_t) addr; + q.query_flags = 0; + int rc; + do { + rc = ioctl(fd, PROCMAP_QUERY, &q); + } while (rc < 0 && errno == EINTR); + if (rc < 0) { + if (errno == ENOENT) + return 0; + return -1; + } + return (q.vma_flags & PROCMAP_QUERY_VMA_WRITABLE) ? 1 : 0; +} +#endif /* PROCMAP_QUERY */ + +/* On Linux kernels older than 6.11, the ioctl doesn't exist and we + have to parse /proc/self/maps using only async-signal-safe functions. */ + +/* Hex digit value, or -1 if c is not a hex digit. + We need this because strtoul is not on the async-signal-safe list. */ +static inline int hexval(char c) +{ + if (c >= '0' && c <= '9') return c - '0'; + if (c >= 'a' && c <= 'f') return c - 'a' + 10; + if (c >= 'A' && c <= 'F') return c - 'A' + 10; + return -1; +} + +/* Parse /proc/self/maps looking for the range containing addr, checking + whether it is writable. Returns 1 if writable, 0 if not writable or + not mapped, and -1 on error. */ +static int crash_maps_check_writable_via_textparse(int fd, uintptr_t addr) +{ + const size_t buf_size = 4096; + char *buf = (char *) mmap(NULL, buf_size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, + -1, 0); + if (buf == MAP_FAILED) + return -1; + + enum { ADDR_START, ADDR_END, PERMS, SKIP_EOL } state = ADDR_START; + uintptr_t start = 0, end = 0; + int perm_idx = 0, writable = 0; + int result = 0; + int done = 0; + + while (!done) { + ssize_t n; + do { + n = read(fd, buf, buf_size); + } while (n < 0 && errno == EINTR); + if (n < 0) { + result = -1; + break; + } + if (n == 0) // EOF + break; + + for (ssize_t i = 0; i < n && !done; i++) { + char c = buf[i]; + switch (state) { + /* We begin parsing a line with the start address of the range */ + case ADDR_START: + if (c == '-') { + /* When we get to the '-', change to parsing the end address */ + state = ADDR_END; + } else { + /* Accumulate one hex digit at at time */ + start = (start << 4) | hexval(c); + } + break; + case ADDR_END: + if (c == ' ') { + /* When we get to a space, we're done with the range and permissions are next */ + state = PERMS; + perm_idx = 0; + } else { + /* Accumulate one hex digit at at time */ + end = (end << 4) | hexval(c); + } + break; + case PERMS: + /* perms are "rwxp". We only care about the 'w' byte at index 1. */ + if (perm_idx++ == 1) { + writable = (c == 'w'); + /* Now that we have the range and writability, check if our + address is in this range */ + if (start > addr) { + done = 1; + } else if (addr < end) { + result = writable; + done = 1; + } else { + state = SKIP_EOL; + } + } + break; + case SKIP_EOL: + if (c == '\n') { + /* When we get to the end of the line, set up to parse the next line */ + state = ADDR_START; + start = 0; + end = 0; + } + break; + } + } + } + + (void) munmap(buf, buf_size); + return result; +} + +/* Probe whether addr is writable in this process by checking + /proc/self/maps. Returns + 1 if the address is mapped in a writable page + 0 if the address is not mapped in a writable page + -1 if an error occurred reading the maps file + We use this to detect if the application has fixed the reason for + a write fault (e.g., libsigsegv for user-level paging). + */ +static inline int crash_maps_check_writable(uintptr_t addr) +{ + int fd = open("/proc/self/maps", O_RDONLY | O_CLOEXEC); + if (fd < 0) return -1; + + int rc; +#ifdef PROCMAP_QUERY + if (crash_maps_query_supported) + rc = crash_maps_check_writable_via_ioctl(fd, addr); + else +#endif + rc = crash_maps_check_writable_via_textparse(fd, addr); + + (void) close(fd); + return rc; +} + +void crash_sigchain_init(void) +{ +#ifdef PROCMAP_QUERY + int fd = open("/proc/self/maps", O_RDONLY | O_CLOEXEC); + if (fd >= 0) { + if (crash_maps_check_writable_via_ioctl(fd, (uintptr_t) &crash_maps_query_supported) == 1) + crash_maps_query_supported = 1; + (void) close(fd); + } +#endif /* PROCMAP_QUERY */ +} + +/* The main entrypoint to the fault recovery detection. */ +int crash_sigchain_fault_resolved(int sig, siginfo_t *info, void *uctx, + unsigned long pc_before) +{ + /* PC advanced inside the application handled. */ + if (extract_pc(uctx) != pc_before) + return 1; + + /* The remaining possible fixes only apply to faulting on an + addresss (SIGSEGV, SIGBUS). */ + if (sig != SIGSEGV && sig != SIGBUS) + return 1; + + /* If we faulted on a write, check if the address is now mapped writable. + If we faulted on a read, retry the read and check if we refault. */ + int rc = pf_is_write(uctx) ? crash_maps_check_writable((uintptr_t) info->si_addr) + : crash_probe_byte_readable(info->si_addr); + return (rc == 1) ? 1 : 0; +} + +/* sigaction-family wrappers */ + +int sigaction_wrapper(int sig, const struct sigaction *act, + struct sigaction *oldact) +{ + if (!crash_sigchain_is_owned(sig)) { + return ((int (*)(int, const struct sigaction *, struct sigaction *)) + orig_sigaction)(sig, act, oldact); + } + + crash_disp_lock(sig); + if (oldact != NULL) get_disposition(sig, oldact); + store_disposition(sig, act); + crash_disp_unlock(sig); + return 0; +} + +static sighandler_t signal_common(int sig, sighandler_t handler, + int flags, void *orig) +{ + if (!crash_sigchain_is_owned(sig)) { + return ((sighandler_t (*)(int, sighandler_t)) orig)(sig, handler); + } + + struct sigaction act, oldact; + memset(&act, 0, sizeof act); + act.sa_handler = handler; + act.sa_flags = flags; + (void) sigaction_wrapper(sig, &act, &oldact); + + return (oldact.sa_flags & SA_SIGINFO) ? SIG_DFL : oldact.sa_handler; +} + +void (*signal_wrapper(int sig, void (*handler)(int)))(int) +{ + return signal_common(sig, handler, SA_RESTART, orig_signal); +} + +void (*bsd_signal_wrapper(int sig, void (*handler)(int)))(int) +{ + return signal_common(sig, handler, SA_RESTART, orig_bsd_signal); +} + +void (*sysv_signal_wrapper(int sig, void (*handler)(int)))(int) +{ + return signal_common(sig, handler, SA_RESETHAND | SA_NODEFER, + orig_sysv_signal); +} diff --git a/src/client/crash_handler/crash_sigchain.h b/src/client/crash_handler/crash_sigchain.h new file mode 100644 index 00000000..6856d0f8 --- /dev/null +++ b/src/client/crash_handler/crash_sigchain.h @@ -0,0 +1,36 @@ +/* +This file is part of Spindle. For copyright information see the COPYRIGHT +file in the top level directory, or at +https://github.com/hpc/Spindle/blob/master/COPYRIGHT + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU Lesser General Public License (as published by the Free Software +Foundation) version 2.1 dated February 1999. This program is distributed in the +hope that it will be useful, but WITHOUT ANY WARRANTY; without even the IMPLIED +WARRANTY OF MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms +and conditions of the GNU Lesser General Public License for more details. You should +have received a copy of the GNU Lesser General Public License along with this +program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA +*/ + +#if !defined(CRASH_SIGCHAIN_H_) +#define CRASH_SIGCHAIN_H_ + +#include +#include + +int crash_sigchain_is_owned(int sig); +void crash_sigchain_register_existing_handler(int sig, const struct sigaction *handler_old); +int crash_sigchain_chain_to_app(int sig, siginfo_t *info, void *ucontext); +void crash_sigchain_init(void); +int crash_sigchain_fault_resolved(int sig, siginfo_t *info, void *uctx, + unsigned long pc_before); + +int sigaction_wrapper(int sig, const struct sigaction *act, + struct sigaction *oldact); +void (*signal_wrapper(int sig, void (*handler)(int)))(int); +void (*bsd_signal_wrapper(int sig, void (*handler)(int)))(int); +void (*sysv_signal_wrapper(int sig, void (*handler)(int)))(int); + +#endif diff --git a/src/client/subaudit/Makefile.am b/src/client/subaudit/Makefile.am index c17b7c6a..0d0cb4a1 100644 --- a/src/client/subaudit/Makefile.am +++ b/src/client/subaudit/Makefile.am @@ -12,7 +12,7 @@ endif AM_CFLAGS = -fvisibility=hidden -AM_CPPFLAGS = -I$(top_srcdir)/../logging -I$(top_srcdir)/../include -I$(top_srcdir)/client -I$(top_srcdir)/client_comlib -I$(top_srcdir)/auditclient -I$(top_srcdir)/../utils +AM_CPPFLAGS = -I$(top_srcdir)/../logging -I$(top_srcdir)/../include -I$(top_srcdir)/client -I$(top_srcdir)/client_comlib -I$(top_srcdir)/auditclient -I$(top_srcdir)/../utils -I$(top_srcdir)/crash_handler BASE_SRCS = subaudit.c intercept_malloc.c update_pltbind.c ../auditclient/auditclient_common.c ../auditclient/patch_linkmap.c ../auditclient/fixlocale.c diff --git a/src/client/subaudit/Makefile.in b/src/client/subaudit/Makefile.in index 0e5d1b06..9869cec9 100644 --- a/src/client/subaudit/Makefile.in +++ b/src/client/subaudit/Makefile.in @@ -397,7 +397,7 @@ top_srcdir = @top_srcdir@ pkglib_LTLIBRARIES = libspindleint.la $(am__append_1) $(am__append_2) \ $(am__append_3) AM_CFLAGS = -fvisibility=hidden -AM_CPPFLAGS = -I$(top_srcdir)/../logging -I$(top_srcdir)/../include -I$(top_srcdir)/client -I$(top_srcdir)/client_comlib -I$(top_srcdir)/auditclient -I$(top_srcdir)/../utils +AM_CPPFLAGS = -I$(top_srcdir)/../logging -I$(top_srcdir)/../include -I$(top_srcdir)/client -I$(top_srcdir)/client_comlib -I$(top_srcdir)/auditclient -I$(top_srcdir)/../utils -I$(top_srcdir)/crash_handler BASE_SRCS = subaudit.c intercept_malloc.c update_pltbind.c ../auditclient/auditclient_common.c ../auditclient/patch_linkmap.c ../auditclient/fixlocale.c AUDITLIB = $(top_builddir)/client/libspindle_audit.la libspindleint_la_SOURCES = preloadlib.c diff --git a/src/fe/config.h.in b/src/fe/config.h.in index 615997be..20a30383 100644 --- a/src/fe/config.h.in +++ b/src/fe/config.h.in @@ -27,6 +27,9 @@ /* Define if were using sockets for client/server communication */ #undef COMM_SOCKET +/* Enable --crash-dedup by default */ +#undef CRASH_DEDUP_ENABLED_BY_DEFAULT + /* Default to using a dedicated process to clean data files */ #undef DEFAULT_CLEANUP_PROC diff --git a/src/fe/configure b/src/fe/configure index 4c11bdc7..73def515 100755 --- a/src/fe/configure +++ b/src/fe/configure @@ -863,6 +863,7 @@ enable_sec_munge enable_sec_keydir enable_sec_launchmon enable_sec_none +enable_crash_dedup with_gcrypt_dir with_munge_dir with_glibc_be_dir @@ -1553,6 +1554,7 @@ Optional Features: --enable-sec-launchmon Enable LaunchmMON to propagate authentication key for connections --enable-sec-none Disable security authentication of connections + --enable-crash-dedup Enable --crash-dedup by default (default: disabled) --enable-remap-pageone Controls mmap behavior when replacing executables. Should be false on older linux systems --enable-slurm Enable support for the SLURM job launcher @@ -17556,6 +17558,20 @@ if test "${enable_sec_none+set}" = set; then : fi +# Check whether --enable-crash-dedup was given. +if test "${enable_crash_dedup+set}" = set; then : + enableval=$enable_crash_dedup; enable_crash_dedup=$enableval +else + enable_crash_dedup=no +fi + + +if test "x$enable_crash_dedup" = xyes; then : + +$as_echo "#define CRASH_DEDUP_ENABLED_BY_DEFAULT 1" >>confdefs.h + +fi + # Check whether --with-gcrypt-dir was given. if test "${with_gcrypt_dir+set}" = set; then : diff --git a/src/fe/startup/config_mgr.cc b/src/fe/startup/config_mgr.cc index 1b9d9ade..e1368642 100644 --- a/src/fe/startup/config_mgr.cc +++ b/src/fe/startup/config_mgr.cc @@ -142,6 +142,12 @@ using namespace std; #define RSHCMD_STR "" #endif +#if defined(CRASH_DEDUP_ENABLED_BY_DEFAULT) +#define DEFAULT_CRASH_DEDUP_STR "true" +#else +#define DEFAULT_CRASH_DEDUP_STR "false" +#endif + #if defined(PYTHON_INST_PREFIX) #define PYTHON_PREFIX_DEFAULT PYTHON_INST_PREFIX #else @@ -294,7 +300,9 @@ void initOptionsList() { confEnableRsh, "enable-rsh", shortRSHMode, groupMisc, cvBool, {}, RSHLAUNCH_STR, "Enable starting daemons with an rsh tree, if the startup mode supports it." }, { confRshCommand, "rsh-command", shortRSHCmd, groupMisc, cvString, {}, RSHCMD_STR, - "The command to run rsh/ssh, when doing RSH startup mode." } + "The command to run rsh/ssh, when doing RSH startup mode." }, + { confCrashDedup, "crash-dedup", shortCrashDedup, groupMisc, cvBool, {}, DEFAULT_CRASH_DEDUP_STR, + "Deduplicate coredumps by crash site, emitting only one coredump per unique site." } } ); } @@ -998,6 +1006,9 @@ bool ConfigMap::toSpindleArgs(spindle_args_t &args, bool alloc_strs) const case confPatchLdso: setopt(args.opts, OPT_PATCHLDSO, boolresult); break; + case confCrashDedup: + setopt(args.opts, OPT_CRASH_HANDLER, boolresult); + break; } } diff --git a/src/fe/startup/config_mgr.h b/src/fe/startup/config_mgr.h index ec3c8135..14f18b7a 100644 --- a/src/fe/startup/config_mgr.h +++ b/src/fe/startup/config_mgr.h @@ -68,7 +68,8 @@ enum SpindleConfigID { confStartMultiSession, confEndSession, confRunSession, - confPatchLdso + confPatchLdso, + confCrashDedup }; enum CmdlineShortOptions { @@ -128,6 +129,7 @@ enum CmdlineShortOptions { shortExecExcludes = 298, shortPatchLdso = 299, shortCachePaths = 300, + shortCrashDedup = 301, }; enum CmdlineGroups { diff --git a/src/fe/startup/spindle_fe.cc b/src/fe/startup/spindle_fe.cc index a038e201..f1daab9e 100644 --- a/src/fe/startup/spindle_fe.cc +++ b/src/fe/startup/spindle_fe.cc @@ -41,7 +41,6 @@ static const char *logging_file = NULL; #endif static const char spindle_bootstrap[] = LIBEXECDIR "/spindle_bootstrap"; static bool sendAndWaitForAlive(); -static void determineCachepathConsensus(); #define STARTUP_TIMEOUT 60 @@ -351,6 +350,7 @@ static void printSpindleFlags(opt_t opts) { printFlag(opts, OPT_NUMA, "OPT_NUMA", ss); printFlag(opts, OPT_OFF, "OPT_OFF", ss); printFlag(opts, OPT_PATCHLDSO, "OPT_PATCHLDSO", ss); + printFlag(opts, OPT_CRASH_HANDLER, "OPT_CRASH_HANDLER", ss); ss << ", "; if (OPT_GET_SEC(opts) == OPT_SEC_MUNGE) ss << "OPT_SEC_MUNGE"; if (OPT_GET_SEC(opts) == OPT_SEC_KEYLMON) ss << "OPT_SEC_KEYLMON"; @@ -433,7 +433,6 @@ int spindleInitFE(const char **hosts, spindle_args_t *params) /* Wait for servers to indicate startup */ sendAndWaitForAlive(); - determineCachepathConsensus(); return 0; } @@ -490,17 +489,6 @@ void markRSHPidReapedFE() clear_fe_rsh_pid(); } -static void determineCachepathConsensus( void ){ - ldcs_message_t consensus_req_msg; - consensus_req_msg.header.type = LDCS_MSG_REQUEST_CACHEPATH_CONSENSUS; - consensus_req_msg.header.len = 0; - consensus_req_msg.data = NULL; - int result = ldcs_audit_server_fe_broadcast(&consensus_req_msg, NULL); - if (result == -1) { - debug_printf("Failure sending cachepath consensus message\n"); - } -} - static bool sendAndWaitForAlive() { int result; diff --git a/src/flux/flux-spindle.c b/src/flux/flux-spindle.c index dd00aa32..a3abeef7 100644 --- a/src/flux/flux-spindle.c +++ b/src/flux/flux-spindle.c @@ -379,6 +379,7 @@ static int sp_getopts (flux_shell_t *shell, struct spindle_ctx *ctx) int pull = 0; int had_error = 0; int numa = 0; + int crash_dedup = 0; const char *relocaout = NULL, *reloclibs = NULL, *relocexec = NULL, *relocpython = NULL; const char *followfork = NULL, *preload = NULL, *level = NULL; const char *pyprefix = NULL, *commpath = NULL; @@ -404,7 +405,7 @@ static int sp_getopts (flux_shell_t *shell, struct spindle_ctx *ctx) * supplied by the user, but not unpacked (This handles typos, etc). */ if (json_unpack_ex (opts, &error, JSON_STRICT, - "{s?i s?i s?i s?i s?s s?s s?s s?s s?s s?s s?s s?i s?s s?s s?s s?s}", + "{s?i s?i s?i s?i s?s s?s s?s s?s s?s s?s s?s s?i s?s s?s s?s s?s s?i}", "noclean", &noclean, "nostrip", &nostrip, "push", &push, @@ -420,7 +421,8 @@ static int sp_getopts (flux_shell_t *shell, struct spindle_ctx *ctx) "numa-files", &numafiles, "preload", &preload, "level", &level, - "cachepaths", &cachepaths) < 0) + "cachepaths", &cachepaths, + "crash-dedup", &crash_dedup) < 0) logerrno_printf_and_return(1, "Error in spindle option: %s\n", error.text); if (noclean) @@ -469,6 +471,9 @@ static int sp_getopts (flux_shell_t *shell, struct spindle_ctx *ctx) if (commpath) { ctx->params.commpath = (char *) commpath; } + if (crash_dedup) { + ctx->params.opts |= OPT_CRASH_HANDLER; + } if (level) { if (strcmp(level, "high") == 0) { ctx->params.opts |= OPT_RELOCAOUT; diff --git a/src/include/ldcs_api.h b/src/include/ldcs_api.h index 0bcd8f40..220eb349 100644 --- a/src/include/ldcs_api.h +++ b/src/include/ldcs_api.h @@ -85,10 +85,10 @@ typedef enum { LDCS_MSG_PICKONE_RESP, LDCS_MSG_ALIVE_REQ, LDCS_MSG_ALIVE_RESP, - LDCS_MSG_REQUEST_CACHEPATH_CONSENSUS, LDCS_MSG_CHOSEN_CACHEPATH_REQUEST, LDCS_MSG_CHOSEN_CACHEPATH, - LDCS_MSG_NO_CACHEPATH_CONSENSUS_YET, + LDCS_MSG_CRASH_REPORT, + LDCS_MSG_CRASH_RESPONSE, LDCS_MSG_UNKNOWN } ldcs_message_ids_t; diff --git a/src/include/spindle_launch.h b/src/include/spindle_launch.h index ca7b8d3d..98121396 100644 --- a/src/include/spindle_launch.h +++ b/src/include/spindle_launch.h @@ -60,7 +60,8 @@ extern "C" { #define OPT_NUMA (1 << 29) /* Enables file replication across NUMA domains */ #define OPT_OFF (1 << 30) /* Turns spindle off, disabling everything */ #define OPT_PATCHLDSO (1 << 31) /* Enables patching of ld.so to intercept stat calls */ - +#define OPT_CRASH_HANDLER ((opt_t) 1 << 32) /* Enables crash handler with coredump deduplication */ + #define OPT_SET_SEC(OPT, X) OPT |= (X << 19) #define OPT_GET_SEC(OPT) ((OPT >> 19) & 7) #define OPT_SEC_MUNGE 0 /* Use munge to validate connections */ diff --git a/src/server/auditserver/Makefile.am b/src/server/auditserver/Makefile.am index c8113dd4..8823ead6 100644 --- a/src/server/auditserver/Makefile.am +++ b/src/server/auditserver/Makefile.am @@ -5,7 +5,7 @@ AM_CPPFLAGS = -I$(top_srcdir)/comlib -I$(top_srcdir)/cache -I$(top_srcdir)/../co LDADD = $(top_builddir)/cache/libldcs_cache.la -lrt #AM_LDFLAGS = -all-static -libserverbase_la_SOURCES = ldcs_audit_server_client_cb.c ldcs_audit_server_server_cb.c ldcs_audit_server_process.c ldcs_audit_server_filemngt.c ldcs_audit_server_handlers.c ldcs_elf_read.c ldcs_audit_server_requestors.c ldcs_audit_server_numa.c msgbundle.c parse_mounts.cc cleanup_proc.cc translate_maps.c force_exit.c $(top_srcdir)/../utils/fileutil.c +libserverbase_la_SOURCES = ldcs_audit_server_client_cb.c ldcs_audit_server_server_cb.c ldcs_audit_server_process.c ldcs_audit_server_filemngt.c ldcs_audit_server_handlers.c ldcs_audit_server_crash_handler.c ldcs_elf_read.c ldcs_audit_server_requestors.c ldcs_audit_server_numa.c msgbundle.c parse_mounts.cc cleanup_proc.cc translate_maps.c force_exit.c $(top_srcdir)/../utils/fileutil.c libserverbase_la_LIBADD = -lpthread #libaudit_server_msocket_la_SOURCES = ldcs_audit_server_md_msocket.c ldcs_audit_server_md_msocket_util.c ldcs_audit_server_md_msocket_topo.c diff --git a/src/server/auditserver/Makefile.in b/src/server/auditserver/Makefile.in index e4527600..cede46ff 100644 --- a/src/server/auditserver/Makefile.in +++ b/src/server/auditserver/Makefile.in @@ -123,10 +123,10 @@ am__dirstamp = $(am__leading_dot)dirstamp am_libserverbase_la_OBJECTS = ldcs_audit_server_client_cb.lo \ ldcs_audit_server_server_cb.lo ldcs_audit_server_process.lo \ ldcs_audit_server_filemngt.lo ldcs_audit_server_handlers.lo \ - ldcs_elf_read.lo ldcs_audit_server_requestors.lo \ - ldcs_audit_server_numa.lo msgbundle.lo parse_mounts.lo \ - cleanup_proc.lo translate_maps.lo force_exit.lo \ - $(top_builddir)/../utils/fileutil.lo + ldcs_audit_server_crash_handler.lo ldcs_elf_read.lo \ + ldcs_audit_server_requestors.lo ldcs_audit_server_numa.lo \ + msgbundle.lo parse_mounts.lo cleanup_proc.lo translate_maps.lo \ + force_exit.lo $(top_builddir)/../utils/fileutil.lo libserverbase_la_OBJECTS = $(am_libserverbase_la_OBJECTS) AM_V_P = $(am__v_P_@AM_V@) am__v_P_ = $(am__v_P_@AM_DEFAULT_V@) @@ -146,6 +146,7 @@ am__maybe_remake_depfiles = depfiles am__depfiles_remade = $(top_builddir)/../utils/$(DEPDIR)/fileutil.Plo \ ./$(DEPDIR)/cleanup_proc.Plo ./$(DEPDIR)/force_exit.Plo \ ./$(DEPDIR)/ldcs_audit_server_client_cb.Plo \ + ./$(DEPDIR)/ldcs_audit_server_crash_handler.Plo \ ./$(DEPDIR)/ldcs_audit_server_filemngt.Plo \ ./$(DEPDIR)/ldcs_audit_server_handlers.Plo \ ./$(DEPDIR)/ldcs_audit_server_md_cobo.Plo \ @@ -380,7 +381,7 @@ noinst_LTLIBRARIES = libaudit_server_cobo.la libserverbase.la AM_CPPFLAGS = -I$(top_srcdir)/comlib -I$(top_srcdir)/cache -I$(top_srcdir)/../cobo -I$(top_srcdir)/../logging -I$(top_srcdir)/../include -I$(top_srcdir)/../utils -DLIBEXECDIR=\"$(pkglibexecdir)\" LDADD = $(top_builddir)/cache/libldcs_cache.la -lrt #AM_LDFLAGS = -all-static -libserverbase_la_SOURCES = ldcs_audit_server_client_cb.c ldcs_audit_server_server_cb.c ldcs_audit_server_process.c ldcs_audit_server_filemngt.c ldcs_audit_server_handlers.c ldcs_elf_read.c ldcs_audit_server_requestors.c ldcs_audit_server_numa.c msgbundle.c parse_mounts.cc cleanup_proc.cc translate_maps.c force_exit.c $(top_srcdir)/../utils/fileutil.c +libserverbase_la_SOURCES = ldcs_audit_server_client_cb.c ldcs_audit_server_server_cb.c ldcs_audit_server_process.c ldcs_audit_server_filemngt.c ldcs_audit_server_handlers.c ldcs_audit_server_crash_handler.c ldcs_elf_read.c ldcs_audit_server_requestors.c ldcs_audit_server_numa.c msgbundle.c parse_mounts.cc cleanup_proc.cc translate_maps.c force_exit.c $(top_srcdir)/../utils/fileutil.c libserverbase_la_LIBADD = -lpthread #libaudit_server_msocket_la_SOURCES = ldcs_audit_server_md_msocket.c ldcs_audit_server_md_msocket_util.c ldcs_audit_server_md_msocket_topo.c @@ -461,6 +462,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/cleanup_proc.Plo@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/force_exit.Plo@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ldcs_audit_server_client_cb.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ldcs_audit_server_crash_handler.Plo@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ldcs_audit_server_filemngt.Plo@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ldcs_audit_server_handlers.Plo@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ldcs_audit_server_md_cobo.Plo@am__quote@ # am--include-marker @@ -665,6 +667,7 @@ distclean: distclean-am -rm -f ./$(DEPDIR)/cleanup_proc.Plo -rm -f ./$(DEPDIR)/force_exit.Plo -rm -f ./$(DEPDIR)/ldcs_audit_server_client_cb.Plo + -rm -f ./$(DEPDIR)/ldcs_audit_server_crash_handler.Plo -rm -f ./$(DEPDIR)/ldcs_audit_server_filemngt.Plo -rm -f ./$(DEPDIR)/ldcs_audit_server_handlers.Plo -rm -f ./$(DEPDIR)/ldcs_audit_server_md_cobo.Plo @@ -725,6 +728,7 @@ maintainer-clean: maintainer-clean-am -rm -f ./$(DEPDIR)/cleanup_proc.Plo -rm -f ./$(DEPDIR)/force_exit.Plo -rm -f ./$(DEPDIR)/ldcs_audit_server_client_cb.Plo + -rm -f ./$(DEPDIR)/ldcs_audit_server_crash_handler.Plo -rm -f ./$(DEPDIR)/ldcs_audit_server_filemngt.Plo -rm -f ./$(DEPDIR)/ldcs_audit_server_handlers.Plo -rm -f ./$(DEPDIR)/ldcs_audit_server_md_cobo.Plo diff --git a/src/server/auditserver/ldcs_audit_server_crash_handler.c b/src/server/auditserver/ldcs_audit_server_crash_handler.c new file mode 100644 index 00000000..bdc25871 --- /dev/null +++ b/src/server/auditserver/ldcs_audit_server_crash_handler.c @@ -0,0 +1,301 @@ +/* +This file is part of Spindle. For copyright information see the COPYRIGHT +file in the top level directory, or at +https://github.com/hpc/Spindle/blob/master/COPYRIGHT + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU Lesser General Public License (as published by the Free Software +Foundation) version 2.1 dated February 1999. This program is distributed in the +hope that it will be useful, but WITHOUT ANY WARRANTY; without even the IMPLIED +WARRANTY OF MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms +and conditions of the GNU Lesser General Public License for more details. You should +have received a copy of the GNU Lesser General Public License along with this +program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA +*/ + +#include +#include +#include +#include + +#include "ldcs_api.h" +#include "ldcs_audit_server_process.h" +#include "ldcs_audit_server_md.h" +#include "ldcs_audit_server_crash_handler.h" +#include "spindle_launch.h" +#include "msgbundle.h" + +static crash_site_entry_t *crash_site_find(ldcs_process_data_t *procdata, + const char *site, size_t site_len); +static crash_site_entry_t *crash_site_insert(ldcs_process_data_t *procdata, + const char *site, size_t site_len); +static char *crash_pack(int32_t rank, const char *site, size_t site_len, + size_t *out_len); +static void send_crash_response_to_waiter(ldcs_process_data_t *procdata, + crash_waiter_t *w, + int winning_rank, + const char *site, size_t site_len); +static int crash_resolve(ldcs_process_data_t *procdata, + crash_site_entry_t *entry, int winning_rank); +static int forward_crash_query_up(ldcs_process_data_t *procdata, + const char *site, size_t site_len, + int first_waiter_rank); +static int crash_parse_report(ldcs_message_t *msg, const char *err_str, + int32_t *rank, + const char **site, size_t *site_len); +static int crash_report_common(ldcs_process_data_t *procdata, + crash_waiter_t *w, int reporter_rank, + const char *site, size_t site_len); + +/* INTERNAL HELPER FUNCTIONS */ + +static crash_site_entry_t *crash_site_find(ldcs_process_data_t *procdata, + const char *site, size_t site_len) +{ + int i; + for (i = 0; i < procdata->crash_sites_count; ++i) { + crash_site_entry_t *e = &procdata->crash_sites[i]; + if (e->site_len == site_len && + memcmp(e->site, site, site_len) == 0) { + return e; + } + } + return NULL; +} + +static crash_site_entry_t *crash_site_insert(ldcs_process_data_t *procdata, + const char *site, size_t site_len) +{ + if (procdata->crash_sites_count >= procdata->crash_sites_cap) { + int new_cap = procdata->crash_sites_cap ? procdata->crash_sites_cap * 2 : 8; + procdata->crash_sites = realloc(procdata->crash_sites, + new_cap * sizeof(*procdata->crash_sites)); + procdata->crash_sites_cap = new_cap; + } + char *copy = malloc(site_len + 1); + memcpy(copy, site, site_len); + copy[site_len] = '\0'; + crash_site_entry_t *e = &procdata->crash_sites[procdata->crash_sites_count++]; + e->site = copy; + e->site_len = site_len; + e->resolved = 0; + return e; +} + +static char *crash_pack(int32_t rank, const char *site, size_t site_len, + size_t *out_len) +{ + size_t total = 2 * sizeof(int32_t) + site_len; + char *buf = malloc(total); + int32_t name_len_v = (int32_t) site_len; + memcpy(buf, &rank, sizeof(int32_t)); + memcpy(buf + 1 * sizeof(int32_t), &name_len_v, sizeof(int32_t)); + memcpy(buf + 2 * sizeof(int32_t), site, site_len); + *out_len = total; + return buf; +} + +static void send_crash_response_to_waiter(ldcs_process_data_t *procdata, + crash_waiter_t *w, + int winning_rank, + const char *site, size_t site_len) +{ + if (w->kind == CRASH_WAITER_LOCAL) { + // Delivering a crash response to a local client + int32_t resp = (int32_t) winning_rank; + ldcs_message_t msg; + msg.header.type = LDCS_MSG_CRASH_RESPONSE; + msg.header.len = sizeof(resp); + msg.data = (char *) &resp; + ldcs_client_t *client = procdata->client_table + w->nc; + if (client->state == LDCS_CLIENT_STATUS_ACTIVE && client->connid >= 0) { + debug_printf2("delivering CRASH_RESPONSE to local nc=%d rank=%d winning=%d\n", + w->nc, w->global_rank, winning_rank); + ldcs_send_msg(client->connid, &msg); + } + } else { + // Delivering a crash response to a child + size_t len = 0; + char *buf = crash_pack((int32_t) winning_rank, site, site_len, &len); + ldcs_message_t msg; + msg.header.type = LDCS_MSG_CRASH_RESPONSE; + msg.header.len = len; + msg.data = buf; + debug_printf2("delivering CRASH_RESPONSE to child peer=%p winning=%d\n", + (void *) w->peer, winning_rank); + spindle_send(procdata, &msg, w->peer); + free(buf); + } +} + +static int crash_resolve(ldcs_process_data_t *procdata, + crash_site_entry_t *entry, int winning_rank) +{ + debug_printf2("delivering site '%s' winning=%d\n", + entry->site, winning_rank); + send_crash_response_to_waiter(procdata, &entry->waiter, + winning_rank, entry->site, entry->site_len); + entry->resolved = 1; + return 0; +} + +static int forward_crash_query_up(ldcs_process_data_t *procdata, + const char *site, size_t site_len, + int first_waiter_rank) +{ + size_t total = 0; + char *buf = crash_pack((int32_t) first_waiter_rank, site, site_len, &total); + ldcs_message_t msg; + msg.header.type = LDCS_MSG_CRASH_REPORT; + msg.header.len = total; + msg.data = buf; + int rc = spindle_forward_query(procdata, &msg); + free(buf); + return rc; +} + +static int crash_parse_report(ldcs_message_t *msg, const char *err_str, + int32_t *rank, + const char **site, size_t *site_len) +{ + if (msg->header.len < 2 * sizeof(int32_t)) { + err_printf("malformed crash report in %s\n", err_str); + return -1; + } + int32_t name_len; + memcpy(rank, msg->data, sizeof(int32_t)); + memcpy(&name_len, msg->data + 1 * sizeof(int32_t), sizeof(int32_t)); + if (name_len <= 0) { + err_printf("bad name_len %d in %s\n", (int) name_len, err_str); + return -1; + } + *site = msg->data + 2 * sizeof(int32_t); + *site_len = (size_t) name_len; + return 0; +} + +static int crash_report_common(ldcs_process_data_t *procdata, + crash_waiter_t *w, int reporter_rank, + const char *site, size_t site_len) +{ + // If we have already seen this crash site before, then we know + // it can't be the winner and can short-circuit and respond + // immediately that this rank was not selected. + crash_site_entry_t *e = crash_site_find(procdata, site, site_len); + if (e) { + debug_printf2("known crash site '%s' (%s); suppressing %s reporter rank=%d\n", + e->site, e->resolved ? "resolved" : "in flight", + w->kind == CRASH_WAITER_LOCAL ? "local" : "child", + reporter_rank); + send_crash_response_to_waiter(procdata, w, -1, site, site_len); + return 0; + } + + // If we haven't seen this crash site before, remember that we have + // seen it and forward up the tree for resolution. + e = crash_site_insert(procdata, site, site_len); + e->waiter = *w; + + // If we reached the root without finding a decision already made, + // then this was the first instance of this crash site; + // select this rank to produce the exemplar coredump. + if (ldcs_audit_server_md_is_responsible(procdata, "")) { + debug_printf2("new crash site '%s' at root; selecting rank %d\n", + e->site, reporter_rank); + return crash_resolve(procdata, e, reporter_rank); + } + + debug_printf2("new crash site '%s' at interior; forwarding upward %d\n", + e->site, reporter_rank); + return forward_crash_query_up(procdata, site, site_len, reporter_rank); +} + +/* PUBLIC API */ + +// Act on a crash report received from a local client +int handle_client_crash_report(ldcs_process_data_t *procdata, + int nc, ldcs_message_t *msg) +{ + if (!(procdata->opts & OPT_CRASH_HANDLER)) { + return 0; + } + + debug_printf2("Received CRASH_REPORT from local client nc=%d (len=%lu)\n", + nc, (unsigned long) msg->header.len); + + int32_t rank_raw; + const char *site; + size_t site_len; + if (crash_parse_report(msg, "crash report from local client", &rank_raw, &site, &site_len) != 0) + return -1; + + crash_waiter_t w; + w.kind = CRASH_WAITER_LOCAL; + w.nc = nc; + w.global_rank = (int) rank_raw; + w.peer = NULL; + return crash_report_common(procdata, &w, (int) rank_raw, site, site_len); +} + +// Act on a crash report passed to us from a child +int handle_crash_report_recv(ldcs_process_data_t *procdata, + node_peer_t peer, ldcs_message_t *msg) +{ + int32_t first_waiter_rank; + const char *site; + size_t site_len; + if (crash_parse_report(msg, "crash report from child", &first_waiter_rank, &site, &site_len) != 0) + return -1; + + crash_waiter_t w; + w.kind = CRASH_WAITER_CHILD; + w.nc = -1; + w.global_rank = -1; + w.peer = peer; + return crash_report_common(procdata, &w, (int) first_waiter_rank, site, site_len); +} + +// Act on a crash response passed to us from our parent +int handle_crash_response_recv(ldcs_process_data_t *procdata, + node_peer_t peer, ldcs_message_t *msg) +{ + if (!ldcs_audit_server_md_is_parent(peer)) { + err_printf("unexpectedly got CRASH_RESPONSE from peer other than parent\n"); + return -1; + } + + int32_t selected_rank; + const char *site; + size_t site_len; + if (crash_parse_report(msg, "crash response from parent", &selected_rank, &site, &site_len) != 0) + return -1; + + crash_site_entry_t *e = crash_site_find(procdata, site, site_len); + if (!e) { + err_printf("unexpectedly got CRASH_RESPONSE for unknown site %.*s\n", (int)site_len, site); + return -1; + } + if (e->resolved) { + err_printf("unexpectedly got CRASH_RESPONSE for already resolved site %.*s\n", (int)site_len, site); + return -1; + } + debug_printf2("received CRASH_RESPONSE site '%.*s' selected=%d\n", (int) site_len, site, (int) selected_rank); + + return crash_resolve(procdata, e, (int) selected_rank); +} + +// Free the tables that store cached crash-site decisions. +void crash_free_tables(ldcs_process_data_t *procdata) +{ + int i; + if (procdata->crash_sites) { + for (i = 0; i < procdata->crash_sites_count; ++i) { + free(procdata->crash_sites[i].site); + } + free(procdata->crash_sites); + procdata->crash_sites = NULL; + procdata->crash_sites_count = 0; + procdata->crash_sites_cap = 0; + } +} diff --git a/src/server/auditserver/ldcs_audit_server_crash_handler.h b/src/server/auditserver/ldcs_audit_server_crash_handler.h new file mode 100644 index 00000000..1a419930 --- /dev/null +++ b/src/server/auditserver/ldcs_audit_server_crash_handler.h @@ -0,0 +1,39 @@ +/* +This file is part of Spindle. For copyright information see the COPYRIGHT +file in the top level directory, or at +https://github.com/hpc/Spindle/blob/master/COPYRIGHT + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU Lesser General Public License (as published by the Free Software +Foundation) version 2.1 dated February 1999. This program is distributed in the +hope that it will be useful, but WITHOUT ANY WARRANTY; without even the IMPLIED +WARRANTY OF MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms +and conditions of the GNU Lesser General Public License for more details. You should +have received a copy of the GNU Lesser General Public License along with this +program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA +*/ + +#ifndef LDCS_AUDIT_SERVER_CRASH_HANDLER_H +#define LDCS_AUDIT_SERVER_CRASH_HANDLER_H + +#include "ldcs_api.h" +#include "ldcs_audit_server_process.h" +#include "ldcs_audit_server_md.h" + +/* CRASH_REPORT from a local client */ +int handle_client_crash_report(ldcs_process_data_t *procdata, + int nc, ldcs_message_t *msg); + +/* CRASH_REPORT from a child */ +int handle_crash_report_recv(ldcs_process_data_t *procdata, + node_peer_t peer, ldcs_message_t *msg); + +/* CRASH_RESPONSE from a parent */ +int handle_crash_response_recv(ldcs_process_data_t *procdata, + node_peer_t peer, ldcs_message_t *msg); + +/* Free the crash-site table at shutdown */ +void crash_free_tables(ldcs_process_data_t *procdata); + +#endif diff --git a/src/server/auditserver/ldcs_audit_server_handlers.c b/src/server/auditserver/ldcs_audit_server_handlers.c index 1afdf2e8..ebfdef13 100644 --- a/src/server/auditserver/ldcs_audit_server_handlers.c +++ b/src/server/auditserver/ldcs_audit_server_handlers.c @@ -35,6 +35,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include "stat_cache.h" #include "global_name.h" #include "ldcs_audit_server_handlers.h" +#include "ldcs_audit_server_crash_handler.h" #include "ldcs_audit_server_requestors.h" #include "spindle_launch.h" #include "pathfn.h" @@ -181,7 +182,6 @@ static int handle_setup_alias(ldcs_process_data_t *procdata, char *pathname, cha static int handle_client_dirlists_req(ldcs_process_data_t *procdata, int nc); static int handle_close_client_query(ldcs_process_data_t *procdata, int nc); static int handle_alive_msg(ldcs_process_data_t *procdata, ldcs_message_t *msg); -static int handle_cachepath_consensus(ldcs_process_data_t *procdata, ldcs_message_t *msg); static int handle_chosen_cachepath_request(ldcs_process_data_t *procdata, int nc); extern void getValidCachePathByIndex( uint64_t validBitIdx, char **realizedCachePath, char **parsedCachePath, char **symbolicCachePath ); @@ -1896,6 +1896,8 @@ int handle_client_message(ldcs_process_data_t *procdata, int nc, ldcs_message_t return handle_client_procmaps_msg(procdata, nc, msg); case LDCS_MSG_PICKONE_REQ: return handle_client_pickone_msg(procdata, nc, msg); + case LDCS_MSG_CRASH_REPORT: + return handle_client_crash_report(procdata, nc, msg); case LDCS_MSG_END: return handle_client_end(procdata, nc); case LDCS_MSG_CHOSEN_CACHEPATH_REQUEST: @@ -1997,8 +1999,10 @@ int handle_server_message(ldcs_process_data_t *procdata, node_peer_t peer, ldcs_ case LDCS_MSG_ALIVE_REQ: case LDCS_MSG_ALIVE_RESP: return handle_alive_msg(procdata, msg); - case LDCS_MSG_REQUEST_CACHEPATH_CONSENSUS: - return handle_cachepath_consensus(procdata, msg); + case LDCS_MSG_CRASH_REPORT: + return handle_crash_report_recv(procdata, peer, msg); + case LDCS_MSG_CRASH_RESPONSE: + return handle_crash_response_recv(procdata, peer, msg); default: err_printf("Received unexpected message from node: %d\n", (int) msg->header.type); assert(0); @@ -2961,35 +2965,26 @@ static int handle_client_pickone_msg(ldcs_process_data_t *procdata, int nc, ldcs } /** - * Handle LDCS_MSG_REQUEST_CACHEPATH_CONSENSUS to determine which cachepaths are - * available across all of the servers. + * Determine which cachepaths are available across all of the servers. */ -static int cachepath_consensus_reached; -static int handle_cachepath_consensus(ldcs_process_data_t *procdata, ldcs_message_t *msg){ - +int handle_cachepath_consensus(ldcs_process_data_t *procdata) +{ int num_children = ldcs_audit_server_md_get_num_children(procdata); - debug_printf( "Processing REQUEST_CACHEPATH_CONSENSUS.\n" ); + debug_printf( "Calculating cachepath consensus.\n" ); debug_printf3( " procdata->cachepath_bitidx = %#"PRIx64"\n", procdata->cachepath_bitidx ); debug_printf3( " procdata->cachepaths = %s\n", procdata->cachepaths ); debug_printf3( " procdata->cachepath = %s [should be null]\n", procdata->cachepath ); debug_printf3( " procdata->commpath = %s\n", procdata->commpath ); debug_printf3( " num_children = %d\n", num_children ); - if (num_children) { - spindle_broadcast(procdata, msg); - debug_printf3( "Successfully broadcast REQUEST_CACHEPATH_CONSENSUS\n" ); - msgbundle_force_flush(procdata); - debug_printf3( "Successfully flushed the broadcast of REQUEST_CACHEPATH_CONSENSUS\n" ); - } - ldcs_audit_server_md_allreduce_AND( &procdata->cachepath_bitidx ); debug_printf3( "The consensus value for procdata->cachepath_bitidx is: %#"PRIx64"\n", procdata->cachepath_bitidx ); - if( procdata->cachepath_bitidx == 0 ){ - err_printf("No valid cachepath path available. Falling back to \"commpath\" path (%s).\n", procdata->commpath); + if( procdata->cachepath_bitidx == 0 ) { + debug_printf("No valid cachepath path available. Falling back to \"commpath\" path (%s).\n", procdata->commpath); procdata->cachepath = procdata->commpath; - }else{ + } else { getValidCachePathByIndex( procdata->cachepath_bitidx, &procdata->cachepath, &procdata->parsed_cachepath, @@ -3003,14 +2998,10 @@ static int handle_cachepath_consensus(ldcs_process_data_t *procdata, ldcs_messag ldcs_audit_server_filemngt_init(procdata->cachepath, procdata->commpath); test_printf(" cachepath=%s\n", procdata->cachepath); - cachepath_consensus_reached = 1; return 0; } -/** - * Handle LDCS_MSG_CHOSEN_CACHEPATH_REQUEST - */ -static int handle_chosen_cachepath_request(ldcs_process_data_t *procdata, int nc){ +static int handle_chosen_cachepath_request(ldcs_process_data_t *procdata, int nc) { ldcs_message_t msg; int connid; ldcs_client_t *client; @@ -3022,18 +3013,12 @@ static int handle_chosen_cachepath_request(ldcs_process_data_t *procdata, int nc return 0; - if( cachepath_consensus_reached ){ - msg.header.type = LDCS_MSG_CHOSEN_CACHEPATH; - msg.header.len = strlen(procdata->cachepath) + 1 + strlen(procdata->parsed_cachepath) + 1; - msg.data = calloc( 1, msg.header.len ); - strcpy( msg.data, procdata->cachepath ); - strcpy( &msg.data[ strlen(procdata->cachepath)+1 ], procdata->parsed_cachepath ); - }else{ - msg.header.type = LDCS_MSG_NO_CACHEPATH_CONSENSUS_YET; - msg.header.len = 0; - msg.data = NULL; - } - + msg.header.type = LDCS_MSG_CHOSEN_CACHEPATH; + msg.header.len = strlen(procdata->cachepath) + 1 + strlen(procdata->parsed_cachepath) + 1; + msg.data = calloc( 1, msg.header.len ); + strcpy( msg.data, procdata->cachepath ); + strcpy( &msg.data[ strlen(procdata->cachepath)+1 ], procdata->parsed_cachepath ); + ldcs_send_msg(connid, &msg); free( msg.data ); procdata->server_stat.clientmsg.cnt++; diff --git a/src/server/auditserver/ldcs_audit_server_handlers.h b/src/server/auditserver/ldcs_audit_server_handlers.h index 8702316c..790e52b3 100644 --- a/src/server/auditserver/ldcs_audit_server_handlers.h +++ b/src/server/auditserver/ldcs_audit_server_handlers.h @@ -25,6 +25,7 @@ int handle_server_error(ldcs_process_data_t *procdata, node_peer_t peer); int handle_client_message(ldcs_process_data_t *procdata, int nc, ldcs_message_t *msg); int handle_client_start(ldcs_process_data_t *procdata, int nc); int handle_client_end(ldcs_process_data_t *procdata, int nc); +int handle_cachepath_consensus(ldcs_process_data_t *procdata); int exit_note_cb(int infd, int serverid, void *data); diff --git a/src/server/auditserver/ldcs_audit_server_process.c b/src/server/auditserver/ldcs_audit_server_process.c index b6ecdbff..8cd7b936 100644 --- a/src/server/auditserver/ldcs_audit_server_process.c +++ b/src/server/auditserver/ldcs_audit_server_process.c @@ -31,6 +31,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include "ldcs_audit_server_filemngt.h" #include "ldcs_audit_server_md.h" #include "ldcs_audit_server_handlers.h" +#include "ldcs_audit_server_crash_handler.h" #include "ldcs_cache.h" #include "spindle_launch.h" #include "ldcs_audit_server_requestors.h" @@ -137,7 +138,7 @@ void stopprofile() int ldcs_audit_server_process(spindle_args_t *args) { - int serverid, fd; + int serverid, fd, result; startprofile(args); @@ -172,7 +173,10 @@ int ldcs_audit_server_process(spindle_args_t *args) ldcs_process_data.num_exited_children_peers = 0; ldcs_process_data.num_exited_parents = 0; ldcs_process_data.num_alives = 0; - + ldcs_process_data.crash_sites = NULL; + ldcs_process_data.crash_sites_count = 0; + ldcs_process_data.crash_sites_cap = 0; + if (ldcs_process_data.opts & OPT_PULL) { debug_printf("Using PULL model\n"); ldcs_process_data.dist_model = LDCS_PULL; @@ -198,7 +202,20 @@ int ldcs_audit_server_process(spindle_args_t *args) if (ldcs_process_data.opts & OPT_PROCCLEAN) init_cleanup_proc(ldcs_process_data.cachepath, ldcs_process_data.commpath); - debug_printf3("Initializing connections for clients at %s and %lu\n", + /* Calculate location of cache */ + debug_printf2("Calculating cache path location\n"); + determineValidCachePaths( + &ldcs_process_data.cachepath_bitidx, + ldcs_process_data.cachepaths, + ldcs_process_data.number ); + result = handle_cachepath_consensus(&ldcs_process_data); + if (result == -1) { + err_printf("Could not determine cachepath consensus\n"); + return -1; + } + + /* Setup connections for clients to start connecting */ + debug_printf2("Initializing connections for clients at %s and %lu\n", ldcs_process_data.commpath, (unsigned long) ldcs_process_data.number); serverid = ldcs_create_server(ldcs_process_data.commpath, ldcs_process_data.number); if (serverid == -1) { @@ -231,12 +248,9 @@ int ldcs_audit_server_process(spindle_args_t *args) if (fd != -1) { ldcs_listen_register_fd(fd, serverid, forceExitCB, (void *) &ldcs_process_data); } - determineValidCachePaths( - &ldcs_process_data.cachepath_bitidx, - ldcs_process_data.cachepaths, - ldcs_process_data.number ); + return 0; -} +} int ldcs_audit_server_run() { @@ -261,7 +275,9 @@ int ldcs_audit_server_run() ldcs_audit_server_md_destroy(&ldcs_process_data); msgbundle_done(&ldcs_process_data); - + + crash_free_tables(&ldcs_process_data); + /* destroy file cache */ if (!(ldcs_process_data.opts & OPT_NOCLEAN)) { ldcs_audit_server_filemngt_clean(); diff --git a/src/server/auditserver/ldcs_audit_server_process.h b/src/server/auditserver/ldcs_audit_server_process.h index 82b60023..4a1734ae 100644 --- a/src/server/auditserver/ldcs_audit_server_process.h +++ b/src/server/auditserver/ldcs_audit_server_process.h @@ -23,9 +23,9 @@ extern "C" { #include "ldcs_api.h" #include "spindle_launch.h" -#include "stat_cache.h" +#include "stat_cache.h" #include "force_exit.h" - + typedef void* requestor_list_t; /* client description structure */ @@ -111,7 +111,28 @@ typedef struct msgbundle_entry_t { struct msgbundle_entry_t *next; char name[16]; } msgbundle_entry_t; - + +typedef enum { + CRASH_WAITER_LOCAL = 0, + CRASH_WAITER_CHILD = 1 +} crash_waiter_kind_t; + +typedef void* node_peer_t; + +typedef struct crash_waiter_t { + crash_waiter_kind_t kind; + int nc; + int global_rank; + node_peer_t peer; +} crash_waiter_t; + +typedef struct crash_site_entry_t { + char *site; + size_t site_len; + int resolved; + crash_waiter_t waiter; +} crash_site_entry_t; + struct ldcs_process_data_struct { int client_table_size; @@ -163,6 +184,10 @@ struct ldcs_process_data_struct requestor_list_t pending_ldso_requests; requestor_list_t completed_ldso_requests; + crash_site_entry_t *crash_sites; + int crash_sites_count; + int crash_sites_cap; + /* multi daemon support */ int md_rank; int md_size; diff --git a/src/server/comlib/ldcs_api_util.c b/src/server/comlib/ldcs_api_util.c index af101815..e9c6cb89 100644 --- a/src/server/comlib/ldcs_api_util.c +++ b/src/server/comlib/ldcs_api_util.c @@ -90,11 +90,11 @@ char* _message_type_to_str (ldcs_message_ids_t type) { STR_CASE(LDCS_MSG_PICKONE_REQ); STR_CASE(LDCS_MSG_PICKONE_RESP); STR_CASE(LDCS_MSG_ALIVE_REQ); - STR_CASE(LDCS_MSG_ALIVE_RESP); - STR_CASE(LDCS_MSG_REQUEST_CACHEPATH_CONSENSUS); + STR_CASE(LDCS_MSG_ALIVE_RESP); STR_CASE(LDCS_MSG_CHOSEN_CACHEPATH_REQUEST); STR_CASE(LDCS_MSG_CHOSEN_CACHEPATH); - STR_CASE(LDCS_MSG_NO_CACHEPATH_CONSENSUS_YET); + STR_CASE(LDCS_MSG_CRASH_REPORT); + STR_CASE(LDCS_MSG_CRASH_RESPONSE); STR_CASE(LDCS_MSG_UNKNOWN); } return "unknown"; diff --git a/src/server/config.h.in b/src/server/config.h.in index 24040627..14c85b47 100644 --- a/src/server/config.h.in +++ b/src/server/config.h.in @@ -27,6 +27,9 @@ /* Define if were using sockets for client/server communication */ #undef COMM_SOCKET +/* Enable --crash-dedup by default */ +#undef CRASH_DEDUP_ENABLED_BY_DEFAULT + /* Defined if GLIBC has an allocation bug */ #undef DTV_ALLOCATION_BUG diff --git a/src/server/configure b/src/server/configure index 83bff908..71d5fb82 100755 --- a/src/server/configure +++ b/src/server/configure @@ -869,6 +869,7 @@ enable_sec_munge enable_sec_keydir enable_sec_launchmon enable_sec_none +enable_crash_dedup with_gcrypt_dir with_munge_dir with_glibc_be_dir @@ -1555,6 +1556,7 @@ Optional Features: --enable-sec-launchmon Enable LaunchmMON to propagate authentication key for connections --enable-sec-none Disable security authentication of connections + --enable-crash-dedup Enable --crash-dedup by default (default: disabled) --enable-remap-pageone Controls mmap behavior when replacing executables. Should be false on older linux systems @@ -17553,6 +17555,20 @@ if test "${enable_sec_none+set}" = set; then : fi +# Check whether --enable-crash-dedup was given. +if test "${enable_crash_dedup+set}" = set; then : + enableval=$enable_crash_dedup; enable_crash_dedup=$enableval +else + enable_crash_dedup=no +fi + + +if test "x$enable_crash_dedup" = xyes; then : + +$as_echo "#define CRASH_DEDUP_ENABLED_BY_DEFAULT 1" >>confdefs.h + +fi + # Check whether --with-gcrypt-dir was given. if test "${with_gcrypt_dir+set}" = set; then : diff --git a/testsuite/Makefile.am b/testsuite/Makefile.am index fa0ea36a..1ce88385 100644 --- a/testsuite/Makefile.am +++ b/testsuite/Makefile.am @@ -3,6 +3,8 @@ noinst_PROGRAMS = libgenerator ABS_TEST_DIR = $(abspath $(top_builddir)/testsuite) BUILT_SOURCES = libtest10.so libtest11.so libtest12.so libtest13.so libtest14.so libtest15.so libtest16.so libtest17.so libtest18.so libtest19.so libtest20.so libtest50.so libtest100.so libtest500.so libtest1000.so libtest2000.so libtest4000.so libtest6000.so libtest8000.so libtest10000.so libtls1.c libtls2.c libtls3.c libtls4.c libtls5.c libtls6.c libtls7.c libtls8.c libtls9.c libtls10.c libtls11.c libtls12.c libtls13.c libtls14.c libtls15.c libtls16.c libtls17.c libtls18.c libtls19.c libtls20.c libsymlink.so libdepC.so libdepB.so libdepA.so libcxxexceptB.so libcxxexceptA.so origin_dir/liboriginlib.so origin_dir/origin_subdir/liborigintarget.so libtestoutput.so libfuncdict.so runTests run_driver run_driver_rm spindle.rc preload_file_list test_driver test_driver_libs retzero_rx retzero_r retzero_x retzero_ badinterp hello_r.py hello_x.py hello_rx.py hello_.py hello_l.py badlink.py spindle_exec_test spindle_deactivated.sh liblocal.so symbind_test interpreter_test interpreter_test_dir/interpreter_test_perl alias/aliastest.py exec_shell.sh exec_shell.tcsh exec_shell_expected_output exec_shell exec_shell_env.sh +BUILT_SOURCES += crash_test crash_test_fixedaddr crash_test_pie libcrashfuncs.so libcrashctor.so libcrashfixed.so run_crash_tests.sh + if BGQ_BLD DYNAMIC_FLAG=-dynamic IS_BLUEGENE=true @@ -420,6 +422,39 @@ interpreter_test_dir/interpreter_test_perl: interpreter_test_perl_template inter interpreter_test: interpreter_test_template $(AM_V_GEN)cp $< $@; chmod 700 $@ +crash_test: $(srcdir)/crash_test.c $(srcdir)/crash_functions.c $(srcdir)/crash_functions.h libcrashfuncs.so libcrashctor.so libcrashfixed.so + $(AM_V_CCLD)$(MPICC) $(CFLAGS) -O0 -fno-omit-frame-pointer -no-pie -Wl,-E \ + -o $@ $(srcdir)/crash_test.c $(srcdir)/crash_functions.c -I$(srcdir) -ldl -lpthread + +CRASH_FIXED_EXE_ADDR ?= 0x60000000 +crash_test_fixedaddr: $(srcdir)/crash_test.c $(srcdir)/crash_functions.c $(srcdir)/crash_functions.h + $(AM_V_CCLD)$(MPICC) $(CFLAGS) -O0 -fno-omit-frame-pointer -no-pie -Wl,-E \ + -Wl,-Ttext-segment=$(CRASH_FIXED_EXE_ADDR) \ + -o $@ $(srcdir)/crash_test.c $(srcdir)/crash_functions.c -I$(srcdir) -ldl -lpthread + +crash_test_pie: $(srcdir)/crash_test.c $(srcdir)/crash_functions.c $(srcdir)/crash_functions.h + $(AM_V_CCLD)$(MPICC) $(CFLAGS) -O0 -fno-omit-frame-pointer -pie -fPIE -Wl,-E \ + -o $@ $(srcdir)/crash_test.c $(srcdir)/crash_functions.c -I$(srcdir) -ldl -lpthread + +libcrashfuncs.so: $(srcdir)/libcrashfuncs.c + $(AM_V_CCLD)$(CC) $(CFLAGS) -O0 -fno-omit-frame-pointer -shared -fPIC \ + -o $@ $(srcdir)/libcrashfuncs.c + +libcrashctor.so: $(srcdir)/libcrashctor.c + $(AM_V_CCLD)$(CC) $(CFLAGS) -O0 -fno-omit-frame-pointer -shared -fPIC \ + -o $@ $(srcdir)/libcrashctor.c + +CRASH_FIXED_LIB_ADDR ?= 0x10000000 +libcrashfixed.so: $(srcdir)/libcrashfixed.c + $(AM_V_CCLD)$(CC) $(CFLAGS) -O0 -fno-omit-frame-pointer -shared -fPIC \ + -Wl,-Ttext-segment=$(CRASH_FIXED_LIB_ADDR) \ + -o $@ $(srcdir)/libcrashfixed.c + +run_crash_tests.sh: $(srcdir)/run_crash_tests_template.sh $(top_builddir)/Makefile + $(AM_V_GEN)$(SED) -e s,TEST_RESOURCE_MANAGER,$(TESTRM),g\;s,SPINDLE_EXEC,$(bindir)/spindle,g\;s,SPINDLE_RC_PATH,$(sysconfdir)/spindle/spindle.rc,g\;s,TEST_RUN_DIR,$(ABS_TEST_DIR),g < $< > $@; chmod 700 $@ + +EXTRA_DIST = run_crash_tests_template.sh crash_test.c crash_functions.c crash_functions.h libcrashfuncs.c libcrashctor.c libcrashfixed.c + exec_shell.sh: $(srcdir)/exec_shell_template.sh $(AM_V_GEN)cp $< $@; chmod 700 $@ @@ -435,4 +470,4 @@ exec_shell_expected_output: $(srcdir)/exec_shell_expected_output_template exec_shell: $(srcdir)/exec_shell.c $(AM_V_CCLD)$(CC) $(CFLAGS) -o $@ $< -CLEANFILES = libtest10.c libtest11.c libtest12.c libtest13.c libtest14.c libtest15.c libtest16.c libtest17.c libtest18.c libtest19.c libtest20.c libtest10.so libtest50.c libtest50.so libtest100.c libtest100.so libtest500.c libtest500.so libtest1000.c libtest1000.so libtest2000.c libtest2000.so libtest4000.c libtest4000.so libtest6000.c libtest6000.so libtest8000.c libtest8000.so libtest10000.c libtest10000.so libsymlink.so libdepA.so libdepB.so libdepC.so libcxxexceptA.so libcxxexceptB.so libtestoutput.so libfuncdict.so runTests run_driver run_driver_rm spindle.rc test_driver test_driver_libs preload_file_list retzero_rx retzero_r retzero_x retzero_ badinterp hello_r.py hello_x.py hello_rx.py hello_.py hello_l.py badlink.py libtls1.c libtls2.c libtls3.c libtls4.c libtls5.c libtls6.c libtls7.c libtls8.c libtls9.c libtls10.c libtls11.c libtls12.c libtls13.c libtls14.c libtls15.c libtls16.c libtls17.c libtls18.c libtls19.c libtls20.c libtls1.so libtls2.so libtls3.so libtls4.so libtls5.so libtls6.so libtls7.so libtls8.so libtls9.so libtls10.so libtls11.so libtls12.so libtls13.so libtls14.so libtls15.so libtls16.so libtls17.so libtls18.so libtls19.so libtls20.so symbind_test libsymbind_a.so libsymbind_b.so libsymbind_c.so libsymbind_d.so libsymbind_e.so libsymbind_f.so libsymbind_g.so interpreter_test interpreter_test_dir/interpreter_test_perl alias exec_shell.sh exec_shell.tcsh exec_shell_env.sh exec_shell_expected_output exec_shell +CLEANFILES = libtest10.c libtest11.c libtest12.c libtest13.c libtest14.c libtest15.c libtest16.c libtest17.c libtest18.c libtest19.c libtest20.c libtest10.so libtest50.c libtest50.so libtest100.c libtest100.so libtest500.c libtest500.so libtest1000.c libtest1000.so libtest2000.c libtest2000.so libtest4000.c libtest4000.so libtest6000.c libtest6000.so libtest8000.c libtest8000.so libtest10000.c libtest10000.so libsymlink.so libdepA.so libdepB.so libdepC.so libcxxexceptA.so libcxxexceptB.so libtestoutput.so libfuncdict.so runTests run_driver run_driver_rm spindle.rc test_driver test_driver_libs preload_file_list retzero_rx retzero_r retzero_x retzero_ badinterp hello_r.py hello_x.py hello_rx.py hello_.py hello_l.py badlink.py libtls1.c libtls2.c libtls3.c libtls4.c libtls5.c libtls6.c libtls7.c libtls8.c libtls9.c libtls10.c libtls11.c libtls12.c libtls13.c libtls14.c libtls15.c libtls16.c libtls17.c libtls18.c libtls19.c libtls20.c libtls1.so libtls2.so libtls3.so libtls4.so libtls5.so libtls6.so libtls7.so libtls8.so libtls9.so libtls10.so libtls11.so libtls12.so libtls13.so libtls14.so libtls15.so libtls16.so libtls17.so libtls18.so libtls19.so libtls20.so symbind_test libsymbind_a.so libsymbind_b.so libsymbind_c.so libsymbind_d.so libsymbind_e.so libsymbind_f.so libsymbind_g.so interpreter_test interpreter_test_dir/interpreter_test_perl alias exec_shell.sh exec_shell.tcsh exec_shell_env.sh exec_shell_expected_output exec_shell crash_test crash_test_fixedaddr crash_test_pie libcrashfuncs.so libcrashctor.so libcrashfixed.so run_crash_tests.sh diff --git a/testsuite/Makefile.in b/testsuite/Makefile.in index 290fe9a1..f3402c0a 100644 --- a/testsuite/Makefile.in +++ b/testsuite/Makefile.in @@ -358,7 +358,10 @@ BUILT_SOURCES = libtest10.so libtest11.so libtest12.so libtest13.so \ spindle_deactivated.sh liblocal.so symbind_test \ interpreter_test interpreter_test_dir/interpreter_test_perl \ alias/aliastest.py exec_shell.sh exec_shell.tcsh \ - exec_shell_expected_output exec_shell exec_shell_env.sh + exec_shell_expected_output exec_shell exec_shell_env.sh \ + crash_test crash_test_fixedaddr crash_test_pie \ + libcrashfuncs.so libcrashctor.so libcrashfixed.so \ + run_crash_tests.sh @BGQ_BLD_FALSE@DYNAMIC_FLAG = @BGQ_BLD_TRUE@DYNAMIC_FLAG = -dynamic @BGQ_BLD_FALSE@IS_BLUEGENE = false @@ -374,6 +377,7 @@ test_driver_libsLDADD = -ltest10 -ltest11 -ltest12 -ltest13 -ltest14 -ltest15 -l test_driver_libsLDFLAGS = -Wl,-E -L$(top_builddir)/testsuite $(MPI_CLDFLAGS) -L$(top_builddir)/src/client/spindle_api -L. $(DYNAMIC_FLAG) -no-install $(LDFLAGS) -Wl,-rpath,$(PWD)/origin_dir -L$(PWD)/origin_dir -Wl,-rpath-link,$(PWD)/origin_dir/origin_subdir -I$(top_builddir) REGLIB_SRC = $(srcdir)/registerlib.c LD_FUNCDICT = -L$(top_builddir)/testsuite -lfuncdict +EXTRA_DIST = run_crash_tests_template.sh crash_test.c crash_functions.c crash_functions.h libcrashfuncs.c libcrashctor.c libcrashfixed.c CLEANFILES = libtest10.c libtest11.c libtest12.c libtest13.c \ libtest14.c libtest15.c libtest16.c libtest17.c libtest18.c \ libtest19.c libtest20.c libtest10.so libtest50.c libtest50.so \ @@ -400,7 +404,9 @@ CLEANFILES = libtest10.c libtest11.c libtest12.c libtest13.c \ libsymbind_g.so interpreter_test \ interpreter_test_dir/interpreter_test_perl alias exec_shell.sh \ exec_shell.tcsh exec_shell_env.sh exec_shell_expected_output \ - exec_shell + exec_shell crash_test crash_test_fixedaddr crash_test_pie \ + libcrashfuncs.so libcrashctor.so libcrashfixed.so \ + run_crash_tests.sh all: $(BUILT_SOURCES) $(MAKE) $(AM_MAKEFLAGS) all-am @@ -1100,6 +1106,37 @@ interpreter_test_dir/interpreter_test_perl: interpreter_test_perl_template inter interpreter_test: interpreter_test_template $(AM_V_GEN)cp $< $@; chmod 700 $@ +crash_test: $(srcdir)/crash_test.c $(srcdir)/crash_functions.c $(srcdir)/crash_functions.h libcrashfuncs.so libcrashctor.so libcrashfixed.so + $(AM_V_CCLD)$(MPICC) $(CFLAGS) -O0 -fno-omit-frame-pointer -no-pie -Wl,-E \ + -o $@ $(srcdir)/crash_test.c $(srcdir)/crash_functions.c -I$(srcdir) -ldl -lpthread + +CRASH_FIXED_EXE_ADDR ?= 0x60000000 +crash_test_fixedaddr: $(srcdir)/crash_test.c $(srcdir)/crash_functions.c $(srcdir)/crash_functions.h + $(AM_V_CCLD)$(MPICC) $(CFLAGS) -O0 -fno-omit-frame-pointer -no-pie -Wl,-E \ + -Wl,-Ttext-segment=$(CRASH_FIXED_EXE_ADDR) \ + -o $@ $(srcdir)/crash_test.c $(srcdir)/crash_functions.c -I$(srcdir) -ldl -lpthread + +crash_test_pie: $(srcdir)/crash_test.c $(srcdir)/crash_functions.c $(srcdir)/crash_functions.h + $(AM_V_CCLD)$(MPICC) $(CFLAGS) -O0 -fno-omit-frame-pointer -pie -fPIE -Wl,-E \ + -o $@ $(srcdir)/crash_test.c $(srcdir)/crash_functions.c -I$(srcdir) -ldl -lpthread + +libcrashfuncs.so: $(srcdir)/libcrashfuncs.c + $(AM_V_CCLD)$(CC) $(CFLAGS) -O0 -fno-omit-frame-pointer -shared -fPIC \ + -o $@ $(srcdir)/libcrashfuncs.c + +libcrashctor.so: $(srcdir)/libcrashctor.c + $(AM_V_CCLD)$(CC) $(CFLAGS) -O0 -fno-omit-frame-pointer -shared -fPIC \ + -o $@ $(srcdir)/libcrashctor.c + +CRASH_FIXED_LIB_ADDR ?= 0x10000000 +libcrashfixed.so: $(srcdir)/libcrashfixed.c + $(AM_V_CCLD)$(CC) $(CFLAGS) -O0 -fno-omit-frame-pointer -shared -fPIC \ + -Wl,-Ttext-segment=$(CRASH_FIXED_LIB_ADDR) \ + -o $@ $(srcdir)/libcrashfixed.c + +run_crash_tests.sh: $(srcdir)/run_crash_tests_template.sh $(top_builddir)/Makefile + $(AM_V_GEN)$(SED) -e s,TEST_RESOURCE_MANAGER,$(TESTRM),g\;s,SPINDLE_EXEC,$(bindir)/spindle,g\;s,SPINDLE_RC_PATH,$(sysconfdir)/spindle/spindle.rc,g\;s,TEST_RUN_DIR,$(ABS_TEST_DIR),g < $< > $@; chmod 700 $@ + exec_shell.sh: $(srcdir)/exec_shell_template.sh $(AM_V_GEN)cp $< $@; chmod 700 $@ diff --git a/testsuite/crash_functions.c b/testsuite/crash_functions.c new file mode 100644 index 00000000..9a9aa4f8 --- /dev/null +++ b/testsuite/crash_functions.c @@ -0,0 +1,86 @@ +/* +This file is part of Spindle. For copyright information see the COPYRIGHT +file in the top level directory, or at +https://github.com/hpc/Spindle/blob/master/COPYRIGHT + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU Lesser General Public License (as published by the Free Software +Foundation) version 2.1 dated February 1999. This program is distributed in the +hope that it will be useful, but WITHOUT ANY WARRANTY; without even the IMPLIED +WARRANTY OF MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms +and conditions of the GNU Lesser General Public License for more details. You should +have received a copy of the GNU Lesser General Public License along with this +program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA +*/ + +#include "crash_functions.h" + +__attribute__((noinline, visibility("default"))) +void crash_function_A(int rank) { + volatile int *p = (int *) 0; + (void) rank; + *p = 0; +} + +__attribute__((noinline, visibility("default"))) +void crash_function_B(int rank) { + volatile int *p = (int *) 0; + (void) rank; + *p = 0; +} + +#define DEFINE_CRASH_N(n) \ + __attribute__((noinline, visibility("default"))) \ + void crash_function_##n(int rank) { \ + volatile int *p = (int *) 0; \ + (void) rank; \ + *p = 0; \ + } + +DEFINE_CRASH_N(0) DEFINE_CRASH_N(1) DEFINE_CRASH_N(2) DEFINE_CRASH_N(3) +DEFINE_CRASH_N(4) DEFINE_CRASH_N(5) DEFINE_CRASH_N(6) DEFINE_CRASH_N(7) +DEFINE_CRASH_N(8) DEFINE_CRASH_N(9) DEFINE_CRASH_N(10) DEFINE_CRASH_N(11) +DEFINE_CRASH_N(12) DEFINE_CRASH_N(13) DEFINE_CRASH_N(14) DEFINE_CRASH_N(15) +DEFINE_CRASH_N(16) DEFINE_CRASH_N(17) DEFINE_CRASH_N(18) DEFINE_CRASH_N(19) +DEFINE_CRASH_N(20) DEFINE_CRASH_N(21) DEFINE_CRASH_N(22) DEFINE_CRASH_N(23) +DEFINE_CRASH_N(24) DEFINE_CRASH_N(25) DEFINE_CRASH_N(26) DEFINE_CRASH_N(27) +DEFINE_CRASH_N(28) DEFINE_CRASH_N(29) DEFINE_CRASH_N(30) DEFINE_CRASH_N(31) +DEFINE_CRASH_N(32) DEFINE_CRASH_N(33) DEFINE_CRASH_N(34) DEFINE_CRASH_N(35) +DEFINE_CRASH_N(36) DEFINE_CRASH_N(37) DEFINE_CRASH_N(38) DEFINE_CRASH_N(39) +DEFINE_CRASH_N(40) DEFINE_CRASH_N(41) DEFINE_CRASH_N(42) DEFINE_CRASH_N(43) +DEFINE_CRASH_N(44) DEFINE_CRASH_N(45) DEFINE_CRASH_N(46) DEFINE_CRASH_N(47) +DEFINE_CRASH_N(48) DEFINE_CRASH_N(49) DEFINE_CRASH_N(50) DEFINE_CRASH_N(51) +DEFINE_CRASH_N(52) DEFINE_CRASH_N(53) DEFINE_CRASH_N(54) DEFINE_CRASH_N(55) +DEFINE_CRASH_N(56) DEFINE_CRASH_N(57) DEFINE_CRASH_N(58) DEFINE_CRASH_N(59) +DEFINE_CRASH_N(60) DEFINE_CRASH_N(61) DEFINE_CRASH_N(62) DEFINE_CRASH_N(63) + +crash_fn_t crash_table[CRASH_TABLE_SIZE] = { + crash_function_0, crash_function_1, crash_function_2, crash_function_3, + crash_function_4, crash_function_5, crash_function_6, crash_function_7, + crash_function_8, crash_function_9, crash_function_10, crash_function_11, + crash_function_12, crash_function_13, crash_function_14, crash_function_15, + crash_function_16, crash_function_17, crash_function_18, crash_function_19, + crash_function_20, crash_function_21, crash_function_22, crash_function_23, + crash_function_24, crash_function_25, crash_function_26, crash_function_27, + crash_function_28, crash_function_29, crash_function_30, crash_function_31, + crash_function_32, crash_function_33, crash_function_34, crash_function_35, + crash_function_36, crash_function_37, crash_function_38, crash_function_39, + crash_function_40, crash_function_41, crash_function_42, crash_function_43, + crash_function_44, crash_function_45, crash_function_46, crash_function_47, + crash_function_48, crash_function_49, crash_function_50, crash_function_51, + crash_function_52, crash_function_53, crash_function_54, crash_function_55, + crash_function_56, crash_function_57, crash_function_58, crash_function_59, + crash_function_60, crash_function_61, crash_function_62, crash_function_63, +}; + +static __attribute__((noinline)) +void static_crash_inner(int rank) { + volatile int *p = (int *) 0; + (void) rank; + *p = 0; +} + +void static_crash(int rank) { + static_crash_inner(rank); +} diff --git a/testsuite/crash_functions.h b/testsuite/crash_functions.h new file mode 100644 index 00000000..34f0a770 --- /dev/null +++ b/testsuite/crash_functions.h @@ -0,0 +1,30 @@ +/* +This file is part of Spindle. For copyright information see the COPYRIGHT +file in the top level directory, or at +https://github.com/hpc/Spindle/blob/master/COPYRIGHT + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU Lesser General Public License (as published by the Free Software +Foundation) version 2.1 dated February 1999. This program is distributed in the +hope that it will be useful, but WITHOUT ANY WARRANTY; without even the IMPLIED +WARRANTY OF MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms +and conditions of the GNU Lesser General Public License for more details. You should +have received a copy of the GNU Lesser General Public License along with this +program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA +*/ + +#ifndef CRASH_FUNCTIONS_H +#define CRASH_FUNCTIONS_H + +void crash_function_A(int rank); +void crash_function_B(int rank); + +#define CRASH_TABLE_SIZE 64 + +typedef void (*crash_fn_t)(int); +extern crash_fn_t crash_table[CRASH_TABLE_SIZE]; + +void static_crash(int rank); + +#endif diff --git a/testsuite/crash_test.c b/testsuite/crash_test.c new file mode 100644 index 00000000..b49c8c2e --- /dev/null +++ b/testsuite/crash_test.c @@ -0,0 +1,838 @@ +/* +This file is part of Spindle. For copyright information see the COPYRIGHT +file in the top level directory, or at +https://github.com/hpc/Spindle/blob/master/COPYRIGHT + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU Lesser General Public License (as published by the Free Software +Foundation) version 2.1 dated February 1999. This program is distributed in the +hope that it will be useful, but WITHOUT ANY WARRANTY; without even the IMPLIED +WARRANTY OF MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms +and conditions of the GNU Lesser General Public License for more details. You should +have received a copy of the GNU Lesser General Public License along with this +program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA +*/ + +/* + * Crash-handler test driver. + * * + * SIGSEGV modes: + * all-same every rank crashes in crash_function_A + * all-different rank N crashes in crash_function_, up to 64 ranks + * partial even ranks crash in crash_function_A, + * odd ranks exit cleanly + * late-straggler rank 0 crashes immediately; others sleep --sleep + * seconds before crashing + * two-groups even ranks call crash_function_A, odd ranks call crash_function_B + * one-crashes rank 0 crashes; others exit cleanly + * in-library every rank dlopen()s libcrashfuncs.so and calls + * crash_in_library() + * in-dlmopen-library + * every rank dlmopen()s libcrashfuncs.so and calls crash_in_library() + * in-fixed-library + * every rank dlopens() a library with a fixed load address + * and calls a crashing function + * in-fixed-dlmopen-library + * every rank dlmopen()s a fixed-load-address library into a new + * namespace and calls a crashing function + * in-library-ctor rank 0 dlopen()s libcrashctor.so, whose constructor + * crashes inside dlopen; others exit cleanly + * span-read no app handler; a read spans two pages (PROT_READ then + * PROT_NONE) and faults in the second page + * + * SIGABRT modes: + * sigabrt every rank calls abort() + * assert every rank issues a failing assert() + * mixed-abort-segv + * even ranks assert, odd ranks segfault + * + * Handler chaining modes + * safepoint app handler fixes segfault on read + * safepoint-then-crash + * app handler fixes segfault on read repeatedly, + * then a real crash occurs on same thread. + * safepoint-bad application handler returns but fails to fix read fault + * safepoint-bad-write + * application handler returns but fails to fix write fault + * safepoint-fix-write + * application handler fixes segfault on write + * safepoint-longjmp + application handler siglongjmp()s out of the handler + * safepoint-span-read + * app handler fixes a read fault whose access spans two pages + * (starts in an accessible page, faults in the next one) + * safepoint-span-write + * app handler fixes a write fault whose access spans two pages + * safepoint-span-bad-read + * app handler returns without fixing a page-spanning read fault + * safepoint-span-bad-write + * app handler returns without fixing a page-spanning write fault + * mmap-sigbus-bad + * app handler does not fix access past the end of an mmap'ed file + * mmap-sigbus-fixed + * app handler fixes access past the end of an mmap'ed file by extending it + * + * no-crash every rank exits cleanly + */ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "crash_functions.h" + +#define SAFEPOINT_RC_INCOMPLETE 103 +#define SAFEPOINT_RC_SETUP_FAILED 104 +#define SAFEPOINT_RC_NOT_TERMINATED 105 + +static void usage(const char *prog) { + fprintf(stderr, + "usage: %s --crash-mode {all-same|all-different|partial|" + "late-straggler|two-groups|one-crashes|in-library|" + "in-dlmopen-library|in-fixed-library|in-fixed-dlmopen-library|" + "in-library-ctor|sigabrt|assert|" + "long-assert|mixed-abort-segv|span-read|" + "safepoint|safepoint-then-crash|" + "safepoint-bad|safepoint-bad-write|safepoint-fix-write|" + "safepoint-span-read|safepoint-span-write|" + "safepoint-span-bad-read|safepoint-span-bad-write|" + "mmap-sigbus-bad|mmap-sigbus-fixed|" + "safepoint-longjmp|safepoint-longjmp-mt|safepoint-concurrent-chain|no-crash}" + " [--sleep ] [--cycles ]\n", + prog); +} + +static inline int addr_in_range(uintptr_t a, void *base, size_t len) { + uintptr_t lo = (uintptr_t) base; + return a >= lo && a < lo + len; +} + +static void *map_trap_page(const char *mode, int rank, long pagesize) { + void *p = mmap(NULL, pagesize, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + fprintf(stderr, "%s rank=%d: mmap failed\n", mode, rank); + return NULL; + } + return p; +} + +static int install_sigsegv_handler(void (*handler)(int, siginfo_t *, void *), + const char *mode, int rank) { + struct sigaction sa; + memset(&sa, 0, sizeof sa); + sa.sa_sigaction = handler; + sa.sa_flags = SA_SIGINFO; + sigemptyset(&sa.sa_mask); + if (sigaction(SIGSEGV, &sa, NULL) != 0) { + fprintf(stderr, "%s rank=%d: sigaction failed\n", mode, rank); + return -1; + } + return 0; +} + +static void handler_die(const char *msg) { + ssize_t w = write(2, msg, strlen(msg)); + (void) w; + _exit(SAFEPOINT_RC_SETUP_FAILED); +} + +static void call_crash_in_library_common(int rank, int use_dlmopen) { + const char *dlopen_name = use_dlmopen ? "dlmopen" : "dlopen"; + void *h = use_dlmopen ? dlmopen(LM_ID_NEWLM, "./libcrashfuncs.so", RTLD_NOW) + : dlopen("./libcrashfuncs.so", RTLD_NOW); + if (!h) + h = use_dlmopen ? dlmopen(LM_ID_NEWLM, "libcrashfuncs.so", RTLD_NOW) + : dlopen("libcrashfuncs.so", RTLD_NOW); + if (!h) { + fprintf(stderr, "rank=%d %s of libcrashfuncs.so failed: %s\n", + rank, dlopen_name, dlerror()); + MPI_Abort(MPI_COMM_WORLD, 3); + return; + } + void (*fn)(int) = (void (*)(int)) dlsym(h, "crash_in_library"); + if (!fn) { + fprintf(stderr, "rank=%d dlsym of crash_in_library failed: %s\n", + rank, dlerror()); + MPI_Abort(MPI_COMM_WORLD, 3); + return; + } + fn(rank); +} + +static void call_crash_in_fixed_library_common(int rank, int use_dlmopen) { + const char *dlopen_name = use_dlmopen ? "dlmopen" : "dlopen"; + void *h = use_dlmopen ? dlmopen(LM_ID_NEWLM, "./libcrashfixed.so", RTLD_NOW) + : dlopen("./libcrashfixed.so", RTLD_NOW); + if (!h) + h = use_dlmopen ? dlmopen(LM_ID_NEWLM, "libcrashfixed.so", RTLD_NOW) + : dlopen("libcrashfixed.so", RTLD_NOW); + if (!h) { + fprintf(stderr, "rank=%d %s of libcrashfixed.so failed: %s\n", + rank, dlopen_name, dlerror()); + MPI_Abort(MPI_COMM_WORLD, 3); + return; + } + void (*fn)(int) = (void (*)(int)) dlsym(h, "crash_in_fixed_library"); + if (!fn) { + fprintf(stderr, "rank=%d dlsym of crash_in_fixed_library failed: %s\n", + rank, dlerror()); + MPI_Abort(MPI_COMM_WORLD, 3); + return; + } + fn(rank); +} + +static void crash_in_library_ctor(int rank) { + if (rank != 0) { + fprintf(stderr, "rank=%d exiting cleanly\n", rank); + fflush(stderr); + MPI_Finalize(); + return; + } + + void *handle = dlopen("./libcrashctor.so", RTLD_NOW); + if (!handle) + handle = dlopen("libcrashctor.so", RTLD_NOW); + + (void) handle; + exit(EXIT_FAILURE); +} + +static void do_assert(int rank) { + assert(rank < 0 && "Assertion message"); +} + +static void do_mixed_abort_segv(int rank) { + if ((rank % 2) == 0) { + assert(0 && "Assertion message"); + } else { + *(volatile int *) 0 = 0; + } +} + +static void do_span_read(int rank) { + long pagesize = sysconf(_SC_PAGESIZE); + unsigned char *region = mmap(NULL, 2 * pagesize, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (region == MAP_FAILED) { + fprintf(stderr, "span-read rank=%d: mmap failed\n", rank); + _exit(SAFEPOINT_RC_SETUP_FAILED); + } + memset(region, 0xAB, 2 * pagesize); + if (mprotect(region, pagesize, PROT_READ) != 0 || + mprotect(region + pagesize, pagesize, PROT_NONE) != 0) { + fprintf(stderr, "span-read rank=%d: mprotect failed\n", rank); + _exit(SAFEPOINT_RC_SETUP_FAILED); + } + + volatile uint64_t v = *(volatile uint64_t *) (region + pagesize - 4); + (void) v; + + fprintf(stderr, "span-read rank=%d: unexpectedly continued after SIGSEGV\n", rank); + _exit(SAFEPOINT_RC_NOT_TERMINATED); +} + +/* Safepoint tests. */ + +#define SAFEPOINT_CYCLES_DEFAULT 100 + +static void *safepoint_page; +static long safepoint_pagesize; +static volatile int safepoint_handler_invocations; + +static void safepoint_nofix_handler(int sig, siginfo_t *info, void *uctx) { + (void) sig; (void) info; (void) uctx; + /* App handler that doesn't fix the problem. */ + safepoint_handler_invocations++; +} + +static int safepoint_result(const char *mode, int rank, int cycles) { + if (safepoint_handler_invocations == 0) { + fprintf(stderr, "%s rank=%d: handler never invoked\n", mode, rank); + return SAFEPOINT_RC_INCOMPLETE; + } + fprintf(stderr, + "%s rank=%d: completed %d safepoint cycles, recorded %d invocations\n", + mode, rank, cycles, safepoint_handler_invocations); + return 0; +} + +static int setup_safepoint(const char *mode, int rank, + void (*handler)(int, siginfo_t *, void *)) { + safepoint_pagesize = sysconf(_SC_PAGESIZE); + safepoint_page = map_trap_page(mode, rank, safepoint_pagesize); + if (safepoint_page == NULL) + return SAFEPOINT_RC_SETUP_FAILED; + if (install_sigsegv_handler(handler, mode, rank) != 0) + return SAFEPOINT_RC_SETUP_FAILED; + return 0; +} + +static void safepoint_sigsegv_handler(int sig, siginfo_t *info, void *uctx) { + (void) sig; (void) info; (void) uctx; + safepoint_handler_invocations++; + + /* Unprotect the page so the read succeeds on retry. */ + if (mprotect(safepoint_page, safepoint_pagesize, PROT_READ) != 0) + handler_die("mprotect failed in safepoint_sigsegv_handler\n"); +} + +static int do_safepoint(int rank, int cycles) { + int rc = setup_safepoint("safepoint", rank, safepoint_sigsegv_handler); + if (rc) + return rc; + + for (int i = 0; i < cycles; i++) { + if (mprotect(safepoint_page, safepoint_pagesize, PROT_NONE) != 0) { + fprintf(stderr, "safepoint rank=%d cycle=%d: mprotect PROT_NONE failed\n", + rank, i); + return SAFEPOINT_RC_SETUP_FAILED; + } + volatile int v = *(volatile int *) safepoint_page; + (void) v; + } + + return safepoint_result("safepoint", rank, cycles); +} + +#define SAFEPOINT_THEN_CRASH_PRECYCLES 5 + +static volatile int stc_safepoint_fixes; +static volatile int stc_realcrash_observed; + +static void safepoint_then_crash_handler(int sig, siginfo_t *info, void *uctx) { + (void) sig; (void) uctx; + + int is_safepoint = 0; + if (info != NULL && info->si_addr != NULL && + addr_in_range((uintptr_t) info->si_addr, safepoint_page, safepoint_pagesize)) + is_safepoint = 1; + + if (is_safepoint) { + stc_safepoint_fixes++; + if (mprotect(safepoint_page, safepoint_pagesize, PROT_READ) != 0) { + _exit(SAFEPOINT_RC_SETUP_FAILED); + } + return; + } + + stc_realcrash_observed = 1; + /* This wasn't in the safepoint page, so restore default disposition and return. */ + signal(SIGSEGV, SIG_DFL); +} + +static int do_safepoint_then_crash(int rank) { + int rc = setup_safepoint("safepoint-then-crash", rank, + safepoint_then_crash_handler); + if (rc) + return rc; + + for (int i = 0; i < SAFEPOINT_THEN_CRASH_PRECYCLES; i++) { + if (mprotect(safepoint_page, safepoint_pagesize, PROT_NONE) != 0) { + fprintf(stderr, + "safepoint-then-crash rank=%d cycle=%d: mprotect PROT_NONE failed\n", + rank, i); + return SAFEPOINT_RC_SETUP_FAILED; + } + volatile int v = *(volatile int *) safepoint_page; + (void) v; + } + fprintf(stderr, + "safepoint-then-crash rank=%d: completed %d safepoint cycles, now triggering real crash\n", + rank, stc_safepoint_fixes); + fflush(stderr); + + crash_function_A(rank); + + fprintf(stderr, "safepoint-then-crash rank=%d: crash_function_A unexpectedly returned\n", rank); + return SAFEPOINT_RC_NOT_TERMINATED; +} + +static int do_safepoint_bad(int rank) { + int rc = setup_safepoint("safepoint-bad", rank, safepoint_nofix_handler); + if (rc) + return rc; + + if (mprotect(safepoint_page, safepoint_pagesize, PROT_NONE) != 0) { + fprintf(stderr, + "safepoint-bad rank=%d: mprotect PROT_NONE failed\n", rank); + return SAFEPOINT_RC_SETUP_FAILED; + } + + volatile int v = *(volatile int *) safepoint_page; + (void) v; + + fprintf(stderr, + "safepoint-bad rank=%d: unexpectedly continued after unrecovered SIGSEGV\n", + rank); + return SAFEPOINT_RC_NOT_TERMINATED; +} + +static int do_safepoint_bad_write(int rank) { + int rc = setup_safepoint("safepoint-bad-write", rank, safepoint_nofix_handler); + if (rc) + return rc; + + if (mprotect(safepoint_page, safepoint_pagesize, PROT_READ) != 0) { + fprintf(stderr, + "safepoint-bad-write rank=%d: mprotect PROT_READ failed\n", + rank); + return SAFEPOINT_RC_SETUP_FAILED; + } + + *(volatile int *) safepoint_page = 0; + + fprintf(stderr, + "safepoint-bad-write rank=%d: unexpectedly continued after unrecovered SIGSEGV\n", + rank); + return SAFEPOINT_RC_NOT_TERMINATED; +} + +static void safepoint_fix_write_handler(int sig, siginfo_t *info, void *uctx) { + (void) sig; (void) info; (void) uctx; + + safepoint_handler_invocations++; + + if (mprotect(safepoint_page, safepoint_pagesize, + PROT_READ | PROT_WRITE) != 0) + handler_die("safepoint-fix-write: mprotect failed in handler\n"); +} + +static int do_safepoint_fix_write(int rank, int cycles) { + int rc = setup_safepoint("safepoint-fix-write", rank, + safepoint_fix_write_handler); + if (rc) return rc; + + for (int i = 0; i < cycles; i++) { + if (mprotect(safepoint_page, safepoint_pagesize, PROT_READ) != 0) { + fprintf(stderr, + "safepoint-fix-write rank=%d: mprotect PROT_READ failed at cycle %d\n", + rank, i); + return SAFEPOINT_RC_SETUP_FAILED; + } + *(volatile int *) safepoint_page = i; + } + + return safepoint_result("safepoint-fix-write", rank, cycles); +} + +static sigjmp_buf longjmp_env; +static volatile int longjmp_cycles_completed; + +static void safepoint_longjmp_handler(int sig, siginfo_t *info, void *uctx) { + (void) sig; (void) info; (void) uctx; + if (mprotect(safepoint_page, safepoint_pagesize, PROT_READ) != 0) + handler_die("safepoint-longjmp: mprotect failed in handler\n"); + longjmp_cycles_completed++; + siglongjmp(longjmp_env, 1); +} + +static int do_safepoint_longjmp(int rank, int cycles) { + int rc = setup_safepoint("safepoint-longjmp", rank, + safepoint_longjmp_handler); + if (rc) + return rc; + + for (int i = 0; i < cycles; i++) { + if (sigsetjmp(longjmp_env, 1) == 0) { + if (mprotect(safepoint_page, safepoint_pagesize, + PROT_NONE) != 0) { + fprintf(stderr, + "safepoint-longjmp rank=%d cycle=%d: mprotect PROT_NONE failed\n", rank, i); + return SAFEPOINT_RC_SETUP_FAILED; + } + volatile int v = *(volatile int *) safepoint_page; + (void) v; + fprintf(stderr, + "safepoint-longjmp rank=%d: unexpectedly continued past siglongjmp\n", rank); + return SAFEPOINT_RC_NOT_TERMINATED; + } + } + + if (longjmp_cycles_completed != cycles) { + fprintf(stderr, + "safepoint-longjmp rank=%d: completed %d of %d cycles\n", + rank, longjmp_cycles_completed, cycles); + return SAFEPOINT_RC_INCOMPLETE; + } + fprintf(stderr, + "safepoint-longjmp rank=%d: completed %d siglongjmp cycles\n", + rank, cycles); + return 0; +} + +static unsigned char *span_region; +static long span_pagesize; + +static unsigned char *span_page2(void) { return span_region + span_pagesize; } + +static volatile uint64_t *span_access(void) { + return (volatile uint64_t *) (span_page2() - 4); +} + +static int setup_span(const char *mode, int rank, + void (*handler)(int, siginfo_t *, void *)) { + span_pagesize = sysconf(_SC_PAGESIZE); + span_region = mmap(NULL, 2 * span_pagesize, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (span_region == MAP_FAILED) { + fprintf(stderr, "%s rank=%d: mmap failed\n", mode, rank); + return SAFEPOINT_RC_SETUP_FAILED; + } + memset(span_region, 0xAB, 2 * span_pagesize); + if (install_sigsegv_handler(handler, mode, rank) != 0) + return SAFEPOINT_RC_SETUP_FAILED; + return 0; +} + +static void span_read_fix_handler(int sig, siginfo_t *info, void *uctx) { + (void) sig; (void) info; (void) uctx; + safepoint_handler_invocations++; + if (mprotect(span_page2(), span_pagesize, PROT_READ) != 0) + handler_die("safepoint-span-read: mprotect failed in handler\n"); +} + +static void span_write_fix_handler(int sig, siginfo_t *info, void *uctx) { + (void) sig; (void) info; (void) uctx; + safepoint_handler_invocations++; + if (mprotect(span_page2(), span_pagesize, PROT_READ | PROT_WRITE) != 0) + handler_die("safepoint-span-write: mprotect failed in handler\n"); +} + +static int do_safepoint_span_read(int rank, int cycles) { + int rc = setup_span("safepoint-span-read", rank, span_read_fix_handler); + if (rc) + return rc; + + for (int i = 0; i < cycles; i++) { + if (mprotect(span_page2(), span_pagesize, PROT_NONE) != 0) { + fprintf(stderr, + "safepoint-span-read rank=%d cycle=%d: mprotect PROT_NONE failed\n", + rank, i); + return SAFEPOINT_RC_SETUP_FAILED; + } + volatile uint64_t v = *span_access(); + (void) v; + } + + return safepoint_result("safepoint-span-read", rank, cycles); +} + +static int do_safepoint_span_write(int rank, int cycles) { + int rc = setup_span("safepoint-span-write", rank, span_write_fix_handler); + if (rc) + return rc; + + for (int i = 0; i < cycles; i++) { + if (mprotect(span_page2(), span_pagesize, PROT_NONE) != 0) { + fprintf(stderr, + "safepoint-span-write rank=%d cycle=%d: mprotect PROT_NONE failed\n", + rank, i); + return SAFEPOINT_RC_SETUP_FAILED; + } + *span_access() = (uint64_t) i; + } + + return safepoint_result("safepoint-span-write", rank, cycles); +} + +static int do_safepoint_span_bad_read(int rank) { + int rc = setup_span("safepoint-span-bad-read", rank, safepoint_nofix_handler); + if (rc) + return rc; + + if (mprotect(span_page2(), span_pagesize, PROT_NONE) != 0) { + fprintf(stderr, + "safepoint-span-bad-read rank=%d: mprotect PROT_NONE failed\n", rank); + return SAFEPOINT_RC_SETUP_FAILED; + } + + volatile uint64_t v = *span_access(); + (void) v; + + fprintf(stderr, + "safepoint-span-bad-read rank=%d: unexpectedly continued after unrecovered SIGSEGV\n", + rank); + return SAFEPOINT_RC_NOT_TERMINATED; +} + +static int do_safepoint_span_bad_write(int rank) { + int rc = setup_span("safepoint-span-bad-write", rank, safepoint_nofix_handler); + if (rc) + return rc; + + if (mprotect(span_page2(), span_pagesize, PROT_NONE) != 0) { + fprintf(stderr, + "safepoint-span-bad-write rank=%d: mprotect PROT_NONE failed\n", rank); + return SAFEPOINT_RC_SETUP_FAILED; + } + + *span_access() = 0; + + fprintf(stderr, + "safepoint-span-bad-write rank=%d: unexpectedly continued after unrecovered SIGSEGV\n", + rank); + return SAFEPOINT_RC_NOT_TERMINATED; +} + +static int mmap_sigbus_fd = -1; +static long mmap_sigbus_pagesize; +static unsigned char *mmap_sigbus_map; + +static volatile char *mmap_sigbus_target(void) { + return (volatile char *) (mmap_sigbus_map + mmap_sigbus_pagesize); +} + +static int setup_mmap_sigbus(const char *mode, int rank, + void (*handler)(int, siginfo_t *, void *)) { + mmap_sigbus_pagesize = sysconf(_SC_PAGESIZE); + + char path[] = "/tmp/spindle_mmap_sigbus_XXXXXX"; + mmap_sigbus_fd = mkstemp(path); + if (mmap_sigbus_fd < 0) { + fprintf(stderr, "%s rank=%d: mkstemp failed\n", mode, rank); + return SAFEPOINT_RC_SETUP_FAILED; + } + (void) unlink(path); + if (ftruncate(mmap_sigbus_fd, (off_t) mmap_sigbus_pagesize) != 0) { + fprintf(stderr, "%s rank=%d: ftruncate failed\n", mode, rank); + return SAFEPOINT_RC_SETUP_FAILED; + } + mmap_sigbus_map = mmap(NULL, 2 * mmap_sigbus_pagesize, + PROT_READ | PROT_WRITE, MAP_SHARED, + mmap_sigbus_fd, 0); + if (mmap_sigbus_map == MAP_FAILED) { + fprintf(stderr, "%s rank=%d: mmap failed\n", mode, rank); + return SAFEPOINT_RC_SETUP_FAILED; + } + + struct sigaction sa; + memset(&sa, 0, sizeof sa); + sa.sa_sigaction = handler; + sa.sa_flags = SA_SIGINFO; + sigemptyset(&sa.sa_mask); + if (sigaction(SIGBUS, &sa, NULL) != 0) { + fprintf(stderr, "%s rank=%d: sigaction(SIGBUS) failed\n", mode, rank); + return SAFEPOINT_RC_SETUP_FAILED; + } + return 0; +} + +static void mmap_sigbus_fix_handler(int sig, siginfo_t *info, void *uctx) { + (void) sig; (void) info; (void) uctx; + safepoint_handler_invocations++; + /* Increase the size of the file so the retried read succeeds */ + if (ftruncate(mmap_sigbus_fd, (off_t) (2 * mmap_sigbus_pagesize)) != 0) + handler_die("ftruncate failed in handler\n"); +} + +static int do_mmap_sigbus_fixed(int rank) { + int rc = setup_mmap_sigbus("mmap-sigbus-fixed", rank, + mmap_sigbus_fix_handler); + if (rc) + return rc; + + /* Read one page past EOF. The app handler extends the file so the retried + read succeeds. */ + volatile char v = *mmap_sigbus_target(); + (void) v; + + if (safepoint_handler_invocations == 0) { + fprintf(stderr, "mmap-sigbus-fixed rank=%d: handler never invoked\n", rank); + return SAFEPOINT_RC_INCOMPLETE; + } + return 0; +} + +static int do_mmap_sigbus_bad(int rank) { + int rc = setup_mmap_sigbus("mmap-sigbus-bad", rank, safepoint_nofix_handler); + if (rc) + return rc; + + volatile char v = *mmap_sigbus_target(); + (void) v; + + fprintf(stderr, + "mmap-sigbus-bad rank=%d: unexpectedly continued after unrecovered SIGBUS\n", + rank); + return SAFEPOINT_RC_NOT_TERMINATED; +} + +static int sleep_seconds = 10; +static int safepoint_cycles = SAFEPOINT_CYCLES_DEFAULT; + +int main(int argc, char **argv) { + const char *mode = NULL; + for (int i = 1; i < argc; i++) { + if (strcmp(argv[i], "--crash-mode") == 0 && i + 1 < argc) { + mode = argv[++i]; + } else if (strncmp(argv[i], "--crash-mode=", 13) == 0) { + mode = argv[i] + 13; + } else if (strcmp(argv[i], "--sleep") == 0 && i + 1 < argc) { + int v = atoi(argv[++i]); + if (v >= 1) sleep_seconds = v; + } else if (strncmp(argv[i], "--sleep=", 8) == 0) { + int v = atoi(argv[i] + 8); + if (v >= 1) sleep_seconds = v; + } else if (strcmp(argv[i], "--cycles") == 0 && i + 1 < argc) { + int v = atoi(argv[++i]); + if (v > 0) safepoint_cycles = v; + } else if (strncmp(argv[i], "--cycles=", 9) == 0) { + int v = atoi(argv[i] + 9); + if (v > 0) safepoint_cycles = v; + } else if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) { + usage(argv[0]); + return 0; + } + } + if (!mode) { + usage(argv[0]); + return 2; + } + + MPI_Init(&argc, &argv); + int rank = 0, size = 1; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &size); + + fprintf(stderr, "crash_test rank=%d size=%d mode=%s pid=%d\n", + rank, size, mode, (int) getpid()); + fflush(stderr); + + + /* create and chdir into per-rank directory so that a given rank's + coredumps go into that directory and we can count the coredumps + per rank. */ + { + char rankdir[64]; + snprintf(rankdir, sizeof rankdir, "rank_%d", rank); + (void) mkdir(rankdir, 0755); + if (chdir(rankdir) != 0) { + fprintf(stderr, "rank=%d chdir(%s) failed\n", rank, rankdir); + } + } + + MPI_Barrier(MPI_COMM_WORLD); + + if (strcmp(mode, "all-same") == 0) { + crash_function_A(rank); + } else if (strcmp(mode, "all-different") == 0) { + int idx = rank % CRASH_TABLE_SIZE; + crash_table[idx](rank); + } else if (strcmp(mode, "partial") == 0) { + if ((rank % 2) == 0) { + crash_function_A(rank); + } else { + fprintf(stderr, "rank=%d exiting cleanly\n", rank); + fflush(stderr); + MPI_Finalize(); + return 0; + } + } else if (strcmp(mode, "late-straggler") == 0) { + int sleep_s = sleep_seconds; + if (rank == 0) { + crash_function_A(rank); + } else { + fprintf(stderr, "rank=%d sleeping %ds before crash\n", rank, sleep_s); + fflush(stderr); + sleep(sleep_s); + crash_function_A(rank); + } + } else if (strcmp(mode, "two-groups") == 0) { + if ((rank % 2) == 0) + crash_function_A(rank); + else + crash_function_B(rank); + } else if (strcmp(mode, "one-crashes") == 0) { + if (rank == 0) { + crash_function_A(rank); + } else { + fprintf(stderr, "rank=%d exiting cleanly\n", rank); + fflush(stderr); + MPI_Finalize(); + return 0; + } + } else if (strcmp(mode, "in-library") == 0) { + call_crash_in_library_common(rank, 0); + } else if (strcmp(mode, "in-dlmopen-library") == 0) { + call_crash_in_library_common(rank, 1); + } else if (strcmp(mode, "in-fixed-library") == 0) { + call_crash_in_fixed_library_common(rank, 0); + } else if (strcmp(mode, "in-fixed-dlmopen-library") == 0) { + call_crash_in_fixed_library_common(rank, 1); + } else if (strcmp(mode, "in-library-ctor") == 0) { + crash_in_library_ctor(rank); + } else if (strcmp(mode, "sigabrt") == 0) { + abort(); + } else if (strcmp(mode, "assert") == 0) { + do_assert(rank); + } else if (strcmp(mode, "mixed-abort-segv") == 0) { + do_mixed_abort_segv(rank); + } else if (strcmp(mode, "span-read") == 0) { + do_span_read(rank); + } else if (strcmp(mode, "safepoint") == 0) { + int rc = do_safepoint(rank, safepoint_cycles); + MPI_Finalize(); + return rc; + } else if (strcmp(mode, "safepoint-then-crash") == 0) { + return do_safepoint_then_crash(rank); + } else if (strcmp(mode, "safepoint-bad") == 0) { + return do_safepoint_bad(rank); + } else if (strcmp(mode, "safepoint-bad-write") == 0) { + return do_safepoint_bad_write(rank); + } else if (strcmp(mode, "safepoint-fix-write") == 0) { + int rc = do_safepoint_fix_write(rank, safepoint_cycles); + MPI_Finalize(); + return rc; + } else if (strcmp(mode, "safepoint-longjmp") == 0) { + int rc = do_safepoint_longjmp(rank, safepoint_cycles); + MPI_Finalize(); + return rc; + } else if (strcmp(mode, "safepoint-span-read") == 0) { + int rc = do_safepoint_span_read(rank, safepoint_cycles); + MPI_Finalize(); + return rc; + } else if (strcmp(mode, "safepoint-span-write") == 0) { + int rc = do_safepoint_span_write(rank, safepoint_cycles); + MPI_Finalize(); + return rc; + } else if (strcmp(mode, "safepoint-span-bad-read") == 0) { + return do_safepoint_span_bad_read(rank); + } else if (strcmp(mode, "safepoint-span-bad-write") == 0) { + return do_safepoint_span_bad_write(rank); + } else if (strcmp(mode, "mmap-sigbus-bad") == 0) { + return do_mmap_sigbus_bad(rank); + } else if (strcmp(mode, "mmap-sigbus-fixed") == 0) { + int rc = do_mmap_sigbus_fixed(rank); + MPI_Finalize(); + return rc; + } else if (strcmp(mode, "no-crash") == 0) { + fprintf(stderr, "rank=%d no-crash, exiting cleanly\n", rank); + fflush(stderr); + MPI_Finalize(); + return 0; + } else { + if (rank == 0) usage(argv[0]); + MPI_Finalize(); + return 2; + } + + MPI_Finalize(); + return 0; +} diff --git a/testsuite/libcrashctor.c b/testsuite/libcrashctor.c new file mode 100644 index 00000000..877d3ce4 --- /dev/null +++ b/testsuite/libcrashctor.c @@ -0,0 +1,7 @@ +#include + +__attribute__((constructor)) +static void ctor_crash(void) +{ + *(volatile int *) 0 = 0; +} diff --git a/testsuite/libcrashfixed.c b/testsuite/libcrashfixed.c new file mode 100644 index 00000000..a48a6f32 --- /dev/null +++ b/testsuite/libcrashfixed.c @@ -0,0 +1,24 @@ +/* +This file is part of Spindle. For copyright information see the COPYRIGHT +file in the top level directory, or at +https://github.com/hpc/Spindle/blob/master/COPYRIGHT + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU Lesser General Public License (as published by the Free Software +Foundation) version 2.1 dated February 1999. This program is distributed in the +hope that it will be useful, but WITHOUT ANY WARRANTY; without even the IMPLIED +WARRANTY OF MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms +and conditions of the GNU Lesser General Public License for more details. You should +have received a copy of the GNU Lesser General Public License along with this +program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA +*/ + +#define _GNU_SOURCE + +__attribute__((noinline, visibility("default"))) +void crash_in_fixed_library(int rank) { + volatile int *p = (int *) 0; + (void) rank; + *p = 0; +} diff --git a/testsuite/libcrashfuncs.c b/testsuite/libcrashfuncs.c new file mode 100644 index 00000000..0cd0b1ba --- /dev/null +++ b/testsuite/libcrashfuncs.c @@ -0,0 +1,24 @@ +/* +This file is part of Spindle. For copyright information see the COPYRIGHT +file in the top level directory, or at +https://github.com/hpc/Spindle/blob/master/COPYRIGHT + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU Lesser General Public License (as published by the Free Software +Foundation) version 2.1 dated February 1999. This program is distributed in the +hope that it will be useful, but WITHOUT ANY WARRANTY; without even the IMPLIED +WARRANTY OF MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms +and conditions of the GNU Lesser General Public License for more details. You should +have received a copy of the GNU Lesser General Public License along with this +program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA +*/ + +#define _GNU_SOURCE + +__attribute__((noinline, visibility("default"))) +void crash_in_library(int rank) { + volatile int *p = (int *) 0; + (void) rank; + *p = 0; +} diff --git a/testsuite/run_crash_tests_template.sh b/testsuite/run_crash_tests_template.sh new file mode 100755 index 00000000..3d4b71c9 --- /dev/null +++ b/testsuite/run_crash_tests_template.sh @@ -0,0 +1,429 @@ +#!/usr/bin/env bash + +# Usage: +# ./run_crash_tests.sh [--launcher=serial|flux|slurm|slurm-plugin] --nodes=N [--scratch=DIR] +# +# If running the tests from a non-shared filesystem, set --scratch to a shared +# filesystem so that the script can count all corefiles produced across all nodes. +# The CI containers' main filesystem is not shared, so a shared volume should be +# mounted across all containers and specified in --scratch + +set -u + +LAUNCHER="TEST_RESOURCE_MANAGER" +NODES="" +CRASH_TEST_SCRATCH="${CRASH_TEST_SCRATCH:-}" +SPINDLE="${SPINDLE:-SPINDLE_EXEC}" +SPINDLE_RC="${SPINDLE_RC:-SPINDLE_RC_PATH}" +TESTDIR="${TESTDIR:-TEST_RUN_DIR}" + +die() { echo "FAIL: $*" >&2; exit 1; } + +# The tests to run +# Fields: +# mode: --mode to pass to crash test runner +# cores: expected number of cores produced; if N, then equal to total number of ranks +# flags: if "multi-rank", skip when running on only one rank; if "clean", expect test to NOT crash +# top_frame_regex: regex that should match the top frame in produced coredumps. +# Note that all threads will be checked, so in multithreaded examples the regex should +# also match anything that could be on threads other than the one that faulted. +# site_regex: optional regex for the crash-site dedup key; if present, the crash site key must +# match the regex for the test to pass +# binary: optional alternate executable to run in place of the default crash_test. +# crash_mode: optional alternate crash mode argument to executable +CRASH_TESTS=( +# mode ; cores ; flags ; top_frame_regex ; site_regex ; binary ; crash_mode + 'all-same ; 1 ; ; crash_function_A ; crash_test\+0x' + 'fixed-address-exe ; 1 ; ; crash_function_A ; crash_test_fixedaddr\+0x ; crash_test_fixedaddr ; all-same' + 'pie-exe ; 1 ; ; crash_function_A ; crash_test_pie\+0x ; crash_test_pie ; all-same' + 'all-different ; N ; ; crash_function_[0-9]+' + 'two-groups ; 2 ; ; crash_function_(A|B)' + 'one-crashes ; 1 ; ; crash_function_A' + 'partial ; 1 ; ; crash_function_A' + 'late-straggler ; 1 ; ; crash_function_A' + 'in-library ; 1 ; ; crash_in_library ; libcrashfuncs\.so\+0x' + 'in-dlmopen-library ; 1 ; ; crash_in_library ; libcrashfuncs\.so\+0x' + 'in-fixed-library ; 1 ; ; crash_in_fixed_library ; libcrashfixed\.so\+0x' + 'in-fixed-dlmopen-library ; 1 ; ; crash_in_fixed_library ; libcrashfixed\.so\+0x' + 'in-library-ctor ; 1 ; ; ctor_crash' + 'sigabrt ; 1 ; ; (__GI_)?raise|abort|pthread_kill' + 'assert ; 1 ; ; (__GI_)?raise|abort|pthread_kill ; abort:.*Assertion' + 'mixed-abort-segv ; 2 ; multi-rank ; (__GI_)?raise|abort|pthread_kill|do_mixed_abort_segv' + 'span-read ; 1 ; ; do_span_read' + 'safepoint ; 0 ; clean ; -' + 'safepoint-then-crash ; 1 ; ; crash_function_A' + 'safepoint-bad ; 1 ; ; do_safepoint_bad' + 'safepoint-bad-write ; 1 ; ; do_safepoint_bad_write' + 'safepoint-fix-write ; 0 ; clean ; -' + 'safepoint-longjmp ; 0 ; clean ; -' + 'safepoint-span-read ; 0 ; clean ; -' + 'safepoint-span-write ; 0 ; clean ; -' + 'safepoint-span-bad-read ; 1 ; ; do_safepoint_span_bad_read' + 'safepoint-span-bad-write ; 1 ; ; do_safepoint_span_bad_write' + 'mmap-sigbus-bad ; 1 ; ; do_mmap_sigbus_bad' + 'mmap-sigbus-fixed ; 0 ; clean ; -' + 'no-crash ; 0 ; clean ; -' +) + +declare -A TEST_CORES TEST_FLAGS TEST_TOPFRAME TEST_SITE TEST_BINARY TEST_CRASHMODE +DEFAULT_MODES=() + +trim() { + local s="$1" + s="${s#"${s%%[![:space:]]*}"}" + s="${s%"${s##*[![:space:]]}"}" + printf '%s' "$s" +} + +parse_table() { + local row mode cores flags top site bin cmode + for row in "${CRASH_TESTS[@]}"; do + IFS=';' read -r mode cores flags top site bin cmode <<<"$row" + mode=$(trim "$mode") + [ -n "$mode" ] || continue + cores=$(trim "$cores") + flags=$(trim "$flags") + top=$(trim "$top") + site=$(trim "$site") + bin=$(trim "$bin") + cmode=$(trim "$cmode") + [ -n "$bin" ] || bin="crash_test" + [ -n "$cmode" ] || cmode="$mode" + TEST_CORES[$mode]="$cores" + TEST_FLAGS[$mode]="$flags" + TEST_TOPFRAME[$mode]="$top" + TEST_SITE[$mode]="$site" + TEST_BINARY[$mode]="$bin" + TEST_CRASHMODE[$mode]="$cmode" + DEFAULT_MODES+=("$mode") + done +} + +has_flag() { + case ",${TEST_FLAGS[$1]:-}," in + *,"$2",*) return 0 ;; + *) return 1 ;; + esac +} + +resolve_cores() { + local mode="$1" val + if [ "$LAUNCHER" = "serial" ]; then + printf '1' + return + fi + val="${TEST_CORES[$mode]}" + [ "$val" = "N" ] && val="$NODES" + printf '%s' "$val" +} + +usage() { + local prog + prog=$(basename "$0") + cat </dev/null 2>&1 || \ + die "crash tests can't run because gdb is not on path" + + case "$LAUNCHER" in + serial|flux|slurm|slurm-plugin) ;; + *) die "unknown launcher" ;; + esac + + if [ "$LAUNCHER" = "serial" ]; then + NODES=1 + elif [ -z "$NODES" ]; then + die "--nodes required" + elif ! [[ "$NODES" =~ ^[1-9][0-9]*$ ]]; then + die "--nodes must be a positive integer (was '$NODES')" + fi + + test -x "$TESTDIR/crash_test" || die "can't find crash test executable" + test -x "$TESTDIR/crash_test_fixedaddr" || die "can't find crash_test_fixedaddr executable" + test -x "$TESTDIR/crash_test_pie" || die "can't find crash_test_pie executable" + test -f "$TESTDIR/libcrashfuncs.so" || die "can't find libcrashfuncs.so" + test -f "$TESTDIR/libcrashfixed.so" || die "can't find libcrashfixed.so" +} + +launch() { + local mode="$1" + local binary="$TESTDIR/${TEST_BINARY[$mode]:-crash_test}" + local crash_mode="${TEST_CRASHMODE[$mode]:-$mode}" + ulimit -c unlimited + # Make libcrashfuncs.so visible to dlopen() from the mode's scratch dir. + export LD_LIBRARY_PATH="$TESTDIR${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" + case "$LAUNCHER" in + serial) + "$SPINDLE" \ + --no-mpi --crash-dedup -- \ + "$binary" --crash-mode "$crash_mode" + ;; + flux) + flux run \ + -o userrc="$SPINDLE_RC" \ + -o spindle.crash-dedup \ + -N"$NODES" -n"$NODES" \ + --env=LD_LIBRARY_PATH \ + -- "$binary" --crash-mode "$crash_mode" + ;; + slurm) + salloc -N"$NODES" -n"$NODES" \ + "$SPINDLE" --crash-dedup -- \ + srun "$binary" --crash-mode "$crash_mode" + ;; + slurm-plugin) + salloc -N"$NODES" -n"$NODES" \ + srun --spindle="--crash-dedup" \ + "$binary" --crash-mode "$crash_mode" + ;; + esac +} + +core_files() { + local dir="$1" + local cp glob search_dir + cp=$(cat /proc/sys/kernel/core_pattern) + case "$cp" in + /*) glob=$(basename "$cp"); search_dir=$(dirname "$cp") ;; + *) glob="$cp"; search_dir="$dir" ;; + esac + # This converts any parameters (%p, %e, etc.) in the core pattern + # into a shell glob so we can match all produced core files + # (for example, "core.%e.%p" becomes "core.*.*") + glob=$(echo "$glob" | sed -E 's|%[-0-9]*[peghistuclIPCsdfkrSTK]|*|g') + find "$search_dir" -maxdepth 3 -type f -name "$glob" 2>/dev/null +} + +count_cores() { + core_files "$1" | wc -l +} + +verify_top_frames() { + local mode="$1" + local dir="$2" + local expected_top="${TEST_TOPFRAME[$mode]:-}" + + [ "$expected_top" = "-" ] && return 0 + [ -z "$expected_top" ] && return 0 + + local binary="$TESTDIR/${TEST_BINARY[$mode]:-crash_test}" + local core top cores + cores=$(core_files "$dir") + for core in $cores; do + [ -e "$core" ] || continue + top=$(gdb -batch -nx \ + -iex 'set print demangle off' \ + -ex 'set pagination off' \ + -ex 'bt 1' "$binary" "$core" 2>/dev/null \ + | grep -oE '#0\s+.*' | head -1) + if ! echo "$top" | grep -qE "$expected_top"; then + echo " core $core: top frame '$top' does not match '$expected_top'" >&2 + return 1 + fi + done + return 0 +} + +# Read the crash site key (library+offset) out of the coredump +read_crash_site() { + local core="$1" + local binary="${2:-$TESTDIR/crash_test}" + local map base cachepath real + # Get the mappings from the coredump and extract the Spindle audit library + map=$(gdb -batch -nx -ex 'set debuginfod enabled off' \ + -ex 'info proc mappings' "$binary" "$core" 2>/dev/null \ + | awk '/-spindlens-dso-libspindle_audit/ && $4 == "0x0" {print; exit}') + base=$(printf '%s' "$map" | awk '{print $1}') + cachepath=$(printf '%s' "$map" | awk '{print $NF}') + # Figure out the path to the Spindle audit library + real=$(printf '%s' "$cachepath" | sed -E 's#^.*/spindle\.[0-9a-f]+##; s#/[0-9]+-spindlens-dso-#/#') + # Open the coredump with the Spindle audit library symbols loaded + # so we can check crash_site_buf + gdb -batch -nx -ex 'set debuginfod enabled off' \ + -ex "add-symbol-file $real -o $base" \ + -ex 'printf "CRASH_SITE=%s\n", crash_site_buf' \ + "$binary" "$core" 2>/dev/null \ + | sed -n 's/^CRASH_SITE=//p' | head -1 +} + +# Check that the crash site key matches the expectation +# to verify that Spindle identified the correct library +verify_crash_site() { + local mode="$1" + local dir="$2" + local expected_site="${TEST_SITE[$mode]:-}" + + # If this test doesn't have a crash site specified, there's nothing to do + [ "$expected_site" = "-" ] && return 0 + [ -z "$expected_site" ] && return 0 + + local binary="$TESTDIR/${TEST_BINARY[$mode]:-crash_test}" + local core cores site + cores=$(core_files "$dir") + for core in $cores; do + site=$(read_crash_site "$core" "$binary") + if [ -z "$site" ]; then + echo " core $core: could not read crash site" >&2 + return 1 + fi + if ! printf '%s' "$site" | grep -qE -- "$expected_site"; then + echo " core $core: crash dedup key '$site' does not match '$expected_site'" >&2 + return 1 + fi + done + return 0 +} + +main() { + parse_args "$@" + parse_table + + if [ -z "$CRASH_TEST_SCRATCH" ]; then + if [ -n "${SPINDLE_TEST_CONTAINER:-}" ]; then + die "--scratch=DIR is required when running in a CI container" + fi + CRASH_TEST_SCRATCH="$TESTDIR/spindle_crash_test" + fi + check_prereqs + + local pass=0 fail=0 + + local modes_to_run + if [ -z "$MODES" ]; then + modes_to_run=("${DEFAULT_MODES[@]}") + else + modes_to_run=() + IFS=',' read -ra specified_modes <<< "$MODES" + for m in "${specified_modes[@]}"; do + m=$(echo "$m" | xargs) + modes_to_run+=("$m") + done + fi + + for mode in "${modes_to_run[@]}"; do + if [ -z "${TEST_CORES[$mode]+set}" ]; then + die "unknown mode '$mode'" + fi + + if has_flag "$mode" multi-rank && \ + { [ "$LAUNCHER" = "serial" ] || [ "$NODES" -lt 2 ]; }; then + echo "SKIP $mode (needs multiple ranks)" + continue + fi + + # Run each test in per-test directory so all the coredumps run in one place + # and we can count them + mkdir -p "$CRASH_TEST_SCRATCH" || die "can't create scratch dir '$CRASH_TEST_SCRATCH'" + local dir + dir=$(mktemp -d "$CRASH_TEST_SCRATCH/$mode.XXXXXX") || die "can't create test dir under '$CRASH_TEST_SCRATCH'" + local launch_rc=0 + ( cd "$dir" && launch "$mode" ) >"$dir/stdout.log" 2>"$dir/stderr.log" || launch_rc=$? + [ "$LAUNCHER" = "serial" ] || sleep 1 + + # If this test is not supposed to crash, verify that it didn't crash + # and exited normally + if has_flag "$mode" clean; then + local actual_cores + actual_cores=$(count_cores "$dir") + if [ "$actual_cores" != "0" ]; then + echo "FAIL $mode: expected 0 dumps, got $actual_cores" + fail=$((fail+1)) + continue + fi + if [ "$launch_rc" != "0" ]; then + echo "FAIL $mode: launcher exited $launch_rc (expected 0)" + fail=$((fail+1)) + continue + fi + echo "PASS $mode (clean exit)" + pass=$((pass+1)) + continue + fi + + # Otherwise, if this test is supposed to crash, verify that we + # got the number of core files that we expect + local want + want=$(resolve_cores "$mode") + local actual + actual=$(count_cores "$dir") + + if [ "$actual" != "$want" ]; then + echo "FAIL $mode: expected $want coredumps, got $actual" + fail=$((fail+1)) + continue + fi + + # And verify that the core files show the expected crash sites + if ! verify_top_frames "$mode" "$dir"; then + echo "FAIL $mode: dump top-frame verification failed" + fail=$((fail+1)) + continue + fi + + # Verify the crash handler's + dedup key + if ! verify_crash_site "$mode" "$dir"; then + echo "FAIL $mode: crash site dedup key verification failed" + fail=$((fail+1)) + continue + fi + + echo "PASS $mode ($actual dumps)" + pass=$((pass+1)) + done + + echo + echo "Summary: $pass passed, $fail failed" + [ "$fail" -eq 0 ] || exit 1 +} + +main "$@"