From 3c91f78d0c3a160354b9438eeeebf8faa801587a Mon Sep 17 00:00:00 2001 From: Matthew LeGendre Date: Mon, 22 Jun 2026 11:17:35 -0700 Subject: [PATCH 01/14] Move cachpath consensus to be calculated at init time rather than on a FE message --- src/client/client_comlib/client_api.c | 51 ++++++++----------- src/fe/startup/spindle_fe.cc | 13 ----- src/include/ldcs_api.h | 2 - .../auditserver/ldcs_audit_server_handlers.c | 50 +++++------------- .../auditserver/ldcs_audit_server_handlers.h | 1 + .../auditserver/ldcs_audit_server_process.c | 22 +++++--- src/server/comlib/ldcs_api_util.c | 2 - 7 files changed, 51 insertions(+), 90 deletions(-) diff --git a/src/client/client_comlib/client_api.c b/src/client/client_comlib/client_api.c index cec31051..07735226 100644 --- a/src/client/client_comlib/client_api.c +++ b/src/client/client_comlib/client_api.c @@ -40,42 +40,31 @@ static struct lock_t comm_lock; int send_cachepath_query( int fd, char **chosen_realized_cachepath, char **chosen_parsed_cachepath){ - int retries = 0, max_retries = 1000, rc = 0; - struct timespec delay_between_retries = { .tv_sec = 0, .tv_nsec = 1000000 }; + int rc = 0; ldcs_message_t message; - char buffer[MAX_PATH_LEN+1]; + char buffer[2*(MAX_PATH_LEN+1)]; buffer[MAX_PATH_LEN] = '\0'; - do{ - message.header.type = LDCS_MSG_CHOSEN_CACHEPATH_REQUEST; - message.header.len = 0; - message.data = buffer; - - COMM_LOCK; - - debug_printf3("sending message of type: CHOSEN_CACHEPATH_REQUEST.\n" ); - rc = client_send_msg(fd, &message); - if( rc != 0 ){ - return rc; - } - rc = client_recv_msg_static(fd, &message, LDCS_READ_BLOCK); - if( rc != 0 ){ - return rc; - } - - COMM_UNLOCK; - - if( message.header.type == LDCS_MSG_NO_CACHEPATH_CONSENSUS_YET ){ - if( retries++ >= max_retries ){ - break; - } - nanosleep( &delay_between_retries, NULL ); - continue; - } - break; + message.header.type = LDCS_MSG_CHOSEN_CACHEPATH_REQUEST; + message.header.len = 0; + message.data = buffer; + + debug_printf3("sending message of type: CHOSEN_CACHEPATH_REQUEST.\n" ); + COMM_LOCK; - }while( 1 ); + rc = client_send_msg(fd, &message); + if( rc != 0 ){ + COMM_UNLOCK; + return rc; + } + rc = client_recv_msg_static(fd, &message, LDCS_READ_BLOCK); + if( rc != 0 ){ + COMM_UNLOCK; + return rc; + } + + COMM_UNLOCK; if (message.header.type != LDCS_MSG_CHOSEN_CACHEPATH || message.header.len > MAX_PATH_LEN) { err_printf("Got unexpected message of type %d\n", (int) message.header.type); diff --git a/src/fe/startup/spindle_fe.cc b/src/fe/startup/spindle_fe.cc index a038e201..0d5c0817 100644 --- a/src/fe/startup/spindle_fe.cc +++ b/src/fe/startup/spindle_fe.cc @@ -41,7 +41,6 @@ static const char *logging_file = NULL; #endif static const char spindle_bootstrap[] = LIBEXECDIR "/spindle_bootstrap"; static bool sendAndWaitForAlive(); -static void determineCachepathConsensus(); #define STARTUP_TIMEOUT 60 @@ -433,7 +432,6 @@ int spindleInitFE(const char **hosts, spindle_args_t *params) /* Wait for servers to indicate startup */ sendAndWaitForAlive(); - determineCachepathConsensus(); return 0; } @@ -490,17 +488,6 @@ void markRSHPidReapedFE() clear_fe_rsh_pid(); } -static void determineCachepathConsensus( void ){ - ldcs_message_t consensus_req_msg; - consensus_req_msg.header.type = LDCS_MSG_REQUEST_CACHEPATH_CONSENSUS; - consensus_req_msg.header.len = 0; - consensus_req_msg.data = NULL; - int result = ldcs_audit_server_fe_broadcast(&consensus_req_msg, NULL); - if (result == -1) { - debug_printf("Failure sending cachepath consensus message\n"); - } -} - static bool sendAndWaitForAlive() { int result; diff --git a/src/include/ldcs_api.h b/src/include/ldcs_api.h index 0bcd8f40..2b261a56 100644 --- a/src/include/ldcs_api.h +++ b/src/include/ldcs_api.h @@ -85,10 +85,8 @@ typedef enum { LDCS_MSG_PICKONE_RESP, LDCS_MSG_ALIVE_REQ, LDCS_MSG_ALIVE_RESP, - LDCS_MSG_REQUEST_CACHEPATH_CONSENSUS, LDCS_MSG_CHOSEN_CACHEPATH_REQUEST, LDCS_MSG_CHOSEN_CACHEPATH, - LDCS_MSG_NO_CACHEPATH_CONSENSUS_YET, LDCS_MSG_UNKNOWN } ldcs_message_ids_t; diff --git a/src/server/auditserver/ldcs_audit_server_handlers.c b/src/server/auditserver/ldcs_audit_server_handlers.c index 1afdf2e8..6b3a6b1c 100644 --- a/src/server/auditserver/ldcs_audit_server_handlers.c +++ b/src/server/auditserver/ldcs_audit_server_handlers.c @@ -181,7 +181,6 @@ static int handle_setup_alias(ldcs_process_data_t *procdata, char *pathname, cha static int handle_client_dirlists_req(ldcs_process_data_t *procdata, int nc); static int handle_close_client_query(ldcs_process_data_t *procdata, int nc); static int handle_alive_msg(ldcs_process_data_t *procdata, ldcs_message_t *msg); -static int handle_cachepath_consensus(ldcs_process_data_t *procdata, ldcs_message_t *msg); static int handle_chosen_cachepath_request(ldcs_process_data_t *procdata, int nc); extern void getValidCachePathByIndex( uint64_t validBitIdx, char **realizedCachePath, char **parsedCachePath, char **symbolicCachePath ); @@ -1997,8 +1996,6 @@ int handle_server_message(ldcs_process_data_t *procdata, node_peer_t peer, ldcs_ case LDCS_MSG_ALIVE_REQ: case LDCS_MSG_ALIVE_RESP: return handle_alive_msg(procdata, msg); - case LDCS_MSG_REQUEST_CACHEPATH_CONSENSUS: - return handle_cachepath_consensus(procdata, msg); default: err_printf("Received unexpected message from node: %d\n", (int) msg->header.type); assert(0); @@ -2961,35 +2958,26 @@ static int handle_client_pickone_msg(ldcs_process_data_t *procdata, int nc, ldcs } /** - * Handle LDCS_MSG_REQUEST_CACHEPATH_CONSENSUS to determine which cachepaths are - * available across all of the servers. + * Determine which cachepaths are available across all of the servers. */ -static int cachepath_consensus_reached; -static int handle_cachepath_consensus(ldcs_process_data_t *procdata, ldcs_message_t *msg){ - +int handle_cachepath_consensus(ldcs_process_data_t *procdata) +{ int num_children = ldcs_audit_server_md_get_num_children(procdata); - debug_printf( "Processing REQUEST_CACHEPATH_CONSENSUS.\n" ); + debug_printf( "Calculating cachepath consensus.\n" ); debug_printf3( " procdata->cachepath_bitidx = %#"PRIx64"\n", procdata->cachepath_bitidx ); debug_printf3( " procdata->cachepaths = %s\n", procdata->cachepaths ); debug_printf3( " procdata->cachepath = %s [should be null]\n", procdata->cachepath ); debug_printf3( " procdata->commpath = %s\n", procdata->commpath ); debug_printf3( " num_children = %d\n", num_children ); - if (num_children) { - spindle_broadcast(procdata, msg); - debug_printf3( "Successfully broadcast REQUEST_CACHEPATH_CONSENSUS\n" ); - msgbundle_force_flush(procdata); - debug_printf3( "Successfully flushed the broadcast of REQUEST_CACHEPATH_CONSENSUS\n" ); - } - ldcs_audit_server_md_allreduce_AND( &procdata->cachepath_bitidx ); debug_printf3( "The consensus value for procdata->cachepath_bitidx is: %#"PRIx64"\n", procdata->cachepath_bitidx ); - if( procdata->cachepath_bitidx == 0 ){ - err_printf("No valid cachepath path available. Falling back to \"commpath\" path (%s).\n", procdata->commpath); + if( procdata->cachepath_bitidx == 0 ) { + debug_printf("No valid cachepath path available. Falling back to \"commpath\" path (%s).\n", procdata->commpath); procdata->cachepath = procdata->commpath; - }else{ + } else { getValidCachePathByIndex( procdata->cachepath_bitidx, &procdata->cachepath, &procdata->parsed_cachepath, @@ -3003,14 +2991,10 @@ static int handle_cachepath_consensus(ldcs_process_data_t *procdata, ldcs_messag ldcs_audit_server_filemngt_init(procdata->cachepath, procdata->commpath); test_printf(" cachepath=%s\n", procdata->cachepath); - cachepath_consensus_reached = 1; return 0; } -/** - * Handle LDCS_MSG_CHOSEN_CACHEPATH_REQUEST - */ -static int handle_chosen_cachepath_request(ldcs_process_data_t *procdata, int nc){ +static int handle_chosen_cachepath_request(ldcs_process_data_t *procdata, int nc) { ldcs_message_t msg; int connid; ldcs_client_t *client; @@ -3022,18 +3006,12 @@ static int handle_chosen_cachepath_request(ldcs_process_data_t *procdata, int nc return 0; - if( cachepath_consensus_reached ){ - msg.header.type = LDCS_MSG_CHOSEN_CACHEPATH; - msg.header.len = strlen(procdata->cachepath) + 1 + strlen(procdata->parsed_cachepath) + 1; - msg.data = calloc( 1, msg.header.len ); - strcpy( msg.data, procdata->cachepath ); - strcpy( &msg.data[ strlen(procdata->cachepath)+1 ], procdata->parsed_cachepath ); - }else{ - msg.header.type = LDCS_MSG_NO_CACHEPATH_CONSENSUS_YET; - msg.header.len = 0; - msg.data = NULL; - } - + msg.header.type = LDCS_MSG_CHOSEN_CACHEPATH; + msg.header.len = strlen(procdata->cachepath) + 1 + strlen(procdata->parsed_cachepath) + 1; + msg.data = calloc( 1, msg.header.len ); + strcpy( msg.data, procdata->cachepath ); + strcpy( &msg.data[ strlen(procdata->cachepath)+1 ], procdata->parsed_cachepath ); + ldcs_send_msg(connid, &msg); free( msg.data ); procdata->server_stat.clientmsg.cnt++; diff --git a/src/server/auditserver/ldcs_audit_server_handlers.h b/src/server/auditserver/ldcs_audit_server_handlers.h index 8702316c..790e52b3 100644 --- a/src/server/auditserver/ldcs_audit_server_handlers.h +++ b/src/server/auditserver/ldcs_audit_server_handlers.h @@ -25,6 +25,7 @@ int handle_server_error(ldcs_process_data_t *procdata, node_peer_t peer); int handle_client_message(ldcs_process_data_t *procdata, int nc, ldcs_message_t *msg); int handle_client_start(ldcs_process_data_t *procdata, int nc); int handle_client_end(ldcs_process_data_t *procdata, int nc); +int handle_cachepath_consensus(ldcs_process_data_t *procdata); int exit_note_cb(int infd, int serverid, void *data); diff --git a/src/server/auditserver/ldcs_audit_server_process.c b/src/server/auditserver/ldcs_audit_server_process.c index b6ecdbff..7e32337d 100644 --- a/src/server/auditserver/ldcs_audit_server_process.c +++ b/src/server/auditserver/ldcs_audit_server_process.c @@ -137,7 +137,7 @@ void stopprofile() int ldcs_audit_server_process(spindle_args_t *args) { - int serverid, fd; + int serverid, fd, result; startprofile(args); @@ -198,7 +198,20 @@ int ldcs_audit_server_process(spindle_args_t *args) if (ldcs_process_data.opts & OPT_PROCCLEAN) init_cleanup_proc(ldcs_process_data.cachepath, ldcs_process_data.commpath); - debug_printf3("Initializing connections for clients at %s and %lu\n", + /* Calculate location of cache */ + debug_printf2("Calculating cache path location\n"); + determineValidCachePaths( + &ldcs_process_data.cachepath_bitidx, + ldcs_process_data.cachepaths, + ldcs_process_data.number ); + result = handle_cachepath_consensus(&ldcs_process_data); + if (result == -1) { + err_printf("Could not determine cachepath consensus\n"); + return -1; + } + + /* Setup connections for clients to start connecting */ + debug_printf2("Initializing connections for clients at %s and %lu\n", ldcs_process_data.commpath, (unsigned long) ldcs_process_data.number); serverid = ldcs_create_server(ldcs_process_data.commpath, ldcs_process_data.number); if (serverid == -1) { @@ -231,10 +244,7 @@ int ldcs_audit_server_process(spindle_args_t *args) if (fd != -1) { ldcs_listen_register_fd(fd, serverid, forceExitCB, (void *) &ldcs_process_data); } - determineValidCachePaths( - &ldcs_process_data.cachepath_bitidx, - ldcs_process_data.cachepaths, - ldcs_process_data.number ); + return 0; } diff --git a/src/server/comlib/ldcs_api_util.c b/src/server/comlib/ldcs_api_util.c index af101815..28a3d90c 100644 --- a/src/server/comlib/ldcs_api_util.c +++ b/src/server/comlib/ldcs_api_util.c @@ -91,10 +91,8 @@ char* _message_type_to_str (ldcs_message_ids_t type) { STR_CASE(LDCS_MSG_PICKONE_RESP); STR_CASE(LDCS_MSG_ALIVE_REQ); STR_CASE(LDCS_MSG_ALIVE_RESP); - STR_CASE(LDCS_MSG_REQUEST_CACHEPATH_CONSENSUS); STR_CASE(LDCS_MSG_CHOSEN_CACHEPATH_REQUEST); STR_CASE(LDCS_MSG_CHOSEN_CACHEPATH); - STR_CASE(LDCS_MSG_NO_CACHEPATH_CONSENSUS_YET); STR_CASE(LDCS_MSG_UNKNOWN); } return "unknown"; From d7e9d1bec5e03ce5002cc776538cc598ee3c8bc5 Mon Sep 17 00:00:00 2001 From: Nicholas Chaimov Date: Mon, 15 Jun 2026 14:58:37 -0700 Subject: [PATCH 02/14] Crash handler: server-side coordination Adds server-side components of crash handling. Client sends LDCS_MSG_CRASH_REPORT to sever. Server propagates request. When a single rank is chosen for a given crashsite, replies with LDCS_MSG_CRASH_RESPONSE. --- src/include/ldcs_api.h | 2 + src/server/auditserver/Makefile.am | 2 +- src/server/auditserver/Makefile.in | 14 +- .../ldcs_audit_server_crash_handler.c | 301 ++++++++++++++++++ .../ldcs_audit_server_crash_handler.h | 39 +++ .../auditserver/ldcs_audit_server_handlers.c | 7 + .../auditserver/ldcs_audit_server_process.c | 12 +- .../auditserver/ldcs_audit_server_process.h | 31 +- src/server/comlib/ldcs_api_util.c | 4 +- 9 files changed, 399 insertions(+), 13 deletions(-) create mode 100644 src/server/auditserver/ldcs_audit_server_crash_handler.c create mode 100644 src/server/auditserver/ldcs_audit_server_crash_handler.h diff --git a/src/include/ldcs_api.h b/src/include/ldcs_api.h index 2b261a56..220eb349 100644 --- a/src/include/ldcs_api.h +++ b/src/include/ldcs_api.h @@ -87,6 +87,8 @@ typedef enum { LDCS_MSG_ALIVE_RESP, LDCS_MSG_CHOSEN_CACHEPATH_REQUEST, LDCS_MSG_CHOSEN_CACHEPATH, + LDCS_MSG_CRASH_REPORT, + LDCS_MSG_CRASH_RESPONSE, LDCS_MSG_UNKNOWN } ldcs_message_ids_t; diff --git a/src/server/auditserver/Makefile.am b/src/server/auditserver/Makefile.am index c8113dd4..8823ead6 100644 --- a/src/server/auditserver/Makefile.am +++ b/src/server/auditserver/Makefile.am @@ -5,7 +5,7 @@ AM_CPPFLAGS = -I$(top_srcdir)/comlib -I$(top_srcdir)/cache -I$(top_srcdir)/../co LDADD = $(top_builddir)/cache/libldcs_cache.la -lrt #AM_LDFLAGS = -all-static -libserverbase_la_SOURCES = ldcs_audit_server_client_cb.c ldcs_audit_server_server_cb.c ldcs_audit_server_process.c ldcs_audit_server_filemngt.c ldcs_audit_server_handlers.c ldcs_elf_read.c ldcs_audit_server_requestors.c ldcs_audit_server_numa.c msgbundle.c parse_mounts.cc cleanup_proc.cc translate_maps.c force_exit.c $(top_srcdir)/../utils/fileutil.c +libserverbase_la_SOURCES = ldcs_audit_server_client_cb.c ldcs_audit_server_server_cb.c ldcs_audit_server_process.c ldcs_audit_server_filemngt.c ldcs_audit_server_handlers.c ldcs_audit_server_crash_handler.c ldcs_elf_read.c ldcs_audit_server_requestors.c ldcs_audit_server_numa.c msgbundle.c parse_mounts.cc cleanup_proc.cc translate_maps.c force_exit.c $(top_srcdir)/../utils/fileutil.c libserverbase_la_LIBADD = -lpthread #libaudit_server_msocket_la_SOURCES = ldcs_audit_server_md_msocket.c ldcs_audit_server_md_msocket_util.c ldcs_audit_server_md_msocket_topo.c diff --git a/src/server/auditserver/Makefile.in b/src/server/auditserver/Makefile.in index e4527600..cede46ff 100644 --- a/src/server/auditserver/Makefile.in +++ b/src/server/auditserver/Makefile.in @@ -123,10 +123,10 @@ am__dirstamp = $(am__leading_dot)dirstamp am_libserverbase_la_OBJECTS = ldcs_audit_server_client_cb.lo \ ldcs_audit_server_server_cb.lo ldcs_audit_server_process.lo \ ldcs_audit_server_filemngt.lo ldcs_audit_server_handlers.lo \ - ldcs_elf_read.lo ldcs_audit_server_requestors.lo \ - ldcs_audit_server_numa.lo msgbundle.lo parse_mounts.lo \ - cleanup_proc.lo translate_maps.lo force_exit.lo \ - $(top_builddir)/../utils/fileutil.lo + ldcs_audit_server_crash_handler.lo ldcs_elf_read.lo \ + ldcs_audit_server_requestors.lo ldcs_audit_server_numa.lo \ + msgbundle.lo parse_mounts.lo cleanup_proc.lo translate_maps.lo \ + force_exit.lo $(top_builddir)/../utils/fileutil.lo libserverbase_la_OBJECTS = $(am_libserverbase_la_OBJECTS) AM_V_P = $(am__v_P_@AM_V@) am__v_P_ = $(am__v_P_@AM_DEFAULT_V@) @@ -146,6 +146,7 @@ am__maybe_remake_depfiles = depfiles am__depfiles_remade = $(top_builddir)/../utils/$(DEPDIR)/fileutil.Plo \ ./$(DEPDIR)/cleanup_proc.Plo ./$(DEPDIR)/force_exit.Plo \ ./$(DEPDIR)/ldcs_audit_server_client_cb.Plo \ + ./$(DEPDIR)/ldcs_audit_server_crash_handler.Plo \ ./$(DEPDIR)/ldcs_audit_server_filemngt.Plo \ ./$(DEPDIR)/ldcs_audit_server_handlers.Plo \ ./$(DEPDIR)/ldcs_audit_server_md_cobo.Plo \ @@ -380,7 +381,7 @@ noinst_LTLIBRARIES = libaudit_server_cobo.la libserverbase.la AM_CPPFLAGS = -I$(top_srcdir)/comlib -I$(top_srcdir)/cache -I$(top_srcdir)/../cobo -I$(top_srcdir)/../logging -I$(top_srcdir)/../include -I$(top_srcdir)/../utils -DLIBEXECDIR=\"$(pkglibexecdir)\" LDADD = $(top_builddir)/cache/libldcs_cache.la -lrt #AM_LDFLAGS = -all-static -libserverbase_la_SOURCES = ldcs_audit_server_client_cb.c ldcs_audit_server_server_cb.c ldcs_audit_server_process.c ldcs_audit_server_filemngt.c ldcs_audit_server_handlers.c ldcs_elf_read.c ldcs_audit_server_requestors.c ldcs_audit_server_numa.c msgbundle.c parse_mounts.cc cleanup_proc.cc translate_maps.c force_exit.c $(top_srcdir)/../utils/fileutil.c +libserverbase_la_SOURCES = ldcs_audit_server_client_cb.c ldcs_audit_server_server_cb.c ldcs_audit_server_process.c ldcs_audit_server_filemngt.c ldcs_audit_server_handlers.c ldcs_audit_server_crash_handler.c ldcs_elf_read.c ldcs_audit_server_requestors.c ldcs_audit_server_numa.c msgbundle.c parse_mounts.cc cleanup_proc.cc translate_maps.c force_exit.c $(top_srcdir)/../utils/fileutil.c libserverbase_la_LIBADD = -lpthread #libaudit_server_msocket_la_SOURCES = ldcs_audit_server_md_msocket.c ldcs_audit_server_md_msocket_util.c ldcs_audit_server_md_msocket_topo.c @@ -461,6 +462,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/cleanup_proc.Plo@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/force_exit.Plo@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ldcs_audit_server_client_cb.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ldcs_audit_server_crash_handler.Plo@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ldcs_audit_server_filemngt.Plo@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ldcs_audit_server_handlers.Plo@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ldcs_audit_server_md_cobo.Plo@am__quote@ # am--include-marker @@ -665,6 +667,7 @@ distclean: distclean-am -rm -f ./$(DEPDIR)/cleanup_proc.Plo -rm -f ./$(DEPDIR)/force_exit.Plo -rm -f ./$(DEPDIR)/ldcs_audit_server_client_cb.Plo + -rm -f ./$(DEPDIR)/ldcs_audit_server_crash_handler.Plo -rm -f ./$(DEPDIR)/ldcs_audit_server_filemngt.Plo -rm -f ./$(DEPDIR)/ldcs_audit_server_handlers.Plo -rm -f ./$(DEPDIR)/ldcs_audit_server_md_cobo.Plo @@ -725,6 +728,7 @@ maintainer-clean: maintainer-clean-am -rm -f ./$(DEPDIR)/cleanup_proc.Plo -rm -f ./$(DEPDIR)/force_exit.Plo -rm -f ./$(DEPDIR)/ldcs_audit_server_client_cb.Plo + -rm -f ./$(DEPDIR)/ldcs_audit_server_crash_handler.Plo -rm -f ./$(DEPDIR)/ldcs_audit_server_filemngt.Plo -rm -f ./$(DEPDIR)/ldcs_audit_server_handlers.Plo -rm -f ./$(DEPDIR)/ldcs_audit_server_md_cobo.Plo diff --git a/src/server/auditserver/ldcs_audit_server_crash_handler.c b/src/server/auditserver/ldcs_audit_server_crash_handler.c new file mode 100644 index 00000000..bdc25871 --- /dev/null +++ b/src/server/auditserver/ldcs_audit_server_crash_handler.c @@ -0,0 +1,301 @@ +/* +This file is part of Spindle. For copyright information see the COPYRIGHT +file in the top level directory, or at +https://github.com/hpc/Spindle/blob/master/COPYRIGHT + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU Lesser General Public License (as published by the Free Software +Foundation) version 2.1 dated February 1999. This program is distributed in the +hope that it will be useful, but WITHOUT ANY WARRANTY; without even the IMPLIED +WARRANTY OF MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms +and conditions of the GNU Lesser General Public License for more details. You should +have received a copy of the GNU Lesser General Public License along with this +program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA +*/ + +#include +#include +#include +#include + +#include "ldcs_api.h" +#include "ldcs_audit_server_process.h" +#include "ldcs_audit_server_md.h" +#include "ldcs_audit_server_crash_handler.h" +#include "spindle_launch.h" +#include "msgbundle.h" + +static crash_site_entry_t *crash_site_find(ldcs_process_data_t *procdata, + const char *site, size_t site_len); +static crash_site_entry_t *crash_site_insert(ldcs_process_data_t *procdata, + const char *site, size_t site_len); +static char *crash_pack(int32_t rank, const char *site, size_t site_len, + size_t *out_len); +static void send_crash_response_to_waiter(ldcs_process_data_t *procdata, + crash_waiter_t *w, + int winning_rank, + const char *site, size_t site_len); +static int crash_resolve(ldcs_process_data_t *procdata, + crash_site_entry_t *entry, int winning_rank); +static int forward_crash_query_up(ldcs_process_data_t *procdata, + const char *site, size_t site_len, + int first_waiter_rank); +static int crash_parse_report(ldcs_message_t *msg, const char *err_str, + int32_t *rank, + const char **site, size_t *site_len); +static int crash_report_common(ldcs_process_data_t *procdata, + crash_waiter_t *w, int reporter_rank, + const char *site, size_t site_len); + +/* INTERNAL HELPER FUNCTIONS */ + +static crash_site_entry_t *crash_site_find(ldcs_process_data_t *procdata, + const char *site, size_t site_len) +{ + int i; + for (i = 0; i < procdata->crash_sites_count; ++i) { + crash_site_entry_t *e = &procdata->crash_sites[i]; + if (e->site_len == site_len && + memcmp(e->site, site, site_len) == 0) { + return e; + } + } + return NULL; +} + +static crash_site_entry_t *crash_site_insert(ldcs_process_data_t *procdata, + const char *site, size_t site_len) +{ + if (procdata->crash_sites_count >= procdata->crash_sites_cap) { + int new_cap = procdata->crash_sites_cap ? procdata->crash_sites_cap * 2 : 8; + procdata->crash_sites = realloc(procdata->crash_sites, + new_cap * sizeof(*procdata->crash_sites)); + procdata->crash_sites_cap = new_cap; + } + char *copy = malloc(site_len + 1); + memcpy(copy, site, site_len); + copy[site_len] = '\0'; + crash_site_entry_t *e = &procdata->crash_sites[procdata->crash_sites_count++]; + e->site = copy; + e->site_len = site_len; + e->resolved = 0; + return e; +} + +static char *crash_pack(int32_t rank, const char *site, size_t site_len, + size_t *out_len) +{ + size_t total = 2 * sizeof(int32_t) + site_len; + char *buf = malloc(total); + int32_t name_len_v = (int32_t) site_len; + memcpy(buf, &rank, sizeof(int32_t)); + memcpy(buf + 1 * sizeof(int32_t), &name_len_v, sizeof(int32_t)); + memcpy(buf + 2 * sizeof(int32_t), site, site_len); + *out_len = total; + return buf; +} + +static void send_crash_response_to_waiter(ldcs_process_data_t *procdata, + crash_waiter_t *w, + int winning_rank, + const char *site, size_t site_len) +{ + if (w->kind == CRASH_WAITER_LOCAL) { + // Delivering a crash response to a local client + int32_t resp = (int32_t) winning_rank; + ldcs_message_t msg; + msg.header.type = LDCS_MSG_CRASH_RESPONSE; + msg.header.len = sizeof(resp); + msg.data = (char *) &resp; + ldcs_client_t *client = procdata->client_table + w->nc; + if (client->state == LDCS_CLIENT_STATUS_ACTIVE && client->connid >= 0) { + debug_printf2("delivering CRASH_RESPONSE to local nc=%d rank=%d winning=%d\n", + w->nc, w->global_rank, winning_rank); + ldcs_send_msg(client->connid, &msg); + } + } else { + // Delivering a crash response to a child + size_t len = 0; + char *buf = crash_pack((int32_t) winning_rank, site, site_len, &len); + ldcs_message_t msg; + msg.header.type = LDCS_MSG_CRASH_RESPONSE; + msg.header.len = len; + msg.data = buf; + debug_printf2("delivering CRASH_RESPONSE to child peer=%p winning=%d\n", + (void *) w->peer, winning_rank); + spindle_send(procdata, &msg, w->peer); + free(buf); + } +} + +static int crash_resolve(ldcs_process_data_t *procdata, + crash_site_entry_t *entry, int winning_rank) +{ + debug_printf2("delivering site '%s' winning=%d\n", + entry->site, winning_rank); + send_crash_response_to_waiter(procdata, &entry->waiter, + winning_rank, entry->site, entry->site_len); + entry->resolved = 1; + return 0; +} + +static int forward_crash_query_up(ldcs_process_data_t *procdata, + const char *site, size_t site_len, + int first_waiter_rank) +{ + size_t total = 0; + char *buf = crash_pack((int32_t) first_waiter_rank, site, site_len, &total); + ldcs_message_t msg; + msg.header.type = LDCS_MSG_CRASH_REPORT; + msg.header.len = total; + msg.data = buf; + int rc = spindle_forward_query(procdata, &msg); + free(buf); + return rc; +} + +static int crash_parse_report(ldcs_message_t *msg, const char *err_str, + int32_t *rank, + const char **site, size_t *site_len) +{ + if (msg->header.len < 2 * sizeof(int32_t)) { + err_printf("malformed crash report in %s\n", err_str); + return -1; + } + int32_t name_len; + memcpy(rank, msg->data, sizeof(int32_t)); + memcpy(&name_len, msg->data + 1 * sizeof(int32_t), sizeof(int32_t)); + if (name_len <= 0) { + err_printf("bad name_len %d in %s\n", (int) name_len, err_str); + return -1; + } + *site = msg->data + 2 * sizeof(int32_t); + *site_len = (size_t) name_len; + return 0; +} + +static int crash_report_common(ldcs_process_data_t *procdata, + crash_waiter_t *w, int reporter_rank, + const char *site, size_t site_len) +{ + // If we have already seen this crash site before, then we know + // it can't be the winner and can short-circuit and respond + // immediately that this rank was not selected. + crash_site_entry_t *e = crash_site_find(procdata, site, site_len); + if (e) { + debug_printf2("known crash site '%s' (%s); suppressing %s reporter rank=%d\n", + e->site, e->resolved ? "resolved" : "in flight", + w->kind == CRASH_WAITER_LOCAL ? "local" : "child", + reporter_rank); + send_crash_response_to_waiter(procdata, w, -1, site, site_len); + return 0; + } + + // If we haven't seen this crash site before, remember that we have + // seen it and forward up the tree for resolution. + e = crash_site_insert(procdata, site, site_len); + e->waiter = *w; + + // If we reached the root without finding a decision already made, + // then this was the first instance of this crash site; + // select this rank to produce the exemplar coredump. + if (ldcs_audit_server_md_is_responsible(procdata, "")) { + debug_printf2("new crash site '%s' at root; selecting rank %d\n", + e->site, reporter_rank); + return crash_resolve(procdata, e, reporter_rank); + } + + debug_printf2("new crash site '%s' at interior; forwarding upward %d\n", + e->site, reporter_rank); + return forward_crash_query_up(procdata, site, site_len, reporter_rank); +} + +/* PUBLIC API */ + +// Act on a crash report received from a local client +int handle_client_crash_report(ldcs_process_data_t *procdata, + int nc, ldcs_message_t *msg) +{ + if (!(procdata->opts & OPT_CRASH_HANDLER)) { + return 0; + } + + debug_printf2("Received CRASH_REPORT from local client nc=%d (len=%lu)\n", + nc, (unsigned long) msg->header.len); + + int32_t rank_raw; + const char *site; + size_t site_len; + if (crash_parse_report(msg, "crash report from local client", &rank_raw, &site, &site_len) != 0) + return -1; + + crash_waiter_t w; + w.kind = CRASH_WAITER_LOCAL; + w.nc = nc; + w.global_rank = (int) rank_raw; + w.peer = NULL; + return crash_report_common(procdata, &w, (int) rank_raw, site, site_len); +} + +// Act on a crash report passed to us from a child +int handle_crash_report_recv(ldcs_process_data_t *procdata, + node_peer_t peer, ldcs_message_t *msg) +{ + int32_t first_waiter_rank; + const char *site; + size_t site_len; + if (crash_parse_report(msg, "crash report from child", &first_waiter_rank, &site, &site_len) != 0) + return -1; + + crash_waiter_t w; + w.kind = CRASH_WAITER_CHILD; + w.nc = -1; + w.global_rank = -1; + w.peer = peer; + return crash_report_common(procdata, &w, (int) first_waiter_rank, site, site_len); +} + +// Act on a crash response passed to us from our parent +int handle_crash_response_recv(ldcs_process_data_t *procdata, + node_peer_t peer, ldcs_message_t *msg) +{ + if (!ldcs_audit_server_md_is_parent(peer)) { + err_printf("unexpectedly got CRASH_RESPONSE from peer other than parent\n"); + return -1; + } + + int32_t selected_rank; + const char *site; + size_t site_len; + if (crash_parse_report(msg, "crash response from parent", &selected_rank, &site, &site_len) != 0) + return -1; + + crash_site_entry_t *e = crash_site_find(procdata, site, site_len); + if (!e) { + err_printf("unexpectedly got CRASH_RESPONSE for unknown site %.*s\n", (int)site_len, site); + return -1; + } + if (e->resolved) { + err_printf("unexpectedly got CRASH_RESPONSE for already resolved site %.*s\n", (int)site_len, site); + return -1; + } + debug_printf2("received CRASH_RESPONSE site '%.*s' selected=%d\n", (int) site_len, site, (int) selected_rank); + + return crash_resolve(procdata, e, (int) selected_rank); +} + +// Free the tables that store cached crash-site decisions. +void crash_free_tables(ldcs_process_data_t *procdata) +{ + int i; + if (procdata->crash_sites) { + for (i = 0; i < procdata->crash_sites_count; ++i) { + free(procdata->crash_sites[i].site); + } + free(procdata->crash_sites); + procdata->crash_sites = NULL; + procdata->crash_sites_count = 0; + procdata->crash_sites_cap = 0; + } +} diff --git a/src/server/auditserver/ldcs_audit_server_crash_handler.h b/src/server/auditserver/ldcs_audit_server_crash_handler.h new file mode 100644 index 00000000..1a419930 --- /dev/null +++ b/src/server/auditserver/ldcs_audit_server_crash_handler.h @@ -0,0 +1,39 @@ +/* +This file is part of Spindle. For copyright information see the COPYRIGHT +file in the top level directory, or at +https://github.com/hpc/Spindle/blob/master/COPYRIGHT + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU Lesser General Public License (as published by the Free Software +Foundation) version 2.1 dated February 1999. This program is distributed in the +hope that it will be useful, but WITHOUT ANY WARRANTY; without even the IMPLIED +WARRANTY OF MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms +and conditions of the GNU Lesser General Public License for more details. You should +have received a copy of the GNU Lesser General Public License along with this +program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA +*/ + +#ifndef LDCS_AUDIT_SERVER_CRASH_HANDLER_H +#define LDCS_AUDIT_SERVER_CRASH_HANDLER_H + +#include "ldcs_api.h" +#include "ldcs_audit_server_process.h" +#include "ldcs_audit_server_md.h" + +/* CRASH_REPORT from a local client */ +int handle_client_crash_report(ldcs_process_data_t *procdata, + int nc, ldcs_message_t *msg); + +/* CRASH_REPORT from a child */ +int handle_crash_report_recv(ldcs_process_data_t *procdata, + node_peer_t peer, ldcs_message_t *msg); + +/* CRASH_RESPONSE from a parent */ +int handle_crash_response_recv(ldcs_process_data_t *procdata, + node_peer_t peer, ldcs_message_t *msg); + +/* Free the crash-site table at shutdown */ +void crash_free_tables(ldcs_process_data_t *procdata); + +#endif diff --git a/src/server/auditserver/ldcs_audit_server_handlers.c b/src/server/auditserver/ldcs_audit_server_handlers.c index 6b3a6b1c..ebfdef13 100644 --- a/src/server/auditserver/ldcs_audit_server_handlers.c +++ b/src/server/auditserver/ldcs_audit_server_handlers.c @@ -35,6 +35,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include "stat_cache.h" #include "global_name.h" #include "ldcs_audit_server_handlers.h" +#include "ldcs_audit_server_crash_handler.h" #include "ldcs_audit_server_requestors.h" #include "spindle_launch.h" #include "pathfn.h" @@ -1895,6 +1896,8 @@ int handle_client_message(ldcs_process_data_t *procdata, int nc, ldcs_message_t return handle_client_procmaps_msg(procdata, nc, msg); case LDCS_MSG_PICKONE_REQ: return handle_client_pickone_msg(procdata, nc, msg); + case LDCS_MSG_CRASH_REPORT: + return handle_client_crash_report(procdata, nc, msg); case LDCS_MSG_END: return handle_client_end(procdata, nc); case LDCS_MSG_CHOSEN_CACHEPATH_REQUEST: @@ -1996,6 +1999,10 @@ int handle_server_message(ldcs_process_data_t *procdata, node_peer_t peer, ldcs_ case LDCS_MSG_ALIVE_REQ: case LDCS_MSG_ALIVE_RESP: return handle_alive_msg(procdata, msg); + case LDCS_MSG_CRASH_REPORT: + return handle_crash_report_recv(procdata, peer, msg); + case LDCS_MSG_CRASH_RESPONSE: + return handle_crash_response_recv(procdata, peer, msg); default: err_printf("Received unexpected message from node: %d\n", (int) msg->header.type); assert(0); diff --git a/src/server/auditserver/ldcs_audit_server_process.c b/src/server/auditserver/ldcs_audit_server_process.c index 7e32337d..8cd7b936 100644 --- a/src/server/auditserver/ldcs_audit_server_process.c +++ b/src/server/auditserver/ldcs_audit_server_process.c @@ -31,6 +31,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include "ldcs_audit_server_filemngt.h" #include "ldcs_audit_server_md.h" #include "ldcs_audit_server_handlers.h" +#include "ldcs_audit_server_crash_handler.h" #include "ldcs_cache.h" #include "spindle_launch.h" #include "ldcs_audit_server_requestors.h" @@ -172,7 +173,10 @@ int ldcs_audit_server_process(spindle_args_t *args) ldcs_process_data.num_exited_children_peers = 0; ldcs_process_data.num_exited_parents = 0; ldcs_process_data.num_alives = 0; - + ldcs_process_data.crash_sites = NULL; + ldcs_process_data.crash_sites_count = 0; + ldcs_process_data.crash_sites_cap = 0; + if (ldcs_process_data.opts & OPT_PULL) { debug_printf("Using PULL model\n"); ldcs_process_data.dist_model = LDCS_PULL; @@ -246,7 +250,7 @@ int ldcs_audit_server_process(spindle_args_t *args) } return 0; -} +} int ldcs_audit_server_run() { @@ -271,7 +275,9 @@ int ldcs_audit_server_run() ldcs_audit_server_md_destroy(&ldcs_process_data); msgbundle_done(&ldcs_process_data); - + + crash_free_tables(&ldcs_process_data); + /* destroy file cache */ if (!(ldcs_process_data.opts & OPT_NOCLEAN)) { ldcs_audit_server_filemngt_clean(); diff --git a/src/server/auditserver/ldcs_audit_server_process.h b/src/server/auditserver/ldcs_audit_server_process.h index 82b60023..4a1734ae 100644 --- a/src/server/auditserver/ldcs_audit_server_process.h +++ b/src/server/auditserver/ldcs_audit_server_process.h @@ -23,9 +23,9 @@ extern "C" { #include "ldcs_api.h" #include "spindle_launch.h" -#include "stat_cache.h" +#include "stat_cache.h" #include "force_exit.h" - + typedef void* requestor_list_t; /* client description structure */ @@ -111,7 +111,28 @@ typedef struct msgbundle_entry_t { struct msgbundle_entry_t *next; char name[16]; } msgbundle_entry_t; - + +typedef enum { + CRASH_WAITER_LOCAL = 0, + CRASH_WAITER_CHILD = 1 +} crash_waiter_kind_t; + +typedef void* node_peer_t; + +typedef struct crash_waiter_t { + crash_waiter_kind_t kind; + int nc; + int global_rank; + node_peer_t peer; +} crash_waiter_t; + +typedef struct crash_site_entry_t { + char *site; + size_t site_len; + int resolved; + crash_waiter_t waiter; +} crash_site_entry_t; + struct ldcs_process_data_struct { int client_table_size; @@ -163,6 +184,10 @@ struct ldcs_process_data_struct requestor_list_t pending_ldso_requests; requestor_list_t completed_ldso_requests; + crash_site_entry_t *crash_sites; + int crash_sites_count; + int crash_sites_cap; + /* multi daemon support */ int md_rank; int md_size; diff --git a/src/server/comlib/ldcs_api_util.c b/src/server/comlib/ldcs_api_util.c index 28a3d90c..e9c6cb89 100644 --- a/src/server/comlib/ldcs_api_util.c +++ b/src/server/comlib/ldcs_api_util.c @@ -90,9 +90,11 @@ char* _message_type_to_str (ldcs_message_ids_t type) { STR_CASE(LDCS_MSG_PICKONE_REQ); STR_CASE(LDCS_MSG_PICKONE_RESP); STR_CASE(LDCS_MSG_ALIVE_REQ); - STR_CASE(LDCS_MSG_ALIVE_RESP); + STR_CASE(LDCS_MSG_ALIVE_RESP); STR_CASE(LDCS_MSG_CHOSEN_CACHEPATH_REQUEST); STR_CASE(LDCS_MSG_CHOSEN_CACHEPATH); + STR_CASE(LDCS_MSG_CRASH_REPORT); + STR_CASE(LDCS_MSG_CRASH_RESPONSE); STR_CASE(LDCS_MSG_UNKNOWN); } return "unknown"; From 59a67097350b3cac905d3bcd2f97d9ebb799b2d0 Mon Sep 17 00:00:00 2001 From: Nicholas Chaimov Date: Mon, 15 Jun 2026 14:58:40 -0700 Subject: [PATCH 03/14] Crash handler: client signal handler Add the client-side crash handler. On a fatal signal, it chains to an application-registered signal handler, if any, and, if the application handler did not fix the fault, sends LDCS_MSG_CRASH_REPORT to the server which selects a single process per crashsite. Non-selected crashsites set their coredump limit to zero. --- src/client/client/Makefile.am | 13 +- src/client/client/client.c | 13 +- src/client/client/client.h | 4 + src/client/client/intercept.c | 10 + src/client/client/intercept.h | 5 + src/client/client/lookup_libc.c | 30 +- src/client/client_comlib/client_api.h | 1 + src/client/client_comlib/client_api_pipe.c | 15 + src/client/client_comlib/client_wrappers.c | 6 + src/client/crash_handler/crash_arch.h | 85 ++++ src/client/crash_handler/crash_fmt.c | 90 ++++ src/client/crash_handler/crash_fmt.h | 26 ++ src/client/crash_handler/crash_handler.c | 318 +++++++++++++ src/client/crash_handler/crash_handler.h | 23 + src/client/crash_handler/crash_io.c | 69 +++ src/client/crash_handler/crash_io.h | 25 ++ src/client/crash_handler/crash_lib_offset.c | 141 ++++++ src/client/crash_handler/crash_lib_offset.h | 27 ++ src/client/crash_handler/crash_sigchain.c | 470 ++++++++++++++++++++ src/client/crash_handler/crash_sigchain.h | 36 ++ 20 files changed, 1404 insertions(+), 3 deletions(-) create mode 100644 src/client/crash_handler/crash_arch.h create mode 100644 src/client/crash_handler/crash_fmt.c create mode 100644 src/client/crash_handler/crash_fmt.h create mode 100644 src/client/crash_handler/crash_handler.c create mode 100644 src/client/crash_handler/crash_handler.h create mode 100644 src/client/crash_handler/crash_io.c create mode 100644 src/client/crash_handler/crash_io.h create mode 100644 src/client/crash_handler/crash_lib_offset.c create mode 100644 src/client/crash_handler/crash_lib_offset.h create mode 100644 src/client/crash_handler/crash_sigchain.c create mode 100644 src/client/crash_handler/crash_sigchain.h diff --git a/src/client/client/Makefile.am b/src/client/client/Makefile.am index b6f03700..3a2a8749 100644 --- a/src/client/client/Makefile.am +++ b/src/client/client/Makefile.am @@ -17,7 +17,18 @@ AM_CPPFLAGS = -I$(top_srcdir)/../logging -I$(top_srcdir)/client_comlib -I$(top_s INTERCEPT_SRCS = intercept_open.c intercept_exec.c intercept_stat.c intercept_readlink.c intercept_spindleapi.c intercept.c spindle_regex.c patch_interception.c realpath.c $(top_srcdir)/../utils/fileutil.c -BASE_SRCS = client.c lookup.c should_intercept.c exec_util.c remap_exec.c lookup_libc.c $(top_srcdir)/../utils/parseloc.c $(top_srcdir)/../utils/getcpu.c adjust_argv.c +BASE_SRCS = client.c \ + lookup.c should_intercept.c exec_util.c remap_exec.c \ + lookup_libc.c adjust_argv.c $(top_srcdir)/../utils/parseloc.c \ + $(top_srcdir)/../utils/getcpu.c + +if BLD_CRASH_HANDLER +BASE_SRCS += $(top_srcdir)/crash_handler/crash_io.c \ + $(top_srcdir)/crash_handler/crash_fmt.c \ + $(top_srcdir)/crash_handler/crash_handler.c \ + $(top_srcdir)/crash_handler/crash_lib_offset.c \ + $(top_srcdir)/crash_handler/crash_sigchain.c +endif libspindlec_socket_la_SOURCES = $(BASE_SRCS) libspindlec_socket_la_LIBADD = $(top_builddir)/client_comlib/libclient_socket.la $(top_builddir)/logging/libspindleclogc.la $(top_builddir)/shm_cache/libshmcache.la diff --git a/src/client/client/client.c b/src/client/client/client.c index 1e2d5705..480518b6 100644 --- a/src/client/client/client.c +++ b/src/client/client/client.c @@ -30,11 +30,14 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include #include -#include "ldcs_api.h" +#include "ldcs_api.h" #include "config.h" #include "client.h" #include "client_heap.h" #include "client_api.h" +#if defined(CRASH_HANDLER_ENABLED) +#include "crash_handler.h" +#endif #include "spindle_launch.h" #include "shmcache.h" #include "ccwarns.h" @@ -364,6 +367,14 @@ int client_init() return -1; init_server_connection(); + +#if defined(CRASH_HANDLER_ENABLED) + if (opts & OPT_CRASH_HANDLER) { + int global_rank = rankinfo[0] * rankinfo[3] + rankinfo[2]; + (void) crash_handler_install(global_rank, ldcsid); + } +#endif /* CRASH_HANDLER_ENABLED */ + intercept_open = (opts & OPT_RELOCPY) ? 1 : 0; intercept_stat = (opts & OPT_RELOCPY || !(opts & OPT_NOHIDE)) ? 1 : 0; intercept_exec = (opts & OPT_RELOCEXEC) ? 1 : 0; diff --git a/src/client/client/client.h b/src/client/client/client.h index 67f1842f..e53543cb 100644 --- a/src/client/client/client.h +++ b/src/client/client/client.h @@ -107,6 +107,10 @@ extern void int_spindle_test_log_msg(char *buffer); typedef void* (*malloc_sig_t)(size_t); malloc_sig_t get_libc_malloc(); +#if defined(CRASH_HANDLER_ENABLED) +void *get_libc_abort_msg(); +#endif + int lookup_libc_symbols(); /* ERRNO_NAME currently refers to a glibc internal symbol. */ diff --git a/src/client/client/intercept.c b/src/client/client/intercept.c index ea73253e..80c4aef4 100644 --- a/src/client/client/intercept.c +++ b/src/client/client/intercept.c @@ -20,6 +20,10 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include #include "intercept.h" #include "client.h" +#include "config.h" +#if defined(CRASH_HANDLER_ENABLED) +#include "crash_sigchain.h" +#endif struct spindle_binding_t spindle_bindings[] = { { "", NULL, "", NULL }, @@ -45,6 +49,12 @@ struct spindle_binding_t spindle_bindings[] = { { "execvp", (void **) &orig_execvp, "execvp_wrapper", (void *) execvp_wrapper }, { "execvpe", (void **) &orig_execvpe, "execvpe_wrapper", (void *) execvpe_wrapper }, { "vfork", (void **) &orig_vfork, "vfork_wrapper", (void *) vfork_wrapper }, +#if defined(CRASH_HANDLER_ENABLED) + { "sigaction", (void **) &orig_sigaction, "sigaction_wrapper", (void *) sigaction_wrapper }, + { "signal", (void **) &orig_signal, "signal_wrapper", (void *) signal_wrapper }, + { "bsd_signal", (void **) &orig_bsd_signal, "bsd_signal_wrapper", (void *) bsd_signal_wrapper }, + { "sysv_signal", (void **) &orig_sysv_signal, "sysv_signal_wrapper", (void *) sysv_signal_wrapper }, +#endif { "readlink", (void **) &orig_readlink, "readlink_wrapper", (void *) readlink_wrapper }, { "readlinkat", (void **) &orig_readlinkat, "readlinkat_wrapper", (void *) readlinkat_wrapper }, { "realpath", (void **) &orig_realpath, "spindle_realpath", (void *) spindle_realpath }, diff --git a/src/client/client/intercept.h b/src/client/client/intercept.h index 4ace2328..3442ef5d 100644 --- a/src/client/client/intercept.h +++ b/src/client/client/intercept.h @@ -54,6 +54,11 @@ extern int (*orig_unsetenv)(const char *name); extern char* (*orig_dlerror)(); extern char* (*orig_realpath)(const char *, char *); +extern void *orig_sigaction; +extern void *orig_signal; +extern void *orig_bsd_signal; +extern void *orig_sysv_signal; + extern int (*orig_spindle_open)(const char *pathname, int flags, ...); extern FILE* (*orig_spindle_fopen)(const char *path, const char *opts); extern int (*orig_spindle_stat)(const char *path, struct stat *buf); diff --git a/src/client/client/lookup_libc.c b/src/client/client/lookup_libc.c index f25497be..20750f96 100644 --- a/src/client/client/lookup_libc.c +++ b/src/client/client/lookup_libc.c @@ -47,6 +47,10 @@ struct gnu_hash_header { static malloc_sig_t mallocfunc = NULL; +#if defined(CRASH_HANDLER_ENABLED) +static void *volatile abort_msg_addr = NULL; +#endif + static signed long lookup_gnu_hash_symbol(const char *name, ElfW(Sym) *syms, char *symnames, struct gnu_hash_header *header) { uint32_t *buckets, *vals; @@ -214,7 +218,24 @@ int lookup_libc_symbols() mallocfunc = (malloc_sig_t) (symtab[result].st_value + libc->l_addr); debug_printf3("Bound mallocfunc to %p\n", mallocfunc); found++; - } + } + +#if defined(CRASH_HANDLER_ENABLED) + result = -1; + if (gnu_hash) + result = lookup_gnu_hash_symbol("__abort_msg", symtab, strtab, (struct gnu_hash_header *) gnu_hash); + if (elf_hash && result == -1) + result = lookup_elf_hash_symbol("__abort_msg", symtab, strtab, (ElfW(Word) *) elf_hash); + if (result == -1) { + debug_printf3("Warning, Could not find symbol __abort_msg in libc\n"); + not_found++; + } + else { + abort_msg_addr = (void *) (symtab[result].st_value + libc->l_addr); + debug_printf3("Bound __abort_msg slot to %p\n", abort_msg_addr); + found++; + } +#endif } libdl_result = lookup_libdl_symbols(); @@ -282,3 +303,10 @@ malloc_sig_t get_libc_malloc() lookup_libc_symbols(); return mallocfunc; } + +#if defined(CRASH_HANDLER_ENABLED) +void *get_libc_abort_msg() +{ + return abort_msg_addr; +} +#endif diff --git a/src/client/client_comlib/client_api.h b/src/client/client_comlib/client_api.h index 3d7c41be..63f92c7a 100644 --- a/src/client/client_comlib/client_api.h +++ b/src/client/client_comlib/client_api.h @@ -54,6 +54,7 @@ char *client_get_connection_string(int fd); int client_send_msg(int connid, ldcs_message_t * msg); int client_recv_msg_static(int fd, ldcs_message_t *msg, ldcs_read_block_t block); int client_recv_msg_dynamic(int fd, ldcs_message_t *msg, ldcs_read_block_t block); +int client_get_raw_fds(int fd, int *read_fd, int *write_fd); int is_client_fd(int connfd, int fd); #endif diff --git a/src/client/client_comlib/client_api_pipe.c b/src/client/client_comlib/client_api_pipe.c index ea57efe5..7f3df457 100644 --- a/src/client/client_comlib/client_api_pipe.c +++ b/src/client/client_comlib/client_api_pipe.c @@ -381,6 +381,21 @@ int is_client_fd(int connfd, int fd) return (fdlist_pipe[connfd].in_fd == fd || fdlist_pipe[connfd].out_fd == fd); } +/* Get raw FDs for use in signal handlers. + We can't use the send/recv functions here in signal handlers + because they are not async-signal-safe. */ +int client_get_raw_fds_pipe(int fd, int *read_fd, int *write_fd) +{ + if (fd < 0 || fd >= MAX_FD) { + *read_fd = -1; + *write_fd = -1; + return -1; + } + *read_fd = fdlist_pipe[fd].in_fd; + *write_fd = fdlist_pipe[fd].out_fd; + return 0; +} + int client_close_connection_pipe(int fd) { int result; diff --git a/src/client/client_comlib/client_wrappers.c b/src/client/client_comlib/client_wrappers.c index 4b73718c..ff8488ec 100644 --- a/src/client/client_comlib/client_wrappers.c +++ b/src/client/client_comlib/client_wrappers.c @@ -43,6 +43,7 @@ extern char *RENAME(client_get_connection_string) (int fd); extern int RENAME(client_send_msg) (int connid, ldcs_message_t * msg); extern int RENAME(client_recv_msg_static) (int fd, ldcs_message_t *msg, ldcs_read_block_t block); extern int RENAME(client_recv_msg_dynamic) (int fd, ldcs_message_t *msg, ldcs_read_block_t block); +extern int RENAME(client_get_raw_fds) (int fd, int *read_fd, int *write_fd); int client_open_connection(char* location, number_t number) { @@ -79,3 +80,8 @@ int client_recv_msg_dynamic(int fd, ldcs_message_t *msg, ldcs_read_block_t block return RENAME(client_recv_msg_dynamic) (fd, msg, block); } +int client_get_raw_fds(int fd, int *read_fd, int *write_fd) +{ + return RENAME(client_get_raw_fds) (fd, read_fd, write_fd); +} + diff --git a/src/client/crash_handler/crash_arch.h b/src/client/crash_handler/crash_arch.h new file mode 100644 index 00000000..8334e75f --- /dev/null +++ b/src/client/crash_handler/crash_arch.h @@ -0,0 +1,85 @@ +/* +This file is part of Spindle. For copyright information see the COPYRIGHT +file in the top level directory, or at +https://github.com/hpc/Spindle/blob/master/COPYRIGHT + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU Lesser General Public License (as published by the Free Software +Foundation) version 2.1 dated February 1999. This program is distributed in the +hope that it will be useful, but WITHOUT ANY WARRANTY; without even the IMPLIED +WARRANTY OF MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms +and conditions of the GNU Lesser General Public License for more details. You should +have received a copy of the GNU Lesser General Public License along with this +program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA +*/ + +#if !defined(CRASH_ARCH_H_) +#define CRASH_ARCH_H_ + +#include "config.h" + +#include +#if defined(arch_aarch64) +#include +#endif + +static inline unsigned long extract_pc(void *uctx) +{ + ucontext_t *uc = (ucontext_t *) uctx; +#if defined(arch_x86_64) + return (unsigned long) uc->uc_mcontext.gregs[REG_RIP]; +#elif defined(arch_aarch64) + return (unsigned long) uc->uc_mcontext.pc; +#elif defined(arch_ppc64) || defined(arch_ppc64le) + return (unsigned long) uc->uc_mcontext.gp_regs[32]; +#else +#error "extract_pc: unsupported architecture" +#endif +} + +/* Unlike x86-64 and ppc64le, for aarch64, whether the fault was a read or a write + * is not found in a default register as a direct field of uc_mcontext. + * The relevant bit is found in the ESR (Exception Syndrome Register). + * The register is accessed by iterating over the extended context in __reserved + * to find the tag ESR_MAGIC; then the next entry is the esr_context. + * See https://blog.linuxplumbersconf.org/2017/ocw/system/presentations/4671/original/plumbers-dm-2017.pdf + * for info on the aarch64 extended context, and see + * https://developer.arm.com/documentation/ddi0595/2020-12/AArch64-Registers/ESR-EL1--Exception-Syndrome-Register--EL1- + * for info on the Exception Syndrome Register. */ +#if defined(arch_aarch64) +static inline const struct _aarch64_ctx *first_aarch64_ctx(const ucontext_t *uc) +{ + return (const struct _aarch64_ctx *) uc->uc_mcontext.__reserved; +} + +static inline const struct _aarch64_ctx *next_aarch64_ctx(const struct _aarch64_ctx *hdr) +{ + return (const struct _aarch64_ctx *) ((const char *) hdr + hdr->size); +} + +static inline int esr_is_write(unsigned long long esr) +{ + return ((esr >> 27) & 0x1f) == 0x12 && ((esr >> 6) & 1) != 0; +} +#endif /* arch_aarch64 */ + +static inline int pf_is_write(void *uctx) +{ + ucontext_t *uc = (ucontext_t *) uctx; +#if defined(arch_x86_64) + return (uc->uc_mcontext.gregs[REG_ERR] & 0x2UL) != 0; +#elif defined(arch_aarch64) + for (const struct _aarch64_ctx *hdr = first_aarch64_ctx(uc); hdr->magic; hdr = next_aarch64_ctx(hdr)) { + if (hdr->magic == ESR_MAGIC) + return esr_is_write(((const struct esr_context *) hdr)->esr); + } + return 0; +#elif defined(arch_ppc64) || defined(arch_ppc64le) + return (uc->uc_mcontext.gp_regs[42] & 0x02000000UL) != 0; +#else +#error "pf_is_write: unsupported architecture" +#endif +} + +#endif diff --git a/src/client/crash_handler/crash_fmt.c b/src/client/crash_handler/crash_fmt.c new file mode 100644 index 00000000..4e2b0ff9 --- /dev/null +++ b/src/client/crash_handler/crash_fmt.c @@ -0,0 +1,90 @@ +/* +This file is part of Spindle. For copyright information see the COPYRIGHT +file in the top level directory, or at +https://github.com/hpc/Spindle/blob/master/COPYRIGHT + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU Lesser General Public License (as published by the Free Software +Foundation) version 2.1 dated February 1999. This program is distributed in the +hope that it will be useful, but WITHOUT ANY WARRANTY; without even the IMPLIED +WARRANTY OF MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms +and conditions of the GNU Lesser General Public License for more details. You should +have received a copy of the GNU Lesser General Public License along with this +program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA +*/ + +#include "crash_fmt.h" + +#include + +/* This file contains async-signal-safe formatting + routines for use in the crash handler. The printf family + of functions are not async-signal-safe. */ + +static size_t crash_fmt_append_str(char *buf, size_t pos, size_t cap, + const char *s) +{ + size_t i = 0; + while (s[i]) { + if (pos + 1 >= cap) + return (size_t) -1; + buf[pos++] = s[i++]; + } + return pos; +} + +static size_t crash_fmt_append_hex(char *buf, size_t pos, size_t cap, + unsigned long v) +{ + char tmp[16]; + int i = 0; + + if (v == 0) { + tmp[i++] = '0'; + } else { + while (v && i < (int) sizeof(tmp)) { + unsigned d = (unsigned) (v & 0xFu); + tmp[i++] = (char) (d < 10 ? '0' + d : 'a' + (d - 10)); + v >>= 4; + } + } + + if (pos + (size_t) i + 1 > cap) + return (size_t) -1; + while (i > 0) + buf[pos++] = tmp[--i]; + return pos; +} + +/* Generates + crash site strings. */ +int crash_fmt_lib_offset(char *buf, size_t buflen, const char *lib, + unsigned long offset) +{ + size_t pos = 0; + + if (buflen == 0) + return -1; + + if (lib[0] != '\0') { + pos = crash_fmt_append_str(buf, pos, buflen, lib); + if (pos == (size_t) -1) + return -1; + if (pos + 1 >= buflen) + return -1; + buf[pos++] = '+'; + } + + pos = crash_fmt_append_str(buf, pos, buflen, "0x"); + if (pos == (size_t) -1) + return -1; + + pos = crash_fmt_append_hex(buf, pos, buflen, offset); + if (pos == (size_t) -1) + return -1; + + if (pos >= buflen) + return -1; + buf[pos] = '\0'; + return 0; +} diff --git a/src/client/crash_handler/crash_fmt.h b/src/client/crash_handler/crash_fmt.h new file mode 100644 index 00000000..7ae87a40 --- /dev/null +++ b/src/client/crash_handler/crash_fmt.h @@ -0,0 +1,26 @@ +/* +This file is part of Spindle. For copyright information see the COPYRIGHT +file in the top level directory, or at +https://github.com/hpc/Spindle/blob/master/COPYRIGHT + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU Lesser General Public License (as published by the Free Software +Foundation) version 2.1 dated February 1999. This program is distributed in the +hope that it will be useful, but WITHOUT ANY WARRANTY; without even the IMPLIED +WARRANTY OF MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms +and conditions of the GNU Lesser General Public License for more details. You should +have received a copy of the GNU Lesser General Public License along with this +program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA +*/ + +#if !defined(CRASH_FMT_H_) +#define CRASH_FMT_H_ + +#include + +/* Format a library+offset pair into buf as "+0x\0". */ +int crash_fmt_lib_offset(char *buf, size_t buflen, const char *lib, + unsigned long offset); + +#endif diff --git a/src/client/crash_handler/crash_handler.c b/src/client/crash_handler/crash_handler.c new file mode 100644 index 00000000..ddf05344 --- /dev/null +++ b/src/client/crash_handler/crash_handler.c @@ -0,0 +1,318 @@ +/* +This file is part of Spindle. For copyright information see the COPYRIGHT +file in the top level directory, or at +https://github.com/hpc/Spindle/blob/master/COPYRIGHT + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU Lesser General Public License (as published by the Free Software +Foundation) version 2.1 dated February 1999. This program is distributed in the +hope that it will be useful, but WITHOUT ANY WARRANTY; without even the IMPLIED +WARRANTY OF MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms +and conditions of the GNU Lesser General Public License for more details. You should +have received a copy of the GNU Lesser General Public License along with this +program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA +*/ + +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "config.h" +#include "ldcs_api.h" +#include "client.h" +#include "client_api.h" +#include "crash_handler.h" +#include "crash_arch.h" +#include "crash_fmt.h" +#include "crash_io.h" +#include "crash_lib_offset.h" +#include "crash_sigchain.h" + +#define CRASH_ALTSTACK_SIZE 65536 +#define CRASH_SITE_BUF_SIZE (PATH_MAX + 32) +#define CRASH_REQ_BUF_SIZE \ + (sizeof(ldcs_message_header_t) + 2 * sizeof(int32_t) + CRASH_SITE_BUF_SIZE) +#define CRASH_ABORT_MSG_MAX (64u * 1024u) + +static int crash_global_rank = -1; +static int crash_read_fd = -1; +static int crash_write_fd = -1; +static int crash_installed = 0; + +static char crash_altstack_buf[CRASH_ALTSTACK_SIZE]; +static char crash_site_buf[CRASH_SITE_BUF_SIZE]; + +static volatile sig_atomic_t handler_active = 0; + +/* abort_msg_s is not defined in any glibc public header + so we provide our own here. If glibc changes this in + a future version, this will have to change here too. */ +struct abort_msg_s { + unsigned int size; + char msg[]; +}; + +/* Given the program counter pc, puts + + into buffer buf of size buflen */ +static void resolve_pc_to_crash_site(unsigned long pc, char *buf, size_t buflen) +{ + if (buflen == 0) return; + buf[0] = '\0'; + if (crash_lib_offset_get_signal_safe(pc, buf, buflen) == 0) + return; + (void) crash_fmt_lib_offset(buf, buflen, "", pc); +} + +/* Builds the crash report message */ +static size_t build_crash_report(char *buf, size_t buflen, int rank, + const char *site) +{ + size_t name_len = strlen(site) + 1; + size_t payload_len = 2 * sizeof(int32_t) + name_len; + size_t total_len = sizeof(ldcs_message_header_t) + payload_len; + if (total_len > buflen) + return 0; + + ldcs_message_header_t hdr; + hdr.type = LDCS_MSG_CRASH_REPORT; + hdr.len = payload_len; + memcpy(buf, &hdr, sizeof(hdr)); + + char *payload = buf + sizeof(hdr); + int32_t rank32 = (int32_t) rank; + int32_t nlen32 = (int32_t) name_len; + memcpy(payload, &rank32, sizeof(rank32)); + memcpy(payload + sizeof(int32_t), &nlen32, sizeof(nlen32)); + memcpy(payload + 2 * sizeof(int32_t), site, name_len); + return total_len; +} + +/* Attempts to read the value of the libc __abort_msg into + buffer buf of size buflen. */ +static size_t read_abort_msg(char *buf, size_t buflen) +{ + struct abort_msg_s *volatile *abort_msg_loc = (struct abort_msg_s *volatile *) get_libc_abort_msg(); + + if (buflen == 0) return 0; + if (abort_msg_loc == NULL) return 0; // We failed to find the symbol + + struct abort_msg_s *p = *abort_msg_loc; + if (p == NULL) return 0; + if (p->size <= sizeof(unsigned int)) return 0; // if not bigger than the header, there is no abort_msg + if (p->size > CRASH_ABORT_MSG_MAX) return 0; + + size_t msg_size = (size_t) p->size - sizeof(unsigned int); + size_t cap = msg_size < buflen - 1 ? msg_size : buflen - 1; + size_t n = 0; + while (n < cap && p->msg[n] != '\0') { + buf[n] = p->msg[n]; + n++; + } + buf[n] = '\0'; + return n; +} + +/* Builds the crash site string according to the signal type. + For SIGABRT, use the abort_msg; otherwise, +. */ +static void crash_build_site(int sig, unsigned long pc, + char *buf, size_t buflen) +{ + if (sig == SIGABRT) { + static const char prefix[] = "abort:"; + const size_t prefix_len = sizeof(prefix) - 1; + memcpy(buf, prefix, prefix_len); + size_t n = read_abort_msg(buf + prefix_len, buflen - prefix_len); + // if we failed to get the abort string, fall back to + + if (n == 0) + resolve_pc_to_crash_site(pc, buf, buflen); + } else { + resolve_pc_to_crash_site(pc, buf, buflen); + } +} + +/* Send the CRASH_REQUEST to the server and read back the CRASH_RESPONSE. + We can't use the normal send/recv here because we're in a signal handler, + so instead do raw read/write to pipe. */ +static int crash_query_server(const char *site, int rank, int32_t *winner) +{ + char req_buf[CRASH_REQ_BUF_SIZE]; + size_t req_len = build_crash_report(req_buf, sizeof req_buf, rank, site); + if (req_len == 0) + return -1; + + if (crash_raw_write(crash_write_fd, req_buf, req_len) != 0) + return -1; + + ldcs_message_header_t resp_hdr; + if (crash_raw_read_exact(crash_read_fd, &resp_hdr, sizeof resp_hdr) != 0) + return -1; + if (resp_hdr.type != LDCS_MSG_CRASH_RESPONSE || + resp_hdr.len != sizeof(int32_t)) + return -1; + + int32_t winning_rank = -1; + if (crash_raw_read_exact(crash_read_fd, &winning_rank, + sizeof winning_rank) != 0) + return -1; + + *winner = winning_rank; + return 0; +} + +/* This is the main crash handler. Chains to application handler, if + installed, and checks whether the application fixed the fault. + If not, we are going to crash, and we report the crash and determine + whether we are the winning rank that will write the coredump for + this specific crash site. */ +static void crash_handler_entry(int sig, siginfo_t *info, void *uctx) +{ + /* Reentrancy flag to detect if we crash again while handling the crash. */ + static __thread sig_atomic_t reentering = 0; + + /* We need to restore errno before returning. */ + int saved_errno = errno; + + /* Get the faulting address. */ + unsigned long pc_before = extract_pc(uctx); + + /* If the application registered its own signal handler, call it. */ + int chained = crash_sigchain_chain_to_app(sig, info, uctx); + + /* We got past the application's signal handler. + If we refault past this point, we crashed in the crash handler; + in that case, give up and exit. */ + if (reentering) { + _exit(128 + sig); + } + reentering = 1; + + /* Check whether the application's signal handler resolved the fault. + If it did, we return and let the instruction re-execute. */ + if (chained && crash_sigchain_fault_resolved(sig, info, uctx, pc_before)) { + reentering = 0; + errno = saved_errno; + return; + } + + /* If we reach this point, the application did NOT fix the issue, so we + know this is a real crash. Now we set the handler_active flag. + This ensures that only the first crash to make it here goes through the + deduplication process. (Since whether to dump is a process-wide decision, + we can't make a different decision for different threads.) + Faulting threads other than the first one pause forever until we're done + with this signal handler, at which point the process will terminate. */ + if (!__sync_bool_compare_and_swap(&handler_active, 0, 1)) { + for (;;) pause(); + } + + if (!crash_installed || crash_write_fd < 0 || crash_read_fd < 0) { + goto reraise; + } + + /* Now we do the actual deduplication part. We get the program counter, + resolve it to + or abort_msg, and pass that crashsite + to the server, which picks one winner per crashsite. */ + unsigned long pc = extract_pc(uctx); + crash_build_site(sig, pc, crash_site_buf, sizeof crash_site_buf); + + int32_t winning_rank = -1; + if (crash_query_server(crash_site_buf, crash_global_rank, + &winning_rank) != 0) + goto reraise; + + if ((int) winning_rank != crash_global_rank) { + /* If we are NOT the winner, we set our own core limit to zero, + preventing us from dumping. If we are the winner, do nothing, + preserving the existing core limit. */ + struct rlimit no_core = { 0, 0 }; + (void) setrlimit(RLIMIT_CORE, &no_core); + } + +reraise: + /* Finally, we restore the default signal handler and return, + terminating the process and producing a coredump if the limit allows.. */ + signal(sig, SIG_DFL); +} + +/* Performs setup and installs the signal handler. */ +int crash_handler_install(int global_rank, int ldcsid_in) +{ + if (crash_installed) + return 0; + + crash_sigchain_init(); + + crash_global_rank = global_rank; + + if (client_get_raw_fds(ldcsid_in, &crash_read_fd, &crash_write_fd) != 0 || + crash_read_fd < 0 || crash_write_fd < 0) { + err_printf("failed to get raw FDs for crash handler\n"); + crash_read_fd = -1; + crash_write_fd = -1; + return 0; + } + + crash_lib_offset_prime(); + + /* Perform a dummy resolution to populate the cache. */ + { + char tmp[256]; + resolve_pc_to_crash_site((unsigned long) &crash_handler_install, + tmp, sizeof tmp); + } + + /* Set up the altstack. + If the reason for a segfault is a stack overflow, the signal handler itself + will have no stack available. We handle this by registering an alternate stack + for the signal handler. However, note that this is per-thread, and currently + we do not register an alternate stack on any thread other than the main thread. + TODO: handle alternate stack on other threads */ + stack_t ss; + memset(&ss, 0, sizeof ss); + ss.ss_sp = crash_altstack_buf; + ss.ss_size = CRASH_ALTSTACK_SIZE; + ss.ss_flags = 0; + if (sigaltstack(&ss, NULL) != 0) { + debug_printf("sigaltstack failed, not installing\n"); + crash_read_fd = -1; + crash_write_fd = -1; + return 0; + } + + /* Install the signal handler. */ + struct sigaction sa; + memset(&sa, 0, sizeof sa); + sa.sa_sigaction = crash_handler_entry; + sa.sa_flags = SA_SIGINFO | SA_ONSTACK | SA_RESTART; + sigemptyset(&sa.sa_mask); + sigaddset(&sa.sa_mask, SIGSEGV); + sigaddset(&sa.sa_mask, SIGBUS); + sigaddset(&sa.sa_mask, SIGFPE); + sigaddset(&sa.sa_mask, SIGILL); + sigaddset(&sa.sa_mask, SIGABRT); + + const int sigs[] = { SIGSEGV, SIGBUS, SIGFPE, SIGILL, SIGABRT }; + for (size_t i = 0; i < sizeof(sigs) / sizeof(sigs[0]); i++) { + /* If a signal handler was already installed by the time Spindle + registers its signal handler, we save it to use in + signal handler chaining. */ + struct sigaction kernel_old; + memset(&kernel_old, 0, sizeof kernel_old); + if (sigaction(sigs[i], &sa, &kernel_old) != 0) { + debug_printf("sigaction(%d) failed when installing crash handler\n", sigs[i]); + continue; + } + crash_sigchain_register_existing_handler(sigs[i], &kernel_old); + } + + crash_installed = 1; + return 0; +} diff --git a/src/client/crash_handler/crash_handler.h b/src/client/crash_handler/crash_handler.h new file mode 100644 index 00000000..c33c9762 --- /dev/null +++ b/src/client/crash_handler/crash_handler.h @@ -0,0 +1,23 @@ +/* +This file is part of Spindle. For copyright information see the COPYRIGHT +file in the top level directory, or at +https://github.com/hpc/Spindle/blob/master/COPYRIGHT + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU Lesser General Public License (as published by the Free Software +Foundation) version 2.1 dated February 1999. This program is distributed in the +hope that it will be useful, but WITHOUT ANY WARRANTY; without even the IMPLIED +WARRANTY OF MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms +and conditions of the GNU Lesser General Public License for more details. You should +have received a copy of the GNU Lesser General Public License along with this +program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA +*/ + +#if !defined(CRASH_HANDLER_H_) +#define CRASH_HANDLER_H_ + +/* Install Spindle's coredump-deduplicating signal handler. */ +int crash_handler_install(int global_rank, int ldcsid); + +#endif diff --git a/src/client/crash_handler/crash_io.c b/src/client/crash_handler/crash_io.c new file mode 100644 index 00000000..aa08cbe7 --- /dev/null +++ b/src/client/crash_handler/crash_io.c @@ -0,0 +1,69 @@ +/* +This file is part of Spindle. For copyright information see the COPYRIGHT +file in the top level directory, or at +https://github.com/hpc/Spindle/blob/master/COPYRIGHT + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU Lesser General Public License (as published by the Free Software +Foundation) version 2.1 dated February 1999. This program is distributed in the +hope that it will be useful, but WITHOUT ANY WARRANTY; without even the IMPLIED +WARRANTY OF MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms +and conditions of the GNU Lesser General Public License for more details. You should +have received a copy of the GNU Lesser General Public License along with this +program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA +*/ + +#include +#include +#include + +#include "crash_io.h" + +/* This file contains async-signal-safe functions for reading from + and writing to file descriptors. Used by the crash handler to + communicate with the server. */ + +int crash_raw_write(int fd, const void *buf, size_t len) +{ + const char *p = (const char *) buf; + size_t left = len; + while (left > 0) { + int saved_errno = errno; + ssize_t n = write(fd, p, left); + int write_errno = errno; + errno = saved_errno; + if (n < 0) { + if (write_errno == EINTR || write_errno == EAGAIN) + continue; + return -1; + } + if (n == 0) + return -1; + p += n; + left -= (size_t) n; + } + return 0; +} + +int crash_raw_read_exact(int fd, void *buf, size_t nbytes) +{ + char *p = (char *) buf; + size_t left = nbytes; + while (left > 0) { + int saved_errno = errno; + ssize_t n = read(fd, p, left); + int read_errno = errno; + errno = saved_errno; + if (n < 0) { + if (read_errno == EINTR || read_errno == EAGAIN) + continue; + return -1; + } + if (n == 0) + return -1; + p += n; + left -= (size_t) n; + } + return 0; +} diff --git a/src/client/crash_handler/crash_io.h b/src/client/crash_handler/crash_io.h new file mode 100644 index 00000000..0d192083 --- /dev/null +++ b/src/client/crash_handler/crash_io.h @@ -0,0 +1,25 @@ +/* +This file is part of Spindle. For copyright information see the COPYRIGHT +file in the top level directory, or at +https://github.com/hpc/Spindle/blob/master/COPYRIGHT + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU Lesser General Public License (as published by the Free Software +Foundation) version 2.1 dated February 1999. This program is distributed in the +hope that it will be useful, but WITHOUT ANY WARRANTY; without even the IMPLIED +WARRANTY OF MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms +and conditions of the GNU Lesser General Public License for more details. You should +have received a copy of the GNU Lesser General Public License along with this +program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA +*/ + +#if !defined(CRASH_IO_H_) +#define CRASH_IO_H_ + +#include + +int crash_raw_write(int fd, const void *buf, size_t len); +int crash_raw_read_exact(int fd, void *buf, size_t nbytes); + +#endif diff --git a/src/client/crash_handler/crash_lib_offset.c b/src/client/crash_handler/crash_lib_offset.c new file mode 100644 index 00000000..b2ffca58 --- /dev/null +++ b/src/client/crash_handler/crash_lib_offset.c @@ -0,0 +1,141 @@ +/* +This file is part of Spindle. For copyright information see the COPYRIGHT +file in the top level directory, or at +https://github.com/hpc/Spindle/blob/master/COPYRIGHT + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU Lesser General Public License (as published by the Free Software +Foundation) version 2.1 dated February 1999. This program is distributed in the +hope that it will be useful, but WITHOUT ANY WARRANTY; without even the IMPLIED +WARRANTY OF MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms +and conditions of the GNU Lesser General Public License for more details. You should +have received a copy of the GNU Lesser General Public License along with this +program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA +*/ + +#define _GNU_SOURCE + +#include "crash_lib_offset.h" +#include "crash_fmt.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define MAX_PATH_LEN 4096 + +struct r_debug_ext_mirror { + struct r_debug base; + void *r_next; +}; + +/* It turns out that we can't get at the program headers via + dl_phdr_iterate in a signal handler, because dl_phdr_iterate + calls malloc, which is not async-signal-safe. So we have get the + program headers through getauxval(AT_PHDR) and iterate ourselves. */ + +static const ElfW(Phdr) *exe_auxv_phdrs; +static unsigned long exe_auxv_phnum; +static char exe_path_cache[MAX_PATH_LEN + 1]; +static char *exe_path_cached; + +/* Get the path to the current executable. This is used for + the part of + when the address + in in the executable. */ +static char *get_executable_path(void) +{ + long r; + + if (exe_path_cached) + return exe_path_cached; + + r = syscall(SYS_readlinkat, AT_FDCWD, "/proc/self/exe", + exe_path_cache, (size_t) MAX_PATH_LEN); + if (r < 0 || r > MAX_PATH_LEN) + return exe_path_cached = (char *) "[EXECUTABLE]"; + exe_path_cache[r] = '\0'; + return exe_path_cached = exe_path_cache; +} + +static int find_pc_et_dyn(unsigned long load_base, unsigned long pc) +{ + const ElfW(Ehdr) *ehdr = (const ElfW(Ehdr) *) load_base; + const ElfW(Phdr) *phdrs; + unsigned int i; + + if (memcmp(ehdr->e_ident, ELFMAG, SELFMAG) != 0) + return 0; + + phdrs = (const ElfW(Phdr) *) (load_base + ehdr->e_phoff); + for (i = 0; i < ehdr->e_phnum; i++) { + if (phdrs[i].p_type != PT_LOAD) + continue; + unsigned long start = load_base + phdrs[i].p_vaddr; + if (pc >= start && pc < start + phdrs[i].p_memsz) + return 1; + } + return 0; +} + +static int find_pc_et_exec(unsigned long pc) +{ + unsigned long i; + + if (!exe_auxv_phdrs) + return 0; + for (i = 0; i < exe_auxv_phnum; i++) { + if (exe_auxv_phdrs[i].p_type != PT_LOAD) + continue; + if (pc >= exe_auxv_phdrs[i].p_vaddr && + pc < exe_auxv_phdrs[i].p_vaddr + exe_auxv_phdrs[i].p_memsz) + return 1; + } + return 0; +} + +static int walk_link_map_list(struct link_map *cur, unsigned long pc, + char *buf, size_t buflen) +{ + for (; cur != NULL; cur = cur->l_next) { + int hit = (cur->l_addr != 0) + ? find_pc_et_dyn(cur->l_addr, pc) + : find_pc_et_exec(pc); + if (hit) { + const char *use_name = + (cur->l_name && cur->l_name[0]) ? cur->l_name + : get_executable_path(); + return crash_fmt_lib_offset(buf, buflen, use_name, pc - cur->l_addr); + } + } + return -1; +} + +int crash_lib_offset_get_signal_safe(unsigned long pc, char *buf, size_t buflen) +{ + const struct r_debug *rd = &_r_debug; + int extended = (_r_debug.r_version >= 2); + + while (rd != NULL) { + if (walk_link_map_list(rd->r_map, pc, buf, buflen) == 0) + return 0; + if (!extended) + break; + rd = (const struct r_debug *) + ((const struct r_debug_ext_mirror *) rd)->r_next; + } + return -1; +} + +void crash_lib_offset_prime(void) +{ + (void) get_executable_path(); + exe_auxv_phdrs = (const ElfW(Phdr) *) getauxval(AT_PHDR); + exe_auxv_phnum = getauxval(AT_PHNUM); +} diff --git a/src/client/crash_handler/crash_lib_offset.h b/src/client/crash_handler/crash_lib_offset.h new file mode 100644 index 00000000..a50bbe69 --- /dev/null +++ b/src/client/crash_handler/crash_lib_offset.h @@ -0,0 +1,27 @@ +/* +This file is part of Spindle. For copyright information see the COPYRIGHT +file in the top level directory, or at +https://github.com/hpc/Spindle/blob/master/COPYRIGHT + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU Lesser General Public License (as published by the Free Software +Foundation) version 2.1 dated February 1999. This program is distributed in the +hope that it will be useful, but WITHOUT ANY WARRANTY; without even the IMPLIED +WARRANTY OF MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms +and conditions of the GNU Lesser General Public License for more details. You should +have received a copy of the GNU Lesser General Public License along with this +program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA +*/ + +#if !defined(CRASH_LIB_OFFSET_H_) +#define CRASH_LIB_OFFSET_H_ + +#include + +void crash_lib_offset_prime(void); + +int crash_lib_offset_get_signal_safe(unsigned long pc, + char *buf, size_t buflen); + +#endif diff --git a/src/client/crash_handler/crash_sigchain.c b/src/client/crash_handler/crash_sigchain.c new file mode 100644 index 00000000..43d87b39 --- /dev/null +++ b/src/client/crash_handler/crash_sigchain.c @@ -0,0 +1,470 @@ +/* +This file is part of Spindle. For copyright information see the COPYRIGHT +file in the top level directory, or at +https://github.com/hpc/Spindle/blob/master/COPYRIGHT + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU Lesser General Public License (as published by the Free Software +Foundation) version 2.1 dated February 1999. This program is distributed in the +hope that it will be useful, but WITHOUT ANY WARRANTY; without even the IMPLIED +WARRANTY OF MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms +and conditions of the GNU Lesser General Public License for more details. You should +have received a copy of the GNU Lesser General Public License along with this +program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA +*/ + +#define _GNU_SOURCE + +#include "crash_sigchain.h" +#include "crash_arch.h" +#include "intercept.h" +#include "spindle_debug.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* This file deals with signal handler chaining. If the application + has registered its own signal handler, we want to run that one first, + before Spindle runs its own. This turns out to be rather complicated, + because the application might do something in its signal handler that + fixes the fault, making it no longer a crash that Spindle should handle. + As an example, the Java and Julia runtimes implement garbage collection + "safepoints" (points in the code where the GC is allowed to run) as reads + to PROT_NONE pages. The signal handler then waits for all threads to reach + a safepoint, runs the GC, then unprotects the page and returns, allowing the + faulting instruction to re-execute. If Spindle ran its own signal handler + after this, safepoints would crash. As another example, libsigsegv uses + a SIGSEGV handler to implement user-level paging. Much of the complexity in + this file comes from handling cases such as these. */ + +/* CAUTION: signal and sigaction are on the async-signal-safe list, so the wrappers + themselves must also be async-signal-safe! */ + +typedef void (*sighandler_t)(int); + +void *orig_sigaction; +void *orig_signal; +void *orig_bsd_signal; +void *orig_sysv_signal; + +struct app_disposition { + void (*handler)(int); + void (*sigaction_fn)(int, siginfo_t *, void *); + sigset_t mask; + int flags; + int has_sigaction; + int active; +}; + +static const int OWNED_SIGS[] = { SIGSEGV, SIGBUS, SIGFPE, SIGILL, SIGABRT }; + +int crash_sigchain_is_owned(int sig) +{ + for (size_t i = 0; i < sizeof(OWNED_SIGS) / sizeof(OWNED_SIGS[0]); i++) { + if (OWNED_SIGS[i] == sig) + return 1; + } + return 0; +} + +/* Storage for the application's registered signal handlers */ +static struct app_disposition disp_store[NSIG]; + +/* Spinlock to protect access to stored app signal dispositions.*/ +static volatile int disp_lock[NSIG]; + +static void crash_disp_lock(int sig) +{ + while (__sync_lock_test_and_set(&disp_lock[sig], 1)) { + sched_yield(); + } +} + +static void crash_disp_unlock(int sig) +{ + __sync_lock_release(&disp_lock[sig]); +} + +/* Caller must hold disp lock for signal */ +static void store_disposition(int sig, const struct sigaction *act) +{ + if (act == NULL) return; + struct app_disposition *d = &disp_store[sig]; + if (!(act->sa_flags & SA_SIGINFO) && + (act->sa_handler == SIG_DFL || act->sa_handler == SIG_IGN)) { + d->active = 0; + return; + } + d->has_sigaction = (act->sa_flags & SA_SIGINFO) ? 1 : 0; + if (d->has_sigaction) { + d->sigaction_fn = act->sa_sigaction; + d->handler = NULL; + } else { + d->handler = act->sa_handler; + d->sigaction_fn = NULL; + } + d->mask = act->sa_mask; + d->flags = act->sa_flags & ~(SA_SIGINFO | SA_ONSTACK | SA_RESTART); + d->active = 1; +} + +/* Caller must hold disp lock for signal */ +static void get_disposition(int sig, struct sigaction *out) +{ + struct app_disposition *d = &disp_store[sig]; + memset(out, 0, sizeof *out); + if (!d->active) { + out->sa_handler = SIG_DFL; + return; + } + if (d->has_sigaction) { + out->sa_sigaction = d->sigaction_fn; + out->sa_flags = d->flags | SA_SIGINFO; + } else { + out->sa_handler = d->handler; + out->sa_flags = d->flags; + } + out->sa_mask = d->mask; +} + +/* This is called during initialization to register a signal handler that + was already registered before Spindle's crash handler was registered */ +void crash_sigchain_register_existing_handler(int sig, const struct sigaction *handler_old) +{ + if (!crash_sigchain_is_owned(sig)) return; + crash_disp_lock(sig); + store_disposition(sig, handler_old); + crash_disp_unlock(sig); + /* This function is not called from a signal handler, so it's safe to log */ + debug_printf2("stored application signal handler for sig %d\n", sig); +} + +/* Run the application signal handler, if present. + Returns 0 if there was no handler to run, 1 if there was an application + handler and we successfully ran it and returned from it. */ +int crash_sigchain_chain_to_app(int sig, siginfo_t *info, void *ucontext) +{ + struct app_disposition local; + + crash_disp_lock(sig); + + /* If there's no application handler, nothing more for us to do */ + if (!disp_store[sig].active) { + crash_disp_unlock(sig); + return 0; + } + + /* Make a local copy of the signal disposition so we can release the lock + before calling the stored signal handler, as it might call sigaction itself */ + local = disp_store[sig]; + + /* If the app requested the handler be reset, do so */ + if (local.flags & SA_RESETHAND) { + disp_store[sig].active = 0; + } + crash_disp_unlock(sig); + + /* Build the mask we want active during the application's handler */ + sigset_t blockset = local.mask; + if (!(local.flags & SA_NODEFER)) sigaddset(&blockset, sig); + sigset_t saved; + (void) sigprocmask(SIG_BLOCK, &blockset, &saved); + + /* Call the application's signal handler */ + if (local.has_sigaction) { + local.sigaction_fn(sig, info, ucontext); + } else { + local.handler(sig); + } + + /* If we reach here, the application handler returned. + Restore the mask to its previous value. */ + (void) sigprocmask(SIG_SETMASK, &saved, NULL); + return 1; +} + +/* Fault resolution detection + This section contains functions used to detect whether the application's + signal handler fixed the fault or not. */ + +#ifdef PROCMAP_QUERY +/* Linux 6.11 and later let us query /proc/pid/maps without textual + parsing. This stores whether we can do that. */ +static int crash_maps_query_supported = 0; +#endif + +/* Probe whether addr is readable in this process by attempting a + 1-byte process_vm_readv from a local stack buffer. Returns + 1 if the read succeeded + 0 if the read faultedd + -1 if the read failed for some reason other than EFAULT + We use this to detect if the application has fixed the reason for + a read fault (e.g., for the Java/Julia safepoint pattern). + */ +static inline int crash_probe_byte_readable(const void *addr) +{ + char buf; + struct iovec local = { &buf, 1 }; + struct iovec remote = { (void *) addr, 1 }; + /* process_vm_readv is intended to let a process read the memory + of its ptrace target, but it can also be used to read one's own + memory. In the event of a fault, it returns EFAULT instead of + triggering a signal. */ + long n = syscall(SYS_process_vm_readv, + (long) getpid(), + &local, 1UL, + &remote, 1UL, + 0UL); + if (n == 1) return 1; // Successfully read 1 byte + if (n == -1 && errno == EFAULT) return 0; // Faulted + return -1; // Neither successfully read nor faulted +} + +/* Checking whether the application handler fixed a write fault is + much more annoying than the read case. For reads, we can safely + just retry the read and see if it succeeded. For writes, however, + we can't safely retry without actually writing into the application's + memory. So for the write case, we instead check /proc/self/maps to see + whether the address in question is writable after the application + handler returns. */ + +#ifdef PROCMAP_QUERY +/* On Linux 6.11 and later, we can read /proc/self/maps via ioctl + instead of having to parse it ourselves. */ +static inline int crash_maps_check_writable_via_ioctl(int fd, uintptr_t addr) +{ + struct procmap_query q; + memset(&q, 0, sizeof q); + q.size = sizeof q; + q.query_addr = (uint64_t) addr; + q.query_flags = 0; + int rc; + do { + rc = ioctl(fd, PROCMAP_QUERY, &q); + } while (rc < 0 && errno == EINTR); + if (rc < 0) { + if (errno == ENOENT) + return 0; + return -1; + } + return (q.vma_flags & PROCMAP_QUERY_VMA_WRITABLE) ? 1 : 0; +} +#endif /* PROCMAP_QUERY */ + +/* On Linux kernels older than 6.11, the ioctl doesn't exist and we + have to parse /proc/self/maps using only async-signal-safe functions. */ + +/* Hex digit value, or -1 if c is not a hex digit. + We need this because strtoul is not on the async-signal-safe list. */ +static inline int hexval(char c) +{ + if (c >= '0' && c <= '9') return c - '0'; + if (c >= 'a' && c <= 'f') return c - 'a' + 10; + if (c >= 'A' && c <= 'F') return c - 'A' + 10; + return -1; +} + +/* Parse /proc/self/maps looking for the range containing addr, checking + whether it is writable. Returns 1 if writable, 0 if not writable or + not mapped, and -1 on error. */ +static int crash_maps_check_writable_via_textparse(int fd, uintptr_t addr) +{ + const size_t buf_size = 4096; + char *buf = (char *) mmap(NULL, buf_size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, + -1, 0); + if (buf == MAP_FAILED) + return -1; + + enum { ADDR_START, ADDR_END, PERMS, SKIP_EOL } state = ADDR_START; + uintptr_t start = 0, end = 0; + int perm_idx = 0, writable = 0; + int result = 0; + int done = 0; + + while (!done) { + ssize_t n; + do { + n = read(fd, buf, buf_size); + } while (n < 0 && errno == EINTR); + if (n < 0) { + result = -1; + break; + } + if (n == 0) // EOF + break; + + for (ssize_t i = 0; i < n && !done; i++) { + char c = buf[i]; + switch (state) { + /* We begin parsing a line with the start address of the range */ + case ADDR_START: + if (c == '-') { + /* When we get to the '-', change to parsing the end address */ + state = ADDR_END; + } else { + /* Accumulate one hex digit at at time */ + start = (start << 4) | hexval(c); + } + break; + case ADDR_END: + if (c == ' ') { + /* When we get to a space, we're done with the range and permissions are next */ + state = PERMS; + perm_idx = 0; + } else { + /* Accumulate one hex digit at at time */ + end = (end << 4) | hexval(c); + } + break; + case PERMS: + /* perms are "rwxp". We only care about the 'w' byte at index 1. */ + if (perm_idx++ == 1) { + writable = (c == 'w'); + /* Now that we have the range and writability, check if our + address is in this range */ + if (start > addr) { + done = 1; + } else if (addr < end) { + result = writable; + done = 1; + } else { + state = SKIP_EOL; + } + } + break; + case SKIP_EOL: + if (c == '\n') { + /* When we get to the end of the line, set up to parse the next line */ + state = ADDR_START; + start = 0; + end = 0; + } + break; + } + } + } + + (void) munmap(buf, buf_size); + return result; +} + +/* Probe whether addr is writable in this process by checking + /proc/self/maps. Returns + 1 if the address is mapped in a writable page + 0 if the address is not mapped in a writable page + -1 if an error occurred reading the maps file + We use this to detect if the application has fixed the reason for + a write fault (e.g., libsigsegv for user-level paging). + */ +static inline int crash_maps_check_writable(uintptr_t addr) +{ + int fd = open("/proc/self/maps", O_RDONLY | O_CLOEXEC); + if (fd < 0) return -1; + + int rc; +#ifdef PROCMAP_QUERY + if (crash_maps_query_supported) + rc = crash_maps_check_writable_via_ioctl(fd, addr); + else +#endif + rc = crash_maps_check_writable_via_textparse(fd, addr); + + (void) close(fd); + return rc; +} + +void crash_sigchain_init(void) +{ +#ifdef PROCMAP_QUERY + int fd = open("/proc/self/maps", O_RDONLY | O_CLOEXEC); + if (fd >= 0) { + if (crash_maps_check_writable_via_ioctl(fd, (uintptr_t) &crash_maps_query_supported) == 1) + crash_maps_query_supported = 1; + (void) close(fd); + } +#endif /* PROCMAP_QUERY */ +} + +/* The main entrypoint to the fault recovery detection. */ +int crash_sigchain_fault_resolved(int sig, siginfo_t *info, void *uctx, + unsigned long pc_before) +{ + /* PC advanced inside the application handled. */ + if (extract_pc(uctx) != pc_before) + return 1; + + /* The remaining possible fixes only apply to faulting on an + addresss (SIGSEGV, SIGBUS). */ + if (sig != SIGSEGV && sig != SIGBUS) + return 1; + + /* If we faulted on a write, check if the address is now mapped writable. + If we faulted on a read, retry the read and check if we refault. */ + int rc = pf_is_write(uctx) ? crash_maps_check_writable((uintptr_t) info->si_addr) + : crash_probe_byte_readable(info->si_addr); + return (rc == 1) ? 1 : 0; +} + +/* sigaction-family wrappers */ + +int sigaction_wrapper(int sig, const struct sigaction *act, + struct sigaction *oldact) +{ + if (!crash_sigchain_is_owned(sig)) { + return ((int (*)(int, const struct sigaction *, struct sigaction *)) + orig_sigaction)(sig, act, oldact); + } + + crash_disp_lock(sig); + if (oldact != NULL) get_disposition(sig, oldact); + store_disposition(sig, act); + crash_disp_unlock(sig); + return 0; +} + +static sighandler_t signal_common(int sig, sighandler_t handler, + int flags, void *orig) +{ + if (!crash_sigchain_is_owned(sig)) { + return ((sighandler_t (*)(int, sighandler_t)) orig)(sig, handler); + } + + struct sigaction act, oldact; + memset(&act, 0, sizeof act); + act.sa_handler = handler; + act.sa_flags = flags; + (void) sigaction_wrapper(sig, &act, &oldact); + + return (oldact.sa_flags & SA_SIGINFO) ? SIG_DFL : oldact.sa_handler; +} + +void (*signal_wrapper(int sig, void (*handler)(int)))(int) +{ + return signal_common(sig, handler, SA_RESTART, orig_signal); +} + +void (*bsd_signal_wrapper(int sig, void (*handler)(int)))(int) +{ + return signal_common(sig, handler, SA_RESTART, orig_bsd_signal); +} + +void (*sysv_signal_wrapper(int sig, void (*handler)(int)))(int) +{ + return signal_common(sig, handler, SA_RESETHAND | SA_NODEFER, + orig_sysv_signal); +} diff --git a/src/client/crash_handler/crash_sigchain.h b/src/client/crash_handler/crash_sigchain.h new file mode 100644 index 00000000..6856d0f8 --- /dev/null +++ b/src/client/crash_handler/crash_sigchain.h @@ -0,0 +1,36 @@ +/* +This file is part of Spindle. For copyright information see the COPYRIGHT +file in the top level directory, or at +https://github.com/hpc/Spindle/blob/master/COPYRIGHT + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU Lesser General Public License (as published by the Free Software +Foundation) version 2.1 dated February 1999. This program is distributed in the +hope that it will be useful, but WITHOUT ANY WARRANTY; without even the IMPLIED +WARRANTY OF MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms +and conditions of the GNU Lesser General Public License for more details. You should +have received a copy of the GNU Lesser General Public License along with this +program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA +*/ + +#if !defined(CRASH_SIGCHAIN_H_) +#define CRASH_SIGCHAIN_H_ + +#include +#include + +int crash_sigchain_is_owned(int sig); +void crash_sigchain_register_existing_handler(int sig, const struct sigaction *handler_old); +int crash_sigchain_chain_to_app(int sig, siginfo_t *info, void *ucontext); +void crash_sigchain_init(void); +int crash_sigchain_fault_resolved(int sig, siginfo_t *info, void *uctx, + unsigned long pc_before); + +int sigaction_wrapper(int sig, const struct sigaction *act, + struct sigaction *oldact); +void (*signal_wrapper(int sig, void (*handler)(int)))(int); +void (*bsd_signal_wrapper(int sig, void (*handler)(int)))(int); +void (*sysv_signal_wrapper(int sig, void (*handler)(int)))(int); + +#endif From dafe83b2c986f93588449d36561b90fb6b4b4b64 Mon Sep 17 00:00:00 2001 From: Nicholas Chaimov Date: Mon, 15 Jun 2026 14:58:40 -0700 Subject: [PATCH 04/14] Crash handler: --crash-dedup option Adds --crash-dedup option to control whether crash deduplication is enabled. --- doc/spindle_launch_README.md | 4 ++++ src/fe/startup/config_mgr.cc | 13 ++++++++++++- src/fe/startup/config_mgr.h | 4 +++- src/fe/startup/spindle_fe.cc | 1 + src/flux/flux-spindle.c | 9 +++++++-- src/include/spindle_launch.h | 3 ++- 6 files changed, 29 insertions(+), 5 deletions(-) diff --git a/doc/spindle_launch_README.md b/doc/spindle_launch_README.md index 38f45eea..a30c8df8 100644 --- a/doc/spindle_launch_README.md +++ b/doc/spindle_launch_README.md @@ -122,6 +122,10 @@ we have to break ABI compatibility. `spindleRunBE()`. Whether the daemons are launched via ssh or rsh can be controlled through command arguments, configure, or an environment variable (SPINDLE_RSH). + - 'OPT_CRASH_HANDLER' - When the application crashes (e.g. SIGSEGV), + Spindle deduplicates the reports by crash site, selecting one + representative per unique site, permitting only those representatives + to write a coredump. - `typedef struct { ... } spindle_args_t` diff --git a/src/fe/startup/config_mgr.cc b/src/fe/startup/config_mgr.cc index 1b9d9ade..e1368642 100644 --- a/src/fe/startup/config_mgr.cc +++ b/src/fe/startup/config_mgr.cc @@ -142,6 +142,12 @@ using namespace std; #define RSHCMD_STR "" #endif +#if defined(CRASH_DEDUP_ENABLED_BY_DEFAULT) +#define DEFAULT_CRASH_DEDUP_STR "true" +#else +#define DEFAULT_CRASH_DEDUP_STR "false" +#endif + #if defined(PYTHON_INST_PREFIX) #define PYTHON_PREFIX_DEFAULT PYTHON_INST_PREFIX #else @@ -294,7 +300,9 @@ void initOptionsList() { confEnableRsh, "enable-rsh", shortRSHMode, groupMisc, cvBool, {}, RSHLAUNCH_STR, "Enable starting daemons with an rsh tree, if the startup mode supports it." }, { confRshCommand, "rsh-command", shortRSHCmd, groupMisc, cvString, {}, RSHCMD_STR, - "The command to run rsh/ssh, when doing RSH startup mode." } + "The command to run rsh/ssh, when doing RSH startup mode." }, + { confCrashDedup, "crash-dedup", shortCrashDedup, groupMisc, cvBool, {}, DEFAULT_CRASH_DEDUP_STR, + "Deduplicate coredumps by crash site, emitting only one coredump per unique site." } } ); } @@ -998,6 +1006,9 @@ bool ConfigMap::toSpindleArgs(spindle_args_t &args, bool alloc_strs) const case confPatchLdso: setopt(args.opts, OPT_PATCHLDSO, boolresult); break; + case confCrashDedup: + setopt(args.opts, OPT_CRASH_HANDLER, boolresult); + break; } } diff --git a/src/fe/startup/config_mgr.h b/src/fe/startup/config_mgr.h index ec3c8135..14f18b7a 100644 --- a/src/fe/startup/config_mgr.h +++ b/src/fe/startup/config_mgr.h @@ -68,7 +68,8 @@ enum SpindleConfigID { confStartMultiSession, confEndSession, confRunSession, - confPatchLdso + confPatchLdso, + confCrashDedup }; enum CmdlineShortOptions { @@ -128,6 +129,7 @@ enum CmdlineShortOptions { shortExecExcludes = 298, shortPatchLdso = 299, shortCachePaths = 300, + shortCrashDedup = 301, }; enum CmdlineGroups { diff --git a/src/fe/startup/spindle_fe.cc b/src/fe/startup/spindle_fe.cc index 0d5c0817..f1daab9e 100644 --- a/src/fe/startup/spindle_fe.cc +++ b/src/fe/startup/spindle_fe.cc @@ -350,6 +350,7 @@ static void printSpindleFlags(opt_t opts) { printFlag(opts, OPT_NUMA, "OPT_NUMA", ss); printFlag(opts, OPT_OFF, "OPT_OFF", ss); printFlag(opts, OPT_PATCHLDSO, "OPT_PATCHLDSO", ss); + printFlag(opts, OPT_CRASH_HANDLER, "OPT_CRASH_HANDLER", ss); ss << ", "; if (OPT_GET_SEC(opts) == OPT_SEC_MUNGE) ss << "OPT_SEC_MUNGE"; if (OPT_GET_SEC(opts) == OPT_SEC_KEYLMON) ss << "OPT_SEC_KEYLMON"; diff --git a/src/flux/flux-spindle.c b/src/flux/flux-spindle.c index dd00aa32..a3abeef7 100644 --- a/src/flux/flux-spindle.c +++ b/src/flux/flux-spindle.c @@ -379,6 +379,7 @@ static int sp_getopts (flux_shell_t *shell, struct spindle_ctx *ctx) int pull = 0; int had_error = 0; int numa = 0; + int crash_dedup = 0; const char *relocaout = NULL, *reloclibs = NULL, *relocexec = NULL, *relocpython = NULL; const char *followfork = NULL, *preload = NULL, *level = NULL; const char *pyprefix = NULL, *commpath = NULL; @@ -404,7 +405,7 @@ static int sp_getopts (flux_shell_t *shell, struct spindle_ctx *ctx) * supplied by the user, but not unpacked (This handles typos, etc). */ if (json_unpack_ex (opts, &error, JSON_STRICT, - "{s?i s?i s?i s?i s?s s?s s?s s?s s?s s?s s?s s?i s?s s?s s?s s?s}", + "{s?i s?i s?i s?i s?s s?s s?s s?s s?s s?s s?s s?i s?s s?s s?s s?s s?i}", "noclean", &noclean, "nostrip", &nostrip, "push", &push, @@ -420,7 +421,8 @@ static int sp_getopts (flux_shell_t *shell, struct spindle_ctx *ctx) "numa-files", &numafiles, "preload", &preload, "level", &level, - "cachepaths", &cachepaths) < 0) + "cachepaths", &cachepaths, + "crash-dedup", &crash_dedup) < 0) logerrno_printf_and_return(1, "Error in spindle option: %s\n", error.text); if (noclean) @@ -469,6 +471,9 @@ static int sp_getopts (flux_shell_t *shell, struct spindle_ctx *ctx) if (commpath) { ctx->params.commpath = (char *) commpath; } + if (crash_dedup) { + ctx->params.opts |= OPT_CRASH_HANDLER; + } if (level) { if (strcmp(level, "high") == 0) { ctx->params.opts |= OPT_RELOCAOUT; diff --git a/src/include/spindle_launch.h b/src/include/spindle_launch.h index ca7b8d3d..98121396 100644 --- a/src/include/spindle_launch.h +++ b/src/include/spindle_launch.h @@ -60,7 +60,8 @@ extern "C" { #define OPT_NUMA (1 << 29) /* Enables file replication across NUMA domains */ #define OPT_OFF (1 << 30) /* Turns spindle off, disabling everything */ #define OPT_PATCHLDSO (1 << 31) /* Enables patching of ld.so to intercept stat calls */ - +#define OPT_CRASH_HANDLER ((opt_t) 1 << 32) /* Enables crash handler with coredump deduplication */ + #define OPT_SET_SEC(OPT, X) OPT |= (X << 19) #define OPT_GET_SEC(OPT) ((OPT >> 19) & 7) #define OPT_SEC_MUNGE 0 /* Use munge to validate connections */ From f51c78fabdd30139d58bad9990c3e544327c7fbf Mon Sep 17 00:00:00 2001 From: Nicholas Chaimov Date: Wed, 17 Jun 2026 11:38:21 -0700 Subject: [PATCH 05/14] Crash handler: configure options Adds --enable-crash-handler to control whether the crash handler is compiled. Adds --enable-crash-dedup to control whether crash deduplication is enabled by default at runtime. --- configure.common.ac | 24 ++++++++++++++++++++++++ src/client/auditclient/Makefile.am | 2 +- src/client/client/Makefile.am | 2 +- src/client/subaudit/Makefile.am | 2 +- 4 files changed, 27 insertions(+), 3 deletions(-) diff --git a/configure.common.ac b/configure.common.ac index baa9c0a9..41da35f0 100644 --- a/configure.common.ac +++ b/configure.common.ac @@ -427,6 +427,30 @@ AC_ARG_ENABLE(sec-none, [AS_HELP_STRING([--enable-sec-none],[Disable security authentication of connections])], [WANT_NOSEC="true";EXPLICIT_SEC="true"],) +AC_ARG_ENABLE([crash-handler], + [AS_HELP_STRING([--disable-crash-handler], + [Build the crash handler (default: enabled)])], + [enable_crash_handler=$enableval], + [enable_crash_handler=yes]) + +AS_IF([test "x$enable_crash_handler" != xno], + [AC_DEFINE([CRASH_HANDLER_ENABLED], [1], + [Define to 1 if the crash handler is built])]) +AM_CONDITIONAL([BLD_CRASH_HANDLER], [test "x$enable_crash_handler" != xno]) + +AC_ARG_ENABLE([crash-dedup], + [AS_HELP_STRING([--enable-crash-dedup], + [Enable --crash-dedup by default (default: disabled)])], + [enable_crash_dedup=$enableval], + [enable_crash_dedup=no]) + +AS_IF([test "x$enable_crash_dedup" = xyes && test "x$enable_crash_handler" = xno], + [AC_MSG_WARN([--enable-crash-dedup ignored because --disable-crash-handler was given])]) + +AS_IF([test "x$enable_crash_dedup" = xyes && test "x$enable_crash_handler" != xno], + [AC_DEFINE([CRASH_DEDUP_ENABLED_BY_DEFAULT], [1], + [Enable --crash-dedup by default])]) + AC_ARG_WITH(gcrypt-dir, [AS_HELP_STRING([--with-gcrypt-dir=DIR],[Look in DIR for gcrypt installation])], [GCRYPT_CFLAGS="-I${withval}/include";GCRYPT_LIBS="-L${withval}/lib"],) diff --git a/src/client/auditclient/Makefile.am b/src/client/auditclient/Makefile.am index 70dead4b..94305b76 100644 --- a/src/client/auditclient/Makefile.am +++ b/src/client/auditclient/Makefile.am @@ -12,7 +12,7 @@ endif AM_CFLAGS = -fvisibility=hidden -AM_CPPFLAGS = -I$(top_srcdir)/../logging -I$(top_srcdir)/../include -I$(top_srcdir)/client -I$(top_srcdir)/client_comlib +AM_CPPFLAGS = -I$(top_srcdir)/../logging -I$(top_srcdir)/../include -I$(top_srcdir)/client -I$(top_srcdir)/client_comlib -I$(top_srcdir)/crash_handler BASE_SRCS = auditclient.c auditclient_common.c patch_linkmap.c redirect.c bindgot.c writablegot.c patch_bad_dtv.c malloc_wrapper.c fixlocale.c if X86_64_BLD diff --git a/src/client/client/Makefile.am b/src/client/client/Makefile.am index 3a2a8749..6af55654 100644 --- a/src/client/client/Makefile.am +++ b/src/client/client/Makefile.am @@ -13,7 +13,7 @@ endif AM_CFLAGS = -fvisibility=hidden -AM_CPPFLAGS = -I$(top_srcdir)/../logging -I$(top_srcdir)/client_comlib -I$(top_srcdir)/../include -I$(top_srcdir)/shm_cache -I$(top_srcdir)/subaudit -I$(top_srcdir)/../utils -I$(top_srcdir)/auditclient +AM_CPPFLAGS = -I$(top_srcdir)/../logging -I$(top_srcdir)/client_comlib -I$(top_srcdir)/../include -I$(top_srcdir)/shm_cache -I$(top_srcdir)/subaudit -I$(top_srcdir)/../utils -I$(top_srcdir)/auditclient -I$(top_srcdir)/client -I$(top_srcdir)/crash_handler INTERCEPT_SRCS = intercept_open.c intercept_exec.c intercept_stat.c intercept_readlink.c intercept_spindleapi.c intercept.c spindle_regex.c patch_interception.c realpath.c $(top_srcdir)/../utils/fileutil.c diff --git a/src/client/subaudit/Makefile.am b/src/client/subaudit/Makefile.am index c17b7c6a..0d0cb4a1 100644 --- a/src/client/subaudit/Makefile.am +++ b/src/client/subaudit/Makefile.am @@ -12,7 +12,7 @@ endif AM_CFLAGS = -fvisibility=hidden -AM_CPPFLAGS = -I$(top_srcdir)/../logging -I$(top_srcdir)/../include -I$(top_srcdir)/client -I$(top_srcdir)/client_comlib -I$(top_srcdir)/auditclient -I$(top_srcdir)/../utils +AM_CPPFLAGS = -I$(top_srcdir)/../logging -I$(top_srcdir)/../include -I$(top_srcdir)/client -I$(top_srcdir)/client_comlib -I$(top_srcdir)/auditclient -I$(top_srcdir)/../utils -I$(top_srcdir)/crash_handler BASE_SRCS = subaudit.c intercept_malloc.c update_pltbind.c ../auditclient/auditclient_common.c ../auditclient/patch_linkmap.c ../auditclient/fixlocale.c From 1afb579a2ff7f5d27925ad79e2512ac9375ec3cb Mon Sep 17 00:00:00 2001 From: Nicholas Chaimov Date: Mon, 15 Jun 2026 14:58:41 -0700 Subject: [PATCH 06/14] Crash handler: testsuite Adds a new script run_crash_tests_template.sh which runs various configurations of the new crash_test.c testsuite. This tests various kinds of crashes, numberes of distinct crashsites, etc. to validate that the crash handler deduplicates them correctly. It also tests chaining to application signal handlers that use the safepoint pattern to fix faults and retry. --- testsuite/Makefile.am | 23 +- testsuite/crash_functions.c | 86 ++++ testsuite/crash_functions.h | 30 ++ testsuite/crash_test.c | 710 ++++++++++++++++++++++++++ testsuite/libcrashctor.c | 7 + testsuite/libcrashfuncs.c | 24 + testsuite/run_crash_tests_template.sh | 348 +++++++++++++ 7 files changed, 1227 insertions(+), 1 deletion(-) create mode 100644 testsuite/crash_functions.c create mode 100644 testsuite/crash_functions.h create mode 100644 testsuite/crash_test.c create mode 100644 testsuite/libcrashctor.c create mode 100644 testsuite/libcrashfuncs.c create mode 100755 testsuite/run_crash_tests_template.sh diff --git a/testsuite/Makefile.am b/testsuite/Makefile.am index fa0ea36a..57929a07 100644 --- a/testsuite/Makefile.am +++ b/testsuite/Makefile.am @@ -3,6 +3,10 @@ noinst_PROGRAMS = libgenerator ABS_TEST_DIR = $(abspath $(top_builddir)/testsuite) BUILT_SOURCES = libtest10.so libtest11.so libtest12.so libtest13.so libtest14.so libtest15.so libtest16.so libtest17.so libtest18.so libtest19.so libtest20.so libtest50.so libtest100.so libtest500.so libtest1000.so libtest2000.so libtest4000.so libtest6000.so libtest8000.so libtest10000.so libtls1.c libtls2.c libtls3.c libtls4.c libtls5.c libtls6.c libtls7.c libtls8.c libtls9.c libtls10.c libtls11.c libtls12.c libtls13.c libtls14.c libtls15.c libtls16.c libtls17.c libtls18.c libtls19.c libtls20.c libsymlink.so libdepC.so libdepB.so libdepA.so libcxxexceptB.so libcxxexceptA.so origin_dir/liboriginlib.so origin_dir/origin_subdir/liborigintarget.so libtestoutput.so libfuncdict.so runTests run_driver run_driver_rm spindle.rc preload_file_list test_driver test_driver_libs retzero_rx retzero_r retzero_x retzero_ badinterp hello_r.py hello_x.py hello_rx.py hello_.py hello_l.py badlink.py spindle_exec_test spindle_deactivated.sh liblocal.so symbind_test interpreter_test interpreter_test_dir/interpreter_test_perl alias/aliastest.py exec_shell.sh exec_shell.tcsh exec_shell_expected_output exec_shell exec_shell_env.sh +if BLD_CRASH_HANDLER +BUILT_SOURCES += crash_test libcrashfuncs.so libcrashctor.so run_crash_tests.sh +endif + if BGQ_BLD DYNAMIC_FLAG=-dynamic IS_BLUEGENE=true @@ -420,6 +424,23 @@ interpreter_test_dir/interpreter_test_perl: interpreter_test_perl_template inter interpreter_test: interpreter_test_template $(AM_V_GEN)cp $< $@; chmod 700 $@ +crash_test: $(srcdir)/crash_test.c $(srcdir)/crash_functions.c $(srcdir)/crash_functions.h libcrashfuncs.so libcrashctor.so + $(AM_V_CCLD)$(MPICC) $(CFLAGS) -O0 -fno-omit-frame-pointer -no-pie -Wl,-E \ + -o $@ $(srcdir)/crash_test.c $(srcdir)/crash_functions.c -I$(srcdir) -ldl -lpthread + +libcrashfuncs.so: $(srcdir)/libcrashfuncs.c + $(AM_V_CCLD)$(CC) $(CFLAGS) -O0 -fno-omit-frame-pointer -shared -fPIC \ + -o $@ $(srcdir)/libcrashfuncs.c + +libcrashctor.so: $(srcdir)/libcrashctor.c + $(AM_V_CCLD)$(CC) $(CFLAGS) -O0 -fno-omit-frame-pointer -shared -fPIC \ + -o $@ $(srcdir)/libcrashctor.c + +run_crash_tests.sh: $(srcdir)/run_crash_tests_template.sh $(top_builddir)/Makefile + $(AM_V_GEN)$(SED) -e s,TEST_RESOURCE_MANAGER,$(TESTRM),g\;s,SPINDLE_EXEC,$(bindir)/spindle,g\;s,SPINDLE_RC_PATH,$(sysconfdir)/spindle/spindle.rc,g\;s,TEST_RUN_DIR,$(ABS_TEST_DIR),g < $< > $@; chmod 700 $@ + +EXTRA_DIST = run_crash_tests_template.sh crash_test.c crash_functions.c crash_functions.h libcrashfuncs.c libcrashctor.c + exec_shell.sh: $(srcdir)/exec_shell_template.sh $(AM_V_GEN)cp $< $@; chmod 700 $@ @@ -435,4 +456,4 @@ exec_shell_expected_output: $(srcdir)/exec_shell_expected_output_template exec_shell: $(srcdir)/exec_shell.c $(AM_V_CCLD)$(CC) $(CFLAGS) -o $@ $< -CLEANFILES = libtest10.c libtest11.c libtest12.c libtest13.c libtest14.c libtest15.c libtest16.c libtest17.c libtest18.c libtest19.c libtest20.c libtest10.so libtest50.c libtest50.so libtest100.c libtest100.so libtest500.c libtest500.so libtest1000.c libtest1000.so libtest2000.c libtest2000.so libtest4000.c libtest4000.so libtest6000.c libtest6000.so libtest8000.c libtest8000.so libtest10000.c libtest10000.so libsymlink.so libdepA.so libdepB.so libdepC.so libcxxexceptA.so libcxxexceptB.so libtestoutput.so libfuncdict.so runTests run_driver run_driver_rm spindle.rc test_driver test_driver_libs preload_file_list retzero_rx retzero_r retzero_x retzero_ badinterp hello_r.py hello_x.py hello_rx.py hello_.py hello_l.py badlink.py libtls1.c libtls2.c libtls3.c libtls4.c libtls5.c libtls6.c libtls7.c libtls8.c libtls9.c libtls10.c libtls11.c libtls12.c libtls13.c libtls14.c libtls15.c libtls16.c libtls17.c libtls18.c libtls19.c libtls20.c libtls1.so libtls2.so libtls3.so libtls4.so libtls5.so libtls6.so libtls7.so libtls8.so libtls9.so libtls10.so libtls11.so libtls12.so libtls13.so libtls14.so libtls15.so libtls16.so libtls17.so libtls18.so libtls19.so libtls20.so symbind_test libsymbind_a.so libsymbind_b.so libsymbind_c.so libsymbind_d.so libsymbind_e.so libsymbind_f.so libsymbind_g.so interpreter_test interpreter_test_dir/interpreter_test_perl alias exec_shell.sh exec_shell.tcsh exec_shell_env.sh exec_shell_expected_output exec_shell +CLEANFILES = libtest10.c libtest11.c libtest12.c libtest13.c libtest14.c libtest15.c libtest16.c libtest17.c libtest18.c libtest19.c libtest20.c libtest10.so libtest50.c libtest50.so libtest100.c libtest100.so libtest500.c libtest500.so libtest1000.c libtest1000.so libtest2000.c libtest2000.so libtest4000.c libtest4000.so libtest6000.c libtest6000.so libtest8000.c libtest8000.so libtest10000.c libtest10000.so libsymlink.so libdepA.so libdepB.so libdepC.so libcxxexceptA.so libcxxexceptB.so libtestoutput.so libfuncdict.so runTests run_driver run_driver_rm spindle.rc test_driver test_driver_libs preload_file_list retzero_rx retzero_r retzero_x retzero_ badinterp hello_r.py hello_x.py hello_rx.py hello_.py hello_l.py badlink.py libtls1.c libtls2.c libtls3.c libtls4.c libtls5.c libtls6.c libtls7.c libtls8.c libtls9.c libtls10.c libtls11.c libtls12.c libtls13.c libtls14.c libtls15.c libtls16.c libtls17.c libtls18.c libtls19.c libtls20.c libtls1.so libtls2.so libtls3.so libtls4.so libtls5.so libtls6.so libtls7.so libtls8.so libtls9.so libtls10.so libtls11.so libtls12.so libtls13.so libtls14.so libtls15.so libtls16.so libtls17.so libtls18.so libtls19.so libtls20.so symbind_test libsymbind_a.so libsymbind_b.so libsymbind_c.so libsymbind_d.so libsymbind_e.so libsymbind_f.so libsymbind_g.so interpreter_test interpreter_test_dir/interpreter_test_perl alias exec_shell.sh exec_shell.tcsh exec_shell_env.sh exec_shell_expected_output exec_shell crash_test libcrashfuncs.so libcrashctor.so run_crash_tests.sh diff --git a/testsuite/crash_functions.c b/testsuite/crash_functions.c new file mode 100644 index 00000000..9a9aa4f8 --- /dev/null +++ b/testsuite/crash_functions.c @@ -0,0 +1,86 @@ +/* +This file is part of Spindle. For copyright information see the COPYRIGHT +file in the top level directory, or at +https://github.com/hpc/Spindle/blob/master/COPYRIGHT + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU Lesser General Public License (as published by the Free Software +Foundation) version 2.1 dated February 1999. This program is distributed in the +hope that it will be useful, but WITHOUT ANY WARRANTY; without even the IMPLIED +WARRANTY OF MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms +and conditions of the GNU Lesser General Public License for more details. You should +have received a copy of the GNU Lesser General Public License along with this +program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA +*/ + +#include "crash_functions.h" + +__attribute__((noinline, visibility("default"))) +void crash_function_A(int rank) { + volatile int *p = (int *) 0; + (void) rank; + *p = 0; +} + +__attribute__((noinline, visibility("default"))) +void crash_function_B(int rank) { + volatile int *p = (int *) 0; + (void) rank; + *p = 0; +} + +#define DEFINE_CRASH_N(n) \ + __attribute__((noinline, visibility("default"))) \ + void crash_function_##n(int rank) { \ + volatile int *p = (int *) 0; \ + (void) rank; \ + *p = 0; \ + } + +DEFINE_CRASH_N(0) DEFINE_CRASH_N(1) DEFINE_CRASH_N(2) DEFINE_CRASH_N(3) +DEFINE_CRASH_N(4) DEFINE_CRASH_N(5) DEFINE_CRASH_N(6) DEFINE_CRASH_N(7) +DEFINE_CRASH_N(8) DEFINE_CRASH_N(9) DEFINE_CRASH_N(10) DEFINE_CRASH_N(11) +DEFINE_CRASH_N(12) DEFINE_CRASH_N(13) DEFINE_CRASH_N(14) DEFINE_CRASH_N(15) +DEFINE_CRASH_N(16) DEFINE_CRASH_N(17) DEFINE_CRASH_N(18) DEFINE_CRASH_N(19) +DEFINE_CRASH_N(20) DEFINE_CRASH_N(21) DEFINE_CRASH_N(22) DEFINE_CRASH_N(23) +DEFINE_CRASH_N(24) DEFINE_CRASH_N(25) DEFINE_CRASH_N(26) DEFINE_CRASH_N(27) +DEFINE_CRASH_N(28) DEFINE_CRASH_N(29) DEFINE_CRASH_N(30) DEFINE_CRASH_N(31) +DEFINE_CRASH_N(32) DEFINE_CRASH_N(33) DEFINE_CRASH_N(34) DEFINE_CRASH_N(35) +DEFINE_CRASH_N(36) DEFINE_CRASH_N(37) DEFINE_CRASH_N(38) DEFINE_CRASH_N(39) +DEFINE_CRASH_N(40) DEFINE_CRASH_N(41) DEFINE_CRASH_N(42) DEFINE_CRASH_N(43) +DEFINE_CRASH_N(44) DEFINE_CRASH_N(45) DEFINE_CRASH_N(46) DEFINE_CRASH_N(47) +DEFINE_CRASH_N(48) DEFINE_CRASH_N(49) DEFINE_CRASH_N(50) DEFINE_CRASH_N(51) +DEFINE_CRASH_N(52) DEFINE_CRASH_N(53) DEFINE_CRASH_N(54) DEFINE_CRASH_N(55) +DEFINE_CRASH_N(56) DEFINE_CRASH_N(57) DEFINE_CRASH_N(58) DEFINE_CRASH_N(59) +DEFINE_CRASH_N(60) DEFINE_CRASH_N(61) DEFINE_CRASH_N(62) DEFINE_CRASH_N(63) + +crash_fn_t crash_table[CRASH_TABLE_SIZE] = { + crash_function_0, crash_function_1, crash_function_2, crash_function_3, + crash_function_4, crash_function_5, crash_function_6, crash_function_7, + crash_function_8, crash_function_9, crash_function_10, crash_function_11, + crash_function_12, crash_function_13, crash_function_14, crash_function_15, + crash_function_16, crash_function_17, crash_function_18, crash_function_19, + crash_function_20, crash_function_21, crash_function_22, crash_function_23, + crash_function_24, crash_function_25, crash_function_26, crash_function_27, + crash_function_28, crash_function_29, crash_function_30, crash_function_31, + crash_function_32, crash_function_33, crash_function_34, crash_function_35, + crash_function_36, crash_function_37, crash_function_38, crash_function_39, + crash_function_40, crash_function_41, crash_function_42, crash_function_43, + crash_function_44, crash_function_45, crash_function_46, crash_function_47, + crash_function_48, crash_function_49, crash_function_50, crash_function_51, + crash_function_52, crash_function_53, crash_function_54, crash_function_55, + crash_function_56, crash_function_57, crash_function_58, crash_function_59, + crash_function_60, crash_function_61, crash_function_62, crash_function_63, +}; + +static __attribute__((noinline)) +void static_crash_inner(int rank) { + volatile int *p = (int *) 0; + (void) rank; + *p = 0; +} + +void static_crash(int rank) { + static_crash_inner(rank); +} diff --git a/testsuite/crash_functions.h b/testsuite/crash_functions.h new file mode 100644 index 00000000..34f0a770 --- /dev/null +++ b/testsuite/crash_functions.h @@ -0,0 +1,30 @@ +/* +This file is part of Spindle. For copyright information see the COPYRIGHT +file in the top level directory, or at +https://github.com/hpc/Spindle/blob/master/COPYRIGHT + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU Lesser General Public License (as published by the Free Software +Foundation) version 2.1 dated February 1999. This program is distributed in the +hope that it will be useful, but WITHOUT ANY WARRANTY; without even the IMPLIED +WARRANTY OF MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms +and conditions of the GNU Lesser General Public License for more details. You should +have received a copy of the GNU Lesser General Public License along with this +program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA +*/ + +#ifndef CRASH_FUNCTIONS_H +#define CRASH_FUNCTIONS_H + +void crash_function_A(int rank); +void crash_function_B(int rank); + +#define CRASH_TABLE_SIZE 64 + +typedef void (*crash_fn_t)(int); +extern crash_fn_t crash_table[CRASH_TABLE_SIZE]; + +void static_crash(int rank); + +#endif diff --git a/testsuite/crash_test.c b/testsuite/crash_test.c new file mode 100644 index 00000000..f49a9f0b --- /dev/null +++ b/testsuite/crash_test.c @@ -0,0 +1,710 @@ +/* +This file is part of Spindle. For copyright information see the COPYRIGHT +file in the top level directory, or at +https://github.com/hpc/Spindle/blob/master/COPYRIGHT + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU Lesser General Public License (as published by the Free Software +Foundation) version 2.1 dated February 1999. This program is distributed in the +hope that it will be useful, but WITHOUT ANY WARRANTY; without even the IMPLIED +WARRANTY OF MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms +and conditions of the GNU Lesser General Public License for more details. You should +have received a copy of the GNU Lesser General Public License along with this +program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA +*/ + +/* + * Crash-handler test driver. + * * + * SIGSEGV modes: + * all-same every rank crashes in crash_function_A + * all-different rank N crashes in crash_function_, up to 64 ranks + * partial even ranks crash in crash_function_A, + * odd ranks exit cleanly + * late-straggler rank 0 crashes immediately; others sleep --sleep + * seconds before crashing + * two-groups even ranks call crash_function_A, odd ranks call crash_function_B + * one-crashes rank 0 crashes; others exit cleanly + * in-library every rank dlopen()s libcrashfuncs.so and calls + * crash_in_library() + * in-dlmopen-library + * every rank dlmopen()s libcrashfuncs.so and calls crash_in_library() + * in-library-ctor rank 0 dlopen()s libcrashctor.so, whose constructor + * crashes inside dlopen; others exit cleanly + * span-read no app handler; a read spans two pages (PROT_READ then + * PROT_NONE) and faults in the second page + * + * SIGABRT modes: + * sigabrt every rank calls abort() + * assert every rank issues a failing assert() + * mixed-abort-segv + * even ranks assert, odd ranks segfault + * + * Handler chaining modes + * safepoint app handler fixes segfault on read + * safepoint-then-crash + * app handler fixes segfault on read repeatedly, + * then a real crash occurs on same thread. + * safepoint-bad application handler returns but fails to fix read fault + * safepoint-bad-write + * application handler returns but fails to fix write fault + * safepoint-fix-write + * application handler fixes segfault on write + * safepoint-longjmp + application handler siglongjmp()s out of the handler + * safepoint-span-read + * app handler fixes a read fault whose access spans two pages + * (starts in an accessible page, faults in the next one) + * safepoint-span-write + * app handler fixes a write fault whose access spans two pages + * safepoint-span-bad-read + * app handler returns without fixing a page-spanning read fault + * safepoint-span-bad-write + * app handler returns without fixing a page-spanning write fault + * + * no-crash every rank exits cleanly + */ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "crash_functions.h" + +#define SAFEPOINT_RC_INCOMPLETE 103 +#define SAFEPOINT_RC_SETUP_FAILED 104 +#define SAFEPOINT_RC_NOT_TERMINATED 105 + +static void usage(const char *prog) { + fprintf(stderr, + "usage: %s --crash-mode {all-same|all-different|partial|" + "late-straggler|two-groups|one-crashes|in-library|" + "in-dlmopen-library|in-library-ctor|sigabrt|assert|" + "long-assert|mixed-abort-segv|span-read|" + "safepoint|safepoint-then-crash|" + "safepoint-bad|safepoint-bad-write|safepoint-fix-write|" + "safepoint-span-read|safepoint-span-write|" + "safepoint-span-bad-read|safepoint-span-bad-write|" + "safepoint-longjmp|safepoint-longjmp-mt|safepoint-concurrent-chain|no-crash}" + " [--sleep ] [--cycles ]\n", + prog); +} + +static inline int addr_in_range(uintptr_t a, void *base, size_t len) { + uintptr_t lo = (uintptr_t) base; + return a >= lo && a < lo + len; +} + +static void *map_trap_page(const char *mode, int rank, long pagesize) { + void *p = mmap(NULL, pagesize, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + fprintf(stderr, "%s rank=%d: mmap failed\n", mode, rank); + return NULL; + } + return p; +} + +static int install_sigsegv_handler(void (*handler)(int, siginfo_t *, void *), + const char *mode, int rank) { + struct sigaction sa; + memset(&sa, 0, sizeof sa); + sa.sa_sigaction = handler; + sa.sa_flags = SA_SIGINFO; + sigemptyset(&sa.sa_mask); + if (sigaction(SIGSEGV, &sa, NULL) != 0) { + fprintf(stderr, "%s rank=%d: sigaction failed\n", mode, rank); + return -1; + } + return 0; +} + +static void handler_die(const char *msg) { + ssize_t w = write(2, msg, strlen(msg)); + (void) w; + _exit(SAFEPOINT_RC_SETUP_FAILED); +} + +static void call_crash_in_library_common(int rank, int use_dlmopen) { + const char *dlopen_name = use_dlmopen ? "dlmopen" : "dlopen"; + void *h = use_dlmopen ? dlmopen(LM_ID_NEWLM, "./libcrashfuncs.so", RTLD_NOW) + : dlopen("./libcrashfuncs.so", RTLD_NOW); + if (!h) + h = use_dlmopen ? dlmopen(LM_ID_NEWLM, "libcrashfuncs.so", RTLD_NOW) + : dlopen("libcrashfuncs.so", RTLD_NOW); + if (!h) { + fprintf(stderr, "rank=%d %s of libcrashfuncs.so failed: %s\n", + rank, dlopen_name, dlerror()); + MPI_Abort(MPI_COMM_WORLD, 3); + return; + } + void (*fn)(int) = (void (*)(int)) dlsym(h, "crash_in_library"); + if (!fn) { + fprintf(stderr, "rank=%d dlsym of crash_in_library failed: %s\n", + rank, dlerror()); + MPI_Abort(MPI_COMM_WORLD, 3); + return; + } + fn(rank); +} + +static void crash_in_library_ctor(int rank) { + if (rank != 0) { + fprintf(stderr, "rank=%d exiting cleanly\n", rank); + fflush(stderr); + MPI_Finalize(); + return; + } + + void *handle = dlopen("./libcrashctor.so", RTLD_NOW); + if (!handle) + handle = dlopen("libcrashctor.so", RTLD_NOW); + + (void) handle; + exit(EXIT_FAILURE); +} + +static void do_assert(int rank) { + assert(rank < 0 && "Assertion message"); +} + +static void do_mixed_abort_segv(int rank) { + if ((rank % 2) == 0) { + assert(0 && "Assertion message"); + } else { + *(volatile int *) 0 = 0; + } +} + +static void do_span_read(int rank) { + long pagesize = sysconf(_SC_PAGESIZE); + unsigned char *region = mmap(NULL, 2 * pagesize, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (region == MAP_FAILED) { + fprintf(stderr, "span-read rank=%d: mmap failed\n", rank); + _exit(SAFEPOINT_RC_SETUP_FAILED); + } + memset(region, 0xAB, 2 * pagesize); + if (mprotect(region, pagesize, PROT_READ) != 0 || + mprotect(region + pagesize, pagesize, PROT_NONE) != 0) { + fprintf(stderr, "span-read rank=%d: mprotect failed\n", rank); + _exit(SAFEPOINT_RC_SETUP_FAILED); + } + + volatile uint64_t v = *(volatile uint64_t *) (region + pagesize - 4); + (void) v; + + fprintf(stderr, "span-read rank=%d: unexpectedly continued after SIGSEGV\n", rank); + _exit(SAFEPOINT_RC_NOT_TERMINATED); +} + +/* Safepoint tests. */ + +#define SAFEPOINT_CYCLES_DEFAULT 100 + +static void *safepoint_page; +static long safepoint_pagesize; +static volatile int safepoint_handler_invocations; + +static void safepoint_nofix_handler(int sig, siginfo_t *info, void *uctx) { + (void) sig; (void) info; (void) uctx; + /* App handler that doesn't fix the problem. */ + safepoint_handler_invocations++; +} + +static int safepoint_result(const char *mode, int rank, int cycles) { + if (safepoint_handler_invocations == 0) { + fprintf(stderr, "%s rank=%d: handler never invoked\n", mode, rank); + return SAFEPOINT_RC_INCOMPLETE; + } + fprintf(stderr, + "%s rank=%d: completed %d safepoint cycles, recorded %d invocations\n", + mode, rank, cycles, safepoint_handler_invocations); + return 0; +} + +static int setup_safepoint(const char *mode, int rank, + void (*handler)(int, siginfo_t *, void *)) { + safepoint_pagesize = sysconf(_SC_PAGESIZE); + safepoint_page = map_trap_page(mode, rank, safepoint_pagesize); + if (safepoint_page == NULL) + return SAFEPOINT_RC_SETUP_FAILED; + if (install_sigsegv_handler(handler, mode, rank) != 0) + return SAFEPOINT_RC_SETUP_FAILED; + return 0; +} + +static void safepoint_sigsegv_handler(int sig, siginfo_t *info, void *uctx) { + (void) sig; (void) info; (void) uctx; + safepoint_handler_invocations++; + + /* Unprotect the page so the read succeeds on retry. */ + if (mprotect(safepoint_page, safepoint_pagesize, PROT_READ) != 0) + handler_die("mprotect failed in safepoint_sigsegv_handler\n"); +} + +static int do_safepoint(int rank, int cycles) { + int rc = setup_safepoint("safepoint", rank, safepoint_sigsegv_handler); + if (rc) + return rc; + + for (int i = 0; i < cycles; i++) { + if (mprotect(safepoint_page, safepoint_pagesize, PROT_NONE) != 0) { + fprintf(stderr, "safepoint rank=%d cycle=%d: mprotect PROT_NONE failed\n", + rank, i); + return SAFEPOINT_RC_SETUP_FAILED; + } + volatile int v = *(volatile int *) safepoint_page; + (void) v; + } + + return safepoint_result("safepoint", rank, cycles); +} + +#define SAFEPOINT_THEN_CRASH_PRECYCLES 5 + +static volatile int stc_safepoint_fixes; +static volatile int stc_realcrash_observed; + +static void safepoint_then_crash_handler(int sig, siginfo_t *info, void *uctx) { + (void) sig; (void) uctx; + + int is_safepoint = 0; + if (info != NULL && info->si_addr != NULL && + addr_in_range((uintptr_t) info->si_addr, safepoint_page, safepoint_pagesize)) + is_safepoint = 1; + + if (is_safepoint) { + stc_safepoint_fixes++; + if (mprotect(safepoint_page, safepoint_pagesize, PROT_READ) != 0) { + _exit(SAFEPOINT_RC_SETUP_FAILED); + } + return; + } + + stc_realcrash_observed = 1; + /* This wasn't in the safepoint page, so restore default disposition and return. */ + signal(SIGSEGV, SIG_DFL); +} + +static int do_safepoint_then_crash(int rank) { + int rc = setup_safepoint("safepoint-then-crash", rank, + safepoint_then_crash_handler); + if (rc) + return rc; + + for (int i = 0; i < SAFEPOINT_THEN_CRASH_PRECYCLES; i++) { + if (mprotect(safepoint_page, safepoint_pagesize, PROT_NONE) != 0) { + fprintf(stderr, + "safepoint-then-crash rank=%d cycle=%d: mprotect PROT_NONE failed\n", + rank, i); + return SAFEPOINT_RC_SETUP_FAILED; + } + volatile int v = *(volatile int *) safepoint_page; + (void) v; + } + fprintf(stderr, + "safepoint-then-crash rank=%d: completed %d safepoint cycles, now triggering real crash\n", + rank, stc_safepoint_fixes); + fflush(stderr); + + crash_function_A(rank); + + fprintf(stderr, "safepoint-then-crash rank=%d: crash_function_A unexpectedly returned\n", rank); + return SAFEPOINT_RC_NOT_TERMINATED; +} + +static int do_safepoint_bad(int rank) { + int rc = setup_safepoint("safepoint-bad", rank, safepoint_nofix_handler); + if (rc) + return rc; + + if (mprotect(safepoint_page, safepoint_pagesize, PROT_NONE) != 0) { + fprintf(stderr, + "safepoint-bad rank=%d: mprotect PROT_NONE failed\n", rank); + return SAFEPOINT_RC_SETUP_FAILED; + } + + volatile int v = *(volatile int *) safepoint_page; + (void) v; + + fprintf(stderr, + "safepoint-bad rank=%d: unexpectedly continued after unrecovered SIGSEGV\n", + rank); + return SAFEPOINT_RC_NOT_TERMINATED; +} + +static int do_safepoint_bad_write(int rank) { + int rc = setup_safepoint("safepoint-bad-write", rank, safepoint_nofix_handler); + if (rc) + return rc; + + if (mprotect(safepoint_page, safepoint_pagesize, PROT_READ) != 0) { + fprintf(stderr, + "safepoint-bad-write rank=%d: mprotect PROT_READ failed\n", + rank); + return SAFEPOINT_RC_SETUP_FAILED; + } + + *(volatile int *) safepoint_page = 0; + + fprintf(stderr, + "safepoint-bad-write rank=%d: unexpectedly continued after unrecovered SIGSEGV\n", + rank); + return SAFEPOINT_RC_NOT_TERMINATED; +} + +static void safepoint_fix_write_handler(int sig, siginfo_t *info, void *uctx) { + (void) sig; (void) info; (void) uctx; + + safepoint_handler_invocations++; + + if (mprotect(safepoint_page, safepoint_pagesize, + PROT_READ | PROT_WRITE) != 0) + handler_die("safepoint-fix-write: mprotect failed in handler\n"); +} + +static int do_safepoint_fix_write(int rank, int cycles) { + int rc = setup_safepoint("safepoint-fix-write", rank, + safepoint_fix_write_handler); + if (rc) return rc; + + for (int i = 0; i < cycles; i++) { + if (mprotect(safepoint_page, safepoint_pagesize, PROT_READ) != 0) { + fprintf(stderr, + "safepoint-fix-write rank=%d: mprotect PROT_READ failed at cycle %d\n", + rank, i); + return SAFEPOINT_RC_SETUP_FAILED; + } + *(volatile int *) safepoint_page = i; + } + + return safepoint_result("safepoint-fix-write", rank, cycles); +} + +static sigjmp_buf longjmp_env; +static volatile int longjmp_cycles_completed; + +static void safepoint_longjmp_handler(int sig, siginfo_t *info, void *uctx) { + (void) sig; (void) info; (void) uctx; + if (mprotect(safepoint_page, safepoint_pagesize, PROT_READ) != 0) + handler_die("safepoint-longjmp: mprotect failed in handler\n"); + longjmp_cycles_completed++; + siglongjmp(longjmp_env, 1); +} + +static int do_safepoint_longjmp(int rank, int cycles) { + int rc = setup_safepoint("safepoint-longjmp", rank, + safepoint_longjmp_handler); + if (rc) + return rc; + + for (int i = 0; i < cycles; i++) { + if (sigsetjmp(longjmp_env, 1) == 0) { + if (mprotect(safepoint_page, safepoint_pagesize, + PROT_NONE) != 0) { + fprintf(stderr, + "safepoint-longjmp rank=%d cycle=%d: mprotect PROT_NONE failed\n", rank, i); + return SAFEPOINT_RC_SETUP_FAILED; + } + volatile int v = *(volatile int *) safepoint_page; + (void) v; + fprintf(stderr, + "safepoint-longjmp rank=%d: unexpectedly continued past siglongjmp\n", rank); + return SAFEPOINT_RC_NOT_TERMINATED; + } + } + + if (longjmp_cycles_completed != cycles) { + fprintf(stderr, + "safepoint-longjmp rank=%d: completed %d of %d cycles\n", + rank, longjmp_cycles_completed, cycles); + return SAFEPOINT_RC_INCOMPLETE; + } + fprintf(stderr, + "safepoint-longjmp rank=%d: completed %d siglongjmp cycles\n", + rank, cycles); + return 0; +} + +static unsigned char *span_region; +static long span_pagesize; + +static unsigned char *span_page2(void) { return span_region + span_pagesize; } + +static volatile uint64_t *span_access(void) { + return (volatile uint64_t *) (span_page2() - 4); +} + +static int setup_span(const char *mode, int rank, + void (*handler)(int, siginfo_t *, void *)) { + span_pagesize = sysconf(_SC_PAGESIZE); + span_region = mmap(NULL, 2 * span_pagesize, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (span_region == MAP_FAILED) { + fprintf(stderr, "%s rank=%d: mmap failed\n", mode, rank); + return SAFEPOINT_RC_SETUP_FAILED; + } + memset(span_region, 0xAB, 2 * span_pagesize); + if (install_sigsegv_handler(handler, mode, rank) != 0) + return SAFEPOINT_RC_SETUP_FAILED; + return 0; +} + +static void span_read_fix_handler(int sig, siginfo_t *info, void *uctx) { + (void) sig; (void) info; (void) uctx; + safepoint_handler_invocations++; + if (mprotect(span_page2(), span_pagesize, PROT_READ) != 0) + handler_die("safepoint-span-read: mprotect failed in handler\n"); +} + +static void span_write_fix_handler(int sig, siginfo_t *info, void *uctx) { + (void) sig; (void) info; (void) uctx; + safepoint_handler_invocations++; + if (mprotect(span_page2(), span_pagesize, PROT_READ | PROT_WRITE) != 0) + handler_die("safepoint-span-write: mprotect failed in handler\n"); +} + +static int do_safepoint_span_read(int rank, int cycles) { + int rc = setup_span("safepoint-span-read", rank, span_read_fix_handler); + if (rc) + return rc; + + for (int i = 0; i < cycles; i++) { + if (mprotect(span_page2(), span_pagesize, PROT_NONE) != 0) { + fprintf(stderr, + "safepoint-span-read rank=%d cycle=%d: mprotect PROT_NONE failed\n", + rank, i); + return SAFEPOINT_RC_SETUP_FAILED; + } + volatile uint64_t v = *span_access(); + (void) v; + } + + return safepoint_result("safepoint-span-read", rank, cycles); +} + +static int do_safepoint_span_write(int rank, int cycles) { + int rc = setup_span("safepoint-span-write", rank, span_write_fix_handler); + if (rc) + return rc; + + for (int i = 0; i < cycles; i++) { + if (mprotect(span_page2(), span_pagesize, PROT_NONE) != 0) { + fprintf(stderr, + "safepoint-span-write rank=%d cycle=%d: mprotect PROT_NONE failed\n", + rank, i); + return SAFEPOINT_RC_SETUP_FAILED; + } + *span_access() = (uint64_t) i; + } + + return safepoint_result("safepoint-span-write", rank, cycles); +} + +static int do_safepoint_span_bad_read(int rank) { + int rc = setup_span("safepoint-span-bad-read", rank, safepoint_nofix_handler); + if (rc) + return rc; + + if (mprotect(span_page2(), span_pagesize, PROT_NONE) != 0) { + fprintf(stderr, + "safepoint-span-bad-read rank=%d: mprotect PROT_NONE failed\n", rank); + return SAFEPOINT_RC_SETUP_FAILED; + } + + volatile uint64_t v = *span_access(); + (void) v; + + fprintf(stderr, + "safepoint-span-bad-read rank=%d: unexpectedly continued after unrecovered SIGSEGV\n", + rank); + return SAFEPOINT_RC_NOT_TERMINATED; +} + +static int do_safepoint_span_bad_write(int rank) { + int rc = setup_span("safepoint-span-bad-write", rank, safepoint_nofix_handler); + if (rc) + return rc; + + if (mprotect(span_page2(), span_pagesize, PROT_NONE) != 0) { + fprintf(stderr, + "safepoint-span-bad-write rank=%d: mprotect PROT_NONE failed\n", rank); + return SAFEPOINT_RC_SETUP_FAILED; + } + + *span_access() = 0; + + fprintf(stderr, + "safepoint-span-bad-write rank=%d: unexpectedly continued after unrecovered SIGSEGV\n", + rank); + return SAFEPOINT_RC_NOT_TERMINATED; +} + +static int sleep_seconds = 10; +static int safepoint_cycles = SAFEPOINT_CYCLES_DEFAULT; + +int main(int argc, char **argv) { + const char *mode = NULL; + for (int i = 1; i < argc; i++) { + if (strcmp(argv[i], "--crash-mode") == 0 && i + 1 < argc) { + mode = argv[++i]; + } else if (strncmp(argv[i], "--crash-mode=", 13) == 0) { + mode = argv[i] + 13; + } else if (strcmp(argv[i], "--sleep") == 0 && i + 1 < argc) { + int v = atoi(argv[++i]); + if (v >= 1) sleep_seconds = v; + } else if (strncmp(argv[i], "--sleep=", 8) == 0) { + int v = atoi(argv[i] + 8); + if (v >= 1) sleep_seconds = v; + } else if (strcmp(argv[i], "--cycles") == 0 && i + 1 < argc) { + int v = atoi(argv[++i]); + if (v > 0) safepoint_cycles = v; + } else if (strncmp(argv[i], "--cycles=", 9) == 0) { + int v = atoi(argv[i] + 9); + if (v > 0) safepoint_cycles = v; + } else if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) { + usage(argv[0]); + return 0; + } + } + if (!mode) { + usage(argv[0]); + return 2; + } + + MPI_Init(&argc, &argv); + int rank = 0, size = 1; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &size); + + fprintf(stderr, "crash_test rank=%d size=%d mode=%s pid=%d\n", + rank, size, mode, (int) getpid()); + fflush(stderr); + + + /* create and chdir into per-rank directory so that a given rank's + coredumps go into that directory and we can count the coredumps + per rank. */ + { + char rankdir[64]; + snprintf(rankdir, sizeof rankdir, "rank_%d", rank); + (void) mkdir(rankdir, 0755); + if (chdir(rankdir) != 0) { + fprintf(stderr, "rank=%d chdir(%s) failed\n", rank, rankdir); + } + } + + MPI_Barrier(MPI_COMM_WORLD); + + if (strcmp(mode, "all-same") == 0) { + crash_function_A(rank); + } else if (strcmp(mode, "all-different") == 0) { + int idx = rank % CRASH_TABLE_SIZE; + crash_table[idx](rank); + } else if (strcmp(mode, "partial") == 0) { + if ((rank % 2) == 0) { + crash_function_A(rank); + } else { + fprintf(stderr, "rank=%d exiting cleanly\n", rank); + fflush(stderr); + MPI_Finalize(); + return 0; + } + } else if (strcmp(mode, "late-straggler") == 0) { + int sleep_s = sleep_seconds; + if (rank == 0) { + crash_function_A(rank); + } else { + fprintf(stderr, "rank=%d sleeping %ds before crash\n", rank, sleep_s); + fflush(stderr); + sleep(sleep_s); + crash_function_A(rank); + } + } else if (strcmp(mode, "two-groups") == 0) { + if ((rank % 2) == 0) + crash_function_A(rank); + else + crash_function_B(rank); + } else if (strcmp(mode, "one-crashes") == 0) { + if (rank == 0) { + crash_function_A(rank); + } else { + fprintf(stderr, "rank=%d exiting cleanly\n", rank); + fflush(stderr); + MPI_Finalize(); + return 0; + } + } else if (strcmp(mode, "in-library") == 0) { + call_crash_in_library_common(rank, 0); + } else if (strcmp(mode, "in-dlmopen-library") == 0) { + call_crash_in_library_common(rank, 1); + } else if (strcmp(mode, "in-library-ctor") == 0) { + crash_in_library_ctor(rank); + } else if (strcmp(mode, "sigabrt") == 0) { + abort(); + } else if (strcmp(mode, "assert") == 0) { + do_assert(rank); + } else if (strcmp(mode, "mixed-abort-segv") == 0) { + do_mixed_abort_segv(rank); + } else if (strcmp(mode, "span-read") == 0) { + do_span_read(rank); + } else if (strcmp(mode, "safepoint") == 0) { + int rc = do_safepoint(rank, safepoint_cycles); + MPI_Finalize(); + return rc; + } else if (strcmp(mode, "safepoint-then-crash") == 0) { + return do_safepoint_then_crash(rank); + } else if (strcmp(mode, "safepoint-bad") == 0) { + return do_safepoint_bad(rank); + } else if (strcmp(mode, "safepoint-bad-write") == 0) { + return do_safepoint_bad_write(rank); + } else if (strcmp(mode, "safepoint-fix-write") == 0) { + int rc = do_safepoint_fix_write(rank, safepoint_cycles); + MPI_Finalize(); + return rc; + } else if (strcmp(mode, "safepoint-longjmp") == 0) { + int rc = do_safepoint_longjmp(rank, safepoint_cycles); + MPI_Finalize(); + return rc; + } else if (strcmp(mode, "safepoint-span-read") == 0) { + int rc = do_safepoint_span_read(rank, safepoint_cycles); + MPI_Finalize(); + return rc; + } else if (strcmp(mode, "safepoint-span-write") == 0) { + int rc = do_safepoint_span_write(rank, safepoint_cycles); + MPI_Finalize(); + return rc; + } else if (strcmp(mode, "safepoint-span-bad-read") == 0) { + return do_safepoint_span_bad_read(rank); + } else if (strcmp(mode, "safepoint-span-bad-write") == 0) { + return do_safepoint_span_bad_write(rank); + } else if (strcmp(mode, "no-crash") == 0) { + fprintf(stderr, "rank=%d no-crash, exiting cleanly\n", rank); + fflush(stderr); + MPI_Finalize(); + return 0; + } else { + if (rank == 0) usage(argv[0]); + MPI_Finalize(); + return 2; + } + + MPI_Finalize(); + return 0; +} diff --git a/testsuite/libcrashctor.c b/testsuite/libcrashctor.c new file mode 100644 index 00000000..877d3ce4 --- /dev/null +++ b/testsuite/libcrashctor.c @@ -0,0 +1,7 @@ +#include + +__attribute__((constructor)) +static void ctor_crash(void) +{ + *(volatile int *) 0 = 0; +} diff --git a/testsuite/libcrashfuncs.c b/testsuite/libcrashfuncs.c new file mode 100644 index 00000000..0cd0b1ba --- /dev/null +++ b/testsuite/libcrashfuncs.c @@ -0,0 +1,24 @@ +/* +This file is part of Spindle. For copyright information see the COPYRIGHT +file in the top level directory, or at +https://github.com/hpc/Spindle/blob/master/COPYRIGHT + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU Lesser General Public License (as published by the Free Software +Foundation) version 2.1 dated February 1999. This program is distributed in the +hope that it will be useful, but WITHOUT ANY WARRANTY; without even the IMPLIED +WARRANTY OF MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms +and conditions of the GNU Lesser General Public License for more details. You should +have received a copy of the GNU Lesser General Public License along with this +program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA +*/ + +#define _GNU_SOURCE + +__attribute__((noinline, visibility("default"))) +void crash_in_library(int rank) { + volatile int *p = (int *) 0; + (void) rank; + *p = 0; +} diff --git a/testsuite/run_crash_tests_template.sh b/testsuite/run_crash_tests_template.sh new file mode 100755 index 00000000..7b53a8d2 --- /dev/null +++ b/testsuite/run_crash_tests_template.sh @@ -0,0 +1,348 @@ +#!/usr/bin/env bash + +# Usage: +# ./run_crash_tests.sh [--launcher=serial|flux|slurm|slurm-plugin] --nodes=N [--scratch=DIR] +# +# If running the tests from a non-shared filesystem, set --scratch to a shared +# filesystem so that the script can count all corefiles produced across all nodes. +# The CI containers' main filesystem is not shared, so a shared volume should be +# mounted across all containers and specified in --scratch + +set -u + +LAUNCHER="TEST_RESOURCE_MANAGER" +NODES="" +CRASH_TEST_SCRATCH="${CRASH_TEST_SCRATCH:-}" +SPINDLE="${SPINDLE:-SPINDLE_EXEC}" +SPINDLE_RC="${SPINDLE_RC:-SPINDLE_RC_PATH}" +TESTDIR="${TESTDIR:-TEST_RUN_DIR}" + +die() { echo "FAIL: $*" >&2; exit 1; } + +# The tests to run +# Fields: +# mode: --mode to pass to crash test runner +# cores: expected number of cores produced; if N, then equal to total number of ranks +# flags: if "multi-rank", skip when running on only one rank; if "clean", expect test to NOT crash +# top_frame_regex: regex that should match the top frame in produced coredumps. +# Note that all threads will be checked, so in multithreaded examples the regex should +# also match anything that could be on threads other than the one that faulted. +CRASH_TESTS=( +# mode ; cores ; flags ; top_frame_regex + 'all-same ; 1 ; ; crash_function_A' + 'all-different ; N ; ; crash_function_[0-9]+' + 'two-groups ; 2 ; ; crash_function_(A|B)' + 'one-crashes ; 1 ; ; crash_function_A' + 'partial ; 1 ; ; crash_function_A' + 'late-straggler ; 1 ; ; crash_function_A' + 'in-library ; 1 ; ; crash_in_library' + 'in-dlmopen-library ; 1 ; ; crash_in_library' + 'in-library-ctor ; 1 ; ; ctor_crash' + 'sigabrt ; 1 ; ; (__GI_)?raise|abort|pthread_kill' + 'assert ; 1 ; ; (__GI_)?raise|abort|pthread_kill' + 'mixed-abort-segv ; 2 ; multi-rank ; (__GI_)?raise|abort|pthread_kill|do_mixed_abort_segv' + 'span-read ; 1 ; ; do_span_read' + 'safepoint ; 0 ; clean ; -' + 'safepoint-then-crash ; 1 ; ; crash_function_A' + 'safepoint-bad ; 1 ; ; do_safepoint_bad' + 'safepoint-bad-write ; 1 ; ; do_safepoint_bad_write' + 'safepoint-fix-write ; 0 ; clean ; -' + 'safepoint-longjmp ; 0 ; clean ; -' + 'safepoint-span-read ; 0 ; clean ; -' + 'safepoint-span-write ; 0 ; clean ; -' + 'safepoint-span-bad-read ; 1 ; ; do_safepoint_span_bad_read' + 'safepoint-span-bad-write ; 1 ; ; do_safepoint_span_bad_write' + 'no-crash ; 0 ; clean ; -' +) + +declare -A TEST_CORES TEST_FLAGS TEST_TOPFRAME +DEFAULT_MODES=() + +trim() { + local s="$1" + s="${s#"${s%%[![:space:]]*}"}" + s="${s%"${s##*[![:space:]]}"}" + printf '%s' "$s" +} + +parse_table() { + local row mode cores flags top + for row in "${CRASH_TESTS[@]}"; do + IFS=';' read -r mode cores flags top <<<"$row" + mode=$(trim "$mode") + [ -n "$mode" ] || continue + cores=$(trim "$cores") + flags=$(trim "$flags") + top=$(trim "$top") + TEST_CORES[$mode]="$cores" + TEST_FLAGS[$mode]="$flags" + TEST_TOPFRAME[$mode]="$top" + DEFAULT_MODES+=("$mode") + done +} + +has_flag() { + case ",${TEST_FLAGS[$1]:-}," in + *,"$2",*) return 0 ;; + *) return 1 ;; + esac +} + +resolve_cores() { + local mode="$1" val + if [ "$LAUNCHER" = "serial" ]; then + printf '1' + return + fi + val="${TEST_CORES[$mode]}" + [ "$val" = "N" ] && val="$NODES" + printf '%s' "$val" +} + +usage() { + local prog + prog=$(basename "$0") + cat </dev/null 2>&1 || \ + die "crash tests can't run because gdb is not on path" + + case "$LAUNCHER" in + serial|flux|slurm|slurm-plugin) ;; + *) die "unknown launcher" ;; + esac + + if [ "$LAUNCHER" = "serial" ]; then + NODES=1 + elif [ -z "$NODES" ]; then + die "--nodes required" + elif ! [[ "$NODES" =~ ^[1-9][0-9]*$ ]]; then + die "--nodes must be a positive integer (was '$NODES')" + fi + + test -x "$TESTDIR/crash_test" || die "can't find crash test executable" + test -f "$TESTDIR/libcrashfuncs.so" || die "can't find libcrashfuncs.so" +} + +launch() { + local mode="$1" + ulimit -c unlimited + # Make libcrashfuncs.so visible to dlopen() from the mode's scratch dir. + export LD_LIBRARY_PATH="$TESTDIR${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" + case "$LAUNCHER" in + serial) + "$SPINDLE" \ + --no-mpi --crash-dedup -- \ + "$TESTDIR/crash_test" --crash-mode "$mode" + ;; + flux) + flux run \ + -o userrc="$SPINDLE_RC" \ + -o spindle.crash-dedup \ + -N"$NODES" -n"$NODES" \ + --env=LD_LIBRARY_PATH \ + -- "$TESTDIR/crash_test" --crash-mode "$mode" + ;; + slurm) + salloc -N"$NODES" -n"$NODES" \ + "$SPINDLE" --crash-dedup -- \ + srun "$TESTDIR/crash_test" --crash-mode "$mode" + ;; + slurm-plugin) + salloc -N"$NODES" -n"$NODES" \ + srun --spindle="--crash-dedup" \ + "$TESTDIR/crash_test" --crash-mode "$mode" + ;; + esac +} + +core_files() { + local dir="$1" + local cp glob search_dir + cp=$(cat /proc/sys/kernel/core_pattern) + case "$cp" in + /*) glob=$(basename "$cp"); search_dir=$(dirname "$cp") ;; + *) glob="$cp"; search_dir="$dir" ;; + esac + # This converts any parameters (%p, %e, etc.) in the core pattern + # into a shell glob so we can match all produced core files + # (for example, "core.%e.%p" becomes "core.*.*") + glob=$(echo "$glob" | sed -E 's|%[-0-9]*[peghistuclIPCsdfkrSTK]|*|g') + find "$search_dir" -maxdepth 3 -type f -name "$glob" 2>/dev/null +} + +count_cores() { + core_files "$1" | wc -l +} + +verify_top_frames() { + local mode="$1" + local dir="$2" + local expected_top="${TEST_TOPFRAME[$mode]:-}" + + [ "$expected_top" = "-" ] && return 0 + [ -z "$expected_top" ] && return 0 + + local core top cores + cores=$(core_files "$dir") + for core in $cores; do + [ -e "$core" ] || continue + top=$(gdb -batch -nx \ + -iex 'set print demangle off' \ + -ex 'set pagination off' \ + -ex 'bt 1' "$TESTDIR/crash_test" "$core" 2>/dev/null \ + | grep -oE '#0\s+.*' | head -1) + if ! echo "$top" | grep -qE "$expected_top"; then + echo " core $core: top frame '$top' does not match '$expected_top'" >&2 + return 1 + fi + done + return 0 +} + +main() { + parse_args "$@" + parse_table + + if [ -z "$CRASH_TEST_SCRATCH" ]; then + if [ -n "${SPINDLE_TEST_CONTAINER:-}" ]; then + die "--scratch=DIR is required when running in a CI container" + fi + CRASH_TEST_SCRATCH="$TESTDIR/spindle_crash_test" + fi + check_prereqs + + local pass=0 fail=0 + + local modes_to_run + if [ -z "$MODES" ]; then + modes_to_run=("${DEFAULT_MODES[@]}") + else + modes_to_run=() + IFS=',' read -ra specified_modes <<< "$MODES" + for m in "${specified_modes[@]}"; do + m=$(echo "$m" | xargs) + modes_to_run+=("$m") + done + fi + + for mode in "${modes_to_run[@]}"; do + if [ -z "${TEST_CORES[$mode]+set}" ]; then + die "unknown mode '$mode'" + fi + + if has_flag "$mode" multi-rank && \ + { [ "$LAUNCHER" = "serial" ] || [ "$NODES" -lt 2 ]; }; then + echo "SKIP $mode (needs multiple ranks)" + continue + fi + + # Run each test in per-test directory so all the coredumps run in one place + # and we can count them + mkdir -p "$CRASH_TEST_SCRATCH" || die "can't create scratch dir '$CRASH_TEST_SCRATCH'" + local dir + dir=$(mktemp -d "$CRASH_TEST_SCRATCH/$mode.XXXXXX") || die "can't create test dir under '$CRASH_TEST_SCRATCH'" + local launch_rc=0 + ( cd "$dir" && launch "$mode" ) >"$dir/stdout.log" 2>"$dir/stderr.log" || launch_rc=$? + [ "$LAUNCHER" = "serial" ] || sleep 1 + + # If this test is not supposed to crash, verify that it didn't crash + # and exited normally + if has_flag "$mode" clean; then + local actual_cores + actual_cores=$(count_cores "$dir") + if [ "$actual_cores" != "0" ]; then + echo "FAIL $mode: expected 0 dumps, got $actual_cores" + fail=$((fail+1)) + continue + fi + if [ "$launch_rc" != "0" ]; then + echo "FAIL $mode: launcher exited $launch_rc (expected 0)" + fail=$((fail+1)) + continue + fi + echo "PASS $mode (clean exit)" + pass=$((pass+1)) + continue + fi + + # Otherwise, if this test is supposed to crash, verify that we + # got the number of core files that we expect + local want + want=$(resolve_cores "$mode") + local actual + actual=$(count_cores "$dir") + + if [ "$actual" != "$want" ]; then + echo "FAIL $mode: expected $want coredumps, got $actual" + fail=$((fail+1)) + continue + fi + + # And verify that the core files show the expected crash sites + if ! verify_top_frames "$mode" "$dir"; then + echo "FAIL $mode: dump top-frame verification failed" + fail=$((fail+1)) + continue + fi + + echo "PASS $mode ($actual dumps)" + pass=$((pass+1)) + done + + echo + echo "Summary: $pass passed, $fail failed" + [ "$fail" -eq 0 ] || exit 1 +} + +main "$@" From 469cb9e04601342cbd8238bb58e97eab0cb5a0d4 Mon Sep 17 00:00:00 2001 From: Nicholas Chaimov Date: Mon, 15 Jun 2026 14:58:41 -0700 Subject: [PATCH 07/14] Crash handler: CI testing Run the new crash handler testsuite in CI. Configure CI runner to generate coredumps, add shared filesystem to aggregate coredumps across nodes in mutli-node tests. Grants CAP_SYS_RESOURCE so ulimit can be set in containers. --- .github/workflows/ci.yml | 47 ++++++++++++++++++- containers/spindle-flux-ubuntu/Dockerfile | 1 + .../spindle-flux-ubuntu/docker-compose.yml | 7 +++ .../scripts/build_spindle.sh | 2 +- .../spindle-flux-ubuntu/scripts/entrypoint.sh | 5 ++ containers/spindle-serial-ubuntu/Dockerfile | 2 + .../scripts/build_spindle.sh | 2 +- .../spindle-slurm-ubuntu/base/Dockerfile | 1 + .../testing-plugin/Dockerfile | 5 ++ .../testing-plugin/docker-compose.yml | 8 ++++ .../testing-plugin/scripts/build_spindle.sh | 2 +- .../testing-plugin/scripts/entrypoint.sh | 11 +++-- .../spindle-slurm-ubuntu/testing/Dockerfile | 5 ++ .../testing/docker-compose.yml | 8 ++++ .../testing/scripts/build_spindle.sh | 2 +- .../testing/scripts/entrypoint.sh | 11 +++-- 16 files changed, 108 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8a5a4b02..7e4708d5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,6 +54,15 @@ jobs: run: | docker exec spindlenode bash -c 'cd Spindle-build/testsuite && ./runTests' + - name: Configure host core_pattern for coredumps + if: ${{ !env.ACT }} + run: | + sudo sysctl -w kernel.core_pattern='core.%p' + + - name: Run spindle-serial-ubuntu crash tests + run: | + docker exec spindlenode bash -c 'cd Spindle-build/testsuite && ./run_crash_tests.sh --launcher=serial --scratch=/tmp/spindle_crash_test' + - name: Bring spindle-serial-ubuntu down id: serial-ubuntu-down if: ${{ always() }} @@ -61,7 +70,7 @@ jobs: run: | cd containers/spindle-serial-ubuntu docker compose down - + spindle-serial-ubuntu_arm64: name: Testsuite (Serial, Ubuntu, arm64) environment: Spindle CI @@ -98,6 +107,15 @@ jobs: run: | docker exec spindlenode bash -c 'cd Spindle-build/testsuite && ./runTests' + - name: Configure host core_pattern for coredumps + if: ${{ !env.ACT }} + run: | + sudo sysctl -w kernel.core_pattern='core.%p' + + - name: Run spindle-serial-ubuntu crash tests + run: | + docker exec spindlenode bash -c 'cd Spindle-build/testsuite && ./run_crash_tests.sh --launcher=serial --scratch=/tmp/spindle_crash_test' + - name: Bring spindle-serial-ubuntu down id: serial-ubuntu-down if: ${{ always() }} @@ -142,6 +160,15 @@ jobs: run: | docker exec node-1 bash -c 'cd Spindle-build/testsuite && flux alloc --nodes=${workers} ./runTests --nodes=${workers} --tasks-per-node=3' + - name: Configure host core_pattern for coredumps + if: ${{ !env.ACT }} + run: | + sudo sysctl -w kernel.core_pattern='core.%p' + + - name: Run spindle-flux-ubuntu crash tests + run: | + docker exec node-1 bash -c 'cd Spindle-build/testsuite && flux alloc --nodes=${workers} ./run_crash_tests.sh --launcher=flux --nodes=${workers} --scratch=/shared/spindle_crash_test' + - name: Bring spindle-flux-ubuntu down id: flux-ubuntu-down if: ${{ always() }} @@ -200,6 +227,15 @@ jobs: run: | docker exec slurm-head bash -c 'cd Spindle-build/testsuite && salloc -n${workers} -N${workers} ./runTests ${workers}' + - name: Configure host core_pattern for coredumps + if: ${{ !env.ACT }} + run: | + sudo sysctl -w kernel.core_pattern='core.%p' + + - name: Run spindle-slurm-ubuntu crash tests + run: | + docker exec slurm-head bash -c 'cd Spindle-build/testsuite && ./run_crash_tests.sh --launcher=slurm --nodes=${workers} --scratch=/shared/spindle_crash_test' + - name: Bring spindle-slurm-ubuntu down id: slurm-ubuntu-down if: ${{ always() }} @@ -321,6 +357,15 @@ jobs: run: | docker exec slurm-plugin-head bash -c 'cd Spindle-build/testsuite && salloc -n${workers} -N${workers} --spindle-session ./runTests ${workers}' + - name: Configure host core_pattern for coredumps + if: ${{ !env.ACT }} + run: | + sudo sysctl -w kernel.core_pattern='core.%p' + + - name: Run spindle-slurm-plugin-ubuntu crash tests + run: | + docker exec slurm-plugin-head bash -c 'cd Spindle-build/testsuite && ./run_crash_tests.sh --launcher=slurm-plugin --nodes=${workers} --scratch=/shared/spindle_crash_test' + - name: Bring spindle-slurm-plugin-ubuntu down id: slurm-ubuntu-down if: ${{ always() }} diff --git a/containers/spindle-flux-ubuntu/Dockerfile b/containers/spindle-flux-ubuntu/Dockerfile index 57badbfc..9bfd7944 100644 --- a/containers/spindle-flux-ubuntu/Dockerfile +++ b/containers/spindle-flux-ubuntu/Dockerfile @@ -7,6 +7,7 @@ ENV workers=${replicas} USER root ENV TMPDIR=/tmp RUN echo 'TMPDIR="/tmp"' >> /etc/environment +ENV SPINDLE_TEST_CONTAINER=1 RUN DEBIAN_FRONTEND="noninteractive" apt-get update \ && apt-get -qq install -y --no-install-recommends \ diff --git a/containers/spindle-flux-ubuntu/docker-compose.yml b/containers/spindle-flux-ubuntu/docker-compose.yml index f2aaf4b4..5903e778 100644 --- a/containers/spindle-flux-ubuntu/docker-compose.yml +++ b/containers/spindle-flux-ubuntu/docker-compose.yml @@ -31,12 +31,19 @@ networks: flux: driver: bridge +# Shared scratch volume so run_crash_tests.sh can count coredumps +# produced by any rank on any node from a single location. +volumes: + crash_scratch: + # Common parameters for all nodes. x-shared-node-parameters: &shared-node-parameters build: *shared-build-context networks: - flux environment: *shared-environment + volumes: + - crash_scratch:/shared cap_add: - SYS_NICE # Required for libnuma diff --git a/containers/spindle-flux-ubuntu/scripts/build_spindle.sh b/containers/spindle-flux-ubuntu/scripts/build_spindle.sh index a63b5d8d..276c59a6 100755 --- a/containers/spindle-flux-ubuntu/scripts/build_spindle.sh +++ b/containers/spindle-flux-ubuntu/scripts/build_spindle.sh @@ -4,7 +4,7 @@ set -euxo pipefail mkdir -p /home/${USER}/Spindle-build cd /home/${USER}/Spindle-build -/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=flux --enable-flux-plugin --with-cachepaths=/tmp/commpath/cachepath --with-commpath=/tmp/commpath CFLAGS="-O2 -g" CXXFLAGS="-O2 -g" +/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=flux --enable-flux-plugin --with-cachepaths=/tmp/commpath/cachepath --with-commpath=/tmp/commpath --enable-crash-dedup CFLAGS="-O2 -g" CXXFLAGS="-O2 -g" make -j$(nproc) make install diff --git a/containers/spindle-flux-ubuntu/scripts/entrypoint.sh b/containers/spindle-flux-ubuntu/scripts/entrypoint.sh index 8d3944ef..95efffdf 100755 --- a/containers/spindle-flux-ubuntu/scripts/entrypoint.sh +++ b/containers/spindle-flux-ubuntu/scripts/entrypoint.sh @@ -22,6 +22,11 @@ thisHost=${thisHost[0]} echo $thisHost export FLUX_FAKE_HOSTNAME=$thisHost +if [ -d /shared ]; then + sudo chown -R "$(id -un):$(id -gn)" /shared + sudo chmod 755 /shared +fi + # Start munged sudo -u munge /usr/sbin/munged diff --git a/containers/spindle-serial-ubuntu/Dockerfile b/containers/spindle-serial-ubuntu/Dockerfile index 62c0cbf5..9fa1b53e 100644 --- a/containers/spindle-serial-ubuntu/Dockerfile +++ b/containers/spindle-serial-ubuntu/Dockerfile @@ -3,6 +3,7 @@ FROM ubuntu:${ubuntu_version} USER root ENV TMPDIR=/tmp RUN echo 'TMPDIR="/tmp"' >> /etc/environment +ENV SPINDLE_TEST_CONTAINER=1 RUN DEBIAN_FRONTEND="noninteractive" apt-get update \ # install latest pkg utils: @@ -26,6 +27,7 @@ RUN DEBIAN_FRONTEND="noninteractive" apt-get -qq install -y --no-install-recomme gfortran-13 \ gcc-13 \ g++-13 \ + gdb \ munge \ libmunge-dev \ libhwloc-dev \ diff --git a/containers/spindle-serial-ubuntu/scripts/build_spindle.sh b/containers/spindle-serial-ubuntu/scripts/build_spindle.sh index 1022ba14..b48e3b53 100755 --- a/containers/spindle-serial-ubuntu/scripts/build_spindle.sh +++ b/containers/spindle-serial-ubuntu/scripts/build_spindle.sh @@ -4,7 +4,7 @@ set -euxo pipefail mkdir -p /home/${USER}/Spindle-build cd /home/${USER}/Spindle-build -/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=serial --with-cachepaths=/tmp/commpath/cachepath --with-commpath=/tmp/commpath CFLAGS="-O2 -g" CXXFLAGS="-O2 -g" +/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=serial --with-cachepaths=/tmp/commpath/cachepath --with-commpath=/tmp/commpath --enable-crash-dedup CFLAGS="-O2 -g" CXXFLAGS="-O2 -g" make -j$(nproc) make install diff --git a/containers/spindle-slurm-ubuntu/base/Dockerfile b/containers/spindle-slurm-ubuntu/base/Dockerfile index d6eee9f2..b571235c 100644 --- a/containers/spindle-slurm-ubuntu/base/Dockerfile +++ b/containers/spindle-slurm-ubuntu/base/Dockerfile @@ -27,6 +27,7 @@ RUN apt-get update \ gfortran-13 \ gcc-13 \ g++-13 \ + gdb \ munge \ libmunge-dev \ libhwloc-dev \ diff --git a/containers/spindle-slurm-ubuntu/testing-plugin/Dockerfile b/containers/spindle-slurm-ubuntu/testing-plugin/Dockerfile index 951480f1..0e7774e3 100644 --- a/containers/spindle-slurm-ubuntu/testing-plugin/Dockerfile +++ b/containers/spindle-slurm-ubuntu/testing-plugin/Dockerfile @@ -4,9 +4,14 @@ ARG replicas=4 ENV workers=${replicas} ENV TMPDIR=/tmp RUN echo 'TMPDIR="/tmp"' >> /etc/environment +ENV SPINDLE_TEST_CONTAINER=1 ARG BUILD_ROOT=containers/spindle-slurm-ubuntu/testing-plugin +RUN apt-get update \ + && DEBIAN_FRONTEND="noninteractive" apt-get -qq install -y --no-install-recommends \ + gdb + # Slurm daemons run as $SLURM_USER ARG SLURM_USER=slurm diff --git a/containers/spindle-slurm-ubuntu/testing-plugin/docker-compose.yml b/containers/spindle-slurm-ubuntu/testing-plugin/docker-compose.yml index 2751a71a..309e64b9 100644 --- a/containers/spindle-slurm-ubuntu/testing-plugin/docker-compose.yml +++ b/containers/spindle-slurm-ubuntu/testing-plugin/docker-compose.yml @@ -35,13 +35,21 @@ networks: slurm: driver: bridge +# Shared scratch volume so run_crash_tests.sh can count coredumps +# written by ranks on any node from the head node. +volumes: + crash_scratch: + # Common parameters for all nodes. x-shared-node-parameters: &shared-node-parameters build: *shared-build-context networks: - slurm + volumes: + - crash_scratch:/shared cap_add: - SYS_NICE # Required for libnuma + - SYS_RESOURCE # Allow changing ulimit x-healthcheck-parameters: &healthcheck-parameters start_period: 3s diff --git a/containers/spindle-slurm-ubuntu/testing-plugin/scripts/build_spindle.sh b/containers/spindle-slurm-ubuntu/testing-plugin/scripts/build_spindle.sh index 2b36be90..8b759415 100755 --- a/containers/spindle-slurm-ubuntu/testing-plugin/scripts/build_spindle.sh +++ b/containers/spindle-slurm-ubuntu/testing-plugin/scripts/build_spindle.sh @@ -3,7 +3,7 @@ set -euxo pipefail mkdir -p /home/${USER}/Spindle-build cd /home/${USER}/Spindle-build -/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=slurm-plugin --enable-slurm-plugin --with-cachepaths=/tmp/commpath/cachepath --with-commpath=/tmp/commpath CFLAGS="-O2 -g" CXXFLAGS="-O2 -g" +/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=slurm-plugin --enable-slurm-plugin --with-cachepaths=/tmp/commpath/cachepath --with-commpath=/tmp/commpath --enable-crash-dedup CFLAGS="-O2 -g" CXXFLAGS="-O2 -g" make -j$(nproc) make install diff --git a/containers/spindle-slurm-ubuntu/testing-plugin/scripts/entrypoint.sh b/containers/spindle-slurm-ubuntu/testing-plugin/scripts/entrypoint.sh index 54d40e48..5ba29ac7 100755 --- a/containers/spindle-slurm-ubuntu/testing-plugin/scripts/entrypoint.sh +++ b/containers/spindle-slurm-ubuntu/testing-plugin/scripts/entrypoint.sh @@ -3,9 +3,14 @@ echo "SLURM_ROLE: ${SLURM_ROLE}" echo "Starting sshd..." -sudo service ssh start +sudo bash -c 'ulimit -c unlimited; service ssh start' echo "Starting munged..." -sudo -u munge /usr/sbin/munged +sudo -u munge /usr/sbin/munged + +if [ -d /shared ]; then + sudo chown -R "$(id -un):$(id -gn)" /shared + sudo chmod 755 /shared +fi if [ "${SLURM_ROLE}" = "db" ]; then echo "Starting slurmdbd..." @@ -15,7 +20,7 @@ elif [ "${SLURM_ROLE}" = "ctl" ] ; then sudo -u slurm /usr/sbin/slurmctld -i -Dvvv elif [ "${SLURM_ROLE}" = "worker" ] ; then echo "Starting slurmd..." - sudo /usr/sbin/slurmd -Dvvv + sudo bash -c 'ulimit -c unlimited; exec env -u PWD /usr/sbin/slurmd -Dvvv' fi sleep inf diff --git a/containers/spindle-slurm-ubuntu/testing/Dockerfile b/containers/spindle-slurm-ubuntu/testing/Dockerfile index 99768535..b73616f8 100644 --- a/containers/spindle-slurm-ubuntu/testing/Dockerfile +++ b/containers/spindle-slurm-ubuntu/testing/Dockerfile @@ -4,9 +4,14 @@ ARG replicas=4 ENV workers=${replicas} ENV TMPDIR=/tmp RUN echo 'TMPDIR="/tmp"' >> /etc/environment +ENV SPINDLE_TEST_CONTAINER=1 ARG BUILD_ROOT=containers/spindle-slurm-ubuntu/testing +RUN apt-get update \ + && DEBIAN_FRONTEND="noninteractive" apt-get -qq install -y --no-install-recommends \ + gdb + # Slurm daemons run as $SLURM_USER ARG SLURM_USER=slurm diff --git a/containers/spindle-slurm-ubuntu/testing/docker-compose.yml b/containers/spindle-slurm-ubuntu/testing/docker-compose.yml index 5d3deb74..6f9ed0dc 100644 --- a/containers/spindle-slurm-ubuntu/testing/docker-compose.yml +++ b/containers/spindle-slurm-ubuntu/testing/docker-compose.yml @@ -35,13 +35,21 @@ networks: slurm: driver: bridge +# Shared scratch volume so run_crash_tests.sh can count coredumps +# written by ranks on any node from the head node. +volumes: + crash_scratch: + # Common parameters for all nodes. x-shared-node-parameters: &shared-node-parameters build: *shared-build-context networks: - slurm + volumes: + - crash_scratch:/shared cap_add: - SYS_NICE # Required for libnuma + - SYS_RESOURCE # Allow changing ulimit x-healthcheck-parameters: &healthcheck-parameters start_period: 3s diff --git a/containers/spindle-slurm-ubuntu/testing/scripts/build_spindle.sh b/containers/spindle-slurm-ubuntu/testing/scripts/build_spindle.sh index 7fcb48e3..4a4c5eee 100755 --- a/containers/spindle-slurm-ubuntu/testing/scripts/build_spindle.sh +++ b/containers/spindle-slurm-ubuntu/testing/scripts/build_spindle.sh @@ -3,7 +3,7 @@ set -euxo pipefail mkdir -p /home/${USER}/Spindle-build cd /home/${USER}/Spindle-build -/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=slurm --with-rsh-launch --with-rsh-cmd=/usr/bin/ssh --with-cachepaths=/tmp/commpath/cachepath --with-commpath=/tmp/commpath CFLAGS="-O2 -g" CXXFLAGS="-O2 -g" +/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=slurm --with-rsh-launch --with-rsh-cmd=/usr/bin/ssh --with-cachepaths=/tmp/commpath/cachepath --with-commpath=/tmp/commpath --enable-crash-dedup CFLAGS="-O2 -g" CXXFLAGS="-O2 -g" make -j$(nproc) make install diff --git a/containers/spindle-slurm-ubuntu/testing/scripts/entrypoint.sh b/containers/spindle-slurm-ubuntu/testing/scripts/entrypoint.sh index 54d40e48..0e2f1a90 100755 --- a/containers/spindle-slurm-ubuntu/testing/scripts/entrypoint.sh +++ b/containers/spindle-slurm-ubuntu/testing/scripts/entrypoint.sh @@ -3,9 +3,14 @@ echo "SLURM_ROLE: ${SLURM_ROLE}" echo "Starting sshd..." -sudo service ssh start +sudo bash -c 'ulimit -c unlimited; service ssh start' echo "Starting munged..." -sudo -u munge /usr/sbin/munged +sudo -u munge /usr/sbin/munged + +if [ -d /shared ]; then + sudo chown -R "$(id -un):$(id -gn)" /shared + sudo chmod 755 /shared +fi if [ "${SLURM_ROLE}" = "db" ]; then echo "Starting slurmdbd..." @@ -15,7 +20,7 @@ elif [ "${SLURM_ROLE}" = "ctl" ] ; then sudo -u slurm /usr/sbin/slurmctld -i -Dvvv elif [ "${SLURM_ROLE}" = "worker" ] ; then echo "Starting slurmd..." - sudo /usr/sbin/slurmd -Dvvv + sudo bash -c 'ulimit -c unlimited; exec /usr/sbin/slurmd -Dvvv' fi sleep inf From f9a51c7600e74277d6e6bd7e6390e1debbe0a917 Mon Sep 17 00:00:00 2001 From: Nicholas Chaimov Date: Mon, 15 Jun 2026 14:58:42 -0700 Subject: [PATCH 08/14] Crash handler: bootstrap autotools-generated files re-generated --- config.h.in | 6 ++ configure | 51 +++++++++++ src/client/auditclient/Makefile.in | 2 +- src/client/client/Makefile.in | 108 ++++++++++++++++++++--- src/client/config.h.in | 6 ++ src/client/configure | 51 +++++++++++ src/client/crash_handler/crash_handler.c | 10 +-- src/client/subaudit/Makefile.in | 2 +- src/fe/config.h.in | 6 ++ src/fe/configure | 51 +++++++++++ src/server/config.h.in | 6 ++ src/server/configure | 51 +++++++++++ testsuite/Makefile.in | 23 ++++- 13 files changed, 354 insertions(+), 19 deletions(-) diff --git a/config.h.in b/config.h.in index 121c551f..2697764a 100644 --- a/config.h.in +++ b/config.h.in @@ -27,6 +27,12 @@ /* Define if were using sockets for client/server communication */ #undef COMM_SOCKET +/* Enable --crash-dedup by default */ +#undef CRASH_DEDUP_ENABLED_BY_DEFAULT + +/* Define to 1 if the crash handler is built */ +#undef CRASH_HANDLER_ENABLED + /* Defined if GLIBC has an allocation bug */ #undef DTV_ALLOCATION_BUG diff --git a/configure b/configure index ac8d1434..f7badb36 100755 --- a/configure +++ b/configure @@ -667,6 +667,8 @@ LAUNCHMON_LIB LAUNCHMON_INC HAVE_LMON LAUNCHMON_STATIC_LIBS +BLD_CRASH_HANDLER_FALSE +BLD_CRASH_HANDLER_TRUE STATICFLAG fluxplugin_ldflags shell_plugindir @@ -879,6 +881,8 @@ enable_sec_munge enable_sec_keydir enable_sec_launchmon enable_sec_none +enable_crash_handler +enable_crash_dedup with_gcrypt_dir with_munge_dir with_glibc_be_dir @@ -1576,6 +1580,8 @@ Optional Features: --enable-sec-launchmon Enable LaunchmMON to propagate authentication key for connections --enable-sec-none Disable security authentication of connections + --disable-crash-handler Build the crash handler (default: enabled) + --enable-crash-dedup Enable --crash-dedup by default (default: disabled) --enable-remap-pageone Controls mmap behavior when replacing executables. Should be false on older linux systems --enable-testsuite Chooses whether to build the Spindle testsuite. @@ -17658,6 +17664,47 @@ if test "${enable_sec_none+set}" = set; then : fi +# Check whether --enable-crash-handler was given. +if test "${enable_crash_handler+set}" = set; then : + enableval=$enable_crash_handler; enable_crash_handler=$enableval +else + enable_crash_handler=yes +fi + + +if test "x$enable_crash_handler" != xno; then : + +$as_echo "#define CRASH_HANDLER_ENABLED 1" >>confdefs.h + +fi + if test "x$enable_crash_handler" != xno; then + BLD_CRASH_HANDLER_TRUE= + BLD_CRASH_HANDLER_FALSE='#' +else + BLD_CRASH_HANDLER_TRUE='#' + BLD_CRASH_HANDLER_FALSE= +fi + + +# Check whether --enable-crash-dedup was given. +if test "${enable_crash_dedup+set}" = set; then : + enableval=$enable_crash_dedup; enable_crash_dedup=$enableval +else + enable_crash_dedup=no +fi + + +if test "x$enable_crash_dedup" = xyes && test "x$enable_crash_handler" = xno; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: --enable-crash-dedup ignored because --disable-crash-handler was given" >&5 +$as_echo "$as_me: WARNING: --enable-crash-dedup ignored because --disable-crash-handler was given" >&2;} +fi + +if test "x$enable_crash_dedup" = xyes && test "x$enable_crash_handler" != xno; then : + +$as_echo "#define CRASH_DEDUP_ENABLED_BY_DEFAULT 1" >>confdefs.h + +fi + # Check whether --with-gcrypt-dir was given. if test "${with_gcrypt_dir+set}" = set; then : @@ -19100,6 +19147,10 @@ if test -z "${AARCH64_BLD_TRUE}" && test -z "${AARCH64_BLD_FALSE}"; then as_fn_error $? "conditional \"AARCH64_BLD\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi +if test -z "${BLD_CRASH_HANDLER_TRUE}" && test -z "${BLD_CRASH_HANDLER_FALSE}"; then + as_fn_error $? "conditional \"BLD_CRASH_HANDLER\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi if test -z "${LMON_TRUE}" && test -z "${LMON_FALSE}"; then as_fn_error $? "conditional \"LMON\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 diff --git a/src/client/auditclient/Makefile.in b/src/client/auditclient/Makefile.in index a972661a..a7bfbe96 100644 --- a/src/client/auditclient/Makefile.in +++ b/src/client/auditclient/Makefile.in @@ -408,7 +408,7 @@ top_builddir = @top_builddir@ top_srcdir = @top_srcdir@ pkglib_LTLIBRARIES = $(am__append_1) $(am__append_2) $(am__append_3) AM_CFLAGS = -fvisibility=hidden -AM_CPPFLAGS = -I$(top_srcdir)/../logging -I$(top_srcdir)/../include -I$(top_srcdir)/client -I$(top_srcdir)/client_comlib +AM_CPPFLAGS = -I$(top_srcdir)/../logging -I$(top_srcdir)/../include -I$(top_srcdir)/client -I$(top_srcdir)/client_comlib -I$(top_srcdir)/crash_handler BASE_SRCS = auditclient.c auditclient_common.c patch_linkmap.c redirect.c bindgot.c writablegot.c patch_bad_dtv.c malloc_wrapper.c fixlocale.c @AARCH64_BLD_TRUE@ARCH_SRCS = auditclient_aarch64.c @PPC64LE_BLD_TRUE@ARCH_SRCS = auditclient_ppc64.c diff --git a/src/client/client/Makefile.in b/src/client/client/Makefile.in index d0241599..a2c82cbe 100644 --- a/src/client/client/Makefile.in +++ b/src/client/client/Makefile.in @@ -93,6 +93,12 @@ target_triplet = @target@ @SOCKETS_TRUE@am__append_1 = libspindlec_socket.la @PIPES_TRUE@am__append_2 = libspindlec_pipe.la @BITER_TRUE@am__append_3 = libspindlec_biter.la +@BLD_CRASH_HANDLER_TRUE@am__append_4 = $(top_srcdir)/crash_handler/crash_io.c \ +@BLD_CRASH_HANDLER_TRUE@ $(top_srcdir)/crash_handler/crash_fmt.c \ +@BLD_CRASH_HANDLER_TRUE@ $(top_srcdir)/crash_handler/crash_handler.c \ +@BLD_CRASH_HANDLER_TRUE@ $(top_srcdir)/crash_handler/crash_lib_offset.c \ +@BLD_CRASH_HANDLER_TRUE@ $(top_srcdir)/crash_handler/crash_sigchain.c + subdir = client ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 am__aclocal_m4_deps = $(top_srcdir)/../../m4/libtool.m4 \ @@ -134,25 +140,57 @@ libspindlec_biter_la_DEPENDENCIES = \ $(top_builddir)/client_comlib/libclient_biter.la \ $(top_builddir)/logging/libspindleclogc.la \ $(top_builddir)/shm_cache/libshmcache.la -am__objects_2 = client.lo lookup.lo should_intercept.lo exec_util.lo \ - remap_exec.lo lookup_libc.lo \ +am__libspindlec_biter_la_SOURCES_DIST = client.c lookup.c \ + should_intercept.c exec_util.c remap_exec.c lookup_libc.c \ + adjust_argv.c $(top_srcdir)/../utils/parseloc.c \ + $(top_srcdir)/../utils/getcpu.c \ + $(top_srcdir)/crash_handler/crash_io.c \ + $(top_srcdir)/crash_handler/crash_fmt.c \ + $(top_srcdir)/crash_handler/crash_handler.c \ + $(top_srcdir)/crash_handler/crash_lib_offset.c \ + $(top_srcdir)/crash_handler/crash_sigchain.c +@BLD_CRASH_HANDLER_TRUE@am__objects_2 = $(top_builddir)/crash_handler/crash_io.lo \ +@BLD_CRASH_HANDLER_TRUE@ $(top_builddir)/crash_handler/crash_fmt.lo \ +@BLD_CRASH_HANDLER_TRUE@ $(top_builddir)/crash_handler/crash_handler.lo \ +@BLD_CRASH_HANDLER_TRUE@ $(top_builddir)/crash_handler/crash_lib_offset.lo \ +@BLD_CRASH_HANDLER_TRUE@ $(top_builddir)/crash_handler/crash_sigchain.lo +am__objects_3 = client.lo lookup.lo should_intercept.lo exec_util.lo \ + remap_exec.lo lookup_libc.lo adjust_argv.lo \ $(top_builddir)/../utils/parseloc.lo \ - $(top_builddir)/../utils/getcpu.lo adjust_argv.lo -am_libspindlec_biter_la_OBJECTS = $(am__objects_2) + $(top_builddir)/../utils/getcpu.lo $(am__objects_2) +am_libspindlec_biter_la_OBJECTS = $(am__objects_3) libspindlec_biter_la_OBJECTS = $(am_libspindlec_biter_la_OBJECTS) @BITER_TRUE@am_libspindlec_biter_la_rpath = libspindlec_pipe_la_DEPENDENCIES = \ $(top_builddir)/client_comlib/libclient_pipe.la \ $(top_builddir)/logging/libspindleclogc.la \ $(top_builddir)/shm_cache/libshmcache.la -am_libspindlec_pipe_la_OBJECTS = $(am__objects_2) +am__libspindlec_pipe_la_SOURCES_DIST = client.c lookup.c \ + should_intercept.c exec_util.c remap_exec.c lookup_libc.c \ + adjust_argv.c $(top_srcdir)/../utils/parseloc.c \ + $(top_srcdir)/../utils/getcpu.c \ + $(top_srcdir)/crash_handler/crash_io.c \ + $(top_srcdir)/crash_handler/crash_fmt.c \ + $(top_srcdir)/crash_handler/crash_handler.c \ + $(top_srcdir)/crash_handler/crash_lib_offset.c \ + $(top_srcdir)/crash_handler/crash_sigchain.c +am_libspindlec_pipe_la_OBJECTS = $(am__objects_3) libspindlec_pipe_la_OBJECTS = $(am_libspindlec_pipe_la_OBJECTS) @PIPES_TRUE@am_libspindlec_pipe_la_rpath = libspindlec_socket_la_DEPENDENCIES = \ $(top_builddir)/client_comlib/libclient_socket.la \ $(top_builddir)/logging/libspindleclogc.la \ $(top_builddir)/shm_cache/libshmcache.la -am_libspindlec_socket_la_OBJECTS = $(am__objects_2) +am__libspindlec_socket_la_SOURCES_DIST = client.c lookup.c \ + should_intercept.c exec_util.c remap_exec.c lookup_libc.c \ + adjust_argv.c $(top_srcdir)/../utils/parseloc.c \ + $(top_srcdir)/../utils/getcpu.c \ + $(top_srcdir)/crash_handler/crash_io.c \ + $(top_srcdir)/crash_handler/crash_fmt.c \ + $(top_srcdir)/crash_handler/crash_handler.c \ + $(top_srcdir)/crash_handler/crash_lib_offset.c \ + $(top_srcdir)/crash_handler/crash_sigchain.c +am_libspindlec_socket_la_OBJECTS = $(am__objects_3) libspindlec_socket_la_OBJECTS = $(am_libspindlec_socket_la_OBJECTS) @SOCKETS_TRUE@am_libspindlec_socket_la_rpath = AM_V_P = $(am__v_P_@AM_V@) @@ -173,6 +211,11 @@ am__maybe_remake_depfiles = depfiles am__depfiles_remade = $(top_builddir)/../utils/$(DEPDIR)/getcpu.Plo \ $(top_builddir)/../utils/$(DEPDIR)/libspindle_audit_la-fileutil.Plo \ $(top_builddir)/../utils/$(DEPDIR)/parseloc.Plo \ + $(top_builddir)/crash_handler/$(DEPDIR)/crash_fmt.Plo \ + $(top_builddir)/crash_handler/$(DEPDIR)/crash_handler.Plo \ + $(top_builddir)/crash_handler/$(DEPDIR)/crash_io.Plo \ + $(top_builddir)/crash_handler/$(DEPDIR)/crash_lib_offset.Plo \ + $(top_builddir)/crash_handler/$(DEPDIR)/crash_sigchain.Plo \ ./$(DEPDIR)/adjust_argv.Plo ./$(DEPDIR)/client.Plo \ ./$(DEPDIR)/exec_util.Plo \ ./$(DEPDIR)/libspindle_audit_la-intercept.Plo \ @@ -209,8 +252,9 @@ SOURCES = $(libspindle_audit_la_SOURCES) \ $(libspindlec_biter_la_SOURCES) $(libspindlec_pipe_la_SOURCES) \ $(libspindlec_socket_la_SOURCES) DIST_SOURCES = $(libspindle_audit_la_SOURCES) \ - $(libspindlec_biter_la_SOURCES) $(libspindlec_pipe_la_SOURCES) \ - $(libspindlec_socket_la_SOURCES) + $(am__libspindlec_biter_la_SOURCES_DIST) \ + $(am__libspindlec_pipe_la_SOURCES_DIST) \ + $(am__libspindlec_socket_la_SOURCES_DIST) am__can_run_installinfo = \ case $$AM_UPDATE_INFO_DIR in \ n|no|NO) false;; \ @@ -375,9 +419,12 @@ top_srcdir = @top_srcdir@ noinst_LTLIBRARIES = libspindle_audit.la $(am__append_1) \ $(am__append_2) $(am__append_3) AM_CFLAGS = -fvisibility=hidden -AM_CPPFLAGS = -I$(top_srcdir)/../logging -I$(top_srcdir)/client_comlib -I$(top_srcdir)/../include -I$(top_srcdir)/shm_cache -I$(top_srcdir)/subaudit -I$(top_srcdir)/../utils -I$(top_srcdir)/auditclient +AM_CPPFLAGS = -I$(top_srcdir)/../logging -I$(top_srcdir)/client_comlib -I$(top_srcdir)/../include -I$(top_srcdir)/shm_cache -I$(top_srcdir)/subaudit -I$(top_srcdir)/../utils -I$(top_srcdir)/auditclient -I$(top_srcdir)/client -I$(top_srcdir)/crash_handler INTERCEPT_SRCS = intercept_open.c intercept_exec.c intercept_stat.c intercept_readlink.c intercept_spindleapi.c intercept.c spindle_regex.c patch_interception.c realpath.c $(top_srcdir)/../utils/fileutil.c -BASE_SRCS = client.c lookup.c should_intercept.c exec_util.c remap_exec.c lookup_libc.c $(top_srcdir)/../utils/parseloc.c $(top_srcdir)/../utils/getcpu.c adjust_argv.c +BASE_SRCS = client.c lookup.c should_intercept.c exec_util.c \ + remap_exec.c lookup_libc.c adjust_argv.c \ + $(top_srcdir)/../utils/parseloc.c \ + $(top_srcdir)/../utils/getcpu.c $(am__append_4) libspindlec_socket_la_SOURCES = $(BASE_SRCS) libspindlec_socket_la_LIBADD = $(top_builddir)/client_comlib/libclient_socket.la $(top_builddir)/logging/libspindleclogc.la $(top_builddir)/shm_cache/libshmcache.la libspindlec_pipe_la_SOURCES = $(BASE_SRCS) @@ -448,6 +495,27 @@ $(top_builddir)/../utils/parseloc.lo: \ $(top_builddir)/../utils/getcpu.lo: \ $(top_builddir)/../utils/$(am__dirstamp) \ $(top_builddir)/../utils/$(DEPDIR)/$(am__dirstamp) +$(top_builddir)/crash_handler/$(am__dirstamp): + @$(MKDIR_P) $(top_builddir)/crash_handler + @: > $(top_builddir)/crash_handler/$(am__dirstamp) +$(top_builddir)/crash_handler/$(DEPDIR)/$(am__dirstamp): + @$(MKDIR_P) $(top_builddir)/crash_handler/$(DEPDIR) + @: > $(top_builddir)/crash_handler/$(DEPDIR)/$(am__dirstamp) +$(top_builddir)/crash_handler/crash_io.lo: \ + $(top_builddir)/crash_handler/$(am__dirstamp) \ + $(top_builddir)/crash_handler/$(DEPDIR)/$(am__dirstamp) +$(top_builddir)/crash_handler/crash_fmt.lo: \ + $(top_builddir)/crash_handler/$(am__dirstamp) \ + $(top_builddir)/crash_handler/$(DEPDIR)/$(am__dirstamp) +$(top_builddir)/crash_handler/crash_handler.lo: \ + $(top_builddir)/crash_handler/$(am__dirstamp) \ + $(top_builddir)/crash_handler/$(DEPDIR)/$(am__dirstamp) +$(top_builddir)/crash_handler/crash_lib_offset.lo: \ + $(top_builddir)/crash_handler/$(am__dirstamp) \ + $(top_builddir)/crash_handler/$(DEPDIR)/$(am__dirstamp) +$(top_builddir)/crash_handler/crash_sigchain.lo: \ + $(top_builddir)/crash_handler/$(am__dirstamp) \ + $(top_builddir)/crash_handler/$(DEPDIR)/$(am__dirstamp) libspindlec_biter.la: $(libspindlec_biter_la_OBJECTS) $(libspindlec_biter_la_DEPENDENCIES) $(EXTRA_libspindlec_biter_la_DEPENDENCIES) $(AM_V_CCLD)$(LINK) $(am_libspindlec_biter_la_rpath) $(libspindlec_biter_la_OBJECTS) $(libspindlec_biter_la_LIBADD) $(LIBS) @@ -462,6 +530,8 @@ mostlyclean-compile: -rm -f *.$(OBJEXT) -rm -f $(top_builddir)/../utils/*.$(OBJEXT) -rm -f $(top_builddir)/../utils/*.lo + -rm -f $(top_builddir)/crash_handler/*.$(OBJEXT) + -rm -f $(top_builddir)/crash_handler/*.lo distclean-compile: -rm -f *.tab.c @@ -469,6 +539,11 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@$(top_builddir)/../utils/$(DEPDIR)/getcpu.Plo@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@$(top_builddir)/../utils/$(DEPDIR)/libspindle_audit_la-fileutil.Plo@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@$(top_builddir)/../utils/$(DEPDIR)/parseloc.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@$(top_builddir)/crash_handler/$(DEPDIR)/crash_fmt.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@$(top_builddir)/crash_handler/$(DEPDIR)/crash_handler.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@$(top_builddir)/crash_handler/$(DEPDIR)/crash_io.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@$(top_builddir)/crash_handler/$(DEPDIR)/crash_lib_offset.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@$(top_builddir)/crash_handler/$(DEPDIR)/crash_sigchain.Plo@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/adjust_argv.Plo@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/client.Plo@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/exec_util.Plo@am__quote@ # am--include-marker @@ -591,6 +666,7 @@ mostlyclean-libtool: clean-libtool: -rm -rf $(top_builddir)/../utils/.libs $(top_builddir)/../utils/_libs + -rm -rf $(top_builddir)/crash_handler/.libs $(top_builddir)/crash_handler/_libs -rm -rf .libs _libs ID: $(am__tagged_files) @@ -710,6 +786,8 @@ distclean-generic: -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES) -test -z "$(top_builddir)/../utils/$(DEPDIR)/$(am__dirstamp)" || rm -f $(top_builddir)/../utils/$(DEPDIR)/$(am__dirstamp) -test -z "$(top_builddir)/../utils/$(am__dirstamp)" || rm -f $(top_builddir)/../utils/$(am__dirstamp) + -test -z "$(top_builddir)/crash_handler/$(DEPDIR)/$(am__dirstamp)" || rm -f $(top_builddir)/crash_handler/$(DEPDIR)/$(am__dirstamp) + -test -z "$(top_builddir)/crash_handler/$(am__dirstamp)" || rm -f $(top_builddir)/crash_handler/$(am__dirstamp) maintainer-clean-generic: @echo "This command is intended for maintainers to use" @@ -723,6 +801,11 @@ distclean: distclean-am -rm -f $(top_builddir)/../utils/$(DEPDIR)/getcpu.Plo -rm -f $(top_builddir)/../utils/$(DEPDIR)/libspindle_audit_la-fileutil.Plo -rm -f $(top_builddir)/../utils/$(DEPDIR)/parseloc.Plo + -rm -f $(top_builddir)/crash_handler/$(DEPDIR)/crash_fmt.Plo + -rm -f $(top_builddir)/crash_handler/$(DEPDIR)/crash_handler.Plo + -rm -f $(top_builddir)/crash_handler/$(DEPDIR)/crash_io.Plo + -rm -f $(top_builddir)/crash_handler/$(DEPDIR)/crash_lib_offset.Plo + -rm -f $(top_builddir)/crash_handler/$(DEPDIR)/crash_sigchain.Plo -rm -f ./$(DEPDIR)/adjust_argv.Plo -rm -f ./$(DEPDIR)/client.Plo -rm -f ./$(DEPDIR)/exec_util.Plo @@ -787,6 +870,11 @@ maintainer-clean: maintainer-clean-am -rm -f $(top_builddir)/../utils/$(DEPDIR)/getcpu.Plo -rm -f $(top_builddir)/../utils/$(DEPDIR)/libspindle_audit_la-fileutil.Plo -rm -f $(top_builddir)/../utils/$(DEPDIR)/parseloc.Plo + -rm -f $(top_builddir)/crash_handler/$(DEPDIR)/crash_fmt.Plo + -rm -f $(top_builddir)/crash_handler/$(DEPDIR)/crash_handler.Plo + -rm -f $(top_builddir)/crash_handler/$(DEPDIR)/crash_io.Plo + -rm -f $(top_builddir)/crash_handler/$(DEPDIR)/crash_lib_offset.Plo + -rm -f $(top_builddir)/crash_handler/$(DEPDIR)/crash_sigchain.Plo -rm -f ./$(DEPDIR)/adjust_argv.Plo -rm -f ./$(DEPDIR)/client.Plo -rm -f ./$(DEPDIR)/exec_util.Plo diff --git a/src/client/config.h.in b/src/client/config.h.in index 044ca9e1..73e75c33 100644 --- a/src/client/config.h.in +++ b/src/client/config.h.in @@ -27,6 +27,12 @@ /* Define if were using sockets for client/server communication */ #undef COMM_SOCKET +/* Enable --crash-dedup by default */ +#undef CRASH_DEDUP_ENABLED_BY_DEFAULT + +/* Define to 1 if the crash handler is built */ +#undef CRASH_HANDLER_ENABLED + /* Defined if GLIBC has an allocation bug */ #undef DTV_ALLOCATION_BUG diff --git a/src/client/configure b/src/client/configure index eb84ee07..d635b861 100755 --- a/src/client/configure +++ b/src/client/configure @@ -637,6 +637,8 @@ LTLIBOBJS LIBOBJS GLIBC_BE_DIR LIBSPINDLE_LIB_VERSION +BLD_CRASH_HANDLER_FALSE +BLD_CRASH_HANDLER_TRUE STATICFLAG fluxplugin_ldflags shell_plugindir @@ -842,6 +844,8 @@ enable_sec_munge enable_sec_keydir enable_sec_launchmon enable_sec_none +enable_crash_handler +enable_crash_dedup with_gcrypt_dir with_munge_dir with_glibc_be_dir @@ -1520,6 +1524,8 @@ Optional Features: --enable-sec-launchmon Enable LaunchmMON to propagate authentication key for connections --enable-sec-none Disable security authentication of connections + --disable-crash-handler Build the crash handler (default: enabled) + --enable-crash-dedup Enable --crash-dedup by default (default: disabled) --enable-remap-pageone Controls mmap behavior when replacing executables. Should be false on older linux systems @@ -13706,6 +13712,47 @@ if test "${enable_sec_none+set}" = set; then : fi +# Check whether --enable-crash-handler was given. +if test "${enable_crash_handler+set}" = set; then : + enableval=$enable_crash_handler; enable_crash_handler=$enableval +else + enable_crash_handler=yes +fi + + +if test "x$enable_crash_handler" != xno; then : + +$as_echo "#define CRASH_HANDLER_ENABLED 1" >>confdefs.h + +fi + if test "x$enable_crash_handler" != xno; then + BLD_CRASH_HANDLER_TRUE= + BLD_CRASH_HANDLER_FALSE='#' +else + BLD_CRASH_HANDLER_TRUE='#' + BLD_CRASH_HANDLER_FALSE= +fi + + +# Check whether --enable-crash-dedup was given. +if test "${enable_crash_dedup+set}" = set; then : + enableval=$enable_crash_dedup; enable_crash_dedup=$enableval +else + enable_crash_dedup=no +fi + + +if test "x$enable_crash_dedup" = xyes && test "x$enable_crash_handler" = xno; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: --enable-crash-dedup ignored because --disable-crash-handler was given" >&5 +$as_echo "$as_me: WARNING: --enable-crash-dedup ignored because --disable-crash-handler was given" >&2;} +fi + +if test "x$enable_crash_dedup" = xyes && test "x$enable_crash_handler" != xno; then : + +$as_echo "#define CRASH_DEDUP_ENABLED_BY_DEFAULT 1" >>confdefs.h + +fi + # Check whether --with-gcrypt-dir was given. if test "${with_gcrypt_dir+set}" = set; then : @@ -13990,6 +14037,10 @@ if test -z "${AARCH64_BLD_TRUE}" && test -z "${AARCH64_BLD_FALSE}"; then as_fn_error $? "conditional \"AARCH64_BLD\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi +if test -z "${BLD_CRASH_HANDLER_TRUE}" && test -z "${BLD_CRASH_HANDLER_FALSE}"; then + as_fn_error $? "conditional \"BLD_CRASH_HANDLER\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi : "${CONFIG_STATUS=./config.status}" ac_write_fail=0 diff --git a/src/client/crash_handler/crash_handler.c b/src/client/crash_handler/crash_handler.c index ddf05344..8647b054 100644 --- a/src/client/crash_handler/crash_handler.c +++ b/src/client/crash_handler/crash_handler.c @@ -304,13 +304,13 @@ int crash_handler_install(int global_rank, int ldcsid_in) /* If a signal handler was already installed by the time Spindle registers its signal handler, we save it to use in signal handler chaining. */ - struct sigaction kernel_old; - memset(&kernel_old, 0, sizeof kernel_old); - if (sigaction(sigs[i], &sa, &kernel_old) != 0) { - debug_printf("sigaction(%d) failed when installing crash handler\n", sigs[i]); + struct sigaction handler_old; + memset(&handler_old, 0, sizeof handler_old); + if (sigaction(sigs[i], &sa, &handler_old) != 0) { + debug_printf("sigaction failed when installing crash handler for signal %d\n", sigs[i]); continue; } - crash_sigchain_register_existing_handler(sigs[i], &kernel_old); + crash_sigchain_register_existing_handler(sigs[i], &handler_old); } crash_installed = 1; diff --git a/src/client/subaudit/Makefile.in b/src/client/subaudit/Makefile.in index 0e5d1b06..9869cec9 100644 --- a/src/client/subaudit/Makefile.in +++ b/src/client/subaudit/Makefile.in @@ -397,7 +397,7 @@ top_srcdir = @top_srcdir@ pkglib_LTLIBRARIES = libspindleint.la $(am__append_1) $(am__append_2) \ $(am__append_3) AM_CFLAGS = -fvisibility=hidden -AM_CPPFLAGS = -I$(top_srcdir)/../logging -I$(top_srcdir)/../include -I$(top_srcdir)/client -I$(top_srcdir)/client_comlib -I$(top_srcdir)/auditclient -I$(top_srcdir)/../utils +AM_CPPFLAGS = -I$(top_srcdir)/../logging -I$(top_srcdir)/../include -I$(top_srcdir)/client -I$(top_srcdir)/client_comlib -I$(top_srcdir)/auditclient -I$(top_srcdir)/../utils -I$(top_srcdir)/crash_handler BASE_SRCS = subaudit.c intercept_malloc.c update_pltbind.c ../auditclient/auditclient_common.c ../auditclient/patch_linkmap.c ../auditclient/fixlocale.c AUDITLIB = $(top_builddir)/client/libspindle_audit.la libspindleint_la_SOURCES = preloadlib.c diff --git a/src/fe/config.h.in b/src/fe/config.h.in index 615997be..315606b8 100644 --- a/src/fe/config.h.in +++ b/src/fe/config.h.in @@ -27,6 +27,12 @@ /* Define if were using sockets for client/server communication */ #undef COMM_SOCKET +/* Enable --crash-dedup by default */ +#undef CRASH_DEDUP_ENABLED_BY_DEFAULT + +/* Define to 1 if the crash handler is built */ +#undef CRASH_HANDLER_ENABLED + /* Default to using a dedicated process to clean data files */ #undef DEFAULT_CLEANUP_PROC diff --git a/src/fe/configure b/src/fe/configure index 4c11bdc7..df0d828d 100755 --- a/src/fe/configure +++ b/src/fe/configure @@ -651,6 +651,8 @@ MUNGE_LIBS MUNGE_CFLAGS GCRYPT_LIBS GCRYPT_CFLAGS +BLD_CRASH_HANDLER_FALSE +BLD_CRASH_HANDLER_TRUE STATICFLAG fluxplugin_ldflags shell_plugindir @@ -863,6 +865,8 @@ enable_sec_munge enable_sec_keydir enable_sec_launchmon enable_sec_none +enable_crash_handler +enable_crash_dedup with_gcrypt_dir with_munge_dir with_glibc_be_dir @@ -1553,6 +1557,8 @@ Optional Features: --enable-sec-launchmon Enable LaunchmMON to propagate authentication key for connections --enable-sec-none Disable security authentication of connections + --disable-crash-handler Build the crash handler (default: enabled) + --enable-crash-dedup Enable --crash-dedup by default (default: disabled) --enable-remap-pageone Controls mmap behavior when replacing executables. Should be false on older linux systems --enable-slurm Enable support for the SLURM job launcher @@ -17556,6 +17562,47 @@ if test "${enable_sec_none+set}" = set; then : fi +# Check whether --enable-crash-handler was given. +if test "${enable_crash_handler+set}" = set; then : + enableval=$enable_crash_handler; enable_crash_handler=$enableval +else + enable_crash_handler=yes +fi + + +if test "x$enable_crash_handler" != xno; then : + +$as_echo "#define CRASH_HANDLER_ENABLED 1" >>confdefs.h + +fi + if test "x$enable_crash_handler" != xno; then + BLD_CRASH_HANDLER_TRUE= + BLD_CRASH_HANDLER_FALSE='#' +else + BLD_CRASH_HANDLER_TRUE='#' + BLD_CRASH_HANDLER_FALSE= +fi + + +# Check whether --enable-crash-dedup was given. +if test "${enable_crash_dedup+set}" = set; then : + enableval=$enable_crash_dedup; enable_crash_dedup=$enableval +else + enable_crash_dedup=no +fi + + +if test "x$enable_crash_dedup" = xyes && test "x$enable_crash_handler" = xno; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: --enable-crash-dedup ignored because --disable-crash-handler was given" >&5 +$as_echo "$as_me: WARNING: --enable-crash-dedup ignored because --disable-crash-handler was given" >&2;} +fi + +if test "x$enable_crash_dedup" = xyes && test "x$enable_crash_handler" != xno; then : + +$as_echo "#define CRASH_DEDUP_ENABLED_BY_DEFAULT 1" >>confdefs.h + +fi + # Check whether --with-gcrypt-dir was given. if test "${with_gcrypt_dir+set}" = set; then : @@ -18374,6 +18421,10 @@ if test -z "${AARCH64_BLD_TRUE}" && test -z "${AARCH64_BLD_FALSE}"; then as_fn_error $? "conditional \"AARCH64_BLD\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi +if test -z "${BLD_CRASH_HANDLER_TRUE}" && test -z "${BLD_CRASH_HANDLER_FALSE}"; then + as_fn_error $? "conditional \"BLD_CRASH_HANDLER\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi if test -z "${LMON_TRUE}" && test -z "${LMON_FALSE}"; then as_fn_error $? "conditional \"LMON\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 diff --git a/src/server/config.h.in b/src/server/config.h.in index 24040627..4b227ec0 100644 --- a/src/server/config.h.in +++ b/src/server/config.h.in @@ -27,6 +27,12 @@ /* Define if were using sockets for client/server communication */ #undef COMM_SOCKET +/* Enable --crash-dedup by default */ +#undef CRASH_DEDUP_ENABLED_BY_DEFAULT + +/* Define to 1 if the crash handler is built */ +#undef CRASH_HANDLER_ENABLED + /* Defined if GLIBC has an allocation bug */ #undef DTV_ALLOCATION_BUG diff --git a/src/server/configure b/src/server/configure index 83bff908..78f8a828 100755 --- a/src/server/configure +++ b/src/server/configure @@ -657,6 +657,8 @@ MUNGE_LIBS MUNGE_CFLAGS GCRYPT_LIBS GCRYPT_CFLAGS +BLD_CRASH_HANDLER_FALSE +BLD_CRASH_HANDLER_TRUE STATICFLAG fluxplugin_ldflags shell_plugindir @@ -869,6 +871,8 @@ enable_sec_munge enable_sec_keydir enable_sec_launchmon enable_sec_none +enable_crash_handler +enable_crash_dedup with_gcrypt_dir with_munge_dir with_glibc_be_dir @@ -1555,6 +1559,8 @@ Optional Features: --enable-sec-launchmon Enable LaunchmMON to propagate authentication key for connections --enable-sec-none Disable security authentication of connections + --disable-crash-handler Build the crash handler (default: enabled) + --enable-crash-dedup Enable --crash-dedup by default (default: disabled) --enable-remap-pageone Controls mmap behavior when replacing executables. Should be false on older linux systems @@ -17553,6 +17559,47 @@ if test "${enable_sec_none+set}" = set; then : fi +# Check whether --enable-crash-handler was given. +if test "${enable_crash_handler+set}" = set; then : + enableval=$enable_crash_handler; enable_crash_handler=$enableval +else + enable_crash_handler=yes +fi + + +if test "x$enable_crash_handler" != xno; then : + +$as_echo "#define CRASH_HANDLER_ENABLED 1" >>confdefs.h + +fi + if test "x$enable_crash_handler" != xno; then + BLD_CRASH_HANDLER_TRUE= + BLD_CRASH_HANDLER_FALSE='#' +else + BLD_CRASH_HANDLER_TRUE='#' + BLD_CRASH_HANDLER_FALSE= +fi + + +# Check whether --enable-crash-dedup was given. +if test "${enable_crash_dedup+set}" = set; then : + enableval=$enable_crash_dedup; enable_crash_dedup=$enableval +else + enable_crash_dedup=no +fi + + +if test "x$enable_crash_dedup" = xyes && test "x$enable_crash_handler" = xno; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: --enable-crash-dedup ignored because --disable-crash-handler was given" >&5 +$as_echo "$as_me: WARNING: --enable-crash-dedup ignored because --disable-crash-handler was given" >&2;} +fi + +if test "x$enable_crash_dedup" = xyes && test "x$enable_crash_handler" != xno; then : + +$as_echo "#define CRASH_DEDUP_ENABLED_BY_DEFAULT 1" >>confdefs.h + +fi + # Check whether --with-gcrypt-dir was given. if test "${with_gcrypt_dir+set}" = set; then : @@ -18418,6 +18465,10 @@ if test -z "${AARCH64_BLD_TRUE}" && test -z "${AARCH64_BLD_FALSE}"; then as_fn_error $? "conditional \"AARCH64_BLD\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi +if test -z "${BLD_CRASH_HANDLER_TRUE}" && test -z "${BLD_CRASH_HANDLER_FALSE}"; then + as_fn_error $? "conditional \"BLD_CRASH_HANDLER\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi if test -z "${LMON_TRUE}" && test -z "${LMON_FALSE}"; then as_fn_error $? "conditional \"LMON\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 diff --git a/testsuite/Makefile.in b/testsuite/Makefile.in index 290fe9a1..3a0b8e13 100644 --- a/testsuite/Makefile.in +++ b/testsuite/Makefile.in @@ -90,6 +90,7 @@ build_triplet = @build@ host_triplet = @host@ target_triplet = @target@ noinst_PROGRAMS = libgenerator$(EXEEXT) +@BLD_CRASH_HANDLER_TRUE@am__append_1 = crash_test libcrashfuncs.so libcrashctor.so run_crash_tests.sh subdir = testsuite ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 am__aclocal_m4_deps = $(top_srcdir)/m4/libtool.m4 \ @@ -358,7 +359,8 @@ BUILT_SOURCES = libtest10.so libtest11.so libtest12.so libtest13.so \ spindle_deactivated.sh liblocal.so symbind_test \ interpreter_test interpreter_test_dir/interpreter_test_perl \ alias/aliastest.py exec_shell.sh exec_shell.tcsh \ - exec_shell_expected_output exec_shell exec_shell_env.sh + exec_shell_expected_output exec_shell exec_shell_env.sh \ + $(am__append_1) @BGQ_BLD_FALSE@DYNAMIC_FLAG = @BGQ_BLD_TRUE@DYNAMIC_FLAG = -dynamic @BGQ_BLD_FALSE@IS_BLUEGENE = false @@ -374,6 +376,7 @@ test_driver_libsLDADD = -ltest10 -ltest11 -ltest12 -ltest13 -ltest14 -ltest15 -l test_driver_libsLDFLAGS = -Wl,-E -L$(top_builddir)/testsuite $(MPI_CLDFLAGS) -L$(top_builddir)/src/client/spindle_api -L. $(DYNAMIC_FLAG) -no-install $(LDFLAGS) -Wl,-rpath,$(PWD)/origin_dir -L$(PWD)/origin_dir -Wl,-rpath-link,$(PWD)/origin_dir/origin_subdir -I$(top_builddir) REGLIB_SRC = $(srcdir)/registerlib.c LD_FUNCDICT = -L$(top_builddir)/testsuite -lfuncdict +EXTRA_DIST = run_crash_tests_template.sh crash_test.c crash_functions.c crash_functions.h libcrashfuncs.c libcrashctor.c CLEANFILES = libtest10.c libtest11.c libtest12.c libtest13.c \ libtest14.c libtest15.c libtest16.c libtest17.c libtest18.c \ libtest19.c libtest20.c libtest10.so libtest50.c libtest50.so \ @@ -400,7 +403,8 @@ CLEANFILES = libtest10.c libtest11.c libtest12.c libtest13.c \ libsymbind_g.so interpreter_test \ interpreter_test_dir/interpreter_test_perl alias exec_shell.sh \ exec_shell.tcsh exec_shell_env.sh exec_shell_expected_output \ - exec_shell + exec_shell crash_test libcrashfuncs.so libcrashctor.so \ + run_crash_tests.sh all: $(BUILT_SOURCES) $(MAKE) $(AM_MAKEFLAGS) all-am @@ -1100,6 +1104,21 @@ interpreter_test_dir/interpreter_test_perl: interpreter_test_perl_template inter interpreter_test: interpreter_test_template $(AM_V_GEN)cp $< $@; chmod 700 $@ +crash_test: $(srcdir)/crash_test.c $(srcdir)/crash_functions.c $(srcdir)/crash_functions.h libcrashfuncs.so libcrashctor.so + $(AM_V_CCLD)$(MPICC) $(CFLAGS) -O0 -fno-omit-frame-pointer -no-pie -Wl,-E \ + -o $@ $(srcdir)/crash_test.c $(srcdir)/crash_functions.c -I$(srcdir) -ldl -lpthread + +libcrashfuncs.so: $(srcdir)/libcrashfuncs.c + $(AM_V_CCLD)$(CC) $(CFLAGS) -O0 -fno-omit-frame-pointer -shared -fPIC \ + -o $@ $(srcdir)/libcrashfuncs.c + +libcrashctor.so: $(srcdir)/libcrashctor.c + $(AM_V_CCLD)$(CC) $(CFLAGS) -O0 -fno-omit-frame-pointer -shared -fPIC \ + -o $@ $(srcdir)/libcrashctor.c + +run_crash_tests.sh: $(srcdir)/run_crash_tests_template.sh $(top_builddir)/Makefile + $(AM_V_GEN)$(SED) -e s,TEST_RESOURCE_MANAGER,$(TESTRM),g\;s,SPINDLE_EXEC,$(bindir)/spindle,g\;s,SPINDLE_RC_PATH,$(sysconfdir)/spindle/spindle.rc,g\;s,TEST_RUN_DIR,$(ABS_TEST_DIR),g < $< > $@; chmod 700 $@ + exec_shell.sh: $(srcdir)/exec_shell_template.sh $(AM_V_GEN)cp $< $@; chmod 700 $@ From 1bbc570219ac39ebda8e98be46a45ff94e95aeed Mon Sep 17 00:00:00 2001 From: Nicholas Chaimov Date: Thu, 18 Jun 2026 22:14:33 -0700 Subject: [PATCH 09/14] Install glibc debug symbols in CI containers --- containers/spindle-flux-ubuntu/Dockerfile | 1 + containers/spindle-serial-ubuntu/Dockerfile | 1 + containers/spindle-slurm-ubuntu/testing-plugin/Dockerfile | 3 ++- containers/spindle-slurm-ubuntu/testing/Dockerfile | 3 ++- 4 files changed, 6 insertions(+), 2 deletions(-) diff --git a/containers/spindle-flux-ubuntu/Dockerfile b/containers/spindle-flux-ubuntu/Dockerfile index 9bfd7944..01cd3625 100644 --- a/containers/spindle-flux-ubuntu/Dockerfile +++ b/containers/spindle-flux-ubuntu/Dockerfile @@ -27,6 +27,7 @@ RUN DEBIAN_FRONTEND="noninteractive" apt-get update \ libgcrypt20 \ libgcrypt20-dev \ gdb \ + libc6-dbg \ software-properties-common ARG USER=fluxuser diff --git a/containers/spindle-serial-ubuntu/Dockerfile b/containers/spindle-serial-ubuntu/Dockerfile index 9fa1b53e..3573f243 100644 --- a/containers/spindle-serial-ubuntu/Dockerfile +++ b/containers/spindle-serial-ubuntu/Dockerfile @@ -28,6 +28,7 @@ RUN DEBIAN_FRONTEND="noninteractive" apt-get -qq install -y --no-install-recomme gcc-13 \ g++-13 \ gdb \ + libc6-dbg \ munge \ libmunge-dev \ libhwloc-dev \ diff --git a/containers/spindle-slurm-ubuntu/testing-plugin/Dockerfile b/containers/spindle-slurm-ubuntu/testing-plugin/Dockerfile index 0e7774e3..a5d95356 100644 --- a/containers/spindle-slurm-ubuntu/testing-plugin/Dockerfile +++ b/containers/spindle-slurm-ubuntu/testing-plugin/Dockerfile @@ -10,7 +10,8 @@ ARG BUILD_ROOT=containers/spindle-slurm-ubuntu/testing-plugin RUN apt-get update \ && DEBIAN_FRONTEND="noninteractive" apt-get -qq install -y --no-install-recommends \ - gdb + gdb \ + libc6-dbg # Slurm daemons run as $SLURM_USER ARG SLURM_USER=slurm diff --git a/containers/spindle-slurm-ubuntu/testing/Dockerfile b/containers/spindle-slurm-ubuntu/testing/Dockerfile index b73616f8..2d0c6202 100644 --- a/containers/spindle-slurm-ubuntu/testing/Dockerfile +++ b/containers/spindle-slurm-ubuntu/testing/Dockerfile @@ -10,7 +10,8 @@ ARG BUILD_ROOT=containers/spindle-slurm-ubuntu/testing RUN apt-get update \ && DEBIAN_FRONTEND="noninteractive" apt-get -qq install -y --no-install-recommends \ - gdb + gdb \ + libc6-dbg # Slurm daemons run as $SLURM_USER ARG SLURM_USER=slurm From e08c537d9899d5a34cf9243fa37b391313bef960 Mon Sep 17 00:00:00 2001 From: Nicholas Chaimov Date: Mon, 29 Jun 2026 16:44:31 -0700 Subject: [PATCH 10/14] Add program header crash tests --- testsuite/Makefile.am | 24 +++++- testsuite/Makefile.in | 25 +++++- testsuite/crash_test.c | 36 ++++++++- testsuite/libcrashfixed.c | 24 ++++++ testsuite/run_crash_tests_template.sh | 109 ++++++++++++++++++++++---- 5 files changed, 194 insertions(+), 24 deletions(-) create mode 100644 testsuite/libcrashfixed.c diff --git a/testsuite/Makefile.am b/testsuite/Makefile.am index 57929a07..70943797 100644 --- a/testsuite/Makefile.am +++ b/testsuite/Makefile.am @@ -4,7 +4,7 @@ ABS_TEST_DIR = $(abspath $(top_builddir)/testsuite) BUILT_SOURCES = libtest10.so libtest11.so libtest12.so libtest13.so libtest14.so libtest15.so libtest16.so libtest17.so libtest18.so libtest19.so libtest20.so libtest50.so libtest100.so libtest500.so libtest1000.so libtest2000.so libtest4000.so libtest6000.so libtest8000.so libtest10000.so libtls1.c libtls2.c libtls3.c libtls4.c libtls5.c libtls6.c libtls7.c libtls8.c libtls9.c libtls10.c libtls11.c libtls12.c libtls13.c libtls14.c libtls15.c libtls16.c libtls17.c libtls18.c libtls19.c libtls20.c libsymlink.so libdepC.so libdepB.so libdepA.so libcxxexceptB.so libcxxexceptA.so origin_dir/liboriginlib.so origin_dir/origin_subdir/liborigintarget.so libtestoutput.so libfuncdict.so runTests run_driver run_driver_rm spindle.rc preload_file_list test_driver test_driver_libs retzero_rx retzero_r retzero_x retzero_ badinterp hello_r.py hello_x.py hello_rx.py hello_.py hello_l.py badlink.py spindle_exec_test spindle_deactivated.sh liblocal.so symbind_test interpreter_test interpreter_test_dir/interpreter_test_perl alias/aliastest.py exec_shell.sh exec_shell.tcsh exec_shell_expected_output exec_shell exec_shell_env.sh if BLD_CRASH_HANDLER -BUILT_SOURCES += crash_test libcrashfuncs.so libcrashctor.so run_crash_tests.sh +BUILT_SOURCES += crash_test crash_test_fixedaddr crash_test_pie libcrashfuncs.so libcrashctor.so libcrashfixed.so run_crash_tests.sh endif if BGQ_BLD @@ -424,10 +424,20 @@ interpreter_test_dir/interpreter_test_perl: interpreter_test_perl_template inter interpreter_test: interpreter_test_template $(AM_V_GEN)cp $< $@; chmod 700 $@ -crash_test: $(srcdir)/crash_test.c $(srcdir)/crash_functions.c $(srcdir)/crash_functions.h libcrashfuncs.so libcrashctor.so +crash_test: $(srcdir)/crash_test.c $(srcdir)/crash_functions.c $(srcdir)/crash_functions.h libcrashfuncs.so libcrashctor.so libcrashfixed.so $(AM_V_CCLD)$(MPICC) $(CFLAGS) -O0 -fno-omit-frame-pointer -no-pie -Wl,-E \ -o $@ $(srcdir)/crash_test.c $(srcdir)/crash_functions.c -I$(srcdir) -ldl -lpthread +CRASH_FIXED_EXE_ADDR ?= 0x60000000 +crash_test_fixedaddr: $(srcdir)/crash_test.c $(srcdir)/crash_functions.c $(srcdir)/crash_functions.h + $(AM_V_CCLD)$(MPICC) $(CFLAGS) -O0 -fno-omit-frame-pointer -no-pie -Wl,-E \ + -Wl,-Ttext-segment=$(CRASH_FIXED_EXE_ADDR) \ + -o $@ $(srcdir)/crash_test.c $(srcdir)/crash_functions.c -I$(srcdir) -ldl -lpthread + +crash_test_pie: $(srcdir)/crash_test.c $(srcdir)/crash_functions.c $(srcdir)/crash_functions.h + $(AM_V_CCLD)$(MPICC) $(CFLAGS) -O0 -fno-omit-frame-pointer -pie -fPIE -Wl,-E \ + -o $@ $(srcdir)/crash_test.c $(srcdir)/crash_functions.c -I$(srcdir) -ldl -lpthread + libcrashfuncs.so: $(srcdir)/libcrashfuncs.c $(AM_V_CCLD)$(CC) $(CFLAGS) -O0 -fno-omit-frame-pointer -shared -fPIC \ -o $@ $(srcdir)/libcrashfuncs.c @@ -436,10 +446,16 @@ libcrashctor.so: $(srcdir)/libcrashctor.c $(AM_V_CCLD)$(CC) $(CFLAGS) -O0 -fno-omit-frame-pointer -shared -fPIC \ -o $@ $(srcdir)/libcrashctor.c +CRASH_FIXED_LIB_ADDR ?= 0x10000000 +libcrashfixed.so: $(srcdir)/libcrashfixed.c + $(AM_V_CCLD)$(CC) $(CFLAGS) -O0 -fno-omit-frame-pointer -shared -fPIC \ + -Wl,-Ttext-segment=$(CRASH_FIXED_LIB_ADDR) \ + -o $@ $(srcdir)/libcrashfixed.c + run_crash_tests.sh: $(srcdir)/run_crash_tests_template.sh $(top_builddir)/Makefile $(AM_V_GEN)$(SED) -e s,TEST_RESOURCE_MANAGER,$(TESTRM),g\;s,SPINDLE_EXEC,$(bindir)/spindle,g\;s,SPINDLE_RC_PATH,$(sysconfdir)/spindle/spindle.rc,g\;s,TEST_RUN_DIR,$(ABS_TEST_DIR),g < $< > $@; chmod 700 $@ -EXTRA_DIST = run_crash_tests_template.sh crash_test.c crash_functions.c crash_functions.h libcrashfuncs.c libcrashctor.c +EXTRA_DIST = run_crash_tests_template.sh crash_test.c crash_functions.c crash_functions.h libcrashfuncs.c libcrashctor.c libcrashfixed.c exec_shell.sh: $(srcdir)/exec_shell_template.sh $(AM_V_GEN)cp $< $@; chmod 700 $@ @@ -456,4 +472,4 @@ exec_shell_expected_output: $(srcdir)/exec_shell_expected_output_template exec_shell: $(srcdir)/exec_shell.c $(AM_V_CCLD)$(CC) $(CFLAGS) -o $@ $< -CLEANFILES = libtest10.c libtest11.c libtest12.c libtest13.c libtest14.c libtest15.c libtest16.c libtest17.c libtest18.c libtest19.c libtest20.c libtest10.so libtest50.c libtest50.so libtest100.c libtest100.so libtest500.c libtest500.so libtest1000.c libtest1000.so libtest2000.c libtest2000.so libtest4000.c libtest4000.so libtest6000.c libtest6000.so libtest8000.c libtest8000.so libtest10000.c libtest10000.so libsymlink.so libdepA.so libdepB.so libdepC.so libcxxexceptA.so libcxxexceptB.so libtestoutput.so libfuncdict.so runTests run_driver run_driver_rm spindle.rc test_driver test_driver_libs preload_file_list retzero_rx retzero_r retzero_x retzero_ badinterp hello_r.py hello_x.py hello_rx.py hello_.py hello_l.py badlink.py libtls1.c libtls2.c libtls3.c libtls4.c libtls5.c libtls6.c libtls7.c libtls8.c libtls9.c libtls10.c libtls11.c libtls12.c libtls13.c libtls14.c libtls15.c libtls16.c libtls17.c libtls18.c libtls19.c libtls20.c libtls1.so libtls2.so libtls3.so libtls4.so libtls5.so libtls6.so libtls7.so libtls8.so libtls9.so libtls10.so libtls11.so libtls12.so libtls13.so libtls14.so libtls15.so libtls16.so libtls17.so libtls18.so libtls19.so libtls20.so symbind_test libsymbind_a.so libsymbind_b.so libsymbind_c.so libsymbind_d.so libsymbind_e.so libsymbind_f.so libsymbind_g.so interpreter_test interpreter_test_dir/interpreter_test_perl alias exec_shell.sh exec_shell.tcsh exec_shell_env.sh exec_shell_expected_output exec_shell crash_test libcrashfuncs.so libcrashctor.so run_crash_tests.sh +CLEANFILES = libtest10.c libtest11.c libtest12.c libtest13.c libtest14.c libtest15.c libtest16.c libtest17.c libtest18.c libtest19.c libtest20.c libtest10.so libtest50.c libtest50.so libtest100.c libtest100.so libtest500.c libtest500.so libtest1000.c libtest1000.so libtest2000.c libtest2000.so libtest4000.c libtest4000.so libtest6000.c libtest6000.so libtest8000.c libtest8000.so libtest10000.c libtest10000.so libsymlink.so libdepA.so libdepB.so libdepC.so libcxxexceptA.so libcxxexceptB.so libtestoutput.so libfuncdict.so runTests run_driver run_driver_rm spindle.rc test_driver test_driver_libs preload_file_list retzero_rx retzero_r retzero_x retzero_ badinterp hello_r.py hello_x.py hello_rx.py hello_.py hello_l.py badlink.py libtls1.c libtls2.c libtls3.c libtls4.c libtls5.c libtls6.c libtls7.c libtls8.c libtls9.c libtls10.c libtls11.c libtls12.c libtls13.c libtls14.c libtls15.c libtls16.c libtls17.c libtls18.c libtls19.c libtls20.c libtls1.so libtls2.so libtls3.so libtls4.so libtls5.so libtls6.so libtls7.so libtls8.so libtls9.so libtls10.so libtls11.so libtls12.so libtls13.so libtls14.so libtls15.so libtls16.so libtls17.so libtls18.so libtls19.so libtls20.so symbind_test libsymbind_a.so libsymbind_b.so libsymbind_c.so libsymbind_d.so libsymbind_e.so libsymbind_f.so libsymbind_g.so interpreter_test interpreter_test_dir/interpreter_test_perl alias exec_shell.sh exec_shell.tcsh exec_shell_env.sh exec_shell_expected_output exec_shell crash_test crash_test_fixedaddr crash_test_pie libcrashfuncs.so libcrashctor.so libcrashfixed.so run_crash_tests.sh diff --git a/testsuite/Makefile.in b/testsuite/Makefile.in index 3a0b8e13..6ab4fcfc 100644 --- a/testsuite/Makefile.in +++ b/testsuite/Makefile.in @@ -90,7 +90,7 @@ build_triplet = @build@ host_triplet = @host@ target_triplet = @target@ noinst_PROGRAMS = libgenerator$(EXEEXT) -@BLD_CRASH_HANDLER_TRUE@am__append_1 = crash_test libcrashfuncs.so libcrashctor.so run_crash_tests.sh +@BLD_CRASH_HANDLER_TRUE@am__append_1 = crash_test crash_test_fixedaddr crash_test_pie libcrashfuncs.so libcrashctor.so libcrashfixed.so run_crash_tests.sh subdir = testsuite ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 am__aclocal_m4_deps = $(top_srcdir)/m4/libtool.m4 \ @@ -376,7 +376,7 @@ test_driver_libsLDADD = -ltest10 -ltest11 -ltest12 -ltest13 -ltest14 -ltest15 -l test_driver_libsLDFLAGS = -Wl,-E -L$(top_builddir)/testsuite $(MPI_CLDFLAGS) -L$(top_builddir)/src/client/spindle_api -L. $(DYNAMIC_FLAG) -no-install $(LDFLAGS) -Wl,-rpath,$(PWD)/origin_dir -L$(PWD)/origin_dir -Wl,-rpath-link,$(PWD)/origin_dir/origin_subdir -I$(top_builddir) REGLIB_SRC = $(srcdir)/registerlib.c LD_FUNCDICT = -L$(top_builddir)/testsuite -lfuncdict -EXTRA_DIST = run_crash_tests_template.sh crash_test.c crash_functions.c crash_functions.h libcrashfuncs.c libcrashctor.c +EXTRA_DIST = run_crash_tests_template.sh crash_test.c crash_functions.c crash_functions.h libcrashfuncs.c libcrashctor.c libcrashfixed.c CLEANFILES = libtest10.c libtest11.c libtest12.c libtest13.c \ libtest14.c libtest15.c libtest16.c libtest17.c libtest18.c \ libtest19.c libtest20.c libtest10.so libtest50.c libtest50.so \ @@ -403,7 +403,8 @@ CLEANFILES = libtest10.c libtest11.c libtest12.c libtest13.c \ libsymbind_g.so interpreter_test \ interpreter_test_dir/interpreter_test_perl alias exec_shell.sh \ exec_shell.tcsh exec_shell_env.sh exec_shell_expected_output \ - exec_shell crash_test libcrashfuncs.so libcrashctor.so \ + exec_shell crash_test crash_test_fixedaddr crash_test_pie \ + libcrashfuncs.so libcrashctor.so libcrashfixed.so \ run_crash_tests.sh all: $(BUILT_SOURCES) $(MAKE) $(AM_MAKEFLAGS) all-am @@ -1104,10 +1105,20 @@ interpreter_test_dir/interpreter_test_perl: interpreter_test_perl_template inter interpreter_test: interpreter_test_template $(AM_V_GEN)cp $< $@; chmod 700 $@ -crash_test: $(srcdir)/crash_test.c $(srcdir)/crash_functions.c $(srcdir)/crash_functions.h libcrashfuncs.so libcrashctor.so +crash_test: $(srcdir)/crash_test.c $(srcdir)/crash_functions.c $(srcdir)/crash_functions.h libcrashfuncs.so libcrashctor.so libcrashfixed.so $(AM_V_CCLD)$(MPICC) $(CFLAGS) -O0 -fno-omit-frame-pointer -no-pie -Wl,-E \ -o $@ $(srcdir)/crash_test.c $(srcdir)/crash_functions.c -I$(srcdir) -ldl -lpthread +CRASH_FIXED_EXE_ADDR ?= 0x60000000 +crash_test_fixedaddr: $(srcdir)/crash_test.c $(srcdir)/crash_functions.c $(srcdir)/crash_functions.h + $(AM_V_CCLD)$(MPICC) $(CFLAGS) -O0 -fno-omit-frame-pointer -no-pie -Wl,-E \ + -Wl,-Ttext-segment=$(CRASH_FIXED_EXE_ADDR) \ + -o $@ $(srcdir)/crash_test.c $(srcdir)/crash_functions.c -I$(srcdir) -ldl -lpthread + +crash_test_pie: $(srcdir)/crash_test.c $(srcdir)/crash_functions.c $(srcdir)/crash_functions.h + $(AM_V_CCLD)$(MPICC) $(CFLAGS) -O0 -fno-omit-frame-pointer -pie -fPIE -Wl,-E \ + -o $@ $(srcdir)/crash_test.c $(srcdir)/crash_functions.c -I$(srcdir) -ldl -lpthread + libcrashfuncs.so: $(srcdir)/libcrashfuncs.c $(AM_V_CCLD)$(CC) $(CFLAGS) -O0 -fno-omit-frame-pointer -shared -fPIC \ -o $@ $(srcdir)/libcrashfuncs.c @@ -1116,6 +1127,12 @@ libcrashctor.so: $(srcdir)/libcrashctor.c $(AM_V_CCLD)$(CC) $(CFLAGS) -O0 -fno-omit-frame-pointer -shared -fPIC \ -o $@ $(srcdir)/libcrashctor.c +CRASH_FIXED_LIB_ADDR ?= 0x10000000 +libcrashfixed.so: $(srcdir)/libcrashfixed.c + $(AM_V_CCLD)$(CC) $(CFLAGS) -O0 -fno-omit-frame-pointer -shared -fPIC \ + -Wl,-Ttext-segment=$(CRASH_FIXED_LIB_ADDR) \ + -o $@ $(srcdir)/libcrashfixed.c + run_crash_tests.sh: $(srcdir)/run_crash_tests_template.sh $(top_builddir)/Makefile $(AM_V_GEN)$(SED) -e s,TEST_RESOURCE_MANAGER,$(TESTRM),g\;s,SPINDLE_EXEC,$(bindir)/spindle,g\;s,SPINDLE_RC_PATH,$(sysconfdir)/spindle/spindle.rc,g\;s,TEST_RUN_DIR,$(ABS_TEST_DIR),g < $< > $@; chmod 700 $@ diff --git a/testsuite/crash_test.c b/testsuite/crash_test.c index f49a9f0b..2c535c18 100644 --- a/testsuite/crash_test.c +++ b/testsuite/crash_test.c @@ -30,6 +30,12 @@ Place, Suite 330, Boston, MA 02111-1307 USA * crash_in_library() * in-dlmopen-library * every rank dlmopen()s libcrashfuncs.so and calls crash_in_library() + * in-fixed-library + * every rank dlopens() a library with a fixed load address + * and calls a crashing function + * in-fixed-dlmopen-library + * every rank dlmopen()s a fixed-load-address library into a new + * namespace and calls a crashing function * in-library-ctor rank 0 dlopen()s libcrashctor.so, whose constructor * crashes inside dlopen; others exit cleanly * span-read no app handler; a read spans two pages (PROT_READ then @@ -94,7 +100,8 @@ static void usage(const char *prog) { fprintf(stderr, "usage: %s --crash-mode {all-same|all-different|partial|" "late-straggler|two-groups|one-crashes|in-library|" - "in-dlmopen-library|in-library-ctor|sigabrt|assert|" + "in-dlmopen-library|in-fixed-library|in-fixed-dlmopen-library|" + "in-library-ctor|sigabrt|assert|" "long-assert|mixed-abort-segv|span-read|" "safepoint|safepoint-then-crash|" "safepoint-bad|safepoint-bad-write|safepoint-fix-write|" @@ -163,6 +170,29 @@ static void call_crash_in_library_common(int rank, int use_dlmopen) { fn(rank); } +static void call_crash_in_fixed_library_common(int rank, int use_dlmopen) { + const char *dlopen_name = use_dlmopen ? "dlmopen" : "dlopen"; + void *h = use_dlmopen ? dlmopen(LM_ID_NEWLM, "./libcrashfixed.so", RTLD_NOW) + : dlopen("./libcrashfixed.so", RTLD_NOW); + if (!h) + h = use_dlmopen ? dlmopen(LM_ID_NEWLM, "libcrashfixed.so", RTLD_NOW) + : dlopen("libcrashfixed.so", RTLD_NOW); + if (!h) { + fprintf(stderr, "rank=%d %s of libcrashfixed.so failed: %s\n", + rank, dlopen_name, dlerror()); + MPI_Abort(MPI_COMM_WORLD, 3); + return; + } + void (*fn)(int) = (void (*)(int)) dlsym(h, "crash_in_fixed_library"); + if (!fn) { + fprintf(stderr, "rank=%d dlsym of crash_in_fixed_library failed: %s\n", + rank, dlerror()); + MPI_Abort(MPI_COMM_WORLD, 3); + return; + } + fn(rank); +} + static void crash_in_library_ctor(int rank) { if (rank != 0) { fprintf(stderr, "rank=%d exiting cleanly\n", rank); @@ -654,6 +684,10 @@ int main(int argc, char **argv) { call_crash_in_library_common(rank, 0); } else if (strcmp(mode, "in-dlmopen-library") == 0) { call_crash_in_library_common(rank, 1); + } else if (strcmp(mode, "in-fixed-library") == 0) { + call_crash_in_fixed_library_common(rank, 0); + } else if (strcmp(mode, "in-fixed-dlmopen-library") == 0) { + call_crash_in_fixed_library_common(rank, 1); } else if (strcmp(mode, "in-library-ctor") == 0) { crash_in_library_ctor(rank); } else if (strcmp(mode, "sigabrt") == 0) { diff --git a/testsuite/libcrashfixed.c b/testsuite/libcrashfixed.c new file mode 100644 index 00000000..a48a6f32 --- /dev/null +++ b/testsuite/libcrashfixed.c @@ -0,0 +1,24 @@ +/* +This file is part of Spindle. For copyright information see the COPYRIGHT +file in the top level directory, or at +https://github.com/hpc/Spindle/blob/master/COPYRIGHT + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU Lesser General Public License (as published by the Free Software +Foundation) version 2.1 dated February 1999. This program is distributed in the +hope that it will be useful, but WITHOUT ANY WARRANTY; without even the IMPLIED +WARRANTY OF MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms +and conditions of the GNU Lesser General Public License for more details. You should +have received a copy of the GNU Lesser General Public License along with this +program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA +*/ + +#define _GNU_SOURCE + +__attribute__((noinline, visibility("default"))) +void crash_in_fixed_library(int rank) { + volatile int *p = (int *) 0; + (void) rank; + *p = 0; +} diff --git a/testsuite/run_crash_tests_template.sh b/testsuite/run_crash_tests_template.sh index 7b53a8d2..e366698c 100755 --- a/testsuite/run_crash_tests_template.sh +++ b/testsuite/run_crash_tests_template.sh @@ -27,19 +27,27 @@ die() { echo "FAIL: $*" >&2; exit 1; } # top_frame_regex: regex that should match the top frame in produced coredumps. # Note that all threads will be checked, so in multithreaded examples the regex should # also match anything that could be on threads other than the one that faulted. +# site_regex: optional regex for the crash-site dedup key; if present, the crash site key must +# match the regex for the test to pass +# binary: optional alternate executable to run in place of the default crash_test. +# crash_mode: optional alternate crash mode argument to executable CRASH_TESTS=( -# mode ; cores ; flags ; top_frame_regex - 'all-same ; 1 ; ; crash_function_A' +# mode ; cores ; flags ; top_frame_regex ; site_regex ; binary ; crash_mode + 'all-same ; 1 ; ; crash_function_A ; crash_test\+0x' + 'fixed-address-exe ; 1 ; ; crash_function_A ; crash_test_fixedaddr\+0x ; crash_test_fixedaddr ; all-same' + 'pie-exe ; 1 ; ; crash_function_A ; crash_test_pie\+0x ; crash_test_pie ; all-same' 'all-different ; N ; ; crash_function_[0-9]+' 'two-groups ; 2 ; ; crash_function_(A|B)' 'one-crashes ; 1 ; ; crash_function_A' 'partial ; 1 ; ; crash_function_A' 'late-straggler ; 1 ; ; crash_function_A' - 'in-library ; 1 ; ; crash_in_library' - 'in-dlmopen-library ; 1 ; ; crash_in_library' + 'in-library ; 1 ; ; crash_in_library ; libcrashfuncs\.so\+0x' + 'in-dlmopen-library ; 1 ; ; crash_in_library ; libcrashfuncs\.so\+0x' + 'in-fixed-library ; 1 ; ; crash_in_fixed_library ; libcrashfixed\.so\+0x' + 'in-fixed-dlmopen-library ; 1 ; ; crash_in_fixed_library ; libcrashfixed\.so\+0x' 'in-library-ctor ; 1 ; ; ctor_crash' 'sigabrt ; 1 ; ; (__GI_)?raise|abort|pthread_kill' - 'assert ; 1 ; ; (__GI_)?raise|abort|pthread_kill' + 'assert ; 1 ; ; (__GI_)?raise|abort|pthread_kill ; abort:.*Assertion' 'mixed-abort-segv ; 2 ; multi-rank ; (__GI_)?raise|abort|pthread_kill|do_mixed_abort_segv' 'span-read ; 1 ; ; do_span_read' 'safepoint ; 0 ; clean ; -' @@ -55,7 +63,7 @@ CRASH_TESTS=( 'no-crash ; 0 ; clean ; -' ) -declare -A TEST_CORES TEST_FLAGS TEST_TOPFRAME +declare -A TEST_CORES TEST_FLAGS TEST_TOPFRAME TEST_SITE TEST_BINARY TEST_CRASHMODE DEFAULT_MODES=() trim() { @@ -66,17 +74,25 @@ trim() { } parse_table() { - local row mode cores flags top + local row mode cores flags top site bin cmode for row in "${CRASH_TESTS[@]}"; do - IFS=';' read -r mode cores flags top <<<"$row" + IFS=';' read -r mode cores flags top site bin cmode <<<"$row" mode=$(trim "$mode") [ -n "$mode" ] || continue cores=$(trim "$cores") flags=$(trim "$flags") top=$(trim "$top") + site=$(trim "$site") + bin=$(trim "$bin") + cmode=$(trim "$cmode") + [ -n "$bin" ] || bin="crash_test" + [ -n "$cmode" ] || cmode="$mode" TEST_CORES[$mode]="$cores" TEST_FLAGS[$mode]="$flags" TEST_TOPFRAME[$mode]="$top" + TEST_SITE[$mode]="$site" + TEST_BINARY[$mode]="$bin" + TEST_CRASHMODE[$mode]="$cmode" DEFAULT_MODES+=("$mode") done } @@ -170,12 +186,17 @@ check_prereqs() { die "--nodes must be a positive integer (was '$NODES')" fi - test -x "$TESTDIR/crash_test" || die "can't find crash test executable" - test -f "$TESTDIR/libcrashfuncs.so" || die "can't find libcrashfuncs.so" + test -x "$TESTDIR/crash_test" || die "can't find crash test executable" + test -x "$TESTDIR/crash_test_fixedaddr" || die "can't find crash_test_fixedaddr executable" + test -x "$TESTDIR/crash_test_pie" || die "can't find crash_test_pie executable" + test -f "$TESTDIR/libcrashfuncs.so" || die "can't find libcrashfuncs.so" + test -f "$TESTDIR/libcrashfixed.so" || die "can't find libcrashfixed.so" } launch() { local mode="$1" + local binary="$TESTDIR/${TEST_BINARY[$mode]:-crash_test}" + local crash_mode="${TEST_CRASHMODE[$mode]:-$mode}" ulimit -c unlimited # Make libcrashfuncs.so visible to dlopen() from the mode's scratch dir. export LD_LIBRARY_PATH="$TESTDIR${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" @@ -183,7 +204,7 @@ launch() { serial) "$SPINDLE" \ --no-mpi --crash-dedup -- \ - "$TESTDIR/crash_test" --crash-mode "$mode" + "$binary" --crash-mode "$crash_mode" ;; flux) flux run \ @@ -191,17 +212,17 @@ launch() { -o spindle.crash-dedup \ -N"$NODES" -n"$NODES" \ --env=LD_LIBRARY_PATH \ - -- "$TESTDIR/crash_test" --crash-mode "$mode" + -- "$binary" --crash-mode "$crash_mode" ;; slurm) salloc -N"$NODES" -n"$NODES" \ "$SPINDLE" --crash-dedup -- \ - srun "$TESTDIR/crash_test" --crash-mode "$mode" + srun "$binary" --crash-mode "$crash_mode" ;; slurm-plugin) salloc -N"$NODES" -n"$NODES" \ srun --spindle="--crash-dedup" \ - "$TESTDIR/crash_test" --crash-mode "$mode" + "$binary" --crash-mode "$crash_mode" ;; esac } @@ -233,6 +254,7 @@ verify_top_frames() { [ "$expected_top" = "-" ] && return 0 [ -z "$expected_top" ] && return 0 + local binary="$TESTDIR/${TEST_BINARY[$mode]:-crash_test}" local core top cores cores=$(core_files "$dir") for core in $cores; do @@ -240,7 +262,7 @@ verify_top_frames() { top=$(gdb -batch -nx \ -iex 'set print demangle off' \ -ex 'set pagination off' \ - -ex 'bt 1' "$TESTDIR/crash_test" "$core" 2>/dev/null \ + -ex 'bt 1' "$binary" "$core" 2>/dev/null \ | grep -oE '#0\s+.*' | head -1) if ! echo "$top" | grep -qE "$expected_top"; then echo " core $core: top frame '$top' does not match '$expected_top'" >&2 @@ -250,6 +272,56 @@ verify_top_frames() { return 0 } +# Read the crash site key (library+offset) out of the coredump +read_crash_site() { + local core="$1" + local binary="${2:-$TESTDIR/crash_test}" + local map base cachepath real + # Get the mappings from the coredump and extract the Spindle audit library + map=$(gdb -batch -nx -ex 'set debuginfod enabled off' \ + -ex 'info proc mappings' "$binary" "$core" 2>/dev/null \ + | awk '/-spindlens-dso-libspindle_audit/ && $4 == "0x0" {print; exit}') + base=$(printf '%s' "$map" | awk '{print $1}') + cachepath=$(printf '%s' "$map" | awk '{print $NF}') + # Figure out the path to the Spindle audit library + real=$(printf '%s' "$cachepath" | sed -E 's#^.*/spindle\.[0-9a-f]+##; s#/[0-9]+-spindlens-dso-#/#') + # Open the coredump with the Spindle audit library symbols loaded + # so we can check crash_site_buf + gdb -batch -nx -ex 'set debuginfod enabled off' \ + -ex "add-symbol-file $real -o $base" \ + -ex 'printf "CRASH_SITE=%s\n", crash_site_buf' \ + "$binary" "$core" 2>/dev/null \ + | sed -n 's/^CRASH_SITE=//p' | head -1 +} + +# Check that the crash site key matches the expectation +# to verify that Spindle identified the correct library +verify_crash_site() { + local mode="$1" + local dir="$2" + local expected_site="${TEST_SITE[$mode]:-}" + + # If this test doesn't have a crash site specified, there's nothing to do + [ "$expected_site" = "-" ] && return 0 + [ -z "$expected_site" ] && return 0 + + local binary="$TESTDIR/${TEST_BINARY[$mode]:-crash_test}" + local core cores site + cores=$(core_files "$dir") + for core in $cores; do + site=$(read_crash_site "$core" "$binary") + if [ -z "$site" ]; then + echo " core $core: could not read crash site" >&2 + return 1 + fi + if ! printf '%s' "$site" | grep -qE -- "$expected_site"; then + echo " core $core: crash dedup key '$site' does not match '$expected_site'" >&2 + return 1 + fi + done + return 0 +} + main() { parse_args "$@" parse_table @@ -336,6 +408,13 @@ main() { continue fi + # Verify the crash handler's + dedup key + if ! verify_crash_site "$mode" "$dir"; then + echo "FAIL $mode: crash site dedup key verification failed" + fail=$((fail+1)) + continue + fi + echo "PASS $mode ($actual dumps)" pass=$((pass+1)) done From 93a6be275978b7a0e78e8fbcbdf66d1a6dea010a Mon Sep 17 00:00:00 2001 From: Nicholas Chaimov Date: Mon, 6 Jul 2026 17:51:16 -0700 Subject: [PATCH 11/14] Read program headers from disk to handle fixed load address --- src/client/crash_handler/crash_lib_offset.c | 72 ++++++++++++++------- 1 file changed, 47 insertions(+), 25 deletions(-) diff --git a/src/client/crash_handler/crash_lib_offset.c b/src/client/crash_handler/crash_lib_offset.c index b2ffca58..f6c45eac 100644 --- a/src/client/crash_handler/crash_lib_offset.c +++ b/src/client/crash_handler/crash_lib_offset.c @@ -38,8 +38,11 @@ struct r_debug_ext_mirror { /* It turns out that we can't get at the program headers via dl_phdr_iterate in a signal handler, because dl_phdr_iterate - calls malloc, which is not async-signal-safe. So we have get the - program headers through getauxval(AT_PHDR) and iterate ourselves. */ + calls malloc, which is not async-signal-safe, and furthermore + dl_phdr_iterate only iterates the objects loaded in the current + namespace, so when called from the Spindle auditclient we only + see the auditclient and its own libc. Instead, we have to read + the program headers from the file. */ static const ElfW(Phdr) *exe_auxv_phdrs; static unsigned long exe_auxv_phnum; @@ -64,37 +67,59 @@ static char *get_executable_path(void) return exe_path_cached = exe_path_cache; } -static int find_pc_et_dyn(unsigned long load_base, unsigned long pc) +/* Check the program headers of the file at path for the given address */ +static int pc_in_object_file(const char *path, unsigned long base, + unsigned long pc) { - const ElfW(Ehdr) *ehdr = (const ElfW(Ehdr) *) load_base; - const ElfW(Phdr) *phdrs; + ElfW(Ehdr) ehdr; + ElfW(Phdr) phdr; + long fd, n; unsigned int i; + unsigned long off; + int found = 0; - if (memcmp(ehdr->e_ident, ELFMAG, SELFMAG) != 0) + fd = syscall(SYS_openat, AT_FDCWD, path, O_RDONLY | O_CLOEXEC, 0); + if (fd < 0) return 0; - phdrs = (const ElfW(Phdr) *) (load_base + ehdr->e_phoff); - for (i = 0; i < ehdr->e_phnum; i++) { - if (phdrs[i].p_type != PT_LOAD) + n = syscall(SYS_pread64, fd, &ehdr, sizeof(ehdr), (off_t) 0); + if (n != (long) sizeof(ehdr)) + goto done; + if (memcmp(ehdr.e_ident, ELFMAG, SELFMAG) != 0) + goto done; + if (ehdr.e_phnum == 0) + goto done; + + off = ehdr.e_phoff; + for (i = 0; i < ehdr.e_phnum; i++, off += ehdr.e_phentsize) { + n = syscall(SYS_pread64, fd, &phdr, sizeof(phdr), (off_t) off); + if (n != (long) sizeof(phdr)) + break; + if (phdr.p_type != PT_LOAD) continue; - unsigned long start = load_base + phdrs[i].p_vaddr; - if (pc >= start && pc < start + phdrs[i].p_memsz) - return 1; + unsigned long start = base + phdr.p_vaddr; + if (pc >= start && pc < start + phdr.p_memsz) { + found = 1; + break; + } } - return 0; + +done: + syscall(SYS_close, fd); + return found; } -static int find_pc_et_exec(unsigned long pc) +static int pc_in_exe(unsigned long base, unsigned long pc) { - unsigned long i; + unsigned long i, start; if (!exe_auxv_phdrs) return 0; for (i = 0; i < exe_auxv_phnum; i++) { if (exe_auxv_phdrs[i].p_type != PT_LOAD) continue; - if (pc >= exe_auxv_phdrs[i].p_vaddr && - pc < exe_auxv_phdrs[i].p_vaddr + exe_auxv_phdrs[i].p_memsz) + start = base + exe_auxv_phdrs[i].p_vaddr; + if (pc >= start && pc < start + exe_auxv_phdrs[i].p_memsz) return 1; } return 0; @@ -104,15 +129,12 @@ static int walk_link_map_list(struct link_map *cur, unsigned long pc, char *buf, size_t buflen) { for (; cur != NULL; cur = cur->l_next) { - int hit = (cur->l_addr != 0) - ? find_pc_et_dyn(cur->l_addr, pc) - : find_pc_et_exec(pc); - if (hit) { - const char *use_name = - (cur->l_name && cur->l_name[0]) ? cur->l_name - : get_executable_path(); + int is_exe = !(cur->l_name && cur->l_name[0]); + const char *use_name = is_exe ? get_executable_path() : cur->l_name; + int hit = is_exe ? pc_in_exe(cur->l_addr, pc) + : pc_in_object_file(use_name, cur->l_addr, pc); + if (hit) return crash_fmt_lib_offset(buf, buflen, use_name, pc - cur->l_addr); - } } return -1; } From 8ac8c681437d32b8dbccea40d1d8a153ab59e076 Mon Sep 17 00:00:00 2001 From: Nicholas Chaimov Date: Tue, 7 Jul 2026 09:46:59 -0700 Subject: [PATCH 12/14] mmap altstack instead of using static buffer --- src/client/crash_handler/crash_handler.c | 31 +++++++++++++++--------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/src/client/crash_handler/crash_handler.c b/src/client/crash_handler/crash_handler.c index 8647b054..8482a30c 100644 --- a/src/client/crash_handler/crash_handler.c +++ b/src/client/crash_handler/crash_handler.c @@ -23,6 +23,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include #include #include +#include #include #include "config.h" @@ -47,7 +48,7 @@ static int crash_read_fd = -1; static int crash_write_fd = -1; static int crash_installed = 0; -static char crash_altstack_buf[CRASH_ALTSTACK_SIZE]; +static char *crash_altstack_buf = NULL; static char crash_site_buf[CRASH_SITE_BUF_SIZE]; static volatile sig_atomic_t handler_active = 0; @@ -275,23 +276,29 @@ int crash_handler_install(int global_rank, int ldcsid_in) for the signal handler. However, note that this is per-thread, and currently we do not register an alternate stack on any thread other than the main thread. TODO: handle alternate stack on other threads */ - stack_t ss; - memset(&ss, 0, sizeof ss); - ss.ss_sp = crash_altstack_buf; - ss.ss_size = CRASH_ALTSTACK_SIZE; - ss.ss_flags = 0; - if (sigaltstack(&ss, NULL) != 0) { - debug_printf("sigaltstack failed, not installing\n"); - crash_read_fd = -1; - crash_write_fd = -1; - return 0; + crash_altstack_buf = mmap(NULL, CRASH_ALTSTACK_SIZE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (crash_altstack_buf == MAP_FAILED) { + crash_altstack_buf = NULL; + } else { + stack_t ss; + memset(&ss, 0, sizeof ss); + ss.ss_sp = crash_altstack_buf; + ss.ss_size = CRASH_ALTSTACK_SIZE; + ss.ss_flags = 0; + if (sigaltstack(&ss, NULL) != 0) { + munmap(crash_altstack_buf, CRASH_ALTSTACK_SIZE); + crash_altstack_buf = NULL; + } } /* Install the signal handler. */ struct sigaction sa; memset(&sa, 0, sizeof sa); sa.sa_sigaction = crash_handler_entry; - sa.sa_flags = SA_SIGINFO | SA_ONSTACK | SA_RESTART; + sa.sa_flags = SA_SIGINFO | SA_RESTART; + if (crash_altstack_buf != NULL) + sa.sa_flags |= SA_ONSTACK; sigemptyset(&sa.sa_mask); sigaddset(&sa.sa_mask, SIGSEGV); sigaddset(&sa.sa_mask, SIGBUS); From 321310afb15d9699542c752f4ba015c1f5736d06 Mon Sep 17 00:00:00 2001 From: Nicholas Chaimov Date: Tue, 7 Jul 2026 10:07:17 -0700 Subject: [PATCH 13/14] Unconditionally compile the signal handler --- config.h.in | 3 -- configure | 37 +------------------ configure.common.ac | 16 +------- src/client/client/Makefile.am | 2 - src/client/client/Makefile.in | 65 +++++++++------------------------ src/client/client/client.c | 4 -- src/client/client/client.h | 2 - src/client/client/intercept.c | 4 -- src/client/client/lookup_libc.c | 6 --- src/client/config.h.in | 3 -- src/client/configure | 37 +------------------ src/fe/config.h.in | 3 -- src/fe/configure | 37 +------------------ src/server/config.h.in | 3 -- src/server/configure | 37 +------------------ testsuite/Makefile.am | 2 - testsuite/Makefile.in | 5 ++- 17 files changed, 26 insertions(+), 240 deletions(-) diff --git a/config.h.in b/config.h.in index 2697764a..00b83ef3 100644 --- a/config.h.in +++ b/config.h.in @@ -30,9 +30,6 @@ /* Enable --crash-dedup by default */ #undef CRASH_DEDUP_ENABLED_BY_DEFAULT -/* Define to 1 if the crash handler is built */ -#undef CRASH_HANDLER_ENABLED - /* Defined if GLIBC has an allocation bug */ #undef DTV_ALLOCATION_BUG diff --git a/configure b/configure index f7badb36..6b54926e 100755 --- a/configure +++ b/configure @@ -667,8 +667,6 @@ LAUNCHMON_LIB LAUNCHMON_INC HAVE_LMON LAUNCHMON_STATIC_LIBS -BLD_CRASH_HANDLER_FALSE -BLD_CRASH_HANDLER_TRUE STATICFLAG fluxplugin_ldflags shell_plugindir @@ -881,7 +879,6 @@ enable_sec_munge enable_sec_keydir enable_sec_launchmon enable_sec_none -enable_crash_handler enable_crash_dedup with_gcrypt_dir with_munge_dir @@ -1580,7 +1577,6 @@ Optional Features: --enable-sec-launchmon Enable LaunchmMON to propagate authentication key for connections --enable-sec-none Disable security authentication of connections - --disable-crash-handler Build the crash handler (default: enabled) --enable-crash-dedup Enable --crash-dedup by default (default: disabled) --enable-remap-pageone Controls mmap behavior when replacing executables. Should be false on older linux systems @@ -17664,28 +17660,6 @@ if test "${enable_sec_none+set}" = set; then : fi -# Check whether --enable-crash-handler was given. -if test "${enable_crash_handler+set}" = set; then : - enableval=$enable_crash_handler; enable_crash_handler=$enableval -else - enable_crash_handler=yes -fi - - -if test "x$enable_crash_handler" != xno; then : - -$as_echo "#define CRASH_HANDLER_ENABLED 1" >>confdefs.h - -fi - if test "x$enable_crash_handler" != xno; then - BLD_CRASH_HANDLER_TRUE= - BLD_CRASH_HANDLER_FALSE='#' -else - BLD_CRASH_HANDLER_TRUE='#' - BLD_CRASH_HANDLER_FALSE= -fi - - # Check whether --enable-crash-dedup was given. if test "${enable_crash_dedup+set}" = set; then : enableval=$enable_crash_dedup; enable_crash_dedup=$enableval @@ -17694,12 +17668,7 @@ else fi -if test "x$enable_crash_dedup" = xyes && test "x$enable_crash_handler" = xno; then : - { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: --enable-crash-dedup ignored because --disable-crash-handler was given" >&5 -$as_echo "$as_me: WARNING: --enable-crash-dedup ignored because --disable-crash-handler was given" >&2;} -fi - -if test "x$enable_crash_dedup" = xyes && test "x$enable_crash_handler" != xno; then : +if test "x$enable_crash_dedup" = xyes; then : $as_echo "#define CRASH_DEDUP_ENABLED_BY_DEFAULT 1" >>confdefs.h @@ -19147,10 +19116,6 @@ if test -z "${AARCH64_BLD_TRUE}" && test -z "${AARCH64_BLD_FALSE}"; then as_fn_error $? "conditional \"AARCH64_BLD\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi -if test -z "${BLD_CRASH_HANDLER_TRUE}" && test -z "${BLD_CRASH_HANDLER_FALSE}"; then - as_fn_error $? "conditional \"BLD_CRASH_HANDLER\" was never defined. -Usually this means the macro was only invoked conditionally." "$LINENO" 5 -fi if test -z "${LMON_TRUE}" && test -z "${LMON_FALSE}"; then as_fn_error $? "conditional \"LMON\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 diff --git a/configure.common.ac b/configure.common.ac index 41da35f0..1b187f2e 100644 --- a/configure.common.ac +++ b/configure.common.ac @@ -427,27 +427,13 @@ AC_ARG_ENABLE(sec-none, [AS_HELP_STRING([--enable-sec-none],[Disable security authentication of connections])], [WANT_NOSEC="true";EXPLICIT_SEC="true"],) -AC_ARG_ENABLE([crash-handler], - [AS_HELP_STRING([--disable-crash-handler], - [Build the crash handler (default: enabled)])], - [enable_crash_handler=$enableval], - [enable_crash_handler=yes]) - -AS_IF([test "x$enable_crash_handler" != xno], - [AC_DEFINE([CRASH_HANDLER_ENABLED], [1], - [Define to 1 if the crash handler is built])]) -AM_CONDITIONAL([BLD_CRASH_HANDLER], [test "x$enable_crash_handler" != xno]) - AC_ARG_ENABLE([crash-dedup], [AS_HELP_STRING([--enable-crash-dedup], [Enable --crash-dedup by default (default: disabled)])], [enable_crash_dedup=$enableval], [enable_crash_dedup=no]) -AS_IF([test "x$enable_crash_dedup" = xyes && test "x$enable_crash_handler" = xno], - [AC_MSG_WARN([--enable-crash-dedup ignored because --disable-crash-handler was given])]) - -AS_IF([test "x$enable_crash_dedup" = xyes && test "x$enable_crash_handler" != xno], +AS_IF([test "x$enable_crash_dedup" = xyes], [AC_DEFINE([CRASH_DEDUP_ENABLED_BY_DEFAULT], [1], [Enable --crash-dedup by default])]) diff --git a/src/client/client/Makefile.am b/src/client/client/Makefile.am index 6af55654..47085b75 100644 --- a/src/client/client/Makefile.am +++ b/src/client/client/Makefile.am @@ -22,13 +22,11 @@ BASE_SRCS = client.c \ lookup_libc.c adjust_argv.c $(top_srcdir)/../utils/parseloc.c \ $(top_srcdir)/../utils/getcpu.c -if BLD_CRASH_HANDLER BASE_SRCS += $(top_srcdir)/crash_handler/crash_io.c \ $(top_srcdir)/crash_handler/crash_fmt.c \ $(top_srcdir)/crash_handler/crash_handler.c \ $(top_srcdir)/crash_handler/crash_lib_offset.c \ $(top_srcdir)/crash_handler/crash_sigchain.c -endif libspindlec_socket_la_SOURCES = $(BASE_SRCS) libspindlec_socket_la_LIBADD = $(top_builddir)/client_comlib/libclient_socket.la $(top_builddir)/logging/libspindleclogc.la $(top_builddir)/shm_cache/libshmcache.la diff --git a/src/client/client/Makefile.in b/src/client/client/Makefile.in index a2c82cbe..d45c5c95 100644 --- a/src/client/client/Makefile.in +++ b/src/client/client/Makefile.in @@ -93,12 +93,6 @@ target_triplet = @target@ @SOCKETS_TRUE@am__append_1 = libspindlec_socket.la @PIPES_TRUE@am__append_2 = libspindlec_pipe.la @BITER_TRUE@am__append_3 = libspindlec_biter.la -@BLD_CRASH_HANDLER_TRUE@am__append_4 = $(top_srcdir)/crash_handler/crash_io.c \ -@BLD_CRASH_HANDLER_TRUE@ $(top_srcdir)/crash_handler/crash_fmt.c \ -@BLD_CRASH_HANDLER_TRUE@ $(top_srcdir)/crash_handler/crash_handler.c \ -@BLD_CRASH_HANDLER_TRUE@ $(top_srcdir)/crash_handler/crash_lib_offset.c \ -@BLD_CRASH_HANDLER_TRUE@ $(top_srcdir)/crash_handler/crash_sigchain.c - subdir = client ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 am__aclocal_m4_deps = $(top_srcdir)/../../m4/libtool.m4 \ @@ -140,57 +134,30 @@ libspindlec_biter_la_DEPENDENCIES = \ $(top_builddir)/client_comlib/libclient_biter.la \ $(top_builddir)/logging/libspindleclogc.la \ $(top_builddir)/shm_cache/libshmcache.la -am__libspindlec_biter_la_SOURCES_DIST = client.c lookup.c \ - should_intercept.c exec_util.c remap_exec.c lookup_libc.c \ - adjust_argv.c $(top_srcdir)/../utils/parseloc.c \ - $(top_srcdir)/../utils/getcpu.c \ - $(top_srcdir)/crash_handler/crash_io.c \ - $(top_srcdir)/crash_handler/crash_fmt.c \ - $(top_srcdir)/crash_handler/crash_handler.c \ - $(top_srcdir)/crash_handler/crash_lib_offset.c \ - $(top_srcdir)/crash_handler/crash_sigchain.c -@BLD_CRASH_HANDLER_TRUE@am__objects_2 = $(top_builddir)/crash_handler/crash_io.lo \ -@BLD_CRASH_HANDLER_TRUE@ $(top_builddir)/crash_handler/crash_fmt.lo \ -@BLD_CRASH_HANDLER_TRUE@ $(top_builddir)/crash_handler/crash_handler.lo \ -@BLD_CRASH_HANDLER_TRUE@ $(top_builddir)/crash_handler/crash_lib_offset.lo \ -@BLD_CRASH_HANDLER_TRUE@ $(top_builddir)/crash_handler/crash_sigchain.lo -am__objects_3 = client.lo lookup.lo should_intercept.lo exec_util.lo \ +am__objects_2 = client.lo lookup.lo should_intercept.lo exec_util.lo \ remap_exec.lo lookup_libc.lo adjust_argv.lo \ $(top_builddir)/../utils/parseloc.lo \ - $(top_builddir)/../utils/getcpu.lo $(am__objects_2) -am_libspindlec_biter_la_OBJECTS = $(am__objects_3) + $(top_builddir)/../utils/getcpu.lo \ + $(top_builddir)/crash_handler/crash_io.lo \ + $(top_builddir)/crash_handler/crash_fmt.lo \ + $(top_builddir)/crash_handler/crash_handler.lo \ + $(top_builddir)/crash_handler/crash_lib_offset.lo \ + $(top_builddir)/crash_handler/crash_sigchain.lo +am_libspindlec_biter_la_OBJECTS = $(am__objects_2) libspindlec_biter_la_OBJECTS = $(am_libspindlec_biter_la_OBJECTS) @BITER_TRUE@am_libspindlec_biter_la_rpath = libspindlec_pipe_la_DEPENDENCIES = \ $(top_builddir)/client_comlib/libclient_pipe.la \ $(top_builddir)/logging/libspindleclogc.la \ $(top_builddir)/shm_cache/libshmcache.la -am__libspindlec_pipe_la_SOURCES_DIST = client.c lookup.c \ - should_intercept.c exec_util.c remap_exec.c lookup_libc.c \ - adjust_argv.c $(top_srcdir)/../utils/parseloc.c \ - $(top_srcdir)/../utils/getcpu.c \ - $(top_srcdir)/crash_handler/crash_io.c \ - $(top_srcdir)/crash_handler/crash_fmt.c \ - $(top_srcdir)/crash_handler/crash_handler.c \ - $(top_srcdir)/crash_handler/crash_lib_offset.c \ - $(top_srcdir)/crash_handler/crash_sigchain.c -am_libspindlec_pipe_la_OBJECTS = $(am__objects_3) +am_libspindlec_pipe_la_OBJECTS = $(am__objects_2) libspindlec_pipe_la_OBJECTS = $(am_libspindlec_pipe_la_OBJECTS) @PIPES_TRUE@am_libspindlec_pipe_la_rpath = libspindlec_socket_la_DEPENDENCIES = \ $(top_builddir)/client_comlib/libclient_socket.la \ $(top_builddir)/logging/libspindleclogc.la \ $(top_builddir)/shm_cache/libshmcache.la -am__libspindlec_socket_la_SOURCES_DIST = client.c lookup.c \ - should_intercept.c exec_util.c remap_exec.c lookup_libc.c \ - adjust_argv.c $(top_srcdir)/../utils/parseloc.c \ - $(top_srcdir)/../utils/getcpu.c \ - $(top_srcdir)/crash_handler/crash_io.c \ - $(top_srcdir)/crash_handler/crash_fmt.c \ - $(top_srcdir)/crash_handler/crash_handler.c \ - $(top_srcdir)/crash_handler/crash_lib_offset.c \ - $(top_srcdir)/crash_handler/crash_sigchain.c -am_libspindlec_socket_la_OBJECTS = $(am__objects_3) +am_libspindlec_socket_la_OBJECTS = $(am__objects_2) libspindlec_socket_la_OBJECTS = $(am_libspindlec_socket_la_OBJECTS) @SOCKETS_TRUE@am_libspindlec_socket_la_rpath = AM_V_P = $(am__v_P_@AM_V@) @@ -252,9 +219,8 @@ SOURCES = $(libspindle_audit_la_SOURCES) \ $(libspindlec_biter_la_SOURCES) $(libspindlec_pipe_la_SOURCES) \ $(libspindlec_socket_la_SOURCES) DIST_SOURCES = $(libspindle_audit_la_SOURCES) \ - $(am__libspindlec_biter_la_SOURCES_DIST) \ - $(am__libspindlec_pipe_la_SOURCES_DIST) \ - $(am__libspindlec_socket_la_SOURCES_DIST) + $(libspindlec_biter_la_SOURCES) $(libspindlec_pipe_la_SOURCES) \ + $(libspindlec_socket_la_SOURCES) am__can_run_installinfo = \ case $$AM_UPDATE_INFO_DIR in \ n|no|NO) false;; \ @@ -424,7 +390,12 @@ INTERCEPT_SRCS = intercept_open.c intercept_exec.c intercept_stat.c intercept_re BASE_SRCS = client.c lookup.c should_intercept.c exec_util.c \ remap_exec.c lookup_libc.c adjust_argv.c \ $(top_srcdir)/../utils/parseloc.c \ - $(top_srcdir)/../utils/getcpu.c $(am__append_4) + $(top_srcdir)/../utils/getcpu.c \ + $(top_srcdir)/crash_handler/crash_io.c \ + $(top_srcdir)/crash_handler/crash_fmt.c \ + $(top_srcdir)/crash_handler/crash_handler.c \ + $(top_srcdir)/crash_handler/crash_lib_offset.c \ + $(top_srcdir)/crash_handler/crash_sigchain.c libspindlec_socket_la_SOURCES = $(BASE_SRCS) libspindlec_socket_la_LIBADD = $(top_builddir)/client_comlib/libclient_socket.la $(top_builddir)/logging/libspindleclogc.la $(top_builddir)/shm_cache/libshmcache.la libspindlec_pipe_la_SOURCES = $(BASE_SRCS) diff --git a/src/client/client/client.c b/src/client/client/client.c index 480518b6..7e5a32a9 100644 --- a/src/client/client/client.c +++ b/src/client/client/client.c @@ -35,9 +35,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include "client.h" #include "client_heap.h" #include "client_api.h" -#if defined(CRASH_HANDLER_ENABLED) #include "crash_handler.h" -#endif #include "spindle_launch.h" #include "shmcache.h" #include "ccwarns.h" @@ -368,12 +366,10 @@ int client_init() init_server_connection(); -#if defined(CRASH_HANDLER_ENABLED) if (opts & OPT_CRASH_HANDLER) { int global_rank = rankinfo[0] * rankinfo[3] + rankinfo[2]; (void) crash_handler_install(global_rank, ldcsid); } -#endif /* CRASH_HANDLER_ENABLED */ intercept_open = (opts & OPT_RELOCPY) ? 1 : 0; intercept_stat = (opts & OPT_RELOCPY || !(opts & OPT_NOHIDE)) ? 1 : 0; diff --git a/src/client/client/client.h b/src/client/client/client.h index e53543cb..7004d577 100644 --- a/src/client/client/client.h +++ b/src/client/client/client.h @@ -107,9 +107,7 @@ extern void int_spindle_test_log_msg(char *buffer); typedef void* (*malloc_sig_t)(size_t); malloc_sig_t get_libc_malloc(); -#if defined(CRASH_HANDLER_ENABLED) void *get_libc_abort_msg(); -#endif int lookup_libc_symbols(); diff --git a/src/client/client/intercept.c b/src/client/client/intercept.c index 80c4aef4..a6bb9d7a 100644 --- a/src/client/client/intercept.c +++ b/src/client/client/intercept.c @@ -21,9 +21,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include "intercept.h" #include "client.h" #include "config.h" -#if defined(CRASH_HANDLER_ENABLED) #include "crash_sigchain.h" -#endif struct spindle_binding_t spindle_bindings[] = { { "", NULL, "", NULL }, @@ -49,12 +47,10 @@ struct spindle_binding_t spindle_bindings[] = { { "execvp", (void **) &orig_execvp, "execvp_wrapper", (void *) execvp_wrapper }, { "execvpe", (void **) &orig_execvpe, "execvpe_wrapper", (void *) execvpe_wrapper }, { "vfork", (void **) &orig_vfork, "vfork_wrapper", (void *) vfork_wrapper }, -#if defined(CRASH_HANDLER_ENABLED) { "sigaction", (void **) &orig_sigaction, "sigaction_wrapper", (void *) sigaction_wrapper }, { "signal", (void **) &orig_signal, "signal_wrapper", (void *) signal_wrapper }, { "bsd_signal", (void **) &orig_bsd_signal, "bsd_signal_wrapper", (void *) bsd_signal_wrapper }, { "sysv_signal", (void **) &orig_sysv_signal, "sysv_signal_wrapper", (void *) sysv_signal_wrapper }, -#endif { "readlink", (void **) &orig_readlink, "readlink_wrapper", (void *) readlink_wrapper }, { "readlinkat", (void **) &orig_readlinkat, "readlinkat_wrapper", (void *) readlinkat_wrapper }, { "realpath", (void **) &orig_realpath, "spindle_realpath", (void *) spindle_realpath }, diff --git a/src/client/client/lookup_libc.c b/src/client/client/lookup_libc.c index 20750f96..51344dfa 100644 --- a/src/client/client/lookup_libc.c +++ b/src/client/client/lookup_libc.c @@ -47,9 +47,7 @@ struct gnu_hash_header { static malloc_sig_t mallocfunc = NULL; -#if defined(CRASH_HANDLER_ENABLED) static void *volatile abort_msg_addr = NULL; -#endif static signed long lookup_gnu_hash_symbol(const char *name, ElfW(Sym) *syms, char *symnames, struct gnu_hash_header *header) { @@ -220,7 +218,6 @@ int lookup_libc_symbols() found++; } -#if defined(CRASH_HANDLER_ENABLED) result = -1; if (gnu_hash) result = lookup_gnu_hash_symbol("__abort_msg", symtab, strtab, (struct gnu_hash_header *) gnu_hash); @@ -235,7 +232,6 @@ int lookup_libc_symbols() debug_printf3("Bound __abort_msg slot to %p\n", abort_msg_addr); found++; } -#endif } libdl_result = lookup_libdl_symbols(); @@ -304,9 +300,7 @@ malloc_sig_t get_libc_malloc() return mallocfunc; } -#if defined(CRASH_HANDLER_ENABLED) void *get_libc_abort_msg() { return abort_msg_addr; } -#endif diff --git a/src/client/config.h.in b/src/client/config.h.in index 73e75c33..872b9278 100644 --- a/src/client/config.h.in +++ b/src/client/config.h.in @@ -30,9 +30,6 @@ /* Enable --crash-dedup by default */ #undef CRASH_DEDUP_ENABLED_BY_DEFAULT -/* Define to 1 if the crash handler is built */ -#undef CRASH_HANDLER_ENABLED - /* Defined if GLIBC has an allocation bug */ #undef DTV_ALLOCATION_BUG diff --git a/src/client/configure b/src/client/configure index d635b861..18f13fa5 100755 --- a/src/client/configure +++ b/src/client/configure @@ -637,8 +637,6 @@ LTLIBOBJS LIBOBJS GLIBC_BE_DIR LIBSPINDLE_LIB_VERSION -BLD_CRASH_HANDLER_FALSE -BLD_CRASH_HANDLER_TRUE STATICFLAG fluxplugin_ldflags shell_plugindir @@ -844,7 +842,6 @@ enable_sec_munge enable_sec_keydir enable_sec_launchmon enable_sec_none -enable_crash_handler enable_crash_dedup with_gcrypt_dir with_munge_dir @@ -1524,7 +1521,6 @@ Optional Features: --enable-sec-launchmon Enable LaunchmMON to propagate authentication key for connections --enable-sec-none Disable security authentication of connections - --disable-crash-handler Build the crash handler (default: enabled) --enable-crash-dedup Enable --crash-dedup by default (default: disabled) --enable-remap-pageone Controls mmap behavior when replacing executables. Should be false on older linux systems @@ -13712,28 +13708,6 @@ if test "${enable_sec_none+set}" = set; then : fi -# Check whether --enable-crash-handler was given. -if test "${enable_crash_handler+set}" = set; then : - enableval=$enable_crash_handler; enable_crash_handler=$enableval -else - enable_crash_handler=yes -fi - - -if test "x$enable_crash_handler" != xno; then : - -$as_echo "#define CRASH_HANDLER_ENABLED 1" >>confdefs.h - -fi - if test "x$enable_crash_handler" != xno; then - BLD_CRASH_HANDLER_TRUE= - BLD_CRASH_HANDLER_FALSE='#' -else - BLD_CRASH_HANDLER_TRUE='#' - BLD_CRASH_HANDLER_FALSE= -fi - - # Check whether --enable-crash-dedup was given. if test "${enable_crash_dedup+set}" = set; then : enableval=$enable_crash_dedup; enable_crash_dedup=$enableval @@ -13742,12 +13716,7 @@ else fi -if test "x$enable_crash_dedup" = xyes && test "x$enable_crash_handler" = xno; then : - { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: --enable-crash-dedup ignored because --disable-crash-handler was given" >&5 -$as_echo "$as_me: WARNING: --enable-crash-dedup ignored because --disable-crash-handler was given" >&2;} -fi - -if test "x$enable_crash_dedup" = xyes && test "x$enable_crash_handler" != xno; then : +if test "x$enable_crash_dedup" = xyes; then : $as_echo "#define CRASH_DEDUP_ENABLED_BY_DEFAULT 1" >>confdefs.h @@ -14037,10 +14006,6 @@ if test -z "${AARCH64_BLD_TRUE}" && test -z "${AARCH64_BLD_FALSE}"; then as_fn_error $? "conditional \"AARCH64_BLD\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi -if test -z "${BLD_CRASH_HANDLER_TRUE}" && test -z "${BLD_CRASH_HANDLER_FALSE}"; then - as_fn_error $? "conditional \"BLD_CRASH_HANDLER\" was never defined. -Usually this means the macro was only invoked conditionally." "$LINENO" 5 -fi : "${CONFIG_STATUS=./config.status}" ac_write_fail=0 diff --git a/src/fe/config.h.in b/src/fe/config.h.in index 315606b8..20a30383 100644 --- a/src/fe/config.h.in +++ b/src/fe/config.h.in @@ -30,9 +30,6 @@ /* Enable --crash-dedup by default */ #undef CRASH_DEDUP_ENABLED_BY_DEFAULT -/* Define to 1 if the crash handler is built */ -#undef CRASH_HANDLER_ENABLED - /* Default to using a dedicated process to clean data files */ #undef DEFAULT_CLEANUP_PROC diff --git a/src/fe/configure b/src/fe/configure index df0d828d..73def515 100755 --- a/src/fe/configure +++ b/src/fe/configure @@ -651,8 +651,6 @@ MUNGE_LIBS MUNGE_CFLAGS GCRYPT_LIBS GCRYPT_CFLAGS -BLD_CRASH_HANDLER_FALSE -BLD_CRASH_HANDLER_TRUE STATICFLAG fluxplugin_ldflags shell_plugindir @@ -865,7 +863,6 @@ enable_sec_munge enable_sec_keydir enable_sec_launchmon enable_sec_none -enable_crash_handler enable_crash_dedup with_gcrypt_dir with_munge_dir @@ -1557,7 +1554,6 @@ Optional Features: --enable-sec-launchmon Enable LaunchmMON to propagate authentication key for connections --enable-sec-none Disable security authentication of connections - --disable-crash-handler Build the crash handler (default: enabled) --enable-crash-dedup Enable --crash-dedup by default (default: disabled) --enable-remap-pageone Controls mmap behavior when replacing executables. Should be false on older linux systems @@ -17562,28 +17558,6 @@ if test "${enable_sec_none+set}" = set; then : fi -# Check whether --enable-crash-handler was given. -if test "${enable_crash_handler+set}" = set; then : - enableval=$enable_crash_handler; enable_crash_handler=$enableval -else - enable_crash_handler=yes -fi - - -if test "x$enable_crash_handler" != xno; then : - -$as_echo "#define CRASH_HANDLER_ENABLED 1" >>confdefs.h - -fi - if test "x$enable_crash_handler" != xno; then - BLD_CRASH_HANDLER_TRUE= - BLD_CRASH_HANDLER_FALSE='#' -else - BLD_CRASH_HANDLER_TRUE='#' - BLD_CRASH_HANDLER_FALSE= -fi - - # Check whether --enable-crash-dedup was given. if test "${enable_crash_dedup+set}" = set; then : enableval=$enable_crash_dedup; enable_crash_dedup=$enableval @@ -17592,12 +17566,7 @@ else fi -if test "x$enable_crash_dedup" = xyes && test "x$enable_crash_handler" = xno; then : - { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: --enable-crash-dedup ignored because --disable-crash-handler was given" >&5 -$as_echo "$as_me: WARNING: --enable-crash-dedup ignored because --disable-crash-handler was given" >&2;} -fi - -if test "x$enable_crash_dedup" = xyes && test "x$enable_crash_handler" != xno; then : +if test "x$enable_crash_dedup" = xyes; then : $as_echo "#define CRASH_DEDUP_ENABLED_BY_DEFAULT 1" >>confdefs.h @@ -18421,10 +18390,6 @@ if test -z "${AARCH64_BLD_TRUE}" && test -z "${AARCH64_BLD_FALSE}"; then as_fn_error $? "conditional \"AARCH64_BLD\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi -if test -z "${BLD_CRASH_HANDLER_TRUE}" && test -z "${BLD_CRASH_HANDLER_FALSE}"; then - as_fn_error $? "conditional \"BLD_CRASH_HANDLER\" was never defined. -Usually this means the macro was only invoked conditionally." "$LINENO" 5 -fi if test -z "${LMON_TRUE}" && test -z "${LMON_FALSE}"; then as_fn_error $? "conditional \"LMON\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 diff --git a/src/server/config.h.in b/src/server/config.h.in index 4b227ec0..14c85b47 100644 --- a/src/server/config.h.in +++ b/src/server/config.h.in @@ -30,9 +30,6 @@ /* Enable --crash-dedup by default */ #undef CRASH_DEDUP_ENABLED_BY_DEFAULT -/* Define to 1 if the crash handler is built */ -#undef CRASH_HANDLER_ENABLED - /* Defined if GLIBC has an allocation bug */ #undef DTV_ALLOCATION_BUG diff --git a/src/server/configure b/src/server/configure index 78f8a828..71d5fb82 100755 --- a/src/server/configure +++ b/src/server/configure @@ -657,8 +657,6 @@ MUNGE_LIBS MUNGE_CFLAGS GCRYPT_LIBS GCRYPT_CFLAGS -BLD_CRASH_HANDLER_FALSE -BLD_CRASH_HANDLER_TRUE STATICFLAG fluxplugin_ldflags shell_plugindir @@ -871,7 +869,6 @@ enable_sec_munge enable_sec_keydir enable_sec_launchmon enable_sec_none -enable_crash_handler enable_crash_dedup with_gcrypt_dir with_munge_dir @@ -1559,7 +1556,6 @@ Optional Features: --enable-sec-launchmon Enable LaunchmMON to propagate authentication key for connections --enable-sec-none Disable security authentication of connections - --disable-crash-handler Build the crash handler (default: enabled) --enable-crash-dedup Enable --crash-dedup by default (default: disabled) --enable-remap-pageone Controls mmap behavior when replacing executables. Should be false on older linux systems @@ -17559,28 +17555,6 @@ if test "${enable_sec_none+set}" = set; then : fi -# Check whether --enable-crash-handler was given. -if test "${enable_crash_handler+set}" = set; then : - enableval=$enable_crash_handler; enable_crash_handler=$enableval -else - enable_crash_handler=yes -fi - - -if test "x$enable_crash_handler" != xno; then : - -$as_echo "#define CRASH_HANDLER_ENABLED 1" >>confdefs.h - -fi - if test "x$enable_crash_handler" != xno; then - BLD_CRASH_HANDLER_TRUE= - BLD_CRASH_HANDLER_FALSE='#' -else - BLD_CRASH_HANDLER_TRUE='#' - BLD_CRASH_HANDLER_FALSE= -fi - - # Check whether --enable-crash-dedup was given. if test "${enable_crash_dedup+set}" = set; then : enableval=$enable_crash_dedup; enable_crash_dedup=$enableval @@ -17589,12 +17563,7 @@ else fi -if test "x$enable_crash_dedup" = xyes && test "x$enable_crash_handler" = xno; then : - { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: --enable-crash-dedup ignored because --disable-crash-handler was given" >&5 -$as_echo "$as_me: WARNING: --enable-crash-dedup ignored because --disable-crash-handler was given" >&2;} -fi - -if test "x$enable_crash_dedup" = xyes && test "x$enable_crash_handler" != xno; then : +if test "x$enable_crash_dedup" = xyes; then : $as_echo "#define CRASH_DEDUP_ENABLED_BY_DEFAULT 1" >>confdefs.h @@ -18465,10 +18434,6 @@ if test -z "${AARCH64_BLD_TRUE}" && test -z "${AARCH64_BLD_FALSE}"; then as_fn_error $? "conditional \"AARCH64_BLD\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi -if test -z "${BLD_CRASH_HANDLER_TRUE}" && test -z "${BLD_CRASH_HANDLER_FALSE}"; then - as_fn_error $? "conditional \"BLD_CRASH_HANDLER\" was never defined. -Usually this means the macro was only invoked conditionally." "$LINENO" 5 -fi if test -z "${LMON_TRUE}" && test -z "${LMON_FALSE}"; then as_fn_error $? "conditional \"LMON\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 diff --git a/testsuite/Makefile.am b/testsuite/Makefile.am index 70943797..1ce88385 100644 --- a/testsuite/Makefile.am +++ b/testsuite/Makefile.am @@ -3,9 +3,7 @@ noinst_PROGRAMS = libgenerator ABS_TEST_DIR = $(abspath $(top_builddir)/testsuite) BUILT_SOURCES = libtest10.so libtest11.so libtest12.so libtest13.so libtest14.so libtest15.so libtest16.so libtest17.so libtest18.so libtest19.so libtest20.so libtest50.so libtest100.so libtest500.so libtest1000.so libtest2000.so libtest4000.so libtest6000.so libtest8000.so libtest10000.so libtls1.c libtls2.c libtls3.c libtls4.c libtls5.c libtls6.c libtls7.c libtls8.c libtls9.c libtls10.c libtls11.c libtls12.c libtls13.c libtls14.c libtls15.c libtls16.c libtls17.c libtls18.c libtls19.c libtls20.c libsymlink.so libdepC.so libdepB.so libdepA.so libcxxexceptB.so libcxxexceptA.so origin_dir/liboriginlib.so origin_dir/origin_subdir/liborigintarget.so libtestoutput.so libfuncdict.so runTests run_driver run_driver_rm spindle.rc preload_file_list test_driver test_driver_libs retzero_rx retzero_r retzero_x retzero_ badinterp hello_r.py hello_x.py hello_rx.py hello_.py hello_l.py badlink.py spindle_exec_test spindle_deactivated.sh liblocal.so symbind_test interpreter_test interpreter_test_dir/interpreter_test_perl alias/aliastest.py exec_shell.sh exec_shell.tcsh exec_shell_expected_output exec_shell exec_shell_env.sh -if BLD_CRASH_HANDLER BUILT_SOURCES += crash_test crash_test_fixedaddr crash_test_pie libcrashfuncs.so libcrashctor.so libcrashfixed.so run_crash_tests.sh -endif if BGQ_BLD DYNAMIC_FLAG=-dynamic diff --git a/testsuite/Makefile.in b/testsuite/Makefile.in index 6ab4fcfc..f3402c0a 100644 --- a/testsuite/Makefile.in +++ b/testsuite/Makefile.in @@ -90,7 +90,6 @@ build_triplet = @build@ host_triplet = @host@ target_triplet = @target@ noinst_PROGRAMS = libgenerator$(EXEEXT) -@BLD_CRASH_HANDLER_TRUE@am__append_1 = crash_test crash_test_fixedaddr crash_test_pie libcrashfuncs.so libcrashctor.so libcrashfixed.so run_crash_tests.sh subdir = testsuite ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 am__aclocal_m4_deps = $(top_srcdir)/m4/libtool.m4 \ @@ -360,7 +359,9 @@ BUILT_SOURCES = libtest10.so libtest11.so libtest12.so libtest13.so \ interpreter_test interpreter_test_dir/interpreter_test_perl \ alias/aliastest.py exec_shell.sh exec_shell.tcsh \ exec_shell_expected_output exec_shell exec_shell_env.sh \ - $(am__append_1) + crash_test crash_test_fixedaddr crash_test_pie \ + libcrashfuncs.so libcrashctor.so libcrashfixed.so \ + run_crash_tests.sh @BGQ_BLD_FALSE@DYNAMIC_FLAG = @BGQ_BLD_TRUE@DYNAMIC_FLAG = -dynamic @BGQ_BLD_FALSE@IS_BLUEGENE = false From 46fa81e3d82e504e53ce6f19444dc0861701b1ec Mon Sep 17 00:00:00 2001 From: Nicholas Chaimov Date: Tue, 7 Jul 2026 10:59:30 -0700 Subject: [PATCH 14/14] Add mmap SIGBUS tests --- testsuite/crash_test.c | 94 +++++++++++++++++++++++++++ testsuite/run_crash_tests_template.sh | 2 + 2 files changed, 96 insertions(+) diff --git a/testsuite/crash_test.c b/testsuite/crash_test.c index 2c535c18..b49c8c2e 100644 --- a/testsuite/crash_test.c +++ b/testsuite/crash_test.c @@ -68,6 +68,10 @@ Place, Suite 330, Boston, MA 02111-1307 USA * app handler returns without fixing a page-spanning read fault * safepoint-span-bad-write * app handler returns without fixing a page-spanning write fault + * mmap-sigbus-bad + * app handler does not fix access past the end of an mmap'ed file + * mmap-sigbus-fixed + * app handler fixes access past the end of an mmap'ed file by extending it * * no-crash every rank exits cleanly */ @@ -107,6 +111,7 @@ static void usage(const char *prog) { "safepoint-bad|safepoint-bad-write|safepoint-fix-write|" "safepoint-span-read|safepoint-span-write|" "safepoint-span-bad-read|safepoint-span-bad-write|" + "mmap-sigbus-bad|mmap-sigbus-fixed|" "safepoint-longjmp|safepoint-longjmp-mt|safepoint-concurrent-chain|no-crash}" " [--sleep ] [--cycles ]\n", prog); @@ -586,6 +591,89 @@ static int do_safepoint_span_bad_write(int rank) { return SAFEPOINT_RC_NOT_TERMINATED; } +static int mmap_sigbus_fd = -1; +static long mmap_sigbus_pagesize; +static unsigned char *mmap_sigbus_map; + +static volatile char *mmap_sigbus_target(void) { + return (volatile char *) (mmap_sigbus_map + mmap_sigbus_pagesize); +} + +static int setup_mmap_sigbus(const char *mode, int rank, + void (*handler)(int, siginfo_t *, void *)) { + mmap_sigbus_pagesize = sysconf(_SC_PAGESIZE); + + char path[] = "/tmp/spindle_mmap_sigbus_XXXXXX"; + mmap_sigbus_fd = mkstemp(path); + if (mmap_sigbus_fd < 0) { + fprintf(stderr, "%s rank=%d: mkstemp failed\n", mode, rank); + return SAFEPOINT_RC_SETUP_FAILED; + } + (void) unlink(path); + if (ftruncate(mmap_sigbus_fd, (off_t) mmap_sigbus_pagesize) != 0) { + fprintf(stderr, "%s rank=%d: ftruncate failed\n", mode, rank); + return SAFEPOINT_RC_SETUP_FAILED; + } + mmap_sigbus_map = mmap(NULL, 2 * mmap_sigbus_pagesize, + PROT_READ | PROT_WRITE, MAP_SHARED, + mmap_sigbus_fd, 0); + if (mmap_sigbus_map == MAP_FAILED) { + fprintf(stderr, "%s rank=%d: mmap failed\n", mode, rank); + return SAFEPOINT_RC_SETUP_FAILED; + } + + struct sigaction sa; + memset(&sa, 0, sizeof sa); + sa.sa_sigaction = handler; + sa.sa_flags = SA_SIGINFO; + sigemptyset(&sa.sa_mask); + if (sigaction(SIGBUS, &sa, NULL) != 0) { + fprintf(stderr, "%s rank=%d: sigaction(SIGBUS) failed\n", mode, rank); + return SAFEPOINT_RC_SETUP_FAILED; + } + return 0; +} + +static void mmap_sigbus_fix_handler(int sig, siginfo_t *info, void *uctx) { + (void) sig; (void) info; (void) uctx; + safepoint_handler_invocations++; + /* Increase the size of the file so the retried read succeeds */ + if (ftruncate(mmap_sigbus_fd, (off_t) (2 * mmap_sigbus_pagesize)) != 0) + handler_die("ftruncate failed in handler\n"); +} + +static int do_mmap_sigbus_fixed(int rank) { + int rc = setup_mmap_sigbus("mmap-sigbus-fixed", rank, + mmap_sigbus_fix_handler); + if (rc) + return rc; + + /* Read one page past EOF. The app handler extends the file so the retried + read succeeds. */ + volatile char v = *mmap_sigbus_target(); + (void) v; + + if (safepoint_handler_invocations == 0) { + fprintf(stderr, "mmap-sigbus-fixed rank=%d: handler never invoked\n", rank); + return SAFEPOINT_RC_INCOMPLETE; + } + return 0; +} + +static int do_mmap_sigbus_bad(int rank) { + int rc = setup_mmap_sigbus("mmap-sigbus-bad", rank, safepoint_nofix_handler); + if (rc) + return rc; + + volatile char v = *mmap_sigbus_target(); + (void) v; + + fprintf(stderr, + "mmap-sigbus-bad rank=%d: unexpectedly continued after unrecovered SIGBUS\n", + rank); + return SAFEPOINT_RC_NOT_TERMINATED; +} + static int sleep_seconds = 10; static int safepoint_cycles = SAFEPOINT_CYCLES_DEFAULT; @@ -728,6 +816,12 @@ int main(int argc, char **argv) { return do_safepoint_span_bad_read(rank); } else if (strcmp(mode, "safepoint-span-bad-write") == 0) { return do_safepoint_span_bad_write(rank); + } else if (strcmp(mode, "mmap-sigbus-bad") == 0) { + return do_mmap_sigbus_bad(rank); + } else if (strcmp(mode, "mmap-sigbus-fixed") == 0) { + int rc = do_mmap_sigbus_fixed(rank); + MPI_Finalize(); + return rc; } else if (strcmp(mode, "no-crash") == 0) { fprintf(stderr, "rank=%d no-crash, exiting cleanly\n", rank); fflush(stderr); diff --git a/testsuite/run_crash_tests_template.sh b/testsuite/run_crash_tests_template.sh index e366698c..3d4b71c9 100755 --- a/testsuite/run_crash_tests_template.sh +++ b/testsuite/run_crash_tests_template.sh @@ -60,6 +60,8 @@ CRASH_TESTS=( 'safepoint-span-write ; 0 ; clean ; -' 'safepoint-span-bad-read ; 1 ; ; do_safepoint_span_bad_read' 'safepoint-span-bad-write ; 1 ; ; do_safepoint_span_bad_write' + 'mmap-sigbus-bad ; 1 ; ; do_mmap_sigbus_bad' + 'mmap-sigbus-fixed ; 0 ; clean ; -' 'no-crash ; 0 ; clean ; -' )