Spank prolog alloc

containers/spindle-slurm-ubuntu/testing-plugin/conf/slurm.conf

@@ -34,6 +34,7 @@ JobAcctGatherFrequency=30
 AccountingStorageType=accounting_storage/slurmdbd
 AccountingStorageHost=slurm-db
 AccountingStoragePort=6819
+PrologFlags=Contain
 NodeName=slurm-node-1 NodeAddr=slurm-node-1 CPUs=3 RealMemory=1000 State=UNKNOWN
 NodeName=slurm-node-2 NodeAddr=slurm-node-2 CPUs=3 RealMemory=1000 State=UNKNOWN
 NodeName=slurm-node-3 NodeAddr=slurm-node-3 CPUs=3 RealMemory=1000 State=UNKNOWN

containers/spindle-slurm-ubuntu/testing-plugin/scripts/build_spindle.sh

@@ -3,7 +3,7 @@ set -euxo pipefail
 
 mkdir -p /home/${USER}/Spindle-build
 cd /home/${USER}/Spindle-build
-/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=slurm-plugin --enable-slurm-plugin --with-cachepaths=/tmp/commpath/cachepath --with-commpath=/tmp/commpath --enable-crash-dedup CFLAGS="-O2 -g" CXXFLAGS="-O2 -g"
+/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=slurm-plugin --enable-slurm-plugin --with-cachepaths=/tmp/commpath/cachepath --with-commpath='$TMPDIR/commpath' --enable-crash-dedup CFLAGS="-O2 -g" CXXFLAGS="-O2 -g"
 make -j$(nproc)
 make install
 

doc/slurm_plugin.md

@@ -0,0 +1,115 @@
+Spindle Slurm plugin
+====================
+
+The Spindle Slurm plugin integrates Spindle into Slurm through the
+SPANK interface as an alternative launch mechanism to the srun wrapper.
+It adds the ability to launch job steps using `srun --spindle`.
+
+## Building and configuring the plugin
+
+Configure Spindle with `--enable-slurm-plugin`:
+
+```bash
+./configure --with-rm=slurm-plugin --enable-slurm-plugin [--with-slurm-dir=/path/to/slurm] ...
+make
+make install
+```
+
+Refer to `INSTALL` for more details on configuring Spindle.
+
+After installation of Spindle, the plugin is installed at
+`$PREFIX/lib/libspindleslurm.so`. It is registered with Slurm by adding the
+following line to `/etc/slurm/plugstack.conf`:
+
+```
+required /path/to/spindle/lib/libspindleslurm.so
+```
+
+## Session launch modes
+
+The manner in which Spindle sessions are started varies depending on 
+the configuration of Spindle and of Slurm. 
+
+When starting a session, the plugin must arrange for Spindle to start
+on each compute node before any step runs within the allocation.
+The most straightforward way to do this is to configure the cluster
+to run job prologs at allocation time.  If your `slurm.conf` includes 
+`PrologFlags=Alloc` (or another flag that implies it: `Contain`, 
+`RunInJob`, `X11`, `ForceRequeueOnFail`, or `NoHold`), then sessions
+will be started on each node of the allocation at the time the allocation
+is made. 
+
+If `PrologFlags=Alloc` or a related setting is *not* used, one of two
+mechanisms is used to start the job on every node:
+
+**RSH launch**: Spindle can use RSH/SSH to launch daemons from the
+frontend (FE) process. To use the RSH launch mode, the cluster must be configured
+such that passwordless ssh can be used to run commands on every compute 
+node within the allocation without any interactive user input.
+This mode is enabled by configuring Spindle with:
+
+```bash
+./configure --with-rm=slurm-plugin --enable-slurm-plugin --with-rsh-launch [--with-rsh-cmd=/usr/bin/ssh] ...
+```
+
+**Dummy srun fallback**: If neither `PrologFlags=Alloc` nor RSH launch is available,
+Spindle will fall back on using a dummy `srun` invocation to force the prolog
+to run on every compute node of the allocation. Note that this has the side-effect
+of consuming step 0, so that the user's first step will instead be numbered 1.
+
+## Using Spindle through the Slurm plugin
+
+### Per-step mode: `--spindle`
+
+Add `--spindle` to any `srun` command to use Spindle for that step.
+Spindle daemons start before the application runs and shut down when
+the step finishes.
+
+```bash
+srun --spindle ./my_application
+```
+
+Additional arguments can be passed to Spindle as an optional value of the argument `--spindle`:
+
+```bash
+srun --spindle="--level=low" ./my_application
+```
+
+### Session mode: `--spindle-session`
+
+Session mode shares a Spindle session across multiple steps.
+The use of sessions in the Slurm plugin differs from its use with
+the other launchers. Unlike the other launchers, sessions are *not*
+started with `spindle --start-session`. Rather, an additional argument
+`--spindle-session` is added to `salloc` and `sbatch`. 
+
+To use a session, include `--spindle-session` when creating the allocation:
+
+```bash
+salloc --spindle-session ...
+```
+
+Then run steps with `--spindle`:
+
+```bash
+srun --spindle ./app1
+srun --spindle ./app2
+srun --spindle ./app3
+```
+
+All steps within the allocation will run in the same Spindle session.
+When the allocation exits, the session will terminate automatically.
+
+Sessions can be used with an `sbatch` script as shown below:
+
+```bash
+#!/bin/bash
+#SBATCH --spindle-session
+#SBATCH -N 4
+#SBATCH -n 4
+
+srun --spindle ./app1
+srun --spindle ./app2
+srun --spindle ./app3
+```
+

src/slurm_plugin/plugin_utils.c

@@ -240,7 +240,7 @@ char **getHostAddrSinfo(unsigned int num_hosts, char **hostlist)
       free(sinfo_cmdline);
    if (!ret && hostaddrlist) {
       for (i = 0; i < num_hosts; i++) free(hostaddrlist[i]);
-      free(hostlist);
+      free(hostaddrlist);
    }
    return ret;
 }
@@ -254,11 +254,11 @@ int isFEHost(char **hostlist, unsigned int num_hosts)
    int feresult = -1;
 
    for (i = 0; i < num_hosts; i++) {
-      if (!last_host || strcmp(hostlist[i], last_host) == 1) {
+      if (!last_host || strcmp(hostlist[i], last_host) > 0) {
          last_host = hostlist[i];
       }
    }
-   sdprintf(2, "last_host = %s\n", last_host ? last_host : NULL);
+   sdprintf(2, "last_host = %s\n", last_host ? last_host : "(null)");
    if (!last_host) {
       error = errno;
       sdprintf(1, "ERROR: Could not get current system's hostname: %s\n", strerror(error));      
@@ -360,15 +360,27 @@ static int createFEExitSocket(char *socket_path)
    int result, sock = -1, retval = -1;
 
    debug_printf("Creating unix socket for session at %s\n", socket_path);
+
+   if (strlen(socket_path) > sizeof(local.sun_path)-1) {
+      err_printf("Session exit socket path too long: %s\n", socket_path);
+      goto done;
+   }
+
    sock = socket(AF_UNIX, SOCK_STREAM, 0);
    if (sock == -1) {
       int error = errno;
       err_printf("Could not create socket for spindle session: %s\n", strerror(error));
       goto done;
    }
 
+   memset(&local, 0, sizeof(local));
    local.sun_family = AF_UNIX;
    strncpy(local.sun_path, socket_path, sizeof(local.sun_path)-1);
+
+   /* If there's an exit socket left over from a previous run that
+    * failed before removing it, remove it here */
+   unlink(socket_path);
+
    result = bind(sock, (struct sockaddr *) &local, sizeof(local));
    if (result == -1) {
       int error = errno;
@@ -419,32 +431,39 @@ int waitForSpankSessionEnd(spindle_args_t *params)
       goto done;
 
    sockfd = createFEExitSocket(socket_path);
-   if (sockfd == -1) 
+   if (sockfd == -1)
       goto done;
-
-   fd = accept(sockfd, NULL, NULL);
-   if (fd == -1) {
-      error = errno;
-      err_printf("Could not accept session exit socket connection: %s\n", strerror(error));
-      goto done;
-   }
 
-   do {
-      result = read(fd, &msg, 1);
-   } while (result == -1 && errno == EINTR);
-   if (result == -1) {
-      error = errno;
-      err_printf("Failed to read from session exit socket: %s\n", strerror(error));
-      goto done;
-   }
-   if (msg != 'q') {
-      error = errno;
-      err_printf("Recieved incorrect msg character: %c\n", msg);
-      goto done;
+   for (;;) {
+      fd = accept(sockfd, NULL, NULL);
+      if (fd == -1) {
+         error = errno;
+         err_printf("Could not accept session exit socket connection: %s\n", strerror(error));
+         goto done;
+      }
+
+      msg = 0;
+      do {
+         result = read(fd, &msg, 1);
+      } while (result == -1 && errno == EINTR);
+
+      if (result == 1 && msg == 'q') {
+         close(fd);
+         fd = -1;
+         sdprintf(2, "Received session exit message\n");
+         retval = 0;
+         break;
+      }
+
+      if (result == -1) {
+         error = errno;
+         err_printf("Failed read from session exit socket: %s\n", strerror(error));
+      } else {
+         sdprintf(2, "Received message other than exit on exit socket");
+      }
+      close(fd);
+      fd = -1;
    }
-
-   sdprintf(2, "Received session exit message\n");   
-   retval = 0;
 
   done:
    if (fd != -1)
@@ -478,6 +497,12 @@ int signalSpankSessionEnd(spindle_args_t *params)
       goto done;
    }
 
+   if (strlen(socket_path) > sizeof(saddr.sun_path)-1) {
+      err_printf("Session exit socket path too long: %s\n", socket_path);
+      goto done;
+   }
+
+   memset(&saddr, 0, sizeof(saddr));
    saddr.sun_family = AF_UNIX;
    strncpy(saddr.sun_path, socket_path, sizeof(saddr.sun_path)-1);
 
@@ -515,6 +540,15 @@ int signalSpankSessionEnd(spindle_args_t *params)
 
 char *unique_file = NULL;
 
+void cleanup_unique_file()
+{
+   if (unique_file) {
+      unlink(unique_file);
+      free(unique_file);
+      unique_file = NULL;
+   }
+}
+
 #define UNIQUE_FILE_NAME "spindle_unique"
 
 int isBEProc(spindle_args_t *params, unsigned int exit_phase)
@@ -526,6 +560,10 @@ int isBEProc(spindle_args_t *params, unsigned int exit_phase)
    int fd = -1, error;
 
    realized_dir = locSpecificDir(params);
+   if (!realized_dir) {
+      sdprintf(1, "ERROR: Could not resolve location directory in isBEProc\n");
+      goto done;
+   }
 
    gethostname(hostname, sizeof(hostname));
    hostname[sizeof(hostname)-1] = '\0';
@@ -540,7 +578,7 @@ int isBEProc(spindle_args_t *params, unsigned int exit_phase)
       strlen(hostname) + 1 +
       strlen(session_id_str) + 1;
 
-   unique_file = (char *) malloc(sizeof(char*) * unique_file_len);
+   unique_file = (char *) malloc(sizeof(char) * unique_file_len);
    snprintf(unique_file, unique_file_len, "%s/%s.%s.%s.%s", realized_dir, UNIQUE_FILE_NAME, phase_name, hostname, session_id_str);
 
    spindle_mkdir(realized_dir);
@@ -687,8 +725,10 @@ void push_env(spank_t spank, saved_env_t **env)
    e->new_spindledebug = readSpankEnv(spank, "SPINDLE_DEBUG");
    e->old_spindledebug = getenv("SPINDLE_DEBUG");
 
-   if (e->new_pwd)
-      chdir(e->new_pwd);
+   if (e->new_pwd) {
+      if (chdir(e->new_pwd) == -1)
+         sdprintf(1, "WARNING: Could not chdir to %s: %s\n", e->new_pwd, strerror(errno));
+   }
 
    if (e->new_home)
       setenv("HOME", e->new_home, 1);
@@ -727,8 +767,10 @@ void pop_env(saved_env_t *env)
    else
       unsetenv("SPINDLE_DEBUG");
 
-   if (env->old_pwd)
-      chdir(env->old_pwd);
+   if (env->old_pwd) {
+      if (chdir(env->old_pwd) == -1)
+         sdprintf(1, "WARNING: Could not chdir to %s: %s\n", env->old_pwd, strerror(errno));
+   }
 
    if (env->new_home)
       free(env->new_home);
@@ -815,7 +857,7 @@ int dropPrivilegeAndRun(dpr_function_t func, uid_t uid, void *input, char **outp
          exit(-1);
       }
       if (output_len) {
-         result = safe_write(pipe_fds[1], output_str, output_len+1);
+         result = safe_write(pipe_fds[1], child_output_str, output_len+1);
          if (result != output_len+1) {
             error = errno;
             fprintf(stderr, "Spindle error.  Could not write result string to pipe: %s\n", strerror(error));
@@ -989,7 +1031,11 @@ pid_t grandchild_fork()
    int result, fork_result = -1;
 
    pipe_fds[0] = pipe_fds[1] = -1;
-   pipe(pipe_fds);
+   result = pipe(pipe_fds);
+   if (result == -1) {
+      sdprintf(1, "ERROR: pipe() failed in grandchild_fork.  Aborting spindle\n");
+      return -1;
+   }
 
    child_pid = fork();
    if (child_pid == -1) {
@@ -1008,7 +1054,7 @@ pid_t grandchild_fork()
          sdprintf(1, "ERROR collecting pid after fork.  Aborting spindle\n");
          goto done;
       }
-      if (!WIFEXITED(status) && WEXITSTATUS(status) != 0) {
+      if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) {
          sdprintf(1, "ERROR with invalid child exit during grandchild fork.  Aborting spindle\n");
          goto done;
       }
@@ -1280,7 +1326,7 @@ char *readSpankEnv(spank_t spank, const char *envname)
       free(buffer);
       buffer = (char *) malloc(buffer_size);
    }
-   if (err == ESPANK_ENV_NOEXIST) {
+   if (err == ESPANK_ENV_NOEXIST || err == ESPANK_NOT_REMOTE || err == ESPANK_BAD_ARG) {
       free(buffer);
       buffer = getenv(envname);
       return buffer ? strdup(buffer) : NULL;

src/slurm_plugin/plugin_utils.h

@@ -45,6 +45,7 @@ char **getHostsParse(unsigned int num_hosts, const char *shortlist);
 
 int isFEHost(char **hostlist, unsigned int num_hosts);
 extern char *unique_file;
+void cleanup_unique_file();
 int isBEProc(spindle_args_t *params, unsigned int exit_phase);
 
 int doesFEExitSocketExist(spindle_args_t *params);