Change the behavior of orterun (mpirun, mpirexec) to search for

argv[0] and the cwd on the target node (i.e., the node where the
executable will be running in all systems except BProc, where the
searches are run on the node where orterun is invoked).
- fork pls now does cwd and argv[0] search in orted
- bproc pls does cwd and argv[0] search in orterun
- cwd behavior slightly different:
  - if user specifies a -wdir to orterun, we chdir() to there; if we
    can't for some reason, abort
  - if user does not specify a -wdir, try to chdir() to the dir where
    orterun was invoked.  If we can't for some reason (e.g., it
    doesn't exist on the target node), then try to chdir($HOME).  If
    we can't do that, then just live with whatever default directory
    we were put in.

This commit was SVN r9068.

NEWS

@@ -32,6 +32,10 @@ version 1.0.
 1.0.2
 -----
 
+- Change the behavior of orterun (mpirun, mpirexec) to search for
+  argv[0] and the cwd on the target node (i.e., the node where the
+  executable will be running in all systems except BProc, where the
+  searches are run on the node where orterun is invoked).
 - Fix race condition in shared memory transport that could cause
   crashes on machines with weak memory consistency models (including
   POWER/PowerPC machines).

orte/mca/pls/base/Makefile.am

@@ -23,6 +23,7 @@ headers += \
 
 libmca_pls_la_SOURCES += \
         base/pls_base_close.c \
+        base/pls_base_context.c \
         base/pls_base_open.c \
         base/pls_base_select.c \
 	base/pls_base_state.c \

orte/mca/pls/base/base.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
  * Copyright (c) 2004-2005 The University of Tennessee and The University
@@ -121,6 +121,30 @@ extern "C" {
     int orte_pls_base_proxy_terminate_job(orte_jobid_t jobid);
     int orte_pls_base_proxy_terminate_proc(const orte_process_name_t *proc);
 
+    /**
+     * Check that the cwd in an app context exists and is accessible.
+     * If the user specified the cwd and we can chdir to it, print an
+     * error and fail.  If the user didn't specify it (i.e., it's a
+     * default), then see if chdir($HOME) would succeed.
+     *
+     * If either chdir() would succeed and do_chdir is true, then
+     * actually do the chdir().
+     *
+     * If we fall back to the chdir($HOME), set context->cwd to be a
+     * string pointing to the home directory name (owned by the
+     * context; safe to free at destruction).
+     */
+    int orte_pls_base_check_context_cwd(orte_app_context_t *context,
+                                        bool do_chdir);
+
+    /**
+     * Check that the app exists and is executable.  If it is not,
+     * print and error and fail.  If it is, and if the app was a naked
+     * executable (i.e., no relative or absolute path), replace the
+     * app with the string containing the absolute pathname to the
+     * exectuable (owned by the context; safe to free at destruction).
+     */
+    int orte_pls_base_check_context_app(orte_app_context_t *context);
 #if defined(c_plusplus) || defined(__cplusplus)
 }
 #endif

orte/mca/pls/base/help-pls-base.txt

@@ -1,6 +1,6 @@
 # -*- text -*-
 #
-# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+# Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
 #                         University Research and Technology
 #                         Corporation.  All rights reserved.
 # Copyright (c) 2004-2005 The University of Tennessee and The University
@@ -22,3 +22,24 @@ No available launching agents were found.
 This is an unusual error; it means that Open RTE was unable to find
 any mechanism to launch proceses, and therefore is unable start the
 process(es) in your application.
+[chdir-error]
+Failed to change to the working directory:
+
+Host:      %s
+Directory: %s
+
+The error returned was "%s".  Execution will now abort.
+[argv0-not-found]
+Failed to find the following executable:
+
+Host:       %s
+Executable: %s
+
+Cannot continue.
+[argv0-not-accessible]
+Failed to find or execute the following executable:
+
+Host:       %s
+Executable: %s
+
+Cannot continue.

orte/mca/pls/base/pls_base_context.c

@@ -0,0 +1,167 @@
+/*
+ * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2006 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "orte_config.h"
+
+#include <string.h>
+#include <stdlib.h>
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+#ifdef HAVE_SYS_PARAM_H
+#include <sys/param.h>
+#endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#include <errno.h>
+
+#include "opal/util/show_help.h"
+#include "opal/util/basename.h"
+#include "opal/util/path.h"
+#include "orte/orte_constants.h"
+#include "orte/mca/pls/base/base.h"
+#include "orte/mca/errmgr/errmgr.h"
+
+
+int orte_pls_base_check_context_cwd(orte_app_context_t *context,
+                                    bool want_chdir)
+{
+    bool good;
+    char *tmp;
+    char hostname[MAXHOSTNAMELEN];
+    struct stat buf;
+
+    /* Use hostname in a few messages below */
+    gethostname(hostname, sizeof(hostname));
+
+    /* If the directory does not exist, or stat() otherwise fails to
+       get info about it, then set good = false. */
+    good = true;
+    if (!(0 == stat(context->cwd, &buf) && S_ISDIR(buf.st_mode))) {
+        good = false;
+    }
+
+    /* If the directory does exist, and we want to chdir, and the
+       chdir fails, then set good = false. */
+    if (good && want_chdir && 0 != chdir(context->cwd)) {
+        good = false;
+    }
+
+    /* If either of the above failed, go into this block */
+    if (!good) {
+        /* See if the directory was a user-specified directory.  If it
+           was, barf because they specifically asked for something we
+           can't provide. */
+        if (context->user_specified_cwd) {
+            opal_show_help("help-pls-base.txt", "chdir-error",
+                           true, hostname, context->cwd, strerror(errno));
+            return ORTE_ERR_NOT_FOUND;
+        }
+        
+        /* If the user didn't specifically ask for it, then it
+           was a system-supplied default directory, so it's ok
+           to not go there.  Try to go to the $HOME directory
+           instead. */
+        tmp = getenv("HOME");
+        if (NULL != tmp) {
+            /* Try $HOME.  Same 2 tests as above. */
+            good = true;
+            if (!(0 == stat(tmp, &buf) && S_ISDIR(buf.st_mode))) {
+                good = false;
+            }
+            if (good && want_chdir && 0 != chdir(tmp)) {
+                good = false;
+            }
+            if (!good) {
+                opal_show_help("help-pls-base.txt", "chdir-error",
+                               true, tmp, strerror(errno));
+                return ORTE_ERR_NOT_FOUND;
+            }
+            
+            /* Reset the pwd in this local copy of the
+               context */
+            free(context->cwd);
+            context->cwd = strdup(tmp);
+        }
+        
+        /* If we couldn't find $HOME, then just take whatever
+           the default directory is -- assumedly there *is*
+           one, or we wouldn't be running... */
+    }
+
+    /* All happy */
+    return ORTE_SUCCESS;
+}
+
+int orte_pls_base_check_context_app(orte_app_context_t *context)
+{
+    char *tmp;
+    char hostname[MAXHOSTNAMELEN];
+
+    /* Use hostname in a few messages below */
+    gethostname(hostname, sizeof(hostname));
+
+    /* If the app is a naked filename, we need to do a path search for
+       it.  orterun will send in whatever the user specified (e.g.,
+       "orterun -np 2 uptime"), so in some cases, we need to search
+       the path to verify that we can find it.  Here's the
+       possibilities:
+           
+       1. The user specified an absolute pathname for the executable.
+          We simply need to verify that it exists and we can run it.
+           
+       2. The user specified a relative pathname for the executable.
+          Ditto with #1 -- based on the cwd, we need to verify that it
+          exists and we can run it.
+           
+       3. The user specified a naked filename.  We need to search the
+          path, find a match, and verify that we can run it.
+
+       Note that in some cases, we won't be doing this work here --
+       bproc, for example, does not use the fork pls for launching, so
+       it does this same work over there. */
+
+    tmp = opal_basename(context->argv[0]);
+    if (strlen(tmp) == strlen(context->argv[0])) {
+        /* If this is a naked executable -- no relative or absolute
+           pathname -- then search the PATH for it */
+        free(tmp);
+        tmp = opal_path_findv(context->argv[0], X_OK, environ, context->cwd);
+        if (NULL == tmp) {
+            opal_show_help("help-pls-base.txt",
+                           "argv0-not-found",
+                           true, hostname, context->argv[0]);
+            return ORTE_ERR_NOT_FOUND;
+        }
+        free(context->app);
+        context->app = tmp;
+    } else {
+        if (0 != access(context->app, X_OK)) {
+            opal_show_help("help-pls-base.txt",
+                           "argv0-not-accessible",
+                           true, hostname, context->argv[0]);
+            return ORTE_ERR_NOT_FOUND;
+        }
+    }
+    
+    /* All was good */
+    return ORTE_SUCCESS;
+}
+

orte/mca/pls/bproc/pls_bproc.c

@@ -391,7 +391,7 @@ static void orte_pls_bproc_setup_env(char *** env)
     int rc;
     int num_env;
 
-    num_env = opal_argv_count(env);
+    num_env = opal_argv_count(*env);
     /* append mca parameters to our environment */
     if(ORTE_SUCCESS != (rc = mca_base_param_build_env(env, &num_env, false))) {
         ORTE_ERROR_LOG(rc);
@@ -803,7 +803,13 @@ int orte_pls_bproc_launch(orte_jobid_t jobid) {
     int num_processes = 0;
     int context = 0;
     size_t idx, j;
+    char cwd_save[OMPI_PATH_MAX + 1];
 
+    if (NULL == getcwd(cwd_save, sizeof(cwd_save))) {
+        ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
+        return ORTE_ERR_NOT_FOUND;
+    }
+    cwd_save[sizeof(cwd_save) - 1] = '\0';
     /* query for the application context and allocated nodes */
     OBJ_CONSTRUCT(&mapping, opal_list_t);
     if(ORTE_SUCCESS != (rc = orte_rmaps_base_get_map(jobid, &mapping))) {
@@ -827,6 +833,28 @@ int orte_pls_bproc_launch(orte_jobid_t jobid) {
      * until we are done launching */
     OPAL_THREAD_LOCK(&mca_pls_bproc_component.lock);
 
+    for (item =  opal_list_get_first(&mapping);
+         item != opal_list_get_end(&mapping);
+         item =  opal_list_get_next(item)) {
+        size_t i;
+        map = (orte_rmaps_base_map_t*) item;
+        for (i = 0; i < map->num_procs; ++i) {
+            orte_app_context_t *context = map->app;
+
+            /* Check that the cwd is sane, but don't chdir there */
+            rc = orte_pls_base_check_context_cwd(context, false);
+            if (ORTE_SUCCESS != rc) {
+                goto cleanup;
+            }
+
+            /* Check that the app exists and is executable */
+            rc = orte_pls_base_check_context_app(context);
+            if (ORTE_SUCCESS != rc) {
+                goto cleanup;
+            }
+        }
+    }
+
     /* create an array to hold the pointers to the node arrays for each app
      * context. Also, create an array to hold the lengths of the node arrays */
     node_array = malloc(opal_list_get_size(&mapping) * sizeof(int *));
@@ -900,6 +928,12 @@ int orte_pls_bproc_launch(orte_jobid_t jobid) {
         item != opal_list_get_end(&mapping);
         item =  opal_list_get_next(item)) {
         map = (orte_rmaps_base_map_t*)item;
+
+        rc = orte_pls_base_check_context_cwd(map->app, true);
+        if (ORTE_SUCCESS != rc) {
+            goto cleanup;
+        }
+
         rc = orte_pls_bproc_launch_app(cellid, jobid, map, num_processes,
                                     vpid_launch, vpid_start, map->app->idx,
                                     node_array[context], node_array_len[context]);
@@ -914,9 +948,11 @@ int orte_pls_bproc_launch(orte_jobid_t jobid) {
 
     mca_pls_bproc_component.done_launching = true;
 cleanup:
+    chdir(cwd_save);
     OPAL_THREAD_UNLOCK(&mca_pls_bproc_component.lock);
-    while(NULL != (item = opal_list_remove_first(&mapping)))
+    while(NULL != (item = opal_list_remove_first(&mapping))) {
         OBJ_RELEASE(item);
+    }
     if(NULL != node_array) {
         free(node_array);
     }

orte/mca/pls/fork/help-orte-pls-fork.txt

@@ -1,6 +1,6 @@
 # -*- text -*-
 #
-# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+# Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
 #                         University Research and Technology
 #                         Corporation.  All rights reserved.
 # Copyright (c) 2004-2005 The University of Tennessee and The University
@@ -19,16 +19,36 @@
 # This is the US/English general help file for Open RTE's orterun.
 #
 [orte-pls-fork:chdir-error]
-Warning: Could not change to the working directory "%s".
+Failed to change to the working directory:
 
-The error returned was "%s".
+Host:      %s
+Directory: %s
 
-This is not necessarily a fatal condition, and execution will continue.
+The error returned was "%s".  Execution will now abort.
+[orte-pls-fork:argv0-not-found]
+Failed to find the following executable:
 
+Host:       %s
+Executable: %s
+
+Cannot continue.
+[orte-pls-fork:argv0-not-accessible]
+Failed to find or execute the following executable:
+
+Host:       %s
+Executable: %s
+
+Cannot continue.
 [orte-pls-fork:execv-error]
 Could not execute the executable "%s": %s
 
 This could mean that your PATH or executable name is wrong, or that you do not
 have the necessary permissions.  Please ensure that the executable is able to be
 found and executed.
+[orte-pls-fork:could-not-kill]
+WARNING: A process refused to die!
 
+Host: %s
+PID:  %d
+
+This process may still be running and/or consuming resources.

orte/mca/pls/fork/pls_fork.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
  * Copyright (c) 2004-2005 The University of Tennessee and The University
@@ -57,6 +57,7 @@ struct orte_pls_fork_component_t {
     int debug;
     int priority;
     int reap;
+    int timeout_before_sigkill;
     size_t num_children;
     opal_mutex_t lock;
     opal_condition_t cond;

orte/mca/pls/fork/pls_fork_component.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
  * Copyright (c) 2004-2005 The University of Tennessee and The University
@@ -101,6 +101,9 @@ int orte_pls_fork_component_open(void)
     mca_base_param_reg_int(c, "reap", 
                            "Whether to wait to reap all children before finalizing or not",
                            false, false, 1, &mca_pls_fork_component.reap);
+    mca_base_param_reg_int(c, "reap_timeout", 
+                           "When killing children processes, first send a SIGTERM, then wait at least this timeout (in seconds), then send a SIGKILL",
+                           false, false, 0, &mca_pls_fork_component.timeout_before_sigkill);
     mca_base_param_reg_int(c, "priority", 
                            "Priority of this component",
                            false, false, 1, &mca_pls_fork_component.priority);

orte/mca/pls/fork/pls_fork_module.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
  * Copyright (c) 2004-2005 The University of Tennessee and The University
@@ -34,6 +34,13 @@
 #include <sys/wait.h>
 #endif
 #include <signal.h>
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+#include <time.h>
+#if HAVE_SYS_PARAM_H
+#include <sys/param.h>
+#endif
 
 #include "orte/orte_constants.h"
 #include "opal/event/event.h"
@@ -41,12 +48,16 @@
 #include "opal/util/output.h"
 #include "opal/mca/paffinity/base/base.h"
 #include "opal/util/show_help.h"
+#include "opal/util/path.h"
+#include "opal/util/basename.h"
+#include "opal/class/opal_value_array.h"
 #include "orte/util/sys_info.h"
 #include "orte/util/univ_info.h"
 #include "opal/util/opal_environ.h"
 #include "orte/util/session_dir.h"
 #include "orte/runtime/orte_wait.h"
 #include "orte/mca/errmgr/errmgr.h"
+#include "orte/mca/errmgr/base/base.h"
 #include "orte/mca/iof/iof.h"
 #include "orte/mca/iof/base/iof_base_setup.h"
 #include "opal/mca/base/mca_base_param.h"
@@ -62,7 +73,6 @@
 #include "orte/mca/soh/base/base.h"
 #include "orte/mca/pls/fork/pls_fork.h"
 
-
 extern char **environ;
 
 #if OMPI_HAVE_POSIX_THREADS && OMPI_THREADS_HAVE_DIFFERENT_PIDS && OMPI_ENABLE_PROGRESS_THREADS
@@ -84,6 +94,59 @@ orte_pls_base_module_1_0_0_t orte_pls_fork_module = {
 static void set_handler_default(int sig);
 
 
+static bool orte_pls_fork_child_died(pid_t pid, unsigned int timeout) 
+{
+    time_t end;
+
+    end = time(NULL) + timeout;
+    while (time(NULL) < end) {
+        if (pid == waitpid(pid, NULL, WNOHANG)) {
+            /* It died -- return success */
+            return true;
+        }
+
+        /* Sleep for a second */
+        sleep(1);
+    }
+
+    /* The child didn't die, so return false */
+    return false;
+}
+
+static void orte_pls_fork_kill_processes(opal_value_array_t *pids)
+{
+    size_t i;
+    pid_t pid;
+
+    for (i = 0; i < opal_value_array_get_size(pids); ++i) {
+        pid = OPAL_VALUE_ARRAY_GET_ITEM(pids, pid_t, i);
+
+        /* Send a sigterm to the process.  If we get ESRCH back, that
+           means the process is already dead, so just proceed on to
+           the reaping of it.  If we get any other error back, just
+           skip it and go on to the next process. */
+        if (0 != kill(pid, SIGTERM) && ESRCH != errno) {
+            continue;
+        }
+
+        /* The kill succeeded.  Wait up to timeout_before_sigkill
+           seconds to see if it died. */
+        
+        if (!orte_pls_fork_child_died(pid, mca_pls_fork_component.timeout_before_sigkill)) {
+            kill(pid, SIGKILL);
+            /* Double check that it actually died */
+            if (!orte_pls_fork_child_died(pid, mca_pls_fork_component.timeout_before_sigkill)) {
+                char hostname[MAXHOSTNAMELEN];
+                gethostname(hostname, sizeof(hostname));
+                
+                opal_show_help("help-orte-pls-fork.txt",
+                               "orte-pls-fork:could-not-kill",
+                               true, hostname, pid);
+            }
+        }
+    }
+}
+
 /*
  *  Wait for a callback indicating the child has completed.
  */
@@ -135,7 +198,7 @@ static int orte_pls_fork_proc(
     int rc;
     sigset_t sigs;
     orte_vpid_t vpid;
-    int i;
+    int i = 0, p[2];
 
     /* should pull this information from MPIRUN instead of going with
        default */
@@ -183,43 +246,55 @@ static int orte_pls_fork_proc(
                         &si,
                         &pi)){
        /* actual error can be got by simply calling GetLastError() */
-       return OMPI_ERROR;
+       return ORTE_ERROR;
     }
     /* get child pid */
     process_id = GetProcessId(&pi);
     pid = (int) process_id;
  }
 #endif
+    /* A pipe is used to communicate between the parent and child to
+       indicate whether the exec ultiimately succeeded or failed.  The
+       child sets the pipe to be close-on-exec; the child only ever
+       writes anything to the pipe if there is an error (e.g.,
+       executable not found, exec() fails, etc.).  The parent does a
+       blocking read on the pipe; if the pipe closed with no data,
+       then the exec() succeeded.  If the parent reads something from
+       the pipe, then the child was letting us know that it failed. */
+    if (pipe(p) < 0) {
+        ORTE_ERROR_LOG(ORTE_ERR_IN_ERRNO);
+        return ORTE_ERR_IN_ERRNO;
+    }
 
+    /* Fork off the child */
     pid = fork();
     if(pid < 0) {
         ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE);
         return ORTE_ERR_OUT_OF_RESOURCE;
     }
 
-    if(pid == 0) {
+    if (pid == 0) {
         char *param, *param2;
         char *uri;
         char **environ_copy;
         long fd, fdmax = sysconf(_SC_OPEN_MAX);
-#if 0
-        /* for gperf - setup a new directory for each executable */
-        char path[PATH_MAX];
 
-        /* set working directory */
-        sprintf(path, "%s/%d", context->cwd, getpid());
-        if(mkdir(path,0777) != 0) {
-            ORTE_ERROR_LOG(ORTE_ERR_BAD_PARAM);
+        /* Setup the pipe to be close-on-exec */
+        close(p[0]);
+        fcntl(p[1], F_SETFD, FD_CLOEXEC);
+
+        /* setup stdout/stderr so that any error messages that we may
+           print out will get displayed back at orterun */
+        orte_iof_base_setup_child(&opts);
+
+        /* Try to change to the context cwd and check that the app
+           exists and is executable */
+        if (ORTE_SUCCESS != orte_pls_base_check_context_cwd(context, true) ||
+            ORTE_SUCCESS != orte_pls_base_check_context_app(context)) {
+            /* Tell the parent that Badness happened */
+            write(p[1], &i, sizeof(int));
+            exit(-1);
         }
-        if(chdir(path) != 0) {
-            ORTE_ERROR_LOG(ORTE_ERR_BAD_PARAM);
-        }
-#else
-        if(chdir(context->cwd) != 0) {
-            opal_show_help("help-orte-pls-fork.txt", "orte-pls-fork:chdir-error",
-                           true, context->cwd, strerror(errno));
-        }
-#endif
         
         /* setup base environment: copy the current environ and merge
            in the app context environ */
@@ -316,9 +391,6 @@ static int orte_pls_fork_proc(
         orte_ns_nds_env_put(&proc->proc_name, vpid_start, vpid_range,
                             &environ_copy);
 
-        /* setup stdout/stderr */
-        orte_iof_base_setup_child(&opts);
-
         /* close all file descriptors w/ exception of stdin/stdout/stderr */
         for(fd=3; fd<fdmax; fd++)
             close(fd);
@@ -355,7 +427,6 @@ static int orte_pls_fork_proc(
         opal_show_help("help-orte-pls-fork.txt", "orte-pls-fork:execv-error",
                        true, context->app, strerror(errno));
         exit(-1);
-
     } else {
 
         /* connect endpoints IOF */
@@ -365,15 +436,41 @@ static int orte_pls_fork_proc(
             return rc;
         }
 
+        /* Wait to read something from the pipe or close */
+        close(p[1]);
+        while (1) {
+            rc = read(p[0], &i, sizeof(int));
+            if (rc < 0) {
+                /* Signal interrupts are ok */
+                if (errno == EINTR) {
+                    continue;
+                }
+                /* Other errno's are bad */
+                return ORTE_ERR_IN_ERRNO;
+                break;
+            } else if (0 == rc) {
+                /* Child was successful in exec'ing! */
+                break;
+            } else {
+                /* Doh -- child failed.  The child already printed a
+                   suitable error message, so disable all
+                   ORTE_ERROR_LOG reporting after this. */
+                return ORTE_ERR_FATAL;
+                break;
+            }
+        }
+
         /* save the pid in the registry */
-        if(ORTE_SUCCESS != (rc = orte_pls_base_set_proc_pid(&proc->proc_name, pid))) {
+        if (ORTE_SUCCESS != 
+            (rc = orte_pls_base_set_proc_pid(&proc->proc_name, pid))) {
             ORTE_ERROR_LOG(rc);
             return rc;
         }
 
-        /* wait for the child process - dont register for wait callback until
-         * after I/O is setup and the pid registered - otherwise can receive the
-         * wait callback before the above is ever completed
+        /* wait for the child process - dont register for wait
+         * callback until after I/O is setup and the pid registered -
+         * otherwise can receive the wait callback before the above is
+         * ever completed
          */
         OPAL_THREAD_LOCK(&mca_pls_fork_component.lock);
         mca_pls_fork_component.num_children++;
@@ -434,7 +531,17 @@ int orte_pls_fork_launch(orte_jobid_t jobid)
                                     vpid_range,
                                     (num_processes > num_processors) ?
                                     false : true, i);
+
             if (ORTE_SUCCESS != rc) {
+                /* Set the state of this process, and all remaining
+                   processes to be launched to ABORTED.  This will
+                   cause the entire job to abort. */
+                for (; i < map->num_procs; ++i) {
+                    orte_soh.set_proc_soh(&map->procs[i]->proc_name, 
+                                          ORTE_PROC_STATE_ABORTED, 0);
+                }
+
+                /* Propagate the error up the stack */
                 ORTE_ERROR_LOG(rc);
                 goto cleanup;
             }
@@ -462,6 +569,11 @@ int orte_pls_fork_terminate_job(orte_jobid_t jobid)
     orte_gpr_value_t** values = NULL;
     size_t i, k, num_values = 0;
     int rc;
+    opal_value_array_t pids;
+
+    /* setup the pid array */
+    OBJ_CONSTRUCT(&pids, opal_value_array_t);
+    opal_value_array_init(&pids, sizeof(pid_t));
 
     /* query the job segment on the registry */
     if(ORTE_SUCCESS != (rc = orte_schema.get_job_segment_name(&segment, jobid))) {
@@ -505,10 +617,17 @@ int orte_pls_fork_terminate_job(orte_jobid_t jobid)
             }
         }
         if (0 != pid) {
-            kill(pid, SIGKILL);
+            opal_value_array_append_item(&pids, &pid);
         }
         OBJ_RELEASE(value);
     }
+
+    /* If we have processes to kill, go kill them */
+    if (opal_value_array_get_size(&pids) > 0) {
+        orte_pls_fork_kill_processes(&pids);
+    }
+    OBJ_DESTRUCT(&pids);
+
     if(NULL != values) {
         free(values);
     }

orte/mca/pls/slurm/pls_slurm_module.c

@@ -481,6 +481,9 @@ static int pls_slurm_start_proc(int argc, char **argv, char **env,
             free(newenv);
         }
 
+        /* JMS -- when not in debug mode, tie stdout/stderr to dev
+           null so we don't see messages from orted */
+
         /* get the srun process out of orterun's process group so that
            signals sent from the shell (like those resulting from
            cntl-c) don't get sent to srun */

orte/mca/rmgr/base/data_type_support/rmgr_data_type_copy_fns.c

@@ -1,8 +1,10 @@
 /*
- * Copyright (c) 2004-2005 The Trustees of Indiana University.
- *                         All rights reserved.
- * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
- *                         All rights reserved.
+ * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
@@ -50,6 +52,7 @@ int orte_rmgr_base_copy_app_context(orte_app_context_t **dest, orte_app_context_
     (*dest)->argv = opal_argv_copy(src->argv);
     (*dest)->env = opal_argv_copy(src->env);
     (*dest)->cwd = strdup(src->cwd);
+    (*dest)->user_specified_cwd = src->user_specified_cwd;
     (*dest)->num_map = src->num_map;
 
     if (0 < src->num_map) {

orte/mca/rmgr/base/data_type_support/rmgr_data_type_packing_fns.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
  * Copyright (c) 2004-2005 The University of Tennessee and The University
@@ -37,7 +37,7 @@ int orte_rmgr_base_pack_app_context(orte_buffer_t *buffer, void *src,
                                  size_t num_vals, orte_data_type_t type)
 {
     int rc, count;
-    int8_t have_prefix;
+    int8_t have_prefix, user_specified;
     size_t i;
     orte_app_context_t **app_context;
 
@@ -105,6 +105,18 @@ int orte_rmgr_base_pack_app_context(orte_buffer_t *buffer, void *src,
             return rc;
         }
 
+        /* pack the user specified cwd flag */
+        if (app_context[i]->user_specified_cwd) {
+            user_specified = 1;
+        } else {
+            user_specified = 0;
+        }
+        if (ORTE_SUCCESS != (rc = orte_dss_pack_buffer(buffer,
+                        (void*)(&user_specified), 1, ORTE_INT8))) {
+            ORTE_ERROR_LOG(rc);
+            return rc;
+        }
+
         /* Pack the map data */
         if (ORTE_SUCCESS != (rc = orte_dss_pack_buffer(buffer,
                 (void*)(&(app_context[i]->num_map)), 1, DSS_TYPE_SIZE_T))) {

orte/mca/rmgr/base/data_type_support/rmgr_data_type_print_fns.c

@@ -1,8 +1,10 @@
 /*
- * Copyright (c) 2004-2005 The Trustees of Indiana University.
- *                         All rights reserved.
- * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
- *                         All rights reserved.
+ * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
@@ -65,7 +67,7 @@ int orte_rmgr_base_print_app_context(char **output, char *prefix, orte_app_conte
         tmp = tmp2;
     }
 
-    asprintf(&tmp2, "%s\n%s\tWorking dir: %s\n%s\tNum maps: %lu", tmp, pfx2, src->cwd,
+    asprintf(&tmp2, "%s\n%s\tWorking dir: %s (user: %d)\n%s\tNum maps: %lu", tmp, pfx2, src->cwd, (int) src->user_specified_cwd,
                     pfx2, (unsigned long)src->num_map);
     free(tmp);
     tmp = tmp2;

orte/mca/rmgr/base/data_type_support/rmgr_data_type_unpacking_fns.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
  * Copyright (c) 2004-2005 The University of Tennessee and The University
@@ -39,7 +39,7 @@ int orte_rmgr_base_unpack_app_context(orte_buffer_t *buffer, void *dest,
     int rc, count;
     orte_app_context_t **app_context;
     size_t i, max_n=1;
-    int8_t have_prefix;
+    int8_t have_prefix, user_specified;
 
     /* unpack into array of app_context objects */
     app_context = (orte_app_context_t**) dest;
@@ -132,6 +132,18 @@ int orte_rmgr_base_unpack_app_context(orte_buffer_t *buffer, void *dest,
             return rc;
         }
 
+        /* unpack the user-specified cwd flag */
+        if (ORTE_SUCCESS != (rc = orte_dss_unpack_buffer(buffer, &user_specified,
+                    &max_n, ORTE_INT8))) {
+            ORTE_ERROR_LOG(rc);
+            return rc;
+        }
+        if (user_specified) {
+            app_context[i]->user_specified_cwd = true;
+        } else {
+            app_context[i]->user_specified_cwd = false;
+        }
+
         /* unpack the map data */
         max_n=1;
         if (ORTE_SUCCESS != (rc = orte_dss_unpack_buffer(buffer, &(app_context[i]->num_map),

orte/mca/rmgr/rmgr_types.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
  * Copyright (c) 2004-2005 The University of Tennessee and The University
@@ -91,6 +91,8 @@ typedef struct {
     char  **env;
     /** Current working directory for this app */
     char   *cwd;
+    /** Whether the cwd was set by the user or by the system */
+    bool user_specified_cwd;
     /** Length of the map_data array, not including the final NULL entry */
     size_t num_map;
     /** Mapping data about how this app should be laid out across CPUs

orte/tools/orterun/help-orterun.txt

@@ -1,6 +1,6 @@
 # -*- text -*-
 #
-# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+# Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
 #                         University Research and Technology
 #                         Corporation.  All rights reserved.
 # Copyright (c) 2004-2005 The University of Tennessee and The University
@@ -34,11 +34,6 @@ requsted more nodes than exist in your cluster).
 
 While probably only useful to Open RTE developers, the error returned
 was %d.
-[orterun:executable-not-found]
-%s could not find the executable "%s".
-
-Please check your PATH and ensure that the executable is able to be
-found and executed.
 [orterun:error-spawning]
 %s was unable to start the specified application.  An attempt has been
 made to clean up all processes that did start.  The error returned was
@@ -66,8 +61,8 @@ argument.
 
 It is possible that you forgot to specify how many processes to run
 via the "-np" argument.
-[orterun:syscall-failed]
-%s encountered a system call failure.  This should not happen, and
+[orterun:call-failed]
+%s encountered a %s call failure.  This should not happen, and
 usually indicates an error within the operating system itself.
 Specifically, the following error occurred:
 

orte/tools/orterun/orterun.c

@@ -1,6 +1,6 @@
 /* -*- C -*-
  *
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
  * Copyright (c) 2004-2005 The University of Tennessee and The University
@@ -44,7 +44,6 @@
 #include "opal/util/cmd_line.h"
 #include "opal/util/opal_environ.h"
 #include "opal/util/output.h"
-#include "opal/util/path.h"
 #include "opal/util/show_help.h"
 #include "opal/util/trace.h"
 
@@ -279,8 +278,8 @@ int orterun(int argc, char *argv[])
     array_size = orte_pointer_array_get_size(apps_pa);
     apps = malloc(sizeof(orte_app_context_t *) * array_size);
     if (NULL == apps) {
-        opal_show_help("help-orterun.txt", "orterun:syscall-failed",
-                       true, orterun_basename, "malloc returned NULL", errno);
+        opal_show_help("help-orterun.txt", "orterun:call-failed",
+                       true, orterun_basename, "system", "malloc returned NULL", errno);
         exit(1);
     }
     num_apps = 0;
@@ -301,8 +300,8 @@ int orterun(int argc, char *argv[])
     }
     proc_infos = malloc(sizeof(struct proc_info_t) * j);
     if (NULL == proc_infos) {
-        opal_show_help("help-orterun.txt", "orterun:syscall-failed",
-                       true, orterun_basename, "malloc returned NULL", errno);
+        opal_show_help("help-orterun.txt", "orterun:call-failed",
+                       true, orterun_basename, "system", "malloc returned NULL", errno);
         exit(1);
     }
     for (i = 0; i < j; ++i) {
@@ -547,10 +546,10 @@ static void dump_aborted_procs(orte_jobid_t jobid)
             proc_infos[rank].exit_status = exit_status;
         }
 
-        if (WIFSIGNALED(exit_status) && rank_found &&
-            !proc_infos[rank].reported) {
+        if (rank_found && !proc_infos[rank].reported) {
             proc_infos[rank].reported = true;
 
+            if (WIFSIGNALED(exit_status)) {
                 if (9 == WTERMSIG(exit_status)) {
                     ++num_killed;
                 } else {
@@ -562,6 +561,7 @@ static void dump_aborted_procs(orte_jobid_t jobid)
                     ++num_aborted;
                 }
             }
+        }
 
         /* If we haven't done so already, hold the exit_status so we
            can return it when exiting.  Specifically, keep the first
@@ -592,7 +592,7 @@ static void job_state_callback(orte_jobid_t jobid, orte_proc_state_t state)
 
     OPAL_THREAD_LOCK(&orterun_globals.lock);
 
-    /* Note that there's only two states that we're interested in
+    /* Note that there's only three states that we're interested in
        here:
 
        ABORTED: which means that one or more processes have aborted
@@ -602,6 +602,9 @@ static void job_state_callback(orte_jobid_t jobid, orte_proc_state_t state)
        TERMINATED: which means that all the processes in the job have
                 completed (normally and/or abnormally).
 
+       AT_STG1: which means that everyone has hit stage gate 1, so we
+                can do the parallel debugger startup stuff.
+
        Remember that the rmgr itself will also be called for the
        ABORTED state and call the pls.terminate_job, which will result
        in killing all the other processes. */
@@ -1159,9 +1162,11 @@ static int create_app(int argc, char* argv[], orte_app_context_t **app_ptr,
 
     if (NULL != orterun_globals.wdir) {
         app->cwd = strdup(orterun_globals.wdir);
+        app->user_specified_cwd = true;
     } else {
         getcwd(cwd, sizeof(cwd));
         app->cwd = strdup(cwd);
+        app->user_specified_cwd = false;
     }
 
     /* Did the user specify a specific prefix for this app_context_t */
@@ -1231,20 +1236,15 @@ static int create_app(int argc, char* argv[], orte_app_context_t **app_ptr,
         goto cleanup;
     }
 
-    /* Find the argv[0] in the path, but only if not absolute or
-       relative pathname was specified */
+    /* Do not try to find argv[0] here -- the starter is responsible
+       for that because it may not be relevant to try to find it on
+       the node where orterun is executing.  So just strdup() argv[0]
+       into app. */
 
-    value = opal_basename(app->argv[0]);
-    if (strlen(value) == strlen(app->argv[0])) {
-        app->app = opal_path_findv(app->argv[0], 0, environ, app->cwd);
-    } else {
     app->app = strdup(app->argv[0]);
-    }
-    free(value);
-
     if (NULL == app->app) {
-        opal_show_help("help-orterun.txt", "orterun:executable-not-found",
-                       true, orterun_basename, app->argv[0], orterun_basename);
+        opal_show_help("help-orterun.txt", "orterun:call-failed",
+                       true, orterun_basename, "library", "strdup returned NULL", errno);
         rc = ORTE_ERR_NOT_FOUND;
         goto cleanup;
     }