Add the ability to directly launch procs via rsh/ssh. Collect common functions in plm/base. Create a new global param to set assume_same_shell, alias'd back to plm_rsh_assume_same_shell (not deprecated).

This commit was SVN r21328.

orte/mca/ess/slave/ess_slave_module.c

@@ -209,14 +209,14 @@ static char* proc_get_hostname(orte_process_name_t *proc)
 
 static uint32_t proc_get_arch(orte_process_name_t *proc)
 {
-    /* if it is me, the answer is my arch */
-    if (proc->jobid == ORTE_PROC_MY_NAME->jobid &&
-        proc->vpid == ORTE_PROC_MY_NAME->vpid) {
-        return orte_process_info.arch;
-    }
-    
-    /* otherwise, no idea */
+    /* can only work in homogeneous environments */
+#if OPAL_ENABLE_HETEROGENEOUS_SUPPORT
+    /* not supported */
     return 0;
+#else
+    /* just return my arch */
+    return orte_process_info.arch;
+#endif
 }
 
 static int update_arch(orte_process_name_t *proc, uint32_t arch)

orte/mca/plm/base/plm_base_open.c

@@ -59,6 +59,7 @@ static void slave_file_construct(orte_slave_files_t *ptr)
 {
     ptr->node = NULL;
     ptr->local = false;
+    ptr->prefix = NULL;
     ptr->bootproxy = NULL;
     ptr->positioned = false;
     OBJ_CONSTRUCT(&ptr->apps, opal_pointer_array_t);
@@ -72,6 +73,7 @@ static void slave_file_destruct(orte_slave_files_t *ptr)
     char *cptr;
     
     if (NULL != ptr->node) free(ptr->node);
+    if (NULL != ptr->prefix) free(ptr->prefix);
     if (NULL != ptr->bootproxy) free(ptr->bootproxy);
     for (i=0; i < ptr->apps.size; i++) {
         if (NULL != (cptr = (char*)opal_pointer_array_get_item(&ptr->apps, i))) {

orte/mca/plm/base/plm_private.h

@@ -48,6 +48,7 @@ typedef struct {
     opal_list_item_t super;
     char *node;
     bool local;
+    char *prefix;
     char *bootproxy;
     bool positioned;
     opal_pointer_array_t apps;
@@ -110,6 +111,12 @@ ORTE_DECLSPEC int orte_plm_base_setup_orted_cmd(int *argc, char ***argv);
 ORTE_DECLSPEC int orte_plm_base_local_slave_launch(orte_job_t *jdata);
 ORTE_DECLSPEC int orte_plm_base_rsh_launch_agent_setup(void);
 ORTE_DECLSPEC void orte_plm_base_local_slave_finalize(void);
+ORTE_DECLSPEC int orte_plm_base_setup_rsh_launch(char *nodename, orte_app_context_t *app,
+                                                 char *rcmd, char ***argv, char **exec_path);
+ORTE_DECLSPEC int orte_plm_base_append_bootproxy_args(orte_app_context_t *app, char ***argv,
+                                                      orte_jobid_t jobid, orte_vpid_t vpid,
+                                                      int num_nodes, orte_vpid_t num_procs, orte_local_rank_t lrank,
+                                                      orte_vpid_t nlocal, int nslots);
 
 /**
  * Heartbeat support

orte/mca/plm/rsh/plm_rsh.h

@@ -64,7 +64,6 @@ int orte_plm_rsh_signal_job(orte_jobid_t, int32_t);
  */
 struct orte_plm_rsh_component_t {
     orte_plm_base_component_t super;
-    bool assume_same_shell;
     bool force_rsh;
     bool disable_qrsh;
     bool using_qrsh;

orte/mca/plm/rsh/plm_rsh_component.c

@@ -135,11 +135,6 @@ int orte_plm_rsh_component_open(void)
                            "Delay (in seconds) between invocations of the remote agent, but only used when the \"debug\" MCA parameter is true, or the top-level MCA debugging is enabled (otherwise this value is ignored)",
                            false, false, 1,
                            &mca_plm_rsh_component.delay);
-    mca_base_param_reg_int(c, "assume_same_shell",
-                           "If set to 1, assume that the shell on the remote node is the same as the shell on the local node.  Otherwise, probe for what the remote shell.",
-                           false, false, 1, &tmp);
-    mca_plm_rsh_component.assume_same_shell = OPAL_INT_TO_BOOL(tmp);
-    
     mca_base_param_reg_int(c, "tree_spawn",
                            "If set to 1, launch via a tree-based topology",
                            false, false, (int)false, &tmp);

orte/mca/plm/rsh/plm_rsh_module.c

@@ -391,7 +391,7 @@ static int setup_shell(orte_plm_rsh_shell_t *rshell,
                          local_shell, orte_plm_rsh_shell_name[local_shell]));
     
     /* What is our remote shell? */
-    if (mca_plm_rsh_component.assume_same_shell) {
+    if (orte_assume_same_shell) {
         remote_shell = local_shell;
         OPAL_OUTPUT_VERBOSE((1, orte_plm_globals.output,
                              "%s plm:rsh: assuming same remote shell as local shell",

orte/mca/plm/rshd/Makefile.am

@@ -0,0 +1,45 @@
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+
+dist_pkgdata_DATA = help-plm-rshd.txt
+
+sources = \
+        plm_rshd.h \
+        plm_rshd_component.c \
+        plm_rshd_module.c
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if OMPI_BUILD_plm_rshd_DSO
+component_noinst =
+component_install = mca_plm_rshd.la
+else
+component_noinst = libmca_plm_rshd.la
+component_install =
+endif
+
+mcacomponentdir = $(pkglibdir)
+mcacomponent_LTLIBRARIES = $(component_install)
+mca_plm_rshd_la_SOURCES = $(sources)
+mca_plm_rshd_la_LDFLAGS = -module -avoid-version
+
+noinst_LTLIBRARIES = $(component_noinst)
+libmca_plm_rshd_la_SOURCES =$(sources)
+libmca_plm_rshd_la_LDFLAGS = -module -avoid-version

orte/mca/plm/rshd/configure.m4

@@ -0,0 +1,24 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+
+# MCA_plm_rshd_CONFIG([action-if-found], [action-if-not-found])
+# -----------------------------------------------------------
+AC_DEFUN([MCA_plm_rshd_CONFIG],[
+    AC_CHECK_FUNC([fork], [$1], [$2])
+])dnl

orte/mca/plm/rshd/configure.params

@@ -0,0 +1,22 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2007      Los Alamos National Security, LLC.  All rights
+#                         reserved. 
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+
+PARAM_CONFIG_FILES="Makefile"

orte/mca/plm/rshd/help-plm-rshd.txt

@@ -0,0 +1,77 @@
+# -*- text -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+# This is the US/English general help file for Open RTE's orterun.
+#
+[no-local-orted]
+The rsh PLS component was not able to find the executable "orted" in
+your PATH or in the directory where Open MPI/OpenRTE was initially installed,
+and therefore cannot continue.
+
+For reference, your current PATH is:
+
+  %s
+
+We also looked for orted in the following directory:
+
+  %s
+
+[multiple-prefixes]
+Specified multiple application contexts using different
+settings for --prefix. Care should be taken, that corresponding
+processes are mapped to different nodes. Having multiple prefixes
+per node is not allowed.
+
+The previously set prefix was
+  %s
+
+the prefix to be set overriding:
+  %s
+
+[concurrency-less-than-zero]
+The value of the MCA parameter "pls_rsh_num_concurrent" is less than
+or equal to zero (%d).  This parameter is used to determine how many
+remote agents (typically rsh or ssh) to invoke concurrently while
+launching parallel jobs.  
+
+This value has automatically be reset to 1; processing will continue.
+
+[deadlock-params]
+The rsh launcher has been given a number of %d concurrent daemons to
+launch and is in a debug-daemons option. However, the total number of
+daemons to launch (%d) is greater than this value. This is a scenario that
+will cause the system to deadlock.
+
+To avoid deadlock, either increase the number of concurrent daemons, or
+remove the debug-daemons flag.
+
+[unknown-user]
+The user (%d) is unknown to the system (i.e. there is no corresponding
+entry in the password file). Please contact your system administrator
+for a fix.
+#
+[cannot-resolve-shell-with-prefix]
+The rsh launcher has been given a prefix to use, but could not determine
+the type of remote shell being used on the remote node. This is a fatal
+error as we cannot determine how to construct the cmd line to set your
+remote LD_LIBRARY_PATH and PATH environmental variables.
+
+The prefix we were given are:
+
+opal_prefix: %s
+prefix_dir: %s

orte/mca/plm/rshd/plm_rshd.h

@@ -0,0 +1,81 @@
+/*
+ * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2006 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+/**
+ * @file:
+ * Part of the rshd launcher. See plm_rshd.h for an overview of how it works.
+ */
+
+#ifndef ORTE_PLM_RSHD_EXPORT_H
+#define ORTE_PLM_RSHD_EXPORT_H
+
+#include "orte_config.h"
+
+#ifdef HAVE_SYS_TIME_H
+#include <sys/time.h>
+#endif
+
+#include "opal/threads/condition.h"
+#include "opal/mca/mca.h"
+
+#include "orte/mca/plm/plm.h"
+
+BEGIN_C_DECLS
+
+/*
+ * Module open / close
+ */
+int orte_plm_rshd_component_open(void);
+int orte_plm_rshd_component_close(void);
+int orte_plm_rshd_component_query(mca_base_module_t **module, int *priority);
+
+/*
+ * Startup / Shutdown
+ */
+int orte_plm_rshd_finalize(void);
+
+/*
+ * Interface
+ */
+int orte_plm_rshd_init(void);
+int orte_plm_rshd_set_hnp_name(void);
+int orte_plm_rshd_launch(orte_job_t *jdata);
+int orte_plm_rshd_terminate_job(orte_jobid_t);
+int orte_plm_rshd_terminate_orteds(void);
+int orte_plm_rshd_signal_job(orte_jobid_t, int32_t);
+
+/**
+ * PLS Component
+ */
+struct orte_plm_rshd_component_t {
+    orte_plm_base_component_t super;
+    bool force_rsh;
+    opal_list_t children;
+    int num_children;
+    int num_concurrent;
+    opal_mutex_t lock;
+    opal_condition_t cond;
+};
+typedef struct orte_plm_rshd_component_t orte_plm_rshd_component_t;
+
+ORTE_MODULE_DECLSPEC extern orte_plm_rshd_component_t mca_plm_rshd_component;
+extern orte_plm_base_module_t orte_plm_rshd_module;
+
+END_C_DECLS
+
+#endif /* ORTE_PLS_RSHD_EXPORT_H */

orte/mca/plm/rshd/plm_rshd_component.c

@@ -0,0 +1,152 @@
+/*
+ * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2006 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007      Los Alamos National Security, LLC.  All rights
+ *                         reserved. 
+ * Copyright (c) 2008-2009 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ *
+ * These symbols are in a file by themselves to provide nice linker
+ * semantics.  Since linkers generally pull in symbols by object
+ * files, keeping these symbols as the only symbols in this file
+ * prevents utility programs such as "ompi_info" from having to import
+ * entire components just to query their version and parameters.
+ */
+
+#include "orte_config.h"
+#include "orte/constants.h"
+
+#include <stdlib.h>
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#include <ctype.h>
+
+#include "opal/util/opal_environ.h"
+#include "opal/util/output.h"
+#include "opal/util/argv.h"
+#include "opal/util/path.h"
+#include "opal/mca/base/mca_base_param.h"
+
+#include "orte/util/name_fns.h"
+#include "orte/runtime/orte_globals.h"
+#include "orte/util/show_help.h"
+
+#include "orte/mca/plm/plm.h"
+#include "orte/mca/plm/base/plm_private.h"
+#include "orte/mca/plm/rshd/plm_rshd.h"
+
+
+/*
+ * Public string showing the plm ompi_rshd component version number
+ */
+const char *mca_plm_rshd_component_version_string =
+  "Open MPI rshd plm MCA component version " ORTE_VERSION;
+
+
+/*
+ * Instantiate the public struct with all of our public information
+ * and pointers to our public functions in it
+ */
+
+orte_plm_rshd_component_t mca_plm_rshd_component = {
+    {
+    /* First, the mca_component_t struct containing meta information
+       about the component itself */
+
+    {
+        ORTE_PLM_BASE_VERSION_2_0_0,
+
+        /* Component name and version */
+        "rshd",
+        ORTE_MAJOR_VERSION,
+        ORTE_MINOR_VERSION,
+        ORTE_RELEASE_VERSION,
+
+        /* Component open and close functions */
+        orte_plm_rshd_component_open,
+        orte_plm_rshd_component_close,
+        orte_plm_rshd_component_query
+    },
+    {
+        /* The component is checkpoint ready */
+        MCA_BASE_METADATA_PARAM_CHECKPOINT
+    }
+    }
+};
+
+
+
+int orte_plm_rshd_component_open(void)
+{
+    int tmp;
+    mca_base_component_t *c = &mca_plm_rshd_component.super.base_version;
+
+    /* initialize globals */
+    OBJ_CONSTRUCT(&mca_plm_rshd_component.lock, opal_mutex_t);
+    OBJ_CONSTRUCT(&mca_plm_rshd_component.cond, opal_condition_t);
+    mca_plm_rshd_component.num_children = 0;
+    OBJ_CONSTRUCT(&mca_plm_rshd_component.children, opal_list_t);
+
+    /* lookup parameters */
+    mca_base_param_reg_int(c, "num_concurrent",
+                           "How many plm_rsh_agent instances to invoke concurrently (must be > 0)",
+                           false, false, 128, &tmp);
+    if (tmp <= 0) {
+        orte_show_help("help-plm-rsh.txt", "concurrency-less-than-zero",
+                       true, tmp);
+        tmp = 1;
+    }
+    mca_plm_rshd_component.num_concurrent = tmp;
+    
+    mca_base_param_reg_int(c, "force_rsh",
+                           "Force the launcher to always use rsh",
+                           false, false, false, &tmp);
+    mca_plm_rshd_component.force_rsh = OPAL_INT_TO_BOOL(tmp);
+    
+    return ORTE_SUCCESS;
+}
+
+
+int orte_plm_rshd_component_query(mca_base_module_t **module, int *priority)
+{
+    if (ORTE_SUCCESS != orte_plm_base_rsh_launch_agent_setup()) {
+        /* this isn't an error - we just cannot be selected */
+        OPAL_OUTPUT_VERBOSE((1, orte_plm_globals.output,
+                             "%s plm:rshd: unable to be used: cannot find path "
+                             "for launching agent \"%s\"\n", 
+                             ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
+                             orte_plm_globals.rsh_agent_argv[0]));
+        *module = NULL;
+        return ORTE_ERROR;
+    }
+    
+    /* we are good - make ourselves available, but only if selected */
+    *priority = 0;
+    *module = (mca_base_module_t *) &orte_plm_rshd_module;
+    return ORTE_SUCCESS;
+}
+
+
+int orte_plm_rshd_component_close(void)
+{
+    /* cleanup state */
+    OBJ_DESTRUCT(&mca_plm_rshd_component.lock);
+    OBJ_DESTRUCT(&mca_plm_rshd_component.cond);
+    OBJ_DESTRUCT(&mca_plm_rshd_component.children);
+
+    return ORTE_SUCCESS;
+}

orte/mca/plm/rshd/plm_rshd_module.c

@@ -0,0 +1,515 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2007 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2006 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2007 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2007      Los Alamos National Security, LLC.  All rights
+ *                         reserved. 
+ * Copyright (c) 2008-2009 Sun Microsystems, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ * These symbols are in a file by themselves to provide nice linker
+ * semantics.  Since linkers generally pull in symbols by object
+ * files, keeping these symbols as the only symbols in this file
+ * prevents utility programs such as "ompi_info" from having to import
+ * entire components just to query their version and parameters.
+ */
+
+#include "orte_config.h"
+#include "orte/constants.h"
+
+#include <stdlib.h>
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#include <errno.h>
+#include <string.h>
+#ifdef HAVE_STRINGS_H
+#include <strings.h>
+#endif
+#ifdef HAVE_SYS_SELECT_H
+#include <sys/select.h>
+#endif
+#ifdef HAVE_SYS_TIME_H
+#include <sys/time.h>
+#endif
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+#ifdef HAVE_SYS_WAIT_H
+#include <sys/wait.h>
+#endif
+#include <fcntl.h>
+#include <signal.h>
+#ifdef HAVE_PWD_H
+#include <pwd.h>
+#endif
+
+#include "opal/mca/installdirs/installdirs.h"
+#include "opal/mca/base/mca_base_param.h"
+#include "opal/util/output.h"
+#include "opal/event/event.h"
+#include "opal/util/argv.h"
+#include "opal/util/opal_environ.h"
+#include "opal/util/basename.h"
+#include "opal/util/bit_ops.h"
+
+#include "orte/util/show_help.h"
+#include "orte/runtime/orte_wait.h"
+#include "orte/runtime/orte_globals.h"
+#include "orte/util/name_fns.h"
+#include "orte/util/nidmap.h"
+#include "orte/util/proc_info.h"
+
+#include "orte/mca/rml/rml.h"
+#include "orte/mca/rml/rml_types.h"
+#include "orte/mca/ess/ess.h"
+#include "orte/mca/ess/base/base.h"
+#include "orte/mca/errmgr/errmgr.h"
+#include "orte/mca/rmaps/rmaps.h"
+#include "orte/mca/routed/routed.h"
+
+#include "orte/mca/plm/plm.h"
+#include "orte/mca/plm/base/base.h"
+#include "orte/mca/plm/base/plm_private.h"
+#include "orte/mca/plm/rshd/plm_rshd.h"
+
+#if OPAL_HAVE_POSIX_THREADS && OPAL_THREADS_HAVE_DIFFERENT_PIDS && OPAL_ENABLE_PROGRESS_THREADS
+static int orte_plm_rshd_launch_threaded(orte_job_t *jdata);
+#endif
+
+orte_plm_base_module_t orte_plm_rshd_module = {
+    orte_plm_rshd_init,
+    orte_plm_base_set_hnp_name,
+#if OPAL_HAVE_POSIX_THREADS && OPAL_THREADS_HAVE_DIFFERENT_PIDS && OPAL_ENABLE_PROGRESS_THREADS
+    orte_plm_rshd_launch_threaded,
+#else
+    orte_plm_rshd_launch,
+#endif
+    NULL,
+    orte_plm_rshd_terminate_job,
+    orte_plm_rshd_terminate_orteds,
+    orte_plm_rshd_signal_job,
+    orte_plm_rshd_finalize
+};
+
+
+/*
+ * Local functions
+ */
+static void set_handler_default(int sig);
+
+/**
+ * Init the module
+ */
+int orte_plm_rshd_init(void)
+{
+    int rc;
+    
+    if (ORTE_SUCCESS != (rc = orte_plm_base_comm_start())) {
+        ORTE_ERROR_LOG(rc);
+    }
+    
+    return rc;
+}
+
+
+/**
+ * Callback on daemon exit.
+ */
+
+static void wait_cb(pid_t pid, int status, void* cbdata)
+{
+    orte_proc_t *proc = (orte_proc_t*)cbdata;
+    orte_job_t *jdata;
+    
+    /* get the associated job object */
+    jdata = orte_get_job_data_object(proc->name.jobid);
+    
+    if (! WIFEXITED(status) || ! WEXITSTATUS(status) == 0) { /* if abnormal exit */
+        
+        OPAL_OUTPUT_VERBOSE((1, orte_plm_globals.output,
+                             "%s proc %d failed with status %d",
+                             ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
+                             (int)proc->name.vpid, WEXITSTATUS(status)));
+        /* note that this daemon failed */
+        proc->state = ORTE_PROC_STATE_ABORTED;
+    }
+    /* increment the #procs terminated so we will exit properly */
+    jdata->num_terminated++;
+
+    /* check for job completion */
+    orte_plm_base_check_job_completed(jdata);
+    
+    /* release any waiting threads */
+    OPAL_THREAD_LOCK(&mca_plm_rshd_component.lock);
+
+    /* decrement our #children */
+    mca_plm_rshd_component.num_children--;
+    
+    /* see if we can allow launching to continue */
+    if (mca_plm_rshd_component.num_children <=
+        mca_plm_rshd_component.num_concurrent ||
+        mca_plm_rshd_component.num_children == 0) {
+        opal_condition_signal(&mca_plm_rshd_component.cond);
+    }
+    
+    OPAL_THREAD_UNLOCK(&mca_plm_rshd_component.lock);
+
+}
+
+
+/* actually ssh the child */
+static void ssh_child(char *cmd, char **argv)
+{
+    char** env;
+    char* var;
+    long fd, fdmax = sysconf(_SC_OPEN_MAX);
+    int fdin;
+    sigset_t sigs;
+
+    /* setup environment */
+    env = opal_argv_copy(orte_launch_environ);
+    
+    /* Don't let ssh slurp all of our stdin! */
+    fdin = open("/dev/null", O_RDWR);
+    dup2(fdin, 0);
+    close(fdin);
+    
+    /* close all file descriptors w/ exception of stdin/stdout/stderr */
+    for(fd=3; fd<fdmax; fd++)
+        close(fd);
+    
+    /* Set signal handlers back to the default.  Do this close
+     to the execve() because the event library may (and likely
+     will) reset them.  If we don't do this, the event
+     library may have left some set that, at least on some
+     OS's, don't get reset via fork() or exec().  Hence, the
+     orted could be unkillable (for example). */
+    
+    set_handler_default(SIGTERM);
+    set_handler_default(SIGINT);
+    set_handler_default(SIGHUP);
+    set_handler_default(SIGPIPE);
+    set_handler_default(SIGCHLD);
+    
+    /* Unblock all signals, for many of the same reasons that
+     we set the default handlers, above.  This is noticable
+     on Linux where the event library blocks SIGTERM, but we
+     don't want that blocked by the orted (or, more
+     specifically, we don't want it to be blocked by the
+     orted and then inherited by the ORTE processes that it
+     forks, making them unkillable by SIGTERM). */
+    sigprocmask(0, 0, &sigs);
+    sigprocmask(SIG_UNBLOCK, &sigs, 0);
+    
+    /* exec the cmd */
+    var = opal_argv_join(argv, ' ');
+    OPAL_OUTPUT_VERBOSE((1, orte_plm_globals.output,
+                         "%s plm:rshd: executing: (%s) [%s]",
+                         ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
+                         cmd, (NULL == var) ? "NULL" : var));
+    if (NULL != var) free(var);
+    
+    execve(cmd, argv, env);
+    opal_output(0, "plm:rshd: execv of %s failed with errno=%s(%d)\n",
+                cmd, strerror(errno), errno);
+    exit(-1);
+
+}
+
+/**
+ * Directly launch each specified process.
+ */
+
+/* When working in this function, ALWAYS jump to "cleanup" if
+ * you encounter an error so that orterun will be woken up and
+ * the job can cleanly terminate
+ */
+int orte_plm_rshd_launch(orte_job_t *jdata)
+{
+    orte_job_map_t *map = NULL;
+    char **argv = NULL;
+    char *cmd, *param;
+    int rc, i;
+    bool failed_launch = true;
+    orte_app_context_t *app;
+    orte_node_t *node;
+    orte_proc_t *proc;
+    orte_jobid_t failed_job = ORTE_JOBID_INVALID;
+    orte_job_state_t job_state = ORTE_JOB_NEVER_LAUNCHED;
+    pid_t pid;
+    
+    if (jdata->controls & ORTE_JOB_CONTROL_LOCAL_SLAVE) {
+        /* if this is a request to launch a local slave,
+         * then we will not be launching an orted - we will
+         * directly ssh the slave process itself. No mapping
+         * is performed to support this - the caller must
+         * provide all the info required to launch the job,
+         * including the target hosts
+         */
+        return orte_plm_base_local_slave_launch(jdata);
+    }
+    
+    /* create a jobid for this job */
+    if (ORTE_SUCCESS != (rc = orte_plm_base_create_jobid(&jdata->jobid))) {
+        ORTE_ERROR_LOG(rc);
+        goto cleanup;
+    }
+    
+    OPAL_OUTPUT_VERBOSE((1, orte_plm_globals.output,
+                         "%s plm:rshd: setting up job %s",
+                         ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
+                         ORTE_JOBID_PRINT(jdata->jobid)));
+    
+    /* setup the job */
+    if (ORTE_SUCCESS != (rc = orte_plm_base_setup_job(jdata))) {
+        ORTE_ERROR_LOG(rc);
+        goto cleanup;
+    }
+
+    /* default to declaring the job launch as having failed */
+    failed_job = jdata->jobid;
+    
+    /* launch each proc */
+    for (i=0; i < jdata->procs->size; i++) {
+        if (NULL == (proc = (orte_proc_t*)opal_pointer_array_get_item(jdata->procs, i))) {
+            continue;
+        }
+        OPAL_OUTPUT_VERBOSE((1, orte_plm_globals.output,
+                             "%s plm:rshd: launching proc %s on node %s",
+                             ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
+                             ORTE_NAME_PRINT(&proc->name), proc->nodename));
+        if (NULL == (app = (orte_app_context_t*)opal_pointer_array_get_item(jdata->apps, proc->app_idx))) {
+            continue;
+        }
+        node = (orte_node_t*)proc->node;
+        /* setup the launch */
+        if (ORTE_SUCCESS != (rc = orte_plm_base_setup_rsh_launch(proc->nodename, app,
+                                                                 "orte-bootproxy.sh",
+                                                                 &argv, &cmd))) {
+            ORTE_ERROR_LOG(rc);
+            goto cleanup;
+        }
+
+        /* add the bootproxy cmd line options */
+        if (ORTE_SUCCESS != (rc = orte_plm_base_append_bootproxy_args(app, &argv,
+                                                                      proc->name.jobid, proc->name.vpid,
+                                                                      jdata->map->num_nodes,
+                                                                      jdata->num_procs, proc->local_rank,
+                                                                      node->num_procs, jdata->total_slots_alloc))) {
+            ORTE_ERROR_LOG(rc);
+            goto cleanup;
+        }
+
+        /* final cmd */
+        if (0 < opal_output_get_verbosity(orte_plm_globals.output)) {
+            param = opal_argv_join(argv, ' ');
+            opal_output(0, "%s plm:rshd: final cmd:\n\t%s",
+                        ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
+                        (NULL == param) ? "NULL" : param);
+            if (NULL != param) free(param);
+        }
+
+        /* fork a child to exec the rsh/ssh session */
+        pid = fork();
+        if (pid < 0) {
+            ORTE_ERROR_LOG(ORTE_ERR_SYS_LIMITS_CHILDREN);
+            rc = ORTE_ERR_SYS_LIMITS_CHILDREN;
+            goto cleanup;
+        }
+        
+        /* child */
+        if (pid == 0) {            
+            /* do the ssh launch - this will exit if it fails */
+            ssh_child(cmd, argv);
+        }
+        /* father */
+        OPAL_THREAD_LOCK(&mca_plm_rshd_component.lock);
+        if (mca_plm_rshd_component.num_children++ >=
+            mca_plm_rshd_component.num_concurrent) {
+            opal_condition_wait(&mca_plm_rshd_component.cond, &mca_plm_rshd_component.lock);
+        }
+        OPAL_THREAD_UNLOCK(&mca_plm_rshd_component.lock);
+        
+        /* cleanup */
+        opal_argv_free(argv);
+        argv = NULL;
+        free(cmd);
+        cmd = NULL;
+        
+        /* setup callback on sigchild - wait until setup above is complete
+         * as the callback can occur in the call to orte_wait_cb
+         */
+        orte_wait_cb(pid, wait_cb, (void*)proc);
+    }
+
+    /* flag the launch as successful */
+    failed_launch = false;
+    
+cleanup:
+    if (NULL != argv) {
+        opal_argv_free(argv);
+    }
+    if (NULL != cmd) {
+        free(cmd);
+    }
+    
+    /* check for failed launch - if so, force terminate */
+    if (failed_launch) {
+        orte_plm_base_launch_failed(failed_job, -1, ORTE_ERROR_DEFAULT_EXIT_CODE, job_state);
+    }
+
+    /* setup a "heartbeat" timer to periodically check on
+     * the state-of-health of the orteds, if requested AND
+     * we actually launched some daemons!
+     */
+    if ((NULL != map) && (0 < map->num_new_daemons)) {
+        orte_plm_base_start_heart();
+    }
+    
+    return rc;
+}
+
+
+/**
+ * Terminate all processes for a given job
+ */
+int orte_plm_rshd_terminate_job(orte_jobid_t jobid)
+{
+    int rc;
+    
+    /* order them to kill their local procs for this job */
+    if (ORTE_SUCCESS != (rc = orte_plm_base_orted_kill_local_procs(jobid))) {
+        ORTE_ERROR_LOG(rc);
+    }
+    
+    return rc;
+}
+
+/**
+ * No orteds to terminate
+ */
+int orte_plm_rshd_terminate_orteds(void)
+{    
+    orte_trigger_event(&orteds_exit);
+    return ORTE_SUCCESS;
+}
+
+int orte_plm_rshd_signal_job(orte_jobid_t jobid, int32_t signal)
+{
+    /* no way to do this */
+    return ORTE_SUCCESS;
+}
+
+int orte_plm_rshd_finalize(void)
+{
+    int rc;
+    
+    /* cleanup any pending recvs */
+    if (ORTE_SUCCESS != (rc = orte_plm_base_comm_stop())) {
+        ORTE_ERROR_LOG(rc);
+    }
+    return rc;
+}
+
+
+/**
+ * Handle threading issues.
+ */
+
+#if OPAL_HAVE_POSIX_THREADS && OPAL_THREADS_HAVE_DIFFERENT_PIDS && OPAL_ENABLE_PROGRESS_THREADS
+
+struct orte_plm_rshd_stack_t {
+    opal_condition_t cond;
+    opal_mutex_t mutex;
+    bool complete;
+    orte_jobid_t jobid;
+    int rc;
+};
+typedef struct orte_plm_rshd_stack_t orte_plm_rshd_stack_t;
+
+static void orte_plm_rshd_stack_construct(orte_plm_rshd_stack_t* stack)
+{
+    OBJ_CONSTRUCT(&stack->mutex, opal_mutex_t);
+    OBJ_CONSTRUCT(&stack->cond, opal_condition_t);
+    stack->rc = 0;
+    stack->complete = false;
+}
+
+static void orte_plm_rshd_stack_destruct(orte_plm_rshd_stack_t* stack)
+{
+    OBJ_DESTRUCT(&stack->mutex);
+    OBJ_DESTRUCT(&stack->cond);
+}
+
+static OBJ_CLASS_INSTANCE(
+    orte_plm_rshd_stack_t,
+    opal_object_t,
+    orte_plm_rshd_stack_construct,
+    orte_plm_rshd_stack_destruct);
+
+static void orte_plm_rshd_launch_cb(int fd, short event, void* args)
+{
+    orte_plm_rshd_stack_t *stack = (orte_plm_rshd_stack_t*)args;
+    OPAL_THREAD_LOCK(&stack->mutex);
+    stack->rc = orte_plm_rshd_launch(stack->jobid);
+    stack->complete = true;
+    opal_condition_signal(&stack->cond);
+    OPAL_THREAD_UNLOCK(&stack->mutex);
+}
+
+static int orte_plm_rshd_launch_threaded(orte_jobid_t jobid)
+{
+    struct timeval tv = { 0, 0 };
+    struct opal_event event;
+    struct orte_plm_rshd_stack_t stack;
+
+    OBJ_CONSTRUCT(&stack, orte_plm_rshd_stack_t);
+
+    stack.jobid = jobid;
+    if( opal_event_progress_thread() ) {
+        stack.rc = orte_plm_rshd_launch( jobid );
+    } else {
+        opal_evtimer_set(&event, orte_plm_rshd_launch_cb, &stack);
+        opal_evtimer_add(&event, &tv);
+
+        OPAL_THREAD_LOCK(&stack.mutex);
+        while (stack.complete == false) {
+            opal_condition_wait(&stack.cond, &stack.mutex);
+        }
+        OPAL_THREAD_UNLOCK(&stack.mutex);
+    }
+    OBJ_DESTRUCT(&stack);
+    return stack.rc;
+}
+
+#endif
+
+
+static void set_handler_default(int sig)
+{
+    struct sigaction act;
+
+    act.sa_handler = SIG_DFL;
+    act.sa_flags = 0;
+    sigemptyset(&act.sa_mask);
+
+    sigaction(sig, &act, (struct sigaction *)0);
+}
+
+

orte/runtime/orte_globals.c

@@ -122,6 +122,7 @@ bool orte_forward_job_control;
 
 /* rsh support */
 char *orte_rsh_agent = NULL;
+bool orte_assume_same_shell = true;
 
 /* orted exit with barrier */
 bool orte_orted_exit_with_barrier = true;

orte/runtime/orte_globals.h

@@ -509,6 +509,7 @@ ORTE_DECLSPEC extern char *orte_xterm;
 
 /* rsh support */
 ORTE_DECLSPEC extern char *orte_rsh_agent;
+ORTE_DECLSPEC extern bool orte_assume_same_shell;
 
 /* whether or not to barrier the orteds upon exit */
 ORTE_DECLSPEC extern bool orte_orted_exit_with_barrier;

orte/runtime/orte_mca_params.c

@@ -276,6 +276,13 @@ int orte_register_params(void)
     mca_base_param_reg_syn_name(tmp, "pls", "rsh_agent", true);
     mca_base_param_reg_syn_name(tmp, "plm", "rsh_agent", true);
     mca_base_param_lookup_string(tmp, &orte_rsh_agent);
+
+    tmp = mca_base_param_reg_int_name("orte", "assume_same_shell",
+                                      "If set to 1, assume that the shell on the remote node is the same as the shell on the local node.  Otherwise, probe for what the remote shell [default: 1]",
+                                      false, false, 1, NULL);
+    mca_base_param_reg_syn_name(tmp, "plm", "rsh_assume_same_shell", true);
+    mca_base_param_lookup_int(tmp, &value);
+    orte_assume_same_shell = OPAL_INT_TO_BOOL(value);
     
 #endif /* ORTE_DISABLE_FULL_SUPPORT */
     

orte/util/proc_info.h

@@ -53,6 +53,7 @@ typedef uint32_t orte_proc_type_t;
 #define ORTE_PROC_MPI           0x0020
 #define ORTE_PROC_APP           0x0030
 #define ORTE_PROC_CM            0x0040
+#define ORTE_PROC_CM_APP        0x0080
 
 #define ORTE_PROC_IS_SINGLETON      (ORTE_PROC_SINGLETON & orte_process_info.proc_type)
 #define ORTE_PROC_IS_DAEMON         (ORTE_PROC_DAEMON & orte_process_info.proc_type)
@@ -62,6 +63,7 @@ typedef uint32_t orte_proc_type_t;
 #define ORTE_PROC_IS_MPI            (ORTE_PROC_MPI & orte_process_info.proc_type)
 #define ORTE_PROC_IS_APP            (ORTE_PROC_APP & orte_process_info.proc_type)
 #define ORTE_PROC_IS_CM             (ORTE_PROC_CM & orte_process_info.proc_type)
+#define ORTE_PROC_IS_CM_APP         (ORTE_PROC_CM_APP & orte_process_info.proc_type)
 
 
 /**