Merge pull request #1525 from rhc54/topic/schizo

Extend the schizo framework

config/orte_config_files.m4

@@ -6,7 +6,7 @@
 #                         Corporation.  All rights reserved.
 # Copyright (c) 2011-2012 Los Alamos National Security, LLC.  All rights
 #                         reserved.
-# Copyright (c) 2015      Intel, Inc. All rights reserved
+# Copyright (c) 2015-2016 Intel, Inc. All rights reserved
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -33,7 +33,6 @@ AC_DEFUN([ORTE_CONFIG_FILES],[
         orte/tools/orte-migrate/Makefile
         orte/tools/orte-info/Makefile
         orte/tools/orte-server/Makefile
-        orte/tools/orte-submit/Makefile
         orte/tools/orte-dvm/Makefile
     ])
 ])

ompi/mca/rte/orte/Makefile.am

@@ -1,7 +1,8 @@
 #
 # Copyright (c) 2012      Los Alamos National Security, LLC.
 #                         All rights reserved.
-# Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2014      Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2016      Intel, Inc. All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -27,7 +28,7 @@ libmca_rte_orte_la_SOURCES =$(sources) $(headers)
 libmca_rte_orte_la_LDFLAGS = -module -avoid-version
 libmca_rte_orte_la_LIBADD = $(top_builddir)/orte/lib@ORTE_LIB_PREFIX@open-rte.la
 
-man_pages = mpirun.1 mpiexec.1 ompi-ps.1 ompi-clean.1 ompi-top.1 ompi-server.1 ompi-dvm.1 ompi-submit.1
+man_pages = mpirun.1 mpiexec.1 ompi-ps.1 ompi-clean.1 ompi-top.1 ompi-server.1 ompi-dvm.1
 
 if WANT_FT
 man_pages += ompi-checkpoint.1 ompi-restart.1
@@ -44,7 +45,6 @@ install-exec-hook:
 	(cd $(DESTDIR)$(bindir); rm -f ompi-top$(EXEEXT); $(LN_S) orte-top$(EXEEXT) ompi-top$(EXEEXT))
 	(cd $(DESTDIR)$(bindir); rm -f ompi-server$(EXEEXT); $(LN_S) orte-server$(EXEEXT) ompi-server$(EXEEXT))
 	(cd $(DESTDIR)$(bindir); rm -f ompi-dvm$(EXEEXT); $(LN_S) orte-dvm$(EXEEXT) ompi-dvm$(EXEEXT))
-	(cd $(DESTDIR)$(bindir); rm -f ompi-submit$(EXEEXT); $(LN_S) orte-submit$(EXEEXT) ompi-submit$(EXEEXT))
 if WANT_FT
 	(cd $(DESTDIR)$(bindir); rm -f ompi-checkpoint$(EXEEXT); $(LN_S) orte-checkpoint$(EXEEXT) ompi-checkpoint$(EXEEXT))
 	(cd $(DESTDIR)$(bindir); rm -f ompi-restart$(EXEEXT); $(LN_S) orte-restart$(EXEEXT) ompi-restart$(EXEEXT))
@@ -58,8 +58,7 @@ uninstall-local:
 		$(DESTDIR)$(bindir)/ompi-clean$(EXEEXT) \
 		$(DESTDIR)$(bindir)/ompi-top$(EXEEXT) \
 		$(DESTDIR)$(bindir)/ompi-server$(EXEEXT) \
-		$(DESTDIR)$(bindir)/ompi-dvm$(EXEEXT) \
+		$(DESTDIR)$(bindir)/ompi-dvm$(EXEEXT)
-		$(DESTDIR)$(bindir)/ompi-submit$(EXEEXT)
 if WANT_FT
 	rm -f $(DESTDIR)$(bindir)/ompi-checkpoint$(EXEEXT) \
 		$(DESTDIR)$(bindir)/ompi-restart$(EXEEXT) \
@@ -122,8 +121,5 @@ ompi-server.1: $(top_builddir)/orte/tools/orte-server/orte-server.1
 ompi-dvm.1: $(top_builddir)/orte/tools/orte-dvm/orte-dvm.1
 	cp -f $(top_builddir)/orte/tools/orte-dvm/orte-dvm.1 ompi-dvm.1
 
-ompi-submit.1: $(top_builddir)/orte/tools/orte-submit/orte-submit.1
-	cp -f $(top_builddir)/orte/tools/orte-submit/orte-submit.1 ompi-submit.1
-
 clean-local:
 	rm -f $(man_pages)

opal/mca/base/base.h

@@ -156,7 +156,7 @@ OPAL_DECLSPEC int mca_base_is_component_required(opal_list_t *components_availab
 /* mca_base_cmd_line.c */
 
 OPAL_DECLSPEC int mca_base_cmd_line_setup(opal_cmd_line_t *cmd);
-OPAL_DECLSPEC int mca_base_cmd_line_process_args(opal_cmd_line_t *cmd,
+OPAL_DECLSPEC int mca_base_cmd_line_process_args(char **argv,
                                                  char ***app_env,
                                                  char ***global_env);
 OPAL_DECLSPEC void mca_base_cmd_line_wrap_args(char **args);

opal/mca/base/mca_base_cmd_line.c

@@ -94,29 +94,25 @@ int mca_base_cmd_line_setup(opal_cmd_line_t *cmd)
 /*
  * Look for and handle any -mca options on the command line
  */
-int mca_base_cmd_line_process_args(opal_cmd_line_t *cmd,
+int mca_base_cmd_line_process_args(char **argv,
                                    char ***context_env, char ***global_env)
 {
-    int i, num_insts, rc;
+    int i, rc;
     char **params;
     char **values;
 
-    /* If no relevant parameters were given, just return */
-
-    if (!opal_cmd_line_is_taken(cmd, OPAL_MCA_CMD_LINE_ID) &&
-        !opal_cmd_line_is_taken(cmd, "g"OPAL_MCA_CMD_LINE_ID)) {
-        return OPAL_SUCCESS;
-    }
-
-    /* Handle app context-specific parameters */
-
-    num_insts = opal_cmd_line_get_ninsts(cmd, OPAL_MCA_CMD_LINE_ID);
     params = values = NULL;
-    for (i = 0; i < num_insts; ++i) {
+    for (i = 0; NULL != argv[i]; ++i) {
-        if (OPAL_SUCCESS != (rc = process_arg(opal_cmd_line_get_param(cmd, OPAL_MCA_CMD_LINE_ID, i, 0),
+        if (0 == strcmp("-"OPAL_MCA_CMD_LINE_ID,  argv[i]) ||
-                                              opal_cmd_line_get_param(cmd, OPAL_MCA_CMD_LINE_ID, i, 1),
+            0 == strcmp("--"OPAL_MCA_CMD_LINE_ID, argv[i])) {
-                                              &params, &values))) {
+            if (NULL == argv[i+1] || NULL == argv[i+2]) {
-            return rc;
+                return OPAL_ERR_BAD_PARAM;
+            }
+            if (OPAL_SUCCESS != (rc = process_arg(argv[i+1], argv[i+2],
+                                                  &params, &values))) {
+                return rc;
+            }
+            i += 2;
         }
     }
     if (NULL != params) {
@@ -125,15 +121,19 @@ int mca_base_cmd_line_process_args(opal_cmd_line_t *cmd,
         opal_argv_free(values);
     }
 
-    /* Handle global parameters */
 
-    num_insts = opal_cmd_line_get_ninsts(cmd, "g"OPAL_MCA_CMD_LINE_ID);
     params = values = NULL;
-    for (i = 0; i < num_insts; ++i) {
+    for (i = 0; NULL != argv[i]; ++i) {
-        if (OPAL_SUCCESS != (rc = process_arg(opal_cmd_line_get_param(cmd, "g"OPAL_MCA_CMD_LINE_ID, i, 0),
+        if (0 == strcmp("-g"OPAL_MCA_CMD_LINE_ID,  argv[i]) ||
-                                              opal_cmd_line_get_param(cmd, "g"OPAL_MCA_CMD_LINE_ID, i, 1),
+            0 == strcmp("--g"OPAL_MCA_CMD_LINE_ID, argv[i])) {
-                                              &params, &values))) {
+            if (NULL == argv[i+1] || NULL == argv[i+2]) {
-            return rc;
+                return OPAL_ERR_BAD_PARAM;
+            }
+            if (OPAL_SUCCESS != (rc = process_arg(argv[i+1], argv[i+2],
+                                                  &params, &values))) {
+                return rc;
+            }
+            i += 2;
         }
     }
     if (NULL != params) {
@@ -190,7 +190,6 @@ static int process_arg(const char *param, const char *value,
 
     /* If we didn't already have an value for the same param, save
        this one away */
-
     opal_argv_append_nosize(params, param);
     opal_argv_append_nosize(values, p1);
     free(p1);

opal/runtime/opal_info_support.c

@@ -207,7 +207,7 @@ int opal_info_init(int argc, char **argv,
         exit(cmd_error ? 1 : 0);
     }
 
-    mca_base_cmd_line_process_args(opal_info_cmd_line, &app_env, &global_env);
+    mca_base_cmd_line_process_args(argv, &app_env, &global_env);
 
 
     /* set the flags */

opal/util/cmd_line.c

@@ -156,7 +156,9 @@ int opal_cmd_line_create(opal_cmd_line_t *cmd,
     }
     OBJ_CONSTRUCT(cmd, opal_cmd_line_t);
 
-    ret = opal_cmd_line_add(cmd, table);
+    if (NULL != table) {
+        ret = opal_cmd_line_add(cmd, table);
+    }
     return ret;
 }
 

orte/mca/ess/hnp/ess_hnp_module.c

@@ -76,6 +76,7 @@
 #include "orte/mca/state/base/base.h"
 #include "orte/mca/state/state.h"
 
+#include "orte/orted/orted_submit.h"
 #include "orte/orted/pmix/pmix_server.h"
 
 #include "orte/util/show_help.h"
@@ -713,6 +714,14 @@ static int rte_init(void)
         goto error;
     }
 
+    /* setup to support debugging */
+    orte_state.add_job_state(ORTE_JOB_STATE_READY_FOR_DEBUGGERS,
+                             orte_debugger_init_after_spawn,
+                             ORTE_SYS_PRI);
+    orte_state.add_job_state(ORTE_JOB_STATE_DEBUGGER_DETACH,
+                             orte_debugger_detached,
+                             ORTE_SYS_PRI);
+
     /* if a tool has launched us and is requesting event reports,
      * then set its contact info into the comm system
      */

orte/mca/grpcomm/base/grpcomm_base_stubs.c

@@ -324,7 +324,7 @@ static int create_dmns(orte_grpcomm_signature_t *sig,
             *dmns = NULL;
             return ORTE_ERR_NOT_FOUND;
         }
-        if (NULL == jdata->map) {
+        if (0 == jdata->map->num_nodes) {
             /* we haven't generated a job map yet - if we are the HNP,
              * then we should only involve ourselves. Otherwise, we have
              * no choice but to abort to avoid hangs */
@@ -340,12 +340,6 @@ static int create_dmns(orte_grpcomm_signature_t *sig,
             *dmns = NULL;
             return ORTE_ERR_NOT_FOUND;
         }
-        /* get the array */
-        if (0 == jdata->map->num_nodes) {
-            ORTE_UPDATE_EXIT_STATUS(ORTE_ERROR_DEFAULT_EXIT_CODE);
-            ORTE_ERROR_LOG(ORTE_ERR_BAD_PARAM);
-            return ORTE_ERR_SILENT;
-        }
         dns = (orte_vpid_t*)malloc(jdata->map->num_nodes * sizeof(vpid));
         nds = 0;
         for (i=0; i < jdata->map->nodes->size && (int)nds < jdata->map->num_nodes; i++) {

orte/mca/plm/base/plm_base_launch_support.c

@@ -1537,6 +1537,9 @@ int orte_plm_base_setup_virtual_machine(orte_job_t *jdata)
         ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
         return ORTE_ERR_NOT_FOUND;
     }
+    if (NULL == daemons->map) {
+        daemons->map = OBJ_NEW(orte_job_map_t);
+    }
     map = daemons->map;
 
     /* if this job is being launched against a fixed DVM, then there is
@@ -1552,8 +1555,7 @@ int orte_plm_base_setup_virtual_machine(orte_job_t *jdata)
      * the virtual machine unless specifically requested to do so
      */
     if (ORTE_JOBID_INVALID != jdata->originator.jobid) {
-        OBJ_CONSTRUCT(&nodes, opal_list_t);
+        if (0 == map->num_nodes) {
-        if (NULL == daemons->map) {
             OPAL_OUTPUT_VERBOSE((5, orte_plm_base_framework.framework_output,
                                  "%s plm:base:setup_vm creating map",
                                  ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
@@ -1562,16 +1564,15 @@ int orte_plm_base_setup_virtual_machine(orte_job_t *jdata)
              * are obviously already here! The ess will already
              * have assigned our node to us.
              */
-            daemons->map = OBJ_NEW(orte_job_map_t);
             node = (orte_node_t*)opal_pointer_array_get_item(orte_node_pool, 0);
-            opal_pointer_array_add(daemons->map->nodes, (void*)node);
+            opal_pointer_array_add(map->nodes, (void*)node);
-            ++(daemons->map->num_nodes);
+            ++(map->num_nodes);
             /* maintain accounting */
             OBJ_RETAIN(node);
             /* mark that this is from a singleton */
             singleton = true;
         }
-        map = daemons->map;
+        OBJ_CONSTRUCT(&nodes, opal_list_t);
         for (i=1; i < orte_node_pool->size; i++) {
             if (NULL == (node = (orte_node_t*)opal_pointer_array_get_item(orte_node_pool, i))) {
                 continue;
@@ -1618,16 +1619,6 @@ int orte_plm_base_setup_virtual_machine(orte_job_t *jdata)
      */
     if (orte_get_attribute(&daemons->attributes, ORTE_JOB_NO_VM, NULL, OPAL_BOOL)) {
         OBJ_CONSTRUCT(&nodes, opal_list_t);
-        if (NULL == daemons->map) {
-            OPAL_OUTPUT_VERBOSE((5, orte_plm_base_framework.framework_output,
-                                 "%s plm:base:setup_vm creating map",
-                                 ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
-            /* this is the first time thru, so the vm is just getting
-             * defined - create a map for it
-             */
-            daemons->map = OBJ_NEW(orte_job_map_t);
-        }
-        map = daemons->map;
         /* loop across all nodes and include those that have
          * num_procs > 0 && no daemon already on them
          */
@@ -1685,23 +1676,21 @@ int orte_plm_base_setup_virtual_machine(orte_job_t *jdata)
         goto process;
     }
 
-    if (NULL == daemons->map) {
+    if (0 == map->num_nodes) {
         OPAL_OUTPUT_VERBOSE((5, orte_plm_base_framework.framework_output,
                              "%s plm:base:setup_vm creating map",
                              ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
         /* this is the first time thru, so the vm is just getting
-         * defined - create a map for it and put us in as we
+         * defined - put us in as we
          * are obviously already here! The ess will already
          * have assigned our node to us.
          */
-        daemons->map = OBJ_NEW(orte_job_map_t);
         node = (orte_node_t*)opal_pointer_array_get_item(orte_node_pool, 0);
-        opal_pointer_array_add(daemons->map->nodes, (void*)node);
+        opal_pointer_array_add(map->nodes, (void*)node);
-        ++(daemons->map->num_nodes);
+        ++(map->num_nodes);
         /* maintain accounting */
         OBJ_RETAIN(node);
     }
-    map = daemons->map;
 
     /* zero-out the number of new daemons as we will compute this
      * each time we are called

orte/mca/rmaps/ppr/rmaps_ppr.c

@@ -110,8 +110,10 @@ static int ppr_mapper(orte_job_t *jdata)
         ORTE_MAPPING_PPR != ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) {
         /* not for us */
         opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
-                            "mca:rmaps:ppr: job %s not using ppr mapper",
+                            "mca:rmaps:ppr: job %s not using ppr mapper PPR %s policy %s",
-                            ORTE_JOBID_PRINT(jdata->jobid));
+                            ORTE_JOBID_PRINT(jdata->jobid),
+                            (NULL == jdata->map->ppr) ? "NULL" : jdata->map->ppr,
+                            (ORTE_MAPPING_PPR == ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) ? "PPRSET" : "PPR NOTSET");
         return ORTE_ERR_TAKE_NEXT_OPTION;
     }
 

orte/mca/schizo/base/base.h

@@ -41,6 +41,7 @@ ORTE_DECLSPEC    int orte_schizo_base_select(void);
 typedef struct {
     /* list of active modules */
     opal_list_t active_modules;
+    char **personalities;
 } orte_schizo_base_t;
 
 /**
@@ -61,15 +62,13 @@ OBJ_CLASS_DECLARATION(orte_schizo_base_active_module_t);
 
 /* the base stub functions */
 ORTE_DECLSPEC const char* orte_schizo_base_print_env(orte_schizo_launch_environ_t env);
-ORTE_DECLSPEC int orte_schizo_base_parse_cli(char **personality,
+ORTE_DECLSPEC int orte_schizo_base_define_cli(opal_cmd_line_t *cli);
-                                             int argc, int start, char **argv);
+ORTE_DECLSPEC int orte_schizo_base_parse_cli(int argc, int start, char **argv);
-ORTE_DECLSPEC int orte_schizo_base_parse_env(char **personality,
+ORTE_DECLSPEC int orte_schizo_base_parse_env(char *path,
-                                             char *path,
                                              opal_cmd_line_t *cmd_line,
                                              char **srcenv,
                                              char ***dstenv);
-ORTE_DECLSPEC int orte_schizo_base_setup_app(char **personality,
+ORTE_DECLSPEC int orte_schizo_base_setup_app(orte_app_context_t *app);
-                                             orte_app_context_t *app);
 ORTE_DECLSPEC int orte_schizo_base_setup_fork(orte_job_t *jdata,
                                               orte_app_context_t *context);
 ORTE_DECLSPEC int orte_schizo_base_setup_child(orte_job_t *jobdat,

orte/mca/schizo/base/schizo_base_frame.c

@@ -37,6 +37,7 @@
  */
 orte_schizo_base_t orte_schizo_base = {{{0}}};
 orte_schizo_base_module_t orte_schizo = {
+    .define_cli = orte_schizo_base_define_cli,
     .parse_cli = orte_schizo_base_parse_cli,
     .parse_env = orte_schizo_base_parse_env,
     .setup_app = orte_schizo_base_setup_app,
@@ -46,10 +47,28 @@ orte_schizo_base_module_t orte_schizo = {
     .finalize = orte_schizo_base_finalize
 };
 
+static char *personalities = NULL;
+
+static int orte_schizo_base_register(mca_base_register_flag_t flags)
+{
+    /* pickup any defined personalities */
+    personalities = NULL;
+    mca_base_var_register("orte", "schizo", "base", "personalities",
+                          "Comma-separated list of personalities",
+                          MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
+                          OPAL_INFO_LVL_9,
+                          MCA_BASE_VAR_SCOPE_READONLY,
+                          &personalities);
+    return ORTE_SUCCESS;
+}
+
 static int orte_schizo_base_close(void)
 {
     /* cleanup globals */
     OPAL_LIST_DESTRUCT(&orte_schizo_base.active_modules);
+    if (NULL != orte_schizo_base.personalities) {
+        opal_argv_free(orte_schizo_base.personalities);
+    }
 
     return mca_base_framework_components_close(&orte_schizo_base_framework, NULL);
 }
@@ -64,6 +83,10 @@ static int orte_schizo_base_open(mca_base_open_flag_t flags)
 
     /* init the globals */
     OBJ_CONSTRUCT(&orte_schizo_base.active_modules, opal_list_t);
+    orte_schizo_base.personalities = NULL;
+    if (NULL != personalities) {
+        orte_schizo_base.personalities = opal_argv_split(personalities, ',');
+    }
 
     /* Open up all available components */
     rc = mca_base_framework_components_open(&orte_schizo_base_framework, flags);
@@ -73,7 +96,8 @@ static int orte_schizo_base_open(mca_base_open_flag_t flags)
 }
 
 MCA_BASE_FRAMEWORK_DECLARE(orte, schizo, "ORTE Schizo Subsystem",
-                           NULL, orte_schizo_base_open, orte_schizo_base_close,
+                           orte_schizo_base_register,
+                           orte_schizo_base_open, orte_schizo_base_close,
                            mca_schizo_base_static_components, 0);
 
 OBJ_CLASS_INSTANCE(orte_schizo_base_active_module_t,

orte/mca/schizo/base/schizo_base_select.c

@@ -28,8 +28,6 @@
  * available.
  */
 
-static bool selected = false;
-
 int orte_schizo_base_select(void)
 {
     mca_base_component_list_item_t *cli = NULL;
@@ -40,11 +38,10 @@ int orte_schizo_base_select(void)
     int rc, priority;
     bool inserted;
 
-    if (selected) {
+    if (0 < opal_list_get_size(&orte_schizo_base.active_modules)) {
         /* ensure we don't do this twice */
         return ORTE_SUCCESS;
     }
-    selected = true;
 
     /* Query all available components and ask if they have a module */
     OPAL_LIST_FOREACH(cli, &orte_schizo_base_framework.framework_components, mca_base_component_list_item_t) {

orte/mca/schizo/base/schizo_base_stubs.c

@@ -37,19 +37,14 @@
     }
 }
 
-int orte_schizo_base_parse_cli(char **personality,
+int orte_schizo_base_define_cli(opal_cmd_line_t *cli)
-                               int argc, int start, char **argv)
 {
     int rc;
     orte_schizo_base_active_module_t *mod;
 
-    if (NULL == personality) {
-        return ORTE_ERR_NOT_SUPPORTED;
-    }
-
     OPAL_LIST_FOREACH(mod, &orte_schizo_base.active_modules, orte_schizo_base_active_module_t) {
-        if (NULL != mod->module->parse_cli) {
+        if (NULL != mod->module->define_cli) {
-            rc = mod->module->parse_cli(personality, argc, start, argv);
+            rc = mod->module->define_cli(cli);
             if (ORTE_SUCCESS != rc && ORTE_ERR_TAKE_NEXT_OPTION != rc) {
                 ORTE_ERROR_LOG(rc);
                 return rc;
@@ -59,8 +54,24 @@ int orte_schizo_base_parse_cli(char **personality,
     return ORTE_SUCCESS;
 }
 
-int orte_schizo_base_parse_env(char **personality,
+int orte_schizo_base_parse_cli(int argc, int start, char **argv)
-                               char *path,
+{
+    int rc;
+    orte_schizo_base_active_module_t *mod;
+
+    OPAL_LIST_FOREACH(mod, &orte_schizo_base.active_modules, orte_schizo_base_active_module_t) {
+        if (NULL != mod->module->parse_cli) {
+            rc = mod->module->parse_cli(argc, start, argv);
+            if (ORTE_SUCCESS != rc && ORTE_ERR_TAKE_NEXT_OPTION != rc) {
+                ORTE_ERROR_LOG(rc);
+                return rc;
+            }
+        }
+    }
+    return ORTE_SUCCESS;
+}
+
+int orte_schizo_base_parse_env(char *path,
                                opal_cmd_line_t *cmd_line,
                                char **srcenv,
                                char ***dstenv)
@@ -70,7 +81,7 @@ int orte_schizo_base_parse_env(char **personality,
 
     OPAL_LIST_FOREACH(mod, &orte_schizo_base.active_modules, orte_schizo_base_active_module_t) {
         if (NULL != mod->module->parse_env) {
-            rc = mod->module->parse_env(personality, path, cmd_line, srcenv, dstenv);
+            rc = mod->module->parse_env(path, cmd_line, srcenv, dstenv);
             if (ORTE_SUCCESS != rc && ORTE_ERR_TAKE_NEXT_OPTION != rc) {
                 ORTE_ERROR_LOG(rc);
                 return rc;
@@ -80,15 +91,14 @@ int orte_schizo_base_parse_env(char **personality,
     return ORTE_SUCCESS;
 }
 
-int orte_schizo_base_setup_app(char **personality,
+int orte_schizo_base_setup_app(orte_app_context_t *app)
-                               orte_app_context_t *app)
 {
     int rc;
     orte_schizo_base_active_module_t *mod;
 
     OPAL_LIST_FOREACH(mod, &orte_schizo_base.active_modules, orte_schizo_base_active_module_t) {
         if (NULL != mod->module->setup_app) {
-            rc = mod->module->setup_app(personality, app);
+            rc = mod->module->setup_app(app);
             if (ORTE_SUCCESS != rc && ORTE_ERR_TAKE_NEXT_OPTION != rc) {
                 ORTE_ERROR_LOG(rc);
                 return rc;

orte/mca/schizo/ompi/schizo_ompi.c

@@ -13,7 +13,7 @@
  *                         All rights reserved.
  * Copyright (c) 2009-2015 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2011      Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2013-2015 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2013-2016 Intel, Inc.  All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
@@ -42,17 +42,17 @@
 #include "orte/mca/errmgr/errmgr.h"
 #include "orte/mca/ess/base/base.h"
 #include "orte/mca/rmaps/rmaps_types.h"
+#include "orte/orted/orted_submit.h"
 #include "orte/util/name_fns.h"
 #include "orte/util/session_dir.h"
 #include "orte/util/show_help.h"
 #include "orte/runtime/orte_globals.h"
 
-#include "orte/mca/schizo/schizo.h"
+#include "orte/mca/schizo/base/base.h"
 
-static int parse_cli(char **personality,
+static int define_cli(opal_cmd_line_t *cli);
-                     int argc, int start, char **argv);
+static int parse_cli(int argc, int start, char **argv);
-static int parse_env(char **personality,
+static int parse_env(char *path,
-                     char *path,
                      opal_cmd_line_t *cmd_line,
                      char **srcenv,
                      char ***dstenv);
@@ -63,14 +63,426 @@ static int setup_child(orte_job_t *jobdat,
                        orte_app_context_t *app);
 
 orte_schizo_base_module_t orte_schizo_ompi_module = {
+    .define_cli = define_cli,
     .parse_cli = parse_cli,
     .parse_env = parse_env,
     .setup_fork = setup_fork,
     .setup_child = setup_child
 };
 
-static int parse_cli(char **personality,
+
-                     int argc, int start, char **argv)
+static opal_cmd_line_init_t cmd_line_init[] = {
+    /* Various "obvious" options */
+    { NULL, 'h', NULL, "help", 0,
+      &orte_cmd_line.help, OPAL_CMD_LINE_TYPE_BOOL,
+      "This help message" },
+    { NULL, 'V', NULL, "version", 0,
+      &orte_cmd_line.version, OPAL_CMD_LINE_TYPE_BOOL,
+      "Print version and exit" },
+    { NULL, 'v', NULL, "verbose", 0,
+      &orte_cmd_line.verbose, OPAL_CMD_LINE_TYPE_BOOL,
+      "Be verbose" },
+    { "orte_execute_quiet", 'q', NULL, "quiet", 0,
+      NULL, OPAL_CMD_LINE_TYPE_BOOL,
+      "Suppress helpful messages" },
+    { NULL, '\0', "report-pid", "report-pid", 1,
+      &orte_cmd_line.report_pid, OPAL_CMD_LINE_TYPE_STRING,
+      "Printout pid on stdout [-], stderr [+], or a file [anything else]" },
+    { NULL, '\0', "report-uri", "report-uri", 1,
+      &orte_cmd_line.report_uri, OPAL_CMD_LINE_TYPE_STRING,
+      "Printout URI on stdout [-], stderr [+], or a file [anything else]" },
+
+    /* exit status reporting */
+    { "orte_report_child_jobs_separately", '\0', "report-child-jobs-separately", "report-child-jobs-separately", 0,
+      NULL, OPAL_CMD_LINE_TYPE_BOOL,
+      "Return the exit status of the primary job only" },
+
+    /* uri of the dvm, or at least where to get it */
+    { NULL, '\0', "hnp", "hnp", 1,
+      &orte_cmd_line.hnp, OPAL_CMD_LINE_TYPE_STRING,
+      "Specify the URI of the HNP, or the name of the file (specified as file:filename) that contains that info" },
+
+    /* hetero apps */
+    { "orte_hetero_apps", '\0', NULL, "hetero-apps", 0,
+        NULL, OPAL_CMD_LINE_TYPE_BOOL,
+    "Indicates that multiple app_contexts are being provided that are a mix of 32/64 bit binaries" },
+
+    /* select XML output */
+    { "orte_xml_output", '\0', "xml", "xml", 0,
+      NULL, OPAL_CMD_LINE_TYPE_BOOL,
+      "Provide all output in XML format" },
+    { "orte_xml_file", '\0', "xml-file", "xml-file", 1,
+      NULL, OPAL_CMD_LINE_TYPE_STRING,
+      "Provide all output in XML format to the specified file" },
+
+    /* tag output */
+    { "orte_tag_output", '\0', "tag-output", "tag-output", 0,
+      NULL, OPAL_CMD_LINE_TYPE_BOOL,
+      "Tag all output with [job,rank]" },
+    { "orte_timestamp_output", '\0', "timestamp-output", "timestamp-output", 0,
+      NULL, OPAL_CMD_LINE_TYPE_BOOL,
+      "Timestamp all application process output" },
+    { "orte_output_filename", '\0', "output-filename", "output-filename", 1,
+      NULL, OPAL_CMD_LINE_TYPE_STRING,
+      "Redirect output from application processes into filename/job/rank/std[out,err,diag]" },
+    { NULL, '\0', "merge-stderr-to-stdout", "merge-stderr-to-stdout", 0,
+      &orte_cmd_line.merge, OPAL_CMD_LINE_TYPE_BOOL,
+      "Merge stderr to stdout for each process"},
+    { "orte_xterm", '\0', "xterm", "xterm", 1,
+      NULL, OPAL_CMD_LINE_TYPE_STRING,
+      "Create a new xterm window and display output from the specified ranks there" },
+
+    /* select stdin option */
+    { NULL, '\0', "stdin", "stdin", 1,
+      &orte_cmd_line.stdin_target, OPAL_CMD_LINE_TYPE_STRING,
+      "Specify procs to receive stdin [rank, all, none] (default: 0, indicating rank 0)" },
+
+    /* request that argv[0] be indexed */
+    { NULL, '\0', "index-argv-by-rank", "index-argv-by-rank", 0,
+      &orte_cmd_line.index_argv, OPAL_CMD_LINE_TYPE_BOOL,
+      "Uniquely index argv[0] for each process using its rank" },
+
+    /* Specify the launch agent to be used */
+    { "orte_launch_agent", '\0', "launch-agent", "launch-agent", 1,
+      NULL, OPAL_CMD_LINE_TYPE_STRING,
+      "Command used to start processes on remote nodes (default: orted)" },
+
+    /* Preload the binary on the remote machine */
+    { NULL, 's', NULL, "preload-binary", 0,
+      &orte_cmd_line.preload_binaries, OPAL_CMD_LINE_TYPE_BOOL,
+      "Preload the binary on the remote machine before starting the remote process." },
+
+    /* Preload files on the remote machine */
+    { NULL, '\0', NULL, "preload-files", 1,
+      &orte_cmd_line.preload_files, OPAL_CMD_LINE_TYPE_STRING,
+      "Preload the comma separated list of files to the remote machines current working directory before starting the remote process." },
+
+#if OPAL_ENABLE_FT_CR == 1
+    /* Tell SStore to preload a snapshot before launch */
+    { NULL, '\0', NULL, "sstore-load", 1,
+      &orte_cmd_line.sstore_load, OPAL_CMD_LINE_TYPE_STRING,
+      "Internal Use Only! Tell SStore to preload a snapshot before launch." },
+#endif
+
+    /* Use an appfile */
+    { NULL, '\0', NULL, "app", 1,
+      &orte_cmd_line.appfile, OPAL_CMD_LINE_TYPE_STRING,
+      "Provide an appfile; ignore all other command line options" },
+
+    /* Number of processes; -c, -n, --n, -np, and --np are all
+       synonyms */
+    { NULL, 'c', "np", "np", 1,
+      &orte_cmd_line.num_procs, OPAL_CMD_LINE_TYPE_INT,
+      "Number of processes to run" },
+    { NULL, '\0', "n", "n", 1,
+      &orte_cmd_line.num_procs, OPAL_CMD_LINE_TYPE_INT,
+      "Number of processes to run" },
+
+    /* maximum size of VM - typically used to subdivide an allocation */
+    { "orte_max_vm_size", '\0', "max-vm-size", "max-vm-size", 1,
+      NULL, OPAL_CMD_LINE_TYPE_INT,
+      "Number of processes to run" },
+
+    /* Set a hostfile */
+    { NULL, '\0', "hostfile", "hostfile", 1,
+      NULL, OPAL_CMD_LINE_TYPE_STRING,
+      "Provide a hostfile" },
+    { NULL, '\0', "machinefile", "machinefile", 1,
+      NULL, OPAL_CMD_LINE_TYPE_STRING,
+      "Provide a hostfile" },
+    { "orte_default_hostfile", '\0', "default-hostfile", "default-hostfile", 1,
+        NULL, OPAL_CMD_LINE_TYPE_STRING,
+      "Provide a default hostfile" },
+    { "opal_if_do_not_resolve", '\0', "do-not-resolve", "do-not-resolve", 0,
+      NULL, OPAL_CMD_LINE_TYPE_BOOL,
+      "Do not attempt to resolve interfaces" },
+
+    /* uri of PMIx publish/lookup server, or at least where to get it */
+    { "pmix_server_uri", '\0', "ompi-server", "ompi-server", 1,
+      NULL, OPAL_CMD_LINE_TYPE_STRING,
+      "Specify the URI of the publish/lookup server, or the name of the file (specified as file:filename) that contains that info" },
+
+    { "carto_file_path", '\0', "cf", "cartofile", 1,
+      NULL, OPAL_CMD_LINE_TYPE_STRING,
+      "Provide a cartography file" },
+
+    { "orte_rankfile", '\0', "rf", "rankfile", 1,
+      NULL, OPAL_CMD_LINE_TYPE_STRING,
+      "Provide a rankfile file" },
+
+    /* Export environment variables; potentially used multiple times,
+       so it does not make sense to set into a variable */
+    { NULL, 'x', NULL, NULL, 1,
+      NULL, OPAL_CMD_LINE_TYPE_NULL,
+      "Export an environment variable, optionally specifying a value (e.g., \"-x foo\" exports the environment variable foo and takes its value from the current environment; \"-x foo=bar\" exports the environment variable name foo and sets its value to \"bar\" in the started processes)" },
+
+      /* Mapping controls */
+    { "rmaps_base_display_map", '\0', "display-map", "display-map", 0,
+      NULL, OPAL_CMD_LINE_TYPE_BOOL,
+      "Display the process map just before launch"},
+    { "rmaps_base_display_devel_map", '\0', "display-devel-map", "display-devel-map", 0,
+       NULL, OPAL_CMD_LINE_TYPE_BOOL,
+       "Display a detailed process map (mostly intended for developers) just before launch"},
+    { "rmaps_base_display_topo_with_map", '\0', "display-topo", "display-topo", 0,
+       NULL, OPAL_CMD_LINE_TYPE_BOOL,
+       "Display the topology as part of the process map (mostly intended for developers) just before launch"},
+    { "rmaps_base_display_diffable_map", '\0', "display-diffable-map", "display-diffable-map", 0,
+       NULL, OPAL_CMD_LINE_TYPE_BOOL,
+       "Display a diffable process map (mostly intended for developers) just before launch"},
+    { NULL, 'H', "host", "host", 1,
+      NULL, OPAL_CMD_LINE_TYPE_STRING,
+      "List of hosts to invoke processes on" },
+    { "rmaps_base_no_schedule_local", '\0', "nolocal", "nolocal", 0,
+      NULL, OPAL_CMD_LINE_TYPE_BOOL,
+      "Do not run any MPI applications on the local node" },
+    { "rmaps_base_no_oversubscribe", '\0', "nooversubscribe", "nooversubscribe", 0,
+      NULL, OPAL_CMD_LINE_TYPE_BOOL,
+      "Nodes are not to be oversubscribed, even if the system supports such operation"},
+    { "rmaps_base_oversubscribe", '\0', "oversubscribe", "oversubscribe", 0,
+      NULL, OPAL_CMD_LINE_TYPE_BOOL,
+      "Nodes are allowed to be oversubscribed, even on a managed system, and overloading of processing elements"},
+    { "rmaps_base_cpus_per_rank", '\0', "cpus-per-proc", "cpus-per-proc", 1,
+      NULL, OPAL_CMD_LINE_TYPE_INT,
+      "Number of cpus to use for each process [default=1]" },
+    { "rmaps_base_cpus_per_rank", '\0', "cpus-per-rank", "cpus-per-rank", 1,
+      NULL, OPAL_CMD_LINE_TYPE_INT,
+      "Synonym for cpus-per-proc" },
+
+    /* backward compatiblity */
+    { "rmaps_base_bycore", '\0', "bycore", "bycore", 0,
+      NULL, OPAL_CMD_LINE_TYPE_BOOL,
+      "Whether to map and rank processes round-robin by core" },
+    { "rmaps_base_bynode", '\0', "bynode", "bynode", 0,
+      NULL, OPAL_CMD_LINE_TYPE_BOOL,
+      "Whether to map and rank processes round-robin by node" },
+    { "rmaps_base_byslot", '\0', "byslot", "byslot", 0,
+      NULL, OPAL_CMD_LINE_TYPE_BOOL,
+      "Whether to map and rank processes round-robin by slot" },
+
+    /* Nperxxx options that do not require topology and are always
+     * available - included for backwards compatibility
+     */
+    { "rmaps_ppr_pernode", '\0', "pernode", "pernode", 0,
+      NULL, OPAL_CMD_LINE_TYPE_BOOL,
+      "Launch one process per available node" },
+    { "rmaps_ppr_n_pernode", '\0', "npernode", "npernode", 1,
+        NULL, OPAL_CMD_LINE_TYPE_INT,
+        "Launch n processes per node on all allocated nodes" },
+    { "rmaps_ppr_n_pernode", '\0', "N", NULL, 1,
+        NULL, OPAL_CMD_LINE_TYPE_INT,
+        "Launch n processes per node on all allocated nodes (synonym for npernode)" },
+
+    /* declare hardware threads as independent cpus */
+    { "hwloc_base_use_hwthreads_as_cpus", '\0', "use-hwthread-cpus", "use-hwthread-cpus", 0,
+      NULL, OPAL_CMD_LINE_TYPE_BOOL,
+      "Use hardware threads as independent cpus" },
+
+    /* include npersocket for backwards compatibility */
+    { "rmaps_ppr_n_persocket", '\0', "npersocket", "npersocket", 1,
+      NULL, OPAL_CMD_LINE_TYPE_INT,
+      "Launch n processes per socket on all allocated nodes" },
+
+    /* Mapping options */
+    { "rmaps_base_mapping_policy", '\0', NULL, "map-by", 1,
+      NULL, OPAL_CMD_LINE_TYPE_STRING,
+      "Mapping Policy [slot | hwthread | core | socket (default) | numa | board | node]" },
+
+      /* Ranking options */
+    { "rmaps_base_ranking_policy", '\0', NULL, "rank-by", 1,
+      NULL, OPAL_CMD_LINE_TYPE_STRING,
+      "Ranking Policy [slot (default) | hwthread | core | socket | numa | board | node]" },
+
+      /* Binding options */
+    { "hwloc_base_binding_policy", '\0', NULL, "bind-to", 1,
+      NULL, OPAL_CMD_LINE_TYPE_STRING,
+      "Policy for binding processes. Allowed values: none, hwthread, core, l1cache, l2cache, l3cache, socket, numa, board (\"none\" is the default when oversubscribed, \"core\" is the default when np<=2, and \"socket\" is the default when np>2). Allowed qualifiers: overload-allowed, if-supported" },
+
+    /* backward compatiblity */
+    { "hwloc_base_bind_to_core", '\0', "bind-to-core", "bind-to-core", 0,
+      NULL, OPAL_CMD_LINE_TYPE_BOOL,
+      "Bind processes to cores" },
+    { "hwloc_base_bind_to_socket", '\0', "bind-to-socket", "bind-to-socket", 0,
+      NULL, OPAL_CMD_LINE_TYPE_BOOL,
+      "Bind processes to sockets" },
+
+    { "hwloc_base_report_bindings", '\0', "report-bindings", "report-bindings", 0,
+      NULL, OPAL_CMD_LINE_TYPE_BOOL,
+      "Whether to report process bindings to stderr" },
+
+    /* slot list option */
+    { "hwloc_base_slot_list", '\0', "slot-list", "slot-list", 1,
+      NULL, OPAL_CMD_LINE_TYPE_STRING,
+      "List of processor IDs to bind processes to [default=NULL]"},
+
+    /* generalized pattern mapping option */
+    { "rmaps_ppr_pattern", '\0', NULL, "ppr", 1,
+        NULL, OPAL_CMD_LINE_TYPE_STRING,
+        "Comma-separated list of number of processes on a given resource type [default: none]" },
+
+    /* Allocation options */
+    { "orte_display_alloc", '\0', "display-allocation", "display-allocation", 0,
+      NULL, OPAL_CMD_LINE_TYPE_BOOL,
+      "Display the allocation being used by this job"},
+    { "orte_display_devel_alloc", '\0', "display-devel-allocation", "display-devel-allocation", 0,
+      NULL, OPAL_CMD_LINE_TYPE_BOOL,
+      "Display a detailed list (mostly intended for developers) of the allocation being used by this job"},
+    { "hwloc_base_cpu_set", '\0', "cpu-set", "cpu-set", 1,
+      NULL, OPAL_CMD_LINE_TYPE_STRING,
+      "Comma-separated list of ranges specifying logical cpus allocated to this job [default: none]"},
+
+    /* mpiexec-like arguments */
+    { NULL, '\0', "wdir", "wdir", 1,
+      &orte_cmd_line.wdir, OPAL_CMD_LINE_TYPE_STRING,
+      "Set the working directory of the started processes" },
+    { NULL, '\0', "wd", "wd", 1,
+      &orte_cmd_line.wdir, OPAL_CMD_LINE_TYPE_STRING,
+      "Synonym for --wdir" },
+    { NULL, '\0', "set-cwd-to-session-dir", "set-cwd-to-session-dir", 0,
+      &orte_cmd_line.set_cwd_to_session_dir, OPAL_CMD_LINE_TYPE_BOOL,
+      "Set the working directory of the started processes to their session directory" },
+    { NULL, '\0', "path", "path", 1,
+      &orte_cmd_line.path, OPAL_CMD_LINE_TYPE_STRING,
+      "PATH to be used to look for executables to start processes" },
+
+    /* User-level debugger arguments */
+    { NULL, '\0', "tv", "tv", 0,
+      &orte_cmd_line.debugger, OPAL_CMD_LINE_TYPE_BOOL,
+      "Deprecated backwards compatibility flag; synonym for \"--debug\"" },
+    { NULL, '\0', "debug", "debug", 0,
+      &orte_cmd_line.debugger, OPAL_CMD_LINE_TYPE_BOOL,
+      "Invoke the user-level debugger indicated by the orte_base_user_debugger MCA parameter" },
+    { "orte_base_user_debugger", '\0', "debugger", "debugger", 1,
+      NULL, OPAL_CMD_LINE_TYPE_STRING,
+      "Sequence of debuggers to search for when \"--debug\" is used" },
+    { "orte_output_debugger_proctable", '\0', "output-proctable", "output-proctable", 0,
+      NULL, OPAL_CMD_LINE_TYPE_BOOL,
+      "Output the debugger proctable after launch" },
+
+    /* OpenRTE arguments */
+    { "orte_debug", 'd', "debug-devel", "debug-devel", 0,
+      NULL, OPAL_CMD_LINE_TYPE_BOOL,
+      "Enable debugging of OpenRTE" },
+
+    { "orte_debug_daemons", '\0', "debug-daemons", "debug-daemons", 0,
+      NULL, OPAL_CMD_LINE_TYPE_INT,
+      "Enable debugging of any OpenRTE daemons used by this application" },
+
+    { "orte_debug_daemons_file", '\0', "debug-daemons-file", "debug-daemons-file", 0,
+      NULL, OPAL_CMD_LINE_TYPE_BOOL,
+      "Enable debugging of any OpenRTE daemons used by this application, storing output in files" },
+
+    { "orte_leave_session_attached", '\0', "leave-session-attached", "leave-session-attached", 0,
+      NULL, OPAL_CMD_LINE_TYPE_BOOL,
+      "Enable debugging of OpenRTE" },
+
+    { "orte_do_not_launch", '\0', "do-not-launch", "do-not-launch", 0,
+      NULL, OPAL_CMD_LINE_TYPE_BOOL,
+      "Perform all necessary operations to prepare to launch the application, but do not actually launch it" },
+
+    { NULL, '\0', NULL, "prefix", 1,
+      NULL, OPAL_CMD_LINE_TYPE_STRING,
+      "Prefix where Open MPI is installed on remote nodes" },
+    { NULL, '\0', NULL, "noprefix", 0,
+      NULL, OPAL_CMD_LINE_TYPE_STRING,
+      "Disable automatic --prefix behavior" },
+
+    { "orte_report_launch_progress", '\0', "show-progress", "show-progress", 0,
+      NULL, OPAL_CMD_LINE_TYPE_BOOL,
+      "Output a brief periodic report on launch progress" },
+
+    { "orte_use_regexp", '\0', "use-regexp", "use-regexp", 0,
+      NULL, OPAL_CMD_LINE_TYPE_BOOL,
+      "Use regular expressions for launch" },
+
+    { "orte_report_events", '\0', "report-events", "report-events", 1,
+      NULL, OPAL_CMD_LINE_TYPE_STRING,
+      "Report events to a tool listening at the specified URI" },
+
+    { "orte_enable_recovery", '\0', "enable-recovery", "enable-recovery", 0,
+      NULL, OPAL_CMD_LINE_TYPE_BOOL,
+      "Enable recovery from process failure [Default = disabled]" },
+
+    { "orte_max_restarts", '\0', "max-restarts", "max-restarts", 1,
+      NULL, OPAL_CMD_LINE_TYPE_INT,
+      "Max number of times to restart a failed process" },
+
+    { "orte_hetero_nodes", '\0', NULL, "hetero-nodes", 0,
+      NULL, OPAL_CMD_LINE_TYPE_BOOL,
+      "Nodes in cluster may differ in topology, so send the topology back from each node [Default = false]" },
+
+#if OPAL_ENABLE_CRDEBUG == 1
+    { "opal_cr_enable_crdebug", '\0', "crdebug", "crdebug", 0,
+      NULL, OPAL_CMD_LINE_TYPE_BOOL,
+      "Enable C/R Debugging" },
+#endif
+
+    { NULL, '\0', "disable-recovery", "disable-recovery", 0,
+      &orte_cmd_line.disable_recovery, OPAL_CMD_LINE_TYPE_BOOL,
+      "Disable recovery (resets all recovery options to off)" },
+
+    { "state_novm_select", '\0', "novm", "novm", 0,
+      NULL, OPAL_CMD_LINE_TYPE_BOOL,
+      "Execute without creating an allocation-spanning virtual machine (only start daemons on nodes hosting application procs)" },
+
+    { NULL, '\0', "staged", "staged", 0,
+      &orte_cmd_line.staged_exec, OPAL_CMD_LINE_TYPE_BOOL,
+      "Used staged execution if inadequate resources are present (cannot support MPI jobs)" },
+
+    { NULL, '\0', "allow-run-as-root", "allow-run-as-root", 0,
+      &orte_cmd_line.run_as_root, OPAL_CMD_LINE_TYPE_BOOL,
+      "Allow execution as root (STRONGLY DISCOURAGED)" },
+
+    { NULL, '\0', "personality", "personality", 1,
+      &orte_cmd_line.personality, OPAL_CMD_LINE_TYPE_STRING,
+      "Comma-separated list of programming model, languages, and containers being used (default=\"ompi\")" },
+
+    { NULL, '\0', "dvm", "dvm", 0,
+      &orte_cmd_line.create_dvm, OPAL_CMD_LINE_TYPE_BOOL,
+      "Create a persistent distributed virtual machine (DVM)" },
+
+    /* tell the dvm to terminate */
+    { NULL, '\0', "terminate", "terminate", 0,
+      &orte_cmd_line.terminate_dvm, OPAL_CMD_LINE_TYPE_BOOL,
+      "Terminate the DVM" },
+
+    /* End of list */
+    { NULL, '\0', NULL, NULL, 0,
+      NULL, OPAL_CMD_LINE_TYPE_NULL, NULL }
+};
+
+static int define_cli(opal_cmd_line_t *cli)
+{
+    int i, rc;
+    bool takeus = false;
+
+    opal_output_verbose(1, orte_schizo_base_framework.framework_output,
+                        "%s schizo:ompi: define_cli",
+                        ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
+
+    /* protect against bozo error */
+    if (NULL == cli) {
+        return ORTE_ERR_BAD_PARAM;
+    }
+
+    if (NULL != orte_schizo_base.personalities) {
+        /* if we aren't included, then ignore us */
+        for (i=0; NULL != orte_schizo_base.personalities[i]; i++) {
+            if (0 == strcmp(orte_schizo_base.personalities[i], "ompi")) {
+                takeus = true;
+                break;
+            }
+        }
+        if (!takeus) {
+            return ORTE_ERR_TAKE_NEXT_OPTION;
+        }
+    }
+
+    /* just add ours to the end */
+    rc = opal_cmd_line_add(cli, cmd_line_init);
+    return rc;
+}
+
+static int parse_cli(int argc, int start, char **argv)
 {
     int i, j, k;
     bool ignore;
@@ -83,15 +495,25 @@ static int parse_cli(char **personality,
     };
     bool takeus = false;
 
-    /* see if we are included */
+    opal_output_verbose(1, orte_schizo_base_framework.framework_output,
-    for (i=0; NULL != personality[i]; i++) {
+                        "%s schizo:ompi: parse_cli",
-        if (0 == strcmp(personality[i], "ompi")) {
+                        ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
-            takeus = true;
+
-            break;
+    /* if they gave us a list of personalities,
+     * see if we are included */
+    if (NULL != orte_schizo_base.personalities) {
+        for (i=0; NULL != orte_schizo_base.personalities[i]; i++) {
+            if (0 == strcmp(orte_schizo_base.personalities[i], "ompi")) {
+                takeus = true;
+                break;
+            }
         }
-    }
+        if (!takeus) {
-    if (!takeus) {
+            return ORTE_ERR_TAKE_NEXT_OPTION;
-        return ORTE_ERR_TAKE_NEXT_OPTION;
+        }
+    } else {
+        /* attempt to auto-detect CLI options that
+         * we recognize */
     }
 
     for (i = 0; i < (argc-start); ++i) {
@@ -162,8 +584,7 @@ static int parse_cli(char **personality,
     return ORTE_SUCCESS;
 }
 
-static int parse_env(char **personality,
+static int parse_env(char *path,
-                     char *path,
                      opal_cmd_line_t *cmd_line,
                      char **srcenv,
                      char ***dstenv)
@@ -175,15 +596,21 @@ static int parse_env(char **personality,
     char **vars;
     bool takeus = false;
 
-    /* see if we are included */
+    opal_output_verbose(1, orte_schizo_base_framework.framework_output,
-    for (i=0; NULL != personality[i]; i++) {
+                        "%s schizo:ompi: parse_env",
-        if (0 == strcmp(personality[i], "ompi")) {
+                        ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
-            takeus = true;
+
-            break;
+    if (NULL != orte_schizo_base.personalities) {
+        /* see if we are included */
+        for (i=0; NULL != orte_schizo_base.personalities[i]; i++) {
+            if (0 == strcmp(orte_schizo_base.personalities[i], "ompi")) {
+                takeus = true;
+                break;
+            }
+        }
+        if (!takeus) {
+            return ORTE_ERR_TAKE_NEXT_OPTION;
         }
-    }
-    if (!takeus) {
-        return ORTE_ERR_TAKE_NEXT_OPTION;
     }
 
     for (i = 0; NULL != srcenv[i]; ++i) {
@@ -307,15 +734,21 @@ static int setup_fork(orte_job_t *jdata,
     char *num_app_ctx;
     bool takeus = false;
 
+    opal_output_verbose(1, orte_schizo_base_framework.framework_output,
+                        "%s schizo:ompi: setup_fork",
+                        ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
+
+    if (NULL != orte_schizo_base.personalities) {
     /* see if we are included */
-    for (i=0; NULL != jdata->personality[i]; i++) {
+        for (i=0; NULL != jdata->personality[i]; i++) {
-        if (0 == strcmp(jdata->personality[i], "ompi")) {
+            if (0 == strcmp(jdata->personality[i], "ompi")) {
-            takeus = true;
+                takeus = true;
-            break;
+                break;
+            }
+        }
+        if (!takeus) {
+            return ORTE_ERR_TAKE_NEXT_OPTION;
         }
-    }
-    if (!takeus) {
-        return ORTE_ERR_TAKE_NEXT_OPTION;
     }
 
     /* see if the mapper thinks we are oversubscribed */
@@ -539,15 +972,21 @@ static int setup_child(orte_job_t *jdata,
     int32_t nrestarts=0, *nrptr;
     bool takeus = false;
 
-    /* see if we are included */
+    opal_output_verbose(1, orte_schizo_base_framework.framework_output,
-    for (i=0; NULL != jdata->personality[i]; i++) {
+                        "%s schizo:ompi: setup_child",
-        if (0 == strcmp(jdata->personality[i], "ompi")) {
+                        ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
-            takeus = true;
+
-            break;
+    if (NULL != orte_schizo_base.personalities) {
+        /* see if we are included */
+        for (i=0; NULL != jdata->personality[i]; i++) {
+            if (0 == strcmp(jdata->personality[i], "ompi")) {
+                takeus = true;
+                break;
+            }
+        }
+        if (!takeus) {
+            return ORTE_ERR_TAKE_NEXT_OPTION;
         }
-    }
-    if (!takeus) {
-        return ORTE_ERR_TAKE_NEXT_OPTION;
     }
 
     /* setup the jobid */

orte/mca/schizo/schizo.h

@@ -44,32 +44,40 @@ BEGIN_C_DECLS
  * things it requires */
 typedef int (*orte_schizo_base_module_init_fn_t)(void);
 
-/* given an argv-array of personalities, parse a tool command line
+/* provide an opportunity for components to add personality and/or
+ * environment-specific command line options. The OPAL cli tools
+ * will add provided options to the CLI definition, and so the
+ * resulting CLI array will include the _union_ of options provided
+ * by the various components. Where there is overlap (i.e., an option
+ * is added that was also defined earlier in the stack), then the
+ * first definition is used. This reflects the higher priority of
+ * the original definition - note that this only impacts the help
+ * message that will be displayed */
+typedef int (*orte_schizo_base_module_define_cli_fn_t)(opal_cmd_line_t *cli);
+
+/* parse a tool command line
  * starting from the given location according to the cmd line options
  * known to this module's personality. First, of course, check that
- * this module is included in the specified array of personalities!
+ * this module is included in the base array of personalities, or is
- * Only one command-line parser is allowed to operate - i.e., if */
+ * automatically recognizable! */
-typedef int (*orte_schizo_base_module_parse_cli_fn_t)(char **personality,
+typedef int (*orte_schizo_base_module_parse_cli_fn_t)(int argc, int start,
-                                                      int argc, int start,
                                                       char **argv);
 
-/* given an argv-array of personalities, parse the environment of the
+/* parse the environment of the
  * tool to extract any personality-specific envars that need to be
  * forward to the app's environment upon execution */
-typedef int (*orte_schizo_base_module_parse_env_fn_t)(char **personality,
+typedef int (*orte_schizo_base_module_parse_env_fn_t)(char *path,
-                                                      char *path,
                                                       opal_cmd_line_t *cmd_line,
                                                       char **srcenv,
                                                       char ***dstenv);
 
-/* given an argv-array of personalities, do whatever preparation work
+/* do whatever preparation work
  * is required to setup the app for execution. This is intended to be
  * used by orterun and other launcher tools to, for example, change
  * an executable's relative-path to an absolute-path, or add a command
  * required for starting a particular kind of application (e.g., adding
  * "java" to start a Java application) */
-typedef int (*orte_schizo_base_module_setup_app_fn_t)(char **personality,
+typedef int (*orte_schizo_base_module_setup_app_fn_t)(orte_app_context_t *app);
-                                                      orte_app_context_t *app);
 
 /* add any personality-specific envars required at the job level prior
  * to beginning to execute local procs */
@@ -107,6 +115,7 @@ typedef void (*orte_schizo_base_module_finalize_fn_t)(void);
  */
 typedef struct {
     orte_schizo_base_module_init_fn_t                   init;
+    orte_schizo_base_module_define_cli_fn_t             define_cli;
     orte_schizo_base_module_parse_cli_fn_t              parse_cli;
     orte_schizo_base_module_parse_env_fn_t              parse_env;
     orte_schizo_base_module_setup_app_fn_t              setup_app;

orte/mca/schizo/singularity/schizo_singularity.c

@@ -29,8 +29,7 @@
 
 #include "schizo_singularity.h"
 
-static int setup_app(char **personality,
+static int setup_app(orte_app_context_t *context);
-                     orte_app_context_t *context);
 static int setup_fork(orte_job_t *jdata,
                       orte_app_context_t *context);
 
@@ -39,18 +38,19 @@ orte_schizo_base_module_t orte_schizo_singularity_module = {
     .setup_fork = setup_fork
 };
 
-static int setup_app(char **personality,
+static int setup_app(orte_app_context_t *app)
-                     orte_app_context_t *app)
 {
     int i;
     char *newenv, *pth, *t2;
     bool takeus = false;
 
-    /* see if we are included */
+    if (NULL != orte_schizo_base.personalities) {
-    for (i=0; NULL != personality[i]; i++) {
+        /* see if we are included */
-        if (0 == strcmp(personality[i], "singularity")) {
+        for (i=0; NULL != orte_schizo_base.personalities[i]; i++) {
-            takeus = true;
+            if (0 == strcmp(orte_schizo_base.personalities[i], "singularity")) {
-            break;
+                takeus = true;
+                break;
+            }
         }
     }
     if (!takeus) {
@@ -113,11 +113,13 @@ static int setup_fork(orte_job_t *jdata,
     char *p, *t2;
     char dir[MAXPATHLEN];
 
-    /* see if we are included */
+    if (NULL != orte_schizo_base.personalities) {
-    for (i=0; NULL != jdata->personality[i]; i++) {
+        /* see if we are included */
-        if (0 == strcmp(jdata->personality[i], "singularity")) {
+        for (i=0; NULL != jdata->personality[i]; i++) {
-            takeus = true;
+            if (0 == strcmp(jdata->personality[i], "singularity")) {
-            break;
+                takeus = true;
+                break;
+            }
         }
     }
     if (!takeus) {

orte/orted/orted_main.c

@@ -263,7 +263,7 @@ int orte_daemon(int argc, char *argv[])
      * Since this process can now handle MCA/GMCA parameters, make sure to
      * process them.
      */
-    mca_base_cmd_line_process_args(cmd_line, &environ, &environ);
+    mca_base_cmd_line_process_args(argv, &environ, &environ);
 
     /* Ensure that enough of OPAL is setup for us to be able to run */
     /*

orte/orted/orted_submit.h

@@ -28,6 +28,13 @@ ORTE_DECLSPEC int orte_submit_job(char *cmd[], int *index,
                                   orte_submit_cbfunc_t launch_cb, void *launch_cbdata,
                                   orte_submit_cbfunc_t complete_cb, void *complete_cbdata);
 ORTE_DECLSPEC int orte_submit_halt(void);
+ORTE_DECLSPEC void orte_debugger_init_after_spawn(int fd, short event, void *arg);
+ORTE_DECLSPEC void orte_debugger_detached(int fd, short event, void *arg);
+
+extern int orte_debugger_attach_fd;
+extern bool orte_debugger_fifo_active;
+extern opal_event_t *orte_debugger_attach;
+extern char MPIR_attach_fifo[];
 
 /**
  * Global struct for catching orte command line options.
@@ -82,6 +89,7 @@ struct orte_cmd_line_t {
     bool merge;
     bool enable_recovery;
     char *hnp;
+    bool staged_exec;
 };
 typedef struct orte_cmd_line_t orte_cmd_line_t;
 ORTE_DECLSPEC extern orte_cmd_line_t orte_cmd_line;

orte/orted/pmix/pmix_server_dyn.c

@@ -169,25 +169,17 @@ int pmix_server_spawn_fn(opal_process_name_t *requestor,
 
     /* create the job object */
     jdata = OBJ_NEW(orte_job_t);
+    jdata->map = OBJ_NEW(orte_job_map_t);
 
     /* transfer the job info across */
     OPAL_LIST_FOREACH(info, job_info, opal_value_t) {
         if (0 == strcmp(info->key, OPAL_PMIX_PERSONALITY)) {
             jdata->personality = opal_argv_split(info->data.string, ',');
         } else if (0 == strcmp(info->key, OPAL_PMIX_MAPPER)) {
-            if (NULL == jdata->map) {
-                jdata->map = OBJ_NEW(orte_job_map_t);
-            }
             jdata->map->req_mapper = strdup(info->data.string);
         } else if (0 == strcmp(info->key, OPAL_PMIX_DISPLAY_MAP)) {
-            if (NULL == jdata->map) {
-                jdata->map = OBJ_NEW(orte_job_map_t);
-            }
             jdata->map->display_map = true;
         } else if (0 == strcmp(info->key, OPAL_PMIX_PPR)) {
-            if (NULL == jdata->map) {
-                jdata->map = OBJ_NEW(orte_job_map_t);
-            }
             if (ORTE_MAPPING_POLICY_IS_SET(jdata->map->mapping)) {
                 /* not allowed to provide multiple mapping policies */
                 orte_show_help("help-orte-rmaps-base.txt", "redefining-policy",
@@ -198,9 +190,6 @@ int pmix_server_spawn_fn(opal_process_name_t *requestor,
             ORTE_SET_MAPPING_DIRECTIVE(jdata->map->mapping, ORTE_MAPPING_PPR);
             jdata->map->ppr = strdup(info->data.string);
         } else if (0 == strcmp(info->key, OPAL_PMIX_MAPBY)) {
-            if (NULL == jdata->map) {
-                jdata->map = OBJ_NEW(orte_job_map_t);
-            }
             if (ORTE_MAPPING_POLICY_IS_SET(jdata->map->mapping)) {
                 /* not allowed to provide multiple mapping policies */
                 orte_show_help("help-orte-rmaps-base.txt", "redefining-policy",
@@ -214,9 +203,6 @@ int pmix_server_spawn_fn(opal_process_name_t *requestor,
                 return rc;
             }
         } else if (0 == strcmp(info->key, OPAL_PMIX_RANKBY)) {
-            if (NULL == jdata->map) {
-                jdata->map = OBJ_NEW(orte_job_map_t);
-            }
             if (ORTE_RANKING_POLICY_IS_SET(jdata->map->ranking)) {
                 /* not allowed to provide multiple ranking policies */
                 orte_show_help("help-orte-rmaps-base.txt", "redefining-policy",
@@ -231,9 +217,6 @@ int pmix_server_spawn_fn(opal_process_name_t *requestor,
                 return rc;
             }
         } else if (0 == strcmp(info->key, OPAL_PMIX_BINDTO)) {
-            if (NULL == jdata->map) {
-                jdata->map = OBJ_NEW(orte_job_map_t);
-            }
             if (OPAL_BINDING_POLICY_IS_SET(jdata->map->binding)) {
                 /* not allowed to provide multiple mapping policies */
                 orte_show_help("help-opal-hwloc-base.txt", "redefining-policy", true,

orte/tools/Makefile.am

@@ -13,7 +13,7 @@
 # Copyright (c) 2006-2008 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2011-2013 Los Alamos National Security, LLC.  All rights
 #                         reserved.
-# Copyright (c) 2014-2015 Intel, Inc.  All rights reserved.
+# Copyright (c) 2014-2016 Intel, Inc.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -36,7 +36,6 @@ SUBDIRS += \
         tools/orte-info \
         tools/orte-migrate \
         tools/orte-server \
-        tools/orte-submit \
         tools/orte-dvm
 
 DIST_SUBDIRS += \
@@ -51,6 +50,5 @@ DIST_SUBDIRS += \
         tools/orte-info \
         tools/orte-migrate \
         tools/orte-server \
-        tools/orte-submit \
         tools/orte-dvm
 

orte/tools/orte-checkpoint/orte-checkpoint.c

@@ -417,7 +417,7 @@ static int parse_args(int argc, char *argv[]) {
     /**
      * Put all of the MCA arguments in the environment
      */
-    mca_base_cmd_line_process_args(&cmd_line, &app_env, &global_env);
+    mca_base_cmd_line_process_args(argc, &app_env, &global_env);
 
     len = opal_argv_count(app_env);
     for(i = 0; i < len; ++i) {

orte/tools/orte-dvm/orte-dvm.c

@@ -234,7 +234,7 @@ int main(int argc, char *argv[])
      *  opal_init_util() since mca_base_cmd_line_process_args() does *not*
      *  depend upon opal_init_util() functionality.
      */
-    if (OPAL_SUCCESS != mca_base_cmd_line_process_args(&cmd_line, &environ, &environ)) {
+    if (OPAL_SUCCESS != mca_base_cmd_line_process_args(argv, &environ, &environ)) {
         exit(1);
     }
 

orte/tools/orte-info/orte-info.c

@@ -184,7 +184,7 @@ int main(int argc, char *argv[])
         exit(cmd_error ? 1 : 0);
     }
 
-    mca_base_cmd_line_process_args(orte_info_cmd_line, &app_env, &global_env);
+    mca_base_cmd_line_process_args(argv, &app_env, &global_env);
 
     /* putenv() all the stuff that we got back from env (in case the
      * user specified some --mca params on the command line).  This

orte/tools/orte-migrate/orte-migrate.c

@@ -304,7 +304,7 @@ static int parse_args(int argc, char *argv[]) {
     /**
      * Put all of the MCA arguments in the environment
      */
-    mca_base_cmd_line_process_args(&cmd_line, &app_env, &global_env);
+    mca_base_cmd_line_process_args(argv, &app_env, &global_env);
 
     len = opal_argv_count(app_env);
     for(i = 0; i < len; ++i) {

orte/tools/orte-restart/orte-restart.c

@@ -461,7 +461,7 @@ static int parse_args(int argc, char *argv[])
     /**
      * Put all of the MCA arguments in the environment
      */
-    mca_base_cmd_line_process_args(&cmd_line, &app_env, &global_env);
+    mca_base_cmd_line_process_args(argv, &app_env, &global_env);
 
     len = opal_argv_count(app_env);
     for(i = 0; i < len; ++i) {

orte/tools/orte-server/orte-server.c

@@ -149,7 +149,7 @@ int main(int argc, char *argv[])
      * Since this process can now handle MCA/GMCA parameters, make sure to
      * process them.
      */
-    mca_base_cmd_line_process_args(cmd_line, &environ, &environ);
+    mca_base_cmd_line_process_args(argv, &environ, &environ);
 
     /* if debug is set, then set orte_debug_flag so that the data server
      * code will output

orte/tools/orte-submit/Makefile.am

@@ -1,57 +0,0 @@
-#
-# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-#                         University Research and Technology
-#                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2005 The University of Tennessee and The University
-#                         of Tennessee Research Foundation.  All rights
-#                         reserved.
-# Copyright (c) 2004-2009 High Performance Computing Center Stuttgart,
-#                         University of Stuttgart.  All rights reserved.
-# Copyright (c) 2004-2005 The Regents of the University of California.
-#                         All rights reserved.
-# Copyright (c) 2008-2014 Cisco Systems, Inc.  All rights reserved.
-# Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
-# Copyright (c) 2015      Intel, Inc.  All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-# This is not quite in the Automake spirit, but we have to do it.
-# Since the totalview portion of the library must be built with -g, we
-# must eliminate the CFLAGS that are passed in here by default (which
-# may already have debugging and/or optimization flags).  We use
-# post-processed forms of the CFLAGS in the library targets down
-# below.
-
-CFLAGS = $(CFLAGS_WITHOUT_OPTFLAGS) $(DEBUGGER_CFLAGS)
-
-include $(top_srcdir)/Makefile.ompi-rules
-
-man_pages = orte-submit.1
-EXTRA_DIST = $(man_pages:.1=.1in)
-
-if OPAL_INSTALL_BINARIES
-
-bin_PROGRAMS = orte-submit
-
-nodist_man_MANS = $(man_pages)
-
-# Ensure that the man pages are rebuilt if the opal_config.h file
-# changes; a "good enough" way to know if configure was run again (and
-# therefore the release date or version may have changed)
-$(nodist_man_MANS): $(top_builddir)/opal/include/opal_config.h
-
-endif # OPAL_INSTALL_BINARIES
-
-orte_submit_SOURCES = \
-        orte-submit.c
-
-orte_submit_LDADD = \
-	$(top_builddir)/orte/lib@ORTE_LIB_PREFIX@open-rte.la \
-	$(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la
-
-distclean-local:
-	rm -f $(man_pages)

orte/tools/orte-submit/orte-submit.c

@@ -1,182 +0,0 @@
-/* -*- C -*-
- *
- * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2008 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2014 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2007-2009 Sun Microsystems, Inc. All rights reserved.
- * Copyright (c) 2007-2013 Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2013-2016 Intel, Inc. All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "orte_config.h"
-#include "orte/constants.h"
-
-#include <string.h>
-#include <stdio.h>
-#include <stdlib.h>
-
-#include "opal/dss/dss.h"
-#include "opal/mca/event/event.h"
-
-#include "orte/mca/errmgr/errmgr.h"
-#include "orte/orted/orted_submit.h"
-#include "orte/runtime/orte_globals.h"
-#include "orte/util/show_help.h"
-
-/*
- * Globals
- */
-typedef struct {
-    int status;
-    volatile bool active;
-    orte_job_t *jdata;
-} orte_submit_status_t;
-
-static void launched(int index, orte_job_t *jdata, int ret, void *cbdata);
-static void completed(int index, orte_job_t *jdata, int ret, void *cbdata);
-
-
-static opal_cmd_line_init_t cmd_line_init[] = {
-    { "orte_execute_quiet", 'q', NULL, "quiet", 0,
-      NULL, OPAL_CMD_LINE_TYPE_BOOL,
-      "Suppress helpful messages" },
-
-    { NULL, '\0', "report-pid", "report-pid", 1,
-      &orte_cmd_line.report_pid, OPAL_CMD_LINE_TYPE_STRING,
-      "Printout pid on stdout [-], stderr [+], or a file [anything else]" },
-    { NULL, '\0', "report-uri", "report-uri", 1,
-      &orte_cmd_line.report_uri, OPAL_CMD_LINE_TYPE_STRING,
-      "Printout URI on stdout [-], stderr [+], or a file [anything else]" },
-
-    /* exit status reporting */
-    { "orte_report_child_jobs_separately", '\0', "report-child-jobs-separately", "report-child-jobs-separately", 0,
-      NULL, OPAL_CMD_LINE_TYPE_BOOL,
-      "Return the exit status of the primary job only" },
-
-    /* select XML output */
-    { "orte_xml_output", '\0', "xml", "xml", 0,
-      NULL, OPAL_CMD_LINE_TYPE_BOOL,
-      "Provide all output in XML format" },
-    { "orte_xml_file", '\0', "xml-file", "xml-file", 1,
-      NULL, OPAL_CMD_LINE_TYPE_STRING,
-      "Provide all output in XML format to the specified file" },
-
-    { "orte_xterm", '\0', "xterm", "xterm", 1,
-      NULL, OPAL_CMD_LINE_TYPE_STRING,
-      "Create a new xterm window and display output from the specified ranks there" },
-
-    /* tell the dvm to terminate */
-    { NULL, '\0', "terminate", "terminate", 0,
-      &orte_cmd_line.terminate_dvm, OPAL_CMD_LINE_TYPE_BOOL,
-      "Terminate the DVM" },
-
-    /* End of list */
-    { NULL, '\0', NULL, NULL, 0,
-      NULL, OPAL_CMD_LINE_TYPE_NULL, NULL }
-};
-
-int main(int argc, char *argv[])
-{
-    int rc;
-    orte_submit_status_t launchst, completest;
-    opal_cmd_line_t cmd_line;
-
-    memset(&orte_cmd_line, 0, sizeof(orte_cmd_line));
-    /* setup our cmd line */
-    opal_cmd_line_create(&cmd_line, cmd_line_init);
-    mca_base_cmd_line_setup(&cmd_line);
-
-    /* initialize the RTE */
-    if (ORTE_SUCCESS != (rc = orte_submit_init(argc, argv, &cmd_line))) {
-        fprintf(stderr, "Init failed due to duplicate command options\n");
-        exit(rc);
-    }
-
-    /* if this is the terminate command, just send it */
-    if (orte_cmd_line.terminate_dvm) {
-        rc = orte_submit_halt();
-        /* just loop the event library - the errmgr
-         * will exit us when the connection to our
-         * HNP closes */
-         while (1) {
-            opal_event_loop(orte_event_base, OPAL_EVLOOP_ONCE);
-         }
-    }
-
-    /* launch whatever job we were given */
-    memset(&launchst, 0, sizeof(launchst));
-    memset(&completest, 0, sizeof(completest));
-    launchst.active = true;
-    completest.active = true;
-    if (ORTE_SUCCESS != (rc = orte_submit_job(argv, NULL,
-                                              launched, &launchst,
-                                              completed, &completest))) {
-        if (ORTE_ERR_OP_IN_PROGRESS == rc) {
-            /* terminate command was given */
-            goto waiting;
-        }
-        opal_output(0, "JOB FAILED TO LAUNCH WITH ERROR %d:%s",
-                    rc, ORTE_ERROR_NAME(rc));
-        goto DONE;
-    }
-
-    // wait for response and unpack the status, jobid
-    while (launchst.active) {
-        opal_event_loop(orte_event_base, OPAL_EVLOOP_ONCE);
-    }
-    if (orte_debug_flag) {
-        opal_output(0, "Job %s has launched", ORTE_JOBID_PRINT(launchst.jdata->jobid));
-    }
-    if (ORTE_SUCCESS != launchst.status) {
-        goto DONE;
-    }
-
-  waiting:
-    while (completest.active) {
-        opal_event_loop(orte_event_base, OPAL_EVLOOP_ONCE);
-    }
-
-  DONE:
-    /* cleanup and leave */
-    orte_submit_finalize();
-
-    if (orte_debug_flag) {
-        fprintf(stderr, "exiting with status %d\n", orte_exit_status);
-    }
-    exit(orte_exit_status);
-}
-
-static void launched(int index, orte_job_t *jdata, int ret, void *cbdata)
-{
-    orte_submit_status_t *launchst = (orte_submit_status_t*)cbdata;
-    launchst->status = ret;
-    ORTE_UPDATE_EXIT_STATUS(ret);
-    OBJ_RETAIN(jdata);
-    launchst->jdata = jdata;
-    launchst->active = false;
-}
-static void completed(int index, orte_job_t *jdata, int ret, void *cbdata)
-{
-    orte_submit_status_t *completest = (orte_submit_status_t*)cbdata;
-    completest->status = ret;
-    ORTE_UPDATE_EXIT_STATUS(ret);
-    OBJ_RETAIN(jdata);
-    completest->jdata = jdata;
-    completest->active = false;
-}

orte/util/Makefile.am

@@ -11,7 +11,7 @@
 #                         All rights reserved.
 # Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
 # Copyright (c) 2014      Cisco Systems, Inc.  All rights reserved.
-# Copyright (c) 2014-2015 Intel, Inc. All rights reserved.
+# Copyright (c) 2014-2016 Intel, Inc. All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -57,7 +57,8 @@ headers += \
         util/nidmap.h \
         util/regex.h \
         util/attr.h \
-        util/listener.h
+        util/listener.h \
+        util/cmd_line.h
 
 lib@ORTE_LIB_PREFIX@open_rte_la_SOURCES += \
         util/error_strings.c \
@@ -76,7 +77,8 @@ lib@ORTE_LIB_PREFIX@open_rte_la_SOURCES += \
         util/nidmap.c \
         util/regex.c \
         util/attr.c \
-        util/listener.c
+        util/listener.c \
+        util/cmd_line.c
 
 # Remove the generated man pages
 distclean-local:

orte/util/cmd_line.c

@@ -0,0 +1,110 @@
+/*
+ * Copyright (c) 2016      Intel, Inc. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "orte_config.h"
+#include "orte/types.h"
+#include "orte/constants.h"
+
+#include <stdio.h>
+#include <string.h>
+
+#include "opal/mca/base/base.h"
+#include "opal/util/cmd_line.h"
+#include "opal/util/printf.h"
+#include "opal/runtime/opal.h"
+
+#include "orte/mca/errmgr/errmgr.h"
+#include "orte/mca/schizo/base/base.h"
+
+#include "orte/util/cmd_line.h"
+
+int orte_cmd_line_create(opal_cmd_line_t *cmd_line,
+                         int argc, char **argv,
+                         char ***context_env, char ***global_env,
+                         bool *version, bool *help)
+{
+    int i, rc;
+
+    if (NULL != version) {
+        *version = false;
+    }
+    if (NULL != help) {
+        *help = false;
+    }
+
+    if (NULL != version) {
+        /* see if they asked for us to print version */
+        for (i=0; NULL != argv[i]; i++) {
+            if (0 == strcmp(argv[i], "--version") ||
+                0 == strcmp(argv[i], "-V")) {
+                *version = true;
+                return ORTE_SUCCESS;
+            }
+        }
+    }
+
+    /* process any mca params */
+    if (OPAL_SUCCESS != (rc = mca_base_cmd_line_process_args(argv, context_env, global_env))) {
+        ORTE_ERROR_LOG(rc);
+        return rc;
+    }
+
+
+    opal_cmd_line_create(cmd_line, NULL);
+
+    /* init the MCA system - will just refcount if already initialized */
+    opal_init_util(NULL, NULL);
+
+    /* open the SCHIZO framework so we can define the cmd line options */
+    if (ORTE_SUCCESS != (rc = mca_base_framework_open(&orte_schizo_base_framework, 0))) {
+        ORTE_ERROR_LOG(rc);
+        return rc;
+    }
+    if (ORTE_SUCCESS != (rc = orte_schizo_base_select())) {
+        ORTE_ERROR_LOG(rc);
+        return rc;
+    }
+    /* define the cli options */
+    if (ORTE_SUCCESS != (rc = orte_schizo.define_cli(cmd_line))) {
+        ORTE_ERROR_LOG(rc);
+        return rc;
+    }
+    /* close the framework for bookkeeping purposes */
+    mca_base_framework_close(&orte_schizo_base_framework);
+
+    /* decrement the opal refcount */
+    opal_finalize_util();
+
+    /* now that options have been defined, finish setup */
+    mca_base_cmd_line_setup(cmd_line);
+
+
+    /* Check for help request - must do this after we setup
+     * the cmd line so the help messages can display */
+    if (NULL != help) {
+        for (i=0; NULL != argv[i]; i++) {
+            if (0 == strcmp(argv[i], "--help") ||
+                0 == strcmp(argv[i], "-h")) {
+                *help = true;
+                return ORTE_SUCCESS;
+            }
+        }
+    }
+
+    /* parse the result to get values */
+    if (OPAL_SUCCESS != (rc = opal_cmd_line_parse(cmd_line, true,
+                                                  argc, argv)) ) {
+        if (OPAL_ERR_SILENT != rc) {
+            fprintf(stderr, "%s: command line error (%s)\n", argv[0],
+                    opal_strerror(rc));
+        }
+        return rc;
+    }
+
+    return ORTE_SUCCESS;
+}

orte/util/cmd_line.h

@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2016      Intel, Inc. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+/** @file:
+ *
+ * Populates global structure with system-specific information.
+ *
+ * Notes: add limits.h, compute size of integer and other types via sizeof(type)*CHAR_BIT
+ *
+ */
+
+#ifndef _ORTE_CMD_LINE_H_
+#define _ORTE_CMD_LINE_H_
+
+#include "orte_config.h"
+
+#ifdef HAVE_STDINT_h
+#include <stdint.h>
+#endif
+
+#include "orte/types.h"
+
+#include "opal/util/cmd_line.h"
+
+BEGIN_C_DECLS
+
+ORTE_DECLSPEC int orte_cmd_line_create(opal_cmd_line_t *cmd_line,
+                                       int argc, char **argv,
+                                       char ***context_env, char ***global_env,
+                                       bool *version, bool *help);
+
+END_C_DECLS
+#endif