Cleanup some cruft and update to coordinate with CM operations:

* don't pass --tree-spawn to the orted cmd line. If someone doesn't want tree-spawn, it shows up as an MCA param anyway
* ensure state/orted component disqualifies itself from CM operations
* clarify the DVM proc_type definitions
* ensure we stop littering the tmp dir with session directories

orte/mca/ess/base/ess_base_std_orted.c

@@ -608,6 +608,10 @@ int orte_ess_base_orted_setup(char **hosts)
     orte_show_help("help-orte-runtime.txt",
                    "orte_init:startup:internal-failure",
                    true, error, ORTE_ERROR_NAME(ret), ret);
+    /* remove our use of the session directory tree */
+    orte_session_dir_finalize(ORTE_PROC_MY_NAME);
+    /* ensure we scrub the session directory tree */
+    orte_session_dir_cleanup(ORTE_JOBID_WILDCARD);
     return ORTE_ERR_SILENT;
 }
 
@@ -645,7 +649,9 @@ int orte_ess_base_orted_finalize(void)
     (void) mca_base_framework_close(&orte_oob_base_framework);
     (void) mca_base_framework_close(&orte_state_base_framework);
     (void) mca_base_framework_close(&opal_dstore_base_framework);
-    /* cleanup any lingering session directories */
+    /* remove our use of the session directory tree */
+    orte_session_dir_finalize(ORTE_PROC_MY_NAME);
+    /* ensure we scrub the session directory tree */
     orte_session_dir_cleanup(ORTE_JOBID_WILDCARD);
     return ORTE_SUCCESS;
 }

orte/mca/ess/hnp/ess_hnp_module.c

@@ -767,6 +767,18 @@ static int rte_init(void)
                        "orte_init:startup:internal-failure",
                        true, error, ORTE_ERROR_NAME(ret), ret);
     }
+    /* remove my contact info file, if we have session directories */
+    if (NULL != orte_process_info.job_session_dir) {
+        jobfam_dir = opal_dirname(orte_process_info.job_session_dir);
+        contact_path = opal_os_path(false, jobfam_dir, "contact.txt", NULL);
+        free(jobfam_dir);
+        unlink(contact_path);
+        free(contact_path);
+    }
+    /* remove our use of the session directory tree */
+    orte_session_dir_finalize(ORTE_PROC_MY_NAME);
+    /* ensure we scrub the session directory tree */
+    orte_session_dir_cleanup(ORTE_JOBID_WILDCARD);
     return ORTE_ERR_SILENT;
 }
 
@@ -827,6 +839,8 @@ static int rte_finalize(void)
     (void) mca_base_framework_close(&orte_rml_base_framework);
     (void) mca_base_framework_close(&orte_oob_base_framework);
 
+    /* remove our use of the session directory tree */
+    orte_session_dir_finalize(ORTE_PROC_MY_NAME);
     /* ensure we scrub the session directory tree */
     orte_session_dir_cleanup(ORTE_JOBID_WILDCARD);
 

orte/mca/plm/base/plm_base_launch_support.c

@@ -1187,9 +1187,11 @@ int orte_plm_base_orted_append_basic_args(int *argc, char ***argv,
         opal_argv_append(argc, argv, "orte_report_bindings");
         opal_argv_append(argc, argv, "1");
     }
-    /* pass our topology signature */
+    if (!ORTE_PROC_IS_CM) {
-    opal_argv_append(argc, argv, "--hnp-topo-sig");
+       /* pass our topology signature */
-    opal_argv_append(argc, argv, orte_topo_signature);
+        opal_argv_append(argc, argv, "--hnp-topo-sig");
+        opal_argv_append(argc, argv, orte_topo_signature);
+    }
     if (orte_hetero_nodes) {
         opal_argv_append(argc, argv, "-"OPAL_MCA_CMD_LINE_ID);
         opal_argv_append(argc, argv, "orte_hetero_nodes");
@@ -1229,7 +1231,7 @@ int orte_plm_base_orted_append_basic_args(int *argc, char ***argv,
 
     /* pass the daemon jobid */
     opal_argv_append(argc, argv, "-"OPAL_MCA_CMD_LINE_ID);
-    opal_argv_append(argc, argv, "orte_ess_jobid");
+    opal_argv_append(argc, argv, "ess_base_jobid");
     if (ORTE_SUCCESS != (rc = orte_util_convert_jobid_to_string(&param, ORTE_PROC_MY_NAME->jobid))) {
         ORTE_ERROR_LOG(rc);
         return rc;
@@ -1240,7 +1242,7 @@ int orte_plm_base_orted_append_basic_args(int *argc, char ***argv,
     /* setup to pass the vpid */
     if (NULL != proc_vpid_index) {
         opal_argv_append(argc, argv, "-"OPAL_MCA_CMD_LINE_ID);
-        opal_argv_append(argc, argv, "orte_ess_vpid");
+        opal_argv_append(argc, argv, "ess_base_vpid");
         *proc_vpid_index = *argc;
         opal_argv_append(argc, argv, "<template>");
     }
@@ -1253,7 +1255,7 @@ int orte_plm_base_orted_append_basic_args(int *argc, char ***argv,
         num_procs = orte_process_info.num_procs;
     }
     opal_argv_append(argc, argv, "-"OPAL_MCA_CMD_LINE_ID);
-    opal_argv_append(argc, argv, "orte_ess_num_procs");
+    opal_argv_append(argc, argv, "ess_base_num_procs");
     asprintf(&param, "%lu", num_procs);
     opal_argv_append(argc, argv, param);
     free(param);
@@ -1288,13 +1290,6 @@ int orte_plm_base_orted_append_basic_args(int *argc, char ***argv,
         free(param);
     }
 
-    /* warn the daemons if we are using a tree spawn pattern so they
-     * know they shouldn't do a rollup on their callback
-     */
-    if (NULL != orte_tree_launch_cmd) {
-        opal_argv_append(argc, argv, "--tree-spawn");
-    }
-
     /* if output-filename was specified, pass that along */
     if (NULL != orte_output_filename) {
         opal_argv_append(argc, argv, "-"OPAL_MCA_CMD_LINE_ID);

orte/mca/plm/rsh/plm_rsh_module.c

@@ -606,10 +606,17 @@ static int setup_launch(int *argcptr, char ***argvptr,
      * Add the basic arguments to the orted command line, including
      * all debug options
      */
-    orte_plm_base_orted_append_basic_args(&argc, &argv,
+    if (ORTE_PROC_IS_CM) {
-                                          "env",
+        orte_plm_base_orted_append_basic_args(&argc, &argv,
-                                          proc_vpid_index,
+                                              NULL,
-                                          NULL);
+                                              proc_vpid_index,
+                                              NULL);
+    } else {
+        orte_plm_base_orted_append_basic_args(&argc, &argv,
+                                              "env",
+                                              proc_vpid_index,
+                                              NULL);
+    }
 
     /* ensure that only the ssh plm is selected on the remote daemon */
     opal_argv_append_nosize(&argv, "-"OPAL_MCA_CMD_LINE_ID);
@@ -660,9 +667,6 @@ static int setup_launch(int *argcptr, char ***argvptr,
     /* protect the params */
     mca_base_cmd_line_wrap_args(argv);
 
-    /* tell the daemon we are in a tree spawn */
-    opal_argv_append(&argc, &argv, "--tree-spawn");
-
     value = opal_argv_join(argv, ' ');
     if (sysconf(_SC_ARG_MAX) < (int)strlen(value)) {
         orte_show_help("help-plm-rsh.txt", "cmd-line-too-long",

orte/mca/state/dvm/state_dvm_component.c

@@ -70,9 +70,14 @@ static int state_dvm_close(void)
 
 static int state_dvm_component_query(mca_base_module_t **module, int *priority)
 {
-    /* we are only used when an envar is set directing it,
+    /* used by DVM masters */
-     * so set our priority very low */
+    if (ORTE_PROC_IS_MASTER) {
+        *priority = 100;
+        *module = (mca_base_module_t *)&orte_state_dvm_module;
+        return ORTE_SUCCESS;
+    }
+
     *priority = 0;
-    *module = (mca_base_module_t *)&orte_state_dvm_module;
+    *module = NULL;
-    return ORTE_SUCCESS;
+    return ORTE_ERR_NOT_AVAILABLE;
 }

orte/mca/state/orted/state_orted_component.c

@@ -71,7 +71,7 @@ static int state_orted_close(void)
 
 static int state_orted_component_query(mca_base_module_t **module, int *priority)
 {
-    if (ORTE_PROC_IS_DAEMON) {
+    if (ORTE_PROC_IS_DAEMON && !ORTE_PROC_IS_CM) {
         /* set our priority high as we are the default for orteds */
         *priority = my_priority;
         *module = (mca_base_module_t *)&orte_state_orted_module;

orte/orted/orted_main.c

@@ -178,10 +178,6 @@ opal_cmd_line_init_t orte_cmd_line_opts[] = {
       NULL, OPAL_CMD_LINE_TYPE_STRING,
       "URI for the parent if tree launch is enabled."},
 
-    { NULL, '\0', NULL, "tree-spawn", 0,
-      &orted_globals.tree_spawn, OPAL_CMD_LINE_TYPE_BOOL,
-      "Tree spawn is underway"},
-
     { NULL, '\0', NULL, "set-sid", 0,
       &orted_globals.set_sid, OPAL_CMD_LINE_TYPE_BOOL,
       "Direct the orted to separate from the current session"},

orte/tools/orte-dvm/orte-dvm.c

@@ -242,17 +242,11 @@ int main(int argc, char *argv[])
         exit(0);
     }
 
-    /* flag that I am the HNP */
-    orte_process_info.proc_type = ORTE_PROC_HNP;
-
     /* Setup MCA params */
     orte_register_params();
 
-    /* specify the DVM state machine */
-    opal_setenv("OMPI_MCA_state", "dvm", true, &environ);
-
     /* Intialize our Open RTE environment */
-    if (ORTE_SUCCESS != (rc = orte_init(&argc, &argv, ORTE_PROC_HNP))) {
+    if (ORTE_SUCCESS != (rc = orte_init(&argc, &argv, ORTE_PROC_MASTER))) {
         /* cannot call ORTE_ERROR_LOG as it could be the errmgr
          * never got loaded!
          */

orte/util/proc_info.h

@@ -58,9 +58,10 @@ typedef uint32_t orte_proc_type_t;
 #define ORTE_PROC_APP           0x0030
 #define ORTE_PROC_CM            0x0040
 #define ORTE_PROC_AGGREGATOR    0x0080
+#define ORTE_PROC_DVM           0x0102  // DVM is DVM+Daemon
 #define ORTE_PROC_IOF_ENDPT     0x1000
 #define ORTE_PROC_SCHEDULER     0x2000
-#define ORTE_PROC_MASTER        0x4000
+#define ORTE_PROC_MASTER        0x4104  // DVM Master is HNP+DVM+Master
 
 #define ORTE_PROC_IS_SINGLETON      (ORTE_PROC_SINGLETON & orte_process_info.proc_type)
 #define ORTE_PROC_IS_DAEMON         (ORTE_PROC_DAEMON & orte_process_info.proc_type)
@@ -71,9 +72,10 @@ typedef uint32_t orte_proc_type_t;
 #define ORTE_PROC_IS_APP            (ORTE_PROC_APP & orte_process_info.proc_type)
 #define ORTE_PROC_IS_CM             (ORTE_PROC_CM & orte_process_info.proc_type)
 #define ORTE_PROC_IS_AGGREGATOR     (ORTE_PROC_AGGREGATOR & orte_process_info.proc_type)
+#define ORTE_PROC_IS_DVM            (ORTE_PROC_IS_DAEMON && (ORTE_PROC_DVM & orte_process_info.proc_type))
 #define ORTE_PROC_IS_IOF_ENDPT      (ORTE_PROC_IOF_ENDPT & orte_process_info.proc_type)
 #define ORTE_PROC_IS_SCHEDULER      (ORTE_PROC_SCHEDULER & orte_process_info.proc_type)
-#define ORTE_PROC_IS_MASTER         (ORTE_PROC_MASTER & orte_process_info.proc_type)
+#define ORTE_PROC_IS_MASTER         (ORTE_PROC_IS_DVM && ORTE_PROC_IS_HNP && (ORTE_PROC_MASTER & orte_process_info.proc_type))
 
 
 /**