Add PMIx key to provide RM with ability to indicate that it will cleanup

session directories provided at through OPAL_PMIX_TMPDIR,
OPAL_PMIX_NSDIR, OPAL_PMIX_PROCDIR

opal/mca/pmix/pmix_types.h

@@ -75,6 +75,7 @@ BEGIN_C_DECLS
 #define OPAL_PMIX_TMPDIR                        "pmix.tmpdir"           // (char*) top-level tmp dir assigned to session
 #define OPAL_PMIX_NSDIR                         "pmix.nsdir"            // (char*) sub-tmpdir assigned to namespace
 #define OPAL_PMIX_PROCDIR                       "pmix.pdir"             // (char*) sub-nsdir assigned to proc
+#define OPAL_PMIX_TDIR_RMCLEAN                  "pmix.tdir.rmclean"     // (bool)  Resource Manager will clean session directories
 
 /* information about relative ranks as assigned by the RM */
 #define OPAL_PMIX_JOBID                         "pmix.jobid"            // (uint32_t) jobid assigned by scheduler

orte/mca/ess/pmi/ess_pmi_module.c

@@ -94,6 +94,7 @@ static int rte_init(void)
     uint16_t u16, *u16ptr;
     char **peers=NULL, *mycpuset, **cpusets=NULL;
     opal_process_name_t wildcard_rank, pname;
+    bool bool_val, tdir_mca_override = false;
     size_t i;
 
     /* run the prolog */
@@ -246,37 +247,51 @@ static int rte_init(void)
     OPAL_MODEX_RECV_VALUE_OPTIONAL(ret, OPAL_PMIX_TMPDIR, &wildcard_rank, &val, OPAL_STRING);
     if (OPAL_SUCCESS == ret && NULL != val) {
         /* TODO: who has precedence - pmix of MCA setting??? */
-        if( NULL == orte_process_info.top_session_dir ){
+        if( NULL != orte_process_info.top_session_dir ){
             orte_process_info.top_session_dir = val;
         } else {
             /* keep the MCA setting */
+            tdir_mca_override = true;
             free(val);
         }
         val = NULL;
     }
 
-    OPAL_MODEX_RECV_VALUE_OPTIONAL(ret, OPAL_PMIX_NSDIR, &wildcard_rank, &val, OPAL_STRING);
-    if (OPAL_SUCCESS == ret && NULL != val) {
-        /* TODO: who has precedence - pmix of MCA setting??? */
-        if( NULL == orte_process_info.job_session_dir ){
-            orte_process_info.job_session_dir = val;
-        } else {
-            /* keep the MCA setting */
-            free(val);
+    if( !tdir_mca_override ){
+        OPAL_MODEX_RECV_VALUE_OPTIONAL(ret, OPAL_PMIX_NSDIR, &wildcard_rank, &val, OPAL_STRING);
+        if (OPAL_SUCCESS == ret && NULL != val) {
+            /* TODO: who has precedence - pmix of MCA setting??? */
+            if( NULL == orte_process_info.job_session_dir ){
+                orte_process_info.job_session_dir = val;
+            } else {
+                /* keep the MCA setting */
+                free(val);
+                tdir_mca_override = true;
+            }
+            val = NULL;
         }
-        val = NULL;
     }
 
-    OPAL_MODEX_RECV_VALUE_OPTIONAL(ret, OPAL_PMIX_PROCDIR, &wildcard_rank, &val, OPAL_STRING);
-    if (OPAL_SUCCESS == ret && NULL != val) {
-        /* TODO: who has precedence - pmix of MCA setting??? */
-        if( NULL == orte_process_info.proc_session_dir ){
-            orte_process_info.proc_session_dir = val;
-        } else {
-            /* keep the MCA setting */
-            free(val);
+    if( !tdir_mca_override ){
+        OPAL_MODEX_RECV_VALUE_OPTIONAL(ret, OPAL_PMIX_PROCDIR, &wildcard_rank, &val, OPAL_STRING);
+        if (OPAL_SUCCESS == ret && NULL != val) {
+            /* TODO: who has precedence - pmix of MCA setting??? */
+            if( NULL == orte_process_info.proc_session_dir ){
+                orte_process_info.proc_session_dir = val;
+            } else {
+                /* keep the MCA setting */
+                tdir_mca_override = true;
+                free(val);
+            }
+            val = NULL;
+        }
+    }
+
+    if( !tdir_mca_override ){
+        OPAL_MODEX_RECV_VALUE_OPTIONAL(ret, OPAL_PMIX_TDIR_RMCLEAN, &wildcard_rank, &bool_val, OPAL_BOOL);
+        if (OPAL_SUCCESS == ret ) {
+            orte_process_info.rm_session_dirs = val;
         }
-        val = NULL;
     }
 
     /* retrieve our topology */

orte/util/proc_info.h

@@ -122,6 +122,7 @@ struct orte_proc_info_t {
     char *jobfam_session_dir;           /**< Session directory for this family of jobs (i.e., share same mpirun) */
     char *job_session_dir;              /**< Session directory for job */
     char *proc_session_dir;             /**< Session directory for the process */
+    bool rm_session_dirs;               /**< Session directories will be cleaned up by RM */
 
     char *sock_stdin;                   /**< Path name to temp file for stdin. */
     char *sock_stdout;                  /**< Path name to temp file for stdout. */

orte/util/session_dir.c

@@ -366,8 +366,8 @@ orte_session_dir_cleanup(orte_jobid_t jobid)
 {
     int rc = ORTE_SUCCESS;
 
-    if (!orte_create_session_dirs ) {
-        /* we haven't created them */
+    if (!orte_create_session_dirs || orte_process_info.rm_session_dirs ) {
+        /* we haven't created them or RM will clean them up for us*/
         return ORTE_SUCCESS;
     }
 
@@ -447,8 +447,8 @@ orte_session_dir_finalize(orte_process_name_t *proc)
     char *tmp;
     char *job_session_dir, *vpid, *proc_session_dir;
 
-    if (!orte_create_session_dirs ) {
-        /* we haven't created them */
+    if (!orte_create_session_dirs || orte_process_info.rm_session_dirs ) {
+        /* we haven't created them or RM will clean them up for us*/
         return ORTE_SUCCESS;
     }