Establish a way for ORTE to tell PMIx the base tmpdir to use, and update PMIx to understand such directives

opal/mca/pmix/pmix2x/pmix/include/pmix/pmix_common.h

@@ -106,6 +106,10 @@ BEGIN_C_DECLS
 #define PMIX_SERVER_TOOL_SUPPORT            "pmix.srvr.tool"        // (bool) The host RM wants to declare itself as willing to
                                                                     //        accept tool connection requests
 #define PMIX_SERVER_PIDINFO                 "pmix.srvr.pidinfo"     // (uint32_t) pid of the target server
+#define PMIX_SERVER_TMPDIR                  "pmix.srvr.tmpdir"      // (char*) temp directory where PMIx server will place
+                                                                    //        client rendezvous points
+#define PMIX_SYSTEM_TMPDIR                  "pmix.sys.tmpdir"       // (char*) temp directory for this system, where PMIx
+                                                                    //        server will place tool rendezvous points
 
 /* identification attributes */
 #define PMIX_USERID                         "pmix.euid"             // (uint32_t) effective user id

opal/mca/pmix/pmix2x/pmix/src/server/pmix_server.c

@@ -67,6 +67,8 @@ pmix_server_globals_t pmix_server_globals = {{{0}}};
 // local variables
 static char *security_mode = NULL;
 static pid_t mypid;
+static char *mytmpdir = NULL;
+static char *systmpdir = NULL;
 
 // local functions for connection support
 static void server_message_handler(struct pmix_peer_t *pr, pmix_usock_hdr_t *hdr,
@@ -181,7 +183,9 @@ static pmix_status_t initialize_server_base(pmix_server_module_t *module)
     security_mode = strdup(pmix_sec.name);
 
     /* find the temp dir */
-    if (NULL == (tdir = getenv("PMIX_SERVER_TMPDIR"))) {
+    if (NULL != mytmpdir) {
+        tdir = mytmpdir;
+    } else if (NULL == (tdir = getenv("PMIX_SERVER_TMPDIR"))) {
         if (NULL == (tdir = getenv("TMPDIR"))) {
             if (NULL == (tdir = getenv("TEMP"))) {
                 if (NULL == (tdir = getenv("TMP"))) {
@@ -233,6 +237,7 @@ PMIX_EXPORT pmix_status_t PMIx_server_init(pmix_server_module_t *module,
     char *pmix_pid, *tdir;
     char **protected = NULL;
     bool protect;
+    bool tool_support = false;
 
     ++pmix_globals.init_cntr;
     if (1 < pmix_globals.init_cntr) {
@@ -289,41 +294,52 @@ PMIX_EXPORT pmix_status_t PMIx_server_init(pmix_server_module_t *module,
                     lt->mode = info[n].value.data.uint32;
                 }
             } else if (0 == strcmp(info[n].key, PMIX_SERVER_TOOL_SUPPORT)) {
-                pmix_listener_t *tl = PMIX_NEW(pmix_listener_t);
+                /* defer processing to ensure we pickup any tmpdir
-                tl -> address.sun_family = AF_UNIX;
+                 * directives before setting location */
-                tl->protocol = PMIX_PROTOCOL_TOOL;
+                tool_support = true;
-                /* Get up to 30 chars of hostname.*/
+            } else if (0 == strcmp(info[n].key, PMIX_SERVER_TMPDIR)) {
-                gethostname(myhostname, myhostnamelen);
+                mytmpdir = strdup(info[n].value.data.string);
-                /* ensure it is NULL terminated */
+            } else if (0 == strcmp(info[n].key, PMIX_SYSTEM_TMPDIR)) {
-                myhostname[myhostnamelen-1] = '\0';
+                systmpdir = strdup(info[n].value.data.string);
-                /* need to put this in the global tmpdir as opposed to
-                 * where the server tmpdir might be */
-                if (NULL == (tdir = getenv("TMPDIR"))) {
-                    if (NULL == (tdir = getenv("TEMP"))) {
-                        if (NULL == (tdir = getenv("TMP"))) {
-                            tdir = "/tmp";
-                        }
-                    }
-                }
-                if (0 > asprintf(&pmix_pid, "%s/pmix.%s.tool.%d", tdir, myhostname, mypid)) {
-                    return PMIX_ERR_NOMEM;
-                }
-                if ((strlen(pmix_pid) + 1) > sizeof(tl->address.sun_path)-1) {
-                    free(pmix_pid);
-                    return PMIX_ERR_INVALID_LENGTH;
-                }
-                snprintf(tl->address.sun_path, sizeof(tl->address.sun_path) - 1, "%s", pmix_pid);
-                free(pmix_pid);
-                /* we don't provide a URI for this listener as we don't pass
-                 * the TOOL connection URI to a child process */
-                pmix_server_globals.tool_connections_allowed = true;
-                pmix_list_append(&pmix_server_globals.listeners, &tl->super);
-                /* push this onto our protected list of keys not
-                 * to be passed to the clients */
-                pmix_argv_append_nosize(&protected, PMIX_SERVER_TOOL_SUPPORT);
             }
         }
     }
+    if (tool_support) {
+        pmix_listener_t *tl = PMIX_NEW(pmix_listener_t);
+        tl -> address.sun_family = AF_UNIX;
+        tl->protocol = PMIX_PROTOCOL_TOOL;
+        /* Get up to 30 chars of hostname.*/
+        gethostname(myhostname, myhostnamelen);
+        /* ensure it is NULL terminated */
+        myhostname[myhostnamelen-1] = '\0';
+        /* need to put this in the global tmpdir as opposed to
+         * where the server tmpdir might be */
+        if (NULL != systmpdir) {
+            tdir = systmpdir;
+        } else if (NULL == (tdir = getenv("TMPDIR"))) {
+            if (NULL == (tdir = getenv("TEMP"))) {
+                if (NULL == (tdir = getenv("TMP"))) {
+                    tdir = "/tmp";
+                }
+            }
+        }
+        if (0 > asprintf(&pmix_pid, "%s/pmix.%s.tool.%d", tdir, myhostname, mypid)) {
+            return PMIX_ERR_NOMEM;
+        }
+        if ((strlen(pmix_pid) + 1) > sizeof(tl->address.sun_path)-1) {
+            free(pmix_pid);
+            return PMIX_ERR_INVALID_LENGTH;
+        }
+        snprintf(tl->address.sun_path, sizeof(tl->address.sun_path) - 1, "%s", pmix_pid);
+        free(pmix_pid);
+        /* we don't provide a URI for this listener as we don't pass
+         * the TOOL connection URI to a child process */
+        pmix_server_globals.tool_connections_allowed = true;
+        pmix_list_append(&pmix_server_globals.listeners, &tl->super);
+        /* push this onto our protected list of keys not
+         * to be passed to the clients */
+        pmix_argv_append_nosize(&protected, PMIX_SERVER_TOOL_SUPPORT);
+    }
 
     /* setup the wildcard recv for inbound messages from clients */
     req = PMIX_NEW(pmix_usock_posted_recv_t);
@@ -408,6 +424,14 @@ static void cleanup_server_state(void)
         free(security_mode);
     }
 
+    if (NULL != mytmpdir) {
+        free(mytmpdir);
+    }
+
+    if (NULL != systmpdir) {
+        free(systmpdir);
+    }
+
     pmix_bfrop_close();
     pmix_sec_finalize();
     pmix_globals_finalize();

opal/mca/pmix/pmix_types.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014-2015 Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2016 Intel, Inc. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -44,6 +44,10 @@ BEGIN_C_DECLS
 #define OPAL_PMIX_SERVER_TOOL_SUPPORT           "pmix.srvr.tool"        // (bool) The host RM wants to declare itself as willing to
                                                                         //        accept tool connection requests
 #define OPAL_PMIX_SERVER_PIDINFO                "pmix.srvr.pidinfo"     // (uint32_t) pid of the target server
+#define OPAL_PMIX_SERVER_TMPDIR                 "pmix.srvr.tmpdir"      // (char*) temp directory where PMIx server will place
+                                                                        //        client rendezvous points
+#define OPAL_PMIX_SYSTEM_TMPDIR                 "pmix.sys.tmpdir"       // (char*) temp directory where PMIx server will place
+                                                                        //        tool rendezvous points
 
 
 /* identification attributes */

orte/orted/pmix/pmix_server.c

@@ -64,6 +64,7 @@
 #include "orte/mca/rml/rml.h"
 #include "orte/mca/rml/base/rml_contact.h"
 #include "orte/util/name_fns.h"
+#include "orte/util/proc_info.h"
 #include "orte/util/session_dir.h"
 #include "orte/util/show_help.h"
 #include "orte/runtime/orte_globals.h"
@@ -249,11 +250,24 @@ int pmix_server_init(void)
         kv->type = OPAL_STRING;
         opal_list_append(&info, &kv->super);
     }
+    /* tell the server to allow tool connections */
     kv = OBJ_NEW(opal_value_t);
     kv->key = strdup(OPAL_PMIX_SERVER_TOOL_SUPPORT);
     kv->type = OPAL_BOOL;
     kv->data.flag = true;
     opal_list_append(&info, &kv->super);
+    /* tell the server our temp directory */
+    kv = OBJ_NEW(opal_value_t);
+    kv->key = strdup(OPAL_PMIX_SERVER_TMPDIR);
+    kv->type = OPAL_STRING;
+    kv->data.string = strdup(orte_process_info.tmpdir_base);
+    opal_list_append(&info, &kv->super);
+    /* use the same for the system temp directory */
+    kv = OBJ_NEW(opal_value_t);
+    kv->key = strdup(OPAL_PMIX_SYSTEM_TMPDIR);
+    kv->type = OPAL_STRING;
+    kv->data.string = strdup(orte_process_info.tmpdir_base);
+    opal_list_append(&info, &kv->super);
 
     /* setup the local server */
     if (ORTE_SUCCESS != (rc = opal_pmix.server_init(&pmix_server, &info))) {