Merge pull request #921 from rhc54/topic/pmix

Sync to PMIx master

opal/mca/pmix/pmix1xx/pmix/Makefile.am

@@ -37,7 +37,8 @@ endif
 
 man_MANS = \
         man/man3/pmix_init.3 \
-        man/man7/pmix.7
+        man/man7/pmix.7 \
+        man/man7/pmix_constants.7
 
 include config/Makefile.am
 include include/Makefile.am

opal/mca/pmix/pmix1xx/pmix/VERSION

@@ -30,7 +30,7 @@ greek=a1
 # command, or with the date (if "git describe" fails) in the form of
 # "date<date>".
 
-repo_rev=git89680d6
+repo_rev=git4dec948
 
 # If tarball_version is not empty, it is used as the version string in
 # the tarball filename, regardless of all other versions listed in
@@ -44,7 +44,7 @@ tarball_version=
 
 # The date when this release was created
 
-date="Sep 10, 2015"
+date="Sep 21, 2015"
 
 # The shared library version of each of PMIx's public libraries.
 # These versions are maintained in accordance with the "Library

opal/mca/pmix/pmix1xx/pmix/include/pmix/pmix_common.h.in

@@ -80,7 +80,6 @@ BEGIN_C_DECLS
 
 /* define maximum value and key sizes */
 #define PMIX_MAX_NSLEN     255
-#define PMIX_MAX_VALLEN   1023
 #define PMIX_MAX_KEYLEN    511
 
 /* define a *wildcard* value for requests involving rank */
@@ -175,6 +174,7 @@ BEGIN_C_DECLS
 #define PMIX_PROC_DATA             "pmix.pdata"        // (pmix_value_array_t) starts with rank, then contains more data
 #define PMIX_NODE_MAP              "pmix.nmap"         // (char*) regex of nodes containing procs for this job
 #define PMIX_PROC_MAP              "pmix.pmap"         // (char*) regex describing procs on each node within this job
+#define PMIX_ANL_MAP               "pmix.anlmap"       // (char*) process mapping in ANL notation (used in PMI-1/PMI-2)
 
 /* attributes used internally to communicate data from the server to the client */
 #define PMIX_PROC_BLOB             "pmix.pblob"        // (pmix_byte_object_t) packed blob of process data
@@ -307,6 +307,7 @@ typedef enum {
 /* define a "persistence" policy for data published by clients */
 typedef enum {
     PMIX_PERSIST_INDEF = 0,   // retain until specifically deleted
+    PMIX_PERSIST_FIRST_READ,  // delete upon first access
     PMIX_PERSIST_PROC,        // retain until publishing process terminates
     PMIX_PERSIST_APP,         // retain until application terminates
     PMIX_PERSIST_SESSION      // retain until session/allocation terminates
@@ -468,13 +469,11 @@ typedef struct {
         }                                       \
     } while(0);
 
-#define PMIX_INFO_LOAD(m, d, k, v, t)                   \
+#define PMIX_INFO_LOAD(m, k, v, t)                      \
-    do {                                                \
     if (NULL != (m)) {                                  \
-        (m)->mandatory = (d);                           \
         (void)strncpy((m)->key, (k), PMIX_MAX_KEYLEN);  \
         pmix_value_load(&((m)->value), (v), (t));       \
-    } while(0);
+    }
 
 
 /****    PMIX LOOKUP RETURN STRUCT    ****/
@@ -827,7 +826,7 @@ const char* PMIx_Get_version(void);
     PMIX_VAL_SET_ ## _field(_v, _field, _val)
 
 #define PMIX_VAL_cmp_val(_val1, _val2)      ((_val1) != (_val2))
-#define PMIX_VAL_cmp_float(_val1, _val2)    (abs((_val1) - (_val2)) > 0.000001)
+#define PMIX_VAL_cmp_float(_val1, _val2)    (((_val1)>(_val2))?(((_val1)-(_val2))>0.000001):(((_val2)-(_val1))>0.000001))
 #define PMIX_VAL_cmp_ptr(_val1, _val2)      strncmp(_val1, _val2, strlen(_val1)+1)
 
 #define PMIX_VAL_CMP_int        PMIX_VAL_cmp_val

opal/mca/pmix/pmix1xx/pmix/man/man7/pmix_constants.7

@@ -0,0 +1,94 @@
+.TH "pmix_constants" "7" "2015\-09\-16" "PMIx Programmer\[aq]s Manual" "\@VERSION\@"
+.SH NAME
+.PP
+PMIx Constants
+.SH SYNOPSIS
+.IP
+.nf
+\f[C]
+#include\ <pmix_common.h>
+\f[]
+.fi
+.SH OVERVIEW
+.PP
+PMIx relies on the following types of constants:
+.PP
+\f[I]Maximum Sizes\f[] : In order to minimize malloc performance
+penalties, PMIx utilizes constant\-sized arrays wherever possible.
+These constants provide the user with the maximum size of the various
+array types.
+.PP
+\f[I]Attributes\f[] : .
+.PP
+\f[I]Errors\f[] : PMIx uses negative error constants, with 0 indicating
+"success".
+.SH MAXIMUM SIZES
+.PP
+The .
+.PP
+\f[I]PMIX_MAX_NSLEN\f[] : The maximum length of a namespace.
+Note that any declaration of an array to hold a key string must include
+one extra space for the terminating \f[I]NULL\f[].
+.PP
+\f[I]PMIX_MAX_KEYLEN\f[] : Maximum length of the key string used in
+structures such as the \f[I]pmix\f[]info_t_.
+Note that any declaration of an array to hold a key string must include
+one extra space for the terminating \f[I]NULL\f[].
+.SH ATTRIBUTES
+.PP
+Define a set of "standard" PMIx attributes that can be queried using the
+PMIx_Get function.
+Implementations (and users) are free to extend as desired \- thus,
+functions calling PMIx_Get must be capable of handling the "not found"
+condition.
+Note that these are attributes of the system and the job as opposed to
+values the application (or underlying programming library) might choose
+to expose \- i.e., they are values provided by the resource manager as
+opposed to the application.
+Thus, these keys are RESERVED for use by PMIx, and users should avoid
+defining any attribute starting with the keyword \f[I]PMIX\f[].
+.PP
+A list of the current PMIx attributes, and the type of their associated
+data value, is provided here.
+.PP
+\f[I]PMIX_ATTR_UNDEF (NULL)\f[] : Used to initialize an attribute field,
+indicating that the attribute has not yet been assigned.
+.PP
+\f[I]PMIX_USERID (uint32_t)\f[] : .
+.PP
+\f[I]PMIX_GRPID (uint32_t)\f[] : An access domain represents a single
+logical connection into a fabric.
+It may map to a single physical or virtual NIC or a port.
+An access domain defines the boundary across which fabric resources may
+be associated.
+Each access domain belongs to a single fabric domain.
+.PP
+\f[I]PMIX_CPUSET (char*)\f[] : .
+.SH ERROR CONSTANTS
+.PP
+\&.
+.PP
+\f[I]PMIX_SUCCESS\f[] : Indicates that the operation was successful.
+.PP
+\f[I]PMIX_ERROR\f[] : A general error code \- an error occurred, but no
+specific reason can be provided.
+.PP
+\f[I]fi_rma \- Remote Memory Access\f[] : RMA transfers are one\-sided
+operations that read or write data directly to a remote memory region.
+Other than defining the appropriate memory region, RMA operations do not
+require interaction at the target side for the data transfer to
+complete.
+.PP
+\f[I]fi_atomic \- Atomic\f[] : Atomic operations can perform one of
+several operations on a remote memory region.
+Atomic operations include well\-known functionality, such as atomic\-add
+and compare\-and\-swap, plus several other pre\-defined calls.
+Unlike other data transfer interfaces, atomic operations are aware of
+the data formatting at the target memory region.
+.SH SEE ALSO
+.PP
+\f[C]pmix\f[](7), \f[C]fi_getinfo\f[](3), \f[C]fi_endpoint\f[](3),
+\f[C]fi_domain\f[](3), \f[C]fi_av\f[](3), \f[C]fi_eq\f[](3),
+\f[C]fi_cq\f[](3), \f[C]fi_cntr\f[](3), \f[C]fi_mr\f[](3)
+.SH AUTHORS
+PMIx.

opal/mca/pmix/pmix1xx/pmix/src/client/pmi1.c

@@ -30,6 +30,8 @@
 #endif
 #include PMIX_EVENT_HEADER
 
+#define ANL_MAPPING "PMI_process_mapping"
+
 #include "src/buffer_ops/buffer_ops.h"
 #include "src/util/argv.h"
 #include "src/util/error.h"
@@ -39,6 +41,7 @@
 static pmix_status_t convert_int(int *value, pmix_value_t *kv);
 static int convert_err(pmix_status_t rc);
 static pmix_proc_t myproc;
+static bool data_commited = false;
 
 int PMI_Init( int *spawned )
 {
@@ -93,6 +96,10 @@ int PMI_KVS_Put(const char kvsname[], const char key[], const char value[])
     pmix_status_t rc;
     pmix_value_t val;
 
+    pmix_output_verbose(2, pmix_globals.debug_output,
+            "PMI_KVS_Put: KVS=%s, key=%s value=%s",
+            kvsname, key, value);
+
     val.type = PMIX_STRING;
     val.data.string = (char*)value;
     rc = PMIx_Put(PMIX_GLOBAL, key, &val);
@@ -104,15 +111,106 @@ int PMI_KVS_Commit(const char kvsname[])
 {
     pmix_status_t rc;
 
+    pmix_output_verbose(2, pmix_globals.debug_output,
+            "PMI_KVS_Commit: KVS=%s", kvsname);
+
     rc = PMIx_Commit();
+    /* PMIx permits only one data commit! */
+    data_commited = true;
     return convert_err(rc);
 }
 
+int PMI_KVS_Get( const char kvsname[], const char key[], char value[], int length)
+{
+    pmix_value_t *val;
+    uint32_t i;
+    static pmix_proc_t proc;
+    uint32_t procnum;
+    proc = myproc;
+    int rc;
+
+    pmix_output_verbose(2, pmix_globals.debug_output,
+                        "PMI_KVS_Get: KVS=%s, key=%s value=%s",
+                        kvsname, key, value);
+
+    /* PMI-1 expects resource manager to set
+     * process mapping in ANL notation. */
+    if( !strcmp(key, ANL_MAPPING) ) {
+        /* we are looking in the job-data. If there is nothing there
+         * we don't want to look in rank's data, thus set rank to widcard */
+        proc.rank = PMIX_RANK_WILDCARD;
+        if( PMIX_SUCCESS == PMIx_Get(&proc, PMIX_ANL_MAP, NULL, 0, &val) &&
+                (NULL != val) && (PMIX_STRING == val->type) ){
+            strncpy(value,val->data.string,length);
+            PMIX_VALUE_FREE(val,1);
+            return PMI_SUCCESS;
+        } else {
+            /* artpol:
+             * Some RM's (i.e. SLURM) already have ANL precomputed. The export it
+             * through PMIX_ANL_MAP variable.
+             * If we haven't found it we want to have our own packing functionality
+             * since it's common.
+             * Somebody else has to write it since I've already done that for
+             * GPL'ed SLURM :) */
+            return PMI_FAIL;
+        }
+    }
+
+    /* We don't know what process keeps this data. So it looks like we need to
+     * check each process.
+     * TODO: Is there any beter way?
+     * WARNING: this may lead to the VERY long HANG's if we ask for the unknown key
+     * before we've done Commit on all nodes. We need a workaround for that.
+     *
+     * SOLUTION: perhaps rovide "OK if nothing" info flag to tell PMIx that
+     * the key supposed to already be there and if nothing there - gave up with
+     * an error and don't try to use direct modex.
+     */
+
+    if (PMIX_SUCCESS != (rc = PMIx_Get(&myproc,PMIX_JOB_SIZE, NULL, 0,&val))) {
+        pmix_output_verbose(2, pmix_globals.debug_output,
+                            "pmi1: executing put for KVS %s, key %s value %s",
+                            kvsname, key, value);
+        return  convert_err(rc);
+    }
+    procnum = val->data.uint32;
+    PMIX_VALUE_FREE(val,1);
+
+    for( i=0; i < procnum; i++){
+        proc.rank = i;
+        if( PMIX_SUCCESS == PMIx_Get(&proc, key, NULL, 0, &val) &&
+                (NULL != val) && (PMIX_STRING == val->type) ){
+            strncpy(value,val->data.string,length);
+            PMIX_VALUE_FREE(val,1);
+            return PMI_SUCCESS;
+        }
+        PMIX_VALUE_FREE(val,1);
+    }
+    return PMI_FAIL;
+}
+
+
 /* Barrier only applies to our own nspace, and we want all
  * data to be collected upon completion */
 int PMI_Barrier(void)
 {
-    return PMIx_Fence(NULL, 0, NULL, 0);
+    pmix_info_t buf;
+    int rc, ninfo = 0;
+    pmix_info_t *info = NULL;
+
+    if( data_commited ){
+        bool  val = 1;
+        info = &buf;
+        PMIX_INFO_CONSTRUCT(info);
+        PMIX_INFO_LOAD(info, PMIX_COLLECT_DATA, &val, PMIX_BOOL );
+        ninfo = 1;
+    }
+    rc = PMIx_Fence(NULL, 0, info, ninfo);
+
+    if( NULL != info ){
+        PMIX_INFO_DESTRUCT(info);
+    }
+    return rc;
 }
 
 int PMI_Get_size(int *size)
@@ -229,8 +327,12 @@ int PMI_Lookup_name(const char service_name[], char port[])
         return convert_err(PMIX_ERR_NOT_FOUND);
     }
 
-    /* return the port */
+    /* return the port - sadly, this API doesn't tell us
-    (void)strncpy(port, pdata.value.data.string, PMIX_MAX_VALLEN);
+     * the size of the port array, and so there is a
+     * potential we could overrun it. As this feature
+     * isn't widely supported in PMI-1, try being
+     * conservative */
+    (void)strncpy(port, pdata.value.data.string, PMIX_MAX_KEYLEN);
     PMIX_PDATA_DESTRUCT(&pdata);
 
     return PMIX_SUCCESS;
@@ -260,7 +362,7 @@ int PMI_Get_id_length_max(int *length)
     if (NULL == length) {
         return PMI_ERR_INVALID_VAL_LENGTH;
     }
-    *length = PMIX_MAX_VALLEN;
+    *length = PMIX_MAX_KEYLEN;
     return PMI_SUCCESS;
 }
 
@@ -327,7 +429,9 @@ int PMI_KVS_Get_value_length_max(int *length)
     if (NULL == length) {
         return PMI_ERR_INVALID_VAL_LENGTH;
     }
-    *length = PMIX_MAX_VALLEN;
+    /* don't give them an enormous size of some implementations
+     * immediately malloc a data block for their use */
+    *length = 4096;
     return PMI_SUCCESS;
 }
 
@@ -484,6 +588,9 @@ static pmix_status_t convert_int(int *value, pmix_value_t *kv)
     case PMIX_SIZE:
         *value = kv->data.size;
         break;
+    case PMIX_BOOL:
+        *value = kv->data.flag;
+        break;
     default:
         /* not an integer type */
         return PMIX_ERR_BAD_PARAM;

opal/mca/pmix/pmix1xx/pmix/src/client/pmi2.c

@@ -304,7 +304,7 @@ int PMI2_Nameserv_lookup(const char service_name[], const PMI_keyval_t *info_ptr
     }
 
     /* return the port */
-    (void)strncpy(port, pdata[0].value.data.string, PMIX_MAX_VALLEN);
+    (void)strncpy(port, pdata[0].value.data.string, portLen);
     PMIX_PDATA_DESTRUCT(&pdata[0]);
 
     if (NULL != info_ptr) {

opal/mca/pmix/pmix1xx/pmix/src/client/pmix_client.c

@@ -266,13 +266,12 @@ int PMIx_Init(pmix_proc_t *proc)
     pmix_output_verbose(2, pmix_globals.debug_output,
                         "pmix: init called");
 
-    pmix_bfrop_open();
-    pmix_usock_init(pmix_client_notify_recv);
-    pmix_sec_init();
-
     /* we require the nspace */
     if (NULL == (evar = getenv("PMIX_NAMESPACE"))) {
         /* let the caller know that the server isn't available yet */
+        pmix_output_close(pmix_globals.debug_output);
+        pmix_output_finalize();
+        pmix_class_finalize();
         return PMIX_ERR_INVALID_NAMESPACE;
     }
     (void)strncpy(proc->nspace, evar, PMIX_MAX_NSLEN);
@@ -285,11 +284,17 @@ int PMIx_Init(pmix_proc_t *proc)
      * then we need to return an error */
     if (NULL == (evar = getenv("PMIX_SERVER_URI"))) {
         /* let the caller know that the server isn't available */
+        pmix_output_close(pmix_globals.debug_output);
+        pmix_output_finalize();
+        pmix_class_finalize();
         return PMIX_ERR_SERVER_NOT_AVAIL;
     }
     uri = pmix_argv_split(evar, ':');
     if (3 != pmix_argv_count(uri)) {
         pmix_argv_free(uri);
+        pmix_output_close(pmix_globals.debug_output);
+        pmix_output_finalize();
+        pmix_class_finalize();
         return PMIX_ERROR;
     }
 
@@ -308,6 +313,9 @@ int PMIx_Init(pmix_proc_t *proc)
     /* if the rendezvous file doesn't exist, that's an error */
     if (0 != access(uri[2], R_OK)) {
         pmix_argv_free(uri);
+        pmix_output_close(pmix_globals.debug_output);
+        pmix_output_finalize();
+        pmix_class_finalize();
         return PMIX_ERR_NOT_FOUND;
     }
     pmix_argv_free(uri);
@@ -315,14 +323,28 @@ int PMIx_Init(pmix_proc_t *proc)
     /* we also require our rank */
     if (NULL == (evar = getenv("PMIX_RANK"))) {
         /* let the caller know that the server isn't available yet */
+        pmix_output_close(pmix_globals.debug_output);
+        pmix_output_finalize();
+        pmix_class_finalize();
         return PMIX_ERR_DATA_VALUE_NOT_FOUND;
     }
     pmix_globals.myid.rank = strtol(evar, NULL, 10);
     proc->rank = pmix_globals.myid.rank;
     pmix_globals.pindex = -1;
 
+    /* setup the support */
+    pmix_bfrop_open();
+    pmix_usock_init(pmix_client_notify_recv);
+    pmix_sec_init();
+
     /* create an event base and progress thread for us */
     if (NULL == (pmix_globals.evbase = pmix_start_progress_thread())) {
+        pmix_sec_finalize();
+        pmix_usock_finalize();
+        pmix_bfrop_close();
+        pmix_output_close(pmix_globals.debug_output);
+        pmix_output_finalize();
+        pmix_class_finalize();
         return -1;
     }
 
@@ -332,6 +354,13 @@ int PMIx_Init(pmix_proc_t *proc)
     /* connect to the server - returns job info if successful */
     if (PMIX_SUCCESS != (rc = connect_to_server(&address, &cb))){
         PMIX_DESTRUCT(&cb);
+        pmix_stop_progress_thread(pmix_globals.evbase);
+        pmix_sec_finalize();
+        pmix_usock_finalize();
+        pmix_bfrop_close();
+        pmix_output_close(pmix_globals.debug_output);
+        pmix_output_finalize();
+        pmix_class_finalize();
         return rc;
     }
     PMIX_WAIT_FOR_COMPLETION(cb.active);
@@ -399,10 +428,6 @@ pmix_status_t PMIx_Finalize(void)
     }
 
     pmix_stop_progress_thread(pmix_globals.evbase);
-    event_base_free(pmix_globals.evbase);
-#ifdef HAVE_LIBEVENT_GLOBAL_SHUTDOWN
-    libevent_global_shutdown();
-#endif
 
     pmix_usock_finalize();
     PMIX_DESTRUCT(&pmix_client_globals.myserver);
@@ -411,6 +436,11 @@ pmix_status_t PMIx_Finalize(void)
     if (0 <= pmix_client_globals.myserver.sd) {
         CLOSE_THE_SOCKET(pmix_client_globals.myserver.sd);
     }
+    event_base_free(pmix_globals.evbase);
+#ifdef HAVE_LIBEVENT_GLOBAL_SHUTDOWN
+    libevent_global_shutdown();
+#endif
+
     pmix_bfrop_close();
     pmix_sec_finalize();
 
@@ -874,6 +904,9 @@ void pmix_client_process_nspace_blob(const char *nspace, pmix_buffer_t *bptr)
             PMIX_CONSTRUCT(&buf2, pmix_buffer_t);
             PMIX_LOAD_BUFFER(&buf2, bo->bytes, bo->size);
             /* protect the data */
+            kptr->value->data.bo.bytes = NULL;
+            kptr->value->data.bo.size = 0;
+            PMIX_RELEASE(kptr);
             bo->bytes = NULL;
             bo->size = 0;
             /* start by unpacking the rank */
@@ -904,7 +937,7 @@ void pmix_client_process_nspace_blob(const char *nspace, pmix_buffer_t *bptr)
                 kp2 = PMIX_NEW(pmix_kval_t);
             }
             /* cleanup */
-            PMIX_DESTRUCT(&buf2);
+            PMIX_DESTRUCT(&buf2);  // releases the original kptr data
             PMIX_RELEASE(kp2);
         } else if (0 == strcmp(kptr->key, PMIX_MAP_BLOB)) {
             /* transfer the byte object for unpacking */
@@ -912,6 +945,9 @@ void pmix_client_process_nspace_blob(const char *nspace, pmix_buffer_t *bptr)
             PMIX_CONSTRUCT(&buf2, pmix_buffer_t);
             PMIX_LOAD_BUFFER(&buf2, bo->bytes, bo->size);
             /* protect the data */
+            kptr->value->data.bo.bytes = NULL;
+            kptr->value->data.bo.size = 0;
+            PMIX_RELEASE(kptr);
             bo->bytes = NULL;
             bo->size = 0;
             /* start by unpacking the number of nodes */
@@ -974,17 +1010,24 @@ void pmix_client_process_nspace_blob(const char *nspace, pmix_buffer_t *bptr)
                 pmix_argv_free(procs);
                 PMIX_DESTRUCT(&kv);
             }
+            /* cleanup */
+            PMIX_DESTRUCT(&buf2);  // releases the original kptr data
         } else {
             /* this is job-level data, so just add it to that hash_table
              * with the wildcard rank */
             if (PMIX_SUCCESS != (rc = pmix_hash_store(&nsptr->internal, PMIX_RANK_WILDCARD, kptr))) {
                 PMIX_ERROR_LOG(rc);
             }
+            /* maintain accounting - but note that the kptr remains
+             * alive and stored in the hash table! So we cannot reuse
+             * it for some other purpose */
             PMIX_RELEASE(kptr);
         }
         kptr = PMIX_NEW(pmix_kval_t);
         cnt = 1;
     }
+    /* need to release the leftover kptr */
+    PMIX_RELEASE(kptr);
 }
 
 static int usock_connect(struct sockaddr *addr)

opal/mca/pmix/pmix1xx/pmix/src/server/pmix_server.c

@@ -753,7 +753,7 @@ pmix_status_t PMIx_server_register_client(const pmix_proc_t *proc,
 /* setup the envars for a child process */
 pmix_status_t PMIx_server_setup_fork(const pmix_proc_t *proc, char ***env)
 {
-    char rankstr[PMIX_MAX_VALLEN+1];
+    char rankstr[128];
 
     pmix_output_verbose(2, pmix_globals.debug_output,
                         "pmix:server setup_fork for nspace %s rank %d",
@@ -762,7 +762,7 @@ pmix_status_t PMIx_server_setup_fork(const pmix_proc_t *proc, char ***env)
     /* pass the nspace */
     pmix_setenv("PMIX_NAMESPACE", proc->nspace, true, env);
     /* pass the rank */
-    (void)snprintf(rankstr, PMIX_MAX_VALLEN, "%d", proc->rank);
+    (void)snprintf(rankstr, 127, "%d", proc->rank);
     pmix_setenv("PMIX_RANK", rankstr, true, env);
     /* pass our rendezvous info */
     pmix_setenv("PMIX_SERVER_URI", myuri, true, env);

opal/mca/pmix/pmix1xx/pmix/src/server/pmix_server_listener.c

@@ -511,6 +511,9 @@ static void connection_handler(int sd, short flags, void* cbdata)
                         "connection_handler: new connection: %d",
                         pnd->sd);
 
+    /* ensure the socket is in blocking mode */
+    pmix_usock_set_blocking(pnd->sd);
+
     /* receive identifier info from the client and authenticate it - the
      * function will lookup and return the peer object if the connection
      * is successfully authenticated */

opal/mca/pmix/pmix1xx/pmix/src/usock/usock.c

@@ -135,9 +135,10 @@ pmix_status_t pmix_usock_send_blocking(int sd, char *ptr, size_t size)
                 continue;
             }
             if (pmix_socket_errno != EINTR) {
-                pmix_output(8, "usock_peer_send_blocking: send() to socket %d failed: %s (%d)\n",
+                pmix_output_verbose(8, pmix_globals.debug_output,
-                            sd, strerror(pmix_socket_errno),
+                                    "usock_peer_send_blocking: send() to socket %d failed: %s (%d)\n",
-                            pmix_socket_errno);
+                                    sd, strerror(pmix_socket_errno),
+                                    pmix_socket_errno);
                 return PMIX_ERR_UNREACH;
             }
             continue;