Sync with PMIx master

Implement direct modex protection to turn off PMIx dstore when direct modex scenario is detected

Signed-off-by: Ralph Castain <rhc@open-mpi.org>

.gitignore

@@ -319,6 +319,7 @@ opal/mca/pmix/pmix*/pmix/src/mca/pinstalldirs/config/pinstall_dirs.h
 opal/mca/pmix/pmix*/pmix/config/autogen_found_items.m4
 opal/mca/pmix/pmix*/pmix/src/include/pmix_config.h
 opal/mca/pmix/pmix*/pmix/src/include/pmix_config.h.in
+opal/mca/pmix/pmix*/pmix/include/pmix_common.h
 opal/mca/pmix/pmix*/pmix/include/pmix_rename.h
 opal/mca/pmix/pmix*/pmix/include/pmix_version.h
 opal/mca/pmix/pmix*/pmix/src/util/keyval/keyval_lex.c

opal/mca/pmix/pmix3x/pmix/VERSION

@@ -30,7 +30,7 @@ greek=
 # command, or with the date (if "git describe" fails) in the form of
 # "date<date>".
 
-repo_rev=git877b4d8
+repo_rev=git9137d98
 
 # If tarball_version is not empty, it is used as the version string in
 # the tarball filename, regardless of all other versions listed in
@@ -44,7 +44,7 @@ tarball_version=
 
 # The date when this release was created
 
-date="Oct 30, 2017"
+date="Nov 07, 2017"
 
 # The shared library version of each of PMIx's public libraries.
 # These versions are maintained in accordance with the "Library

opal/mca/pmix/pmix3x/pmix/config/pmix.m4

@@ -819,6 +819,8 @@ AC_DEFUN([PMIX_SETUP_CORE],[
 
     pmix_show_subtitle "Final output"
 
+    AC_CONFIG_HEADERS(pmix_config_prefix[include/pmix_common.h])
+
     AC_CONFIG_FILES(
         pmix_config_prefix[Makefile]
         pmix_config_prefix[config/Makefile]

opal/mca/pmix/pmix3x/pmix/examples/debugger.c

@@ -232,7 +232,7 @@ int main(int argc, char **argv)
     pmix_info_t *info;
     pmix_app_t *app;
     size_t ninfo, napps;
-    char *tdir, *filename, *sdir, *str, *nspace = NULL;
+    char *nspace = NULL;
     char appspace[PMIX_MAX_NSLEN+1];
     int i;
     pmix_query_t *query;
@@ -241,7 +241,6 @@ int main(int argc, char **argv)
     bool cospawn = false, stop_on_exec = false;
     char cwd[1024];
     pmix_status_t code = PMIX_ERR_JOB_TERMINATED;
-    char hostname[1024];
     mylock_t mylock;
 
     /* Process any arguments we were given */

opal/mca/pmix/pmix3x/pmix/include/Makefile.am

@@ -13,7 +13,6 @@
 if WANT_PRIMARY_HEADERS
 include_HEADERS = \
         pmix.h \
-        pmix_common.h \
         pmix_server.h \
         pmix_tool.h
 
@@ -24,6 +23,7 @@ include_HEADERS += \
 endif
 
 nodist_include_HEADERS = \
+    pmix_common.h \
     pmix_version.h \
     pmix_rename.h
 

opal/mca/pmix/pmix3x/pmix/include/pmix_common.h.in

@@ -61,12 +61,16 @@
 #include <unistd.h> /* for uid_t and gid_t */
 #include <sys/types.h> /* for uid_t and gid_t */
 
-#ifdef PMIX_HAVE_VISIBILITY
+/* Whether C compiler supports -fvisibility */
+#undef PMIX_HAVE_VISIBILITY
+
+#if PMIX_HAVE_VISIBILITY == 1
 #define PMIX_EXPORT __attribute__((__visibility__("default")))
 #else
 #define PMIX_EXPORT
 #endif
 
+
 #include <pmix_rename.h>
 #include <pmix_version.h>
 

opal/mca/pmix/pmix3x/pmix/src/client/pmix_client.c

@@ -97,7 +97,7 @@ static void pmix_client_notify_recv(struct pmix_peer_t *peer,
     pmix_event_chain_t *chain;
     size_t ninfo;
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_client_globals.base_output,
                         "pmix:client_notify_recv - processing event");
 
     /* a zero-byte buffer indicates that this recv is being
@@ -182,7 +182,7 @@ static void pmix_client_notify_recv(struct pmix_peer_t *peer,
     /* now put the callback object tag in the last element */
     PMIX_INFO_LOAD(&chain->info[ninfo], PMIX_EVENT_RETURN_OBJECT, NULL, PMIX_POINTER);
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_client_globals.base_output,
                         "[%s:%d] pmix:client_notify_recv - processing event %d, calling errhandler",
                         pmix_globals.myid.nspace, pmix_globals.myid.rank, chain->status);
 
@@ -191,7 +191,7 @@ static void pmix_client_notify_recv(struct pmix_peer_t *peer,
 
   error:
     /* we always need to return */
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_client_globals.base_output,
                         "pmix:client_notify_recv - unpack error status =%d, calling def errhandler", rc);
     chain = PMIX_NEW(pmix_event_chain_t);
     if (NULL == chain) {
@@ -212,7 +212,7 @@ static void wait_cbfunc(struct pmix_peer_t *pr,
 {
     pmix_lock_t *lock = (pmix_lock_t*)cbdata;
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_client_globals.base_output,
                         "pmix:client wait_cbfunc received");
     PMIX_WAKEUP_THREAD(lock);
 }
@@ -418,8 +418,15 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc,
     PMIX_CONSTRUCT(&pmix_globals.notifications, pmix_ring_buffer_t);
     pmix_ring_buffer_init(&pmix_globals.notifications, 256);
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_client_globals.base_output,
                         "pmix: init called");
+    /* setup the base verbosity */
+    if (0 < pmix_client_globals.base_verbose) {
+        /* set default output */
+        pmix_client_globals.base_output = pmix_output_open(NULL);
+        pmix_output_set_verbosity(pmix_client_globals.base_output,
+                                  pmix_client_globals.base_verbose);
+    }
 
     /* we require our nspace */
     if (NULL == (evar = getenv("PMIX_NAMESPACE"))) {
@@ -618,7 +625,7 @@ static void fin_timeout(int sd, short args, void *cbdata)
     pmix_client_timeout_t *tev;
     tev = (pmix_client_timeout_t*)cbdata;
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_client_globals.base_output,
                         "pmix:client finwait timeout fired");
     if (tev->active) {
         tev->active = false;
@@ -633,7 +640,7 @@ static void finwait_cbfunc(struct pmix_peer_t *pr,
     pmix_client_timeout_t *tev;
     tev = (pmix_client_timeout_t*)cbdata;
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_client_globals.base_output,
                         "pmix:client finwait_cbfunc received");
     if (tev->active) {
         tev->active = false;
@@ -658,7 +665,7 @@ PMIX_EXPORT pmix_status_t PMIx_Finalize(const pmix_info_t info[], size_t ninfo)
     }
     pmix_globals.init_cntr = 0;
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_client_globals.base_output,
                         "%s:%d pmix:client finalize called",
                         pmix_globals.myid.nspace, pmix_globals.myid.rank);
 
@@ -696,7 +703,7 @@ PMIX_EXPORT pmix_status_t PMIx_Finalize(const pmix_info_t info[], size_t ninfo)
         }
 
 
-        pmix_output_verbose(2, pmix_globals.debug_output,
+        pmix_output_verbose(2, pmix_client_globals.base_output,
                              "%s:%d pmix:client sending finalize sync to server",
                              pmix_globals.myid.nspace, pmix_globals.myid.rank);
 
@@ -723,7 +730,7 @@ PMIX_EXPORT pmix_status_t PMIx_Finalize(const pmix_info_t info[], size_t ninfo)
             pmix_event_del(&tev.ev);
         }
 
-        pmix_output_verbose(2, pmix_globals.debug_output,
+        pmix_output_verbose(2, pmix_client_globals.base_output,
                              "%s:%d pmix:client finalize sync received",
                              pmix_globals.myid.nspace, pmix_globals.myid.rank);
     }
@@ -760,7 +767,7 @@ PMIX_EXPORT pmix_status_t PMIx_Abort(int flag, const char msg[],
     pmix_status_t rc;
     pmix_lock_t reglock;
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_client_globals.base_output,
                         "pmix:client abort called");
 
     PMIX_ACQUIRE_THREAD(&pmix_global_lock);
@@ -911,7 +918,7 @@ PMIX_EXPORT pmix_status_t PMIx_Put(pmix_scope_t scope, const char key[], pmix_va
     pmix_cb_t *cb;
     pmix_status_t rc;
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_client_globals.base_output,
                         "pmix: executing put for key %s type %d",
                         key, val->type);
 

opal/mca/pmix/pmix3x/pmix/src/client/pmix_client_get.c

@@ -91,6 +91,12 @@ PMIX_EXPORT pmix_status_t PMIx_Get(const pmix_proc_t *proc, const char key[],
     }
     PMIX_RELEASE_THREAD(&pmix_global_lock);
 
+    pmix_output_verbose(2, pmix_client_globals.get_verbose,
+                        "pmix:client get for %s:%d key %s",
+                        (NULL == proc) ? "NULL" : proc->nspace,
+                        (NULL == proc) ? PMIX_RANK_UNDEF : proc->rank,
+                        (NULL == key) ? "NULL" : key);
+
     /* create a callback object as we need to pass it to the
      * recv routine so we know which callback to use when
      * the return message is recvd */
@@ -108,7 +114,7 @@ PMIX_EXPORT pmix_status_t PMIx_Get(const pmix_proc_t *proc, const char key[],
     }
     PMIX_RELEASE(cb);
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_client_globals.get_verbose,
                         "pmix:client get completed");
 
     return rc;
@@ -142,12 +148,16 @@ PMIX_EXPORT pmix_status_t PMIx_Get_nb(const pmix_proc_t *proc, const char *key,
      * Either case is supported. However, we don't currently
      * support the case where -both- values are NULL */
     if (NULL == proc && NULL == key) {
+        pmix_output_verbose(2, pmix_client_globals.get_verbose,
+                            "pmix: get_nb value error - both proc and key are NULL");
         return PMIX_ERR_BAD_PARAM;
     }
 
     /* if the key is NULL, the rank cannot be WILDCARD as
      * we cannot return all info from every rank */
     if (NULL != proc && PMIX_RANK_WILDCARD == proc->rank && NULL == key) {
+        pmix_output_verbose(2, pmix_client_globals.get_verbose,
+                            "pmix: get_nb value error - WILDCARD rank and key is NULL");
         return PMIX_ERR_BAD_PARAM;
     }
 
@@ -168,7 +178,7 @@ PMIX_EXPORT pmix_status_t PMIx_Get_nb(const pmix_proc_t *proc, const char *key,
         rank = proc->rank;
     }
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_client_globals.get_verbose,
                         "pmix: get_nb value for proc %s:%u key %s",
                         nm, rank, (NULL == key) ? "NULL" : key);
 
@@ -273,7 +283,7 @@ static void _getnb_cbfunc(struct pmix_peer_t *pr,
     pmix_proc_t proc;
     pmix_kval_t *kv;
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_client_globals.get_verbose,
                         "pmix: get_nb callback recvd");
 
     if (NULL == cb) {
@@ -479,7 +489,7 @@ static void _getnbfn(int fd, short flags, void *cbdata)
     /* cb was passed to us from another thread - acquire it */
     PMIX_ACQUIRE_OBJECT(cb);
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_client_globals.get_verbose,
                         "pmix: getnbfn value for proc %s:%u key %s",
                         cb->pname.nspace, cb->pname.rank,
                         (NULL == cb->key) ? "NULL" : cb->key);
@@ -517,9 +527,13 @@ static void _getnbfn(int fd, short flags, void *cbdata)
     cb->copy = true;
     PMIX_GDS_FETCH_KV(rc, pmix_globals.mypeer, cb);
     if (PMIX_SUCCESS == rc) {
+        pmix_output_verbose(5, pmix_client_globals.get_verbose,
+                            "pmix:client data found in internal storage");
         rc = process_values(&val, cb);
         goto respond;
     }
+    pmix_output_verbose(5, pmix_client_globals.get_verbose,
+                        "pmix:client data NOT found in internal storage");
 
     /* if the key is NULL or starts with "pmix", then they are looking
      * for data that was provided by the server at startup */
@@ -530,6 +544,8 @@ static void _getnbfn(int fd, short flags, void *cbdata)
         cb->copy = true;
         PMIX_GDS_FETCH_KV(rc, pmix_client_globals.myserver, cb);
         if (PMIX_SUCCESS != rc) {
+            pmix_output_verbose(5, pmix_client_globals.get_verbose,
+                                "pmix:client job-level data NOT found");
             if (0 != strncmp(cb->pname.nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN)) {
                 /* we are asking about the job-level info from another
                  * namespace. It seems that we don't have it - go and
@@ -542,6 +558,8 @@ static void _getnbfn(int fd, short flags, void *cbdata)
                  * job-level info. In some cases, a server may elect not
                  * to provide info at init to save memory */
                 if (immediate) {
+                    pmix_output_verbose(5, pmix_client_globals.get_verbose,
+                                        "pmix:client IMMEDIATE given - querying data");
                     /* the direct modex request doesn't pass a key as it
                      * was intended to support non-job-level information.
                      * So instead, we will use the PMIx_Query function
@@ -560,11 +578,17 @@ static void _getnbfn(int fd, short flags, void *cbdata)
                     return;
                 }
                 /* we should have had this info, so respond with the error */
+                pmix_output_verbose(5, pmix_client_globals.get_verbose,
+                                    "pmix:client returning NOT FOUND error");
                 goto respond;
             } else {
+                pmix_output_verbose(5, pmix_client_globals.get_verbose,
+                                    "pmix:client NULL KEY - returning error");
                 goto respond;
             }
         }
+        pmix_output_verbose(5, pmix_client_globals.get_verbose,
+                            "pmix:client job-level data NOT found");
         rc = process_values(&val, cb);
         goto respond;
     } else {
@@ -619,7 +643,7 @@ static void _getnbfn(int fd, short flags, void *cbdata)
      * us to attempt to retrieve it from the server */
     if (optional) {
         /* they don't want us to try and retrieve it */
-        pmix_output_verbose(2, pmix_globals.debug_output,
+        pmix_output_verbose(2, pmix_client_globals.get_verbose,
                             "PMIx_Get key=%s for rank = %u, namespace = %s was not found - request was optional",
                             cb->key, cb->pname.rank, cb->pname.nspace);
         rc = PMIX_ERR_NOT_FOUND;
@@ -647,7 +671,7 @@ static void _getnbfn(int fd, short flags, void *cbdata)
         goto respond;
     }
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_client_globals.get_verbose,
                         "%s:%d REQUESTING DATA FROM SERVER FOR %s:%d KEY %s",
                         pmix_globals.myid.nspace, pmix_globals.myid.rank,
                         cb->pname.nspace, cb->pname.rank, cb->key);

opal/mca/pmix/pmix3x/pmix/src/client/pmix_client_ops.h

@@ -22,6 +22,27 @@ BEGIN_C_DECLS
 typedef struct {
     pmix_peer_t *myserver;          // messaging support to/from my server
     pmix_list_t pending_requests;   // list of pmix_cb_t pending data requests
+    // verbosity for client get operations
+    int get_output;
+    int get_verbose;
+    // verbosity for client connect operations
+    int connect_output;
+    int connect_verbose;
+    // verbosity for client fence operations
+    int fence_output;
+    int fence_verbose;
+    // verbosity for client pub operations
+    int pub_output;
+    int pub_verbose;
+    // verbosity for client spawn operations
+    int spawn_output;
+    int spawn_verbose;
+    // verbosity for client event operations
+    int event_output;
+    int event_verbose;
+    // verbosity for basic client functions
+    int base_output;
+    int base_verbose;
 } pmix_client_globals_t;
 
 PMIX_EXPORT extern pmix_client_globals_t pmix_client_globals;

opal/mca/pmix/pmix3x/pmix/src/event/pmix_event_notification.c

@@ -65,7 +65,7 @@ PMIX_EXPORT pmix_status_t PMIx_Notify_event(pmix_status_t status,
         rc = pmix_server_notify_client_of_event(status, source, range,
                                                 info, ninfo,
                                                 cbfunc, cbdata);
-        pmix_output_verbose(2, pmix_globals.debug_output,
+        pmix_output_verbose(2, pmix_server_globals.event_output,
                             "pmix_server_notify_event source = %s:%d event_status = %d, rc= %d",
                             (NULL == source) ? "UNKNOWN" : source->nspace,
                             (NULL == source) ? PMIX_RANK_WILDCARD : source->rank, status, rc);
@@ -73,7 +73,7 @@ PMIX_EXPORT pmix_status_t PMIx_Notify_event(pmix_status_t status,
         rc = notify_server_of_event(status, source, range,
                                     info, ninfo,
                                     cbfunc, cbdata);
-        pmix_output_verbose(2, pmix_globals.debug_output,
+        pmix_output_verbose(2, pmix_client_globals.event_output,
                             "pmix_client_notify_event source = %s:%d event_status =%d, rc=%d",
                             (NULL == source) ? pmix_globals.myid.nspace : source->nspace,
                             (NULL == source) ? pmix_globals.myid.rank : source->rank, status, rc);
@@ -116,7 +116,7 @@ static pmix_status_t notify_server_of_event(pmix_status_t status,
     size_t n;
     pmix_notify_caddy_t *cd, *rbout;
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_client_globals.event_output,
                         "client: notifying server %s:%d of status %s for range %s",
                         pmix_globals.myid.nspace, pmix_globals.myid.rank,
                         PMIx_Error_string(status), PMIx_Data_range_string(range));
@@ -243,7 +243,7 @@ static pmix_status_t notify_server_of_event(pmix_status_t status,
         cb->cbfunc.opfn = cbfunc;
         cb->cbdata = cbdata;
         /* send to the server */
-        pmix_output_verbose(2, pmix_globals.debug_output,
+        pmix_output_verbose(2, pmix_client_globals.event_output,
                             "client: notifying server %s:%d - sending",
                             pmix_globals.myid.nspace, pmix_globals.myid.rank);
         PMIX_PTL_SEND_RECV(rc, pmix_client_globals.myserver,
@@ -263,7 +263,7 @@ static pmix_status_t notify_server_of_event(pmix_status_t status,
     return PMIX_SUCCESS;
 
   cleanup:
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_client_globals.event_output,
                         "client: notifying server - unable to send");
     PMIX_RELEASE(msg);
     /* we were unable to send anything, so we just return the error */
@@ -803,7 +803,7 @@ static void _notify_client_event(int sd, short args, void *cbdata)
     /* need to acquire the object from its originating thread */
     PMIX_ACQUIRE_OBJECT(cd);
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_server_globals.event_output,
                         "pmix_server: _notify_client_event notifying clients of event %s range %s type %s",
                         PMIx_Error_string(cd->status),
                         PMIx_Data_range_string(cd->range),
@@ -882,7 +882,7 @@ static void _notify_client_event(int sd, short args, void *cbdata)
                             continue;
                         }
                     }
-                    pmix_output_verbose(2, pmix_globals.debug_output,
+                    pmix_output_verbose(2, pmix_server_globals.event_output,
                                         "pmix_server: notifying client %s:%u on status %s",
                                         pr->peer->info->pname.nspace, pr->peer->info->pname.rank,
                                         PMIx_Error_string(cd->status));
@@ -1007,7 +1007,7 @@ pmix_status_t pmix_server_notify_client_of_event(pmix_status_t status,
     pmix_notify_caddy_t *cd;
     size_t n;
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_server_globals.event_output,
                         "pmix_server: notify client of event %s",
                         PMIx_Error_string(status));
 
@@ -1079,7 +1079,7 @@ pmix_status_t pmix_server_notify_client_of_event(pmix_status_t status,
     cd->cbfunc = cbfunc;
     cd->cbdata = cbdata;
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_server_globals.event_output,
                         "pmix_server_notify_event status =%d, source = %s:%d, ninfo =%lu",
                          status, cd->source.nspace, cd->source.rank, ninfo);
 

opal/mca/pmix/pmix3x/pmix/src/mca/psensor/file/psensor_file.h

@@ -29,7 +29,7 @@ typedef struct {
     pmix_list_t trackers;
 } pmix_psensor_file_component_t;
 
-extern pmix_psensor_file_component_t mca_psensor_file_component;
+PMIX_EXPORT extern pmix_psensor_file_component_t mca_psensor_file_component;
 extern pmix_psensor_base_module_t pmix_psensor_file_module;
 
 

opal/mca/pmix/pmix3x/pmix/src/runtime/pmix_init.c

@@ -50,6 +50,7 @@
 #include "src/mca/preg/base/base.h"
 #include "src/mca/ptl/base/base.h"
 
+#include "src/client/pmix_client_ops.h"
 #include "src/event/pmix_event.h"
 #include "src/include/types.h"
 #include "src/util/error.h"
@@ -159,6 +160,45 @@ int pmix_rte_init(pmix_proc_type_t type,
     PMIX_CONSTRUCT(&pmix_globals.notifications, pmix_ring_buffer_t);
     pmix_ring_buffer_init(&pmix_globals.notifications, 256);
 
+    /* Setup client verbosities as all procs are allowed to
+     * access client APIs */
+    if (0 < pmix_client_globals.get_verbose) {
+        /* set default output */
+        pmix_client_globals.get_output = pmix_output_open(NULL);
+        pmix_output_set_verbosity(pmix_client_globals.get_output,
+                                  pmix_client_globals.get_verbose);
+    }
+    if (0 < pmix_client_globals.connect_verbose) {
+        /* set default output */
+        pmix_client_globals.connect_output = pmix_output_open(NULL);
+        pmix_output_set_verbosity(pmix_client_globals.connect_output,
+                                  pmix_client_globals.connect_verbose);
+    }
+    if (0 < pmix_client_globals.fence_verbose) {
+        /* set default output */
+        pmix_client_globals.fence_output = pmix_output_open(NULL);
+        pmix_output_set_verbosity(pmix_client_globals.fence_output,
+                                  pmix_client_globals.fence_verbose);
+    }
+    if (0 < pmix_client_globals.pub_verbose) {
+        /* set default output */
+        pmix_client_globals.pub_output = pmix_output_open(NULL);
+        pmix_output_set_verbosity(pmix_client_globals.pub_output,
+                                  pmix_client_globals.pub_verbose);
+    }
+    if (0 < pmix_client_globals.spawn_verbose) {
+        /* set default output */
+        pmix_client_globals.spawn_output = pmix_output_open(NULL);
+        pmix_output_set_verbosity(pmix_client_globals.spawn_output,
+                                  pmix_client_globals.spawn_verbose);
+    }
+    if (0 < pmix_client_globals.event_verbose) {
+        /* set default output */
+        pmix_client_globals.event_output = pmix_output_open(NULL);
+        pmix_output_set_verbosity(pmix_client_globals.event_output,
+                                  pmix_client_globals.event_verbose);
+    }
+
     /* get our effective id's */
     pmix_globals.uid = geteuid();
     pmix_globals.gid = getegid();
@@ -174,8 +214,8 @@ int pmix_rte_init(pmix_proc_type_t type,
         ret = PMIX_ERR_NOMEM;
         goto return_error;
     }
-    /* whatever our declared proc type, we are definitely v2.1 */
+    /* whatever our declared proc type, we are definitely v3.0 */
-    pmix_globals.mypeer->proc_type = type | PMIX_PROC_V21;
+    pmix_globals.mypeer->proc_type = type | PMIX_PROC_V3;
     /* create an nspace object for ourselves - we will
      * fill in the nspace name later */
     pmix_globals.mypeer->nptr = PMIX_NEW(pmix_nspace_t);

opal/mca/pmix/pmix3x/pmix/src/runtime/pmix_params.c

@@ -35,6 +35,8 @@
 #include "src/mca/base/pmix_mca_base_var.h"
 #include "src/runtime/pmix_rte.h"
 #include "src/util/timings.h"
+#include "src/client/pmix_client_ops.h"
+#include "src/server/pmix_server_ops.h"
 
 #if PMIX_ENABLE_TIMING
 char *pmix_timing_output = NULL;
@@ -106,6 +108,93 @@ pmix_status_t pmix_register_params(void)
                                   PMIX_INFO_LVL_1, PMIX_MCA_BASE_VAR_SCOPE_ALL,
                                   &pmix_suppress_missing_data_warning);
 
+    /****   CLIENT: VERBOSE OUTPUT PARAMS   ****/
+    (void) pmix_mca_base_var_register ("pmix", "pmix", "client", "get_verbose",
+                                  "Verbosity for client get operations",
+                                  PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                  PMIX_INFO_LVL_1, PMIX_MCA_BASE_VAR_SCOPE_ALL,
+                                  &pmix_client_globals.get_verbose);
+
+    (void) pmix_mca_base_var_register ("pmix", "pmix", "client", "connect_verbose",
+                                  "Verbosity for client connect operations",
+                                  PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                  PMIX_INFO_LVL_1, PMIX_MCA_BASE_VAR_SCOPE_ALL,
+                                  &pmix_client_globals.connect_verbose);
+
+    (void) pmix_mca_base_var_register ("pmix", "pmix", "client", "fence_verbose",
+                                  "Verbosity for client fence operations",
+                                  PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                  PMIX_INFO_LVL_1, PMIX_MCA_BASE_VAR_SCOPE_ALL,
+                                  &pmix_client_globals.fence_verbose);
+
+    (void) pmix_mca_base_var_register ("pmix", "pmix", "client", "pub_verbose",
+                                  "Verbosity for client publish, lookup, and unpublish operations",
+                                  PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                  PMIX_INFO_LVL_1, PMIX_MCA_BASE_VAR_SCOPE_ALL,
+                                  &pmix_client_globals.pub_verbose);
+
+    (void) pmix_mca_base_var_register ("pmix", "pmix", "client", "spawn_verbose",
+                                  "Verbosity for client spawn operations",
+                                  PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                  PMIX_INFO_LVL_1, PMIX_MCA_BASE_VAR_SCOPE_ALL,
+                                  &pmix_client_globals.spawn_verbose);
+
+    (void) pmix_mca_base_var_register ("pmix", "pmix", "client", "event_verbose",
+                                  "Verbosity for eventt spawn operations",
+                                  PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                  PMIX_INFO_LVL_1, PMIX_MCA_BASE_VAR_SCOPE_ALL,
+                                  &pmix_client_globals.event_verbose);
+
+    (void) pmix_mca_base_var_register ("pmix", "pmix", "client", "base_verbose",
+                                  "Verbosity for basic client operations",
+                                  PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                  PMIX_INFO_LVL_1, PMIX_MCA_BASE_VAR_SCOPE_ALL,
+                                  &pmix_client_globals.base_verbose);
+
+    /****   SERVER: VERBOSE OUTPUT PARAMS   ****/
+    (void) pmix_mca_base_var_register ("pmix", "pmix", "server", "get_verbose",
+                                  "Verbosity for server get operations",
+                                  PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                  PMIX_INFO_LVL_1, PMIX_MCA_BASE_VAR_SCOPE_ALL,
+                                  &pmix_server_globals.get_verbose);
+
+    (void) pmix_mca_base_var_register ("pmix", "pmix", "server", "connect_verbose",
+                                  "Verbosity for server connect operations",
+                                  PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                  PMIX_INFO_LVL_1, PMIX_MCA_BASE_VAR_SCOPE_ALL,
+                                  &pmix_server_globals.connect_verbose);
+
+    (void) pmix_mca_base_var_register ("pmix", "pmix", "server", "fence_verbose",
+                                  "Verbosity for server fence operations",
+                                  PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                  PMIX_INFO_LVL_1, PMIX_MCA_BASE_VAR_SCOPE_ALL,
+                                  &pmix_server_globals.fence_verbose);
+
+    (void) pmix_mca_base_var_register ("pmix", "pmix", "server", "pub_verbose",
+                                  "Verbosity for server publish, lookup, and unpublish operations",
+                                  PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                  PMIX_INFO_LVL_1, PMIX_MCA_BASE_VAR_SCOPE_ALL,
+                                  &pmix_server_globals.pub_verbose);
+
+    (void) pmix_mca_base_var_register ("pmix", "pmix", "server", "spawn_verbose",
+                                  "Verbosity for server spawn operations",
+                                  PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                  PMIX_INFO_LVL_1, PMIX_MCA_BASE_VAR_SCOPE_ALL,
+                                  &pmix_server_globals.spawn_verbose);
+
+    (void) pmix_mca_base_var_register ("pmix", "pmix", "server", "event_verbose",
+                                  "Verbosity for server event operations",
+                                  PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                  PMIX_INFO_LVL_1, PMIX_MCA_BASE_VAR_SCOPE_ALL,
+                                  &pmix_server_globals.event_verbose);
+
+    (void) pmix_mca_base_var_register ("pmix", "pmix", "server", "base_verbose",
+                                  "Verbosity for basic server operations",
+                                  PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                  PMIX_INFO_LVL_1, PMIX_MCA_BASE_VAR_SCOPE_ALL,
+                                  &pmix_server_globals.base_verbose);
+
+
     return PMIX_SUCCESS;
 }
 

opal/mca/pmix/pmix3x/pmix/src/server/pmix_server.c

@@ -109,7 +109,7 @@ PMIX_EXPORT pmix_status_t PMIx_server_init(pmix_server_module_t *module,
 
     PMIX_ACQUIRE_THREAD(&pmix_global_lock);
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_server_globals.base_output,
                         "pmix:server init called");
 
     /* setup the runtime - this init's the globals,
@@ -130,9 +130,54 @@ PMIX_EXPORT pmix_status_t PMIx_server_init(pmix_server_module_t *module,
     PMIX_CONSTRUCT(&pmix_server_globals.local_reqs, pmix_list_t);
     PMIX_CONSTRUCT(&pmix_server_globals.nspaces, pmix_list_t);
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_server_globals.base_output,
                         "pmix:server init called");
 
+    /* setup the server verbosities */
+    if (0 < pmix_server_globals.get_verbose) {
+        /* set default output */
+        pmix_server_globals.get_output = pmix_output_open(NULL);
+        pmix_output_set_verbosity(pmix_server_globals.get_output,
+                                  pmix_server_globals.get_verbose);
+    }
+    if (0 < pmix_server_globals.connect_verbose) {
+        /* set default output */
+        pmix_server_globals.connect_output = pmix_output_open(NULL);
+        pmix_output_set_verbosity(pmix_server_globals.connect_output,
+                                  pmix_server_globals.connect_verbose);
+    }
+    if (0 < pmix_server_globals.fence_verbose) {
+        /* set default output */
+        pmix_server_globals.fence_output = pmix_output_open(NULL);
+        pmix_output_set_verbosity(pmix_server_globals.fence_output,
+                                  pmix_server_globals.fence_verbose);
+    }
+    if (0 < pmix_server_globals.pub_verbose) {
+        /* set default output */
+        pmix_server_globals.pub_output = pmix_output_open(NULL);
+        pmix_output_set_verbosity(pmix_server_globals.pub_output,
+                                  pmix_server_globals.pub_verbose);
+    }
+    if (0 < pmix_server_globals.spawn_verbose) {
+        /* set default output */
+        pmix_server_globals.spawn_output = pmix_output_open(NULL);
+        pmix_output_set_verbosity(pmix_server_globals.spawn_output,
+                                  pmix_server_globals.spawn_verbose);
+    }
+    if (0 < pmix_server_globals.event_verbose) {
+        /* set default output */
+        pmix_server_globals.event_output = pmix_output_open(NULL);
+        pmix_output_set_verbosity(pmix_server_globals.event_output,
+                                  pmix_server_globals.event_verbose);
+    }
+    /* setup the base verbosity */
+    if (0 < pmix_server_globals.base_verbose) {
+        /* set default output */
+        pmix_server_globals.base_output = pmix_output_open(NULL);
+        pmix_output_set_verbosity(pmix_server_globals.base_output,
+                                  pmix_server_globals.base_verbose);
+    }
+
     /* setup the function pointers */
     memset(&pmix_host_server, 0, sizeof(pmix_server_module_t));
     pmix_host_server = *module;
@@ -316,7 +361,7 @@ PMIX_EXPORT pmix_status_t PMIx_server_finalize(void)
     pmix_globals.init_cntr = 0;
     PMIX_RELEASE_THREAD(&pmix_global_lock);
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_server_globals.base_output,
                         "pmix:server finalize called");
 
     if (!pmix_globals.external_evbase) {
@@ -359,7 +404,7 @@ PMIX_EXPORT pmix_status_t PMIx_server_finalize(void)
     }
     pmix_rte_finalize();
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_server_globals.base_output,
                         "pmix:server finalize complete");
 
     return PMIX_SUCCESS;
@@ -374,7 +419,7 @@ static void _register_nspace(int sd, short args, void *cbdata)
 
     PMIX_ACQUIRE_OBJECT(caddy);
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_server_globals.base_output,
                         "pmix:server _register_nspace %s", cd->proc.nspace);
 
     /* see if we already have this nspace */
@@ -469,7 +514,7 @@ static void _deregister_nspace(int sd, short args, void *cbdata)
 
     PMIX_ACQUIRE_OBJECT(cd);
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_server_globals.base_output,
                         "pmix:server _deregister_nspace %s",
                         cd->proc.nspace);
 
@@ -501,7 +546,7 @@ PMIX_EXPORT void PMIx_server_deregister_nspace(const char nspace[],
 {
     pmix_setup_caddy_t *cd;
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_server_globals.base_output,
                         "pmix:server deregister nspace %s",
                         nspace);
 
@@ -583,7 +628,7 @@ void pmix_server_execute_collective(int sd, short args, void *cbdata)
         PMIX_BFROPS_PACK(rc, peer, &bucket, &tmp, 1, PMIX_BYTE);
 
         if (PMIX_COLLECT_YES == trk->collect_type) {
-            pmix_output_verbose(2, pmix_globals.debug_output,
+            pmix_output_verbose(2, pmix_server_globals.base_output,
                                 "fence - assembling data");
             first = true;
             PMIX_CONSTRUCT(&pnames, pmix_list_t);
@@ -692,7 +737,7 @@ static void _register_client(int sd, short args, void *cbdata)
 
     PMIX_ACQUIRE_OBJECT(cd);
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_server_globals.base_output,
                         "pmix:server _register_client for nspace %s rank %d",
                         cd->proc.nspace, cd->proc.rank);
 
@@ -814,7 +859,7 @@ PMIX_EXPORT pmix_status_t PMIx_server_register_client(const pmix_proc_t *proc,
     }
     PMIX_RELEASE_THREAD(&pmix_global_lock);
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_server_globals.base_output,
                         "pmix:server register client %s:%d",
                         proc->nspace, proc->rank);
 
@@ -844,7 +889,7 @@ static void _deregister_client(int sd, short args, void *cbdata)
 
     PMIX_ACQUIRE_OBJECT(cd);
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_server_globals.base_output,
                         "pmix:server _deregister_client for nspace %s rank %d",
                         cd->proc.nspace, cd->proc.rank);
 
@@ -891,7 +936,7 @@ PMIX_EXPORT void PMIx_server_deregister_client(const pmix_proc_t *proc,
     }
     PMIX_RELEASE_THREAD(&pmix_global_lock);
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_server_globals.base_output,
                         "pmix:server deregister client %s:%d",
                         proc->nspace, proc->rank);
 
@@ -928,7 +973,7 @@ PMIX_EXPORT pmix_status_t PMIx_server_setup_fork(const pmix_proc_t *proc, char *
     }
     PMIX_RELEASE_THREAD(&pmix_global_lock);
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_server_globals.base_output,
                         "pmix:server setup_fork for nspace %s rank %d",
                         proc->nspace, proc->rank);
 
@@ -995,7 +1040,7 @@ static void _dmodex_req(int sd, short args, void *cbdata)
 
     PMIX_ACQUIRE_OBJECT(cd);
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_server_globals.base_output,
                         "DMODX LOOKING FOR %s:%d",
                         cd->proc.nspace, cd->proc.rank);
 
@@ -1129,7 +1174,7 @@ PMIX_EXPORT pmix_status_t PMIx_server_dmodex_request(const pmix_proc_t *proc,
         return PMIX_ERR_BAD_PARAM;
     }
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_server_globals.base_output,
                         "pmix:server dmodex request%s:%d",
                         proc->nspace, proc->rank);
 
@@ -1687,7 +1732,7 @@ static void _mdxcbfunc(int sd, short argc, void *cbdata)
             PMIX_ERROR_LOG(ret);
             goto cleanup;
         }
-        pmix_output_verbose(2, pmix_globals.debug_output,
+        pmix_output_verbose(2, pmix_server_globals.base_output,
                             "server:modex_cbfunc reply being sent to %s:%u",
                             cd->peer->info->pname.nspace, cd->peer->info->pname.rank);
         PMIX_SERVER_QUEUE_REPLY(cd->peer, cd->hdr.tag, reply);
@@ -1720,7 +1765,7 @@ static void modex_cbfunc(pmix_status_t status, const char *data, size_t ndata, v
     pmix_server_trkr_t *tracker = (pmix_server_trkr_t*)cbdata;
     pmix_shift_caddy_t *scd;
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_server_globals.base_output,
                         "server:modex_cbfunc called with %d bytes", (int)ndata);
 
     if (NULL == tracker) {
@@ -1757,7 +1802,7 @@ static void get_cbfunc(pmix_status_t status, const char *data, size_t ndata, voi
     pmix_buffer_t *reply, buf;
     pmix_status_t rc;
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_server_globals.base_output,
                         "server:get_cbfunc called with %d bytes", (int)ndata);
 
     /* no need to thread-shift here as no global data is accessed */
@@ -1790,10 +1835,10 @@ static void get_cbfunc(pmix_status_t status, const char *data, size_t ndata, voi
     buf.bytes_used = 0;
     PMIX_DESTRUCT(&buf);
     /* send the data to the requestor */
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_server_globals.base_output,
                         "server:get_cbfunc reply being sent to %s:%u",
                         cd->peer->info->pname.nspace, cd->peer->info->pname.rank);
-    pmix_output_hexdump(10, pmix_globals.debug_output,
+    pmix_output_hexdump(10, pmix_server_globals.base_output,
                         reply->base_ptr, (reply->bytes_used < 256 ? reply->bytes_used : 256));
 
     PMIX_SERVER_QUEUE_REPLY(cd->peer, cd->hdr.tag, reply);
@@ -1966,7 +2011,7 @@ static void _cnct(int sd, short args, void *cbdata)
                 PMIX_DESTRUCT(&pbkt);
             }
         }
-        pmix_output_verbose(2, pmix_globals.debug_output,
+        pmix_output_verbose(2, pmix_server_globals.base_output,
                             "server:cnct_cbfunc reply being sent to %s:%u",
                             cd->peer->info->pname.nspace, cd->peer->info->pname.rank);
         PMIX_SERVER_QUEUE_REPLY(cd->peer, cd->hdr.tag, reply);
@@ -1990,7 +2035,7 @@ static void cnct_cbfunc(pmix_status_t status,
     pmix_server_trkr_t *tracker = (pmix_server_trkr_t*)cbdata;
     pmix_shift_caddy_t *scd;
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_server_globals.base_output,
                         "server:cnct_cbfunc called with nspace %s",
                         (NULL == nspace) ? "NULL" : nspace);
 
@@ -2040,7 +2085,7 @@ static void _discnct(int sd, short args, void *cbdata)
             PMIX_RELEASE(reply);
             goto cleanup;
         }
-        pmix_output_verbose(2, pmix_globals.debug_output,
+        pmix_output_verbose(2, pmix_server_globals.base_output,
                             "server:cnct_cbfunc reply being sent to %s:%u",
                             cd->peer->info->pname.nspace, cd->peer->info->pname.rank);
         PMIX_SERVER_QUEUE_REPLY(cd->peer, cd->hdr.tag, reply);
@@ -2061,7 +2106,7 @@ static void discnct_cbfunc(pmix_status_t status, void *cbdata)
     pmix_server_trkr_t *tracker = (pmix_server_trkr_t*)cbdata;
     pmix_shift_caddy_t *scd;
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_server_globals.base_output,
                         "server:discnct_cbfunc called on nspace %s",
                         (NULL == tracker) ? "NULL" : tracker->pname.nspace);
 
@@ -2088,7 +2133,7 @@ static void regevents_cbfunc(pmix_status_t status, void *cbdata)
     pmix_server_caddy_t *cd = (pmix_server_caddy_t*) cbdata;
     pmix_buffer_t *reply;
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_server_globals.base_output,
                         "server:regevents_cbfunc called status = %d", status);
 
     reply = PMIX_NEW(pmix_buffer_t);
@@ -2112,7 +2157,7 @@ static void notifyerror_cbfunc (pmix_status_t status, void *cbdata)
     pmix_server_caddy_t *cd = (pmix_server_caddy_t*) cbdata;
     pmix_buffer_t *reply;
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_server_globals.base_output,
                         "server:notifyerror_cbfunc called status = %d", status);
 
     reply = PMIX_NEW(pmix_buffer_t);
@@ -2142,7 +2187,7 @@ static void query_cbfunc(pmix_status_t status,
     pmix_buffer_t *reply;
     pmix_status_t rc;
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_server_globals.base_output,
                         "pmix:query callback with status %d", status);
 
     reply = PMIX_NEW(pmix_buffer_t);
@@ -2219,7 +2264,7 @@ static pmix_status_t server_switchyard(pmix_peer_t *peer, uint32_t tag,
         PMIX_ERROR_LOG(rc);
         return rc;
     }
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_server_globals.base_output,
                         "recvd pmix cmd %d from %s:%u",
                         cmd, peer->info->pname.nspace, peer->info->pname.rank);
 
@@ -2281,7 +2326,7 @@ static pmix_status_t server_switchyard(pmix_peer_t *peer, uint32_t tag,
     }
 
     if (PMIX_FINALIZE_CMD == cmd) {
-        pmix_output_verbose(2, pmix_globals.debug_output,
+        pmix_output_verbose(2, pmix_server_globals.base_output,
                             "recvd FINALIZE");
         /* mark that this peer called finalize */
         peer->finalized = true;
@@ -2433,7 +2478,7 @@ static void server_message_handler(struct pmix_peer_t *pr,
     pmix_buffer_t *reply;
     pmix_status_t rc, ret;
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_server_globals.base_output,
                         "SWITCHYARD for %s:%u:%d",
                         peer->info->pname.nspace,
                         peer->info->pname.rank, peer->sd);

opal/mca/pmix/pmix3x/pmix/src/server/pmix_server_get.c

@@ -130,7 +130,7 @@ pmix_status_t pmix_server_get(pmix_buffer_t *buf,
     size_t sz, n;
     pmix_peer_t *peer;
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_server_globals.get_output,
                         "recvd GET");
 
     /* setup */
@@ -191,7 +191,7 @@ pmix_status_t pmix_server_get(pmix_buffer_t *buf,
         }
     }
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_server_globals.get_output,
                         "%s:%d EXECUTE GET FOR %s:%d ON BEHALF OF %s:%d",
                         pmix_globals.myid.nspace,
                         pmix_globals.myid.rank, nspace, rank,
@@ -320,8 +320,17 @@ pmix_status_t pmix_server_get(pmix_buffer_t *buf,
      * we do know how many clients to expect, so first check to see if
      * all clients have been registered with us */
      if (!nptr->all_registered) {
+        pmix_output_verbose(2, pmix_server_globals.get_output,
+                            "%s:%d NSPACE %s not all registered",
+                            pmix_globals.myid.nspace,
+                            pmix_globals.myid.rank, nspace);
+
         if (localonly) {
             /* the client asked that we not wait, so return now */
+            pmix_output_verbose(2, pmix_server_globals.get_output,
+                                "%s:%d CLIENT REQUESTED IMMEDIATE",
+                                pmix_globals.myid.nspace,
+                                pmix_globals.myid.rank);
             return PMIX_ERR_NOT_FOUND;
         }
         /* we cannot do anything further, so just track this request
@@ -331,6 +340,10 @@ pmix_status_t pmix_server_get(pmix_buffer_t *buf,
         if (PMIX_ERR_NOMEM == rc) {
             PMIX_INFO_FREE(info, ninfo);
         }
+        pmix_output_verbose(2, pmix_server_globals.get_output,
+                            "%s:%d TRACKER CREATED - WAITING",
+                            pmix_globals.myid.nspace,
+                            pmix_globals.myid.rank);
         return rc;
     }
 
@@ -342,9 +355,18 @@ pmix_status_t pmix_server_get(pmix_buffer_t *buf,
         return rc;
     }
 
+    pmix_output_verbose(2, pmix_server_globals.get_output,
+                        "%s:%d DATA NOT FOUND",
+                        pmix_globals.myid.nspace,
+                        pmix_globals.myid.rank);
+
     /* If we get here, then we don't have the data at this time. If
      * the user doesn't want to look for it, then we are done */
     if (localonly) {
+        pmix_output_verbose(2, pmix_server_globals.get_output,
+                            "%s:%d CLIENT REQUESTED IMMEDIATE",
+                            pmix_globals.myid.nspace,
+                            pmix_globals.myid.rank);
         return PMIX_ERR_NOT_FOUND;
     }
 
@@ -379,6 +401,10 @@ pmix_status_t pmix_server_get(pmix_buffer_t *buf,
     if (NULL != pmix_host_server.direct_modex) {
         rc = pmix_host_server.direct_modex(&lcd->proc, info, ninfo, dmdx_cbfunc, lcd);
     } else {
+        pmix_output_verbose(2, pmix_server_globals.get_output,
+                            "%s:%d NO SERVER SUPPORT",
+                            pmix_globals.myid.nspace,
+                            pmix_globals.myid.rank);
         /* if we don't have direct modex feature, just respond with "not found" */
         cbfunc(PMIX_ERR_NOT_FOUND, NULL, 0, cbdata, NULL, NULL);
         PMIX_INFO_FREE(info, ninfo);
@@ -503,6 +529,11 @@ static pmix_status_t _satisfy_request(pmix_nspace_t *nptr, pmix_rank_t rank,
     size_t sz = 0;
     pmix_scope_t scope = PMIX_SCOPE_UNDEF;
 
+    pmix_output_verbose(2, pmix_server_globals.get_output,
+                        "%s:%d SATISFY REQUEST CALLED",
+                        pmix_globals.myid.nspace,
+                        pmix_globals.myid.rank);
+
     /* check to see if this data already has been
      * obtained as a result of a prior direct modex request from
      * a remote peer, or due to data from a local client
@@ -582,6 +613,7 @@ static pmix_status_t _satisfy_request(pmix_nspace_t *nptr, pmix_rank_t rank,
             /* assemble the provided data into a byte object */
             PMIX_GDS_ASSEMB_KVS_REQ(rc, cd->peer, &proc, &cb.kvs, &pkt, cd);
             if (rc != PMIX_SUCCESS) {
+                PMIX_ERROR_LOG(rc);
                 PMIX_DESTRUCT(&pkt);
                 PMIX_DESTRUCT(&pbkt);
                 PMIX_DESTRUCT(&cb);
@@ -595,6 +627,7 @@ static pmix_status_t _satisfy_request(pmix_nspace_t *nptr, pmix_rank_t rank,
                 PMIX_CONSTRUCT(&xfer, pmix_buffer_t);
                 PMIX_BFROPS_PACK(rc, cd->peer, &xfer, &pkt, 1, PMIX_BUFFER);
                 if (PMIX_SUCCESS != rc) {
+                    PMIX_ERROR_LOG(rc);
                     PMIX_DESTRUCT(&pkt);
                     PMIX_DESTRUCT(&pbkt);
                     PMIX_DESTRUCT(&xfer);
@@ -610,6 +643,7 @@ static pmix_status_t _satisfy_request(pmix_nspace_t *nptr, pmix_rank_t rank,
             /* pack it for transmission */
             PMIX_BFROPS_PACK(rc, cd->peer, &pbkt, &bo, 1, PMIX_BYTE_OBJECT);
             if (PMIX_SUCCESS != rc) {
+                PMIX_ERROR_LOG(rc);
                 PMIX_DESTRUCT(&pbkt);
                 PMIX_DESTRUCT(&cb);
                 return rc;
@@ -638,6 +672,7 @@ static pmix_status_t _satisfy_request(pmix_nspace_t *nptr, pmix_rank_t rank,
             /* assemble the provided data into a byte object */
             PMIX_GDS_ASSEMB_KVS_REQ(rc, cd->peer, &proc, &cb.kvs, &pkt, cd);
             if (rc != PMIX_SUCCESS) {
+                PMIX_ERROR_LOG(rc);
                 PMIX_DESTRUCT(&pkt);
                 PMIX_DESTRUCT(&pbkt);
                 PMIX_DESTRUCT(&cb);
@@ -650,6 +685,7 @@ static pmix_status_t _satisfy_request(pmix_nspace_t *nptr, pmix_rank_t rank,
                 /* pack the rank */
                 PMIX_BFROPS_PACK(rc, cd->peer, &pbkt, &rank, 1, PMIX_PROC_RANK);
                 if (PMIX_SUCCESS != rc) {
+                    PMIX_ERROR_LOG(rc);
                     PMIX_DESTRUCT(&pkt);
                     PMIX_DESTRUCT(&pbkt);
                     PMIX_DESTRUCT(&cb);
@@ -658,6 +694,7 @@ static pmix_status_t _satisfy_request(pmix_nspace_t *nptr, pmix_rank_t rank,
                 /* now pack the data itself as a buffer */
                 PMIX_BFROPS_PACK(rc, cd->peer, &pbkt, &pkt, 1, PMIX_BUFFER);
                 if (PMIX_SUCCESS != rc) {
+                    PMIX_ERROR_LOG(rc);
                     PMIX_DESTRUCT(&pkt);
                     PMIX_DESTRUCT(&pbkt);
                     PMIX_DESTRUCT(&cb);
@@ -670,6 +707,7 @@ static pmix_status_t _satisfy_request(pmix_nspace_t *nptr, pmix_rank_t rank,
                 /* pack it for transmission */
                 PMIX_BFROPS_PACK(rc, cd->peer, &pbkt, &bo, 1, PMIX_BYTE_OBJECT);
                 if (PMIX_SUCCESS != rc) {
+                    PMIX_ERROR_LOG(rc);
                     PMIX_DESTRUCT(&pbkt);
                     PMIX_DESTRUCT(&cb);
                     return rc;
@@ -687,12 +725,6 @@ static pmix_status_t _satisfy_request(pmix_nspace_t *nptr, pmix_rank_t rank,
         return rc;
     }
 
-    if ((PMIX_LOCAL == scope) && !found) {
-        /* pass PMIX_ERR_NOT_FOUND for local request if it's not found*/
-        cbfunc(PMIX_ERR_NOT_FOUND, NULL, 0, cbdata, NULL, NULL);
-        return PMIX_SUCCESS;
-    }
-
     return PMIX_ERR_NOT_FOUND;
 }
 
@@ -774,7 +806,7 @@ static void _process_dmdx_reply(int fd, short args, void *cbdata)
 
     PMIX_ACQUIRE_OBJECT(caddy);
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_server_globals.get_output,
                     "[%s:%d] process dmdx reply from %s:%u",
                     __FILE__, __LINE__,
                     caddy->lcd->proc.nspace, caddy->lcd->proc.rank);
@@ -903,7 +935,7 @@ static void dmdx_cbfunc(pmix_status_t status,
     caddy->data   = data;
     caddy->ndata  = ndata;
     caddy->lcd    = (pmix_dmdx_local_t *)cbdata;
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_server_globals.get_output,
                         "[%s:%d] queue dmdx reply for %s:%u",
                         __FILE__, __LINE__,
                         caddy->lcd->proc.nspace, caddy->lcd->proc.rank);

opal/mca/pmix/pmix3x/pmix/src/server/pmix_server_ops.c

@@ -70,7 +70,7 @@ pmix_status_t pmix_server_abort(pmix_peer_t *peer, pmix_buffer_t *buf,
     pmix_proc_t *procs = NULL;
     pmix_proc_t proc;
 
-    pmix_output_verbose(2, pmix_globals.debug_output, "recvd ABORT");
+    pmix_output_verbose(2, pmix_server_globals.base_output, "recvd ABORT");
 
     /* unpack the status */
     cnt = 1;
@@ -156,7 +156,7 @@ pmix_status_t pmix_server_commit(pmix_peer_t *peer, pmix_buffer_t *buf)
     (void)strncpy(proc.nspace, nptr->nspace, PMIX_MAX_NSLEN);
     proc.rank = info->pname.rank;
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_server_globals.base_output,
                         "%s:%d EXECUTE COMMIT FOR %s:%d",
                         pmix_globals.myid.nspace,
                         pmix_globals.myid.rank,
@@ -290,7 +290,7 @@ static pmix_server_trkr_t* get_tracker(pmix_proc_t *procs,
     size_t i, j;
     size_t matches;
 
-    pmix_output_verbose(5, pmix_globals.debug_output,
+    pmix_output_verbose(5, pmix_server_globals.base_output,
                         "get_tracker called with %d procs", (int)nprocs);
 
     /* bozo check - should never happen outside of programmer error */
@@ -360,7 +360,7 @@ static pmix_server_trkr_t* new_tracker(pmix_proc_t *procs,
     pmix_nspace_t *nptr, *ns;
     pmix_rank_info_t *info;
 
-    pmix_output_verbose(5, pmix_globals.debug_output,
+    pmix_output_verbose(5, pmix_server_globals.base_output,
                         "new_tracker called with %d procs", (int)nprocs);
 
     /* bozo check - should never happen outside of programmer error */
@@ -369,7 +369,7 @@ static pmix_server_trkr_t* new_tracker(pmix_proc_t *procs,
         return NULL;
     }
 
-    pmix_output_verbose(5, pmix_globals.debug_output,
+    pmix_output_verbose(5, pmix_server_globals.base_output,
                         "adding new tracker with %d procs", (int)nprocs);
 
     /* this tracker is new - create it */
@@ -406,7 +406,7 @@ static pmix_server_trkr_t* new_tracker(pmix_proc_t *procs,
         }
         if (NULL == nptr) {
             /* cannot be a local proc */
-            pmix_output_verbose(5, pmix_globals.debug_output,
+            pmix_output_verbose(5, pmix_server_globals.base_output,
                                 "new_tracker: unknown nspace %s",
                                 procs[i].nspace);
             continue;
@@ -416,7 +416,7 @@ static pmix_server_trkr_t* new_tracker(pmix_proc_t *procs,
             /* nope, so no point in going further on this one - we'll
              * process it once all the procs are known */
             all_def = false;
-            pmix_output_verbose(5, pmix_globals.debug_output,
+            pmix_output_verbose(5, pmix_server_globals.base_output,
                                 "new_tracker: all clients not registered nspace %s",
                                 procs[i].nspace);
             /* we have to continue processing the list of procs
@@ -427,7 +427,7 @@ static pmix_server_trkr_t* new_tracker(pmix_proc_t *procs,
         PMIX_LIST_FOREACH(info, &nptr->ranks, pmix_rank_info_t) {
             if (procs[i].rank == info->pname.rank ||
                 PMIX_RANK_WILDCARD == procs[i].rank) {
-                    pmix_output_verbose(5, pmix_globals.debug_output,
+                    pmix_output_verbose(5, pmix_server_globals.base_output,
                                         "adding local proc %s.%d to tracker",
                                         info->pname.nspace, info->pname.rank);
                 /* track the count */
@@ -466,7 +466,7 @@ pmix_status_t pmix_server_fence(pmix_server_caddy_t *cd,
     pmix_info_t *info = NULL;
     size_t ninfo=0, n;
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_server_globals.fence_output,
                         "recvd FENCE");
 
     if (NULL == pmix_host_server.fence_nb) {
@@ -480,7 +480,7 @@ pmix_status_t pmix_server_fence(pmix_server_caddy_t *cd,
     if (PMIX_SUCCESS != rc) {
         return rc;
     }
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_server_globals.fence_output,
                         "recvd fence from %s:%u with %d procs",
                         cd->peer->info->pname.nspace, cd->peer->info->pname.rank, (int)nprocs);
     /* there must be at least one as the client has to at least provide
@@ -588,7 +588,7 @@ pmix_status_t pmix_server_fence(pmix_server_caddy_t *cd,
      * across all participants has been completed */
     if (trk->def_complete &&
         pmix_list_get_size(&trk->local_cbs) == trk->nlocal) {
-        pmix_output_verbose(2, pmix_globals.debug_output,
+        pmix_output_verbose(2, pmix_server_globals.base_output,
                             "fence complete");
         /* if the user asked us to collect data, then we have
          * to provide any locally collected data to the host
@@ -606,7 +606,7 @@ pmix_status_t pmix_server_fence(pmix_server_caddy_t *cd,
                          &tmp, 1, PMIX_BYTE);
 
         if (PMIX_COLLECT_YES == trk->collect_type) {
-            pmix_output_verbose(2, pmix_globals.debug_output,
+            pmix_output_verbose(2, pmix_server_globals.fence_output,
                                 "fence - assembling data");
             PMIX_LIST_FOREACH(scd, &trk->local_cbs, pmix_server_caddy_t) {
                 /* get any remote contribution - note that there
@@ -710,7 +710,7 @@ pmix_status_t pmix_server_publish(pmix_peer_t *peer,
     pmix_proc_t proc;
     uint32_t uid;
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_server_globals.pub_output,
                         "recvd PUBLISH");
 
     if (NULL == pmix_host_server.publish) {
@@ -805,7 +805,7 @@ pmix_status_t pmix_server_lookup(pmix_peer_t *peer,
     pmix_proc_t proc;
     uint32_t uid;
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_server_globals.pub_output,
                         "recvd LOOKUP");
 
     if (NULL == pmix_host_server.lookup) {
@@ -901,7 +901,7 @@ pmix_status_t pmix_server_unpublish(pmix_peer_t *peer,
     pmix_proc_t proc;
     uint32_t uid;
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_server_globals.pub_output,
                         "recvd UNPUBLISH");
 
     if (NULL == pmix_host_server.unpublish) {
@@ -1014,7 +1014,7 @@ pmix_status_t pmix_server_spawn(pmix_peer_t *peer,
     pmix_proc_t proc;
     size_t ninfo;
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_server_globals.spawn_output,
                         "recvd SPAWN");
 
     if (NULL == pmix_host_server.spawn) {
@@ -1225,7 +1225,7 @@ pmix_status_t pmix_server_connect(pmix_server_caddy_t *cd,
     size_t nprocs, ninfo;
     pmix_server_trkr_t *trk;
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_server_globals.connect_output,
                         "recvd CONNECT from peer %s:%d",
                         cd->peer->info->pname.nspace,
                         cd->peer->info->pname.rank);
@@ -1355,7 +1355,7 @@ pmix_status_t pmix_server_register_events(pmix_peer_t *peer,
     pmix_buffer_t *relay;
     pmix_cmd_t cmd = PMIX_NOTIFY_CMD;
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_server_globals.event_output,
                         "recvd register events");
 
     /* unpack the number of codes */
@@ -1528,7 +1528,7 @@ pmix_status_t pmix_server_register_events(pmix_peer_t *peer,
         scd->opcbfunc = cbfunc;
         scd->cbdata = cbdata;
         if (PMIX_SUCCESS != (rc = pmix_host_server.register_events(scd->codes, scd->ncodes, scd->info, scd->ninfo, opcbfunc, scd))) {
-            pmix_output_verbose(2, pmix_globals.debug_output,
+            pmix_output_verbose(2, pmix_server_globals.event_output,
                                  "server register events: host server reg events returned rc =%d", rc);
             if (NULL != scd->codes) {
                 free(scd->codes);
@@ -1543,7 +1543,7 @@ pmix_status_t pmix_server_register_events(pmix_peer_t *peer,
     }
 
   cleanup:
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_server_globals.event_output,
                         "server register events: ninfo =%lu rc =%d", ninfo, rc);
     /* be sure to execute the callback */
     if (NULL != cbfunc) {
@@ -1659,7 +1659,7 @@ void pmix_server_deregister_events(pmix_peer_t *peer,
     pmix_regevents_info_t *reginfo_next;
     pmix_peer_events_info_t *prev;
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_server_globals.event_output,
                         "recvd deregister events");
 
     /* unpack codes and process until done */
@@ -1713,7 +1713,7 @@ pmix_status_t pmix_server_event_recvd_from_client(pmix_peer_t *peer,
     pmix_status_t rc;
     pmix_notify_caddy_t *cd;
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_server_globals.event_output,
                         "recvd event notification from client");
 
     if (NULL == pmix_host_server.notify_event) {
@@ -1804,7 +1804,7 @@ pmix_status_t pmix_server_query(pmix_peer_t *peer,
     pmix_query_caddy_t *cd;
     pmix_proc_t proc;
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_server_globals.base_output,
                         "recvd query from client");
 
     if (NULL == pmix_host_server.query) {
@@ -1872,7 +1872,7 @@ pmix_status_t pmix_server_log(pmix_peer_t *peer,
     pmix_shift_caddy_t *cd;
     pmix_proc_t proc;
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_server_globals.base_output,
                         "recvd log from client");
 
     if (NULL == pmix_host_server.log) {
@@ -1946,7 +1946,7 @@ pmix_status_t pmix_server_alloc(pmix_peer_t *peer,
     pmix_proc_t proc;
     pmix_alloc_directive_t directive;
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_server_globals.base_output,
                         "recvd query from client");
 
     if (NULL == pmix_host_server.allocate) {
@@ -2012,7 +2012,7 @@ pmix_status_t pmix_server_job_ctrl(pmix_peer_t *peer,
     pmix_query_caddy_t *cd;
     pmix_proc_t proc;
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_server_globals.base_output,
                         "recvd job control request from client");
 
     if (NULL == pmix_host_server.job_control) {
@@ -2088,7 +2088,7 @@ pmix_status_t pmix_server_monitor(pmix_peer_t *peer,
     pmix_query_caddy_t *cd;
     pmix_proc_t proc;
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_server_globals.base_output,
                         "recvd monitor request from client");
 
     if (NULL == pmix_host_server.monitor) {

opal/mca/pmix/pmix3x/pmix/src/server/pmix_server_ops.h

@@ -107,6 +107,28 @@ typedef struct {
     pmix_list_t gdata;                      // cache of data given to me for passing to all clients
     pmix_list_t events;                     // list of pmix_regevents_info_t registered events
     bool tool_connections_allowed;
+    // verbosity for server get operations
+    int get_output;
+    int get_verbose;
+    // verbosity for server connect operations
+    int connect_output;
+    int connect_verbose;
+    // verbosity for server fence operations
+    int fence_output;
+    int fence_verbose;
+    // verbosity for server pub operations
+    int pub_output;
+    int pub_verbose;
+    // verbosity for server spawn operations
+    int spawn_output;
+    int spawn_verbose;
+    // verbosity for server event operations
+    int event_output;
+    int event_verbose;
+    // verbosity for basic server functions
+    int base_output;
+    int base_verbose;
+
 } pmix_server_globals_t;
 
 #define PMIX_GDS_CADDY(c, p, t)                \

opal/mca/pmix/pmix3x/pmix/src/threads/threads.h

@@ -75,12 +75,10 @@ typedef struct {
         PMIX_CONSTRUCT(&(l)->mutex, pmix_mutex_t);      \
         pthread_cond_init(&(l)->cond, NULL);            \
         (l)->active = true;                             \
-        PMIX_POST_OBJECT((l));                          \
     } while(0)
 
 #define PMIX_DESTRUCT_LOCK(l)               \
     do {                                    \
-        PMIX_ACQUIRE_OBJECT((l));           \
         PMIX_DESTRUCT(&(l)->mutex);         \
         pthread_cond_destroy(&(l)->cond);   \
     } while(0)

opal/mca/pmix/pmix3x/pmix/test/simple/simpdmodex.c

@@ -84,7 +84,7 @@ static void valcbfunc(pmix_status_t status,
         }
         pmix_output(0, "%s:%d PMIx_Get_nb Key %s returned correctly", myproc.nspace, myproc.rank, key);
     } else {
-        pmix_output(0, "%s:%d PMIx_Get_nb Key %s failed", myproc.nspace, myproc.rank, key);
+        pmix_output(0, "%s:%d PMIx_Get_nb Key %s failed: %s", myproc.nspace, myproc.rank, key, PMIx_Error_string(status));
     }
  done:
     free(key);
@@ -100,6 +100,11 @@ int main(int argc, char **argv)
     pmix_proc_t proc;
     uint32_t n, num_gets;
     bool active;
+    bool dofence = true;
+
+    if (NULL != getenv("PMIX_SIMPDMODEX_ASYNC")) {
+        dofence = false;
+    }
 
     /* init us */
     if (PMIX_SUCCESS != (rc = PMIx_Init(&myproc, NULL, 0))) {
@@ -148,6 +153,7 @@ int main(int argc, char **argv)
      * if a "get" is received prior to data being provided */
     if (0 == myproc.rank) {
         sleep(2);
+        pmix_output(0, "\n\n\nWOKE UP");
     }
 
     /* commit the data to the server */
@@ -156,14 +162,16 @@ int main(int argc, char **argv)
         goto done;
     }
 
-    /* call fence_nb, but don't return any data */
+    if (dofence) {
-    PMIX_PROC_CONSTRUCT(&proc);
+        /* call fence_nb, but don't return any data */
-    (void)strncpy(proc.nspace, myproc.nspace, PMIX_MAX_NSLEN);
+        PMIX_PROC_CONSTRUCT(&proc);
-    proc.rank = PMIX_RANK_WILDCARD;
+        (void)strncpy(proc.nspace, myproc.nspace, PMIX_MAX_NSLEN);
-    active = true;
+        proc.rank = PMIX_RANK_WILDCARD;
-    if (PMIX_SUCCESS != (rc = PMIx_Fence_nb(&proc, 1, NULL, 0, opcbfunc, &active))) {
+        active = true;
-        pmix_output(0, "Client ns %s rank %d: PMIx_Fence failed: %d", myproc.nspace, myproc.rank, rc);
+        if (PMIX_SUCCESS != (rc = PMIx_Fence_nb(&proc, 1, NULL, 0, opcbfunc, &active))) {
-        goto done;
+            pmix_output(0, "Client ns %s rank %d: PMIx_Fence failed: %d", myproc.nspace, myproc.rank, rc);
+            goto done;
+        }
     }
 
     /* get the committed data - ask for someone who doesn't exist as well */
@@ -186,8 +194,10 @@ int main(int argc, char **argv)
         ++num_gets;
     }
 
-    /* wait for the first fence to finish */
+    if (dofence) {
-    PMIX_WAIT_FOR_COMPLETION(active);
+        /* wait for the first fence to finish */
+        PMIX_WAIT_FOR_COMPLETION(active);
+    }
 
     /* wait for all my "get" calls to complete */
     while (getcount < num_gets) {

opal/mca/pmix/pmix3x/pmix3x.h

@@ -37,12 +37,6 @@
 
 BEGIN_C_DECLS
 
-#ifdef OPAL_C_HAVE_VISIBILITY
-#define PMIX_HAVE_VISIBILITY 1
-#else
-#undef PMIX_HAVE_VISIBILITY
-#endif
-
 typedef struct {
   opal_pmix_base_component_t super;
   opal_list_t jobids;

opal/mca/pmix/pmix3x/pmix3x_client.c

@@ -99,6 +99,11 @@ int pmix3x_client_init(opal_list_t *ilist)
         ninfo = 0;
     }
 
+    /* check for direct modex use-case */
+    if (opal_pmix_base_async_modex && !opal_pmix_collect_all_data) {
+        opal_setenv("PMIX_MCA_gds", "hash", true, &environ);
+    }
+
     OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
     rc = PMIx_Init(&my_proc, pinfo, ninfo);
     if (NULL != pinfo) {

opal/mca/pmix/pmix3x/pmix3x_server_south.c

@@ -128,6 +128,11 @@ int pmix3x_server_init(opal_pmix_server_module_t *module,
         }
     }
 
+    /* check for direct modex use-case */
+    if (opal_pmix_base_async_modex && !opal_pmix_collect_all_data) {
+        opal_setenv("PMIX_MCA_gds", "hash", true, &environ);
+    }
+
     /* insert ourselves into our list of jobids - it will be the
      * first, and so we'll check it first */
     job = OBJ_NEW(opal_pmix3x_jobid_trkr_t);