Sync to PMIx v3.0rc and add ext4x

Sync to the draft rc for PMIx v3.0. Add an external component for PMIx master, which is at v4.0

Signed-off-by: Ralph Castain <rhc@open-mpi.org>

config/opal_check_pmi.m4

@@ -13,7 +13,7 @@
 # Copyright (c) 2009-2015 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2011-2014 Los Alamos National Security, LLC. All rights
 #                         reserved.
-# Copyright (c) 2014-2017 Intel, Inc. All rights reserved.
+# Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
 # Copyright (c) 2014-2018 Research Organization for Information Science
 #                         and Technology (RIST).  All rights reserved.
 # Copyright (c) 2016      IBM Corporation.  All rights reserved.
@@ -311,6 +311,19 @@ AC_DEFUN([OPAL_CHECK_PMIX],[
 
            # if it does exist, then we need to parse it to find
            # the actual release series
+           AS_IF([test "$opal_external_pmix_version_found" = "0"],
+                 [AC_MSG_CHECKING([version 4x])
+                  AC_PREPROC_IFELSE([AC_LANG_PROGRAM([
+                                                      #include <pmix_version.h>
+                                                      #if (PMIX_VERSION_MAJOR != 4L)
+                                                      #error "not version 4"
+                                                      #endif
+                                                      ], [])],
+                                    [AC_MSG_RESULT([found])
+                                     opal_external_pmix_version=4x
+                                     opal_external_pmix_version_found=1],
+                                    [AC_MSG_RESULT([not found])])])
+
            AS_IF([test "$opal_external_pmix_version_found" = "0"],
                  [AC_MSG_CHECKING([version 3x])
                   AC_PREPROC_IFELSE([AC_LANG_PROGRAM([

opal/mca/pmix/ext4x/Makefile.am

@@ -0,0 +1,54 @@
+#
+# Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
+# Copyright (c) 2015      Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2015-2018 Research Organization for Information Science
+#                         and Technology (RIST). All rights reserved.
+# Copyright (c) 2017      IBM Corporation.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+dist_opaldata_DATA = help-pmix-ext4x.txt
+
+sources = \
+        ext4x.c \
+        ext4x_client.c \
+        ext4x_component.c \
+        ext4x_server_north.c \
+        ext4x_server_south.c \
+        ext4x_local.c
+
+headers = \
+        ext4x.h
+
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if MCA_BUILD_opal_pmix_ext4x_DSO
+component_noinst =
+component_install = mca_pmix_ext4x.la
+else
+component_noinst = libmca_pmix_ext4x.la
+component_install =
+endif
+
+mcacomponentdir = $(opallibdir)
+mcacomponent_LTLIBRARIES = $(component_install)
+mca_pmix_ext4x_la_SOURCES = $(sources)
+mca_pmix_ext4x_la_CFLAGS = $(opal_pmix_ext4x_CFLAGS)
+mca_pmix_ext4x_la_CPPFLAGS =$(opal_pmix_ext4x_CPPFLAGS)
+mca_pmix_ext4x_la_LDFLAGS = -module -avoid-version $(opal_pmix_ext4x_LDFLAGS)
+mca_pmix_ext4x_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la \
+	$(opal_pmix_ext4x_LIBS)
+
+noinst_LTLIBRARIES = $(component_noinst)
+libmca_pmix_ext4x_la_SOURCES =$(sources)
+libmca_pmix_ext4x_la_CFLAGS = $(opal_pmix_ext4x_CFLAGS)
+libmca_pmix_ext4x_la_CPPFLAGS = $(opal_pmix_ext4x_CPPFLAGS)
+libmca_pmix_ext4x_la_LDFLAGS = -module -avoid-version $(opal_pmix_ext4x_LDFLAGS)
+libmca_pmix_ext4x_la_LIBADD = $(opal_pmix_ext4x_LIBS)

opal/mca/pmix/ext4x/configure.m4

@@ -0,0 +1,64 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2011-2013 Los Alamos National Security, LLC.
+#                         All rights reserved.
+# Copyright (c) 2010-2015 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2013-2018 Intel, Inc. All rights reserved.
+# Copyright (c) 2015-2017 Research Organization for Information Science
+#                         and Technology (RIST). All rights reserved.
+# Copyright (c) 2014-2015 Mellanox Technologies, Inc.
+#                         All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# MCA_pmix_ext4x_CONFIG([action-if-found], [action-if-not-found])
+# -----------------------------------------------------------
+AC_DEFUN([MCA_opal_pmix_ext4x_CONFIG],[
+    AC_CONFIG_FILES([opal/mca/pmix/ext4x/Makefile])
+
+    AS_IF([test "$opal_external_pmix_happy" = "yes"],
+          [ # check for the 4.x version
+           AC_MSG_CHECKING([if external component is version 4.x])
+           AS_IF([test "$opal_external_pmix_version" = "4x"],
+                 [AC_MSG_RESULT([yes])
+                  AS_IF([test "$opal_event_external_support" != "yes"],
+                        [AC_MSG_WARN([EXTERNAL PMIX SUPPORT REQUIRES USE OF EXTERNAL LIBEVENT])
+                         AC_MSG_WARN([AND HWLOC LIBRARIES. THESE LIBRARIES MUST POINT TO THE])
+                         AC_MSG_WARN([SAME ONES USED TO BUILD PMIX OR ELSE UNPREDICTABLE])
+                         AC_MSG_WARN([BEHAVIOR MAY RESULT])
+                         AC_MSG_ERROR([PLEASE CORRECT THE CONFIGURE COMMAND LINE AND REBUILD])])
+                  opal_pmix_external_4x_happy=yes],
+                 [AC_MSG_RESULT([no])
+                  opal_pmix_external_4x_happy=no])
+
+           AS_IF([test "$opal_pmix_external_4x_happy" = "yes"],
+                 [$1
+                  # need to set the wrapper flags for static builds
+                  pmix_ext4x_WRAPPER_EXTRA_LDFLAGS=$opal_external_pmix_LDFLAGS
+                  pmix_ext4x_WRAPPER_EXTRA_LIBS=$opal_external_pmix_LIBS],
+                 [$2])],
+          [$2])
+
+    opal_pmix_ext4x_CPPFLAGS=$opal_external_pmix_CPPFLAGS
+    opal_pmix_ext4x_LDFLAGS=$opal_external_pmix_LDFLAGS
+    opal_pmix_ext4x_LIBS=$opal_external_pmix_LIBS
+
+    AC_SUBST([opal_pmix_ext4x_CPPFLAGS])
+    AC_SUBST([opal_pmix_ext4x_LDFLAGS])
+    AC_SUBST([opal_pmix_ext4x_LIBS])
+
+])dnl

opal/mca/pmix/ext4x/ext4x.h

@@ -0,0 +1,351 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2015 Mellanox Technologies, Inc.
+ *                         All rights reserved.
+ * Copyright (c) 2016      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2017      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef MCA_PMIX_PMIX2X_H
+#define MCA_PMIX_PMIX2X_H
+
+#include "opal_config.h"
+
+#ifdef HAVE_SYS_SOCKET_H
+#include <sys/socket.h>
+#endif
+#ifdef HAVE_SYS_UN_H
+#include <sys/un.h>
+#endif
+
+#include "opal/class/opal_list.h"
+#include "opal/mca/mca.h"
+#include "opal/mca/event/event.h"
+#include "opal/util/proc.h"
+
+#include "opal/mca/pmix/base/base.h"
+#include "pmix_server.h"
+#include "pmix_common.h"
+
+BEGIN_C_DECLS
+
+typedef struct {
+    opal_pmix_base_component_t super;
+    pmix_proc_t myproc;
+    opal_list_t jobids;
+    bool native_launch;
+    size_t evindex;
+    opal_list_t events;
+    int cache_size;
+    opal_list_t cache;
+    opal_list_t dmdx;
+    bool silence_warning;
+} mca_pmix_ext4x_component_t;
+
+OPAL_DECLSPEC extern mca_pmix_ext4x_component_t mca_pmix_ext4x_component;
+
+OPAL_DECLSPEC extern const opal_pmix_base_module_t opal_pmix_ext4x_module;
+
+/****  INTERNAL OBJECTS  ****/
+typedef struct {
+    opal_list_item_t super;
+    opal_jobid_t jobid;
+    char nspace[PMIX_MAX_NSLEN + 1];
+} opal_ext4x_jobid_trkr_t;
+OBJ_CLASS_DECLARATION(opal_ext4x_jobid_trkr_t);
+
+typedef struct {
+    opal_list_item_t super;
+    opal_pmix_lock_t lock;
+    size_t index;
+    opal_pmix_notification_fn_t handler;
+    void *cbdata;
+} opal_ext4x_event_t;
+OBJ_CLASS_DECLARATION(opal_ext4x_event_t);
+
+typedef struct {
+    opal_list_item_t super;
+    char *nspace;
+    pmix_modex_cbfunc_t cbfunc;
+    void *cbdata;
+} opal_ext4x_dmx_trkr_t;
+OBJ_CLASS_DECLARATION(opal_ext4x_dmx_trkr_t);
+
+typedef struct {
+    opal_object_t super;
+    opal_event_t ev;
+    pmix_status_t status;
+    char *nspace;
+    pmix_proc_t p;
+    pmix_proc_t *procs;
+    size_t nprocs;
+    pmix_pdata_t *pdata;
+    size_t npdata;
+    pmix_proc_t *error_procs;
+    size_t nerror_procs;
+    pmix_info_t *info;
+    size_t ninfo;
+    pmix_app_t *apps;
+    size_t sz;
+    opal_pmix_lock_t lock;
+    opal_list_t *codes;
+    pmix_status_t *pcodes;
+    size_t ncodes;
+    pmix_query_t *queries;
+    size_t nqueries;
+    opal_ext4x_event_t *event;
+    opal_pmix_op_cbfunc_t opcbfunc;
+    opal_pmix_modex_cbfunc_t mdxcbfunc;
+    opal_pmix_value_cbfunc_t valcbfunc;
+    opal_pmix_lookup_cbfunc_t lkcbfunc;
+    opal_pmix_spawn_cbfunc_t spcbfunc;
+    opal_pmix_evhandler_reg_cbfunc_t evregcbfunc;
+    opal_pmix_info_cbfunc_t qcbfunc;
+    opal_pmix_setup_application_cbfunc_t setupcbfunc;
+    void *cbdata;
+} ext4x_opcaddy_t;
+OBJ_CLASS_DECLARATION(ext4x_opcaddy_t);
+
+typedef struct {
+    opal_object_t super;
+    opal_list_t procs;
+    opal_list_t info;
+    opal_list_t apps;
+    pmix_op_cbfunc_t opcbfunc;
+    pmix_dmodex_response_fn_t dmdxfunc;
+    pmix_modex_cbfunc_t mdxcbfunc;
+    pmix_lookup_cbfunc_t lkupcbfunc;
+    pmix_spawn_cbfunc_t spwncbfunc;
+    pmix_info_cbfunc_t infocbfunc;
+    pmix_tool_connection_cbfunc_t toolcbfunc;
+    void *cbdata;
+    opal_pmix_release_cbfunc_t odmdxfunc;
+    void *ocbdata;
+} ext4x_opalcaddy_t;
+OBJ_CLASS_DECLARATION(ext4x_opalcaddy_t);
+
+typedef struct {
+    opal_object_t super;
+    opal_event_t ev;
+    opal_pmix_lock_t lock;
+    const char *msg;
+    char *strings;
+    size_t id;
+    int status;
+    opal_process_name_t pname;
+    opal_jobid_t jobid;
+    const opal_process_name_t *source;
+    opal_pmix_data_range_t range;
+    bool nondefault;
+    size_t handler;
+    opal_value_t *val;
+    opal_list_t *event_codes;
+    opal_list_t *info;
+    opal_list_t results;
+    opal_pmix_notification_fn_t evhandler;
+    opal_pmix_evhandler_reg_cbfunc_t cbfunc;
+    opal_pmix_op_cbfunc_t opcbfunc;
+    pmix_event_notification_cbfunc_fn_t pmixcbfunc;
+    opal_pmix_value_cbfunc_t valcbfunc;
+    opal_pmix_lookup_cbfunc_t lkcbfunc;
+    void *cbdata;
+} ext4x_threadshift_t;
+OBJ_CLASS_DECLARATION(ext4x_threadshift_t);
+
+#define OPAL_PMIX_OP_THREADSHIFT(e, fn, cb, cd)             \
+    do {                                                    \
+        ext4x_threadshift_t *_cd;                          \
+        _cd = OBJ_NEW(ext4x_threadshift_t);                \
+        _cd->handler = (e);                                 \
+        _cd->opcbfunc = (cb);                               \
+        _cd->cbdata = (cd);                                 \
+        opal_event_assign(&((_cd)->ev), opal_pmix_base.evbase,   \
+                          -1, EV_WRITE, (fn), (_cd));            \
+        OPAL_POST_OBJECT(_cd);                              \
+        opal_event_active(&((_cd)->ev), EV_WRITE, 1);            \
+    } while(0)
+
+#define OPAL_PMIX_THREADSHIFT(e, i, eh, fn, cb, cd)         \
+    do {                                                    \
+        ext4x_threadshift_t *_cd;                          \
+        _cd = OBJ_NEW(ext4x_threadshift_t);                \
+        _cd->event_codes = (e);                             \
+        _cd->info = (i);                                    \
+        _cd->evhandler = (eh);                              \
+        _cd->cbfunc = (cb);                                 \
+        _cd->cbdata = (cd);                                 \
+        opal_event_assign(&((_cd)->ev), opal_pmix_base.evbase,  \
+                          -1, EV_WRITE, (fn), (_cd));           \
+        OPAL_POST_OBJECT(_cd);                              \
+        opal_event_active(&((_cd)->ev), EV_WRITE, 1);           \
+    } while(0)
+
+#define OPAL_PMIX_NOTIFY_THREADSHIFT(s, sr, r, i, fn, cb, cd)   \
+    do {                                                        \
+        ext4x_threadshift_t *_cd;                              \
+        _cd = OBJ_NEW(ext4x_threadshift_t);                    \
+        _cd->status = (s);                                      \
+        _cd->source = (sr);                                     \
+        _cd->range = (r);                                       \
+        _cd->info = (i);                                        \
+        _cd->opcbfunc = (cb);                                   \
+        _cd->cbdata = (cd);                                     \
+        opal_event_assign(&((_cd)->ev), opal_pmix_base.evbase,  \
+                     -1, EV_WRITE, (fn), (_cd));                \
+        OPAL_POST_OBJECT(_cd);                                  \
+        opal_event_active(&((_cd)->ev), EV_WRITE, 1);           \
+    } while(0)
+
+#define OPAL_PMIX2X_THREADSHIFT(p, cb)                          \
+    do {                                                        \
+        opal_event_assign(&((p)->ev), opal_pmix_base.evbase,    \
+                          -1, EV_WRITE, (cb), (p));             \
+        OPAL_POST_OBJECT(p);                                    \
+        opal_event_active(&((p)->ev), EV_WRITE, 1);             \
+    } while(0)
+
+/****  CLIENT FUNCTIONS  ****/
+OPAL_MODULE_DECLSPEC int ext4x_client_init(opal_list_t *ilist);
+OPAL_MODULE_DECLSPEC int ext4x_client_finalize(void);
+OPAL_MODULE_DECLSPEC int ext4x_initialized(void);
+OPAL_MODULE_DECLSPEC int ext4x_abort(int flag, const char *msg,
+                                     opal_list_t *procs);
+OPAL_MODULE_DECLSPEC int ext4x_commit(void);
+OPAL_MODULE_DECLSPEC int ext4x_fence(opal_list_t *procs, int collect_data);
+OPAL_MODULE_DECLSPEC int ext4x_fencenb(opal_list_t *procs, int collect_data,
+                                       opal_pmix_op_cbfunc_t cbfunc, void *cbdata);
+OPAL_MODULE_DECLSPEC int ext4x_put(opal_pmix_scope_t scope,
+                                     opal_value_t *val);
+OPAL_MODULE_DECLSPEC int ext4x_get(const opal_process_name_t *proc, const char *key,
+                                   opal_list_t *info, opal_value_t **val);
+OPAL_MODULE_DECLSPEC int ext4x_getnb(const opal_process_name_t *proc, const char *key,
+                                     opal_list_t *info,
+                                     opal_pmix_value_cbfunc_t cbfunc, void *cbdata);
+OPAL_MODULE_DECLSPEC int ext4x_publish(opal_list_t *info);
+OPAL_MODULE_DECLSPEC int ext4x_publishnb(opal_list_t *info,
+                                         opal_pmix_op_cbfunc_t cbfunc, void *cbdata);
+OPAL_MODULE_DECLSPEC int ext4x_lookup(opal_list_t *data, opal_list_t *info);
+OPAL_MODULE_DECLSPEC int ext4x_lookupnb(char **keys, opal_list_t *info,
+                                        opal_pmix_lookup_cbfunc_t cbfunc, void *cbdata);
+OPAL_MODULE_DECLSPEC int ext4x_unpublish(char **keys, opal_list_t *info);
+OPAL_MODULE_DECLSPEC int ext4x_unpublishnb(char **keys, opal_list_t *info,
+                                           opal_pmix_op_cbfunc_t cbfunc, void *cbdata);
+OPAL_MODULE_DECLSPEC int ext4x_spawn(opal_list_t *job_info, opal_list_t *apps, opal_jobid_t *jobid);
+OPAL_MODULE_DECLSPEC int ext4x_spawnnb(opal_list_t *job_info, opal_list_t *apps,
+                                       opal_pmix_spawn_cbfunc_t cbfunc, void *cbdata);
+OPAL_MODULE_DECLSPEC int ext4x_connect(opal_list_t *procs);
+OPAL_MODULE_DECLSPEC int ext4x_connectnb(opal_list_t *procs,
+                                         opal_pmix_op_cbfunc_t cbfunc,
+                                         void *cbdata);
+OPAL_MODULE_DECLSPEC int ext4x_disconnect(opal_list_t *procs);
+OPAL_MODULE_DECLSPEC int ext4x_disconnectnb(opal_list_t *procs,
+                                            opal_pmix_op_cbfunc_t cbfunc,
+                                            void *cbdata);
+OPAL_MODULE_DECLSPEC int ext4x_resolve_peers(const char *nodename, opal_jobid_t jobid,
+                                             opal_list_t *procs);
+OPAL_MODULE_DECLSPEC int ext4x_resolve_nodes(opal_jobid_t jobid, char **nodelist);
+OPAL_MODULE_DECLSPEC int ext4x_allocate(opal_pmix_alloc_directive_t directive,
+                                         opal_list_t *info,
+                                         opal_pmix_info_cbfunc_t cbfunc, void *cbdata);
+OPAL_MODULE_DECLSPEC int ext4x_job_control(opal_list_t *targets,
+                                            opal_list_t *directives,
+                                            opal_pmix_info_cbfunc_t cbfunc, void *cbdata);
+
+/****  TOOL FUNCTIONS  ****/
+OPAL_MODULE_DECLSPEC int ext4x_tool_init(opal_list_t *info);
+OPAL_MODULE_DECLSPEC int ext4x_tool_fini(void);
+
+/****  COMMON FUNCTIONS  ****/
+OPAL_MODULE_DECLSPEC int ext4x_store_local(const opal_process_name_t *proc,
+                                             opal_value_t *val);
+
+/****  SERVER SOUTHBOUND FUNCTIONS  ****/
+OPAL_MODULE_DECLSPEC int ext4x_server_init(opal_pmix_server_module_t *module,
+                                           opal_list_t *info);
+OPAL_MODULE_DECLSPEC int ext4x_server_finalize(void);
+OPAL_MODULE_DECLSPEC int ext4x_server_gen_regex(const char *input, char **regex);
+OPAL_MODULE_DECLSPEC int ext4x_server_gen_ppn(const char *input, char **ppn);
+OPAL_MODULE_DECLSPEC int ext4x_server_register_nspace(opal_jobid_t jobid,
+                                                      int nlocalprocs,
+                                                      opal_list_t *info,
+                                                      opal_pmix_op_cbfunc_t cbfunc,
+                                                      void *cbdata);
+OPAL_MODULE_DECLSPEC void ext4x_server_deregister_nspace(opal_jobid_t jobid,
+                                                          opal_pmix_op_cbfunc_t cbfunc,
+                                                          void *cbdata);
+OPAL_MODULE_DECLSPEC int ext4x_server_register_client(const opal_process_name_t *proc,
+                                                      uid_t uid, gid_t gid,
+                                                      void *server_object,
+                                                      opal_pmix_op_cbfunc_t cbfunc,
+                                                      void *cbdata);
+OPAL_MODULE_DECLSPEC void ext4x_server_deregister_client(const opal_process_name_t *proc,
+                                                          opal_pmix_op_cbfunc_t cbfunc,
+                                                          void *cbdata);
+OPAL_MODULE_DECLSPEC int ext4x_server_setup_fork(const opal_process_name_t *proc, char ***env);
+OPAL_MODULE_DECLSPEC int ext4x_server_dmodex(const opal_process_name_t *proc,
+                                             opal_pmix_modex_cbfunc_t cbfunc, void *cbdata);
+OPAL_MODULE_DECLSPEC int ext4x_server_notify_event(int status,
+                                                    const opal_process_name_t *source,
+                                                    opal_list_t *info,
+                                                    opal_pmix_op_cbfunc_t cbfunc, void *cbdata);
+
+OPAL_MODULE_DECLSPEC int ext4x_server_iof_push(const opal_process_name_t *source,
+                                                opal_pmix_iof_channel_t channel,
+                                                unsigned char *data, size_t nbytes);
+
+OPAL_MODULE_DECLSPEC int ext4x_server_setup_application(opal_jobid_t jobid,
+                                                         opal_list_t *info,
+                                                         opal_pmix_setup_application_cbfunc_t cbfunc, void *cbdata);
+OPAL_MODULE_DECLSPEC int ext4x_server_setup_local_support(opal_jobid_t jobid,
+                                                           opal_list_t *info,
+                                                           opal_pmix_op_cbfunc_t cbfunc, void *cbdata);
+
+/****  COMPONENT UTILITY FUNCTIONS  ****/
+OPAL_MODULE_DECLSPEC int opal_pmix_ext4x_check_evars(void);
+
+OPAL_MODULE_DECLSPEC void ext4x_event_hdlr(size_t evhdlr_registration_id,
+                                            pmix_status_t status, const pmix_proc_t *source,
+                                            pmix_info_t info[], size_t ninfo,
+                                            pmix_info_t results[], size_t nresults,
+                                            pmix_event_notification_cbfunc_fn_t cbfunc,
+                                            void *cbdata);
+OPAL_MODULE_DECLSPEC pmix_status_t ext4x_convert_opalrc(int rc);
+OPAL_MODULE_DECLSPEC int ext4x_convert_rc(pmix_status_t rc);
+
+OPAL_MODULE_DECLSPEC opal_vpid_t ext4x_convert_rank(pmix_rank_t rank);
+OPAL_MODULE_DECLSPEC pmix_rank_t ext4x_convert_opalrank(opal_vpid_t vpid);
+
+OPAL_MODULE_DECLSPEC opal_pmix_scope_t ext4x_convert_scope(pmix_scope_t scope);
+OPAL_MODULE_DECLSPEC pmix_scope_t ext4x_convert_opalscope(opal_pmix_scope_t scope);
+
+OPAL_MODULE_DECLSPEC pmix_data_range_t ext4x_convert_opalrange(opal_pmix_data_range_t range);
+OPAL_MODULE_DECLSPEC opal_pmix_data_range_t ext4x_convert_range(pmix_data_range_t range);
+
+OPAL_MODULE_DECLSPEC opal_pmix_persistence_t ext4x_convert_persist(pmix_persistence_t scope);
+OPAL_MODULE_DECLSPEC pmix_persistence_t ext4x_convert_opalpersist(opal_pmix_persistence_t scope);
+
+OPAL_MODULE_DECLSPEC void ext4x_value_load(pmix_value_t *v,
+                                            opal_value_t *kv);
+OPAL_MODULE_DECLSPEC int ext4x_value_unload(opal_value_t *kv,
+                                            const pmix_value_t *v);
+
+OPAL_MODULE_DECLSPEC opal_pmix_alloc_directive_t ext4x_convert_allocdir(pmix_alloc_directive_t dir);
+
+OPAL_MODULE_DECLSPEC char* ext4x_convert_jobid(opal_jobid_t jobid);
+
+OPAL_MODULE_DECLSPEC int ext4x_convert_state(pmix_proc_state_t state);
+
+OPAL_MODULE_DECLSPEC pmix_proc_state_t ext4x_convert_opalstate(int state);
+
+
+END_C_DECLS
+
+#endif /* MCA_PMIX_EXTERNAL_H */

opal/mca/pmix/ext4x/ext4x_component.c

@@ -0,0 +1,153 @@
+/*
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2015 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016-2018 Cisco Systems, Inc.  All rights reserved
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ * These symbols are in a file by themselves to provide nice linker
+ * semantics.  Since linkers generally pull in symbols by object
+ * files, keeping these symbols as the only symbols in this file
+ * prevents utility programs such as "ompi_info" from having to import
+ * entire components just to query their version and parameters.
+ */
+
+#include "opal_config.h"
+
+#include "opal/constants.h"
+#include "opal/class/opal_list.h"
+#include "opal/util/proc.h"
+#include "opal/util/show_help.h"
+#include "opal/mca/pmix/pmix.h"
+#include "ext4x.h"
+
+/*
+ * Public string showing the pmix external component version number
+ */
+const char *opal_pmix_ext4x_component_version_string =
+    "OPAL ext4x MCA component version " OPAL_VERSION;
+
+/*
+ * Local function
+ */
+static int external_register(void);
+static int external_open(void);
+static int external_close(void);
+static int external_component_query(mca_base_module_t **module, int *priority);
+
+/*
+ * Local variable
+ */
+static char *pmix_library_version = NULL;
+
+
+/*
+ * Instantiate the public struct with all of our public information
+ * and pointers to our public functions in it
+ */
+
+mca_pmix_ext4x_component_t mca_pmix_ext4x_component = {
+    {
+    /* First, the mca_component_t struct containing meta information
+       about the component itself */
+
+        .base_version = {
+        /* Indicate that we are a pmix v1.1.0 component (which also
+           implies a specific MCA version) */
+
+            OPAL_PMIX_BASE_VERSION_2_0_0,
+
+        /* Component name and version */
+
+            .mca_component_name = "ext4x",
+            MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION,
+                                  OPAL_RELEASE_VERSION),
+
+        /* Component open and close functions */
+
+            .mca_open_component = external_open,
+            .mca_close_component = external_close,
+            .mca_query_component = external_component_query,
+            .mca_register_component_params = external_register
+        },
+        /* Next the MCA v1.0.0 component meta data */
+        .base_data = {
+        /* The component is checkpoint ready */
+            MCA_BASE_METADATA_PARAM_CHECKPOINT
+        }
+    },
+    .native_launch = false
+};
+
+static int external_register(void)
+{
+    mca_base_component_t *component = &mca_pmix_ext4x_component.super.base_version;
+
+    mca_pmix_ext4x_component.silence_warning = false;
+    (void) mca_base_component_var_register (component, "silence_warning",
+                                            "Silence warning about PMIX_INSTALL_PREFIX",
+                                            MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
+                                            OPAL_INFO_LVL_4,
+                                            MCA_BASE_VAR_SCOPE_READONLY,
+                                            &mca_pmix_ext4x_component.silence_warning);
+
+    asprintf(&pmix_library_version,
+             "PMIx library version %s (embedded in Open MPI)", PMIx_Get_version());
+    (void) mca_base_component_var_register(component, "library_version",
+                                           "Version of the underlying PMIx library",
+                                           MCA_BASE_VAR_TYPE_STRING,
+                                           NULL, 0, 0,
+                                           OPAL_INFO_LVL_4,
+                                           MCA_BASE_VAR_SCOPE_CONSTANT,
+                                           &pmix_library_version);
+
+    return OPAL_SUCCESS;
+}
+
+static int external_open(void)
+{
+    const char *version;
+
+    mca_pmix_ext4x_component.evindex = 0;
+    OBJ_CONSTRUCT(&mca_pmix_ext4x_component.jobids, opal_list_t);
+    OBJ_CONSTRUCT(&mca_pmix_ext4x_component.events, opal_list_t);
+    OBJ_CONSTRUCT(&mca_pmix_ext4x_component.dmdx, opal_list_t);
+
+    version = PMIx_Get_version();
+    if ('3' != version[0]) {
+        opal_show_help("help-pmix-base.txt",
+                       "incorrect-pmix", true, version, "v3.x");
+        return OPAL_ERROR;
+    }
+    return OPAL_SUCCESS;
+}
+
+static int external_close(void)
+{
+    OPAL_LIST_DESTRUCT(&mca_pmix_ext4x_component.jobids);
+    OPAL_LIST_DESTRUCT(&mca_pmix_ext4x_component.events);
+    OPAL_LIST_DESTRUCT(&mca_pmix_ext4x_component.dmdx);
+    return OPAL_SUCCESS;
+}
+
+
+static int external_component_query(mca_base_module_t **module, int *priority)
+{
+    char *t, *id;
+
+    /* see if a PMIx server is present */
+    if (NULL != (t = getenv("PMIX_SERVER_URI")) ||
+        NULL != (id = getenv("PMIX_ID"))) {
+        /* if PMIx is present, then we are a client and need to use it */
+        *priority = 100;
+    } else {
+        /* we could be a server, so we still need to be considered */
+        *priority = 5;
+    }
+    *module = (mca_base_module_t *)&opal_pmix_ext4x_module;
+    return OPAL_SUCCESS;
+}

opal/mca/pmix/ext4x/ext4x_local.c

@@ -0,0 +1,27 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2017 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2014-2015 Mellanox Technologies, Inc.
+ *                         All rights reserved.
+ * Copyright (c) 2016      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2017      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "opal_config.h"
+#include "opal/constants.h"
+
+#include "ext4x.h"
+
+int opal_pmix_ext4x_check_evars(void)
+{
+    /* a dummy function */
+    return OPAL_SUCCESS;
+}

opal/mca/pmix/ext4x/ext4x_server_south.c

@@ -0,0 +1,790 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2017 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2014-2016 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014-2017 Mellanox Technologies, Inc.
+ *                         All rights reserved.
+ * Copyright (c) 2016      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2017      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "opal_config.h"
+#include "opal/constants.h"
+#include "opal/types.h"
+
+#ifdef HAVE_STRING_H
+#include <string.h>
+#endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+#include "opal/dss/dss.h"
+#include "opal/mca/event/event.h"
+#include "opal/mca/hwloc/base/base.h"
+#include "opal/runtime/opal.h"
+#include "opal/runtime/opal_progress_threads.h"
+#include "opal/threads/threads.h"
+#include "opal/util/argv.h"
+#include "opal/util/error.h"
+#include "opal/util/output.h"
+#include "opal/util/opal_environ.h"
+#include "opal/util/proc.h"
+#include "opal/util/show_help.h"
+#include "opal/mca/pmix/base/base.h"
+#include "ext4x.h"
+
+#include "pmix.h"
+#include "pmix_server.h"
+
+/****    S.O.U.T.H.B.O.U.N.D   I.N.T.E.R.F.A.C.E.S     ****/
+
+/* These are the interfaces used by the OMPI/ORTE/OPAL layer to call
+ * down into the embedded PMIx server. */
+
+extern pmix_server_module_t mymodule;
+extern opal_pmix_server_module_t *host_module;
+static char *dbgvalue=NULL;
+
+static void errreg_cbfunc (pmix_status_t status,
+                          size_t errhandler_ref,
+                          void *cbdata)
+{
+    opal_ext4x_event_t *ev = (opal_ext4x_event_t*)cbdata;
+
+    OPAL_ACQUIRE_OBJECT(ev);
+    ev->index = errhandler_ref;
+    opal_output_verbose(5, opal_pmix_base_framework.framework_output,
+                        "PMIX server errreg_cbfunc - error handler registered status=%d, reference=%lu",
+                         status, (unsigned long)errhandler_ref);
+    OPAL_POST_OBJECT(ev);
+    OPAL_PMIX_WAKEUP_THREAD(&ev->lock);
+}
+
+static void opcbfunc(pmix_status_t status, void *cbdata)
+{
+    ext4x_opcaddy_t *op = (ext4x_opcaddy_t*)cbdata;
+
+    OPAL_ACQUIRE_OBJECT(op);
+
+    if (NULL != op->opcbfunc) {
+        op->opcbfunc(ext4x_convert_rc(status), op->cbdata);
+    }
+    OBJ_RELEASE(op);
+}
+
+static void lkcbfunc(pmix_status_t status, void *cbdata)
+{
+    opal_pmix_lock_t *lk = (opal_pmix_lock_t*)cbdata;
+
+    OPAL_POST_OBJECT(lk);
+    lk->status = ext4x_convert_rc(status);
+    OPAL_PMIX_WAKEUP_THREAD(lk);
+}
+
+int ext4x_server_init(opal_pmix_server_module_t *module,
+                      opal_list_t *info)
+{
+    pmix_status_t rc;
+    int dbg;
+    opal_value_t *kv;
+    pmix_info_t *pinfo;
+    size_t sz, n;
+    opal_ext4x_event_t *event;
+    opal_ext4x_jobid_trkr_t *job;
+    opal_pmix_lock_t lk;
+
+    OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock);
+
+    if (0 == opal_pmix_base.initialized) {
+        if (0 < (dbg = opal_output_get_verbosity(opal_pmix_base_framework.framework_output))) {
+            asprintf(&dbgvalue, "PMIX_DEBUG=%d", dbg);
+            putenv(dbgvalue);
+        }
+        /* check the evars for a mismatch */
+        if (OPAL_SUCCESS != (dbg = opal_pmix_ext4x_check_evars())) {
+            OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
+            return dbg;
+        }
+    }
+    ++opal_pmix_base.initialized;
+
+    /* convert the list to an array of pmix_info_t */
+    sz = 2 + ((NULL==info)?0:opal_list_get_size(info));
+    PMIX_INFO_CREATE(pinfo, sz);
+    n = 0;
+    if (NULL != info) {
+        OPAL_LIST_FOREACH(kv, info, opal_value_t) {
+            (void)strncpy(pinfo[n].key, kv->key, PMIX_MAX_KEYLEN);
+            ext4x_value_load(&pinfo[n].value, kv);
+            ++n;
+        }
+    }
+
+    /* check for direct modex use-case */
+    if (opal_pmix_base_async_modex && !opal_pmix_collect_all_data) {
+        opal_setenv("PMIX_MCA_gds", "hash", true, &environ);
+    }
+
+    /* insert ourselves into our list of jobids - it will be the
+     * first, and so we'll check it first */
+    job = OBJ_NEW(opal_ext4x_jobid_trkr_t);
+    (void)opal_snprintf_jobid(job->nspace, PMIX_MAX_NSLEN, OPAL_PROC_MY_NAME.jobid);
+    job->jobid = OPAL_PROC_MY_NAME.jobid;
+    opal_list_append(&mca_pmix_ext4x_component.jobids, &job->super);
+    OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
+
+    /* add our nspace and rank to the array going down to the PMIx server */
+    PMIX_INFO_LOAD(&pinfo[sz-2], PMIX_SERVER_NSPACE, job->nspace, PMIX_STRING);
+    PMIX_INFO_LOAD(&pinfo[sz-1], PMIX_SERVER_RANK, &OPAL_PROC_MY_NAME.vpid, PMIX_PROC_RANK);
+    if (PMIX_SUCCESS != (rc = PMIx_server_init(&mymodule, pinfo, sz))) {
+        PMIX_INFO_FREE(pinfo, sz);
+        return ext4x_convert_rc(rc);
+    }
+    PMIX_INFO_FREE(pinfo, sz);
+
+    /* record the host module */
+    host_module = module;
+
+    /* register the default event handler */
+    event = OBJ_NEW(opal_ext4x_event_t);
+    opal_list_append(&mca_pmix_ext4x_component.events, &event->super);
+    PMIX_INFO_CREATE(pinfo, 1);
+    PMIX_INFO_LOAD(&pinfo[0], PMIX_EVENT_HDLR_NAME, "OPAL-PMIX-2X-SERVER-DEFAULT", PMIX_STRING);
+    PMIx_Register_event_handler(NULL, 0, pinfo, 1, ext4x_event_hdlr, errreg_cbfunc, (void*)event);
+    OPAL_PMIX_WAIT_THREAD(&event->lock);
+    PMIX_INFO_FREE(pinfo, 1);
+
+    /* as we might want to use some client-side functions, be sure
+     * to register our own nspace */
+    OPAL_PMIX_CONSTRUCT_LOCK(&lk);
+    PMIX_INFO_CREATE(pinfo, 1);
+    PMIX_INFO_LOAD(&pinfo[0], PMIX_REGISTER_NODATA, NULL, PMIX_BOOL);
+    PMIx_server_register_nspace(job->nspace, 1, pinfo, 1, lkcbfunc, (void*)&lk);
+    OPAL_PMIX_WAIT_THREAD(&lk);
+    OPAL_PMIX_DESTRUCT_LOCK(&lk);
+    PMIX_INFO_FREE(pinfo, 1);
+
+    return OPAL_SUCCESS;
+}
+
+static void dereg_cbfunc(pmix_status_t st, void *cbdata)
+{
+    opal_ext4x_event_t *ev = (opal_ext4x_event_t*)cbdata;
+    OPAL_PMIX_WAKEUP_THREAD(&ev->lock);
+}
+
+int ext4x_server_finalize(void)
+{
+    pmix_status_t rc;
+    opal_ext4x_event_t *event, *ev2;
+
+    OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock);
+    --opal_pmix_base.initialized;
+
+    if (0 < opal_pmix_base.initialized) {
+        /* deregister all event handlers */
+        OPAL_LIST_FOREACH_SAFE(event, ev2, &mca_pmix_ext4x_component.events, opal_ext4x_event_t) {
+            OPAL_PMIX_DESTRUCT_LOCK(&event->lock);
+            OPAL_PMIX_CONSTRUCT_LOCK(&event->lock);
+            PMIx_Deregister_event_handler(event->index, dereg_cbfunc, (void*)event);
+            OPAL_PMIX_WAIT_THREAD(&event->lock);
+            opal_list_remove_item(&mca_pmix_ext4x_component.events, &event->super);
+            OBJ_RELEASE(event);
+        }
+    }
+    OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
+    rc = PMIx_server_finalize();
+    return ext4x_convert_rc(rc);
+}
+
+int ext4x_server_gen_regex(const char *input, char **regex)
+{
+    pmix_status_t rc;
+
+    OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock);
+    if (0 >= opal_pmix_base.initialized) {
+        OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
+        return OPAL_ERR_NOT_INITIALIZED;
+    }
+    OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
+
+    rc = PMIx_generate_regex(input, regex);
+    return ext4x_convert_rc(rc);
+}
+
+
+int ext4x_server_gen_ppn(const char *input, char **ppn)
+{
+    pmix_status_t rc;
+
+    OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock);
+    if (0 >= opal_pmix_base.initialized) {
+        OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
+        return OPAL_ERR_NOT_INITIALIZED;
+    }
+    OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
+
+    rc = PMIx_generate_ppn(input, ppn);
+    return ext4x_convert_rc(rc);
+}
+
+int ext4x_server_register_nspace(opal_jobid_t jobid,
+                                 int nlocalprocs,
+                                 opal_list_t *info,
+                                 opal_pmix_op_cbfunc_t cbfunc,
+                                 void *cbdata)
+{
+    opal_value_t *kv, *k2;
+    pmix_info_t *pinfo = NULL, *pmap;
+    size_t sz, szmap, m, n;
+    char nspace[PMIX_MAX_NSLEN];
+    pmix_status_t rc;
+    opal_list_t *pmapinfo;
+    opal_ext4x_jobid_trkr_t *job;
+    opal_pmix_lock_t lock;
+    int ret;
+
+    OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock);
+    if (0 >= opal_pmix_base.initialized) {
+        OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
+        return OPAL_ERR_NOT_INITIALIZED;
+    }
+
+    /* convert the jobid */
+    (void)opal_snprintf_jobid(nspace, PMIX_MAX_NSLEN, jobid);
+
+    /* store this job in our list of known nspaces */
+    job = OBJ_NEW(opal_ext4x_jobid_trkr_t);
+    (void)strncpy(job->nspace, nspace, PMIX_MAX_NSLEN);
+    job->jobid = jobid;
+    opal_list_append(&mca_pmix_ext4x_component.jobids, &job->super);
+    OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
+
+    /* convert the list to an array of pmix_info_t */
+    if (NULL != info && 0 < (sz = opal_list_get_size(info))) {
+        PMIX_INFO_CREATE(pinfo, sz);
+        n = 0;
+        OPAL_LIST_FOREACH(kv, info, opal_value_t) {
+            (void)strncpy(pinfo[n].key, kv->key, PMIX_MAX_KEYLEN);
+            if (0 == strcmp(kv->key, OPAL_PMIX_PROC_DATA)) {
+                pinfo[n].value.type = PMIX_DATA_ARRAY;
+                /* the value contains a list of values - convert
+                 * that list to another array */
+                pmapinfo = (opal_list_t*)kv->data.ptr;
+                szmap = opal_list_get_size(pmapinfo);
+                if (0 < szmap) {
+                    PMIX_INFO_CREATE(pmap, szmap);
+                    pinfo[n].value.data.darray = (pmix_data_array_t*)calloc(1, sizeof(pmix_data_array_t));
+                    pinfo[n].value.data.darray->type = PMIX_INFO;
+                    pinfo[n].value.data.darray->array = (struct pmix_info_t*)pmap;
+                    pinfo[n].value.data.darray->size = szmap;
+                    m = 0;
+                    OPAL_LIST_FOREACH(k2, pmapinfo, opal_value_t) {
+                        (void)strncpy(pmap[m].key, k2->key, PMIX_MAX_KEYLEN);
+                        ext4x_value_load(&pmap[m].value, k2);
+                        ++m;
+                    }
+                }
+                OPAL_LIST_RELEASE(pmapinfo);
+            } else {
+                ext4x_value_load(&pinfo[n].value, kv);
+            }
+            ++n;
+        }
+    } else {
+        sz = 0;
+        pinfo = NULL;
+    }
+
+    OPAL_PMIX_CONSTRUCT_LOCK(&lock);
+    rc = PMIx_server_register_nspace(nspace, nlocalprocs, pinfo, sz,
+                                     lkcbfunc, (void*)&lock);
+    if (PMIX_SUCCESS == rc) {
+        OPAL_PMIX_WAIT_THREAD(&lock);
+    }
+    OPAL_PMIX_DESTRUCT_LOCK(&lock);
+
+    if (NULL != pinfo) {
+        PMIX_INFO_FREE(pinfo, sz);
+    }
+
+    ret = ext4x_convert_rc(rc);
+
+    /* release the caller */
+    if (NULL != cbfunc) {
+        cbfunc(ret, cbdata);
+    }
+    return ret;
+}
+
+void ext4x_server_deregister_nspace(opal_jobid_t jobid,
+                                     opal_pmix_op_cbfunc_t cbfunc,
+                                     void *cbdata)
+{
+    opal_ext4x_jobid_trkr_t *jptr;
+    opal_pmix_lock_t lock;
+
+    OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock);
+    if (0 >= opal_pmix_base.initialized) {
+        OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
+        /* release the caller */
+        if (NULL != cbfunc) {
+            cbfunc(OPAL_ERR_NOT_INITIALIZED, cbdata);
+        }
+        return;
+    }
+
+    /* if we don't already have it, we can ignore this */
+    OPAL_LIST_FOREACH(jptr, &mca_pmix_ext4x_component.jobids, opal_ext4x_jobid_trkr_t) {
+        if (jptr->jobid == jobid) {
+            /* found it - tell the server to deregister */
+            OPAL_PMIX_CONSTRUCT_LOCK(&lock);
+            OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
+            PMIx_server_deregister_nspace(jptr->nspace, lkcbfunc, (void*)&lock);
+            OPAL_PMIX_WAIT_THREAD(&lock);
+            OPAL_PMIX_DESTRUCT_LOCK(&lock);
+            /* now get rid of it from our list */
+            OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock);
+            opal_list_remove_item(&mca_pmix_ext4x_component.jobids, &jptr->super);
+            OBJ_RELEASE(jptr);
+            break;
+        }
+    }
+
+    OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
+    /* release the caller */
+    if (NULL != cbfunc) {
+        cbfunc(OPAL_SUCCESS, cbdata);
+    }
+}
+
+int ext4x_server_register_client(const opal_process_name_t *proc,
+                                 uid_t uid, gid_t gid,
+                                 void *server_object,
+                                 opal_pmix_op_cbfunc_t cbfunc,
+                                 void *cbdata)
+{
+    pmix_status_t rc;
+    pmix_proc_t p;
+    opal_pmix_lock_t lock;
+
+    OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock);
+    if (0 >= opal_pmix_base.initialized) {
+        OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
+        return OPAL_ERR_NOT_INITIALIZED;
+    }
+    OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
+
+    /* convert the jobid */
+    (void)opal_snprintf_jobid(p.nspace, PMIX_MAX_NSLEN, proc->jobid);
+    p.rank = ext4x_convert_opalrank(proc->vpid);
+
+    OPAL_PMIX_CONSTRUCT_LOCK(&lock);
+    rc = PMIx_server_register_client(&p, uid, gid, server_object,
+                                     lkcbfunc, (void*)&lock);
+    if (PMIX_SUCCESS == rc) {
+        OPAL_PMIX_WAIT_THREAD(&lock);
+    }
+    OPAL_PMIX_DESTRUCT_LOCK(&lock);
+    return ext4x_convert_rc(rc);
+}
+
+/* tell the local PMIx server to cleanup this client as it is
+ * done executing */
+void ext4x_server_deregister_client(const opal_process_name_t *proc,
+                                     opal_pmix_op_cbfunc_t cbfunc,
+                                     void *cbdata)
+{
+    opal_ext4x_jobid_trkr_t *jptr;
+    pmix_proc_t p;
+    opal_pmix_lock_t lock;
+
+    OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock);
+    if (0 >= opal_pmix_base.initialized) {
+        OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
+        if (NULL != cbfunc) {
+            cbfunc(OPAL_ERR_NOT_INITIALIZED, cbdata);
+        }
+        return;
+    }
+
+    /* if we don't already have it, we can ignore this */
+    OPAL_LIST_FOREACH(jptr, &mca_pmix_ext4x_component.jobids, opal_ext4x_jobid_trkr_t) {
+        if (jptr->jobid == proc->jobid) {
+            /* found it - tell the server to deregister */
+            (void)strncpy(p.nspace, jptr->nspace, PMIX_MAX_NSLEN);
+            p.rank = ext4x_convert_opalrank(proc->vpid);
+            OPAL_PMIX_CONSTRUCT_LOCK(&lock);
+            OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
+            PMIx_server_deregister_client(&p, lkcbfunc, (void*)&lock);
+            OPAL_PMIX_WAIT_THREAD(&lock);
+            OPAL_PMIX_DESTRUCT_LOCK(&lock);
+            OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock);
+            break;
+        }
+    }
+    OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
+    if (NULL != cbfunc) {
+        cbfunc(OPAL_SUCCESS, cbdata);
+    }
+}
+
+/* have the local PMIx server setup the environment for this client */
+int ext4x_server_setup_fork(const opal_process_name_t *proc, char ***env)
+{
+    pmix_status_t rc;
+    pmix_proc_t p;
+
+    OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock);
+    if (0 >= opal_pmix_base.initialized) {
+        OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
+        return OPAL_ERR_NOT_INITIALIZED;
+    }
+    OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
+
+    /* convert the jobid */
+    (void)opal_snprintf_jobid(p.nspace, PMIX_MAX_NSLEN, proc->jobid);
+    p.rank = ext4x_convert_opalrank(proc->vpid);
+
+    rc = PMIx_server_setup_fork(&p, env);
+    return ext4x_convert_rc(rc);
+}
+
+/* this is the call back up from the embedded PMIx server that
+ * will contain the returned data. Note that the embedded server
+ * "owns" the data and will free it upon return from this function */
+static void dmdx_response(pmix_status_t status, char *data, size_t sz, void *cbdata)
+{
+    int rc;
+    ext4x_opcaddy_t *op = (ext4x_opcaddy_t*)cbdata;
+
+    rc = ext4x_convert_rc(status);
+    if (NULL != op->mdxcbfunc) {
+        op->mdxcbfunc(rc, data, sz, op->cbdata, NULL, NULL);
+    }
+    OBJ_RELEASE(op);
+}
+
+/* request modex data for a local proc from the PMIx server */
+int ext4x_server_dmodex(const opal_process_name_t *proc,
+                        opal_pmix_modex_cbfunc_t cbfunc, void *cbdata)
+{
+    ext4x_opcaddy_t *op;
+    pmix_status_t rc;
+
+    OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock);
+    if (0 >= opal_pmix_base.initialized) {
+        OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
+        return OPAL_ERR_NOT_INITIALIZED;
+    }
+    OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
+
+    /* setup the caddy */
+    op = OBJ_NEW(ext4x_opcaddy_t);
+    op->mdxcbfunc = cbfunc;
+    op->cbdata = cbdata;
+
+    /* convert the jobid */
+    (void)opal_snprintf_jobid(op->p.nspace, PMIX_MAX_NSLEN, proc->jobid);
+    op->p.rank = ext4x_convert_opalrank(proc->vpid);
+
+    /* find the internally-cached data for this proc */
+    rc = PMIx_server_dmodex_request(&op->p, dmdx_response, op);
+    if (PMIX_SUCCESS != rc) {
+        OBJ_RELEASE(op);
+    }
+    return ext4x_convert_rc(rc);
+}
+
+/* tell the PMIx server to notify its local clients of an event */
+int ext4x_server_notify_event(int status,
+                               const opal_process_name_t *source,
+                               opal_list_t *info,
+                               opal_pmix_op_cbfunc_t cbfunc, void *cbdata)
+{
+    opal_value_t *kv;
+    pmix_info_t *pinfo;
+    size_t sz, n;
+    pmix_status_t rc;
+    ext4x_opcaddy_t *op;
+
+    OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock);
+    if (0 >= opal_pmix_base.initialized) {
+        OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
+        return OPAL_ERR_NOT_INITIALIZED;
+    }
+    OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
+
+    /* convert the list to an array of pmix_info_t */
+    if (NULL != info && 0 < (sz = opal_list_get_size(info))) {
+        PMIX_INFO_CREATE(pinfo, sz);
+        n = 0;
+        OPAL_LIST_FOREACH(kv, info, opal_value_t) {
+            (void)strncpy(pinfo[n].key, kv->key, PMIX_MAX_KEYLEN);
+            if (0 == strcmp(kv->key, OPAL_PMIX_JOB_TERM_STATUS)) {
+                pinfo[n].value.type = PMIX_STATUS;
+                pinfo[n].value.data.status = ext4x_convert_opalrc(kv->data.integer);
+            } else {
+                ext4x_value_load(&pinfo[n].value, kv);
+            }
+            ++n;
+        }
+    } else {
+        sz = 0;
+        pinfo = NULL;
+    }
+    /* setup the caddy */
+    op = OBJ_NEW(ext4x_opcaddy_t);
+    op->info = pinfo;
+    op->sz = sz;
+    op->opcbfunc = cbfunc;
+    op->cbdata = cbdata;
+    /* convert the jobid */
+    if (NULL == source) {
+        (void)opal_snprintf_jobid(op->p.nspace, PMIX_MAX_NSLEN, OPAL_JOBID_INVALID);
+        op->p.rank = ext4x_convert_opalrank(OPAL_VPID_INVALID);
+    } else {
+        (void)opal_snprintf_jobid(op->p.nspace, PMIX_MAX_NSLEN, source->jobid);
+        op->p.rank = ext4x_convert_opalrank(source->vpid);
+    }
+
+
+    rc = ext4x_convert_opalrc(status);
+    /* the range must be nonlocal so the server will pass
+     * the event down to its local clients */
+    rc = PMIx_Notify_event(rc, &op->p, PMIX_RANGE_SESSION,
+                           pinfo, sz, opcbfunc, op);
+    if (PMIX_SUCCESS != rc) {
+        OBJ_RELEASE(op);
+    }
+    return ext4x_convert_rc(rc);
+}
+
+int ext4x_server_iof_push(const opal_process_name_t *source,
+                           opal_pmix_iof_channel_t channel,
+                           unsigned char *data, size_t nbytes)
+{
+    ext4x_opcaddy_t *op;
+    pmix_byte_object_t bo;
+    pmix_iof_channel_t pchan;
+    opal_pmix_lock_t lock;
+    pmix_status_t rc;
+    int ret;
+
+    opal_output_verbose(2, opal_pmix_base_framework.framework_output,
+                        "%s IOF push from %s with %d bytes",
+                        OPAL_NAME_PRINT(OPAL_PROC_MY_NAME),
+                        OPAL_NAME_PRINT(*source), (int)nbytes);
+
+    OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock);
+    if (0 >= opal_pmix_base.initialized) {
+        OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
+        return OPAL_ERR_NOT_INITIALIZED;
+    }
+    OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
+
+    /* setup the caddy */
+    op = OBJ_NEW(ext4x_opcaddy_t);
+    /* convert the source */
+    (void)opal_snprintf_jobid(op->p.nspace, PMIX_MAX_NSLEN, source->jobid);
+    op->p.rank = ext4x_convert_opalrank(source->vpid);
+    /* convert the channel */
+    pchan = 0;
+    if (OPAL_PMIX_FWD_STDIN_CHANNEL & channel) {
+        pchan |= PMIX_FWD_STDIN_CHANNEL;
+    }
+    if (OPAL_PMIX_FWD_STDOUT_CHANNEL & channel) {
+        pchan |= PMIX_FWD_STDOUT_CHANNEL;
+    }
+    if (OPAL_PMIX_FWD_STDERR_CHANNEL & channel) {
+        pchan |= PMIX_FWD_STDERR_CHANNEL;
+    }
+    if (OPAL_PMIX_FWD_STDDIAG_CHANNEL & channel) {
+        pchan |= PMIX_FWD_STDDIAG_CHANNEL;
+    }
+
+    /* setup the byte object */
+    PMIX_BYTE_OBJECT_CONSTRUCT(&bo);
+    if (0 < nbytes) {
+        bo.bytes = (char*)data;
+    }
+    bo.size = nbytes;
+
+    /* push the IO */
+    OPAL_PMIX_CONSTRUCT_LOCK(&lock);
+    rc = PMIx_server_IOF_deliver(&op->p, pchan, &bo, NULL, 0, lkcbfunc, (void*)&lock);
+    if (PMIX_SUCCESS != rc) {
+        ret = ext4x_convert_rc(rc);
+    } else {
+        /* wait for completion */
+        OPAL_PMIX_WAIT_THREAD(&lock);
+        ret = lock.status;
+        OPAL_PMIX_DESTRUCT_LOCK(&lock);
+    }
+    /* cleanup */
+    OBJ_RELEASE(op);
+
+    return ret;
+}
+
+static void final_cleanup(int status, void *cbdata)
+{
+    ext4x_opalcaddy_t *opalcaddy = (ext4x_opalcaddy_t*)cbdata;
+    OBJ_RELEASE(opalcaddy);
+}
+
+static void setup_cbfunc(pmix_status_t status,
+                         pmix_info_t info[], size_t ninfo,
+                         void *provided_cbdata,
+                         pmix_op_cbfunc_t cbfunc, void *cbdata)
+{
+    ext4x_opcaddy_t *op = (ext4x_opcaddy_t*)provided_cbdata;
+    ext4x_opalcaddy_t *opalcaddy;
+    size_t n;
+    opal_value_t *iptr;
+    int rc;
+    pmix_status_t ret = PMIX_SUCCESS;
+
+    /* setup the caddy */
+    opalcaddy = OBJ_NEW(ext4x_opalcaddy_t);
+
+    rc = ext4x_convert_rc(status);
+    if (OPAL_SUCCESS == rc && NULL != info) {
+        /* need to convert the info array to a list */
+        for (n=0; n < ninfo; n++) {
+            iptr = OBJ_NEW(opal_value_t);
+            opal_list_append(&opalcaddy->info, &iptr->super);
+            iptr->key = strdup(info[n].key);
+            if (OPAL_SUCCESS != (rc = ext4x_value_unload(iptr, &info[n].value))) {
+                OBJ_RELEASE(opalcaddy);
+                ret = ext4x_convert_opalrc(rc);
+                goto done;
+            }
+        }
+    }
+
+  done:
+    /* release our caller */
+    if (NULL != cbfunc) {
+        cbfunc(ret, cbdata);
+    }
+    /* pass what we have upstairs */
+    if (NULL != op->setupcbfunc) {
+        op->setupcbfunc(rc, &opalcaddy->info, op->cbdata,
+                        final_cleanup, opalcaddy);
+    }
+    OBJ_RELEASE(op);
+}
+
+int ext4x_server_setup_application(opal_jobid_t jobid,
+                                    opal_list_t *info,
+                                    opal_pmix_setup_application_cbfunc_t cbfunc, void *cbdata)
+{
+    opal_value_t *kv;
+    pmix_info_t *pinfo;
+    size_t sz, n;
+    pmix_status_t rc;
+    ext4x_opcaddy_t *op;
+
+    opal_output_verbose(2, opal_pmix_base_framework.framework_output,
+                        "%s setup application for job %s",
+                        OPAL_NAME_PRINT(OPAL_PROC_MY_NAME),
+                        OPAL_JOBID_PRINT(jobid));
+
+    OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock);
+    if (0 >= opal_pmix_base.initialized) {
+        OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
+        return OPAL_ERR_NOT_INITIALIZED;
+    }
+    OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
+
+    /* convert the list to an array of pmix_info_t */
+    if (NULL != info && 0 < (sz = opal_list_get_size(info))) {
+        PMIX_INFO_CREATE(pinfo, sz);
+        n = 0;
+        OPAL_LIST_FOREACH(kv, info, opal_value_t) {
+            (void)strncpy(pinfo[n].key, kv->key, PMIX_MAX_KEYLEN);
+            ext4x_value_load(&pinfo[n].value, kv);
+            ++n;
+        }
+    } else {
+        sz = 0;
+        pinfo = NULL;
+    }
+    /* setup the caddy */
+    op = OBJ_NEW(ext4x_opcaddy_t);
+    op->info = pinfo;
+    op->sz = sz;
+    op->setupcbfunc = cbfunc;
+    op->cbdata = cbdata;
+    /* convert the jobid */
+    (void)opal_snprintf_jobid(op->p.nspace, PMIX_MAX_NSLEN, jobid);
+
+    rc = PMIx_server_setup_application(op->p.nspace, op->info, op->sz,
+                                       setup_cbfunc, op);
+    if (PMIX_SUCCESS != rc) {
+        OBJ_RELEASE(op);
+    }
+    return ext4x_convert_rc(rc);
+}
+
+int ext4x_server_setup_local_support(opal_jobid_t jobid,
+                                      opal_list_t *info,
+                                      opal_pmix_op_cbfunc_t cbfunc, void *cbdata)
+{
+    opal_value_t *kv;
+    pmix_info_t *pinfo;
+    size_t sz, n;
+    pmix_status_t rc;
+    ext4x_opcaddy_t *op;
+
+    opal_output_verbose(2, opal_pmix_base_framework.framework_output,
+                        "%s setup local support for job %s",
+                        OPAL_NAME_PRINT(OPAL_PROC_MY_NAME),
+                        OPAL_JOBID_PRINT(jobid));
+
+    OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock);
+    if (0 >= opal_pmix_base.initialized) {
+        OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
+        return OPAL_ERR_NOT_INITIALIZED;
+    }
+    OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
+
+    /* convert the list to an array of pmix_info_t */
+    if (NULL != info && 0 < (sz = opal_list_get_size(info))) {
+        PMIX_INFO_CREATE(pinfo, sz);
+        n = 0;
+        OPAL_LIST_FOREACH(kv, info, opal_value_t) {
+            (void)strncpy(pinfo[n].key, kv->key, PMIX_MAX_KEYLEN);
+            ext4x_value_load(&pinfo[n].value, kv);
+            ++n;
+        }
+    } else {
+        sz = 0;
+        pinfo = NULL;
+    }
+    /* setup the caddy */
+    op = OBJ_NEW(ext4x_opcaddy_t);
+    op->info = pinfo;
+    op->sz = sz;
+    op->opcbfunc = cbfunc;
+    op->cbdata = cbdata;
+    /* convert the jobid */
+    (void)opal_snprintf_jobid(op->p.nspace, PMIX_MAX_NSLEN, jobid);
+
+    rc = PMIx_server_setup_local_support(op->p.nspace, op->info, op->sz,
+                                         opcbfunc, op);
+    if (PMIX_SUCCESS != rc) {
+        OBJ_RELEASE(op);
+    }
+    return ext4x_convert_rc(rc);
+}

opal/mca/pmix/ext4x/help-pmix-ext4x.txt

@@ -0,0 +1,32 @@
+# -*- text -*-
+#
+# Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2017-2018 Intel, Inc. All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+# This is the US/English help file for Open MPI MCA error messages.
+#
+[evars]
+We found conflicting directives regarding the location of OPAL vs PMIx
+installation directories:
+
+%s
+
+This usually indicates that OMPI was configured to use its internal copy
+of PMIx, but another installation of PMIx is also in use on this system
+and could potentially cause confusion between the two sets of plugins.
+Please either unset the indicated environment variables, or configure
+OMPI to use the external PMIx installation.

opal/mca/pmix/pmix3x/pmix/Makefile.am

@@ -11,7 +11,7 @@
 #                         All rights reserved.
 # Copyright (c) 2006-2016 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2012-2013 Los Alamos National Security, Inc.  All rights reserved.
-# Copyright (c) 2013-2017 Intel, Inc. All rights reserved.
+# Copyright (c) 2013-2018 Intel, Inc.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -23,7 +23,7 @@
 # via AC_CONFIG_MACRO_DIR in configure.ac.
 ACLOCAL_AMFLAGS = -I ./config
 
-SUBDIRS = config contrib include src etc
+SUBDIRS = config contrib include src etc bindings
 
 
 headers =

opal/mca/pmix/pmix3x/pmix/NEWS

@@ -21,8 +21,59 @@ example, a bug might be fixed in the master, and then moved to the
 current release as well as the "stable" bug fix release branch.
 
 
-Master (not on release branches yet)
+3.0.0 -- TBD
 ------------------------------------
+**** NOTE: This release implements the complete PMIX v3.0 Standard
+**** and therefore includes a number of new APIs and features. These
+**** can be tracked by their RFC's on the community website:
+**** https://pmix.org/pmix-standard.
+- Added blocking forms of several existing APIs:
+    - PMIx_Log
+    - PMIx_Allocation_request
+    - PMIx_Job_control
+    - PMIx_Process_monitor
+- Added support for getting/validating security credentials
+    - PMIx_Get_credential, PMIx_Validate_credential
+- Extended support for debuggers/tools
+    - Added IO forwarding support allowing tools to request
+      forwarding of output from specific application procs,
+      and to forward their input to specified target procs
+    - Extended tool attributes to support synchronization
+      during startup of applications. This includes the
+      ability to modify an application's environment
+      (including support for LD_PRELOAD) and define an
+      alternate fork/exec agent
+    - Added ability for a tool to switch server connections
+      so it can first connect to a system-level server to
+      launch a starter program, and then reconnect to that
+      starter for debugging purposes
+- Extended network support to collect network inventory by
+     either rolling it up from individual nodes or by direct
+     query of fabric managers. Added an API by which the
+     host can inject any rolled up inventory into the local
+     PMIx server. Applications and/or the host RM can access
+     the inventory via the PMIx_Query function.
+- Added the ability for applications and/or tools to register
+     files and directories for cleanup upon their termination
+- Added support for inter-library coordination within a process
+- Extended PMIx_Log support by adding plugin support for new
+     channels, including local/remote syslog and email. Added
+     attributes to query available channels and to tag and
+     format output.
+
+
+
+
+2.1.1 -- 23 Feb 2018
+----------------------
+- Fix direct modex when receiving new nspace
+- Resolve direct modex of job-level info
+- Fix a bug in attribute configuration checks
+- Fix a couple of bugs in unpacking of direct modex job-level data
+- Correcly handle application setup data during "instant on" launch
+- add a PMIX_BYTE_OBJECT_LOAD convenience macro
+- Fix two early "free" bugs
+- Add an example PMI-1 client program
 
 
 2.1.0 -- 1 Feb 2018
@@ -40,7 +91,7 @@ Master (not on release branches yet)
   sets of numbers
 
 
-2.0.3 -- TBD
+2.0.3 -- 1 Feb 2018
 ----------------------
 - Fix event notification so all sides of multi-library get notified
   of other library's existence

opal/mca/pmix/pmix3x/pmix/VERSION

@@ -30,7 +30,7 @@ greek=
 # command, or with the date (if "git describe" fails) in the form of
 # "date<date>".
 
-repo_rev=gitf0b8151
+repo_rev=gita493add
 
 # If tarball_version is not empty, it is used as the version string in
 # the tarball filename, regardless of all other versions listed in
@@ -44,7 +44,7 @@ tarball_version=
 
 # The date when this release was created
 
-date="Mar 24, 2018"
+date="Jun 10, 2018"
 
 # The shared library version of each of PMIx's public libraries.
 # These versions are maintained in accordance with the "Library
@@ -75,6 +75,6 @@ date="Mar 24, 2018"
 # Version numbers are described in the Libtool current:revision:age
 # format.
 
-libpmix_so_version=0:0:0
-libpmi_so_version=0:0:0
-libpmi2_so_version=0:0:0
+libpmix_so_version=4:0:2
+libpmi_so_version=1:0:0
+libpmi2_so_version=1:0:0

opal/mca/pmix/pmix3x/pmix/bindings/Makefile.am

@@ -0,0 +1,22 @@
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2009 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2006-2010 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2012-2013 Los Alamos National Security, Inc.  All rights reserved.
+# Copyright (c) 2013-2018 Intel, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+SUBDIRS = python

opal/mca/pmix/pmix3x/pmix/bindings/README

@@ -0,0 +1,19 @@
+Copyright (c) 2016-2018 Intel, Inc.  All rights reserved.
+
+$COPYRIGHT$
+
+Additional copyrights may follow
+
+$HEADER$
+
+===========================================================================
+
+This is where bindings of PMIx functions to alternative programming languages
+such as Python reside. All functions defined in the public headers have been
+provided with a wrapper. Note that there is no restriction on the number of
+wrappers that can exist, nor on what type of function is wrapped.
+
+There is only one rule to observe: you can wrap a framework, but you cannot wrap a
+specific plugin within that framework. This constraint flows from the fact that
+plugins are only accessed via the framework interface - thus, there is no way to
+guarantee that a particular plugin will be the active selection.

opal/mca/pmix/pmix3x/pmix/bindings/python/Makefile.am

@@ -0,0 +1,39 @@
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2009 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2006-2010 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2012-2013 Los Alamos National Security, Inc.  All rights reserved.
+# Copyright (c) 2013-2018 Intel, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+helpers = setup.py client.py server.py cpmix.pxd pmix.pyx
+
+if WANT_PYTHON_BINDINGS
+
+install-exec-local: $(helpers)
+	$(PYTHON) setup.py build_ext --include-dirs="$(top_builddir)/include" --library-dirs="$(DESTDIR)$(libdir)" --user
+	$(PYTHON) setup.py install --prefix="$(DESTDIR)$(prefix)"
+
+uninstall-hook:
+	rm -f $(pythondir)/pmix*.so
+	rm -f $(pythondir)/pypmix-*.egg-info
+
+CLEANFILES += pmix.c
+
+clean-local:
+	rm -rf build
+
+endif

opal/mca/pmix/pmix3x/pmix/bindings/python/README

@@ -0,0 +1,49 @@
+===========================================================================
+Cython-based Python wrapper for PMIx
+===========================================================================
+
+
+Example
+-------
+
+This example starts up a persistent DVM and then spawns some tasks using
+Python.
+
+$ virtualenv ve
+$ source ve/bin/activate
+$ pip install orte-cffi
+$ orte-dvm --report-uri dvm_uri
+$ python examples/submit.py
+
+
+Create a distfile
+----------------------------------------
+
+If you want to create a sdist file:
+
+$ virtualenv ve
+$ source ve/bin/activate
+$ python setup.py sdist
+
+
+Uploading sdist to pypi
+-----------------------
+
+Assuming you have admin privileges to the pypi package repository for this
+package, a new version can be uploaded using twine:
+
+$ virtualenv ve
+$ source ve/bin/activate
+$ pip install twine
+$ twine upload dist/orte-cffi-`python setup.py --version`.tar.gz
+
+
+Building (for development purposes only)
+----------------------------------------
+
+If you want to create a non-pip build:
+
+$ virtualenv ve
+$ source ve/bin/activate
+$ pip install cffi
+$ python src/orte-cffi/build.py

opal/mca/pmix/pmix3x/pmix/config/pmix.m4

@@ -727,6 +727,14 @@ AC_DEFUN([PMIX_SETUP_CORE],[
 
     PMIX_LIBEVENT_CONFIG
 
+    ##################################
+    # HWLOC
+    ##################################
+    pmix_show_title "HWLOC"
+
+    PMIX_HWLOC_CONFIG
+
+
     ##################################
     # ZLIB COMPRESSION
     ##################################
@@ -825,6 +833,8 @@ AC_DEFUN([PMIX_SETUP_CORE],[
 
     AC_CONFIG_FILES(
         pmix_config_prefix[Makefile]
+        pmix_config_prefix[bindings/Makefile]
+        pmix_config_prefix[bindings/python/Makefile]
         pmix_config_prefix[config/Makefile]
         pmix_config_prefix[etc/Makefile]
         pmix_config_prefix[include/Makefile]
@@ -832,7 +842,7 @@ AC_DEFUN([PMIX_SETUP_CORE],[
         pmix_config_prefix[src/util/keyval/Makefile]
         pmix_config_prefix[src/mca/base/Makefile]
         pmix_config_prefix[src/tools/pevent/Makefile]
-        pmix_config_prefix[src/tools/pinfo/Makefile]
+        pmix_config_prefix[src/tools/pmix_info/Makefile]
         pmix_config_prefix[src/tools/plookup/Makefile]
         pmix_config_prefix[src/tools/pps/Makefile]
         )
@@ -1118,6 +1128,44 @@ fi
 
 AM_CONDITIONAL([PMIX_INSTALL_BINARIES], [test $WANT_PMIX_BINARIES -eq 1])
 
+#
+# Install Python bindings?
+#
+AC_MSG_CHECKING([if want install Python bindings])
+AC_ARG_ENABLE(python-bindings,
+              AC_HELP_STRING([--enable-python-bindings],
+                             [enable Python bindings (default: disabled)]))
+if test "$enable_python_bindings" != "yes"; then
+    AC_MSG_RESULT([no])
+    WANT_PYTHON_BINDINGS=0
+else
+    AC_MSG_RESULT([yes])
+    WANT_PYTHON_BINDINGS=1
+fi
+
+AM_CONDITIONAL([WANT_PYTHON_BINDINGS], [test $WANT_PYTHON_BINDINGS -eq 1])
+
+if test "$WANT_PYTHON_BINDINGS" = "1"; then
+    AM_PATH_PYTHON([2.7], [python_happy=1], [python_happy=0])
+    if test "$python_happy" = "0"; then
+        AC_MSG_WARN([Python bindings were enabled, but no suitable])
+        AC_MSG_WARN([interpreter was found. PMIx requires at least])
+        AC_MSG_WARN([Python v2.7 to provide Python bindings])
+        AC_MSG_ERROR([Cannot continue])
+    fi
+
+    AC_MSG_CHECKING([if Cython package installed])
+    have_cython=esyscmd(config/pmix_check_cython.py)
+    if test "$have_cython" = "0"; then
+        AC_MSG_RESULT([yes])
+    else
+        AC_MSG_RESULT([no])
+        AC_MSG_WARN([Python bindings were enabled, but the Cython])
+        AC_MSG_WARN([package was not found. PMIx Python bindings])
+        AC_MSG_WARN([require that the Cython package be installed])
+        AC_MSG_ERROR([Cannot continue])
+    fi
+fi
 
 ])dnl
 

opal/mca/pmix/pmix3x/pmix/config/pmix_setup_hwloc.m4

@@ -0,0 +1,106 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2009-2015 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2013      Los Alamos National Security, LLC.  All rights reserved.
+# Copyright (c) 2013-2018 Intel, Inc. All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# MCA_hwloc_CONFIG([action-if-found], [action-if-not-found])
+# --------------------------------------------------------------------
+AC_DEFUN([PMIX_HWLOC_CONFIG],[
+    PMIX_VAR_SCOPE_PUSH([pmix_hwloc_dir pmix_hwloc_libdir pmix_hwloc_standard_lib_location pmix_hwloc_standard_header_location])
+
+    AC_ARG_WITH([hwloc],
+                [AC_HELP_STRING([--with-hwloc=DIR],
+                                [Search for hwloc headers and libraries in DIR ])])
+
+    AC_ARG_WITH([hwloc-libdir],
+                [AC_HELP_STRING([--with-hwloc-libdir=DIR],
+                                [Search for hwloc libraries in DIR ])])
+
+    pmix_hwloc_support=0
+
+    if test "$with_hwloc" != "no"; then
+        AC_MSG_CHECKING([for hwloc in])
+        if test ! -z "$with_hwloc" && test "$with_hwloc" != "yes"; then
+            pmix_hwloc_dir=$with_hwloc
+            pmix_hwloc_standard_header_location=no
+            pmix_hwloc_standard_lib_location=no
+            AS_IF([test -z "$with_hwloc_libdir" || test "$with_hwloc_libdir" = "yes"],
+                  [if test -d $with_hwloc/lib; then
+                       pmix_hwloc_libdir=$with_hwloc/lib
+                   elif test -d $with_hwloc/lib64; then
+                       pmix_hwloc_libdir=$with_hwloc/lib64
+                   else
+                       AC_MSG_RESULT([Could not find $with_hwloc/lib or $with_hwloc/lib64])
+                       AC_MSG_ERROR([Can not continue])
+                   fi
+                   AC_MSG_RESULT([$pmix_hwloc_dir and $pmix_hwloc_libdir])],
+                  [AC_MSG_RESULT([$with_hwloc_libdir])])
+        else
+            AC_MSG_RESULT([(default search paths)])
+            pmix_hwloc_standard_header_location=yes
+            pmix_hwloc_standard_lib_location=yes
+        fi
+        AS_IF([test ! -z "$with_hwloc_libdir" && test "$with_hwloc_libdir" != "yes"],
+              [pmix_hwloc_libdir="$with_hwloc_libdir"
+               pmix_hwloc_standard_lib_location=no])
+
+        PMIX_CHECK_PACKAGE([pmix_hwloc],
+                           [hwloc.h],
+                           [hwloc],
+                           [hwloc_topology_init],
+                           [-lhwloc],
+                           [$pmix_hwloc_dir],
+                           [$pmix_hwloc_libdir],
+                           [pmix_hwloc_support=1],
+                           [pmix_hwloc_support=0])
+        if test $pmix_hwloc_support = "1"; then
+            LIBS="$LIBS -lhwloc"
+            PMIX_EMBEDDED_LIBS="$PMIX_EMBEDDED_LIBS -lhwloc"
+            if test "$pmix_hwloc_standard_header_location" != "yes"; then
+                PMIX_EMBEDDED_CPPFLAGS="$PMIX_EMBEDDED_CPPFLAGS $pmix_hwloc_CPPFLAGS"
+                CPPFLAGS="$CPPFLAGS $pmix_hwloc_CPPFLAGS"
+            fi
+            if test "$pmix_hwloc_standard_lib_location" != "yes"; then
+                PMIX_EMBEDDED_LDFLAGS="$PMIX_EMBEDDED_LDFLAGS $pmix_hwloc_LDFLAGS"
+                LDFLAGS="$LDFLAGS $pmix_hwloc_LDFLAGS"
+            fi
+        fi
+    fi
+
+    if test ! -z "$with_hwloc" && test "$with_hwloc" != "no" && test "$pmix_hwloc_support" != "1"; then
+        AC_MSG_WARN([HWLOC SUPPORT REQUESTED AND NOT FOUND])
+        AC_MSG_ERROR([CANNOT CONTINUE])
+    fi
+
+    if test $pmix_hwloc_support = "1"; then
+        AC_MSG_CHECKING([if external hwloc version is 1.5 or greater])
+        AC_COMPILE_IFELSE(
+              [AC_LANG_PROGRAM([[#include <hwloc.h>]],
+              [[
+    #if HWLOC_API_VERSION < 0x00010500
+    #error "hwloc API version is less than 0x00010500"
+    #endif
+              ]])],
+              [AC_MSG_RESULT([yes])],
+              [AC_MSG_RESULT([no])
+               AC_MSG_ERROR([Cannot continue])])
+    fi
+
+    AC_MSG_CHECKING([will hwloc support be built])
+    if test "$pmix_hwloc_support" != "1"; then
+        AC_MSG_RESULT([no])
+    else
+        AC_MSG_RESULT([yes])
+    fi
+
+    AC_DEFINE_UNQUOTED([PMIX_HAVE_HWLOC], [$pmix_hwloc_support],
+                       [Whether or not we have hwloc support])
+    PMIX_VAR_SCOPE_POP
+])dnl

opal/mca/pmix/pmix3x/pmix/contrib/perf_tools/Makefile

@@ -1,21 +1,29 @@
-PMIX_BASE = <pmix-path>
-PMIX_INC= -I$(PMIX_BASE)/include/
-PMIX_LIB= -L$(PMIX_BASE)/lib/ -lpmix
+PMIX_BASE = /tmp/artemp/pmix
+SLURM_BASE = /tmp/artemp/slurm
 
-PMI2_BASE = /usr/
-#PMI2_INC= -I$(PMI2_BASE)/include/
-#PMI2_LIB= -L$(PMI2_BASE)/lib/ -lpmi2
-PMI2_LIB= -lpmi2
+PMIX_INC= -I$(PMIX_BASE)/include/
+PMIX_LIB= -L$(PMIX_BASE)/lib/ -L$(PMIX_BASE)/lib64/ -lpmix
+
+PMI2_BASE = $(SLURM_BASE)
+PMI2_INC= -I$(PMI2_BASE)/include/
+PMI2_LIB= -L$(PMI2_BASE)/lib/ -L$(PMI2_BASE)/lib64/ -lpmi2
+
+PMI1_BASE = $(SLURM_BASE)
+PMI1_INC= -I$(PMI1_BASE)/include/
+PMI1_LIB= -L$(PMI1_BASE)/lib/ -L$(PMI1_BASE)/lib64/ -lpmi
 
 CFLAGS = -O2 -g
 
-all: pmix pmi2
+all: pmix_intra_perf pmi2_intra_perf pmi1_intra_perf
 
-pmix: pmi_intra_perf.c pmi.h pmix.c
+pmix_intra_perf: pmi_intra_perf.c pmi.h pmix.c
 	gcc $(PMIX_INC) $(CFLAGS) -o pmix_intra_perf pmi_intra_perf.c pmix.c $(PMIX_LIB) -lrt
 
-pmi2: pmi_intra_perf.c pmi.h pmi2.c pmi2_pmap_parser.c pmi2_pmap_parser.h pmi2_utils.c pmi2_utils.h
+pmi2_intra_perf: pmi_intra_perf.c pmi.h pmi2.c pmi2_pmap_parser.c pmi2_pmap_parser.h pmi2_utils.c pmi2_utils.h
 	gcc $(PMI2_INC) $(CFLAGS) -o pmi2_intra_perf pmi_intra_perf.c pmi2.c pmi2_utils.c pmi2_pmap_parser.c -lrt $(PMI2_LIB)
 
+pmi1_intra_perf: pmi_intra_perf.c pmi.h pmi1.c pmi2_utils.c pmi2_utils.h
+	gcc $(PMI1_INC) $(CFLAGS) -o pmi1_intra_perf pmi_intra_perf.c pmi1.c pmi2_utils.c -lrt $(PMI1_LIB)
+
 clean:
-	rm -f pmix_intra_perf pmi2_intra_perf
+	rm -f pmix_intra_perf pmi2_intra_perf pmi1_intra_perf

opal/mca/pmix/pmix3x/pmix/contrib/perf_tools/pmi.h

@@ -15,6 +15,7 @@
 
 void pmi_init(int *rank, int *size);
 void pmi_get_local_ranks(int **local_ranks, int *local_cnt);
+void pmi_get_shmem_size(char *is_avail, size_t *cum_size);
 void pmi_put_key_loc(char *key, int *key_val, int key_size);
 void pmi_put_key_rem(char *key, int *key_val, int key_size);
 void pmi_put_double(char *key, double val);

opal/mca/pmix/pmix3x/pmix/contrib/perf_tools/pmi2.c

@@ -75,6 +75,11 @@ void pmi_get_local_ranks(int **local_ranks, int *local_cnt)
     free(pmapping);
 }
 
+void pmi_get_shmem_size(char *is_avail, size_t *size)
+{
+    *is_avail = 0;
+}
+
 void pmi_put_key_loc(char *key, int *key_val, int key_size)
 {
     char *encoded = pmi_encode(key_val, key_size * sizeof(int));
@@ -169,7 +174,7 @@ void pmi_get_key_rem(int rank, char *key_name, int **key_val, int *key_size)
     free(tmp);
 }
 
-float pmi_get_double(int rank, char *key)
+double pmi_get_double(int rank, char *key)
 {
     int len, rc;
     size_t tmp_size;

opal/mca/pmix/pmix3x/pmix/contrib/perf_tools/pmi_intra_perf.c

@@ -67,6 +67,10 @@ void parse_options(int argc, char **argv)
         switch (c) {
         case 's':
             key_size = atoi(optarg);
+            /* Make sure that we transform it to int as
+             * this is what will be the key value type
+             */
+            key_size = key_size / 4 + !!(key_size % 4);
             break;
         case 'c':
             key_count = atoi(optarg);
@@ -159,9 +163,11 @@ int main(int argc, char **argv)
     int *local_ranks, local_cnt;
     int *remote_ranks, remote_cnt;
     double start, total_start, get_loc_time = 0, get_rem_time = 0, put_loc_time = 0,
-            put_rem_time = 0, commit_time = 0, fence_time = 0, init_time = 0, total_time = 0;
+           put_rem_time = 0, commit_time = 0, fence_time = 0, init_time = 0, total_time = 0;
     int get_loc_cnt = 0, get_rem_cnt = 0, put_loc_cnt = 0, put_rem_cnt = 0;
     double mem_pss = 0.0, mem_rss = 0.0;
+    char have_shmem;
+    size_t shmem_job_info, shmem_all;
 
     parse_options(argc, argv);
 
@@ -177,6 +183,14 @@ int main(int argc, char **argv)
         fill_remote_ranks(local_ranks, local_cnt, remote_ranks, nproc);
     }
 
+    pmi_get_shmem_size(&have_shmem, &shmem_job_info);
+
+    /*
+     * Make sure that no other rank started publishing keys in the dstore
+     * before we finished with shmem size screening
+     */
+    pmi_fence( 0 );
+
     if( 0 == rank && debug_on ){
         int i;
         fprintf(stderr,"%d: local ranks: ", rank);
@@ -302,6 +316,7 @@ int main(int argc, char **argv)
                 commit_time, fence_time);
     }
 
+    pmi_get_shmem_size(&have_shmem, &shmem_all);
     /*
      * The barrier ensures that all procs finished key fetching
      * we had issues with dstor/lockless case evaluation
@@ -501,10 +516,12 @@ int main(int argc, char **argv)
         fprintf(stderr,"total:         max %lf min %lf\n", max_total_time, min_total_time);
         fprintf(stderr,"mem:           loc %0.2lf avg %0.2lf min %0.2lf max %0.2lf total %0.2lf Kb\n",
                 mem_pss, cum_mem_pss / nproc, min_mem_pss, max_mem_pss, cum_mem_pss);
+        if( have_shmem ) {
+            fprintf(stderr,"shmem:         job_info: %0.2lf total %0.2lf Kb\n",
+                    (double)shmem_job_info / 1024, (double)shmem_all / 1024);
+        }
 
-
-        /* debug printout */
-/*
+        /* debug printout *//*
         for(i = 0; i < nproc; i++){
             double val;
             printf("%d: ", i);

opal/mca/pmix/pmix3x/pmix/contrib/perf_tools/pmix.c

@@ -10,9 +10,16 @@
  */
 
 #include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <time.h>
+#include <getopt.h>
+#include <limits.h>
+#include <string.h>
 #include <pmix.h>
 
 pmix_proc_t this_proc;
+static int _int_size = 0;
 
 void pmi_init(int *rank, int *size)
 {
@@ -40,7 +47,7 @@ void pmi_init(int *rank, int *size)
         fprintf(stderr, "Client ns %s rank %d: PMIx_Get job size failed: %d", this_proc.nspace, this_proc.rank, rc);
         abort();
     }
-    *size = val->data.uint32;
+    _int_size = *size = val->data.uint32;
     *rank = this_proc.rank;
     PMIX_VALUE_RELEASE(val);
 }
@@ -81,6 +88,94 @@ void pmi_get_local_ranks(int **local_ranks, int *local_cnt)
     }
 }
 
+/* WARNING: should match one in
+ * src/mca/gds/ds12/gds_dstore.h
+ */
+typedef struct {
+    size_t rank;
+    size_t offset;
+    size_t count;
+} rank_meta_info;
+
+#define DSTORE_INIT_SEG "initial-pmix_shared-segment"
+#define DSTORE_META_SEG "smseg"
+#define DSTORE_DATA_SEG "smdataseg"
+
+typedef enum { other_seg, init_seg, meta_seg, data_seg } seg_type_t;
+
+struct {
+    char *sign;
+    seg_type_t type;
+} segments[] = {
+    {DSTORE_INIT_SEG, init_seg},
+    {DSTORE_META_SEG, meta_seg},
+    {DSTORE_DATA_SEG, data_seg},
+};
+
+static char *_get_maps_file(char *line)
+{
+    char *token = NULL;
+    char *saveptr = NULL;
+    int i = 0;
+
+    token = strtok_r(line, "-\n", &saveptr);
+    if(NULL == token) {
+        return NULL;
+    }
+    return (char*)strtoul(token, &saveptr, 16);
+}
+
+static seg_type_t
+is_dstor_region(char *line, char **base)
+{
+    int i;
+    seg_type_t type = other_seg;
+    for(i=0; i<3; i++){
+        if( strstr(line, segments[i].sign) ){
+            if(!(*base = _get_maps_file(line)) ){
+                return other_seg;
+            }
+            return segments[i].type;
+        }
+    }
+    return other_seg;
+}
+
+void pmi_get_shmem_size(char *is_avail, size_t *cum_size)
+{
+    char maps_path[PATH_MAX] = "/proc/self/maps";
+    FILE *maps_fp = NULL;
+    char *line = NULL;
+    size_t size = 0, meta_size = 0, data_size = 0;
+    char *base;
+
+    if (NULL == (maps_fp = fopen(maps_path, "r"))) {
+        abort();
+    }
+
+    while ((size = getline(&line, &size, maps_fp)) != -1) {
+        seg_type_t type = is_dstor_region(line, &base);
+        switch(type) {
+        case other_seg:
+        case init_seg:
+        case meta_seg:
+            break;
+        case data_seg:{
+            data_size += *((size_t*)base);
+            break;
+        }
+        }
+    }
+    free(line);
+    fclose(maps_fp);
+    *is_avail = 0;
+    *cum_size = data_size;
+    if( *cum_size > 0 ) {
+        *is_avail = 1;
+        *cum_size += sizeof(rank_meta_info) * _int_size;
+    }
+}
+
 static void _put_key(char *key, int *key_val, int key_size, pmix_scope_t scope)
 {
     pmix_value_t value;

opal/mca/pmix/pmix3x/pmix/contrib/pmix.spec

@@ -12,7 +12,7 @@
 # Copyright (c) 2006-2016 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2013      Mellanox Technologies, Inc.
 #                         All rights reserved.
-# Copyright (c) 2015-2017 Intel, Inc. All rights reserved.
+# Copyright (c) 2015-2018 Intel, Inc.  All rights reserved.
 # Copyright (c) 2015      Research Organization for Information Science
 #                         and Technology (RIST). All rights reserved.
 # $COPYRIGHT$

opal/mca/pmix/pmix3x/pmix/examples/Makefile.am

@@ -11,7 +11,7 @@
 #                         All rights reserved.
 # Copyright (c) 2006-2010 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2012-2013 Los Alamos National Security, Inc.  All rights reserved.
-# Copyright (c) 2013-2017 Intel, Inc.  All rights reserved.
+# Copyright (c) 2013-2018 Intel, Inc. All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -21,7 +21,7 @@
 
 AM_CPPFLAGS = -I$(top_builddir)/src -I$(top_builddir)/src/include -I$(top_builddir)/include -I$(top_builddir)/include/pmix
 
-noinst_PROGRAMS = client client2 dmodex dynamic fault pub tool debugger debuggerd alloc jctrl
+noinst_PROGRAMS = client client2 dmodex dynamic fault pub pubi tool debugger debuggerd alloc jctrl
 if !WANT_HIDDEN
 # these examples use internal symbols
 # use --disable-visibility
@@ -68,6 +68,10 @@ pub_SOURCES = pub.c
 pub_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS)
 pub_LDADD = $(top_builddir)/src/libpmix.la
 
+pubi_SOURCES = pubi.c
+pubi_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS)
+pubi_LDADD = $(top_builddir)/src/libpmix.la
+
 tool_SOURCES = tool.c
 tool_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS)
 tool_LDADD = $(top_builddir)/src/libpmix.la

opal/mca/pmix/pmix3x/pmix/examples/debuggerd.c

@@ -13,7 +13,7 @@
  *                         All rights reserved.
  * Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2011      Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2013-2017 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2013-2018 Intel, Inc.  All rights reserved.
  * Copyright (c) 2015      Mellanox Technologies, Inc.  All rights reserved.
  * $COPYRIGHT$
  *

opal/mca/pmix/pmix3x/pmix/examples/dynamic.c

@@ -13,7 +13,7 @@
  *                         All rights reserved.
  * Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2011      Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2013-2017 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2013-2018 Intel, Inc. All rights reserved.
  * Copyright (c) 2015      Mellanox Technologies, Inc.  All rights reserved.
  * Copyright (c) 2016      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
@@ -44,7 +44,7 @@ int main(int argc, char **argv)
     int rc;
     pmix_value_t value;
     pmix_value_t *val = &value;
-    pmix_proc_t proc, newproc;
+    pmix_proc_t proc;
     uint32_t nprocs;
     char nsp2[PMIX_MAX_NSLEN+1];
     pmix_app_t *app;
@@ -135,13 +135,13 @@ int main(int argc, char **argv)
     /* just cycle the connect/disconnect functions */
     (void)strncpy(proc.nspace, myproc.nspace, PMIX_MAX_NSLEN);
     proc.rank = PMIX_RANK_WILDCARD;
-    if (PMIX_SUCCESS != (rc = PMIx_Connect(&proc, 1, NULL, 0, newproc.nspace, &newproc.rank))) {
+    if (PMIX_SUCCESS != (rc = PMIx_Connect(&proc, 1, NULL, 0))) {
         fprintf(stderr, "Client ns %s rank %d: PMIx_Connect failed: %d\n", myproc.nspace, myproc.rank, rc);
         goto done;
     }
-    fprintf(stderr, "Client ns %s rank %d: PMIx_Connect succeeded - new ID: %s:%d\n",
-            myproc.nspace, myproc.rank, newproc.nspace, newproc.rank);
-    if (PMIX_SUCCESS != (rc = PMIx_Disconnect(newproc.nspace, NULL, 0))) {
+    fprintf(stderr, "Client ns %s rank %d: PMIx_Connect succeeded\n",
+            myproc.nspace, myproc.rank);
+    if (PMIX_SUCCESS != (rc = PMIx_Disconnect(&proc, 1, NULL, 0))) {
         fprintf(stderr, "Client ns %s rank %d: PMIx_Disonnect failed: %d\n", myproc.nspace, myproc.rank, rc);
         goto done;
     }

opal/mca/pmix/pmix3x/pmix/examples/pub.c

@@ -13,7 +13,7 @@
  *                         All rights reserved.
  * Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2011      Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2013-2016 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2013-2018 Intel, Inc. All rights reserved.
  * Copyright (c) 2015      Mellanox Technologies, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
@@ -69,6 +69,7 @@ int main(int argc, char **argv)
 
     /* publish something */
     if (0 == myproc.rank) {
+        fprintf(stderr, "%s:%d publishing two keys\n", myproc.nspace, myproc.rank);
         PMIX_INFO_CREATE(info, 2);
         (void)strncpy(info[0].key, "FOOBAR", PMIX_MAX_KEYLEN);
         info[0].value.type = PMIX_UINT8;
@@ -80,6 +81,7 @@ int main(int argc, char **argv)
             fprintf(stderr, "Client ns %s rank %d: PMIx_Publish failed: %d\n", myproc.nspace, myproc.rank, rc);
             goto done;
         }
+        fprintf(stderr, "%s:%d publish complete\n", myproc.nspace, myproc.rank);
         PMIX_INFO_FREE(info, 2);
     }
 
@@ -93,6 +95,7 @@ int main(int argc, char **argv)
     /* lookup something */
     if (0 != myproc.rank) {
         PMIX_PDATA_CREATE(pdata, 1);
+        fprintf(stderr, "%s:%d looking up key FOOBAR\n", myproc.nspace, myproc.rank);
         (void)strncpy(pdata[0].key, "FOOBAR", PMIX_MAX_KEYLEN);
         if (PMIX_SUCCESS != (rc = PMIx_Lookup(pdata, 1, NULL, 0))) {
             fprintf(stderr, "Client ns %s rank %d: PMIx_Lookup failed: %d\n", myproc.nspace, myproc.rank, rc);
@@ -136,6 +139,7 @@ int main(int argc, char **argv)
         keys[1] = "PANDA";
         keys[2] = NULL;
 
+        fprintf(stderr, "%s:%d unpublishing two keys\n", myproc.nspace, myproc.rank);
         if (PMIX_SUCCESS != (rc = PMIx_Unpublish(keys, NULL, 0))) {
             fprintf(stderr, "Client ns %s rank %d: PMIx_Unpublish failed: %d\n", myproc.nspace, myproc.rank, rc);
             free(keys);

opal/mca/pmix/pmix3x/pmix/examples/pubi.c

@@ -0,0 +1,173 @@
+/*
+ * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2011 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2013 Los Alamos National Security, LLC.
+ *                         All rights reserved.
+ * Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2011      Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2013-2018 Intel, Inc. All rights reserved.
+ * Copyright (c) 2015      Mellanox Technologies, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <time.h>
+
+#include <pmix.h>
+
+int main(int argc, char **argv)
+{
+    pmix_proc_t myproc;
+    int rc;
+    pmix_value_t value;
+    pmix_value_t *val = &value;
+    pmix_proc_t proc;
+    uint32_t nprocs;
+    pmix_info_t *info;
+    pmix_pdata_t *pdata;
+    size_t n;
+
+    /* init us */
+    if (PMIX_SUCCESS != (rc = PMIx_Init(&myproc, NULL, 0))) {
+        fprintf(stderr, "Client ns %s rank %d: PMIx_Init failed: %d\n", myproc.nspace, myproc.rank, rc);
+        exit(0);
+    }
+    fprintf(stderr, "Client ns %s rank %d: Running\n", myproc.nspace, myproc.rank);
+
+    /* get our universe size */
+    PMIX_PROC_CONSTRUCT(&proc);
+    (void)strncpy(proc.nspace, myproc.nspace, PMIX_MAX_NSLEN);
+    proc.rank = PMIX_RANK_WILDCARD;
+    if (PMIX_SUCCESS != (rc = PMIx_Get(&proc, PMIX_UNIV_SIZE, NULL, 0, &val))) {
+        fprintf(stderr, "Client ns %s rank %d: PMIx_Get universe size failed: %d\n", myproc.nspace, myproc.rank, rc);
+        goto done;
+    }
+    nprocs = val->data.uint32;
+    PMIX_VALUE_RELEASE(val);
+    fprintf(stderr, "Client %s:%d universe size %d\n", myproc.nspace, myproc.rank, nprocs);
+
+    /* publish something */
+    if (0 == myproc.rank) {
+        fprintf(stderr, "%s:%d publishing two keys\n", myproc.nspace, myproc.rank);
+        PMIX_INFO_CREATE(info, 2);
+        (void)strncpy(info[0].key, "FOOBAR", PMIX_MAX_KEYLEN);
+        info[0].value.type = PMIX_UINT8;
+        info[0].value.data.uint8 = 1;
+        (void)strncpy(info[1].key, "PANDA", PMIX_MAX_KEYLEN);
+        info[1].value.type = PMIX_SIZE;
+        info[1].value.data.size = 123456;
+        if (PMIX_SUCCESS != (rc = PMIx_Publish(info, 2))) {
+            fprintf(stderr, "Client ns %s rank %d: PMIx_Publish failed: %d\n", myproc.nspace, myproc.rank, rc);
+            goto done;
+        }
+        fprintf(stderr, "%s:%d publish complete\n", myproc.nspace, myproc.rank);
+        PMIX_INFO_FREE(info, 2);
+    }
+
+    /* lookup something */
+    if (0 != myproc.rank) {
+        PMIX_PDATA_CREATE(pdata, 2);
+        fprintf(stderr, "%s:%d looking up key FOOBAR\n", myproc.nspace, myproc.rank);
+        (void)strncpy(pdata[0].key, "FOOBAR", PMIX_MAX_KEYLEN);
+        (void)strncpy(pdata[1].key, "PANDA", PMIX_MAX_KEYLEN);
+        PMIX_INFO_CREATE(info, 1);
+        rc = 0;
+        PMIX_INFO_LOAD(&info[0], PMIX_WAIT, &rc, PMIX_INT);
+        if (PMIX_SUCCESS != (rc = PMIx_Lookup(pdata, 2, info, 1))) {
+            fprintf(stderr, "Client ns %s rank %d: PMIx_Lookup failed: %d\n", myproc.nspace, myproc.rank, rc);
+            goto done;
+        }
+        PMIX_INFO_FREE(info, 1);
+        /* check the return for value and source */
+        for (n=0; n < 2; n++) {
+            if (0 != strncmp(myproc.nspace, pdata[n].proc.nspace, PMIX_MAX_NSLEN)) {
+                fprintf(stderr, "Client ns %s rank %d: PMIx_Lookup returned wrong nspace: %s\n",
+                            myproc.nspace, myproc.rank, pdata[n].proc.nspace);
+                goto done;
+            }
+            if (0 != pdata[n].proc.rank) {
+                fprintf(stderr, "Client ns %s rank %d: PMIx_Lookup returned wrong rank: %d\n",
+                            myproc.nspace, myproc.rank, pdata[n].proc.rank);
+                goto done;
+            }
+        }
+        if (PMIX_UINT8 != pdata[0].value.type) {
+            fprintf(stderr, "Client ns %s rank %d: PMIx_Lookup returned wrong type: %d\n",
+                        myproc.nspace, myproc.rank, pdata[0].value.type);
+            goto done;
+        }
+        if (1 != pdata[0].value.data.uint8) {
+            fprintf(stderr, "Client ns %s rank %d: PMIx_Lookup returned wrong value: %d\n",
+                        myproc.nspace, myproc.rank, (int)pdata[0].value.data.uint8);
+            goto done;
+        }
+        if (PMIX_SIZE != pdata[1].value.type) {
+            fprintf(stderr, "Client ns %s rank %d: PMIx_Lookup returned wrong type: %d\n",
+                        myproc.nspace, myproc.rank, pdata[1].value.type);
+            goto done;
+        }
+        if (123456 != pdata[1].value.data.size) {
+            fprintf(stderr, "Client ns %s rank %d: PMIx_Lookup returned wrong value: %d\n",
+                        myproc.nspace, myproc.rank, (int)pdata[1].value.data.size);
+            goto done;
+        }
+        PMIX_PDATA_FREE(pdata, 2);
+        fprintf(stderr, "PUBLISH-LOOKUP SUCCEEDED\n");
+    }
+
+    /* call fence so rank 0 waits before leaving */
+    if (PMIX_SUCCESS != (rc = PMIx_Fence(&proc, 1, NULL, 0))) {
+        fprintf(stderr, "Client ns %s rank %d: PMIx_Fence failed: %d\n", myproc.nspace, myproc.rank, rc);
+        goto done;
+    }
+
+    if (0 == myproc.rank) {
+        char **keys;
+        keys = (char**)malloc(3 * sizeof(char*));
+        keys[0] = "FOOBAR";
+        keys[1] = "PANDA";
+        keys[2] = NULL;
+
+        fprintf(stderr, "%s:%d unpublishing two keys\n", myproc.nspace, myproc.rank);
+        if (PMIX_SUCCESS != (rc = PMIx_Unpublish(keys, NULL, 0))) {
+            fprintf(stderr, "Client ns %s rank %d: PMIx_Unpublish failed: %d\n", myproc.nspace, myproc.rank, rc);
+            free(keys);
+            goto done;
+        }
+        free(keys);
+        fprintf(stderr, "UNPUBLISH SUCCEEDED\n");
+    }
+
+    /* call fence again so everyone waits for rank 0 before leaving */
+    if (PMIX_SUCCESS != (rc = PMIx_Fence(&proc, 1, NULL, 0))) {
+        fprintf(stderr, "Client ns %s rank %d: PMIx_Fence failed: %d\n", myproc.nspace, myproc.rank, rc);
+        goto done;
+    }
+
+ done:
+    /* finalize us */
+    fprintf(stderr, "Client ns %s rank %d: Finalizing\n", myproc.nspace, myproc.rank);
+    if (PMIX_SUCCESS != (rc = PMIx_Finalize(NULL, 0))) {
+        fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize failed: %d\n", myproc.nspace, myproc.rank, rc);
+    } else {
+        fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize successfully completed\n", myproc.nspace, myproc.rank);
+    }
+    fflush(stderr);
+    return(0);
+}

opal/mca/pmix/pmix3x/pmix/examples/server.c

@@ -13,7 +13,7 @@
  *                         All rights reserved.
  * Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2011      Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2013-2017 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2013-2018 Intel, Inc. All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2016      IBM Corporation.  All rights reserved.
@@ -80,8 +80,8 @@ static pmix_status_t spawn_fn(const pmix_proc_t *proc,
                               pmix_spawn_cbfunc_t cbfunc, void *cbdata);
 static pmix_status_t connect_fn(const pmix_proc_t procs[], size_t nprocs,
                                 const pmix_info_t info[], size_t ninfo,
-                                pmix_connect_cbfunc_t cbfunc, void *cbdata);
-static pmix_status_t disconnect_fn(const char nspace[],
+                                pmix_op_cbfunc_t cbfunc, void *cbdata);
+static pmix_status_t disconnect_fn(const pmix_proc_t procs[], size_t nprocs,
                                    const pmix_info_t info[], size_t ninfo,
                                    pmix_op_cbfunc_t cbfunc, void *cbdata);
 static pmix_status_t register_event_fn(pmix_status_t *codes, size_t ncodes,
@@ -134,6 +134,14 @@ PMIX_CLASS_INSTANCE(pmix_locdat_t,
                     pmix_list_item_t,
                     NULL, NULL);
 
+#define PMIX_WAIT_FOR_COMPLETION(a)             \
+    do {                                        \
+        while ((a)) {                           \
+            usleep(10);                         \
+        }                                       \
+        PMIX_ACQUIRE_OBJECT((a));               \
+    } while (0)
+
 typedef struct {
     pmix_object_t super;
     volatile bool active;
@@ -740,7 +748,7 @@ static pmix_status_t spawn_fn(const pmix_proc_t *proc,
 
 static pmix_status_t connect_fn(const pmix_proc_t procs[], size_t nprocs,
                                 const pmix_info_t info[], size_t ninfo,
-                                pmix_connect_cbfunc_t cbfunc, void *cbdata)
+                                pmix_op_cbfunc_t cbfunc, void *cbdata)
 {
     pmix_output(0, "SERVER: CONNECT");
 
@@ -748,14 +756,14 @@ static pmix_status_t connect_fn(const pmix_proc_t procs[], size_t nprocs,
      * resource manager for handling */
 
     if (NULL != cbfunc) {
-        cbfunc(PMIX_SUCCESS, "FOOBAR", 1, cbdata);
+        cbfunc(PMIX_SUCCESS, cbdata);
     }
 
     return PMIX_SUCCESS;
 }
 
 
-static pmix_status_t disconnect_fn(const char nspace[],
+static pmix_status_t disconnect_fn(const pmix_proc_t procs[], size_t nprocs,
                                    const pmix_info_t info[], size_t ninfo,
                                    pmix_op_cbfunc_t cbfunc, void *cbdata)
 {

opal/mca/pmix/pmix3x/pmix/include/pmix.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013-2018 Intel, Inc. All rights reserved.
+ * Copyright (c) 2013-2018 Intel, Inc.  All rights reserved.
  * Copyright (c) 2016      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  *
@@ -353,9 +353,11 @@ PMIX_EXPORT pmix_status_t PMIx_Spawn_nb(const pmix_info_t job_info[], size_t nin
  * appropriate action. Note that different resource managers may respond to
  * failures in different manners.
  *
- * The host RM will assign a unique nspace to the resulting process group. The
- * new nspace must be provided when disconnecting from the group. All procs are
- * required to disconnect from the nspace prior to terminating.
+ * The callback function is to be called once all participating processes have
+ * called connect. The server is required to return any job-level info for the
+ * connecting processes that might not already have - i.e., if the connect
+ * request involves procs from different nspaces, then each proc shall receive
+ * the job-level info from those nspaces other than their own.
  *
  * Note: a process can only engage in _one_ connect operation involving the identical
  * set of processes at a time. However, a process _can_ be simultaneously engaged
@@ -364,61 +366,24 @@ PMIX_EXPORT pmix_status_t PMIx_Spawn_nb(const pmix_info_t job_info[], size_t nin
  * As in the case of the fence operation, the info array can be used to pass
  * user-level directives regarding the algorithm to be used for the collective
  * operation involved in the "connect", timeout constraints, and other options
- * available from the host RM.
- *
- * The server is required to return any job-level info for the connecting
- * processes that they might not already have - i.e., if the connect request
- * involves procs from different nspaces, then each proc shall receive the
- * job-level info from those nspaces other than their own.
- */
-
-
-/* The blocking form of this call must provide a character array of size
- * PMIX_MAX_NSLEN+1 for the assigned nspace of the resulting group, and a pointer
- * to an pmix_rank_t location where the new rank of this process in the assigned
- * nspace can be returned. Calls will return once the specified operation
- * is complete (i.e., all participants have called PMIx_Connect).
- */
-
+ * available from the host RM */
 PMIX_EXPORT pmix_status_t PMIx_Connect(const pmix_proc_t procs[], size_t nprocs,
-                                       const pmix_info_t info[], size_t ninfo,
-                                       char nspace[], pmix_rank_t *newrank);
-
-/* The callback function for the non-blocking form of the PMIx_Connect
- * operation will be called once the operation is complete. Any participant
- * that fails to call "connect" prior to terminating will cause the
- * operation to return a "failed" status to all other participants. This
- * is the default behavior in the absence of any provided directive.
- *
- * Some additional info keys are provided for this operation:
- *
- * (a) PMIX_CONNECT_NOTIFY_EACH: generate a local event notification using
- *     the PMIX_PROC_HAS_CONNECTED event each time a process connects
- *
- * (b) PMIX_CONNECT_NOTIFY_REQ: notify each of the indicated procs that
- *     they are requested to connect using the PMIX_CONNECT_REQUESTED event
- *
- * (c) PMIX_CONNECT_OPTIONAL: participation is optional - do not return
- *     error if procs terminate without having connected
- */
+                                       const pmix_info_t info[], size_t ninfo);
 
 PMIX_EXPORT pmix_status_t PMIx_Connect_nb(const pmix_proc_t procs[], size_t nprocs,
                                           const pmix_info_t info[], size_t ninfo,
-                                          pmix_connect_cbfunc_t cbfunc, void *cbdata);
+                                          pmix_op_cbfunc_t cbfunc, void *cbdata);
 
-/* Disconnect this process from a specified nspace. An error will be returned
- * if the specified nspace is not recognized. The info array is used as above.
- *
- * Processes that terminate while connected to other processes will generate a
- * "termination error" event that will be reported to any process in the connected
- * group that has registered for such events. Calls to "disconnect" that include the
- * PMIX_CONNECT_NOTIFY_EACH info key will cause other processes in the nspace to receive
- * an event notification of the disconnect, if they are registered for such events.
- */
-PMIX_EXPORT pmix_status_t PMIx_Disconnect(const char nspace[],
+/* Disconnect a previously connected set of processes. An error will be returned
+ * if the specified set of procs was not previously "connected". As above, a process
+ * may be involved in multiple simultaneous disconnect operations. However, a process
+ * is not allowed to reconnect to a set of procs that has not fully completed
+ * disconnect - i.e., you have to fully disconnect before you can reconnect to the
+ * _same_ group of processes. The info array is used as above. */
+PMIX_EXPORT pmix_status_t PMIx_Disconnect(const pmix_proc_t procs[], size_t nprocs,
                                           const pmix_info_t info[], size_t ninfo);
 
-PMIX_EXPORT pmix_status_t PMIx_Disconnect_nb(const char nspace[],
+PMIX_EXPORT pmix_status_t PMIx_Disconnect_nb(const pmix_proc_t ranges[], size_t nprocs,
                                              const pmix_info_t info[], size_t ninfo,
                                              pmix_op_cbfunc_t cbfunc, void *cbdata);
 
@@ -468,6 +433,9 @@ PMIX_EXPORT pmix_status_t PMIx_Query_info_nb(pmix_query_t queries[], size_t nque
  * has been completed. The data array must be maintained until
  * the callback is provided
  */
+PMIX_EXPORT pmix_status_t PMIx_Log(const pmix_info_t data[], size_t ndata,
+                                   const pmix_info_t directives[], size_t ndirs);
+
 PMIX_EXPORT pmix_status_t PMIx_Log_nb(const pmix_info_t data[], size_t ndata,
                                       const pmix_info_t directives[], size_t ndirs,
                                       pmix_op_cbfunc_t cbfunc, void *cbdata);
@@ -504,6 +472,9 @@ PMIX_EXPORT pmix_status_t PMIx_Log_nb(const pmix_info_t data[], size_t ndata,
  *   lieue of preemption. A corresponding ability to "reacquire" resources
  *   previously released is included.
  */
+PMIX_EXPORT pmix_status_t PMIx_Allocation_request(pmix_alloc_directive_t directive,
+                                                  pmix_info_t *info, size_t ninfo);
+
 PMIX_EXPORT pmix_status_t PMIx_Allocation_request_nb(pmix_alloc_directive_t directive,
                                                      pmix_info_t *info, size_t ninfo,
                                                      pmix_info_cbfunc_t cbfunc, void *cbdata);
@@ -522,6 +493,9 @@ PMIX_EXPORT pmix_status_t PMIx_Allocation_request_nb(pmix_alloc_directive_t dire
  * when the callback function completes - this will be used to release
  * any provided pmix_info_t array.
  */
+PMIX_EXPORT pmix_status_t PMIx_Job_control(const pmix_proc_t targets[], size_t ntargets,
+                                           const pmix_info_t directives[], size_t ndirs);
+
 PMIX_EXPORT pmix_status_t PMIx_Job_control_nb(const pmix_proc_t targets[], size_t ntargets,
                                               const pmix_info_t directives[], size_t ndirs,
                                               pmix_info_cbfunc_t cbfunc, void *cbdata);
@@ -553,6 +527,9 @@ PMIX_EXPORT pmix_status_t PMIx_Job_control_nb(const pmix_proc_t targets[], size_
  *
  * Note: a process can send a heartbeat to the server using the PMIx_Heartbeat
  * macro provided below*/
+PMIX_EXPORT pmix_status_t PMIx_Process_monitor(const pmix_info_t *monitor, pmix_status_t error,
+                                               const pmix_info_t directives[], size_t ndirs);
+
 PMIX_EXPORT pmix_status_t PMIx_Process_monitor_nb(const pmix_info_t *monitor, pmix_status_t error,
                                                   const pmix_info_t directives[], size_t ndirs,
                                                   pmix_info_cbfunc_t cbfunc, void *cbdata);

opal/mca/pmix/pmix3x/pmix/include/pmix_common.h.in

@@ -1,7 +1,7 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2013-2018 Intel, Inc. All rights reserved.
- * Copyright (c) 2016-2017 Research Organization for Information Science
+ * Copyright (c) 2016-2018 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2016      IBM Corporation.  All rights reserved.
  * Copyright (c) 2016-2017 Mellanox Technologies, Inc.
@@ -99,11 +99,23 @@ typedef uint32_t pmix_rank_t;
  * data for the given key from every rank that posted
  * that key */
 #define PMIX_RANK_WILDCARD  UINT32_MAX-1
-
 /* other special rank values will be used to define
  * groups of ranks for use in collectives */
 #define PMIX_RANK_LOCAL_NODE    UINT32_MAX-2        // all ranks on local node
+/* define an invalid value */
+#define PMIX_RANK_INVALID   UINT32_MAX-3
 
+/****  PMIX ENVIRONMENTAL PARAMETERS  ****/
+/* There are a few environmental parameters used by PMIx for
+ * various operations. While there is no "definition" of them
+ * as values, we do record them here for informational purposes.
+ *
+ * PMIX_LAUNCHER_PAUSE_FOR_TOOL - if set to non-zero value, instructs
+ * launchers (e.g., "prun") to stop prior to spawning the application until
+ * a tool can connect with further instructions. This envar will be
+ * set by the tool and is _not_ intended for the direct use of users.
+ *
+ */
 
 /* define a set of "standard" PMIx attributes that can
  * be queried. Implementations (and users) are free to extend as
@@ -132,7 +144,9 @@ typedef uint32_t pmix_rank_t;
 #define PMIX_SERVER_ENABLE_MONITORING       "pmix.srv.monitor"      // (bool) Enable PMIx internal monitoring by server
 #define PMIX_SERVER_NSPACE                  "pmix.srv.nspace"       // (char*) Name of the nspace to use for this server
 #define PMIX_SERVER_RANK                    "pmix.srv.rank"         // (pmix_rank_t) Rank of this server
-
+#define PMIX_SERVER_GATEWAY                 "pmix.srv.gway"         // (bool) Server is acting as a gateway for PMIx requests
+                                                                    //        that cannot be serviced on backend nodes
+                                                                    //        (e.g., logging to email)
 
 /* tool-related attributes */
 #define PMIX_TOOL_NSPACE                    "pmix.tool.nspace"      // (char*) Name of the nspace to use for this tool
@@ -148,6 +162,8 @@ typedef uint32_t pmix_rank_t;
 #define PMIX_TOOL_DO_NOT_CONNECT            "pmix.tool.nocon"       // (bool) the tool wants to use internal PMIx support, but does
                                                                     //        not want to connect to a PMIx server
                                                                     //        from the specified processes to this tool
+#define PMIX_RECONNECT_SERVER               "pmix.cnct.recon"       // (bool) tool is requesting to change server connections
+#define PMIX_LAUNCHER                       "pmix.tool.launcher"    // (bool) tool is a launcher and needs rendezvous files created
 
 /* identification attributes */
 #define PMIX_USERID                         "pmix.euid"             // (uint32_t) effective user id
@@ -189,6 +205,7 @@ typedef uint32_t pmix_rank_t;
 #define PMIX_TCP_DISABLE_IPV4               "pmix.tcp.disipv4"      // (bool) true to disable IPv4 family
 #define PMIX_TCP_DISABLE_IPV6               "pmix.tcp.disipv6"      // (bool) true to disable IPv6 family
 
+
 /* attributes for GDS */
 #define PMIX_GDS_MODULE                     "pmix.gds.mod"          // (char*) comma-delimited string of desired modules
 
@@ -207,7 +224,7 @@ typedef uint32_t pmix_rank_t;
 
 /* information about relative ranks as assigned by the RM */
 #define PMIX_CLUSTER_ID                     "pmix.clid"             // (char*) a string name for the cluster this proc is executing on
-#define PMIX_PROCID                         "pmix.procid"           // (pmix_proc_t) process identifier
+#define PMIX_PROCID                         "pmix.procid"           // (pmix_proc_t*) process identifier
 #define PMIX_NSPACE                         "pmix.nspace"           // (char*) nspace of a job
 #define PMIX_JOBID                          "pmix.jobid"            // (char*) jobid assigned by scheduler
 #define PMIX_APPNUM                         "pmix.appnum"           // (uint32_t) app number within the job
@@ -234,6 +251,7 @@ typedef uint32_t pmix_rank_t;
 #define PMIX_LOCALITY                       "pmix.loc"              // (uint16_t) relative locality of two procs
 #define PMIX_PARENT_ID                      "pmix.parent"           // (pmix_proc_t*) identifier of the process that called PMIx_Spawn
                                                                     //                to launch this proc's application
+#define PMIX_EXIT_CODE                      "pmix.exit.code"        // (int) exit code returned when proc terminated
 
 
 /* size info */
@@ -246,16 +264,20 @@ typedef uint32_t pmix_rank_t;
 #define PMIX_MAX_PROCS                      "pmix.max.size"         // (uint32_t) max #procs for this job
 #define PMIX_NUM_NODES                      "pmix.num.nodes"        // (uint32_t) #nodes in this nspace
 
+
 /* Memory info */
 #define PMIX_AVAIL_PHYS_MEMORY              "pmix.pmem"             // (uint64_t) total available physical memory on this node
 #define PMIX_DAEMON_MEMORY                  "pmix.dmn.mem"          // (float) Mbytes of memory currently used by daemon
 #define PMIX_CLIENT_AVG_MEMORY              "pmix.cl.mem.avg"       // (float) Average Mbytes of memory used by client processes
 
+
 /* topology info */
 #define PMIX_NET_TOPO                       "pmix.ntopo"            // (char*) xml-representation of network topology
 #define PMIX_LOCAL_TOPO                     "pmix.ltopo"            // (char*) xml-representation of local node topology
 #define PMIX_NODE_LIST                      "pmix.nlist"            // (char*) comma-delimited list of nodes running procs for this job
 #define PMIX_TOPOLOGY                       "pmix.topo"             // (hwloc_topology_t) pointer to the PMIx client's internal topology object
+#define PMIX_TOPOLOGY_XML                   "pmix.topo.xml"         // (char*) XML-based description of topology
+#define PMIX_TOPOLOGY_FILE                  "pmix.topo.file"        // (char*) full path to file containing XML topology description
 #define PMIX_TOPOLOGY_SIGNATURE             "pmix.toposig"          // (char*) topology signature string
 #define PMIX_LOCALITY_STRING                "pmix.locstr"           // (char*) string describing a proc's location
 #define PMIX_HWLOC_SHMEM_ADDR               "pmix.hwlocaddr"        // (size_t) address of HWLOC shared memory segment
@@ -263,6 +285,8 @@ typedef uint32_t pmix_rank_t;
 #define PMIX_HWLOC_SHMEM_FILE               "pmix.hwlocfile"        // (char*) path to HWLOC shared memory file
 #define PMIX_HWLOC_XML_V1                   "pmix.hwlocxml1"        // (char*) XML representation of local topology using HWLOC v1.x format
 #define PMIX_HWLOC_XML_V2                   "pmix.hwlocxml2"        // (char*) XML representation of local topology using HWLOC v2.x format
+#define PMIX_HWLOC_SHARE_TOPO               "pmix.hwlocsh"          // (bool) Share the HWLOC topology via shared memory
+#define PMIX_HWLOC_HOLE_KIND                "pmix.hwlocholek"       // (char*) Kind of VM "hole" HWLOC should use for shared memory
 
 
 /* request-related info */
@@ -291,7 +315,7 @@ typedef uint32_t pmix_rank_t;
 /* attributes used by host server to pass data to the server convenience library - the
  * data will then be parsed and provided to the local clients */
 #define PMIX_REGISTER_NODATA                "pmix.reg.nodata"       // (bool) Registration is for nspace only, do not copy job data
-#define PMIX_PROC_DATA                      "pmix.pdata"            // (pmix_data_array_t) starts with rank, then contains more data
+#define PMIX_PROC_DATA                      "pmix.pdata"            // (pmix_data_array_t*) starts with rank, then contains more data
 #define PMIX_NODE_MAP                       "pmix.nmap"             // (char*) regex of nodes containing procs for this job
 #define PMIX_PROC_MAP                       "pmix.pmap"             // (char*) regex describing procs on each node within this job
 #define PMIX_ANL_MAP                        "pmix.anlmap"           // (char*) process mapping in ANL notation (used in PMI-1/PMI-2)
@@ -317,7 +341,7 @@ typedef uint32_t pmix_rank_t;
 #define PMIX_EVENT_HDLR_PREPEND             "pmix.evprepend"        // (bool) prepend this handler to the precedence list within its category
 #define PMIX_EVENT_HDLR_APPEND              "pmix.evappend"         // (bool) append this handler to the precedence list within its category
 #define PMIX_EVENT_CUSTOM_RANGE             "pmix.evrange"          // (pmix_data_array_t*) array of pmix_proc_t defining range of event notification
-#define PMIX_EVENT_AFFECTED_PROC            "pmix.evproc"           // (pmix_proc_t) single proc that was affected
+#define PMIX_EVENT_AFFECTED_PROC            "pmix.evproc"           // (pmix_proc_t*) single proc that was affected
 #define PMIX_EVENT_AFFECTED_PROCS           "pmix.evaffected"       // (pmix_data_array_t*) array of pmix_proc_t defining affected procs
 #define PMIX_EVENT_NON_DEFAULT              "pmix.evnondef"         // (bool) event is not to be delivered to default event handlers
 #define PMIX_EVENT_RETURN_OBJECT            "pmix.evobject"         // (void*) object to be returned whenever the registered cbfunc is invoked
@@ -335,7 +359,8 @@ typedef uint32_t pmix_rank_t;
 #define PMIX_EVENT_ACTION_TIMEOUT           "pmix.evtimeout"        // (int) time in sec before RM will execute error response
 #define PMIX_EVENT_NO_TERMINATION           "pmix.evnoterm"         // (bool) indicates that the handler has satisfactorily handled
                                                                     //        the event and believes termination of the application is not required
-#define PMIX_EVENT_WANT_TERMINATION         "pmix.evterm"           // (bool) indicates that the handler has determined that the application should be terminated
+#define PMIX_EVENT_WANT_TERMINATION         "pmix.evterm"           // (bool) indicates that the handler has determined that the
+                                                                    //        application should be terminated
 
 
 /* attributes used to describe "spawn" directives */
@@ -355,8 +380,8 @@ typedef uint32_t pmix_rank_t;
 #define PMIX_PRELOAD_BIN                    "pmix.preloadbin"       // (bool) preload binaries
 #define PMIX_PRELOAD_FILES                  "pmix.preloadfiles"     // (char*) comma-delimited list of files to pre-position
 #define PMIX_NON_PMI                        "pmix.nonpmi"           // (bool) spawned procs will not call PMIx_Init
-#define PMIX_STDIN_TGT                      "pmix.stdin"            // (pmix_proc_t) proc that is to receive stdin
-                                                                    //               (PMIX_RANK_WILDCARD = all in given nspace)
+#define PMIX_STDIN_TGT                      "pmix.stdin"            // (pmix_proc_t*) proc that is to receive stdin
+                                                                    //                (PMIX_RANK_WILDCARD = all in given nspace)
 #define PMIX_DEBUGGER_DAEMONS               "pmix.debugger"         // (bool) spawned app consists of debugger daemons
 #define PMIX_COSPAWN_APP                    "pmix.cospawn"          // (bool) designated app is to be spawned as a disconnected
                                                                     //        job - i.e., not part of the "comm_world" of the job
@@ -376,7 +401,7 @@ typedef uint32_t pmix_rank_t;
 #define PMIX_JOB_CONTINUOUS                 "pmix.continuous"       // (bool) application is continuous, all failed procs should
                                                                         //        be immediately restarted
 #define PMIX_MAX_RESTARTS                   "pmix.maxrestarts"      // (uint32_t) max number of times to restart a job
-#define PMIX_FWD_STDIN                      "pmix.fwd.stdin"        // (bool) forward the stdin from this process to the spawned processes
+#define PMIX_FWD_STDIN                      "pmix.fwd.stdin"        // (bool) forward the stdin from this process to the target processes
 #define PMIX_FWD_STDOUT                     "pmix.fwd.stdout"       // (bool) forward stdout from the spawned processes to this process (typically used by a tool)
 #define PMIX_FWD_STDERR                     "pmix.fwd.stderr"       // (bool) forward stderr from the spawned processes to this process (typically used by a tool)
 #define PMIX_FWD_STDDIAG                    "pmix.fwd.stddiag"      // (bool) if a diagnostic channel exists, forward any output on it
@@ -401,9 +426,9 @@ typedef uint32_t pmix_rank_t;
 #define PMIX_QUERY_QUEUE_LIST               "pmix.qry.qlst"         // (char*) request a comma-delimited list of scheduler queues
 #define PMIX_QUERY_QUEUE_STATUS             "pmix.qry.qst"          // (TBD) status of a specified scheduler queue
 #define PMIX_QUERY_PROC_TABLE               "pmix.qry.ptable"       // (char*) input nspace of job whose info is being requested
-                                                                    //         returns (pmix_data_array_t) an array of pmix_proc_info_t
+                                                                    //         returns (pmix_data_array_t*) an array of pmix_proc_info_t
 #define PMIX_QUERY_LOCAL_PROC_TABLE         "pmix.qry.lptable"      // (char*) input nspace of job whose info is being requested
-                                                                    //         returns (pmix_data_array_t) an array of pmix_proc_info_t for
+                                                                    //         returns (pmix_data_array_t*) an array of pmix_proc_info_t for
                                                                     //         procs in job on same node
 #define PMIX_QUERY_AUTHORIZATIONS           "pmix.qry.auths"        // (bool) return operations tool is authorized to perform
 #define PMIX_QUERY_SPAWN_SUPPORT            "pmix.qry.spawn"        // (bool) return a comma-delimited list of supported spawn attributes
@@ -418,21 +443,46 @@ typedef uint32_t pmix_rank_t;
                                                                     //         for the specified nspace
 
 /* log attributes */
+#define PMIX_LOG_SOURCE                     "pmix.log.source"       // (pmix_proc_t*) ID of source of the log request
 #define PMIX_LOG_STDERR                     "pmix.log.stderr"       // (char*) log string to stderr
 #define PMIX_LOG_STDOUT                     "pmix.log.stdout"       // (char*) log string to stdout
-#define PMIX_LOG_SYSLOG                     "pmix.log.syslog"       // (char*) log data to syslog - defaults to ERROR priority unless
+#define PMIX_LOG_SYSLOG                     "pmix.log.syslog"       // (char*) log message to syslog - defaults to ERROR priority. Will log
+                                                                    //         to global syslog if available, otherwise to local syslog
+#define PMIX_LOG_LOCAL_SYSLOG               "pmix.log.lsys"         // (char*) log msg to local syslog - defaults to ERROR priority
+#define PMIX_LOG_GLOBAL_SYSLOG              "pmix.log.gsys"         // (char*) forward data to system "master" and log msg to that syslog
+#define PMIX_LOG_SYSLOG_PRI                 "pmix.log.syspri"       // (int) syslog priority level
+
+#define PMIX_LOG_TIMESTAMP                  "pmix.log.tstmp"        // (time_t) timestamp for log report
+#define PMIX_LOG_GENERATE_TIMESTAMP         "pmix.log.gtstmp"       // (bool) generate timestamp for log
+#define PMIX_LOG_TAG_OUTPUT                 "pmix.log.tag"          // (bool) label the output stream with the channel name (e.g., "stdout")
+#define PMIX_LOG_TIMESTAMP_OUTPUT           "pmix.log.tsout"        // (bool) print timestamp in output string
+#define PMIX_LOG_XML_OUTPUT                 "pmix.log.xml"          // (bool) print the output stream in xml format
+#define PMIX_LOG_ONCE                       "pmix.log.once"         // (bool) only log this once with whichever channel can first support it
 #define PMIX_LOG_MSG                        "pmix.log.msg"          // (pmix_byte_object_t) message blob to be sent somewhere
-#define PMIX_LOG_EMAIL                      "pmix.log.email"        // (pmix_data_array_t) log via email based on pmix_info_t containing directives
+
+#define PMIX_LOG_EMAIL                      "pmix.log.email"        // (pmix_data_array_t*) log via email based on array of pmix_info_t
+                                                                    //         containing directives
 #define PMIX_LOG_EMAIL_ADDR                 "pmix.log.emaddr"       // (char*) comma-delimited list of email addresses that are to recv msg
+#define PMIX_LOG_EMAIL_SENDER_ADDR          "pmix.log.emfaddr"      // (char*) return email address of sender
 #define PMIX_LOG_EMAIL_SUBJECT              "pmix.log.emsub"        // (char*) subject line for email
 #define PMIX_LOG_EMAIL_MSG                  "pmix.log.emmsg"        // (char*) msg to be included in email
+#define PMIX_LOG_EMAIL_SERVER               "pmix.log.esrvr"        // (char*) hostname (or IP addr) of estmp server
+#define PMIX_LOG_EMAIL_SRVR_PORT            "pmix.log.esrvrprt"     // (int32_t) port the email server is listening to
+
+#define PMIX_LOG_GLOBAL_DATASTORE           "pmix.log.gstore"       // (bool)
+#define PMIX_LOG_JOB_RECORD                 "pmix.log.jrec"         // (bool) log the provided information to the RM's job record
+
 
 /* debugger attributes */
 #define PMIX_DEBUG_STOP_ON_EXEC             "pmix.dbg.exec"         // (bool) job is being spawned under debugger - instruct it to pause on start
 #define PMIX_DEBUG_STOP_IN_INIT             "pmix.dbg.init"         // (bool) instruct job to stop during PMIx init
 #define PMIX_DEBUG_WAIT_FOR_NOTIFY          "pmix.dbg.notify"       // (bool) block at desired point until receiving debugger release notification
-#define PMIX_DEBUG_JOB                      "pmix.dbg.job"          // (char*) nspace of the job to be debugged - the RM/PMIx server are
+#define PMIX_DEBUG_JOB                      "pmix.dbg.job"          // (char*) nspace of the job assigned to this debugger to be debugged. Note
+                                                                    //         that id's, pids, and other info on the procs is available
+                                                                    //         via a query for the nspace's local or global proctable
 #define PMIX_DEBUG_WAITING_FOR_NOTIFY       "pmix.dbg.waiting"      // (bool) job to be debugged is waiting for a release
+#define PMIX_DEBUG_JOB_DIRECTIVES           "pmix.dbg.jdirs"        // (pmix_data_array_t*) array of job-level directives
+#define PMIX_DEBUG_APP_DIRECTIVES           "pmix.dbg.adirs"        // (pmix_data_array_t*) array of app-level directives
 
 
 /* Resource Manager identification */
@@ -442,6 +492,7 @@ typedef uint32_t pmix_rank_t;
 /* environmental variable operation attributes */
 #define PMIX_SET_ENVAR                      "pmix.envar.set"          // (pmix_envar_t*) set the envar to the given value,
                                                                       //                 overwriting any pre-existing one
+#define PMIX_ADD_ENVAR                      "pmix.envar.add"          // (pmix_envar_t*) add envar, but do not overwrite any existing one
 #define PMIX_UNSET_ENVAR                    "pmix.envar.unset"        // (char*) unset the envar, if present
 #define PMIX_PREPEND_ENVAR                  "pmix.envar.prepnd"       // (pmix_envar_t*) prepend the given value to the
                                                                       //                 specified envar using the separator
@@ -459,14 +510,41 @@ typedef uint32_t pmix_rank_t;
 #define PMIX_ALLOC_NUM_CPU_LIST             "pmix.alloc.ncpulist"   // (char*) regex of #cpus for each node
 #define PMIX_ALLOC_CPU_LIST                 "pmix.alloc.cpulist"    // (char*) regex of specific cpus indicating the cpus involved.
 #define PMIX_ALLOC_MEM_SIZE                 "pmix.alloc.msize"      // (float) number of Mbytes
-#define PMIX_ALLOC_NETWORK                  "pmix.alloc.net"        // (array) array of pmix_info_t describing network resources. If not
-                                                                    //         given as part of an info struct that identifies the
-                                                                    //         impacted nodes, then the description will be applied
-                                                                    //         across all nodes in the requestor's allocation
-#define PMIX_ALLOC_NETWORK_ID               "pmix.alloc.netid"      // (char*) name of network
+#define PMIX_ALLOC_NETWORK                  "pmix.alloc.net"        // (pmix_data_array_t*) Array of pmix_info_t describing
+                                                                    //         network resource request. This must include at least:
+                                                                    //           * PMIX_ALLOC_NETWORK_ID
+                                                                    //           * PMIX_ALLOC_NETWORK_TYPE
+                                                                    //           * PMIX_ALLOC_NETWORK_ENDPTS
+                                                                    //         plus whatever other descriptors are desired
+#define PMIX_ALLOC_NETWORK_ID               "pmix.alloc.netid"      // (char*) key to be used when accessing this requested network allocation. The
+                                                                    //         allocation will be returned/stored as a pmix_data_array_t of
+                                                                    //         pmix_info_t indexed by this key and containing at least one
+                                                                    //         entry with the same key and the allocated resource description.
+                                                                    //         The type of the included value depends upon the network
+                                                                    //         support. For example, a TCP allocation might consist of a
+                                                                    //         comma-delimited string of socket ranges such as
+                                                                    //         "32000-32100,33005,38123-38146". Additional entries will consist
+                                                                    //         of any provided resource request directives, along with their
+                                                                    //         assigned values. Examples include:
+                                                                    //           * PMIX_ALLOC_NETWORK_TYPE - the type of resources provided
+                                                                    //           * PMIX_ALLOC_NETWORK_PLANE - if applicable, what plane the
+                                                                    //               resources were assigned from
+                                                                    //           * PMIX_ALLOC_NETWORK_QOS - the assigned QoS
+                                                                    //           * PMIX_ALLOC_BANDWIDTH - the allocated bandwidth
+                                                                    //           * PMIX_ALLOC_NETWORK_SEC_KEY - a security key for the requested
+                                                                    //               network allocation
+                                                                    //         NOTE: the assigned values may differ from those requested,
+                                                                    //         especially if the "required" flag was not set in the request
 #define PMIX_ALLOC_BANDWIDTH                "pmix.alloc.bw"         // (float) Mbits/sec
 #define PMIX_ALLOC_NETWORK_QOS              "pmix.alloc.netqos"     // (char*) quality of service level
-#define PMIX_ALLOC_TIME                     "pmix.alloc.time"       // (uint32_t) time in seconds
+#define PMIX_ALLOC_TIME                     "pmix.alloc.time"       // (uint32_t) time in seconds that the allocation shall remain valid
+#define PMIX_ALLOC_NETWORK_TYPE             "pmix.alloc.nettype"    // (char*) type of desired transport (e.g., tcp, udp)
+#define PMIX_ALLOC_NETWORK_PLANE            "pmix.alloc.netplane"   // (char*) id string for the NIC (aka plane) to be used for this allocation
+                                                                    //         (e.g., CIDR for Ethernet)
+#define PMIX_ALLOC_NETWORK_ENDPTS           "pmix.alloc.endpts"     // (size_t) number of endpoints to allocate per process
+#define PMIX_ALLOC_NETWORK_ENDPTS_NODE      "pmix.alloc.endpts.nd"  // (size_t) number of endpoints to allocate per node
+#define PMIX_ALLOC_NETWORK_SEC_KEY          "pmix.alloc.nsec"       // (pmix_byte_object_t) network security key
+
 
 /* job control attributes */
 #define PMIX_JOB_CTRL_ID                    "pmix.jctrl.id"         // (char*) provide a string identifier for this request
@@ -520,32 +598,44 @@ typedef uint32_t pmix_rank_t;
                                                                     //            generating the event
 
 /* security attributes */
-#define PMIX_CRED_TYPE                      "pmix.sec.ctype"        //  when passed in PMIx_Get_credential, a prioritized,
+#define PMIX_CRED_TYPE                      "pmix.sec.ctype"        // (char*) when passed in PMIx_Get_credential, a prioritized,
                                                                     // comma-delimited list of desired credential types for use
                                                                     // in environments where multiple authentication mechanisms
                                                                     // may be available. When returned in a callback function, a
                                                                     // string identifier of the credential type
+#define PMIX_CRYPTO_KEY                     "pmix.sec.key"          // (pmix_byte_object_t) blob containing crypto key
+
 
 /* IO Forwarding Attributes */
-#define PMIX_IOF_CACHE_SIZE        "pmix.iof.csize"       // (uint32_t) requested size of the server cache in bytes for each specified channel.
-                                                          //            By default, the server is allowed (but not required) to drop
-                                                          //            all bytes received beyond the max size
-#define PMIX_IOF_DROP_OLDEST       "pmix.iof.old"         // (bool) in an overflow situation, drop the oldest bytes to make room in the cache
-#define PMIX_IOF_DROP_NEWEST       "pmix.iof.new"         // (bool) in an overflow situation, drop any new bytes received until room becomes
-                                                          //        available in the cache (default)
-#define PMIX_IOF_BUFFERING_SIZE    "pmix.iof.bsize"       // (uint32_t) basically controls grouping of IO on the specified channel(s) to
-                                                          //            avoid being called every time a bit of IO arrives. The library
-                                                          //            will execute the callback whenever the specified number of bytes
-                                                          //            becomes available. Any remaining buffered data will be "flushed"
-                                                          //            upon call to deregister the respective channel
-#define PMIX_IOF_BUFFERING_TIME    "pmix.iof.btime"       // (uint32_t) max time in seconds to buffer IO before delivering it. Used in conjunction
-                                                          //            with buffering size, this prevents IO from being held indefinitely
-                                                          //            while waiting for another payload to arrive
-#define PMIX_IOF_COMPLETE          "pmix.iof.cmp"         // (bool) indicates whether or not the specified IO channel has been closed
-                                                          //        by the source
-#define PMIX_IOF_PUSH_STDIN        "pmix.iof.stdin"       // (bool) Used by a tool to request that the PMIx library collect
-                                                          //        the tool's stdin and forward it to the procs specified in
-                                                          //        the PMIx_IOF_push call
+#define PMIX_IOF_CACHE_SIZE                 "pmix.iof.csize"        // (uint32_t) requested size of the server cache in bytes for each specified channel.
+                                                                    //            By default, the server is allowed (but not required) to drop
+                                                                    //            all bytes received beyond the max size
+#define PMIX_IOF_DROP_OLDEST                "pmix.iof.old"          // (bool) in an overflow situation, drop the oldest bytes to make room in the cache
+#define PMIX_IOF_DROP_NEWEST                "pmix.iof.new"          // (bool) in an overflow situation, drop any new bytes received until room becomes
+                                                                    //        available in the cache (default)
+#define PMIX_IOF_BUFFERING_SIZE             "pmix.iof.bsize"        // (uint32_t) basically controls grouping of IO on the specified channel(s) to
+                                                                    //            avoid being called every time a bit of IO arrives. The library
+                                                                    //            will execute the callback whenever the specified number of bytes
+                                                                    //            becomes available. Any remaining buffered data will be "flushed"
+                                                                    //            upon call to deregister the respective channel
+#define PMIX_IOF_BUFFERING_TIME             "pmix.iof.btime"        // (uint32_t) max time in seconds to buffer IO before delivering it. Used in conjunction
+                                                                    //            with buffering size, this prevents IO from being held indefinitely
+                                                                    //            while waiting for another payload to arrive
+#define PMIX_IOF_COMPLETE                   "pmix.iof.cmp"          // (bool) indicates whether or not the specified IO channel has been closed
+                                                                    //        by the source
+#define PMIX_IOF_PUSH_STDIN                 "pmix.iof.stdin"        // (bool) Used by a tool to request that the PMIx library collect
+                                                                    //        the tool's stdin and forward it to the procs specified in
+                                                                    //        the PMIx_IOF_push call
+#define PMIX_IOF_TAG_OUTPUT                 "pmix.iof.tag"          // (bool) Tag output with the channel it comes from
+#define PMIX_IOF_TIMESTAMP_OUTPUT           "pmix.iof.ts"           // (bool) Timestamp output
+#define PMIX_IOF_XML_OUTPUT                 "pmix.iof.xml"          // (bool) Format output in XML
+
+
+/* Attributes for controlling contents of application setup data */
+#define PMIX_SETUP_APP_ENVARS               "pmix.setup.env"        // (bool) harvest and include relevant envars
+#define PMIX_SETUP_APP_NONENVARS            "pmix.setup.nenv"       // (bool) include all non-envar data
+#define PMIX_SETUP_APP_ALL                  "pmix.setup.all"        // (bool) include all relevant data
+
 
 /****    PROCESS STATE DEFINITIONS    ****/
 typedef uint8_t pmix_proc_state_t;
@@ -576,11 +666,13 @@ typedef uint8_t pmix_proc_state_t;
 #define PMIX_PROC_STATE_ABORTED_BY_SIG          (PMIX_PROC_STATE_ERROR +  4)  /* process aborted by signal */
 #define PMIX_PROC_STATE_TERM_WO_SYNC            (PMIX_PROC_STATE_ERROR +  5)  /* process exit'd w/o calling PMIx_Finalize */
 #define PMIX_PROC_STATE_COMM_FAILED             (PMIX_PROC_STATE_ERROR +  6)  /* process communication has failed */
-#define PMIX_PROC_STATE_CALLED_ABORT            (PMIX_PROC_STATE_ERROR +  7)  /* process called "PMIx_Abort" */
-#define PMIX_PROC_STATE_MIGRATING               (PMIX_PROC_STATE_ERROR +  8)  /* process failed and is waiting for resources before restarting */
-#define PMIX_PROC_STATE_CANNOT_RESTART          (PMIX_PROC_STATE_ERROR +  9)  /* process failed and cannot be restarted */
-#define PMIX_PROC_STATE_TERM_NON_ZERO           (PMIX_PROC_STATE_ERROR + 10)  /* process exited with a non-zero status, indicating abnormal */
-#define PMIX_PROC_STATE_FAILED_TO_LAUNCH        (PMIX_PROC_STATE_ERROR + 11)  /* unable to launch process */
+#define PMIX_PROC_STATE_SENSOR_BOUND_EXCEEDED   (PMIX_PROC_STATE_ERROR +  7)  /* process exceeded a sensor limit */
+#define PMIX_PROC_STATE_CALLED_ABORT            (PMIX_PROC_STATE_ERROR +  8)  /* process called "PMIx_Abort" */
+#define PMIX_PROC_STATE_HEARTBEAT_FAILED        (PMIX_PROC_STATE_ERROR +  9)  /* process failed to send heartbeat w/in time limit */
+#define PMIX_PROC_STATE_MIGRATING               (PMIX_PROC_STATE_ERROR + 10)  /* process failed and is waiting for resources before restarting */
+#define PMIX_PROC_STATE_CANNOT_RESTART          (PMIX_PROC_STATE_ERROR + 11)  /* process failed and cannot be restarted */
+#define PMIX_PROC_STATE_TERM_NON_ZERO           (PMIX_PROC_STATE_ERROR + 12)  /* process exited with a non-zero status, indicating abnormal */
+#define PMIX_PROC_STATE_FAILED_TO_LAUNCH        (PMIX_PROC_STATE_ERROR + 13)  /* unable to launch process */
 
 
 /****    PMIX ERROR CONSTANTS    ****/
@@ -686,6 +778,10 @@ typedef int pmix_status_t;
 #define PMIX_MODEL_RESOURCES                    (PMIX_ERR_OP_BASE - 21)     // model resource usage has changed
 #define PMIX_OPENMP_PARALLEL_ENTERED            (PMIX_ERR_OP_BASE - 22)     // an OpenMP parallel region has been entered
 #define PMIX_OPENMP_PARALLEL_EXITED             (PMIX_ERR_OP_BASE - 23)     // an OpenMP parallel region has completed
+#define PMIX_LAUNCH_DIRECTIVE                   (PMIX_ERR_OP_BASE - 24)
+#define PMIX_LAUNCHER_READY                     (PMIX_ERR_OP_BASE - 25)
+#define PMIX_OPERATION_IN_PROGRESS              (PMIX_ERR_OP_BASE - 26)
+
 
 /* define a starting point for system error constants so
  * we avoid renumbering when making additions */
@@ -806,6 +902,7 @@ typedef uint8_t pmix_data_range_t;
 #define PMIX_RANGE_GLOBAL       5   // data available to all procs
 #define PMIX_RANGE_CUSTOM       6   // range is specified in a pmix_info_t
 #define PMIX_RANGE_PROC_LOCAL   7   // restrict range to the local proc
+#define PMIX_RANGE_INVALID   UINT8_MAX
 
 /* define a "persistence" policy for data published by clients */
 typedef uint8_t pmix_persistence_t;
@@ -814,12 +911,16 @@ typedef uint8_t pmix_persistence_t;
 #define PMIX_PERSIST_PROC           2   // retain until publishing process terminates
 #define PMIX_PERSIST_APP            3   // retain until application terminates
 #define PMIX_PERSIST_SESSION        4   // retain until session/allocation terminates
+#define PMIX_PERSIST_INVALID   UINT8_MAX
 
 /* define a set of bit-mask flags for specifying behavior of
  * command directives via pmix_info_t arrays */
 typedef uint32_t pmix_info_directives_t;
-#define PMIX_INFO_REQD          0x0001
-
+#define PMIX_INFO_REQD          0x00000001
+/* the top 16-bits are reserved for internal use by
+ * implementers - these may be changed inside the
+ * PMIx library */
+#define PMIX_INFO_DIR_RESERVED 0xffff0000
 
 /* define a set of directives for allocation requests */
 typedef uint8_t pmix_alloc_directive_t;
@@ -935,9 +1036,11 @@ typedef struct {
     do {                               \
         if (NULL != (m)->envar) {      \
             free((m)->envar);          \
+            (m)->envar = NULL;         \
         }                              \
         if (NULL != (m)->value) {      \
             free((m)->value);          \
+            (m)->value = NULL;         \
         }                              \
     } while(0)
 #define PMIX_ENVAR_LOAD(m, e, v, s)    \
@@ -986,9 +1089,28 @@ typedef struct pmix_data_buffer {
     do {                                    \
         if (NULL != (m)->base_ptr) {        \
             free((m)->base_ptr);            \
+            (m)->base_ptr = NULL;           \
         }                                   \
+        (m)->pack_ptr = NULL;               \
+        (m)->unpack_ptr = NULL;             \
+        (m)->bytes_allocated = 0;           \
+        (m)->bytes_used = 0;                \
     } while (0)
+#define PMIX_DATA_BUFFER_LOAD(b, d, s)          \
+    do {                                        \
+        (b)->base_ptr = (char*)(d);             \
+        (b)->pack_ptr = (b)->base_ptr + (s);    \
+        (b)->unpack_ptr = (b)->base_ptr;        \
+        (b)->bytes_allocated = (s);             \
+        (b)->bytes_used = (s);                  \
+    } while(0)
 
+#define PMIX_DATA_BUFFER_UNLOAD(b, d, s)    \
+    do {                                    \
+        (d) = (b)->base_ptr;                \
+        (s) = (b)->bytes_used;              \
+        (b)->base_ptr = NULL;               \
+    } while(0)
 
 /****    PMIX PROC OBJECT    ****/
 typedef struct pmix_proc {
@@ -1017,6 +1139,7 @@ typedef struct pmix_proc {
     do {                                        \
         if (NULL != (m)) {                      \
             free((m));                          \
+            (m) = NULL;                         \
         }                                       \
     } while (0)
 
@@ -1082,9 +1205,11 @@ typedef struct pmix_proc_info {
     do {                                        \
         if (NULL != (m)->hostname) {            \
             free((m)->hostname);                \
+            (m)->hostname = NULL;               \
         }                                       \
         if (NULL != (m)->executable_name) {     \
             free((m)->executable_name);         \
+            (m)->executable_name = NULL;        \
         }                                       \
     } while(0)
 
@@ -1108,6 +1233,16 @@ typedef struct pmix_data_array {
     size_t size;
     void *array;
 } pmix_data_array_t;
+#define PMIX_DATA_ARRAY_CONSTRUCT(m, n, t)         \
+    do {                                           \
+        (m)->type = (t);                           \
+        (m)->size = (n);                           \
+    } while(0)
+#define PMIX_DATA_ARRAY_CREATE(m, n, t)                                   \
+    do {                                                                  \
+        (m) = (pmix_data_array_t*)calloc(1, sizeof(pmix_data_array_t));   \
+        PMIX_DATA_ARRAY_CONSTRUCT((m), (n), (t));                         \
+    } while(0)
 
 typedef struct pmix_info_array {
     size_t size;
@@ -1179,6 +1314,7 @@ typedef struct pmix_value {
     do {                            \
         PMIX_VALUE_DESTRUCT((m));   \
         free((m));                  \
+        (m) = NULL;                 \
     } while (0)
 
 /* initialize a single value struct */
@@ -1189,86 +1325,7 @@ typedef struct pmix_value {
     } while (0)
 
 /* release the memory in the value struct data field */
-#define PMIX_VALUE_DESTRUCT(m)                                                      \
-    do {                                                                            \
-        size_t _n;                                                                  \
-        if (PMIX_STRING == (m)->type) {                                             \
-            if (NULL != (m)->data.string) {                                         \
-                free((m)->data.string);                                             \
-            }                                                                       \
-        } else if ((PMIX_BYTE_OBJECT == (m)->type) ||                               \
-                   (PMIX_COMPRESSED_STRING == (m)->type)) {                         \
-            if (NULL != (m)->data.bo.bytes) {                                       \
-                free((m)->data.bo.bytes);                                           \
-            }                                                                       \
-        } else if (PMIX_DATA_ARRAY == (m)->type) {                                  \
-            if (NULL != (m)->data.darray && NULL != (m)->data.darray->array) {      \
-                if (PMIX_STRING == (m)->data.darray->type) {                        \
-                    char **_str = (char**)(m)->data.darray->array;                  \
-                    for (_n=0; _n < (m)->data.darray->size; _n++) {                 \
-                        if (NULL != _str[_n]) {                                     \
-                            free(_str[_n]);                                         \
-                        }                                                           \
-                    }                                                               \
-                } else if (PMIX_PROC_INFO == (m)->data.darray->type) {              \
-                    pmix_proc_info_t *_info =                                       \
-                                (pmix_proc_info_t*)(m)->data.darray->array;         \
-                    for (_n=0; _n < (m)->data.darray->size; _n++) {                 \
-                        PMIX_PROC_INFO_DESTRUCT(&_info[_n]);                        \
-                    }                                                               \
-                } else if (PMIX_INFO == (m)->data.darray->type) {                   \
-                    pmix_info_t *_info =                                            \
-                                (pmix_info_t*)(m)->data.darray->array;              \
-                    for (_n=0; _n < (m)->data.darray->size; _n++) {                 \
-                        /* cannot use info destruct as that loops back */           \
-                        if (PMIX_STRING == _info[_n].value.type) {                  \
-                            if (NULL != _info[_n].value.data.string) {              \
-                                free(_info[_n].value.data.string);                  \
-                            }                                                       \
-                        } else if (PMIX_BYTE_OBJECT == _info[_n].value.type) {      \
-                            if (NULL != _info[_n].value.data.bo.bytes) {            \
-                                free(_info[_n].value.data.bo.bytes);                \
-                            }                                                       \
-                        } else if (PMIX_PROC_INFO == _info[_n].value.type) {        \
-                            PMIX_PROC_INFO_DESTRUCT(_info[_n].value.data.pinfo);    \
-                        }                                                           \
-                    }                                                               \
-                } else if (PMIX_BYTE_OBJECT == (m)->data.darray->type) {            \
-                    pmix_byte_object_t *_obj =                                      \
-                                (pmix_byte_object_t*)(m)->data.darray->array;       \
-                    for (_n=0; _n < (m)->data.darray->size; _n++) {                 \
-                        if (NULL != _obj[_n].bytes) {                               \
-                            free(_obj[_n].bytes);                                   \
-                        }                                                           \
-                    }                                                               \
-                }                                                                   \
-                free((m)->data.darray->array);                                      \
-            }                                                                       \
-            if (NULL != (m)->data.darray) {                                         \
-                free((m)->data.darray);                                             \
-            }                                                                       \
-        /**** DEPRECATED ****/                                                      \
-        } else if (PMIX_INFO_ARRAY == (m)->type) {                                  \
-            pmix_info_t *_p = (pmix_info_t*)((m)->data.array->array);               \
-            for (_n=0; _n < (m)->data.array->size; _n++) {                          \
-                if (PMIX_STRING == _p[_n].value.type) {                             \
-                    if (NULL != _p[_n].value.data.string) {                         \
-                        free(_p[_n].value.data.string);                             \
-                    }                                                               \
-                } else if (PMIX_BYTE_OBJECT == _p[_n].value.type) {                 \
-                    if (NULL != _p[_n].value.data.bo.bytes) {                       \
-                        free(_p[_n].value.data.bo.bytes);                           \
-                    }                                                               \
-                } else if (PMIX_PROC_INFO == _p[_n].value.type) {                   \
-                    PMIX_PROC_INFO_DESTRUCT(_p[_n].value.data.pinfo);               \
-                }                                                                   \
-            }                                                                       \
-            free(_p);                                                               \
-        /********************/                                                      \
-        } else if (PMIX_ENVAR == (m)->type) {                                       \
-            PMIX_ENVAR_DESTRUCT(&(m)->data.envar);                                  \
-        }                                                                           \
-    } while (0)
+#define PMIX_VALUE_DESTRUCT(m) pmix_value_destruct(m)
 
 #define PMIX_VALUE_FREE(m, n)                           \
     do {                                                \
@@ -1278,9 +1335,44 @@ typedef struct pmix_value {
                 PMIX_VALUE_DESTRUCT(&((m)[_s]));        \
             }                                           \
             free((m));                                  \
+            (m) = NULL;                                 \
         }                                               \
     } while (0)
 
+#define PMIX_VALUE_GET_NUMBER(s, m, n, t)               \
+    do {                                                \
+        (s) = PMIX_SUCCESS;                             \
+        if (PMIX_SIZE == (m)->type) {                   \
+            (n) = (t)((m)->data.size);                  \
+        } else if (PMIX_INT == (m)->type) {             \
+            (n) = (t)((m)->data.integer);               \
+        } else if (PMIX_INT8 == (m)->type) {            \
+            (n) = (t)((m)->data.int8);                  \
+        } else if (PMIX_INT16 == (m)->type) {           \
+            (n) = (t)((m)->data.int16);                 \
+        } else if (PMIX_INT32 == (m)->type) {           \
+            (n) = (t)((m)->data.int32);                 \
+        } else if (PMIX_INT64 == (m)->type) {           \
+            (n) = (t)((m)->data.int64);                 \
+        } else if (PMIX_UINT == (m)->type) {            \
+            (n) = (t)((m)->data.uint);                  \
+        } else if (PMIX_UINT8 == (m)->type) {           \
+            (n) = (t)((m)->data.uint8);                 \
+        } else if (PMIX_UINT16 == (m)->type) {          \
+            (n) = (t)((m)->data.uint16);                \
+        } else if (PMIX_UINT32 == (m)->type) {          \
+            (n) = (t)((m)->data.uint32);                \
+        } else if (PMIX_UINT64 == (m)->type) {          \
+            (n) = (t)((m)->data.uint64);                \
+        } else if (PMIX_FLOAT == (m)->type) {           \
+            (n) = (t)((m)->data.fval);                  \
+        } else if (PMIX_DOUBLE == (m)->type) {          \
+            (n) = (t)((m)->data.dval);                  \
+        } else {                                        \
+            (s) = PMIX_ERR_BAD_PARAM;                   \
+        }                                               \
+    } while(0)
+
 /* expose some functions that are resolved in the
  * PMIx library, but part of a header that
  * includes internal functions - we don't
@@ -1291,6 +1383,10 @@ void pmix_value_load(pmix_value_t *v, const void *data, pmix_data_type_t type);
 #define PMIX_VALUE_LOAD(v, d, t) \
     pmix_value_load((v), (d), (t))
 
+pmix_status_t pmix_value_unload(pmix_value_t *kv, void **data, size_t *sz);
+#define PMIX_VALUE_UNLOAD(r, k, d, s)      \
+    (r) = pmix_value_unload((k), (d), (s))
+
 pmix_status_t pmix_value_xfer(pmix_value_t *kv, pmix_value_t *src);
 #define PMIX_VALUE_XFER(r, v, s)                                \
     do {                                                        \
@@ -1347,25 +1443,36 @@ struct pmix_info_t {
                 PMIX_INFO_DESTRUCT(&((m)[_s])); \
             }                                   \
             free((m));                          \
+            (m) = NULL;                         \
         }                                       \
     } while (0)
 
-#define PMIX_INFO_LOAD(m, k, v, t)                      \
-    do {                                                \
-        (void)strncpy((m)->key, (k), PMIX_MAX_KEYLEN);  \
-        pmix_value_load(&((m)->value), (v), (t));       \
-    } while (0)
-#define PMIX_INFO_XFER(d, s)                                \
+#define PMIX_INFO_LOAD(m, k, v, t)                          \
     do {                                                    \
-        (void)strncpy((d)->key, (s)->key, PMIX_MAX_KEYLEN); \
-        (d)->flags = (s)->flags;                      \
-        pmix_value_xfer(&(d)->value, &(s)->value);          \
+        if (NULL != (k)) {                                  \
+            (void)strncpy((m)->key, (k), PMIX_MAX_KEYLEN);  \
+        }                                                   \
+        (m)->flags = 0;                                     \
+        pmix_value_load(&((m)->value), (v), (t));           \
+    } while (0)
+#define PMIX_INFO_XFER(d, s)                                    \
+    do {                                                        \
+        if (NULL != (s)->key) {                                 \
+            (void)strncpy((d)->key, (s)->key, PMIX_MAX_KEYLEN); \
+        }                                                       \
+        (d)->flags = (s)->flags;                                \
+        pmix_value_xfer(&(d)->value, &(s)->value);              \
     } while(0)
 
 #define PMIX_INFO_REQUIRED(m)       \
-    (m)->flags |= PMIX_INFO_REQD;
+    (m)->flags |= PMIX_INFO_REQD
 #define PMIX_INFO_OPTIONAL(m)       \
-    (m)->flags &= ~PMIX_INFO_REQD;
+    (m)->flags &= ~PMIX_INFO_REQD
+
+#define PMIX_INFO_IS_REQUIRED(m)    \
+    (m)->flags & PMIX_INFO_REQD
+#define PMIX_INFO_IS_OPTIONAL(m)    \
+    !((m)->flags & PMIX_INFO_REQD)
 
 #define PMIX_INFO_UNLOAD(r, v, l)                              \
     do {                                                       \
@@ -1380,7 +1487,9 @@ struct pmix_info_t {
                 (r) = PMIX_ERR_NOMEM;                          \
                 break;                                         \
             }                                                  \
-            _kv->key = strdup(_info[_n].key);                  \
+            if (NULL != _info[_n].key) {                       \
+                _kv->key = strdup(_info[_n].key);              \
+            }                                                  \
             PMIX_VALUE_XFER((r), _kv->value, &_info[_n].value);\
             if (PMIX_SUCCESS != (r)) {                         \
                 PMIX_RELEASE(_kv);                             \
@@ -1415,6 +1524,7 @@ typedef struct pmix_pdata {
     do {                                        \
         PMIX_VALUE_DESTRUCT(&(m)->value);       \
         free((m));                              \
+        (m) = NULL;                             \
     } while (0)
 
 #define PMIX_PDATA_CONSTRUCT(m)                 \
@@ -1436,6 +1546,7 @@ typedef struct pmix_pdata {
                 PMIX_PDATA_DESTRUCT(&((m)[_s]));        \
             }                                           \
             free((m));                                  \
+            (m) = NULL;                                 \
         }                                               \
     } while (0)
 
@@ -1482,6 +1593,7 @@ typedef struct pmix_app {
     do {                                        \
         PMIX_APP_DESTRUCT((m));                 \
         free((m));                              \
+        (m) = NULL;                             \
     } while (0)
 
 #define PMIX_APP_CONSTRUCT(m)                   \
@@ -1494,27 +1606,32 @@ typedef struct pmix_app {
         size_t _ii;                                             \
         if (NULL != (m)->cmd) {                                 \
             free((m)->cmd);                                     \
+            (m)->cmd = NULL;                                    \
         }                                                       \
         if (NULL != (m)->argv) {                                \
             for (_ii=0; NULL != (m)->argv[_ii]; _ii++) {        \
                 free((m)->argv[_ii]);                           \
             }                                                   \
             free((m)->argv);                                    \
+            (m)->argv = NULL;                                   \
         }                                                       \
         if (NULL != (m)->env) {                                 \
             for (_ii=0; NULL != (m)->env[_ii]; _ii++) {         \
                 free((m)->env[_ii]);                            \
             }                                                   \
             free((m)->env);                                     \
+            (m)->env = NULL;                                    \
         }                                                       \
         if (NULL != (m)->cwd) {                                 \
             free((m)->cwd);                                     \
+            (m)->cwd = NULL;                                    \
         }                                                       \
         if (NULL != (m)->info) {                                \
             for (_ii=0; _ii < (m)->ninfo; _ii++) {              \
                 PMIX_INFO_DESTRUCT(&(m)->info[_ii]);            \
             }                                                   \
             free((m)->info);                                    \
+            (m)->info = NULL;                                   \
         }                                                       \
     } while (0)
 
@@ -1526,6 +1643,7 @@ typedef struct pmix_app {
                 PMIX_APP_DESTRUCT(&((m)[_s]));  \
             }                                   \
             free((m));                          \
+            (m) = NULL;                         \
         }                                       \
     } while (0)
 
@@ -1546,6 +1664,7 @@ typedef struct pmix_query {
     do {                            \
         PMIX_QUERY_DESTRUCT((m));   \
         free((m));                  \
+        (m) = NULL;                 \
     } while (0)
 
 #define PMIX_QUERY_CONSTRUCT(m)                 \
@@ -1561,12 +1680,14 @@ typedef struct pmix_query {
                 free((m)->keys[_ii]);                           \
             }                                                   \
             free((m)->keys);                                    \
+            (m)->keys = NULL;                                   \
         }                                                       \
         if (NULL != (m)->qualifiers) {                          \
             for (_ii=0; _ii < (m)->nqual; _ii++) {              \
                 PMIX_INFO_DESTRUCT(&(m)->qualifiers[_ii]);      \
             }                                                   \
             free((m)->qualifiers);                              \
+            (m)->qualifiers = NULL;                             \
         }                                                       \
     } while (0)
 
@@ -1578,6 +1699,7 @@ typedef struct pmix_query {
                 PMIX_QUERY_DESTRUCT(&((m)[_s]));    \
             }                                       \
             free((m));                              \
+            (m) = NULL;                             \
         }                                           \
     } while (0)
 
@@ -1601,6 +1723,7 @@ typedef struct pmix_modex_data {
     do {                                        \
         PMIX_MODEX_DESTRUCT((m));               \
         free((m));                              \
+        (m) = NULL;                             \
     } while (0)
 
 #define PMIX_MODEX_CONSTRUCT(m)                         \
@@ -1612,6 +1735,7 @@ typedef struct pmix_modex_data {
     do {                                        \
         if (NULL != (m)->blob) {                \
             free((m)->blob);                    \
+            (m)->blob = NULL;                   \
         }                                       \
     } while (0)
 
@@ -1623,6 +1747,7 @@ typedef struct pmix_modex_data {
                 PMIX_MODEX_DESTRUCT(&((m)[_s]));        \
             }                                           \
             free((m));                                  \
+            (m) = NULL;                                 \
         }                                               \
     } while (0)
 
@@ -1655,18 +1780,6 @@ typedef void (*pmix_modex_cbfunc_t)(pmix_status_t status,
 typedef void (*pmix_spawn_cbfunc_t)(pmix_status_t status,
                                     char nspace[], void *cbdata);
 
-/* define a callback function for calls to PMIx_Connect_nb - the function
- * will be called upon completion of the command. The status will indicate
- * whether or not the connect operation succeeded. The nspace will contain
- * the new identity assigned by the host RM to the specified group of
- * processes, and the rank will be the rank of this process within that new
- * group. Note that the returned nspace value may be
- * released by the library upon return from the callback function, so
- * the receiver must copy it if it needs to be retained */
-typedef void (*pmix_connect_cbfunc_t)(pmix_status_t status,
-                                      char nspace[], int rank,
-                                      void *cbdata);
-
 /* define a callback for common operations that simply return
  * a status. Examples include the non-blocking versions of
  * Fence, Connect, and Disconnect */
@@ -1908,13 +2021,8 @@ PMIX_EXPORT void PMIx_Deregister_event_handler(size_t evhdlr_ref,
                                                pmix_op_cbfunc_t cbfunc,
                                                void *cbdata);
 
-/* Report an event to a process for notification via any
- * registered evhdlr. The evhdlr registration can be
- * called by both the server and the client application. On the
- * server side, the evhdlr is used to report events detected
- * by PMIx to the host server for handling. On the client side,
- * the evhdlr is used to notify the process of events
- * reported by the server - e.g., the failure of another process.
+/* Report an event for notification via any
+ * registered evhdlr.
  *
  * This function allows the host server to direct the server
  * convenience library to notify all registered local procs of
@@ -1922,22 +2030,37 @@ PMIX_EXPORT void PMIx_Deregister_event_handler(size_t evhdlr_ref,
  * The status indicates the event being reported.
  *
  * The client application can also call this function to notify the
- * resource manager of an event it encountered. It can request the host
- * server to notify the indicated processes about the event.
+ * resource manager and/or other processes of an event it encountered.
+ * It can also be used to asynchronously notify other parts of its
+ * own internal process - e.g., for one library to notify another
+ * when initialized inside the process.
  *
- * The array of procs identifies the processes that will be impacted
- * by the event. This could consist of a single process, or a number
- * of processes.
+ * status - status code indicating the event being reported
  *
- * The info array contains any further info the RM can and/or chooses
- * to provide.
+ * source - the process that generated the event
  *
- * The callback function will be called upon completion of the
- * notify_event function's actions. Note that any messages will
- * have been queued, but may not have been transmitted by this
- * time. Note that the caller is required to maintain the input
- * data until the callback function has been executed!
-*/
+ * range - the range in which the event is to be reported. For example,
+ *         a value of PMIX_RANGE_LOCAL would instruct the system
+ *         to only notify procs on the same local node as the
+ *         event generator.
+ *
+ * info - an array of pmix_info_t structures provided by the event
+ *        generator to pass any additional information about the
+ *        event. This can include an array of pmix_proc_t structs
+ *        describing the processes impacted by the event, the nature
+ *        of the event and its severity, etc. The precise contents
+ *        of the array will depend on the event generator.
+ *
+ * ninfo - number of elements in the info array
+ *
+ * cbfunc - callback function to be called upon completion of the
+ *          notify_event function's actions. Note that any messages
+ *          will have been queued, but may not have been transmitted
+ *          by this time. Note that the caller is required to maintain
+ *          the input data until the callback function has been executed!
+ *
+ * cbdata - the caller's provided void* object
+ */
 PMIX_EXPORT pmix_status_t PMIx_Notify_event(pmix_status_t status,
                                             const pmix_proc_t *source,
                                             pmix_data_range_t range,
@@ -2268,6 +2391,81 @@ PMIX_EXPORT pmix_status_t PMIx_Data_copy_payload(pmix_data_buffer_t *dest,
 #define PMIX_VAL_FREE(_v) \
      PMIx_free_value_data(_v)
 
+static inline void pmix_value_destruct(pmix_value_t * m) {
+    size_t _n;
+    if (PMIX_STRING == (m)->type) {
+        if (NULL != (m)->data.string) {
+            free((m)->data.string);
+            (m)->data.string = NULL;
+        }
+    } else if ((PMIX_BYTE_OBJECT == (m)->type) ||
+               (PMIX_COMPRESSED_STRING == (m)->type)) {
+        if (NULL != (m)->data.bo.bytes) {
+            free((m)->data.bo.bytes);
+            (m)->data.bo.bytes = NULL;
+            (m)->data.bo.size = 0;
+        }
+    } else if (PMIX_DATA_ARRAY == (m)->type) {
+        if (NULL != (m)->data.darray && NULL != (m)->data.darray->array) {
+            if (PMIX_STRING == (m)->data.darray->type) {
+                char **_str = (char**)(m)->data.darray->array;
+                for (_n=0; _n < (m)->data.darray->size; _n++) {
+                    if (NULL != _str[_n]) {
+                        free(_str[_n]);
+                    }
+                }
+            } else if (PMIX_PROC_INFO == (m)->data.darray->type) {
+                pmix_proc_info_t *_info =
+                            (pmix_proc_info_t*)(m)->data.darray->array;
+                for (_n=0; _n < (m)->data.darray->size; _n++) {
+                    PMIX_PROC_INFO_DESTRUCT(&_info[_n]);
+                }
+            } else if (PMIX_INFO == (m)->data.darray->type) {
+                pmix_info_t *_info =
+                            (pmix_info_t*)(m)->data.darray->array;
+                for (_n=0; _n < (m)->data.darray->size; _n++) {
+                    pmix_value_destruct(&_info[_n].value);
+                }
+            } else if (PMIX_BYTE_OBJECT == (m)->data.darray->type) {
+                pmix_byte_object_t *_obj =
+                            (pmix_byte_object_t*)(m)->data.darray->array;
+                for (_n=0; _n < (m)->data.darray->size; _n++) {
+                    if (NULL != _obj[_n].bytes) {
+                        free(_obj[_n].bytes);
+                    }
+                }
+            }
+            free((m)->data.darray->array);
+            (m)->data.darray->array = NULL;
+            (m)->data.darray->size = 0;
+        }
+        if (NULL != (m)->data.darray) {
+            free((m)->data.darray);
+            (m)->data.darray = NULL;
+        }
+    /**** DEPRECATED ****/
+    } else if (PMIX_INFO_ARRAY == (m)->type) {
+        pmix_info_t *_p = (pmix_info_t*)((m)->data.array->array);
+        for (_n=0; _n < (m)->data.array->size; _n++) {
+            if (PMIX_STRING == _p[_n].value.type) {
+                if (NULL != _p[_n].value.data.string) {
+                    free(_p[_n].value.data.string);
+                }
+            } else if (PMIX_BYTE_OBJECT == _p[_n].value.type) {
+                if (NULL != _p[_n].value.data.bo.bytes) {
+                    free(_p[_n].value.data.bo.bytes);
+                }
+            } else if (PMIX_PROC_INFO == _p[_n].value.type) {
+                PMIX_PROC_INFO_DESTRUCT(_p[_n].value.data.pinfo);
+            }
+        }
+        free(_p);
+    /********************/
+    } else if (PMIX_ENVAR == (m)->type) {
+        PMIX_ENVAR_DESTRUCT(&(m)->data.envar);
+    }
+}
+
 #if defined(c_plusplus) || defined(__cplusplus)
 }
 #endif

opal/mca/pmix/pmix3x/pmix/include/pmix_server.h

@@ -198,48 +198,38 @@ typedef pmix_status_t (*pmix_server_spawn_fn_t)(const pmix_proc_t *proc,
                                                 const pmix_app_t apps[], size_t napps,
                                                 pmix_spawn_cbfunc_t cbfunc, void *cbdata);
 
-/* Record the specified processes as "connected". This means that:
+/* Record the specified processes as "connected". This means that the resource
+ * manager should treat the failure of any process in the specified group as
+ * a reportable event, and take appropriate action. The callback function is
+ * to be called once all participating processes have called connect. Note that
+ * a process can only engage in *one* connect operation involving the identical
+ * set of procs at a time. However, a process *can* be simultaneously engaged
+ * in multiple connect operations, each involving a different set of procs
  *
- * (a) the resource manager should treat the specified group as
- *     a group when reporting events.
- *
- * (b) processes can address the group by the newly assigned nspace
- *     when passing requests
- *
- * As in the case of the fence operation, the info array can be used to pass
- * user-level directives regarding the algorithm to be used for the collective
- * operation involved in the "connect", timeout constraints, and other options
- * available from the host RM.
- *
- * The callback function will be called once the operation is complete. Any
- * participant that fails to call "connect" prior to terminating will cause the
- * operation to return a "failed" status to all other participants. This
- * is the default behavior in the absence of any provided directive.
- *
- * Some additional info keys are provided for this operation:
- *
- * (a) PMIX_CONNECT_NOTIFY_EACH: generate a local event notification using
- *     the PMIX_PROC_HAS_CONNECTED event each time a process connects
- *
- * (b) PMIX_CONNECT_NOTIFY_REQ: notify each of the indicated procs that
- *     they are requested to connect using the PMIX_CONNECT_REQUESTED event
- *
- * (c) PMIX_CONNECT_OPTIONAL: participation is optional - do not return
- *     error if procs terminate without having connected
- */
+ * Note also that this is a collective operation within the client library, and
+ * thus the client will be blocked until all procs participate. Thus, the info
+ * array can be used to pass user directives, including a timeout.
+ * The directives are optional _unless_ the _mandatory_ flag
+ * has been set - in such cases, the host RM is required to return an error
+ * if the directive cannot be met. */
 typedef pmix_status_t (*pmix_server_connect_fn_t)(const pmix_proc_t procs[], size_t nprocs,
                                                   const pmix_info_t info[], size_t ninfo,
-                                                  pmix_connect_cbfunc_t cbfunc, void *cbdata);
+                                                  pmix_op_cbfunc_t cbfunc, void *cbdata);
 
-/* Disconnect this process from a specified nspace. An error will be returned
- * if the specified nspace is not recognized. The info array is used as above.
- *
- * Processes that terminate while connected to other processes will generate a
- * "termination error" event that will be reported to any process in the connected
- * group that has registered for such events. Calls to "disconnect" that include the
- * PMIX_CONNECT_NOTIFY_EACH info key will cause other processes in the nspace to receive
- * an event notification of the disconnect, if they are registered for such events. */
-typedef pmix_status_t (*pmix_server_disconnect_fn_t)(const char nspace[],
+/* Disconnect a previously connected set of processes. An error should be returned
+ * if the specified set of procs was not previously "connected". As above, a process
+ * may be involved in multiple simultaneous disconnect operations. However, a process
+ * is not allowed to reconnect to a set of ranges that has not fully completed
+ * disconnect - i.e., you have to fully disconnect before you can reconnect to the
+ * same group of processes.
+  *
+ * Note also that this is a collective operation within the client library, and
+ * thus the client will be blocked until all procs participate. Thus, the info
+ * array can be used to pass user directives, including a timeout.
+ * The directives are optional _unless_ the _mandatory_ flag
+ * has been set - in such cases, the host RM is required to return an error
+ * if the directive cannot be met. */
+typedef pmix_status_t (*pmix_server_disconnect_fn_t)(const pmix_proc_t procs[], size_t nprocs,
                                                      const pmix_info_t info[], size_t ninfo,
                                                      pmix_op_cbfunc_t cbfunc, void *cbdata);
 
@@ -326,7 +316,24 @@ typedef void (*pmix_server_tool_connection_fn_t)(pmix_info_t *info, size_t ninfo
                                                  pmix_tool_connection_cbfunc_t cbfunc,
                                                  void *cbdata);
 
-/* Log data on behalf of a client */
+/* Log data on behalf of a client. Calls to the host thru this
+ * function must _NOT_ call the PMIx_Log API as this will
+ * trigger an infinite loop. Instead, the implementation must
+ * perform one of three operations:
+ *
+ * (a) transfer the data+directives to a "gateway" server
+ *     where they can be logged. Gateways are designated
+ *     servers on nodes (typically service nodes) where
+ *     centralized logging is supported. The data+directives
+ *     may be passed to the PMIx_Log API once arriving at
+ *     that destination.
+ *
+ * (b) transfer the data to a logging channel outside of
+ *     PMIx, but directly supported by the host
+ *
+ * (c) return an error to the caller indicating that the
+ *     requested action is not supported
+ */
 typedef void (*pmix_server_log_fn_t)(const pmix_proc_t *client,
                                      const pmix_info_t data[], size_t ndata,
                                      const pmix_info_t directives[], size_t ndirs,
@@ -662,12 +669,13 @@ typedef void (*pmix_setup_application_cbfunc_t)(pmix_status_t status,
                                                 pmix_op_cbfunc_t cbfunc, void *cbdata);
 
 /* Provide a function by which the resource manager can request
- * any application-specific environmental variables prior to
- * launch of an application. For example, network libraries may
- * opt to provide security credentials for the application. This
- * is defined as a non-blocking operation in case network
- * libraries need to perform some action before responding. The
- * returned env will be distributed along with the application */
+ * any application-specific environmental variables, resource
+ * assignments, and/or other data prior to launch of an application.
+ * For example, network libraries may opt to provide security
+ * credentials for the application. This is defined as a non-blocking
+ * operation in case network libraries need to perform some action
+ * before responding. Any returned env will be distributed along
+ * with the application */
 PMIX_EXPORT pmix_status_t PMIx_server_setup_application(const char nspace[],
                                                         pmix_info_t info[], size_t ninfo,
                                                         pmix_setup_application_cbfunc_t cbfunc, void *cbdata);
@@ -676,6 +684,13 @@ PMIX_EXPORT pmix_status_t PMIx_server_setup_application(const char nspace[],
  * any application-specific operations prior to spawning local
  * clients of a given application. For example, a network library
  * might need to setup the local driver for "instant on" addressing.
+ * Data provided in the info array will be stored in the job-info
+ * region for the nspace. Operations included in the info array
+ * will be cached until the server calls PMIx_server_setup_fork,
+ * thereby indicating that local clients of this nspace will exist.
+ * Operations indicated by the provided data will only be executed
+ * for the first local client - i.e., they will only be executed
+ * once for a given nspace
  */
 PMIX_EXPORT pmix_status_t PMIx_server_setup_local_support(const char nspace[],
                                                           pmix_info_t info[], size_t ninfo,
@@ -713,6 +728,26 @@ PMIX_EXPORT pmix_status_t PMIx_server_IOF_deliver(const pmix_proc_t *source,
                                                   const pmix_info_t info[], size_t ninfo,
                                                   pmix_op_cbfunc_t cbfunc, void *cbdata);
 
+/* Collect inventory of local resources. This is a non-blocking
+ * API as it may involve somewhat lengthy operations to obtain
+ * the requested information. Servers designated as "gateways"
+ * and whose plugins support collection of infrastructure info
+ * (e.g., switch and fabric topology, connectivity maps) shall
+ * return that information - plugins on non-gateway servers
+ * shall only return the node-local inventory. */
+PMIX_EXPORT pmix_status_t PMIx_server_collect_inventory(pmix_info_t directives[], size_t ndirs,
+                                                        pmix_info_cbfunc_t cbfunc, void *cbdata);
+
+/* Deliver collected inventory for archiving by the corresponding
+ * plugins. Typically executed on a "gateway" associated with the
+ * system scheduler to enable use of inventory information by the
+ * the scheduling algorithm. May also be used on compute nodes to
+ * store a broader picture of the system for access by applications,
+ * if desired */
+PMIX_EXPORT pmix_status_t PMIx_server_deliver_inventory(pmix_info_t info[], size_t ninfo,
+                                                        pmix_info_t directives[], size_t ndirs,
+                                                        pmix_op_cbfunc_t cbfunc, void *cbdata);
+
 #if defined(c_plusplus) || defined(__cplusplus)
 }
 #endif

opal/mca/pmix/pmix3x/pmix/include/pmix_tool.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013-2017 Intel, Inc. All rights reserved.
+ * Copyright (c) 2013-2018 Intel, Inc.  All rights reserved.
  * Copyright (c) 2015      Artem Y. Polyakov <artpol84@gmail.com>.
  *                         All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
@@ -98,6 +98,36 @@ PMIX_EXPORT pmix_status_t PMIx_tool_init(pmix_proc_t *proc,
  * operation. */
 PMIX_EXPORT pmix_status_t PMIx_tool_finalize(void);
 
+/* Switch server connection. Closes the connection, if existing, to a server
+ * and establishes a connection to the specified server. The target server can
+ * be given as:
+ *
+ * - PMIX_CONNECT_TO_SYSTEM: connect solely to the system server
+ *
+ * - PMIX_CONNECT_SYSTEM_FIRST: a request to use the system server first,
+ *   if existing, and then look for the server specified in a different
+ *   attribute
+ *
+ * - PMIX_SERVER_URI: connect to the server at the given URI
+ *
+ * - PMIX_SERVER_NSPACE: connect to the server of a given nspace
+ *
+ * - PMIX_SERVER_PIDINFO: connect to a server embedded in the process with
+ *   the given pid
+ *
+ * Passing a _NULL_ value for the info array pointer is not allowed and will
+ * result in return of an error.
+ *
+ * NOTE: PMIx does not currently support on-the-fly changes to the tool's
+ * identifier. Thus, the new server must be under the same nspace manager
+ * (e.g., host RM) as the prior server so that the original nspace remains
+ * a unique assignment. The proc parameter is included here for obsolence
+ * protection in case this constraint is someday removed. Meantime, the
+ * proc parameter will be filled with the tool's existing nspace/rank, and
+ * the caller is welcome to pass _NULL_ in that location
+ */
+PMIX_EXPORT pmix_status_t PMIx_tool_connect_to_server(pmix_proc_t *proc,
+                                                      pmix_info_t info[], size_t ninfo);
 
 #if defined(c_plusplus) || defined(__cplusplus)
 }

opal/mca/pmix/pmix3x/pmix/src/Makefile.am

@@ -11,7 +11,7 @@
 #                         All rights reserved.
 # Copyright (c) 2006-2016 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2012-2013 Los Alamos National Security, Inc.  All rights reserved.
-# Copyright (c) 2013-2017 Intel, Inc. All rights reserved.
+# Copyright (c) 2013-2018 Intel, Inc. All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -103,6 +103,7 @@ include runtime/Makefile.include
 include tool/Makefile.include
 include tools/Makefile.include
 include common/Makefile.include
+include hwloc/Makefile.include
 
 MAINTAINERCLEANFILES = Makefile.in config.h config.h.in
 DISTCLEANFILES = Makefile

opal/mca/pmix/pmix3x/pmix/src/class/Makefile.include

@@ -10,7 +10,7 @@
 #                         University of Stuttgart.  All rights reserved.
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
-# Copyright (c) 2013-2016 Intel, Inc. All rights reserved
+# Copyright (c) 2013-2018 Intel, Inc. All rights reserved.
 # Copyright (c) 2016      Cisco Systems, Inc.  All rights reserved.
 # $COPYRIGHT$
 #
@@ -24,6 +24,7 @@
 
 # Source code files
 headers += \
+        class/pmix_bitmap.h \
         class/pmix_object.h \
         class/pmix_list.h \
         class/pmix_pointer_array.h \
@@ -33,6 +34,7 @@ headers += \
         class/pmix_value_array.h
 
 sources += \
+        class/pmix_bitmap.c \
         class/pmix_object.c \
         class/pmix_list.c \
         class/pmix_pointer_array.c \

opal/mca/pmix/pmix3x/pmix/src/class/pmix_bitmap.c

@@ -0,0 +1,412 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2014 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include <src/include/pmix_config.h>
+
+#include <stdio.h>
+#include <limits.h>
+
+#include "pmix_common.h"
+#include "src/class/pmix_bitmap.h"
+
+/* The number of bits in the underlying type of the bitmap field
+ * in the pmix_bitmap_t struct
+ */
+#define SIZE_OF_BASE_TYPE  64
+
+static void pmix_bitmap_construct(pmix_bitmap_t *bm);
+static void pmix_bitmap_destruct(pmix_bitmap_t *bm);
+
+PMIX_CLASS_INSTANCE(pmix_bitmap_t, pmix_object_t,
+                    pmix_bitmap_construct, pmix_bitmap_destruct);
+
+
+static void
+pmix_bitmap_construct(pmix_bitmap_t *bm)
+{
+    bm->bitmap = NULL;
+    bm->array_size = 0;
+    bm->max_size = INT_MAX;
+}
+
+
+static void
+pmix_bitmap_destruct(pmix_bitmap_t *bm)
+{
+    if (NULL != bm->bitmap) {
+        free(bm->bitmap);
+        bm->bitmap = NULL;
+    }
+}
+
+
+int pmix_bitmap_set_max_size (pmix_bitmap_t *bm, int max_size)
+{
+    if (NULL == bm) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    /*
+     * Only if the caller wants to set the maximum size,
+     * we set it (in numbers of bits!), otherwise it is
+     * set to INT_MAX in the constructor.
+     */
+    bm->max_size = (int)(((size_t)max_size + SIZE_OF_BASE_TYPE - 1) / SIZE_OF_BASE_TYPE);
+
+    return PMIX_SUCCESS;
+}
+
+
+int
+pmix_bitmap_init(pmix_bitmap_t *bm, int size)
+{
+    /*
+     * Only if the caller set the maximum size before initializing,
+     * we test here (in numbers of bits!)
+     * By default, the max size is INT_MAX, set in the constructor.
+     */
+    if ((size <= 0) || (NULL == bm) || (size > bm->max_size)) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    bm->array_size = (int)(((size_t)size + SIZE_OF_BASE_TYPE - 1) / SIZE_OF_BASE_TYPE);
+    if( NULL != bm->bitmap ) {
+        free(bm->bitmap);
+        if(bm->max_size < bm->array_size)
+            bm->max_size = bm->array_size;
+    }
+    bm->bitmap = (uint64_t*) malloc(bm->array_size * sizeof(uint64_t));
+    if (NULL == bm->bitmap) {
+        return PMIX_ERR_OUT_OF_RESOURCE;
+    }
+
+    pmix_bitmap_clear_all_bits(bm);
+    return PMIX_SUCCESS;
+}
+
+
+int
+pmix_bitmap_set_bit(pmix_bitmap_t *bm, int bit)
+{
+    int index, offset, new_size;
+
+    if ((bit < 0) || (NULL == bm) || (bit > bm->max_size)) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    index = bit / SIZE_OF_BASE_TYPE;
+    offset = bit % SIZE_OF_BASE_TYPE;
+
+    if (index >= bm->array_size) {
+
+        /* We need to allocate more space for the bitmap, since we are
+         out of range. We don't throw any error here, because this is
+         valid and we simply expand the bitmap */
+
+        new_size = index + 1;
+        if( new_size > bm->max_size )
+            new_size = bm->max_size;
+
+        /* New size is just a multiple of the original size to fit in
+         the index. */
+        bm->bitmap = (uint64_t*)realloc(bm->bitmap, new_size*sizeof(uint64_t));
+        if (NULL == bm->bitmap) {
+            return PMIX_ERR_OUT_OF_RESOURCE;
+        }
+
+        /* zero out the new elements */
+        memset(&bm->bitmap[bm->array_size], 0, (new_size - bm->array_size) * sizeof(uint64_t));
+
+        /* Update the array_size */
+        bm->array_size = new_size;
+    }
+
+    /* Now set the bit */
+    bm->bitmap[index] |= (1UL << offset);
+
+    return PMIX_SUCCESS;
+}
+
+
+int
+pmix_bitmap_clear_bit(pmix_bitmap_t *bm, int bit)
+{
+    int index, offset;
+
+    if ((bit < 0) || NULL == bm || (bit >= (bm->array_size * SIZE_OF_BASE_TYPE))) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    index = bit / SIZE_OF_BASE_TYPE;
+    offset = bit % SIZE_OF_BASE_TYPE;
+
+    bm->bitmap[index] &= ~(1UL << offset);
+    return PMIX_SUCCESS;
+}
+
+
+bool
+pmix_bitmap_is_set_bit(pmix_bitmap_t *bm, int bit)
+{
+    int index, offset;
+
+    if ((bit < 0) || NULL == bm || (bit >= (bm->array_size * SIZE_OF_BASE_TYPE))) {
+        return false;
+    }
+
+    index = bit / SIZE_OF_BASE_TYPE;
+    offset = bit % SIZE_OF_BASE_TYPE;
+
+    if (0 != (bm->bitmap[index] & (1UL << offset))) {
+        return true;
+    }
+
+    return false;
+}
+
+
+int
+pmix_bitmap_clear_all_bits(pmix_bitmap_t *bm)
+{
+    if (NULL == bm) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    memset(bm->bitmap, 0, bm->array_size * sizeof(uint64_t));
+    return PMIX_SUCCESS;
+}
+
+
+int
+pmix_bitmap_set_all_bits(pmix_bitmap_t *bm)
+{
+    if (NULL == bm) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    memset(bm->bitmap, 0xff, bm->array_size * sizeof(uint64_t));
+
+    return PMIX_SUCCESS;
+}
+
+
+int
+pmix_bitmap_find_and_set_first_unset_bit(pmix_bitmap_t *bm, int *position)
+{
+    int i = 0;
+    uint64_t temp, all_ones = 0xffffffffffffffffUL;
+
+    if (NULL == bm) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    /* Neglect all which don't have an unset bit */
+    *position = 0;
+    while((i < bm->array_size) && (bm->bitmap[i] == all_ones)) {
+        ++i;
+    }
+
+    if (i == bm->array_size) {
+        /* increase the bitmap size then */
+        *position = bm->array_size * SIZE_OF_BASE_TYPE;
+        return pmix_bitmap_set_bit(bm, *position);
+    }
+
+    /* This one has an unset bit, find its bit number */
+
+    temp = bm->bitmap[i];
+    bm->bitmap[i] |= (bm->bitmap[i] + 1); /* Set the first zero bit */
+    temp ^= bm->bitmap[i];  /* Compute the change: the first unset bit in the original number */
+    while( !(temp & 0x1) ) {
+        ++(*position);
+        temp >>= 1;
+    }
+
+    (*position) += i * SIZE_OF_BASE_TYPE;
+    return PMIX_SUCCESS;
+}
+
+int pmix_bitmap_bitwise_and_inplace(pmix_bitmap_t *dest, pmix_bitmap_t *right)
+{
+    int i;
+
+    /*
+     * Sanity check
+     */
+    if( NULL == dest || NULL == right ) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+    if( dest->array_size != right->array_size ) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    /*
+     * Bitwise AND
+     */
+    for(i = 0; i < dest->array_size; ++i) {
+        dest->bitmap[i] &= right->bitmap[i];
+    }
+
+    return PMIX_SUCCESS;
+}
+
+int pmix_bitmap_bitwise_or_inplace(pmix_bitmap_t *dest, pmix_bitmap_t *right)
+{
+    int i;
+
+    /*
+     * Sanity check
+     */
+    if( NULL == dest || NULL == right ) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+    if( dest->array_size != right->array_size ) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    /*
+     * Bitwise OR
+     */
+    for(i = 0; i < dest->array_size; ++i) {
+        dest->bitmap[i] |= right->bitmap[i];
+    }
+
+    return PMIX_SUCCESS;
+}
+
+int pmix_bitmap_bitwise_xor_inplace(pmix_bitmap_t *dest, pmix_bitmap_t *right)
+{
+    int i;
+
+    /*
+     * Sanity check
+     */
+    if( NULL == dest || NULL == right ) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+    if( dest->array_size != right->array_size ) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    /*
+     * Bitwise XOR
+     */
+    for(i = 0; i < dest->array_size; ++i) {
+        dest->bitmap[i] ^= right->bitmap[i];
+    }
+
+    return PMIX_SUCCESS;
+}
+
+bool pmix_bitmap_are_different(pmix_bitmap_t *left, pmix_bitmap_t *right)
+{
+    int i;
+
+    /*
+     * Sanity check
+     */
+    if( NULL == left || NULL == right ) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    if( pmix_bitmap_size(left) != pmix_bitmap_size(right) ) {
+        return true;
+    }
+
+    /*
+     * Direct comparison
+     */
+    for(i = 0; i < left->array_size; ++i) {
+        if( left->bitmap[i] != right->bitmap[i] ) {
+            return true;
+        }
+    }
+
+    return false;
+}
+
+char * pmix_bitmap_get_string(pmix_bitmap_t *bitmap)
+{
+    int i;
+    char *bitmap_str = NULL;
+
+    if( NULL == bitmap) {
+        return NULL;
+    }
+
+    bitmap_str = malloc(bitmap->array_size * SIZE_OF_BASE_TYPE + 1);
+    if (NULL == bitmap_str) {
+        return NULL;
+    }
+    bitmap_str[bitmap->array_size * SIZE_OF_BASE_TYPE] = '\0';
+
+    for( i = 0; i < (bitmap->array_size * SIZE_OF_BASE_TYPE); ++i) {
+        if( pmix_bitmap_is_set_bit(bitmap, i) ) {
+            bitmap_str[i] = 'X';
+        } else {
+            bitmap_str[i] = '_';
+        }
+    }
+
+    return bitmap_str;
+}
+
+int pmix_bitmap_num_unset_bits(pmix_bitmap_t *bm, int len)
+{
+    return (len - pmix_bitmap_num_set_bits(bm, len));
+}
+
+int pmix_bitmap_num_set_bits(pmix_bitmap_t *bm, int len)
+{
+    int i, cnt = 0;
+    uint64_t val;
+
+#if PMIX_ENABLE_DEBUG
+    if ((len < 0) || NULL == bm || (len >= (bm->array_size * SIZE_OF_BASE_TYPE))) {
+        return 0;
+    }
+#endif
+
+    for(i = 0; i < len; ++i) {
+        if( 0 == (val = bm->bitmap[i]) ) continue;
+        /*  Peter Wegner in CACM 3 (1960), 322. This method goes through as many
+         *  iterations as there are set bits. */
+        for( ; val; cnt++ ) {
+            val &= val - 1;  /* clear the least significant bit set */
+        }
+    }
+
+    return cnt;
+}
+
+bool pmix_bitmap_is_clear(pmix_bitmap_t *bm)
+{
+    int i;
+
+    for (i = 0; i < bm->array_size; ++i) {
+        if (0 != bm->bitmap[i]) {
+            return false;
+        }
+    }
+    return true;
+}

opal/mca/pmix/pmix3x/pmix/src/class/pmix_bitmap.h

@@ -0,0 +1,259 @@
+/* -*- Mode: C; c-basic-offset:4 ; -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2014 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2018      Intel, Inc. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+
+/** @file
+ *
+ *  A bitmap implementation. The bits start off with 0, so this bitmap
+ *  has bits numbered as bit 0, bit 1, bit 2 and so on. This bitmap
+ *  has auto-expansion capabilities, that is once the size is set
+ *  during init, it can be automatically expanded by setting the bit
+ *  beyond the current size. But note, this is allowed just when the
+ *  bit is set -- so the valid functions are set_bit and
+ *  find_and_set_bit. Other functions like clear, if passed a bit
+ *  outside the initialized range will result in an error.
+ *
+ *  To allow these bitmaps to track fortran handles (which MPI defines
+ *  to be Fortran INTEGER), we offer a pmix_bitmap_set_max_size, so that
+ *  the upper layer can ask to never have more than
+ *  OMPI_FORTRAN_HANDLE_MAX, which is min(INT_MAX, fortran INTEGER max).
+ */
+
+#ifndef PMIX_BITMAP_H
+#define PMIX_BITMAP_H
+
+#include <src/include/pmix_config.h>
+
+#include <string.h>
+
+#include "src/class/pmix_object.h"
+
+BEGIN_C_DECLS
+
+struct pmix_bitmap_t {
+    pmix_object_t  super;       /**< Subclass of pmix_object_t */
+    uint64_t      *bitmap;      /**< The actual bitmap array of characters */
+    int            array_size;  /**< The actual array size that maintains the bitmap */
+    int            max_size;    /**< The maximum size that this bitmap may grow (optional) */
+};
+
+typedef struct pmix_bitmap_t pmix_bitmap_t;
+
+PMIX_EXPORT PMIX_CLASS_DECLARATION(pmix_bitmap_t);
+
+/**
+ * Set the maximum size of the bitmap.
+ * May be reset any time, but HAS TO BE SET BEFORE pmix_bitmap_init!
+ *
+ * @param  bitmap     The input bitmap (IN)
+ * @param  max_size   The maximum size of the bitmap in terms of bits (IN)
+ * @return PMIX error code or success
+ *
+ */
+PMIX_EXPORT int pmix_bitmap_set_max_size (pmix_bitmap_t *bm, int max_size);
+
+
+/**
+ * Initializes the bitmap and sets its size. This must be called
+ * before the bitmap can be actually used
+ *
+ * @param  bitmap The input bitmap (IN)
+ * @param  size   The initial size of the bitmap in terms of bits (IN)
+ * @return PMIX error code or success
+ *
+ */
+PMIX_EXPORT int pmix_bitmap_init (pmix_bitmap_t *bm, int size);
+
+
+/**
+ * Set a bit of the bitmap. If the bit asked for is beyond the current
+ * size of the bitmap, then the bitmap is extended to accomodate the
+ * bit
+ *
+ * @param  bitmap The input bitmap (IN)
+ * @param  bit    The bit which is to be set (IN)
+ * @return PMIX error code or success
+ *
+ */
+PMIX_EXPORT int pmix_bitmap_set_bit(pmix_bitmap_t *bm, int bit);
+
+
+/**
+ * Clear/unset a bit of the bitmap. If the bit is beyond the current
+ * size of the bitmap, an error is returned
+ *
+ * @param  bitmap The input bitmap (IN)
+ * @param  bit    The bit which is to be cleared (IN)
+ * @return PMIX error code if the bit is out of range, else success
+ *
+ */
+PMIX_EXPORT int pmix_bitmap_clear_bit(pmix_bitmap_t *bm, int bit);
+
+
+/**
+  * Find out if a bit is set in the bitmap
+  *
+  * @param  bitmap  The input bitmap (IN)
+  * @param  bit     The bit which is to be checked (IN)
+  * @return true    if the bit is set
+  *         false   if the bit is not set OR the index
+  *                 is outside the bounds of the provided
+  *                 bitmap
+  *
+  */
+PMIX_EXPORT bool pmix_bitmap_is_set_bit(pmix_bitmap_t *bm, int bit);
+
+
+/**
+ * Find the first clear bit in the bitmap and set it
+ *
+ * @param  bitmap     The input bitmap (IN)
+ * @param  position   Position of the first clear bit (OUT)
+
+ * @return err        PMIX_SUCCESS on success
+ */
+PMIX_EXPORT int pmix_bitmap_find_and_set_first_unset_bit(pmix_bitmap_t *bm,
+                                                           int *position);
+
+
+/**
+ * Clear all bits in the bitmap
+ *
+ * @param bitmap The input bitmap (IN)
+ * @return PMIX error code if bm is NULL
+ *
+ */
+PMIX_EXPORT int pmix_bitmap_clear_all_bits(pmix_bitmap_t *bm);
+
+
+/**
+ * Set all bits in the bitmap
+ * @param bitmap The input bitmap (IN)
+ * @return PMIX error code if bm is NULL
+ *
+ */
+PMIX_EXPORT int pmix_bitmap_set_all_bits(pmix_bitmap_t *bm);
+
+
+/**
+ * Gives the current size (number of bits) in the bitmap. This is the
+ * legal (accessible) number of bits
+ *
+ * @param bitmap The input bitmap (IN)
+ * @return PMIX error code if bm is NULL
+ *
+ */
+static inline int pmix_bitmap_size(pmix_bitmap_t *bm)
+{
+    return (NULL == bm) ? 0 : (bm->array_size * ((int) (sizeof(*bm->bitmap) * 8)));
+}
+
+
+/**
+ * Copy a bitmap
+ *
+ * @param dest Pointer to the destination bitmap
+ * @param src Pointer to the source bitmap
+ * @ return PMIX error code if something goes wrong
+ */
+static inline void pmix_bitmap_copy(pmix_bitmap_t *dest, pmix_bitmap_t *src)
+{
+    if( dest->array_size < src->array_size ) {
+        if( NULL != dest->bitmap) free(dest->bitmap);
+        dest->max_size = src->max_size;
+        dest->bitmap = (uint64_t*)malloc(src->array_size*sizeof(uint64_t));
+    }
+    memcpy(dest->bitmap, src->bitmap, src->array_size * sizeof(uint64_t));
+    dest->array_size = src->array_size;
+}
+
+/**
+ * Bitwise AND operator (inplace)
+ *
+ * @param dest Pointer to the bitmap that should be modified
+ * @param right Point to the other bitmap in the operation
+ * @return PMIX error code if the length of the two bitmaps is not equal or one is NULL.
+ */
+PMIX_EXPORT int pmix_bitmap_bitwise_and_inplace(pmix_bitmap_t *dest, pmix_bitmap_t *right);
+
+/**
+ * Bitwise OR operator (inplace)
+ *
+ * @param dest Pointer to the bitmap that should be modified
+ * @param right Point to the other bitmap in the operation
+ * @return PMIX error code if the length of the two bitmaps is not equal or one is NULL.
+ */
+PMIX_EXPORT int pmix_bitmap_bitwise_or_inplace(pmix_bitmap_t *dest, pmix_bitmap_t *right);
+
+/**
+ * Bitwise XOR operator (inplace)
+ *
+ * @param dest Pointer to the bitmap that should be modified
+ * @param right Point to the other bitmap in the operation
+ * @return PMIX error code if the length of the two bitmaps is not equal or one is NULL.
+ */
+PMIX_EXPORT int pmix_bitmap_bitwise_xor_inplace(pmix_bitmap_t *dest, pmix_bitmap_t *right);
+
+/**
+ * If the bitmaps are different
+ *
+ * @param left Pointer to a bitmap
+ * @param right Pointer to another bitmap
+ * @return true if different, false if the same
+ */
+PMIX_EXPORT bool pmix_bitmap_are_different(pmix_bitmap_t *left, pmix_bitmap_t *right);
+
+/**
+ * Get a string representation of the bitmap.
+ * Useful for debugging.
+ *
+ * @param bitmap Point to the bitmap to represent
+ * @return Pointer to the string (caller must free if not NULL)
+ */
+PMIX_EXPORT char * pmix_bitmap_get_string(pmix_bitmap_t *bitmap);
+
+/**
+ * Return the number of 'unset' bits, upto the specified length
+ *
+ * @param bitmap Pointer to the bitmap
+ * @param len Number of bits to check
+ * @return Integer
+ */
+PMIX_EXPORT int pmix_bitmap_num_unset_bits(pmix_bitmap_t *bm, int len);
+
+/**
+ * Return the number of 'set' bits, upto the specified length
+ *
+ * @param bitmap Pointer to the bitmap
+ * @param len Number of bits to check
+ * @return Integer
+ */
+PMIX_EXPORT int pmix_bitmap_num_set_bits(pmix_bitmap_t *bm, int len);
+
+/**
+ * Check a bitmap to see if any bit is set
+ */
+PMIX_EXPORT bool pmix_bitmap_is_clear(pmix_bitmap_t *bm);
+
+END_C_DECLS
+
+#endif

opal/mca/pmix/pmix3x/pmix/src/class/pmix_hotel.c

@@ -2,7 +2,7 @@
 /*
  * Copyright (c) 2012-2016 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2012      Los Alamos National Security, LLC. All rights reserved
- * Copyright (c) 2015-2018 Intel, Inc. All rights reserved.
+ * Copyright (c) 2015-2018 Intel, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow

opal/mca/pmix/pmix3x/pmix/src/class/pmix_hotel.h

@@ -2,7 +2,7 @@
 /*
  * Copyright (c) 2012-2016 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2012      Los Alamos National Security, LLC. All rights reserved
- * Copyright (c) 2015-2018 Intel, Inc. All rights reserved.
+ * Copyright (c) 2015-2018 Intel, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow

opal/mca/pmix/pmix3x/pmix/src/client/pmi2.c

@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2014-2017 Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2016      Mellanox Technologies, Inc.
@@ -327,8 +327,6 @@ PMIX_EXPORT int PMI2_Job_Connect(const char jobid[], PMI2_Connect_comm_t *conn)
 {
     pmix_status_t rc = PMIX_SUCCESS;
     pmix_proc_t proc;
-    char nspace[PMIX_MAX_NSLEN+1];
-    pmix_rank_t rank;
 
     PMI2_CHECK();
 
@@ -343,14 +341,14 @@ PMIX_EXPORT int PMI2_Job_Connect(const char jobid[], PMI2_Connect_comm_t *conn)
     memset(proc.nspace, 0, sizeof(proc.nspace));
     (void)strncpy(proc.nspace, (jobid ? jobid : proc.nspace), sizeof(proc.nspace)-1);
     proc.rank = PMIX_RANK_WILDCARD;
-    rc = PMIx_Connect(&proc, 1, NULL, 0, nspace, &rank);
+    rc = PMIx_Connect(&proc, 1, NULL, 0);
     return convert_err(rc);
 }
 
 PMIX_EXPORT int PMI2_Job_Disconnect(const char jobid[])
 {
     pmix_status_t rc = PMIX_SUCCESS;
-    char nspace[PMIX_MAX_NSLEN+1];
+    pmix_proc_t proc;
 
     PMI2_CHECK();
 
@@ -358,9 +356,10 @@ PMIX_EXPORT int PMI2_Job_Disconnect(const char jobid[])
         return PMI2_SUCCESS;
     }
 
-    memset(nspace, 0, sizeof(nspace));
-    (void)strncpy(nspace, (jobid ? jobid : nspace), sizeof(nspace)-1);
-    rc = PMIx_Disconnect(nspace, NULL, 0);
+    memset(proc.nspace, 0, sizeof(proc.nspace));
+    (void)strncpy(proc.nspace, (jobid ? jobid : proc.nspace), sizeof(proc.nspace)-1);
+    proc.rank = PMIX_RANK_WILDCARD;
+    rc = PMIx_Disconnect(&proc, 1, NULL, 0);
     return convert_err(rc);
 }
 

opal/mca/pmix/pmix3x/pmix/src/client/pmix_client.c

@@ -1,7 +1,7 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2014-2018 Intel, Inc.  All rights reserved.
- * Copyright (c) 2014-2017 Research Organization for Information Science
+ * Copyright (c) 2014-2018 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2014      Artem Y. Polyakov <artpol84@gmail.com>.
  *                         All rights reserved.
@@ -66,6 +66,7 @@ static const char pmix_version_string[] = PMIX_VERSION;
 #include "src/util/compress.h"
 #include "src/util/error.h"
 #include "src/util/hash.h"
+#include "src/util/name_fns.h"
 #include "src/util/output.h"
 #include "src/runtime/pmix_progress_threads.h"
 #include "src/runtime/pmix_rte.h"
@@ -154,9 +155,9 @@ static void pmix_client_notify_recv(struct pmix_peer_t *peer,
         goto error;
     }
 
-    /* we always leave space for a callback object */
-    chain->ninfo = ninfo + 1;
-    PMIX_INFO_CREATE(chain->info, chain->ninfo);
+    /* we always leave space for event hdlr name and a callback object */
+    chain->nallocated = ninfo + 2;
+    PMIX_INFO_CREATE(chain->info, chain->nallocated);
     if (NULL == chain->info) {
         PMIX_ERROR_LOG(PMIX_ERR_NOMEM);
         PMIX_RELEASE(chain);
@@ -164,6 +165,7 @@ static void pmix_client_notify_recv(struct pmix_peer_t *peer,
     }
 
     if (0 < ninfo) {
+        chain->ninfo = ninfo;
         cnt = ninfo;
         PMIX_BFROPS_UNPACK(rc, pmix_client_globals.myserver,
                            buf, chain->info, &cnt, PMIX_INFO);
@@ -180,12 +182,10 @@ static void pmix_client_notify_recv(struct pmix_peer_t *peer,
             }
         }
     }
-    /* now put the callback object tag in the last element */
-    PMIX_INFO_LOAD(&chain->info[ninfo], PMIX_EVENT_RETURN_OBJECT, NULL, PMIX_POINTER);
 
     pmix_output_verbose(2, pmix_client_globals.base_output,
-                        "[%s:%d] pmix:client_notify_recv - processing event %d, calling errhandler",
-                        pmix_globals.myid.nspace, pmix_globals.myid.rank, chain->status);
+                        "[%s:%d] pmix:client_notify_recv - processing event %s, calling errhandler",
+                        pmix_globals.myid.nspace, pmix_globals.myid.rank, PMIx_Error_string(chain->status));
 
     pmix_invoke_local_event_hdlr(chain);
     return;
@@ -243,6 +243,7 @@ static void job_data(struct pmix_peer_t *pr,
     PMIX_GDS_STORE_JOB_INFO(cb->status,
                             pmix_client_globals.myserver,
                             nspace, buf);
+    free(nspace);
     cb->status = PMIX_SUCCESS;
     PMIX_POST_OBJECT(cb);
     PMIX_WAKEUP_THREAD(&cb->lock);
@@ -253,6 +254,18 @@ PMIX_EXPORT const char* PMIx_Get_version(void)
     return pmix_version_string;
 }
 
+/* event handler registration callback */
+static void evhandler_reg_callbk(pmix_status_t status,
+                                 size_t evhandler_ref,
+                                 void *cbdata)
+{
+    pmix_lock_t *lock = (pmix_lock_t*)cbdata;
+
+    lock->status = status;
+    PMIX_WAKEUP_THREAD(lock);
+}
+
+
 static void notification_fn(size_t evhdlr_registration_id,
                             pmix_status_t status,
                             const pmix_proc_t *source,
@@ -261,13 +274,38 @@ static void notification_fn(size_t evhdlr_registration_id,
                             pmix_event_notification_cbfunc_fn_t cbfunc,
                             void *cbdata)
 {
-    pmix_lock_t *reglock = (pmix_lock_t*)cbdata;
+    pmix_lock_t *lock=NULL;
+    char *name = NULL;
+    size_t n;
+
+    if (NULL != info) {
+        lock = NULL;
+        for (n=0; n < ninfo; n++) {
+            if (0 == strncmp(info[n].key, PMIX_EVENT_RETURN_OBJECT, PMIX_MAX_KEYLEN)) {
+                lock = (pmix_lock_t*)info[n].value.data.ptr;
+            } else if (0 == strncmp(info[n].key, PMIX_EVENT_HDLR_NAME, PMIX_MAX_KEYLEN)) {
+                name = info[n].value.data.string;
+            }
+        }
+        /* if the object wasn't returned, then that is an error */
+        if (NULL == lock) {
+            pmix_output_verbose(2, pmix_client_globals.base_output,
+                                "event handler %s failed to return object",
+                                (NULL == name) ? "NULL" : name);
+            /* let the event handler progress */
+            if (NULL != cbfunc) {
+                cbfunc(PMIX_SUCCESS, NULL, 0, NULL, NULL, cbdata);
+            }
+            return;
+        }
+    }
+    if (NULL != lock) {
+        PMIX_WAKEUP_THREAD(lock);
+    }
 
     if (NULL != cbfunc) {
         cbfunc(PMIX_EVENT_ACTION_COMPLETE, NULL, 0, NULL, NULL, cbdata);
     }
-    PMIX_WAKEUP_THREAD(reglock);
-
 }
 
 typedef struct {
@@ -381,11 +419,7 @@ static void client_iof_handler(struct pmix_peer_t *pr,
         return;
     }
     if (NULL != bo.bytes && 0 < bo.size) {
-        if (channel & PMIX_FWD_STDOUT_CHANNEL) {
-            pmix_iof_write_output(&source, channel, &bo, &pmix_client_globals.iof_stdout.wev);
-        } else {
-            pmix_iof_write_output(&source, channel, &bo, &pmix_client_globals.iof_stderr.wev);
-        }
+        pmix_iof_write_output(&source, channel, &bo, NULL);
     }
     PMIX_BYTE_OBJECT_DESTRUCT(&bo);
 }
@@ -395,15 +429,14 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc,
 {
     char *evar;
     pmix_status_t rc;
-    pmix_nspace_t *nsptr;
     pmix_cb_t cb;
     pmix_buffer_t *req;
     pmix_cmd_t cmd = PMIX_REQ_CMD;
     pmix_status_t code = PMIX_ERR_DEBUGGER_RELEASE;
     pmix_proc_t wildcard;
-    pmix_info_t ginfo;
+    pmix_info_t ginfo, evinfo[2];
     pmix_value_t *val = NULL;
-    pmix_lock_t reglock;
+    pmix_lock_t reglock, releaselock;
     size_t n;
     bool found;
     pmix_ptl_posted_recv_t *rcv;
@@ -430,7 +463,7 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc,
         return PMIX_SUCCESS;
     }
     /* if we don't see the required info, then we cannot init */
-    if (NULL == getenv("PMIX_NAMESPACE")) {
+    if (NULL == (evar = getenv("PMIX_NAMESPACE"))) {
         PMIX_RELEASE_THREAD(&pmix_global_lock);
         return PMIX_ERR_INVALID_NAMESPACE;
     }
@@ -472,9 +505,6 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc,
         PMIX_RELEASE_THREAD(&pmix_global_lock);
         return PMIX_ERR_NOMEM;
     }
-    /* construct the global notification ring buffer */
-    PMIX_CONSTRUCT(&pmix_globals.notifications, pmix_ring_buffer_t);
-    pmix_ring_buffer_init(&pmix_globals.notifications, 256);
 
     pmix_output_verbose(2, pmix_client_globals.base_output,
                         "pmix: init called");
@@ -487,23 +517,12 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc,
     }
 
     /* we require our nspace */
-    if (NULL == (evar = getenv("PMIX_NAMESPACE"))) {
-        /* let the caller know that the server isn't available yet */
-        PMIX_RELEASE_THREAD(&pmix_global_lock);
-        return PMIX_ERR_INVALID_NAMESPACE;
-    }
     if (NULL != proc) {
         (void)strncpy(proc->nspace, evar, PMIX_MAX_NSLEN);
     }
     (void)strncpy(pmix_globals.myid.nspace, evar, PMIX_MAX_NSLEN);
-    /* create a pmix_nspace_t object for our peer */
-    nsptr = PMIX_NEW(pmix_nspace_t);
-    if (NULL == nsptr){
-        PMIX_RELEASE_THREAD(&pmix_global_lock);
-        return PMIX_ERR_NOMEM;
-    }
-    nsptr->nspace = strdup(evar);
-    pmix_globals.mypeer->nptr = nsptr;
+    /* set the global pmix_nspace_t object for our peer */
+    pmix_globals.mypeer->nptr->nspace = strdup(evar);
 
     /* we also require our rank */
     if (NULL == (evar = getenv("PMIX_RANK"))) {
@@ -643,12 +662,20 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc,
         /* if the value was found, then we need to wait for debugger attach here */
         /* register for the debugger release notification */
         PMIX_CONSTRUCT_LOCK(&reglock);
-        PMIX_POST_OBJECT(&reglock);
-        PMIx_Register_event_handler(&code, 1, NULL, 0,
-                                    notification_fn, NULL, (void*)&reglock);
-        /* wait for it to arrive */
+        PMIX_CONSTRUCT_LOCK(&releaselock);
+        PMIX_INFO_LOAD(&evinfo[0], PMIX_EVENT_RETURN_OBJECT, &releaselock, PMIX_POINTER);
+        PMIX_INFO_LOAD(&evinfo[1], PMIX_EVENT_HDLR_NAME, "WAIT-FOR-DEBUGGER", PMIX_STRING);
+
+        PMIx_Register_event_handler(&code, 1, evinfo, 2,
+                                    notification_fn, evhandler_reg_callbk, (void*)&reglock);
+        /* wait for registration to complete */
         PMIX_WAIT_THREAD(&reglock);
         PMIX_DESTRUCT_LOCK(&reglock);
+        PMIX_INFO_DESTRUCT(&evinfo[0]);
+        PMIX_INFO_DESTRUCT(&evinfo[1]);
+        /* wait for release to arrive */
+        PMIX_WAIT_THREAD(&releaselock);
+        PMIX_DESTRUCT_LOCK(&releaselock);
     }
     PMIX_INFO_DESTRUCT(&ginfo);
 
@@ -656,7 +683,6 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc,
     if (NULL != info) {
         _check_for_notify(info, ninfo);
     }
-
     return PMIX_SUCCESS;
 }
 
@@ -810,6 +836,7 @@ PMIX_EXPORT pmix_status_t PMIx_Finalize(const pmix_info_t info[], size_t ninfo)
             PMIX_RELEASE(peer);
         }
     }
+    PMIX_DESTRUCT(&pmix_client_globals.peers);
 
     if (0 <= pmix_client_globals.myserver->sd) {
         CLOSE_THE_SOCKET(pmix_client_globals.myserver->sd);
@@ -820,8 +847,15 @@ PMIX_EXPORT pmix_status_t PMIx_Finalize(const pmix_info_t info[], size_t ninfo)
 
 
     pmix_rte_finalize();
+    if (NULL != pmix_globals.mypeer) {
+        PMIX_RELEASE(pmix_globals.mypeer);
+    }
+
     PMIX_RELEASE_THREAD(&pmix_global_lock);
 
+    /* finalize the class/object system */
+    pmix_class_finalize();
+
     return PMIX_SUCCESS;
 }
 
@@ -1175,11 +1209,26 @@ static void _commitfn(int sd, short args, void *cbdata)
     return rc;
 }
 
+static void _resolve_peers(int sd, short args, void *cbdata)
+{
+    pmix_cb_t *cb = (pmix_cb_t*)cbdata;
+
+    cb->status = pmix_preg.resolve_peers(cb->key, cb->pname.nspace,
+                                         &cb->procs, &cb->nprocs);
+    /* post the data so the receiving thread can acquire it */
+    PMIX_POST_OBJECT(cb);
+    PMIX_WAKEUP_THREAD(&cb->lock);
+}
+
 /* need to thread-shift this request */
 PMIX_EXPORT pmix_status_t PMIx_Resolve_peers(const char *nodename,
                                              const char *nspace,
                                              pmix_proc_t **procs, size_t *nprocs)
 {
+    pmix_cb_t *cb;
+    pmix_status_t rc;
+    pmix_proc_t proc;
+
     PMIX_ACQUIRE_THREAD(&pmix_global_lock);
     if (pmix_globals.init_cntr <= 0) {
         PMIX_RELEASE_THREAD(&pmix_global_lock);
@@ -1187,16 +1236,71 @@ PMIX_EXPORT pmix_status_t PMIx_Resolve_peers(const char *nodename,
     }
     PMIX_RELEASE_THREAD(&pmix_global_lock);
 
-    /* set default */
-    *procs = NULL;
-    *nprocs = 0;
 
-    return pmix_preg.resolve_peers(nodename, nspace, procs, nprocs);
+    cb = PMIX_NEW(pmix_cb_t);
+    cb->key = (char*)nodename;
+    cb->pname.nspace = strdup(nspace);
+
+    PMIX_THREADSHIFT(cb, _resolve_peers);
+
+    /* wait for the result */
+    PMIX_WAIT_THREAD(&cb->lock);
+
+    /* if the nspace wasn't found, then we need to
+     * ask the server for that info */
+    if (PMIX_ERR_INVALID_NAMESPACE == cb->status) {
+        (void)strncpy(proc.nspace, nspace, PMIX_MAX_NSLEN);
+        proc.rank = PMIX_RANK_WILDCARD;
+        /* any key will suffice as it will bring down
+         * the entire data blob */
+        rc = PMIx_Get(&proc, PMIX_UNIV_SIZE, NULL, 0, NULL);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_RELEASE(cb);
+            return rc;
+        }
+        /* retry the fetch */
+        cb->lock.active = true;
+        PMIX_THREADSHIFT(cb, _resolve_peers);
+        PMIX_WAIT_THREAD(&cb->lock);
+    }
+    *procs = cb->procs;
+    *nprocs = cb->nprocs;
+
+    rc = cb->status;
+    PMIX_RELEASE(cb);
+    return rc;
+}
+
+static void _resolve_nodes(int fd, short args, void *cbdata)
+{
+    pmix_cb_t *cb = (pmix_cb_t*)cbdata;
+    char *regex, **names;
+
+    /* get a regular expression describing the PMIX_NODE_MAP */
+    cb->status = pmix_preg.resolve_nodes(cb->pname.nspace, &regex);
+    if (PMIX_SUCCESS == cb->status) {
+        /* parse it into an argv array of names */
+        cb->status = pmix_preg.parse_nodes(regex, &names);
+        if (PMIX_SUCCESS == cb->status) {
+            /* assemble it into a comma-delimited list */
+            cb->key = pmix_argv_join(names, ',');
+            pmix_argv_free(names);
+        } else {
+            free(regex);
+        }
+    }
+    /* post the data so the receiving thread can acquire it */
+    PMIX_POST_OBJECT(cb);
+    PMIX_WAKEUP_THREAD(&cb->lock);
 }
 
 /* need to thread-shift this request */
 PMIX_EXPORT pmix_status_t PMIx_Resolve_nodes(const char *nspace, char **nodelist)
 {
+    pmix_cb_t *cb;
+    pmix_status_t rc;
+    pmix_proc_t proc;
+
     PMIX_ACQUIRE_THREAD(&pmix_global_lock);
     if (pmix_globals.init_cntr <= 0) {
         PMIX_RELEASE_THREAD(&pmix_global_lock);
@@ -1204,8 +1308,35 @@ PMIX_EXPORT pmix_status_t PMIx_Resolve_nodes(const char *nspace, char **nodelist
     }
     PMIX_RELEASE_THREAD(&pmix_global_lock);
 
-    /* set default */
-    *nodelist = NULL;
+    cb = PMIX_NEW(pmix_cb_t);
+    cb->pname.nspace = strdup(nspace);
 
-    return pmix_preg.resolve_nodes(nspace, nodelist);
+    PMIX_THREADSHIFT(cb, _resolve_nodes);
+
+    /* wait for the result */
+    PMIX_WAIT_THREAD(&cb->lock);
+
+    /* if the nspace wasn't found, then we need to
+     * ask the server for that info */
+    if (PMIX_ERR_INVALID_NAMESPACE == cb->status) {
+        (void)strncpy(proc.nspace, nspace, PMIX_MAX_NSLEN);
+        proc.rank = PMIX_RANK_WILDCARD;
+        /* any key will suffice as it will bring down
+         * the entire data blob */
+        rc = PMIx_Get(&proc, PMIX_UNIV_SIZE, NULL, 0, NULL);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_RELEASE(cb);
+            return rc;
+        }
+        /* retry the fetch */
+        cb->lock.active = true;
+        PMIX_THREADSHIFT(cb, _resolve_nodes);
+        PMIX_WAIT_THREAD(&cb->lock);
+    }
+    /* the string we want is in the key field */
+    *nodelist = cb->key;
+
+    rc = cb->status;
+    PMIX_RELEASE(cb);
+    return rc;
 }

opal/mca/pmix/pmix3x/pmix/src/client/pmix_client_connect.c

@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2014-2017 Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
  * Copyright (c) 2014-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2014      Artem Y. Polyakov <artpol84@gmail.com>.
@@ -62,26 +62,17 @@
 static void wait_cbfunc(struct pmix_peer_t *pr,
                         pmix_ptl_hdr_t *hdr,
                         pmix_buffer_t *buf, void *cbdata);
-static void discbfunc(struct pmix_peer_t *pr,
-                      pmix_ptl_hdr_t *hdr,
-                      pmix_buffer_t *buf, void *cbdata);
-static void cnct_cbfunc(pmix_status_t status,
-                        char nspace[], int rank,
-                        void *cbdata);
-
 static void op_cbfunc(pmix_status_t status, void *cbdata);
 
 PMIX_EXPORT pmix_status_t PMIx_Connect(const pmix_proc_t procs[], size_t nprocs,
-                                       const pmix_info_t info[], size_t ninfo,
-                                       char nspace[], pmix_rank_t *newrank)
+                                       const pmix_info_t info[], size_t ninfo)
 {
     pmix_status_t rc;
     pmix_cb_t *cb;
-    size_t n;
 
     PMIX_ACQUIRE_THREAD(&pmix_global_lock);
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_client_globals.connect_output,
                         "pmix: connect called");
 
     if (pmix_globals.init_cntr <= 0) {
@@ -100,18 +91,9 @@ PMIX_EXPORT pmix_status_t PMIx_Connect(const pmix_proc_t procs[], size_t nprocs,
      * recv routine so we know which callback to use when
      * the return message is recvd */
     cb = PMIX_NEW(pmix_cb_t);
-    /* see if this connect request was to return a new nspace/rank, or
-     * was just an exchange of info */
-    cb->checked = true;
-    for (n=0; n < ninfo; n++) {
-        if (0 == strncmp(info[n].key, PMIX_CONNECT_XCHG_ONLY, PMIX_MAX_KEYLEN)) {
-            cb->checked = false;
-            break;
-        }
-    }
 
     /* push the message into our event base to send to the server */
-    if (PMIX_SUCCESS != (rc = PMIx_Connect_nb(procs, nprocs, info, ninfo, cnct_cbfunc, cb))) {
+    if (PMIX_SUCCESS != (rc = PMIx_Connect_nb(procs, nprocs, info, ninfo, op_cbfunc, cb))) {
         PMIX_RELEASE(cb);
         return rc;
     }
@@ -119,15 +101,6 @@ PMIX_EXPORT pmix_status_t PMIx_Connect(const pmix_proc_t procs[], size_t nprocs,
     /* wait for the connect to complete */
     PMIX_WAIT_THREAD(&cb->lock);
     rc = cb->status;
-
-    if (cb->checked && PMIX_SUCCESS == rc) {
-        if (NULL != nspace) {
-            (void)strncpy(nspace, cb->pname.nspace, PMIX_MAX_NSLEN);
-        }
-        if (NULL != newrank) {
-            *newrank = cb->pname.rank;
-        }
-    }
     PMIX_RELEASE(cb);
 
     pmix_output_verbose(2, pmix_globals.debug_output,
@@ -138,18 +111,17 @@ PMIX_EXPORT pmix_status_t PMIx_Connect(const pmix_proc_t procs[], size_t nprocs,
 
 PMIX_EXPORT pmix_status_t PMIx_Connect_nb(const pmix_proc_t procs[], size_t nprocs,
                                           const pmix_info_t info[], size_t ninfo,
-                                          pmix_connect_cbfunc_t cbfunc, void *cbdata)
+                                          pmix_op_cbfunc_t cbfunc, void *cbdata)
 {
     pmix_buffer_t *msg;
     pmix_cmd_t cmd = PMIX_CONNECTNB_CMD;
     pmix_status_t rc;
     pmix_cb_t *cb;
-    size_t n;
 
     PMIX_ACQUIRE_THREAD(&pmix_global_lock);
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
-                        "pmix: connect called");
+    pmix_output_verbose(2, pmix_client_globals.connect_output,
+                        "pmix:connect_nb called");
 
     if (pmix_globals.init_cntr <= 0) {
         PMIX_RELEASE_THREAD(&pmix_global_lock);
@@ -213,19 +185,9 @@ PMIX_EXPORT pmix_status_t PMIx_Connect_nb(const pmix_proc_t procs[], size_t npro
      * recv routine so we know which callback to use when
      * the return message is recvd */
     cb = PMIX_NEW(pmix_cb_t);
-    cb->cbfunc.cnctfn = cbfunc;
+    cb->cbfunc.opfn = cbfunc;
     cb->cbdata = cbdata;
 
-    /* see if this connect request was to return a new nspace/rank, or
-     * was just an exchange of info */
-    cb->checked = true;
-    for (n=0; n < ninfo; n++) {
-        if (0 == strncmp(info[n].key, PMIX_CONNECT_XCHG_ONLY, PMIX_MAX_KEYLEN)) {
-            cb->checked = false;
-            break;
-        }
-    }
-
     /* push the message into our event base to send to the server */
     PMIX_PTL_SEND_RECV(rc, pmix_client_globals.myserver,
                        msg, wait_cbfunc, (void*)cb);
@@ -237,7 +199,7 @@ PMIX_EXPORT pmix_status_t PMIx_Connect_nb(const pmix_proc_t procs[], size_t npro
     return rc;
 }
 
-PMIX_EXPORT pmix_status_t PMIx_Disconnect(const char nspace[],
+PMIX_EXPORT pmix_status_t PMIx_Disconnect(const pmix_proc_t procs[], size_t nprocs,
                                           const pmix_info_t info[], size_t ninfo)
 {
     pmix_status_t rc;
@@ -261,7 +223,7 @@ PMIX_EXPORT pmix_status_t PMIx_Disconnect(const char nspace[],
      * the return message is recvd */
     cb = PMIX_NEW(pmix_cb_t);
 
-    if (PMIX_SUCCESS != (rc = PMIx_Disconnect_nb(nspace, info, ninfo, op_cbfunc, cb))) {
+    if (PMIX_SUCCESS != (rc = PMIx_Disconnect_nb(procs, nprocs, info, ninfo, op_cbfunc, cb))) {
         PMIX_RELEASE(cb);
         return rc;
     }
@@ -277,7 +239,7 @@ PMIX_EXPORT pmix_status_t PMIx_Disconnect(const char nspace[],
     return rc;
 }
 
-PMIX_EXPORT pmix_status_t PMIx_Disconnect_nb(const char nspace[],
+PMIX_EXPORT pmix_status_t PMIx_Disconnect_nb(const pmix_proc_t procs[], size_t nprocs,
                                              const pmix_info_t info[], size_t ninfo,
                                              pmix_op_cbfunc_t cbfunc, void *cbdata)
 {
@@ -291,6 +253,13 @@ PMIX_EXPORT pmix_status_t PMIx_Disconnect_nb(const char nspace[],
     pmix_output_verbose(2, pmix_globals.debug_output,
                         "pmix: disconnect called");
 
+    size_t cnt;
+    for (cnt = 0; cnt < nprocs; cnt++) {
+        if (0 != strcmp(pmix_globals.myid.nspace, procs[cnt].nspace)) {
+            PMIX_GDS_DEL_NSPACE(rc, procs[cnt].nspace);
+        }
+    }
+
     if (pmix_globals.init_cntr <= 0) {
         PMIX_RELEASE_THREAD(&pmix_global_lock);
         return PMIX_ERR_INIT;
@@ -304,15 +273,10 @@ PMIX_EXPORT pmix_status_t PMIx_Disconnect_nb(const char nspace[],
     PMIX_RELEASE_THREAD(&pmix_global_lock);
 
     /* check for bozo input */
-    if (NULL == nspace) {
+    if (NULL == procs || 0 >= nprocs) {
         return PMIX_ERR_BAD_PARAM;
     }
 
-    /* release our internal resources */
-    if (0 != strncmp(pmix_globals.myid.nspace, nspace, PMIX_MAX_NSLEN)) {
-        PMIX_GDS_DEL_NSPACE(rc, nspace);
-    }
-
     msg = PMIX_NEW(pmix_buffer_t);
     /* pack the cmd */
     PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
@@ -322,9 +286,15 @@ PMIX_EXPORT pmix_status_t PMIx_Disconnect_nb(const char nspace[],
         return rc;
     }
 
-    /* pack the nspace */
+    /* pack the number of procs */
     PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
-                     msg, &nspace, 1, PMIX_STRING);
+                     msg, &nprocs, 1, PMIX_SIZE);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        return rc;
+    }
+    PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
+                     msg, procs, nprocs, PMIX_PROC);
     if (PMIX_SUCCESS != rc) {
         PMIX_ERROR_LOG(rc);
         return rc;
@@ -357,7 +327,7 @@ PMIX_EXPORT pmix_status_t PMIx_Disconnect_nb(const char nspace[],
 
     /* push the message into our event base to send to the server */
     PMIX_PTL_SEND_RECV(rc, pmix_client_globals.myserver,
-                       msg, discbfunc, (void*)cb);
+                       msg, wait_cbfunc, (void*)cb);
     if (PMIX_SUCCESS != rc) {
         PMIX_RELEASE(msg);
         PMIX_RELEASE(cb);
@@ -380,7 +350,6 @@ static void wait_cbfunc(struct pmix_peer_t *pr,
     char *nspace;
     pmix_buffer_t bkt;
     pmix_byte_object_t bo;
-    pmix_proc_t pname;
 
     pmix_output_verbose(2, pmix_globals.debug_output,
                         "pmix:client recv callback activated with %d bytes",
@@ -398,10 +367,6 @@ static void wait_cbfunc(struct pmix_peer_t *pr,
         goto report;
     }
 
-    /* set the default nspace/rank */
-    memset(pname.nspace, 0, PMIX_MAX_NSLEN+1);
-    pname.rank = PMIX_RANK_UNDEF;
-
     /* unpack the returned status */
     cnt = 1;
     PMIX_BFROPS_UNPACK(rc, pmix_client_globals.myserver,
@@ -410,23 +375,6 @@ static void wait_cbfunc(struct pmix_peer_t *pr,
         PMIX_ERROR_LOG(rc);
         ret = rc;
     }
-
-    if (PMIX_SUCCESS != ret) {
-        goto report;
-    }
-
-    if (cb->checked) {
-        /* unpack the returned nspace/rank */
-        cnt = 1;
-        PMIX_BFROPS_UNPACK(rc, pmix_client_globals.myserver,
-                           buf, &pname, &cnt, PMIX_PROC);
-        if (PMIX_SUCCESS != rc) {
-            PMIX_ERROR_LOG(rc);
-            ret = rc;
-            goto report;
-        }
-    }
-
     /* connect has to also pass back data from all nspace's involved in
      * the operation, including our own. Each will come as a byte object */
     cnt = 1;
@@ -463,47 +411,6 @@ static void wait_cbfunc(struct pmix_peer_t *pr,
         ret = rc;
     }
 
-  report:
-    if (NULL != cb->cbfunc.cnctfn) {
-        cb->cbfunc.cnctfn(ret, pname.nspace, pname.rank, cb->cbdata);
-    }
-    PMIX_RELEASE(cb);
-}
-
-static void discbfunc(struct pmix_peer_t *pr,
-                      pmix_ptl_hdr_t *hdr,
-                      pmix_buffer_t *buf, void *cbdata)
-{
-    pmix_cb_t *cb = (pmix_cb_t*)cbdata;
-    pmix_status_t rc;
-    pmix_status_t ret;
-    int32_t cnt;
-
-    pmix_output_verbose(2, pmix_globals.debug_output,
-                        "pmix:client recv callback activated with %d bytes",
-                        (NULL == buf) ? -1 : (int)buf->bytes_used);
-
-    if (NULL == buf) {
-        ret = PMIX_ERR_BAD_PARAM;
-        goto report;
-    }
-
-    /* a zero-byte buffer indicates that this recv is being
-     * completed due to a lost connection */
-    if (PMIX_BUFFER_IS_EMPTY(buf)) {
-        ret = PMIX_ERR_UNREACH;
-        goto report;
-    }
-
-    /* unpack the returned status */
-    cnt = 1;
-    PMIX_BFROPS_UNPACK(rc, pmix_client_globals.myserver,
-                       buf, &ret, &cnt, PMIX_STATUS);
-    if (PMIX_SUCCESS != rc) {
-        PMIX_ERROR_LOG(rc);
-        ret = rc;
-    }
-
   report:
     if (NULL != cb->cbfunc.opfn) {
         cb->cbfunc.opfn(ret, cb->cbdata);
@@ -519,18 +426,3 @@ static void op_cbfunc(pmix_status_t status, void *cbdata)
     PMIX_POST_OBJECT(cb);
     PMIX_WAKEUP_THREAD(&cb->lock);
 }
-
-static void cnct_cbfunc(pmix_status_t status,
-                        char nspace[], int rank,
-                        void *cbdata)
-{
-    pmix_cb_t *cb = (pmix_cb_t*)cbdata;
-
-    cb->status = status;
-    if (NULL != nspace) {
-        cb->pname.nspace = strdup(nspace);
-    }
-    cb->pname.rank = rank;
-    PMIX_POST_OBJECT(cb);
-    PMIX_WAKEUP_THREAD(&cb->lock);
-}

opal/mca/pmix/pmix3x/pmix/src/client/pmix_client_get.c

@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2014-2017 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
  * Copyright (c) 2014-2016 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2014      Artem Y. Polyakov <artpol84@gmail.com>.
@@ -111,6 +111,7 @@ PMIX_EXPORT pmix_status_t PMIx_Get(const pmix_proc_t *proc, const char key[],
     rc = cb->status;
     if (NULL != val) {
         *val = cb->value;
+        cb->value = NULL;
     }
     PMIX_RELEASE(cb);
 

opal/mca/pmix/pmix3x/pmix/src/common/pmix_control.c

@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2014-2017 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
  * Copyright (c) 2016      Mellanox Technologies, Inc.
  *                         All rights reserved.
  * Copyright (c) 2016      IBM Corporation.  All rights reserved.
@@ -24,6 +24,7 @@
 #include "src/threads/threads.h"
 #include "src/util/argv.h"
 #include "src/util/error.h"
+#include "src/util/name_fns.h"
 #include "src/util/output.h"
 #include "src/mca/bfrops/bfrops.h"
 #include "src/mca/ptl/ptl.h"
@@ -108,6 +109,59 @@ static void query_cbfunc(struct pmix_peer_t *peer,
     PMIX_RELEASE(cd);
 }
 
+static void acb(pmix_status_t status,
+                pmix_info_t *info, size_t ninfo,
+                void *cbdata,
+                pmix_release_cbfunc_t release_fn,
+                void *release_cbdata)
+{
+    pmix_cb_t *cb = (pmix_cb_t*)cbdata;
+    cb->status = status;
+    if (NULL != release_fn) {
+        release_fn(release_cbdata);
+    }
+    PMIX_WAKEUP_THREAD(&cb->lock);
+}
+
+PMIX_EXPORT pmix_status_t PMIx_Job_control(const pmix_proc_t targets[], size_t ntargets,
+                                           const pmix_info_t directives[], size_t ndirs)
+{
+    pmix_cb_t cb;
+    pmix_status_t rc;
+
+    PMIX_ACQUIRE_THREAD(&pmix_global_lock);
+
+    if (pmix_globals.init_cntr <= 0) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return PMIX_ERR_INIT;
+    }
+    PMIX_RELEASE_THREAD(&pmix_global_lock);
+
+    pmix_output_verbose(2, pmix_globals.debug_output,
+                        "%s pmix:job_ctrl", PMIX_NAME_PRINT(&pmix_globals.myid));
+
+    /* create a callback object as we need to pass it to the
+     * recv routine so we know which callback to use when
+     * the return message is recvd */
+    PMIX_CONSTRUCT(&cb, pmix_cb_t);
+    if (PMIX_SUCCESS != (rc = PMIx_Job_control_nb(targets, ntargets,
+                                                  directives, ndirs,
+                                                  acb, &cb))) {
+        PMIX_DESTRUCT(&cb);
+        return rc;
+    }
+
+    /* wait for the operation to complete */
+    PMIX_WAIT_THREAD(&cb.lock);
+    rc = cb.status;
+    PMIX_DESTRUCT(&cb);
+
+    pmix_output_verbose(2, pmix_globals.debug_output,
+                        "pmix:job_ctrl completed");
+
+    return rc;
+}
+
 PMIX_EXPORT pmix_status_t PMIx_Job_control_nb(const pmix_proc_t targets[], size_t ntargets,
                                               const pmix_info_t directives[], size_t ndirs,
                                               pmix_info_cbfunc_t cbfunc, void *cbdata)
@@ -127,16 +181,11 @@ PMIX_EXPORT pmix_status_t PMIx_Job_control_nb(const pmix_proc_t targets[], size_
         return PMIX_ERR_INIT;
     }
 
-    /* if we aren't connected, don't attempt to send */
-    if (!PMIX_PROC_IS_SERVER(pmix_globals.mypeer) && !pmix_globals.connected) {
-        PMIX_RELEASE_THREAD(&pmix_global_lock);
-        return PMIX_ERR_UNREACH;
-    }
-    PMIX_RELEASE_THREAD(&pmix_global_lock);
-
     /* if we are the server, then we just issue the request and
      * return the response */
-    if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) {
+    if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer) &&
+        !PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
         if (NULL == pmix_host_server.job_control) {
             /* nothing we can do */
             return PMIX_ERR_NOT_SUPPORTED;
@@ -150,6 +199,13 @@ PMIX_EXPORT pmix_status_t PMIx_Job_control_nb(const pmix_proc_t targets[], size_
         return rc;
     }
 
+    /* we need to send, so check for connection */
+    if (!pmix_globals.connected) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return PMIX_ERR_UNREACH;
+    }
+    PMIX_RELEASE_THREAD(&pmix_global_lock);
+
     /* if we are a client, then relay this request to the server */
     msg = PMIX_NEW(pmix_buffer_t);
     /* pack the cmd */
@@ -171,7 +227,7 @@ PMIX_EXPORT pmix_status_t PMIx_Job_control_nb(const pmix_proc_t targets[], size_
     }
     /* remember, the targets can be NULL to indicate that the operation
      * is to be done against all members of our nspace */
-    if (0 < ntargets) {
+    if (NULL != targets && 0 < ntargets) {
         /* pack the targets */
         PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
                          msg, targets, ntargets, PMIX_PROC);
@@ -190,7 +246,7 @@ PMIX_EXPORT pmix_status_t PMIx_Job_control_nb(const pmix_proc_t targets[], size_
         PMIX_RELEASE(msg);
         return rc;
     }
-    if (0 < ndirs) {
+    if (NULL != directives && 0 < ndirs) {
         PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
                          msg, directives, ndirs, PMIX_INFO);
         if (PMIX_SUCCESS != rc) {
@@ -218,6 +274,45 @@ PMIX_EXPORT pmix_status_t PMIx_Job_control_nb(const pmix_proc_t targets[], size_
     return rc;
 }
 
+PMIX_EXPORT pmix_status_t PMIx_Process_monitor(const pmix_info_t *monitor, pmix_status_t error,
+                                               const pmix_info_t directives[], size_t ndirs)
+{
+    pmix_cb_t cb;
+    pmix_status_t rc;
+
+    PMIX_ACQUIRE_THREAD(&pmix_global_lock);
+
+    if (pmix_globals.init_cntr <= 0) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return PMIX_ERR_INIT;
+    }
+    PMIX_RELEASE_THREAD(&pmix_global_lock);
+
+    pmix_output_verbose(2, pmix_globals.debug_output,
+                        "%s pmix:monitor", PMIX_NAME_PRINT(&pmix_globals.myid));
+
+    /* create a callback object as we need to pass it to the
+     * recv routine so we know which callback to use when
+     * the return message is recvd */
+    PMIX_CONSTRUCT(&cb, pmix_cb_t);
+    if (PMIX_SUCCESS != (rc = PMIx_Process_monitor_nb(monitor, error,
+                                                      directives, ndirs,
+                                                      acb, &cb))) {
+        PMIX_DESTRUCT(&cb);
+        return rc;
+    }
+
+    /* wait for the operation to complete */
+    PMIX_WAIT_THREAD(&cb.lock);
+    rc = cb.status;
+    PMIX_DESTRUCT(&cb);
+
+    pmix_output_verbose(2, pmix_globals.debug_output,
+                        "pmix:monitor completed");
+
+    return rc;
+}
+
 PMIX_EXPORT pmix_status_t PMIx_Process_monitor_nb(const pmix_info_t *monitor, pmix_status_t error,
                                                   const pmix_info_t directives[], size_t ndirs,
                                                   pmix_info_cbfunc_t cbfunc, void *cbdata)
@@ -237,16 +332,11 @@ PMIX_EXPORT pmix_status_t PMIx_Process_monitor_nb(const pmix_info_t *monitor, pm
         return PMIX_ERR_INIT;
     }
 
-    /* if we aren't connected, don't attempt to send */
-    if (!PMIX_PROC_IS_SERVER(pmix_globals.mypeer) && !pmix_globals.connected) {
-        PMIX_RELEASE_THREAD(&pmix_global_lock);
-        return PMIX_ERR_UNREACH;
-    }
-    PMIX_RELEASE_THREAD(&pmix_global_lock);
-
     /* if we are the server, then we just issue the request and
      * return the response */
-    if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) {
+    if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer) &&
+        !PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
         if (NULL == pmix_host_server.monitor) {
             /* nothing we can do */
             return PMIX_ERR_NOT_SUPPORTED;
@@ -258,6 +348,13 @@ PMIX_EXPORT pmix_status_t PMIx_Process_monitor_nb(const pmix_info_t *monitor, pm
         return rc;
     }
 
+    /* we need to send, so check for connection */
+    if (!pmix_globals.connected) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return PMIX_ERR_UNREACH;
+    }
+    PMIX_RELEASE_THREAD(&pmix_global_lock);
+
     /* if we are a client, then relay this request to the server */
     msg = PMIX_NEW(pmix_buffer_t);
     /* pack the cmd */

opal/mca/pmix/pmix3x/pmix/src/common/pmix_data.c

@@ -76,7 +76,12 @@ static pmix_peer_t* find_peer(const pmix_proc_t *proc)
     pmix_value_t *value;
     int i;
 
-    if (NULL == proc) {
+    if (NULL == proc ) {
+        return pmix_globals.mypeer;
+    }
+
+    /* if the target is someone in my nspace, then use my own peer */
+    if (0 == strncmp(proc->nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN)) {
         return pmix_globals.mypeer;
     }
 

opal/mca/pmix/pmix3x/pmix/src/common/pmix_iof.c

@@ -83,7 +83,8 @@ PMIX_EXPORT pmix_status_t PMIx_IOF_pull(const pmix_proc_t procs[], size_t nprocs
     }
 
     /* if we are a server, we cannot do this */
-    if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) {
+    if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer) &&
+        !PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) {
         PMIX_RELEASE_THREAD(&pmix_global_lock);
         return PMIX_ERR_NOT_SUPPORTED;
     }
@@ -236,7 +237,8 @@ pmix_status_t PMIx_IOF_push(const pmix_proc_t targets[], size_t ntargets,
 
     /* if we are not a server, then we send the provided
      * data to our server for processing */
-    if (!PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) {
+    if (!PMIX_PROC_IS_SERVER(pmix_globals.mypeer) ||
+        PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) {
         msg = PMIX_NEW(pmix_buffer_t);
         if (NULL == msg) {
             return PMIX_ERR_NOMEM;
@@ -318,7 +320,7 @@ pmix_status_t PMIx_IOF_push(const pmix_proc_t targets[], size_t ntargets,
 pmix_status_t pmix_iof_write_output(const pmix_proc_t *name,
                                     pmix_iof_channel_t stream,
                                     const pmix_byte_object_t *bo,
-                                    pmix_iof_write_event_t *channel)
+                                    pmix_iof_flags_t *flags)
 {
     char starttag[PMIX_IOF_BASE_TAG_MAX], endtag[PMIX_IOF_BASE_TAG_MAX], *suffix;
     pmix_iof_write_output_t *output;
@@ -326,6 +328,25 @@ pmix_status_t pmix_iof_write_output(const pmix_proc_t *name,
     int j, k, starttaglen, endtaglen, num_buffered;
     bool endtagged;
     char qprint[10];
+    pmix_iof_write_event_t *channel;
+    pmix_iof_flags_t myflags;
+
+    if (PMIX_FWD_STDOUT_CHANNEL & stream) {
+        channel = &pmix_client_globals.iof_stdout.wev;
+    } else {
+        channel = &pmix_client_globals.iof_stderr.wev;
+    }
+    if (NULL == flags) {
+        myflags.xml = pmix_globals.xml_output;
+        if (pmix_globals.timestamp_output) {
+            time(&myflags.timestamp);
+        } else {
+            myflags.timestamp = 0;
+        }
+        myflags.tag = pmix_globals.tag_output;
+    } else {
+        myflags = *flags;
+    }
 
     PMIX_OUTPUT_VERBOSE((1, pmix_client_globals.iof_output,
                          "%s write:output setting up to write %lu bytes to %s for %s on fd %d",
@@ -337,6 +358,8 @@ pmix_status_t pmix_iof_write_output(const pmix_proc_t *name,
 
     /* setup output object */
     output = PMIX_NEW(pmix_iof_write_output_t);
+    memset(starttag, 0, PMIX_IOF_BASE_TAG_MAX);
+    memset(endtag, 0, PMIX_IOF_BASE_TAG_MAX);
 
     /* write output data to the corresponding tag */
     if (PMIX_FWD_STDIN_CHANNEL & stream) {
@@ -370,22 +393,20 @@ pmix_status_t pmix_iof_write_output(const pmix_proc_t *name,
     /* if this is to be xml tagged, create a tag with the correct syntax - we do not allow
      * timestamping of xml output
      */
-    if (pmix_globals.xml_output) {
+    if (myflags.xml) {
         snprintf(starttag, PMIX_IOF_BASE_TAG_MAX, "<%s rank=\"%s\">", suffix, PMIX_RANK_PRINT(name->rank));
         snprintf(endtag, PMIX_IOF_BASE_TAG_MAX, "</%s>", suffix);
         goto construct;
     }
 
     /* if we are to timestamp output, start the tag with that */
-    if (pmix_globals.timestamp_output) {
-        time_t mytime;
+    if (0 < myflags.timestamp) {
         char *cptr;
         /* get the timestamp */
-        time(&mytime);
-        cptr = ctime(&mytime);
+        cptr = ctime(&myflags.timestamp);
         cptr[strlen(cptr)-1] = '\0';  /* remove trailing newline */
 
-        if (pmix_globals.tag_output) {
+        if (myflags.tag) {
             /* if we want it tagged as well, use both */
             snprintf(starttag, PMIX_IOF_BASE_TAG_MAX, "%s[%s]<%s>:",
                      cptr, PMIX_NAME_PRINT(name), suffix);
@@ -398,7 +419,7 @@ pmix_status_t pmix_iof_write_output(const pmix_proc_t *name,
         goto construct;
     }
 
-    if (pmix_globals.tag_output) {
+    if (myflags.tag) {
         snprintf(starttag, PMIX_IOF_BASE_TAG_MAX, "[%s]<%s>:",
                  PMIX_NAME_PRINT(name), suffix);
         /* no endtag for this option */
@@ -431,7 +452,7 @@ pmix_status_t pmix_iof_write_output(const pmix_proc_t *name,
      * and replace those with the tag
      */
     for (i=0; i < bo->size && k < PMIX_IOF_BASE_TAGGED_OUT_MAX; i++) {
-        if (pmix_globals.xml_output) {
+        if (myflags.xml) {
             if ('&' == bo->bytes[i]) {
                 if (k+5 >= PMIX_IOF_BASE_TAGGED_OUT_MAX) {
                     PMIX_ERROR_LOG(PMIX_ERR_OUT_OF_RESOURCE);

opal/mca/pmix/pmix3x/pmix/src/common/pmix_iof.h

@@ -104,6 +104,15 @@ typedef struct {
 PMIX_EXPORT PMIX_CLASS_DECLARATION(pmix_iof_read_event_t);
 
 
+/* define a struct to hold booleans controlling the
+ * format/contents of the output */
+typedef struct {
+    bool xml;
+    time_t timestamp;
+    bool tag;
+} pmix_iof_flags_t;
+
+
 /* Write event macro's */
 
 static inline bool
@@ -184,7 +193,7 @@ PMIX_EXPORT pmix_status_t pmix_iof_flush(void);
 PMIX_EXPORT pmix_status_t pmix_iof_write_output(const pmix_proc_t *name,
                                                 pmix_iof_channel_t stream,
                                                 const pmix_byte_object_t *bo,
-                                                pmix_iof_write_event_t *channel);
+                                                pmix_iof_flags_t *flags);
 PMIX_EXPORT void pmix_iof_static_dump_output(pmix_iof_sink_t *sink);
 PMIX_EXPORT void pmix_iof_write_handler(int fd, short event, void *cbdata);
 PMIX_EXPORT void pmix_iof_stdin_write_handler(int fd, short event, void *cbdata);

opal/mca/pmix/pmix3x/pmix/src/common/pmix_log.c

@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2014-2017 Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
  * Copyright (c) 2016      Mellanox Technologies, Inc.
  *                         All rights reserved.
  * Copyright (c) 2016      IBM Corporation.  All rights reserved.
@@ -24,14 +24,22 @@
 #include "src/threads/threads.h"
 #include "src/util/argv.h"
 #include "src/util/error.h"
+#include "src/util/name_fns.h"
 #include "src/util/output.h"
 #include "src/mca/bfrops/bfrops.h"
-#include "src/mca/ptl/ptl.h"
+#include "src/mca/plog/base/base.h"
 
 #include "src/client/pmix_client_ops.h"
 #include "src/server/pmix_server_ops.h"
 #include "src/include/pmix_globals.h"
 
+static void opcbfunc(pmix_status_t status, void *cbdata)
+{
+    pmix_cb_t *cb = (pmix_cb_t*)cbdata;
+    cb->status = status;
+    PMIX_WAKEUP_THREAD(&cb->lock);
+}
+
 static void log_cbfunc(struct pmix_peer_t *peer,
                        pmix_ptl_hdr_t *hdr,
                        pmix_buffer_t *buf, void *cbdata)
@@ -53,6 +61,55 @@ static void log_cbfunc(struct pmix_peer_t *peer,
     PMIX_RELEASE(cd);
 }
 
+PMIX_EXPORT pmix_status_t PMIx_Log(const pmix_info_t data[], size_t ndata,
+                                   const pmix_info_t directives[], size_t ndirs)
+{
+    pmix_cb_t cb;
+    pmix_status_t rc;
+
+    PMIX_ACQUIRE_THREAD(&pmix_global_lock);
+
+    if (pmix_globals.init_cntr <= 0) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return PMIX_ERR_INIT;
+    }
+    PMIX_RELEASE_THREAD(&pmix_global_lock);
+
+    pmix_output_verbose(2, pmix_plog_base_framework.framework_output,
+                        "%s pmix:log", PMIX_NAME_PRINT(&pmix_globals.myid));
+
+    /* create a callback object as we need to pass it to the
+     * recv routine so we know which callback to use when
+     * the return message is recvd */
+    PMIX_CONSTRUCT(&cb, pmix_cb_t);
+    if (PMIX_SUCCESS != (rc = PMIx_Log_nb(data, ndata, directives,
+                                          ndirs, opcbfunc, &cb))) {
+        PMIX_DESTRUCT(&cb);
+        return rc;
+    }
+
+    /* wait for the operation to complete */
+    PMIX_WAIT_THREAD(&cb.lock);
+    rc = cb.status;
+    PMIX_DESTRUCT(&cb);
+
+    pmix_output_verbose(2, pmix_plog_base_framework.framework_output,
+                        "pmix:log completed");
+
+    return rc;
+}
+
+static void localcbfunc(pmix_status_t status, void *cbdata)
+{
+    pmix_shift_caddy_t *cd = (pmix_shift_caddy_t*)cbdata;
+
+    PMIX_INFO_FREE(cd->directives, cd->ndirs);
+    if (NULL != cd->cbfunc.opcbfn) {
+        cd->cbfunc.opcbfn(status, cd->cbdata);
+    }
+    PMIX_RELEASE(cd);
+}
+
 PMIX_EXPORT pmix_status_t PMIx_Log_nb(const pmix_info_t data[], size_t ndata,
                                       const pmix_info_t directives[], size_t ndirs,
                                       pmix_op_cbfunc_t cbfunc, void *cbdata)
@@ -62,6 +119,9 @@ PMIX_EXPORT pmix_status_t PMIx_Log_nb(const pmix_info_t data[], size_t ndata,
     pmix_cmd_t cmd = PMIX_LOG_CMD;
     pmix_buffer_t *msg;
     pmix_status_t rc;
+    size_t n;
+    time_t timestamp = 0;
+    pmix_proc_t *source = NULL;
 
     PMIX_ACQUIRE_THREAD(&pmix_global_lock);
 
@@ -73,32 +133,38 @@ PMIX_EXPORT pmix_status_t PMIx_Log_nb(const pmix_info_t data[], size_t ndata,
         return PMIX_ERR_INIT;
     }
 
-    /* if we aren't connected, don't attempt to send */
-    if (!PMIX_PROC_IS_SERVER(pmix_globals.mypeer) && !pmix_globals.connected) {
-        PMIX_RELEASE_THREAD(&pmix_global_lock);
-        return PMIX_ERR_UNREACH;
-    }
-    PMIX_RELEASE_THREAD(&pmix_global_lock);
-
     if (0 == ndata || NULL == data) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
         return PMIX_ERR_BAD_PARAM;
     }
 
-    /* if we are the server, then we just log and
-     * return the response */
-    if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) {
-            if (NULL == pmix_host_server.log) {
-                /* nothing we can do */
-                return PMIX_ERR_NOT_SUPPORTED;
+    /* check the directives - if they requested a timestamp, then
+     * get the time, also look for a source */
+    if (NULL != directives) {
+        for (n=0; n < ndirs; n++) {
+            if (0 == strncmp(directives[n].key, PMIX_LOG_GENERATE_TIMESTAMP, PMIX_MAX_KEYLEN)) {
+                if (PMIX_INFO_TRUE(&directives[n])) {
+                    /* pickup the timestamp */
+                    timestamp = time(NULL);
+                }
+            } else if (0 == strncmp(directives[n].key, PMIX_LOG_SOURCE, PMIX_MAX_KEYLEN)) {
+                source = directives[n].value.data.proc;
             }
-            pmix_output_verbose(2, pmix_globals.debug_output,
-                                "pmix:log handed to RM");
-            pmix_host_server.log(&pmix_globals.myid,
-                                 data, ndata, directives, ndirs,
-                                 cbfunc, cbdata);
-            rc = PMIX_SUCCESS;
-    } else {
-        /* if we are a client, then relay this request to the server */
+        }
+    }
+
+    /* if we are a client or tool, we never do this ourselves - we
+     * always pass this request to our server for execution */
+    if (!PMIX_PROC_IS_SERVER(pmix_globals.mypeer) &&
+        !PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) {
+        /* if we aren't connected, don't attempt to send */
+        if (!pmix_globals.connected) {
+            PMIX_RELEASE_THREAD(&pmix_global_lock);
+            return PMIX_ERR_UNREACH;
+        }
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+
+        /* if we are not a server, then relay this request to the server */
         cd = PMIX_NEW(pmix_shift_caddy_t);
         cd->cbfunc.opcbfn = cbfunc;
         cd->cbdata = cbdata;
@@ -111,6 +177,17 @@ PMIX_EXPORT pmix_status_t PMIx_Log_nb(const pmix_info_t data[], size_t ndata,
             PMIX_RELEASE(cd);
             return rc;
         }
+        /* provide the timestamp - zero will indicate
+         * that it wasn't taken */
+        PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
+                         msg, &timestamp, 1, PMIX_TIME);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            PMIX_RELEASE(msg);
+            PMIX_RELEASE(cd);
+            return rc;
+        }
+        /* pack the number of data entries */
         PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
                          msg, &ndata, 1, PMIX_SIZE);
         if (PMIX_SUCCESS != rc) {
@@ -119,13 +196,15 @@ PMIX_EXPORT pmix_status_t PMIx_Log_nb(const pmix_info_t data[], size_t ndata,
             PMIX_RELEASE(cd);
             return rc;
         }
-        PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
-                         msg, data, ndata, PMIX_INFO);
-        if (PMIX_SUCCESS != rc) {
-            PMIX_ERROR_LOG(rc);
-            PMIX_RELEASE(msg);
-            PMIX_RELEASE(cd);
-            return rc;
+        if (0 < ndata) {
+            PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
+                             msg, data, ndata, PMIX_INFO);
+            if (PMIX_SUCCESS != rc) {
+                PMIX_ERROR_LOG(rc);
+                PMIX_RELEASE(msg);
+                PMIX_RELEASE(cd);
+                return rc;
+            }
         }
         PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
                          msg, &ndirs, 1, PMIX_SIZE);
@@ -146,7 +225,7 @@ PMIX_EXPORT pmix_status_t PMIx_Log_nb(const pmix_info_t data[], size_t ndata,
             }
         }
 
-        pmix_output_verbose(2, pmix_globals.debug_output,
+        pmix_output_verbose(2, pmix_plog_base_framework.framework_output,
                             "pmix:log sending to server");
         PMIX_PTL_SEND_RECV(rc, pmix_client_globals.myserver,
                            msg, log_cbfunc, (void*)cd);
@@ -154,6 +233,41 @@ PMIX_EXPORT pmix_status_t PMIx_Log_nb(const pmix_info_t data[], size_t ndata,
             PMIX_ERROR_LOG(rc);
             PMIX_RELEASE(cd);
         }
+        return rc;
     }
+    PMIX_RELEASE_THREAD(&pmix_global_lock);
+
+    /* if no recorded source was found, then we must be it */
+    if (NULL == source) {
+        source = &pmix_globals.myid;
+        cd = PMIX_NEW(pmix_shift_caddy_t);
+        cd->cbfunc.opcbfn = cbfunc;
+        cd->cbdata = cbdata;
+        cd->ndirs = ndirs + 1;
+        PMIX_INFO_CREATE(cd->directives, cd->ndirs);
+        for (n=0; n < ndirs; n++) {
+            PMIX_INFO_XFER(&cd->directives[n], (pmix_info_t*)&directives[n]);
+        }
+        PMIX_INFO_LOAD(&cd->directives[ndirs], PMIX_LOG_SOURCE, &source, PMIX_PROC);
+        /* call down to process the request - the various components
+         * will thread shift as required */
+        rc = pmix_plog.log(source, data, ndata, cd->directives, cd->ndirs, localcbfunc, cd);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_INFO_FREE(cd->directives, cd->ndirs);
+            PMIX_RELEASE(cd);
+        }
+    } else if (0 == strncmp(source->nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN) &&
+               source->rank == pmix_globals.myid.rank) {
+        /* if I am the recorded source, then this is a re-submission of
+         * something that got "upcalled" by a prior call. In this case,
+         * we return a "not supported" error as clearly we couldn't
+         * handle it, and neither could our host */
+        rc = PMIX_ERR_NOT_SUPPORTED;
+    } else {
+        /* call down to process the request - the various components
+         * will thread shift as required */
+        rc = pmix_plog.log(source, data, ndata, directives, ndirs, cbfunc, cbdata);
+    }
+
     return rc;
 }

opal/mca/pmix/pmix3x/pmix/src/common/pmix_query.c

@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2014-2017 Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
  * Copyright (c) 2016      Mellanox Technologies, Inc.
  *                         All rights reserved.
  * Copyright (c) 2016      IBM Corporation.  All rights reserved.
@@ -24,6 +24,7 @@
 #include "src/threads/threads.h"
 #include "src/util/argv.h"
 #include "src/util/error.h"
+#include "src/util/name_fns.h"
 #include "src/util/output.h"
 #include "src/mca/bfrops/bfrops.h"
 #include "src/mca/ptl/ptl.h"
@@ -63,6 +64,7 @@ static void query_cbfunc(struct pmix_peer_t *peer,
     PMIX_BFROPS_UNPACK(rc, peer, buf, &results->status, &cnt, PMIX_STATUS);
     if (PMIX_SUCCESS != rc) {
         PMIX_ERROR_LOG(rc);
+        results->status = rc;
         goto complete;
     }
     if (PMIX_SUCCESS != results->status) {
@@ -74,6 +76,7 @@ static void query_cbfunc(struct pmix_peer_t *peer,
     PMIX_BFROPS_UNPACK(rc, peer, buf, &results->ninfo, &cnt, PMIX_SIZE);
     if (PMIX_SUCCESS != rc) {
         PMIX_ERROR_LOG(rc);
+        results->status = rc;
         goto complete;
     }
     if (0 < results->ninfo) {
@@ -82,6 +85,7 @@ static void query_cbfunc(struct pmix_peer_t *peer,
         PMIX_BFROPS_UNPACK(rc, peer, buf, results->info, &cnt, PMIX_INFO);
         if (PMIX_SUCCESS != rc) {
             PMIX_ERROR_LOG(rc);
+            results->status = rc;
             goto complete;
         }
     }
@@ -115,20 +119,16 @@ PMIX_EXPORT pmix_status_t PMIx_Query_info_nb(pmix_query_t queries[], size_t nque
         return PMIX_ERR_INIT;
     }
 
-    /* if we aren't connected, don't attempt to send */
-    if (!PMIX_PROC_IS_SERVER(pmix_globals.mypeer) && !pmix_globals.connected) {
-        PMIX_RELEASE_THREAD(&pmix_global_lock);
-        return PMIX_ERR_UNREACH;
-    }
-    PMIX_RELEASE_THREAD(&pmix_global_lock);
-
     if (0 == nqueries || NULL == queries) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
         return PMIX_ERR_BAD_PARAM;
     }
 
     /* if we are the server, then we just issue the query and
      * return the response */
-    if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) {
+    if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer) &&
+        !PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
         if (NULL == pmix_host_server.query) {
             /* nothing we can do */
             return PMIX_ERR_NOT_SUPPORTED;
@@ -138,45 +138,104 @@ PMIX_EXPORT pmix_status_t PMIx_Query_info_nb(pmix_query_t queries[], size_t nque
         pmix_host_server.query(&pmix_globals.myid,
                                queries, nqueries,
                                cbfunc, cbdata);
-        rc = PMIX_SUCCESS;
-    } else {
-        /* if we are a client, then relay this request to the server */
-        cd = PMIX_NEW(pmix_query_caddy_t);
-        cd->cbfunc = cbfunc;
-        cd->cbdata = cbdata;
-        msg = PMIX_NEW(pmix_buffer_t);
-        PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
-                         msg, &cmd, 1, PMIX_COMMAND);
-        if (PMIX_SUCCESS != rc) {
-            PMIX_ERROR_LOG(rc);
-            PMIX_RELEASE(msg);
-            PMIX_RELEASE(cd);
-            return rc;
-        }
-        PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
-                         msg, &nqueries, 1, PMIX_SIZE);
-        if (PMIX_SUCCESS != rc) {
-            PMIX_ERROR_LOG(rc);
-            PMIX_RELEASE(msg);
-            PMIX_RELEASE(cd);
-            return rc;
-        }
-        PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
-                         msg, queries, nqueries, PMIX_QUERY);
-        if (PMIX_SUCCESS != rc) {
-            PMIX_ERROR_LOG(rc);
-            PMIX_RELEASE(msg);
-            PMIX_RELEASE(cd);
-            return rc;
-        }
-        pmix_output_verbose(2, pmix_globals.debug_output,
-                            "pmix:query sending to server");
-        PMIX_PTL_SEND_RECV(rc, pmix_client_globals.myserver,
-                           msg, query_cbfunc, (void*)cd);
-        if (PMIX_SUCCESS != rc) {
-            PMIX_RELEASE(cd);
-        }
+        return PMIX_SUCCESS;
     }
+
+    /* if we aren't connected, don't attempt to send */
+    if (!pmix_globals.connected) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return PMIX_ERR_UNREACH;
+    }
+    PMIX_RELEASE_THREAD(&pmix_global_lock);
+
+    /* if we are a client, then relay this request to the server */
+    cd = PMIX_NEW(pmix_query_caddy_t);
+    cd->cbfunc = cbfunc;
+    cd->cbdata = cbdata;
+    msg = PMIX_NEW(pmix_buffer_t);
+    PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
+                     msg, &cmd, 1, PMIX_COMMAND);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_RELEASE(msg);
+        PMIX_RELEASE(cd);
+        return rc;
+    }
+    PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
+                     msg, &nqueries, 1, PMIX_SIZE);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_RELEASE(msg);
+        PMIX_RELEASE(cd);
+        return rc;
+    }
+    PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
+                     msg, queries, nqueries, PMIX_QUERY);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_RELEASE(msg);
+        PMIX_RELEASE(cd);
+        return rc;
+    }
+    pmix_output_verbose(2, pmix_globals.debug_output,
+                        "pmix:query sending to server");
+    PMIX_PTL_SEND_RECV(rc, pmix_client_globals.myserver,
+                       msg, query_cbfunc, (void*)cd);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_RELEASE(cd);
+    }
+    return rc;
+}
+
+static void acb(pmix_status_t status,
+                pmix_info_t *info, size_t ninfo,
+                void *cbdata,
+                pmix_release_cbfunc_t release_fn,
+                void *release_cbdata)
+{
+    pmix_cb_t *cb = (pmix_cb_t*)cbdata;
+    cb->status = status;
+    if (NULL != release_fn) {
+        release_fn(release_cbdata);
+    }
+    PMIX_WAKEUP_THREAD(&cb->lock);
+}
+
+PMIX_EXPORT pmix_status_t PMIx_Allocation_request(pmix_alloc_directive_t directive,
+                                                  pmix_info_t *info, size_t ninfo)
+{
+    pmix_cb_t cb;
+    pmix_status_t rc;
+
+    PMIX_ACQUIRE_THREAD(&pmix_global_lock);
+
+    if (pmix_globals.init_cntr <= 0) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return PMIX_ERR_INIT;
+    }
+    PMIX_RELEASE_THREAD(&pmix_global_lock);
+
+    pmix_output_verbose(2, pmix_globals.debug_output,
+                        "%s pmix:allocate", PMIX_NAME_PRINT(&pmix_globals.myid));
+
+    /* create a callback object as we need to pass it to the
+     * recv routine so we know which callback to use when
+     * the return message is recvd */
+    PMIX_CONSTRUCT(&cb, pmix_cb_t);
+    if (PMIX_SUCCESS != (rc = PMIx_Allocation_request_nb(directive, info, ninfo,
+                                                         acb, &cb))) {
+        PMIX_DESTRUCT(&cb);
+        return rc;
+    }
+
+    /* wait for the operation to complete */
+    PMIX_WAIT_THREAD(&cb.lock);
+    rc = cb.status;
+    PMIX_DESTRUCT(&cb);
+
+    pmix_output_verbose(2, pmix_globals.debug_output,
+                        "pmix:allocate completed");
+
     return rc;
 }
 
@@ -192,13 +251,18 @@ PMIX_EXPORT pmix_status_t PMIx_Allocation_request_nb(pmix_alloc_directive_t dire
     pmix_output_verbose(2, pmix_globals.debug_output,
                         "pmix: allocate called");
 
+    PMIX_ACQUIRE_THREAD(&pmix_global_lock);
+
     if (pmix_globals.init_cntr <= 0) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
         return PMIX_ERR_INIT;
     }
 
     /* if we are the server, then we just issue the request and
      * return the response */
-    if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) {
+    if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer) &&
+        !PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
         if (NULL == pmix_host_server.allocate) {
             /* nothing we can do */
             return PMIX_ERR_NOT_SUPPORTED;
@@ -216,8 +280,10 @@ PMIX_EXPORT pmix_status_t PMIx_Allocation_request_nb(pmix_alloc_directive_t dire
 
     /* if we aren't connected, don't attempt to send */
     if (!pmix_globals.connected) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
         return PMIX_ERR_UNREACH;
     }
+    PMIX_RELEASE_THREAD(&pmix_global_lock);
 
     msg = PMIX_NEW(pmix_buffer_t);
     /* pack the cmd */

opal/mca/pmix/pmix3x/pmix/src/common/pmix_security.c

@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2014-2017 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
  * Copyright (c) 2016      Mellanox Technologies, Inc.
  *                         All rights reserved.
  * Copyright (c) 2016      IBM Corporation.  All rights reserved.
@@ -131,7 +131,8 @@ PMIX_EXPORT pmix_status_t PMIx_Get_credential(const pmix_info_t info[], size_t n
     }
 
     /* if we are the server */
-    if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) {
+    if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer) &&
+        !PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) {
         PMIX_RELEASE_THREAD(&pmix_global_lock);
         /* if the host doesn't support this operation,
          * see if we can generate it ourselves */
@@ -316,7 +317,8 @@ PMIX_EXPORT pmix_status_t PMIx_Validate_credential(const pmix_byte_object_t *cre
     }
 
     /* if we are the server */
-    if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) {
+    if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer) &&
+        !PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) {
         PMIX_RELEASE_THREAD(&pmix_global_lock);
         /* if the host doesn't support this operation,
          * see if we can validate it ourselves */

opal/mca/pmix/pmix3x/pmix/src/common/pmix_strings.c

@@ -71,8 +71,12 @@ PMIX_EXPORT const char* PMIx_Proc_state_string(pmix_proc_state_t state)
             return "PROC TERMINATED WITHOUT CALLING PMIx_Finalize";
         case PMIX_PROC_STATE_COMM_FAILED:
             return "PROC LOST COMMUNICATION";
+        case PMIX_PROC_STATE_SENSOR_BOUND_EXCEEDED:
+            return "PROC SENSOR BOUND EXCEEDED";
         case PMIX_PROC_STATE_CALLED_ABORT:
             return "PROC CALLED PMIx_Abort";
+        case PMIX_PROC_STATE_HEARTBEAT_FAILED:
+            return "PROC FAILED TO REPORT HEARTBEAT";
         case PMIX_PROC_STATE_MIGRATING:
             return "PROC WAITING TO MIGRATE";
         case PMIX_PROC_STATE_CANNOT_RESTART:
@@ -117,6 +121,8 @@ PMIX_EXPORT const char* PMIx_Persistence_string(pmix_persistence_t persist)
             return "RETAIN UNTIL APPLICATION OF PUBLISHING PROCESS TERMINATES";
         case PMIX_PERSIST_SESSION:
             return "RETAIN UNTIL ALLOCATION OF PUBLISHING PROCESS TERMINATES";
+        case PMIX_PERSIST_INVALID:
+            return "INVALID";
         default:
             return "UNKNOWN PERSISTENCE";
     }
@@ -141,6 +147,8 @@ PMIX_EXPORT const char* PMIx_Data_range_string(pmix_data_range_t range)
             return "AVAIL AS SPECIFIED IN DIRECTIVES";
         case PMIX_RANGE_PROC_LOCAL:
             return "AVAIL ON LOCAL PROC ONLY";
+        case PMIX_RANGE_INVALID:
+            return "INVALID";
         default:
             return "UNKNOWN";
     }

opal/mca/pmix/pmix3x/pmix/src/event/pmix_event.h

@@ -10,7 +10,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2015-2017 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2015-2018 Intel, Inc. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -53,7 +53,22 @@ typedef struct {
     size_t index;
     uint8_t precedence;
     char *locator;
+    pmix_proc_t source;  // who generated this event
+    /* When registering for events, callers can specify
+     * the range of sources from which they are willing
+     * to receive notifications - e.g., for callers to
+     * define different handlers for events coming from
+     * the RM vs those coming from their peers. We use
+     * the rng field to track these values upon registration.
+     */
     pmix_range_trkr_t rng;
+    /* For registration, we use the affected field to track
+     * the range of procs that, if affected by the event,
+     * should cause the handler to be called (subject, of
+     * course, to any rng constraints).
+     */
+    pmix_proc_t *affected;
+    size_t naffected;
     pmix_notification_fn_t evhdlr;
     void *cbobject;
     pmix_status_t *codes;
@@ -102,8 +117,11 @@ typedef struct pmix_event_chain_t {
     bool endchain;
     pmix_proc_t source;
     pmix_data_range_t range;
+    pmix_proc_t *affected;
+    size_t naffected;
     pmix_info_t *info;
     size_t ninfo;
+    size_t nallocated;
     pmix_info_t *results;
     size_t nresults;
     pmix_event_hdlr_t *evhdlr;
@@ -117,6 +135,13 @@ PMIX_CLASS_DECLARATION(pmix_event_chain_t);
  * affected, plus any additional info provided by the server */
 void pmix_invoke_local_event_hdlr(pmix_event_chain_t *chain);
 
+bool pmix_notify_check_range(pmix_range_trkr_t *rng,
+                             const pmix_proc_t *proc);
+
+bool pmix_notify_check_affected(pmix_proc_t *interested, size_t ninterested,
+                                pmix_proc_t *affected, size_t naffected);
+
+
 /* invoke the server event notification handler */
 pmix_status_t pmix_server_notify_client_of_event(pmix_status_t status,
                                                  const pmix_proc_t *source,
@@ -150,16 +175,11 @@ void pmix_event_timeout_cb(int fd, short flags, void *arg);
                           (p)->nptr->nspace,                                        \
                           PMIX_MAX_NSLEN);                                          \
             ch->source.rank = (p)->info->pname.rank;                                \
-            ch->ninfo = 2;                                                          \
+            ch->ninfo = 0;                                                          \
+            ch->nallocated = 2;                                                     \
             ch->final_cbfunc = (f);                                                 \
             ch->final_cbdata = ch;                                                  \
-            PMIX_INFO_CREATE(ch->info, ch->ninfo);                                  \
-            PMIX_INFO_LOAD(&ch->info[0],                                            \
-                           PMIX_EVENT_HDLR_NAME,                                    \
-                           NULL, PMIX_STRING);                                      \
-            PMIX_INFO_LOAD(&ch->info[1],                                            \
-                           PMIX_EVENT_RETURN_OBJECT,                                \
-                           NULL, PMIX_POINTER);                                     \
+            PMIX_INFO_CREATE(ch->info, ch->nallocated);                             \
             /* cache it */                                                          \
             pmix_list_append(&pmix_globals.cached_events, &ch->super);              \
             ch->timer_active = true;                                                \
@@ -171,7 +191,7 @@ void pmix_event_timeout_cb(int fd, short flags, void *arg);
             /* add this peer to the array of sources */                             \
             (void)strncpy(proc.nspace, (p)->nptr->nspace, PMIX_MAX_NSLEN);          \
             proc.rank = (p)->info->pname.rank;                                      \
-            ninfo = ch->ninfo + 1;                                                  \
+            ninfo = ch->nallocated + 1;                                             \
             PMIX_INFO_CREATE(info, ninfo);                                          \
             /* must keep the hdlr name and return object at the end, so prepend */  \
             PMIX_INFO_LOAD(&info[0], PMIX_PROCID,                                   \
@@ -179,9 +199,10 @@ void pmix_event_timeout_cb(int fd, short flags, void *arg);
             for (n=0; n < ch->ninfo; n++) {                                         \
                 PMIX_INFO_XFER(&info[n+1], &ch->info[n]);                           \
             }                                                                       \
-            PMIX_INFO_FREE(ch->info, ch->ninfo);                                    \
+            PMIX_INFO_FREE(ch->info, ch->nallocated);                               \
+            ch->nallocated = ninfo;                                                 \
             ch->info = info;                                                        \
-            ch->ninfo = ninfo;                                                      \
+            ch->ninfo = ninfo - 2;                                                  \
             /* reset the timer */                                                   \
             pmix_event_del(&ch->ev);                                                \
             PMIX_POST_OBJECT(ch);                                                   \

opal/mca/pmix/pmix3x/pmix/src/event/pmix_event_notification.c

@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2014-2017 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
  * Copyright (c) 2017      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2017      IBM Corporation. All rights reserved.
@@ -33,8 +33,6 @@ static pmix_status_t notify_server_of_event(pmix_status_t status,
                                             pmix_info_t info[], size_t ninfo,
                                             pmix_op_cbfunc_t cbfunc, void *cbdata);
 
-static bool check_range(pmix_range_trkr_t *range, const pmix_proc_t *proc);
-
 /* if we are a client, we call this function to notify the server of
  * an event. If we are a server, our host RM will call this function
  * to notify us of an event */
@@ -53,15 +51,10 @@ PMIX_EXPORT pmix_status_t PMIx_Notify_event(pmix_status_t status,
         return PMIX_ERR_INIT;
     }
 
-    /* if we aren't connected, don't attempt to send */
-    if (!PMIX_PROC_IS_SERVER(pmix_globals.mypeer) && !pmix_globals.connected) {
+
+    if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer) &&
+        !PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) {
         PMIX_RELEASE_THREAD(&pmix_global_lock);
-        return PMIX_ERR_UNREACH;
-    }
-    PMIX_RELEASE_THREAD(&pmix_global_lock);
-
-
-    if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) {
         rc = pmix_server_notify_client_of_event(status, source, range,
                                                 info, ninfo,
                                                 cbfunc, cbdata);
@@ -69,15 +62,23 @@ PMIX_EXPORT pmix_status_t PMIx_Notify_event(pmix_status_t status,
                             "pmix_server_notify_event source = %s:%d event_status = %d, rc= %d",
                             (NULL == source) ? "UNKNOWN" : source->nspace,
                             (NULL == source) ? PMIX_RANK_WILDCARD : source->rank, status, rc);
-    } else {
-        rc = notify_server_of_event(status, source, range,
-                                    info, ninfo,
-                                    cbfunc, cbdata);
-        pmix_output_verbose(2, pmix_client_globals.event_output,
-                            "pmix_client_notify_event source = %s:%d event_status =%d, rc=%d",
-                            (NULL == source) ? pmix_globals.myid.nspace : source->nspace,
-                            (NULL == source) ? pmix_globals.myid.rank : source->rank, status, rc);
+        return rc;
     }
+
+    /* if we aren't connected, don't attempt to send */
+    if (!pmix_globals.connected) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return PMIX_ERR_UNREACH;
+    }
+    PMIX_RELEASE_THREAD(&pmix_global_lock);
+
+    rc = notify_server_of_event(status, source, range,
+                                info, ninfo,
+                                cbfunc, cbdata);
+    pmix_output_verbose(2, pmix_client_globals.event_output,
+                        "pmix_client_notify_event source = %s:%d event_status =%d, rc=%d",
+                        (NULL == source) ? pmix_globals.myid.nspace : source->nspace,
+                        (NULL == source) ? pmix_globals.myid.rank : source->rank, status, rc);
     return rc;
 }
 
@@ -167,22 +168,17 @@ static pmix_status_t notify_server_of_event(pmix_status_t status,
     chain->status = status;
     (void)strncpy(chain->source.nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN);
     chain->source.rank = pmix_globals.myid.rank;
-    /* we always leave space for a callback object and
-     * the evhandler name. */
-    chain->ninfo = ninfo + 2;
-    PMIX_INFO_CREATE(chain->info, chain->ninfo);
+    /* we always leave space for event hdlr name and a callback object */
+    chain->nallocated = ninfo + 2;
+    PMIX_INFO_CREATE(chain->info, chain->nallocated);
 
     if (0 < ninfo) {
+        chain->ninfo = ninfo;
         /* need to copy the info */
         for (n=0; n < ninfo; n++) {
             PMIX_INFO_XFER(&chain->info[n], &info[n]);
         }
     }
-    /* add the evhandler name tag - we
-     * will fill it in as each handler is called */
-    PMIX_INFO_LOAD(&chain->info[chain->ninfo-2], PMIX_EVENT_HDLR_NAME, NULL, PMIX_STRING);
-    /* now add the callback object tag */
-    PMIX_INFO_LOAD(&chain->info[chain->ninfo-1], PMIX_EVENT_RETURN_OBJECT, NULL, PMIX_POINTER);
 
     /* we need to cache this event so we can pass it into
      * ourselves should someone later register for it */
@@ -204,6 +200,7 @@ static pmix_status_t notify_server_of_event(pmix_status_t status,
             PMIX_INFO_XFER(&cd->info[n], &chain->info[n]);
             if (0 == strncmp(cd->info[n].key, PMIX_EVENT_NON_DEFAULT, PMIX_MAX_KEYLEN)) {
                 cd->nondefault = true;
+                chain->nondefault = true;
             } else if (0 == strncmp(cd->info[n].key, PMIX_EVENT_CUSTOM_RANGE, PMIX_MAX_KEYLEN)) {
                 /* provides an array of pmix_proc_t identifying the procs
                  * that are to receive this notification, or a single pmix_proc_t  */
@@ -222,6 +219,40 @@ static pmix_status_t notify_server_of_event(pmix_status_t status,
                     PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
                     return PMIX_ERR_BAD_PARAM;
                 }
+            } else if (0 == strncmp(cd->info[n].key, PMIX_EVENT_AFFECTED_PROC, PMIX_MAX_KEYLEN)) {
+                PMIX_PROC_CREATE(cd->affected, 1);
+                if (NULL == cd->affected) {
+                    rc = PMIX_ERR_NOMEM;
+                    goto cleanup;
+                }
+                cd->naffected = 1;
+                memcpy(cd->affected, cd->info[n].value.data.proc, sizeof(pmix_proc_t));
+                /* need to do the same for chain so it can be correctly processed */
+                PMIX_PROC_CREATE(chain->affected, 1);
+                if (NULL == chain->affected) {
+                    rc = PMIX_ERR_NOMEM;
+                    goto cleanup;
+                }
+                chain->naffected = 1;
+                memcpy(chain->affected, cd->info[n].value.data.proc, sizeof(pmix_proc_t));
+            } else if (0 == strncmp(cd->info[n].key, PMIX_EVENT_AFFECTED_PROCS, PMIX_MAX_KEYLEN)) {
+                cd->naffected = cd->info[n].value.data.darray->size;
+                PMIX_PROC_CREATE(cd->affected, cd->naffected);
+                if (NULL == cd->affected) {
+                    cd->naffected = 0;
+                    rc = PMIX_ERR_NOMEM;
+                    goto cleanup;
+                }
+                memcpy(cd->affected, cd->info[n].value.data.darray->array, cd->naffected * sizeof(pmix_proc_t));
+                /* need to do the same for chain so it can be correctly processed */
+                chain->naffected = cd->info[n].value.data.darray->size;
+                PMIX_PROC_CREATE(chain->affected, chain->naffected);
+                if (NULL == chain->affected) {
+                    chain->naffected = 0;
+                    rc = PMIX_ERR_NOMEM;
+                    goto cleanup;
+                }
+                memcpy(chain->affected, cd->info[n].value.data.darray->array, chain->naffected * sizeof(pmix_proc_t));
             }
         }
     }
@@ -265,7 +296,9 @@ static pmix_status_t notify_server_of_event(pmix_status_t status,
   cleanup:
     pmix_output_verbose(2, pmix_client_globals.event_output,
                         "client: notifying server - unable to send");
-    PMIX_RELEASE(msg);
+    if (NULL != msg) {
+        PMIX_RELEASE(msg);
+    }
     /* we were unable to send anything, so we just return the error */
     return rc;
 }
@@ -332,6 +365,10 @@ static void progress_local_event_hdlr(pmix_status_t status,
     /* pass along the new ones */
     chain->results = newinfo;
     chain->nresults = cnt;
+    /* clear any loaded name and object */
+    chain->ninfo = chain->nallocated - 2;
+    PMIX_INFO_DESTRUCT(&chain->info[chain->nallocated-2]);
+    PMIX_INFO_DESTRUCT(&chain->info[chain->nallocated-1]);
 
     /* if the caller indicates that the chain is completed,
      * or we completed the "last" event */
@@ -348,28 +385,22 @@ static void progress_local_event_hdlr(pmix_status_t status,
         while (pmix_list_get_end(&pmix_globals.events.single_events) != (item = pmix_list_get_next(item))) {
             nxt = (pmix_event_hdlr_t*)item;
             if (nxt->codes[0] == chain->status &&
-                check_range(&nxt->rng, &chain->source)) {
+                pmix_notify_check_range(&nxt->rng, &chain->source) &&
+                pmix_notify_check_affected(nxt->affected, nxt->naffected,
+                                           chain->affected, chain->naffected)) {
                 chain->evhdlr = nxt;
-                /* update the handler name in case they want to reference it */
-                for (n=0; n < chain->ninfo; n++) {
-                    if (0 == strncmp(chain->info[n].key, PMIX_EVENT_HDLR_NAME, PMIX_MAX_KEYLEN)) {
-                        if (NULL != chain->info[n].value.data.string) {
-                            free(chain->info[n].value.data.string);
-                        }
-                        if (NULL != chain->evhdlr->name) {
-                            chain->info[n].value.data.string = strdup(chain->evhdlr->name);
-                        }
-                        break;
-                    }
+                /* reset our count to the info provided by the caller */
+                chain->ninfo = chain->nallocated - 2;
+                /* if the handler has a name, then provide it */
+                if (NULL != chain->evhdlr->name) {
+                    PMIX_INFO_LOAD(&chain->info[chain->ninfo], PMIX_EVENT_HDLR_NAME, chain->evhdlr->name, PMIX_STRING);
+                    chain->ninfo++;
                 }
-                /* update the evhdlr cbobject */
-                for (n=0; n < chain->ninfo; n++) {
-                    if (0 == strncmp(chain->info[n].key, PMIX_EVENT_RETURN_OBJECT, PMIX_MAX_KEYLEN)) {
-                        if (NULL != chain->evhdlr->name) {
-                            chain->info[n].value.data.ptr = chain->evhdlr->cbobject;
-                        }
-                        break;
-                    }
+
+                /* if there is an evhdlr cbobject, provide it */
+                if (NULL != chain->evhdlr->cbobject) {
+                    PMIX_INFO_LOAD(&chain->info[chain->ninfo], PMIX_EVENT_RETURN_OBJECT, chain->evhdlr->cbobject, PMIX_POINTER);
+                    chain->ninfo++;
                 }
                 nxt->evhdlr(nxt->index,
                             chain->status, &chain->source,
@@ -394,7 +425,9 @@ static void progress_local_event_hdlr(pmix_status_t status,
         }
         while (pmix_list_get_end(&pmix_globals.events.multi_events) != (item = pmix_list_get_next(item))) {
             nxt = (pmix_event_hdlr_t*)item;
-            if (!check_range(&nxt->rng, &chain->source)) {
+            if (!pmix_notify_check_range(&nxt->rng, &chain->source) &&
+                !pmix_notify_check_affected(nxt->affected, nxt->naffected,
+                                            chain->affected, chain->naffected)) {
                 continue;
             }
             for (n=0; n < nxt->ncodes; n++) {
@@ -402,26 +435,18 @@ static void progress_local_event_hdlr(pmix_status_t status,
                  * the source fits within it */
                 if (nxt->codes[n] == chain->status) {
                     chain->evhdlr = nxt;
-                    /* update the handler name in case they want to reference it */
-                    for (n=0; n < chain->ninfo; n++) {
-                        if (0 == strncmp(chain->info[n].key, PMIX_EVENT_HDLR_NAME, PMIX_MAX_KEYLEN)) {
-                            if (NULL != chain->info[n].value.data.string) {
-                                free(chain->info[n].value.data.string);
-                            }
-                            if (NULL != chain->evhdlr->name) {
-                                chain->info[n].value.data.string = strdup(chain->evhdlr->name);
-                            }
-                            break;
-                        }
+                    /* reset our count to the info provided by the caller */
+                    chain->ninfo = chain->nallocated - 2;
+                    /* if the handler has a name, then provide it */
+                    if (NULL != chain->evhdlr->name) {
+                        PMIX_INFO_LOAD(&chain->info[chain->ninfo], PMIX_EVENT_HDLR_NAME, chain->evhdlr->name, PMIX_STRING);
+                        chain->ninfo++;
                     }
-                    /* update the evhdlr cbobject */
-                    for (n=0; n < chain->ninfo; n++) {
-                        if (0 == strncmp(chain->info[n].key, PMIX_EVENT_RETURN_OBJECT, PMIX_MAX_KEYLEN)) {
-                            if (NULL != chain->evhdlr->name) {
-                                chain->info[n].value.data.ptr = chain->evhdlr->cbobject;
-                            }
-                            break;
-                        }
+
+                    /* if there is an evhdlr cbobject, provide it */
+                    if (NULL != chain->evhdlr->cbobject) {
+                        PMIX_INFO_LOAD(&chain->info[chain->ninfo], PMIX_EVENT_RETURN_OBJECT, chain->evhdlr->cbobject, PMIX_POINTER);
+                        chain->ninfo++;
                     }
                     nxt->evhdlr(nxt->index,
                                 chain->status, &chain->source,
@@ -446,28 +471,22 @@ static void progress_local_event_hdlr(pmix_status_t status,
             nxt = (pmix_event_hdlr_t*)item;
             /* if this event handler provided a range, check to see if
              * the source fits within it */
-            if (check_range(&nxt->rng, &chain->source)) {
+            if (pmix_notify_check_range(&nxt->rng, &chain->source) &&
+                pmix_notify_check_affected(nxt->affected, nxt->naffected,
+                                           chain->affected, chain->naffected)) {
                 chain->evhdlr = nxt;
-                /* update the handler name in case they want to reference it */
-                for (n=0; n < chain->ninfo; n++) {
-                    if (0 == strncmp(chain->info[n].key, PMIX_EVENT_HDLR_NAME, PMIX_MAX_KEYLEN)) {
-                        if (NULL != chain->info[n].value.data.string) {
-                            free(chain->info[n].value.data.string);
-                        }
-                        if (NULL != chain->evhdlr->name) {
-                            chain->info[n].value.data.string = strdup(chain->evhdlr->name);
-                        }
-                        break;
-                    }
+                /* reset our count to the info provided by the caller */
+                chain->ninfo = chain->nallocated - 2;
+                /* if the handler has a name, then provide it */
+                if (NULL != chain->evhdlr->name) {
+                    PMIX_INFO_LOAD(&chain->info[chain->ninfo], PMIX_EVENT_HDLR_NAME, chain->evhdlr->name, PMIX_STRING);
+                    chain->ninfo++;
                 }
-                /* update the evhdlr cbobject */
-                for (n=0; n < chain->ninfo; n++) {
-                    if (0 == strncmp(chain->info[n].key, PMIX_EVENT_RETURN_OBJECT, PMIX_MAX_KEYLEN)) {
-                        if (NULL != chain->evhdlr->name) {
-                            chain->info[n].value.data.ptr = chain->evhdlr->cbobject;
-                        }
-                        break;
-                    }
+
+                /* if there is an evhdlr cbobject, provide it */
+                if (NULL != chain->evhdlr->cbobject) {
+                    PMIX_INFO_LOAD(&chain->info[chain->ninfo], PMIX_EVENT_RETURN_OBJECT, chain->evhdlr->cbobject, PMIX_POINTER);
+                    chain->ninfo++;
                 }
                 nxt->evhdlr(nxt->index,
                             chain->status, &chain->source,
@@ -482,31 +501,25 @@ static void progress_local_event_hdlr(pmix_status_t status,
     /* if we registered a "last" handler, and it fits the given range
      * and code, then invoke it now */
     if (NULL != pmix_globals.events.last &&
-        check_range(&pmix_globals.events.last->rng, &chain->source)) {
+        pmix_notify_check_range(&pmix_globals.events.last->rng, &chain->source) &&
+        pmix_notify_check_affected(nxt->affected, nxt->naffected,
+                                   chain->affected, chain->naffected)) {
         chain->endchain = true;  // ensure we don't do this again
         if (1 == pmix_globals.events.last->ncodes &&
             pmix_globals.events.last->codes[0] == chain->status) {
             chain->evhdlr = pmix_globals.events.last;
-            /* update the handler name in case they want to reference it */
-            for (n=0; n < chain->ninfo; n++) {
-                if (0 == strncmp(chain->info[n].key, PMIX_EVENT_HDLR_NAME, PMIX_MAX_KEYLEN)) {
-                    if (NULL != chain->info[n].value.data.string) {
-                        free(chain->info[n].value.data.string);
-                    }
-                    if (NULL != chain->evhdlr->name) {
-                        chain->info[n].value.data.string = strdup(chain->evhdlr->name);
-                    }
-                    break;
-                }
+            /* reset our count to the info provided by the caller */
+            chain->ninfo = chain->nallocated - 2;
+            /* if the handler has a name, then provide it */
+            if (NULL != chain->evhdlr->name) {
+                PMIX_INFO_LOAD(&chain->info[chain->ninfo], PMIX_EVENT_HDLR_NAME, chain->evhdlr->name, PMIX_STRING);
+                chain->ninfo++;
             }
-            /* update the evhdlr cbobject */
-            for (n=0; n < chain->ninfo; n++) {
-                if (0 == strncmp(chain->info[n].key, PMIX_EVENT_RETURN_OBJECT, PMIX_MAX_KEYLEN)) {
-                    if (NULL != chain->evhdlr->name) {
-                        chain->info[n].value.data.ptr = chain->evhdlr->cbobject;
-                    }
-                    break;
-                }
+
+            /* if there is an evhdlr cbobject, provide it */
+            if (NULL != chain->evhdlr->cbobject) {
+                PMIX_INFO_LOAD(&chain->info[chain->ninfo], PMIX_EVENT_RETURN_OBJECT, chain->evhdlr->cbobject, PMIX_POINTER);
+                chain->ninfo++;
             }
             chain->evhdlr->evhdlr(chain->evhdlr->index,
                                   chain->status, &chain->source,
@@ -519,26 +532,18 @@ static void progress_local_event_hdlr(pmix_status_t status,
             for (n=0; n < pmix_globals.events.last->ncodes; n++) {
                 if (pmix_globals.events.last->codes[n] == chain->status) {
                     chain->evhdlr = pmix_globals.events.last;
-                    /* update the handler name in case they want to reference it */
-                    for (n=0; n < chain->ninfo; n++) {
-                        if (0 == strncmp(chain->info[n].key, PMIX_EVENT_HDLR_NAME, PMIX_MAX_KEYLEN)) {
-                            if (NULL != chain->info[n].value.data.string) {
-                                free(chain->info[n].value.data.string);
-                            }
-                            if (NULL != chain->evhdlr->name) {
-                                chain->info[n].value.data.string = strdup(chain->evhdlr->name);
-                            }
-                            break;
-                        }
+                    /* reset our count to the info provided by the caller */
+                    chain->ninfo = chain->nallocated - 2;
+                    /* if the handler has a name, then provide it */
+                    if (NULL != chain->evhdlr->name) {
+                        PMIX_INFO_LOAD(&chain->info[chain->ninfo], PMIX_EVENT_HDLR_NAME, chain->evhdlr->name, PMIX_STRING);
+                        chain->ninfo++;
                     }
-                    /* update the evhdlr cbobject */
-                    for (n=0; n < chain->ninfo; n++) {
-                        if (0 == strncmp(chain->info[n].key, PMIX_EVENT_RETURN_OBJECT, PMIX_MAX_KEYLEN)) {
-                            if (NULL != chain->evhdlr->name) {
-                                chain->info[n].value.data.ptr = chain->evhdlr->cbobject;
-                            }
-                            break;
-                        }
+
+                    /* if there is an evhdlr cbobject, provide it */
+                    if (NULL != chain->evhdlr->cbobject) {
+                        PMIX_INFO_LOAD(&chain->info[chain->ninfo], PMIX_EVENT_RETURN_OBJECT, chain->evhdlr->cbobject, PMIX_POINTER);
+                        chain->ninfo++;
                     }
                     chain->evhdlr->evhdlr(chain->evhdlr->index,
                                           chain->status, &chain->source,
@@ -551,26 +556,18 @@ static void progress_local_event_hdlr(pmix_status_t status,
         } else {
             /* gets run for all codes */
             chain->evhdlr = pmix_globals.events.last;
-            /* update the handler name in case they want to reference it */
-            for (n=0; n < chain->ninfo; n++) {
-                if (0 == strncmp(chain->info[n].key, PMIX_EVENT_HDLR_NAME, PMIX_MAX_KEYLEN)) {
-                    if (NULL != chain->info[n].value.data.string) {
-                        free(chain->info[n].value.data.string);
-                    }
-                    if (NULL != chain->evhdlr->name) {
-                        chain->info[n].value.data.string = strdup(chain->evhdlr->name);
-                    }
-                    break;
-                }
+            /* reset our count to the info provided by the caller */
+            chain->ninfo = chain->nallocated - 2;
+            /* if the handler has a name, then provide it */
+            if (NULL != chain->evhdlr->name) {
+                PMIX_INFO_LOAD(&chain->info[chain->ninfo], PMIX_EVENT_HDLR_NAME, chain->evhdlr->name, PMIX_STRING);
+                chain->ninfo++;
             }
-            /* update the evhdlr cbobject */
-            for (n=0; n < chain->ninfo; n++) {
-                if (0 == strncmp(chain->info[n].key, PMIX_EVENT_RETURN_OBJECT, PMIX_MAX_KEYLEN)) {
-                    if (NULL != chain->evhdlr->name) {
-                        chain->info[n].value.data.ptr = chain->evhdlr->cbobject;
-                    }
-                    break;
-                }
+
+            /* if there is an evhdlr cbobject, provide it */
+            if (NULL != chain->evhdlr->cbobject) {
+                PMIX_INFO_LOAD(&chain->info[chain->ninfo], PMIX_EVENT_RETURN_OBJECT, chain->evhdlr->cbobject, PMIX_POINTER);
+                chain->ninfo++;
             }
             chain->evhdlr->evhdlr(chain->evhdlr->index,
                                   chain->status, &chain->source,
@@ -613,15 +610,15 @@ void pmix_invoke_local_event_hdlr(pmix_event_chain_t *chain)
     pmix_status_t rc = PMIX_SUCCESS;
     bool found;
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_client_globals.event_output,
                         "%s:%d invoke_local_event_hdlr for status %s",
                         pmix_globals.myid.nspace, pmix_globals.myid.rank,
                         PMIx_Error_string(chain->status));
 
     /* sanity check */
     if (NULL == chain->info) {
-        /* should never happen as the return object must
-         * at least be there, even if it is NULL */
+        /* should never happen as space must always be
+         * reserved for handler name and callback object*/
         rc = PMIX_ERR_BAD_PARAM;
         goto complete;
     }
@@ -638,7 +635,9 @@ void pmix_invoke_local_event_hdlr(pmix_event_chain_t *chain)
     if (NULL != pmix_globals.events.first) {
         if (1 == pmix_globals.events.first->ncodes &&
             pmix_globals.events.first->codes[0] == chain->status &&
-            check_range(&pmix_globals.events.first->rng, &chain->source)) {
+            pmix_notify_check_range(&pmix_globals.events.first->rng, &chain->source) &&
+            pmix_notify_check_affected(pmix_globals.events.first->affected, pmix_globals.events.first->naffected,
+                                       chain->affected, chain->naffected)) {
             /* invoke the handler */
             chain->evhdlr = pmix_globals.events.first;
             goto invk;
@@ -653,14 +652,14 @@ void pmix_invoke_local_event_hdlr(pmix_event_chain_t *chain)
             }
             /* if this event handler provided a range, check to see if
              * the source fits within it */
-            if (found && check_range(&pmix_globals.events.first->rng, &chain->source)) {
+            if (found && pmix_notify_check_range(&pmix_globals.events.first->rng, &chain->source)) {
                 /* invoke the handler */
                 chain->evhdlr = pmix_globals.events.first;
                 goto invk;
             }
         } else {
             /* take all codes for a default handler */
-            if (check_range(&pmix_globals.events.first->rng, &chain->source)) {
+            if (pmix_notify_check_range(&pmix_globals.events.first->rng, &chain->source)) {
                 /* invoke the handler */
                 chain->evhdlr = pmix_globals.events.first;
                 goto invk;
@@ -672,7 +671,9 @@ void pmix_invoke_local_event_hdlr(pmix_event_chain_t *chain)
     /* cycle thru the single-event registrations first */
     PMIX_LIST_FOREACH(evhdlr, &pmix_globals.events.single_events, pmix_event_hdlr_t) {
         if (evhdlr->codes[0] == chain->status) {
-            if (check_range(&evhdlr->rng, &chain->source)) {
+            if (pmix_notify_check_range(&evhdlr->rng, &chain->source) &&
+                pmix_notify_check_affected(evhdlr->affected, evhdlr->naffected,
+                                           chain->affected, chain->naffected)) {
                 /* invoke the handler */
                 chain->evhdlr = evhdlr;
                 goto invk;
@@ -685,7 +686,9 @@ void pmix_invoke_local_event_hdlr(pmix_event_chain_t *chain)
     PMIX_LIST_FOREACH(evhdlr, &pmix_globals.events.multi_events, pmix_event_hdlr_t) {
         for (i=0; i < evhdlr->ncodes; i++) {
             if (evhdlr->codes[i] == chain->status) {
-                if (check_range(&evhdlr->rng, &chain->source)) {
+                if (pmix_notify_check_range(&evhdlr->rng, &chain->source) &&
+                    pmix_notify_check_affected(evhdlr->affected, evhdlr->naffected,
+                                               chain->affected, chain->naffected)) {
                     /* invoke the handler */
                     chain->evhdlr = evhdlr;
                     goto invk;
@@ -698,7 +701,9 @@ void pmix_invoke_local_event_hdlr(pmix_event_chain_t *chain)
     if (!chain->nondefault) {
         /* pass it to any default handlers */
         PMIX_LIST_FOREACH(evhdlr, &pmix_globals.events.default_events, pmix_event_hdlr_t) {
-            if (check_range(&evhdlr->rng, &chain->source)) {
+            if (pmix_notify_check_range(&evhdlr->rng, &chain->source) &&
+                pmix_notify_check_affected(evhdlr->affected, evhdlr->naffected,
+                                           chain->affected, chain->naffected)) {
                 /* invoke the handler */
                 chain->evhdlr = evhdlr;
                 goto invk;
@@ -709,7 +714,9 @@ void pmix_invoke_local_event_hdlr(pmix_event_chain_t *chain)
     /* if we registered a "last" handler, and it fits the given range
      * and code, then invoke it now */
     if (NULL != pmix_globals.events.last &&
-        check_range(&pmix_globals.events.last->rng, &chain->source)) {
+        pmix_notify_check_range(&pmix_globals.events.last->rng, &chain->source) &&
+        pmix_notify_check_affected(pmix_globals.events.last->affected, pmix_globals.events.last->naffected,
+                                   chain->affected, chain->naffected)) {
         chain->endchain = true;  // ensure we don't do this again
         if (1 == pmix_globals.events.last->ncodes &&
             pmix_globals.events.last->codes[0] == chain->status) {
@@ -741,29 +748,23 @@ void pmix_invoke_local_event_hdlr(pmix_event_chain_t *chain)
 
 
   invk:
-    /* update the handler name in case they want to reference it */
-    for (i=0; i < chain->ninfo; i++) {
-        if (0 == strncmp(chain->info[i].key, PMIX_EVENT_HDLR_NAME, PMIX_MAX_KEYLEN)) {
-            if (NULL != chain->info[i].value.data.string) {
-                free(chain->info[i].value.data.string);
-            }
-            if (NULL != chain->evhdlr->name) {
-                chain->info[i].value.data.string = strdup(chain->evhdlr->name);
-            }
-            break;
-        }
+    /* start with the chain holding only the given info */
+    chain->ninfo = chain->nallocated - 2;
+
+    /* if the handler has a name, then provide it */
+    if (NULL != chain->evhdlr->name) {
+        PMIX_INFO_LOAD(&chain->info[chain->ninfo], PMIX_EVENT_HDLR_NAME, chain->evhdlr->name, PMIX_STRING);
+        chain->ninfo++;
     }
-    /* update the evhdlr cbobject */
-    for (i=0; i < chain->ninfo; i++) {
-        if (0 == strncmp(chain->info[i].key, PMIX_EVENT_RETURN_OBJECT, PMIX_MAX_KEYLEN)) {
-            if (NULL != chain->evhdlr->name) {
-                chain->info[i].value.data.ptr = chain->evhdlr->cbobject;
-            }
-            break;
-        }
+
+    /* if there is an evhdlr cbobject, provide it */
+    if (NULL != chain->evhdlr->cbobject) {
+        PMIX_INFO_LOAD(&chain->info[chain->ninfo], PMIX_EVENT_RETURN_OBJECT, chain->evhdlr->cbobject, PMIX_POINTER);
+        chain->ninfo++;
     }
+
     /* invoke the handler */
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_client_globals.event_output,
                         "[%s:%d] INVOKING EVHDLR %s", __FILE__, __LINE__,
                         (NULL == chain->evhdlr->name) ?
                         "NULL" : chain->evhdlr->name);
@@ -822,7 +823,6 @@ static void _notify_client_event(int sd, short args, void *cbdata)
             }
         }
     }
-
     if (holdcd) {
         /* we cannot know if everyone who wants this notice has had a chance
          * to register for it - the notice may be coming too early. So cache
@@ -965,19 +965,72 @@ static void _notify_client_event(int sd, short args, void *cbdata)
     chain->source.rank = cd->source.rank;
     /* we always leave space for a callback object and
      * the evhandler name. */
-    chain->ninfo = cd->ninfo + 2;
-    PMIX_INFO_CREATE(chain->info, chain->ninfo);
+    chain->nallocated = cd->ninfo + 2;
+    PMIX_INFO_CREATE(chain->info, chain->nallocated);
     if (0 < cd->ninfo) {
+        chain->ninfo = cd->ninfo;
         /* need to copy the info */
         for (n=0; n < cd->ninfo; n++) {
             PMIX_INFO_XFER(&chain->info[n], &cd->info[n]);
+            if (0 == strncmp(cd->info[n].key, PMIX_EVENT_NON_DEFAULT, PMIX_MAX_KEYLEN)) {
+                cd->nondefault = true;
+                chain->nondefault = true;
+            } else if (0 == strncmp(cd->info[n].key, PMIX_EVENT_CUSTOM_RANGE, PMIX_MAX_KEYLEN)) {
+                /* provides an array of pmix_proc_t identifying the procs
+                 * that are to receive this notification, or a single pmix_proc_t  */
+                if (PMIX_DATA_ARRAY == cd->info[n].value.type &&
+                    NULL != cd->info[n].value.data.darray &&
+                    NULL != cd->info[n].value.data.darray->array) {
+                    cd->ntargets = cd->info[n].value.data.darray->size;
+                    PMIX_PROC_CREATE(cd->targets, cd->ntargets);
+                    memcpy(cd->targets, cd->info[n].value.data.darray->array, cd->ntargets * sizeof(pmix_proc_t));
+                } else if (PMIX_PROC == cd->info[n].value.type) {
+                    cd->ntargets = 1;
+                    PMIX_PROC_CREATE(cd->targets, cd->ntargets);
+                    memcpy(cd->targets, cd->info[n].value.data.proc, sizeof(pmix_proc_t));
+                } else {
+                    /* this is an error */
+                    PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
+                    PMIX_RELEASE(chain);
+                    return;
+                }
+            } else if (0 == strncmp(cd->info[n].key, PMIX_EVENT_AFFECTED_PROC, PMIX_MAX_KEYLEN)) {
+                PMIX_PROC_CREATE(cd->affected, 1);
+                if (NULL == cd->affected) {
+                    PMIX_RELEASE(chain);
+                    return;
+                }
+                cd->naffected = 1;
+                memcpy(cd->affected, cd->info[n].value.data.proc, sizeof(pmix_proc_t));
+                /* need to do the same for chain so it can be correctly processed */
+                PMIX_PROC_CREATE(chain->affected, 1);
+                if (NULL == chain->affected) {
+                    PMIX_RELEASE(chain);
+                    return;
+                }
+                chain->naffected = 1;
+                memcpy(chain->affected, cd->info[n].value.data.proc, sizeof(pmix_proc_t));
+            } else if (0 == strncmp(cd->info[n].key, PMIX_EVENT_AFFECTED_PROCS, PMIX_MAX_KEYLEN)) {
+                cd->naffected = cd->info[n].value.data.darray->size;
+                PMIX_PROC_CREATE(cd->affected, cd->naffected);
+                if (NULL == cd->affected) {
+                    cd->naffected = 0;
+                    PMIX_RELEASE(chain);
+                    return;
+                }
+                memcpy(cd->affected, cd->info[n].value.data.darray->array, cd->naffected * sizeof(pmix_proc_t));
+                /* need to do the same for chain so it can be correctly processed */
+                chain->naffected = cd->info[n].value.data.darray->size;
+                PMIX_PROC_CREATE(chain->affected, chain->naffected);
+                if (NULL == chain->affected) {
+                    chain->naffected = 0;
+                    PMIX_RELEASE(chain);
+                    return;
+                }
+                memcpy(chain->affected, cd->info[n].value.data.darray->array, chain->naffected * sizeof(pmix_proc_t));
+            }
         }
     }
-    /* put the evhandler name tag in the next-to-last element - we
-     * will fill it in as each handler is called */
-    PMIX_INFO_LOAD(&chain->info[chain->ninfo-2], PMIX_EVENT_HDLR_NAME, NULL, PMIX_STRING);
-    /* now put the callback object tag in the last element */
-    PMIX_INFO_LOAD(&chain->info[chain->ninfo-1], PMIX_EVENT_RETURN_OBJECT, NULL, PMIX_POINTER);
     /* process it */
     pmix_invoke_local_event_hdlr(chain);
 
@@ -1057,6 +1110,7 @@ pmix_status_t pmix_server_notify_client_of_event(pmix_status_t status,
             }
         }
     }
+
     /*
      * If the range is PMIX_RANGE_NAMESPACE, then they should not have set a
      * PMIX_EVENT_CUSTOM_RANGE info object or at least we should ignore it
@@ -1089,8 +1143,8 @@ pmix_status_t pmix_server_notify_client_of_event(pmix_status_t status,
     return PMIX_SUCCESS;
 }
 
-static bool check_range(pmix_range_trkr_t *rng,
-                        const pmix_proc_t *proc)
+bool pmix_notify_check_range(pmix_range_trkr_t *rng,
+                             const pmix_proc_t *proc)
 {
     size_t n;
 
@@ -1138,6 +1192,37 @@ static bool check_range(pmix_range_trkr_t *rng,
     return false;
 }
 
+bool pmix_notify_check_affected(pmix_proc_t *interested, size_t ninterested,
+                                pmix_proc_t *affected, size_t naffected)
+{
+    size_t m, n;
+
+    /* if they didn't restrict their interests, then accept it */
+    if (NULL == interested) {
+        return true;
+    }
+    /* if we weren't given the affected procs, then accept it */
+    if (NULL == affected) {
+        return true;
+    }
+    /* check if the two overlap */
+    for (n=0; n < naffected; n++) {
+        for (m=0; m < ninterested; m++) {
+            if (0 != strncmp(affected[n].nspace, interested[m].nspace, PMIX_MAX_NSLEN)) {
+                continue;
+            }
+            if (PMIX_RANK_WILDCARD == interested[m].rank ||
+                PMIX_RANK_WILDCARD == affected[n].rank ||
+                affected[n].rank == interested[m].rank) {
+                return true;
+            }
+        }
+    }
+    /* if we get here, then this proc isn't in range */
+    return false;
+
+}
+
 void pmix_event_timeout_cb(int fd, short flags, void *arg)
 {
     pmix_event_chain_t *ch = (pmix_event_chain_t*)arg;
@@ -1151,7 +1236,8 @@ void pmix_event_timeout_cb(int fd, short flags, void *arg)
     pmix_list_remove_item(&pmix_globals.cached_events, &ch->super);
 
     /* process this event thru the regular channels */
-    if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) {
+    if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer) &&
+        !PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) {
         pmix_server_notify_client_of_event(ch->status, &ch->source,
                                            ch->range, ch->info, ch->ninfo,
                                            ch->final_cbfunc, ch->final_cbdata);
@@ -1171,6 +1257,8 @@ static void sevcon(pmix_event_hdlr_t *p)
     p->rng.range = PMIX_RANGE_UNDEF;
     p->rng.procs = NULL;
     p->rng.nprocs = 0;
+    p->affected = NULL;
+    p->naffected = 0;
     p->evhdlr = NULL;
     p->cbobject = NULL;
     p->codes = NULL;
@@ -1187,6 +1275,9 @@ static void sevdes(pmix_event_hdlr_t *p)
     if (NULL != p->rng.procs) {
         free(p->rng.procs);
     }
+    if (NULL != p->affected) {
+        PMIX_PROC_FREE(p->affected, p->naffected);
+    }
     if (NULL != p->codes) {
         free(p->codes);
     }
@@ -1238,8 +1329,11 @@ static void chcon(pmix_event_chain_t *p)
     p->nondefault = false;
     p->endchain = false;
     p->range = PMIX_RANGE_UNDEF;
+    p->affected = NULL;
+    p->naffected = 0;
     p->info = NULL;
     p->ninfo = 0;
+    p->nallocated = 0;
     p->results = NULL;
     p->nresults = 0;
     p->evhdlr = NULL;
@@ -1251,8 +1345,11 @@ static void chdes(pmix_event_chain_t *p)
     if (p->timer_active) {
         pmix_event_del(&p->ev);
     }
+    if (NULL != p->affected) {
+        PMIX_PROC_FREE(p->affected, p->naffected);
+    }
     if (NULL != p->info) {
-        PMIX_INFO_FREE(p->info, p->ninfo);
+        PMIX_INFO_FREE(p->info, p->nallocated);
     }
     if (NULL != p->results) {
         PMIX_INFO_FREE(p->results, p->nresults);

opal/mca/pmix/pmix3x/pmix/src/event/pmix_event_registration.c

@@ -1,7 +1,7 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2014-2017 Intel, Inc.  All rights reserved.
- * Copyright (c) 2017      Research Organization for Information Science
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
+ * Copyright (c) 2017-2018 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
  *
@@ -40,6 +40,8 @@
     size_t ncodes;
     pmix_info_t *info;
     size_t ninfo;
+    pmix_proc_t *affected;
+    size_t naffected;
     pmix_notification_fn_t evhdlr;
     pmix_hdlr_reg_cbfunc_t evregcbfn;
     void *cbdata;
@@ -55,12 +57,17 @@ static void rscon(pmix_rshift_caddy_t *p)
     p->ncodes = 0;
     p->info = NULL;
     p->ninfo = 0;
+    p->affected = NULL;
+    p->naffected = 0;
     p->evhdlr = NULL;
     p->evregcbfn = NULL;
     p->cbdata = NULL;
 }
 static void rsdes(pmix_rshift_caddy_t *p)
 {
+    if (0 < p->ncodes) {
+        free(p->codes);
+    }
     if (NULL != p->cd) {
         PMIX_RELEASE(p->cd);
     }
@@ -80,7 +87,7 @@ static void regevents_cbfunc(struct pmix_peer_t *peer, pmix_ptl_hdr_t *hdr,
     int cnt;
     size_t index = rb->index;
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_client_globals.event_output,
                         "pmix: regevents callback recvd");
 
     /* unpack the status code */
@@ -230,7 +237,7 @@ static pmix_status_t _add_hdlr(pmix_rshift_caddy_t *cd, pmix_list_t *xfer)
     pmix_active_code_t *active;
     pmix_status_t rc;
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_client_globals.event_output,
                         "pmix: _add_hdlr");
 
     /* check to see if we have an active registration on these codes */
@@ -299,15 +306,16 @@ static pmix_status_t _add_hdlr(pmix_rshift_caddy_t *cd, pmix_list_t *xfer)
      * type with our server, or if we have directives, then we need to notify
      * the server - however, don't do this for a v1 server as the event
      * notification system there doesn't work */
-    if (!PMIX_PROC_IS_SERVER(pmix_globals.mypeer) && pmix_globals.connected &&
+    if ((!PMIX_PROC_IS_SERVER(pmix_globals.mypeer) || PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) &&
+        pmix_globals.connected &&
         !PMIX_PROC_IS_V1(pmix_client_globals.myserver) &&
        (need_register || 0 < pmix_list_get_size(xfer))) {
-        pmix_output_verbose(2, pmix_globals.debug_output,
+        pmix_output_verbose(2, pmix_client_globals.event_output,
                             "pmix: _add_hdlr sending to server");
         /* send the directives to the server - we will ack this
          * registration upon return from there */
         if (PMIX_SUCCESS != (rc = _send_to_server(cd2))) {
-            pmix_output_verbose(2, pmix_globals.debug_output,
+            pmix_output_verbose(2, pmix_client_globals.event_output,
                                 "pmix: add_hdlr - pack send_to_server failed status=%d", rc);
             if (NULL != cd2->info) {
                 PMIX_INFO_FREE(cd2->info, cd2->ninfo);
@@ -320,9 +328,10 @@ static pmix_status_t _add_hdlr(pmix_rshift_caddy_t *cd, pmix_list_t *xfer)
 
     /* if we are a server and are registering for events, then we only contact
      * our host if we want environmental events */
-    if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer) && cd->enviro &&
+    if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer) &&
+        !PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer) && cd->enviro &&
         NULL != pmix_host_server.register_events) {
-        pmix_output_verbose(2, pmix_globals.debug_output,
+        pmix_output_verbose(2, pmix_client_globals.event_output,
                             "pmix: _add_hdlr registering with server");
         if (PMIX_SUCCESS != (rc = pmix_host_server.register_events(cd->codes, cd->ncodes,
                                                                    cd2->info, cd2->ninfo,
@@ -357,8 +366,10 @@ static void check_cached_events(pmix_rshift_caddy_t *cd)
         }
         found = false;
         if (NULL == cd->codes) {
-            /* they registered a default event handler - always matches */
-            found = true;
+            if (!ncd->nondefault) {
+                /* they registered a default event handler - always matches */
+                found = true;
+            }
         } else {
             for (n=0; n < cd->ncodes; n++) {
                 if (cd->codes[n] == ncd->status) {
@@ -367,49 +378,72 @@ static void check_cached_events(pmix_rshift_caddy_t *cd)
                 }
             }
         }
-        if (found) {
-           /* if we were given specific targets, check if we are one */
-            if (NULL != ncd->targets) {
-                matched = false;
-                for (n=0; n < ncd->ntargets; n++) {
-                    if (0 != strncmp(pmix_globals.myid.nspace, ncd->targets[n].nspace, PMIX_MAX_NSLEN)) {
-                        continue;
-                    }
-                    if (PMIX_RANK_WILDCARD == ncd->targets[n].rank ||
-                        pmix_globals.myid.rank == ncd->targets[n].rank) {
-                        matched = true;
-                        break;
-                    }
-                }
-                if (!matched) {
-                    /* do not notify this one */
+        if (!found) {
+            continue;
+        }
+       /* if we were given specific targets, check if we are one */
+        if (NULL != ncd->targets) {
+            matched = false;
+            for (n=0; n < ncd->ntargets; n++) {
+                if (0 != strncmp(pmix_globals.myid.nspace, ncd->targets[n].nspace, PMIX_MAX_NSLEN)) {
                     continue;
                 }
-            }
-           /* all matches - notify */
-            chain = PMIX_NEW(pmix_event_chain_t);
-            chain->status = ncd->status;
-            (void)strncpy(chain->source.nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN);
-            chain->source.rank = pmix_globals.myid.rank;
-            /* we already left space for evhandler name plus
-             * a callback object when we cached the notification */
-            chain->ninfo = ncd->ninfo;
-            PMIX_INFO_CREATE(chain->info, chain->ninfo);
-            if (0 < cd->ninfo) {
-                /* need to copy the info */
-                for (n=0; n < ncd->ninfo; n++) {
-                    PMIX_INFO_XFER(&chain->info[n], &ncd->info[n]);
-                    if (0 == strncmp(chain->info[n].key, PMIX_EVENT_NON_DEFAULT, PMIX_MAX_KEYLEN)) {
-                        chain->nondefault = true;
-                    }
+                if (PMIX_RANK_WILDCARD == ncd->targets[n].rank ||
+                    pmix_globals.myid.rank == ncd->targets[n].rank) {
+                    matched = true;
+                    break;
                 }
             }
-            /* we don't want this chain to propagate, so indicate it
-             * should only be run as a single-shot */
-            chain->endchain = true;
-            /* now notify any matching registered callbacks we have */
-            pmix_invoke_local_event_hdlr(chain);
+            if (!matched) {
+                /* do not notify this one */
+                continue;
+            }
         }
+       /* if they specified affected proc(s) they wanted to know about, check */
+       if (!pmix_notify_check_affected(cd->affected, cd->naffected,
+                                       ncd->affected, ncd->naffected)) {
+           continue;
+       }
+       /* create the chain */
+        chain = PMIX_NEW(pmix_event_chain_t);
+        chain->status = ncd->status;
+        (void)strncpy(chain->source.nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN);
+        chain->source.rank = pmix_globals.myid.rank;
+        /* we always leave space for event hdlr name and a callback object */
+        chain->nallocated = ncd->ninfo + 2;
+        PMIX_INFO_CREATE(chain->info, chain->nallocated);
+        if (0 < cd->ninfo) {
+            chain->ninfo = ncd->ninfo;
+            /* need to copy the info */
+            for (n=0; n < ncd->ninfo; n++) {
+                PMIX_INFO_XFER(&chain->info[n], &ncd->info[n]);
+                if (0 == strncmp(ncd->info[n].key, PMIX_EVENT_NON_DEFAULT, PMIX_MAX_KEYLEN)) {
+                    chain->nondefault = true;
+                } else if (0 == strncmp(ncd->info[n].key, PMIX_EVENT_AFFECTED_PROC, PMIX_MAX_KEYLEN)) {
+                    PMIX_PROC_CREATE(chain->affected, 1);
+                    if (NULL == chain->affected) {
+                        PMIX_RELEASE(chain);
+                        return;
+                    }
+                    chain->naffected = 1;
+                    memcpy(chain->affected, ncd->info[n].value.data.proc, sizeof(pmix_proc_t));
+                } else if (0 == strncmp(ncd->info[n].key, PMIX_EVENT_AFFECTED_PROCS, PMIX_MAX_KEYLEN)) {
+                    chain->naffected = ncd->info[n].value.data.darray->size;
+                    PMIX_PROC_CREATE(chain->affected, chain->naffected);
+                    if (NULL == chain->affected) {
+                        chain->naffected = 0;
+                        PMIX_RELEASE(chain);
+                        return;
+                    }
+                    memcpy(chain->affected, ncd->info[n].value.data.darray->array, chain->naffected * sizeof(pmix_proc_t));
+                }
+            }
+        }
+        /* we don't want this chain to propagate, so indicate it
+         * should only be run as a single-shot */
+        chain->endchain = true;
+        /* now notify any matching registered callbacks we have */
+        pmix_invoke_local_event_hdlr(chain);
     }
 }
 
@@ -433,7 +467,7 @@ static void reg_event_hdlr(int sd, short args, void *cbdata)
     /* need to acquire the object from its originating thread */
     PMIX_ACQUIRE_OBJECT(cd);
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_client_globals.event_output,
                         "pmix: register event_hdlr with %d infos", (int)cd->ninfo);
 
     PMIX_CONSTRUCT(&xfer, pmix_list_t);
@@ -482,6 +516,12 @@ static void reg_event_hdlr(int sd, short args, void *cbdata)
             } else if (0 == strncmp(cd->info[n].key, PMIX_EVENT_CUSTOM_RANGE, PMIX_MAX_KEYLEN)) {
                 parray = (pmix_proc_t*)cd->info[n].value.data.darray->array;
                 nprocs = cd->info[n].value.data.darray->size;
+            } else if (0 == strncmp(cd->info[n].key, PMIX_EVENT_AFFECTED_PROC, PMIX_MAX_KEYLEN)) {
+                cd->affected = cd->info[n].value.data.proc;
+                cd->naffected = 1;
+            } else if (0 == strncmp(cd->info[n].key, PMIX_EVENT_AFFECTED_PROCS, PMIX_MAX_KEYLEN)) {
+                cd->affected = (pmix_proc_t*)cd->info[n].value.data.darray->array;
+                cd->naffected = cd->info[n].value.data.darray->size;
             } else {
                 ixfer = PMIX_NEW(pmix_info_caddy_t);
                 ixfer->info = &cd->info[n];
@@ -525,6 +565,17 @@ static void reg_event_hdlr(int sd, short args, void *cbdata)
             }
             memcpy(evhdlr->rng.procs, parray, nprocs * sizeof(pmix_proc_t));
         }
+        if (NULL != cd->affected && 0 < cd->naffected) {
+            evhdlr->naffected = cd->naffected;
+            PMIX_PROC_CREATE(evhdlr->affected, cd->naffected);
+            if (NULL == evhdlr->affected) {
+                index = UINT_MAX;
+                rc = PMIX_ERR_EVENT_REGISTRATION;
+                PMIX_RELEASE(evhdlr);
+                goto ack;
+            }
+            memcpy(evhdlr->affected, cd->affected, cd->naffected * sizeof(pmix_proc_t));
+        }
         evhdlr->evhdlr = cd->evhdlr;
         evhdlr->cbobject = cbobject;
         if (NULL != cd->codes) {
@@ -599,6 +650,17 @@ static void reg_event_hdlr(int sd, short args, void *cbdata)
         }
         memcpy(evhdlr->rng.procs, parray, nprocs * sizeof(pmix_proc_t));
     }
+    if (NULL != cd->affected && 0 < cd->naffected) {
+        evhdlr->naffected = cd->naffected;
+        PMIX_PROC_CREATE(evhdlr->affected, cd->naffected);
+        if (NULL == evhdlr->affected) {
+            index = UINT_MAX;
+            rc = PMIX_ERR_EVENT_REGISTRATION;
+            PMIX_RELEASE(evhdlr);
+            goto ack;
+        }
+        memcpy(evhdlr->affected, cd->affected, cd->naffected * sizeof(pmix_proc_t));
+    }
     evhdlr->evhdlr = cd->evhdlr;
     evhdlr->cbobject = cbobject;
     if (NULL == cd->codes) {
@@ -810,7 +872,7 @@ PMIX_EXPORT void PMIx_Register_event_handler(pmix_status_t codes[], size_t ncode
     cd->evregcbfn = cbfunc;
     cd->cbdata = cbdata;
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_client_globals.event_output,
                         "pmix_register_event_hdlr shifting to progress thread");
 
     PMIX_THREADSHIFT(cd, reg_event_hdlr);
@@ -832,7 +894,8 @@ static void dereg_event_hdlr(int sd, short args, void *cbdata)
 
     /* if I am not the server, and I am connected, then I need
      * to notify the server to remove my registration */
-    if (!PMIX_PROC_IS_SERVER(pmix_globals.mypeer) && pmix_globals.connected) {
+    if ((!PMIX_PROC_IS_SERVER(pmix_globals.mypeer) || PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) &&
+        pmix_globals.connected) {
         msg = PMIX_NEW(pmix_buffer_t);
         PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
                          msg, &cmd, 1, PMIX_COMMAND);
@@ -1025,7 +1088,7 @@ PMIX_EXPORT void PMIx_Deregister_event_handler(size_t event_hdlr_ref,
     cd->cbdata = cbdata;
     cd->ref = event_hdlr_ref;
 
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output_verbose(2, pmix_client_globals.event_output,
                         "pmix_deregister_event_hdlr shifting to progress thread");
     PMIX_THREADSHIFT(cd, dereg_event_hdlr);
 }

opal/mca/pmix/pmix3x/pmix/src/hwloc/Makefile.include

@@ -0,0 +1,15 @@
+# -*- makefile -*-
+#
+# Copyright (c) 2018      Intel, Inc. All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+sources += \
+        hwloc/hwloc.c
+
+headers += \
+        hwloc/hwloc-internal.h

opal/mca/pmix/pmix3x/pmix/src/hwloc/hwloc-internal.h

@@ -0,0 +1,60 @@
+/*
+ * Copyright (c) 2011-2017 Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2016      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ *
+ * Copyright (c) 2016-2018 Intel, Inc. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ * this file is included in the rest of
+ * the code base via pmix/hwloc/hwloc-internal.h.
+ */
+
+#ifndef PMIX_HWLOC_INTERNAL_H
+#define PMIX_HWLOC_INTERNAL_H
+
+
+#include <src/include/pmix_config.h>
+#include <pmix_common.h>
+
+#if PMIX_HAVE_HWLOC
+#include <hwloc.h>
+
+#if HWLOC_API_VERSION < 0x00010b00
+#define HWLOC_OBJ_NUMANODE HWLOC_OBJ_NODE
+#define HWLOC_OBJ_PACKAGE HWLOC_OBJ_SOCKET
+#endif
+
+extern hwloc_topology_t pmix_hwloc_topology;
+#endif
+
+BEGIN_C_DECLS
+
+typedef enum {
+    VM_HOLE_NONE = -1,
+    VM_HOLE_BEGIN = 0,        /* use hole at the very beginning */
+    VM_HOLE_AFTER_HEAP = 1,   /* use hole right after heap */
+    VM_HOLE_BEFORE_STACK = 2, /* use hole right before stack */
+    VM_HOLE_BIGGEST = 3,      /* use biggest hole */
+    VM_HOLE_IN_LIBS = 4,      /* use biggest hole between heap and stack */
+    VM_HOLE_CUSTOM = 5,       /* use given address if available */
+} pmix_hwloc_vm_hole_kind_t;
+
+typedef enum {
+    VM_MAP_FILE = 0,
+    VM_MAP_ANONYMOUS = 1,
+    VM_MAP_HEAP = 2,
+    VM_MAP_STACK = 3,
+    VM_MAP_OTHER = 4 /* vsyscall/vdso/vvar shouldn't occur since we stop after stack */
+} pmix_hwloc_vm_map_kind_t;
+
+PMIX_EXPORT pmix_status_t pmix_hwloc_get_topology(pmix_info_t *info, size_t ninfo);
+PMIX_EXPORT void pmix_hwloc_cleanup(void);
+
+END_C_DECLS
+
+#endif /* PMIX_HWLOC_INTERNAL_H */

opal/mca/pmix/pmix3x/pmix/src/hwloc/hwloc.c

@@ -0,0 +1,741 @@
+/*
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
+ * Copyright (c) 2017      Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2017      Inria.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include <src/include/pmix_config.h>
+#include <pmix_common.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <assert.h>
+#include <errno.h>
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif  /* HAVE_UNISTD_H */
+#include <string.h>
+#include <sys/mman.h>
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+#if HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+
+#include "src/util/error.h"
+#include "src/util/fd.h"
+#include "src/util/path.h"
+#include "src/mca/bfrops/bfrops_types.h"
+#include "src/server/pmix_server_ops.h"
+#include "hwloc-internal.h"
+
+#if PMIX_HAVE_HWLOC
+
+#if HWLOC_API_VERSION >= 0x20000
+#include <hwloc/shmem.h>
+#endif
+
+
+PMIX_EXPORT hwloc_topology_t pmix_hwloc_topology = NULL;
+static bool external_topology = false;
+
+#if HWLOC_API_VERSION >= 0x20000
+static size_t shmemsize = 0;
+static size_t shmemaddr;
+static char *shmemfile = NULL;
+static int shmemfd = -1;
+
+static int parse_map_line(const char *line,
+                          unsigned long *beginp,
+                          unsigned long *endp,
+                          pmix_hwloc_vm_map_kind_t *kindp);
+static int use_hole(unsigned long holebegin,
+                    unsigned long holesize,
+                    unsigned long *addrp,
+                    unsigned long size);
+static int find_hole(pmix_hwloc_vm_hole_kind_t hkind,
+                     size_t *addrp,
+                     size_t size);
+static int enough_space(const char *filename,
+                        size_t space_req,
+                        uint64_t *space_avail,
+                        bool *result);
+#endif
+
+static int set_flags(hwloc_topology_t topo, unsigned int flags)
+{
+    #if HWLOC_API_VERSION < 0x20000
+            flags = HWLOC_TOPOLOGY_FLAG_IO_DEVICES;
+    #else
+            int ret = hwloc_topology_set_io_types_filter(topo, HWLOC_TYPE_FILTER_KEEP_IMPORTANT);
+            if (0 != ret) return ret;
+    #endif
+    if (0 != hwloc_topology_set_flags(topo, flags)) {
+        return PMIX_ERR_INIT;
+    }
+    return PMIX_SUCCESS;
+}
+#endif
+
+pmix_status_t pmix_hwloc_get_topology(pmix_info_t *info, size_t ninfo)
+{
+#if PMIX_HAVE_HWLOC
+    size_t n;
+    bool save_xml_v1 = false;
+    bool save_xml_v2 = false;
+#if HWLOC_API_VERSION < 0x20000
+    bool save_xml_v2_reqd = false;
+#endif
+    bool share_topo = false;
+    bool share_reqd = false;
+    pmix_kval_t *kp2;
+    char *xml;
+    int sz;
+    pmix_status_t rc;
+#if HWLOC_API_VERSION >= 0x20000
+    pmix_hwloc_vm_hole_kind_t hole = VM_HOLE_BIGGEST;
+#endif
+
+    /* we only do something if specifically requested */
+    if (NULL == info) {
+        return PMIX_SUCCESS;
+    }
+
+    /* check for directives */
+    for (n=0; n < ninfo; n++) {
+        if (0 == strncmp(info[n].key, PMIX_TOPOLOGY, PMIX_MAX_KEYLEN)) {
+            /* if the pointer is NULL, then they want us to
+             * get the topology - it not NULL, then they
+             * are giving us the topology */
+            if (NULL != pmix_hwloc_topology) {
+                /* cannot have two topologies */
+                PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
+                return PMIX_ERR_BAD_PARAM;
+            }
+            if (NULL != info[n].value.data.ptr) {
+                pmix_hwloc_topology = (hwloc_topology_t)info[n].value.data.ptr;
+                external_topology = true;
+            } else {
+                if (0 != hwloc_topology_init(&pmix_hwloc_topology)) {
+                    return PMIX_ERR_INIT;
+                }
+
+                if (0 != set_flags(pmix_hwloc_topology, 0)) {
+                    hwloc_topology_destroy(pmix_hwloc_topology);
+                    return PMIX_ERR_INIT;
+                }
+
+                if (0 != hwloc_topology_load(pmix_hwloc_topology)) {
+                    PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED);
+                    hwloc_topology_destroy(pmix_hwloc_topology);
+                    return PMIX_ERR_NOT_SUPPORTED;
+                }
+            }
+        } else if (0 == strncmp(info[n].key, PMIX_HWLOC_XML_V1, PMIX_MAX_KEYLEN)) {
+            /* if the string pointer is NULL or empty, then they
+             * want us to create it and store it for later sharing.
+             * if non-NULL, then this is the topology we are to use */
+            if (NULL == info[n].value.data.string) {
+                save_xml_v1 = true;
+            } else if (NULL != pmix_hwloc_topology) {
+                /* cannot have two topologies */
+                PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
+                return PMIX_ERR_BAD_PARAM;
+            } else {
+                /* load the topology */
+                if (0 != hwloc_topology_init(&pmix_hwloc_topology)) {
+                    return PMIX_ERROR;
+                }
+                if (0 != hwloc_topology_set_xmlbuffer(pmix_hwloc_topology,
+                                                      info[n].value.data.string,
+                                                      strlen(info[n].value.data.string))) {
+                    hwloc_topology_destroy(pmix_hwloc_topology);
+                    return PMIX_ERROR;
+                }
+                /* since we are loading this from an external source, we have to
+                 * explicitly set a flag so hwloc sets things up correctly
+                 */
+                if (0 != set_flags(pmix_hwloc_topology, HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM)) {
+                    hwloc_topology_destroy(pmix_hwloc_topology);
+                    return PMIX_ERROR;
+                }
+                /* now load the topology */
+                if (0 != hwloc_topology_load(pmix_hwloc_topology)) {
+                    hwloc_topology_destroy(pmix_hwloc_topology);
+                    return PMIX_ERROR;
+                }
+                /* store the string */
+                kp2 = PMIX_NEW(pmix_kval_t);
+                if (NULL == kp2) {
+                    return PMIX_ERR_NOMEM;
+                }
+                kp2->key = strdup(info[n].key);
+                PMIX_VALUE_XFER(rc, kp2->value, &info[n].value);
+                if (PMIX_SUCCESS != rc) {
+                    PMIX_ERROR_LOG(rc);
+                    PMIX_RELEASE(kp2);
+                    return rc;
+                }
+                pmix_list_append(&pmix_server_globals.gdata, &kp2->super);
+            }
+        } else if (0 == strncmp(info[n].key, PMIX_HWLOC_XML_V2, PMIX_MAX_KEYLEN)) {
+            /* if the string pointer is NULL or empty, then they
+             * want us to create it and store it for later sharing.
+             * if non-NULL, then this is the topology we are to use */
+            if (NULL == info[n].value.data.string) {
+                save_xml_v2 = true;
+#if HWLOC_API_VERSION < 0x20000
+                save_xml_v2_reqd = PMIX_INFO_REQUIRED(&info[n]);
+#endif
+            } else if (NULL != pmix_hwloc_topology) {
+                /* cannot have two topologies */
+                PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
+                return PMIX_ERR_BAD_PARAM;
+            } else {
+                /* load the topology */
+                if (0 != hwloc_topology_init(&pmix_hwloc_topology)) {
+                    return PMIX_ERROR;
+                }
+                if (0 != hwloc_topology_set_xmlbuffer(pmix_hwloc_topology,
+                                                      info[n].value.data.string,
+                                                      strlen(info[n].value.data.string))) {
+                    hwloc_topology_destroy(pmix_hwloc_topology);
+                    return PMIX_ERROR;
+                }
+                /* since we are loading this from an external source, we have to
+                 * explicitly set a flag so hwloc sets things up correctly
+                 */
+                if (0 != set_flags(pmix_hwloc_topology, HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM)) {
+                    hwloc_topology_destroy(pmix_hwloc_topology);
+                    return PMIX_ERROR;
+                }
+                /* now load the topology */
+                if (0 != hwloc_topology_load(pmix_hwloc_topology)) {
+                    hwloc_topology_destroy(pmix_hwloc_topology);
+                    return PMIX_ERROR;
+                }
+                /* store the string */
+                kp2 = PMIX_NEW(pmix_kval_t);
+                if (NULL == kp2) {
+                    return PMIX_ERR_NOMEM;
+                }
+                kp2->key = strdup(info[n].key);
+                PMIX_VALUE_XFER(rc, kp2->value, &info[n].value);
+                if (PMIX_SUCCESS != rc) {
+                    PMIX_ERROR_LOG(rc);
+                    PMIX_RELEASE(kp2);
+                    return rc;
+                }
+                pmix_list_append(&pmix_server_globals.gdata, &kp2->super);
+            }
+        } else if (0 == strncmp(info[n].key, PMIX_TOPOLOGY_FILE, PMIX_MAX_KEYLEN)) {
+            if (NULL == info[n].value.data.string) {
+                PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
+                return PMIX_ERR_BAD_PARAM;
+            } else if (NULL != pmix_hwloc_topology) {
+                /* cannot have two topologies */
+                PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
+                return PMIX_ERR_BAD_PARAM;
+            } else {
+                if (0 != hwloc_topology_init(&pmix_hwloc_topology)) {
+                    return PMIX_ERR_NOT_SUPPORTED;
+                }
+                if (0 != hwloc_topology_set_xml(pmix_hwloc_topology, info[n].value.data.string)) {
+                    hwloc_topology_destroy(pmix_hwloc_topology);
+                    return PMIX_ERR_NOT_SUPPORTED;
+                }
+                /* since we are loading this from an external source, we have to
+                 * explicitly set a flag so hwloc sets things up correctly
+                 */
+                if (0 != set_flags(pmix_hwloc_topology, HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM)) {
+                    hwloc_topology_destroy(pmix_hwloc_topology);
+                    return PMIX_ERR_NOT_SUPPORTED;
+                }
+                if (0 != hwloc_topology_load(pmix_hwloc_topology)) {
+                    hwloc_topology_destroy(pmix_hwloc_topology);
+                     return PMIX_ERR_NOT_SUPPORTED;
+                }
+                /* store the filename */
+                kp2 = PMIX_NEW(pmix_kval_t);
+                if (NULL == kp2) {
+                    return PMIX_ERR_NOMEM;
+                }
+                kp2->key = strdup(info[n].key);
+                PMIX_VALUE_XFER(rc, kp2->value, &info[n].value);
+                if (PMIX_SUCCESS != rc) {
+                    PMIX_ERROR_LOG(rc);
+                    PMIX_RELEASE(kp2);
+                    return rc;
+                }
+                pmix_list_append(&pmix_server_globals.gdata, &kp2->super);
+            }
+        } else if (0 == strncmp(info[n].key, PMIX_HWLOC_SHARE_TOPO, PMIX_MAX_KEYLEN)) {
+            share_topo = PMIX_INFO_TRUE(&info[n]);
+            share_reqd = PMIX_INFO_IS_REQUIRED(&info[n]);
+        } else if (0 == strncmp(info[n].key, PMIX_HWLOC_HOLE_KIND, PMIX_MAX_KEYLEN)) {
+#if HWLOC_API_VERSION >= 0x20000
+            if (0 == strcasecmp(info[n].value.data.string, "none")) {
+                hole = VM_HOLE_NONE;
+            } else if (0 == strcasecmp(info[n].value.data.string, "begin")) {
+                hole = VM_HOLE_BEGIN;
+            } else if (0 == strcasecmp(info[n].value.data.string, "biggest")) {
+                hole = VM_HOLE_BIGGEST;
+            } else if (0 == strcasecmp(info[n].value.data.string, "libs")) {
+                hole = VM_HOLE_IN_LIBS;
+            } else if (0 == strcasecmp(info[n].value.data.string, "heap")) {
+                hole = VM_HOLE_AFTER_HEAP;
+            } else if (0 == strcasecmp(info[n].value.data.string, "stack")) {
+                hole = VM_HOLE_BEFORE_STACK;
+            } else {
+                PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
+                return PMIX_ERR_BAD_PARAM;
+            }
+#endif
+        }
+    }
+
+    if (save_xml_v1) {
+        /* create the XML string */
+#if HWLOC_API_VERSION >= 0x20000
+        if (0 != hwloc_topology_export_xmlbuffer(pmix_hwloc_topology, &xml, &sz, HWLOC_TOPOLOGY_EXPORT_XML_FLAG_V1)) {
+            PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED);
+            return PMIX_ERR_NOT_SUPPORTED;
+        }
+#else
+        if (0 != hwloc_topology_export_xmlbuffer(pmix_hwloc_topology, &xml, &sz)) {
+            PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED);
+            return PMIX_ERR_NOT_SUPPORTED;
+        }
+#endif
+        /* store it */
+        kp2 = PMIX_NEW(pmix_kval_t);
+        if (NULL == kp2) {
+            return PMIX_ERR_NOMEM;
+        }
+        kp2->key = strdup(PMIX_HWLOC_XML_V1);
+        PMIX_VALUE_LOAD(kp2->value, xml, PMIX_STRING);
+        hwloc_free_xmlbuffer(pmix_hwloc_topology, xml);
+        pmix_list_append(&pmix_server_globals.gdata, &kp2->super);
+    }
+    if (save_xml_v2) {
+        /* create the XML string */
+#if HWLOC_API_VERSION >= 0x20000
+        if (0 != hwloc_topology_export_xmlbuffer(pmix_hwloc_topology, &xml, &sz, 0)) {
+            PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED);
+            return PMIX_ERR_NOT_SUPPORTED;
+        }
+        /* store it */
+        kp2 = PMIX_NEW(pmix_kval_t);
+        if (NULL == kp2) {
+            return PMIX_ERR_NOMEM;
+        }
+        kp2->key = strdup(PMIX_HWLOC_XML_V1);
+        PMIX_VALUE_LOAD(kp2->value, xml, PMIX_STRING);
+        hwloc_free_xmlbuffer(pmix_hwloc_topology, xml);
+        pmix_list_append(&pmix_server_globals.gdata, &kp2->super);
+#else
+        if (save_xml_v2_reqd) {
+            PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED);
+            return PMIX_ERR_NOT_SUPPORTED;
+        }
+#endif
+    }
+
+    if (share_topo) {
+#if HWLOC_API_VERSION < 0x20000
+        if (share_reqd) {
+            return PMIX_ERR_NOT_SUPPORTED;
+        }
+#else
+        pmix_status_t rc;
+        bool space_available = false;
+        uint64_t amount_space_avail = 0;
+
+        if (VM_HOLE_NONE == hole) {
+            return PMIX_SUCCESS;
+        }
+
+        /* get the size of the topology shared memory segment */
+        if (0 != hwloc_shmem_topology_get_length(pmix_hwloc_topology, &shmemsize, 0)) {
+            if (share_reqd) {
+                PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED);
+                return PMIX_ERR_NOT_SUPPORTED;
+            }
+            return PMIX_SUCCESS;
+        }
+
+        if (PMIX_SUCCESS != (rc = find_hole(hole, &shmemaddr, shmemsize))) {
+            /* we couldn't find a hole, so don't use the shmem support */
+            if (share_reqd) {
+                PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED);
+                return PMIX_ERR_NOT_SUPPORTED;
+            }
+            return PMIX_SUCCESS;
+        }
+        /* create the shmem file in our session dir so it
+         * will automatically get cleaned up */
+        asprintf(&shmemfile, "%s/hwloc.sm", pmix_server_globals.tmpdir);
+        /* let's make sure we have enough space for the backing file */
+        if (PMIX_SUCCESS != (rc = enough_space(shmemfile, shmemsize,
+                                               &amount_space_avail,
+                                               &space_available))) {
+            free(shmemfile);
+            shmemfile = NULL;
+            if (share_reqd) {
+                PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED);
+                return PMIX_ERR_NOT_SUPPORTED;
+            } else {
+                return PMIX_SUCCESS;
+            }
+        }
+        if (!space_available) {
+            free(shmemfile);
+            shmemfile = NULL;
+            if (share_reqd) {
+                PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED);
+                return PMIX_ERR_NOT_SUPPORTED;
+            } else {
+                return PMIX_SUCCESS;
+            }
+        }
+        /* enough space is available, so create the segment */
+        if (-1 == (shmemfd = open(shmemfile, O_CREAT | O_RDWR, 0600))) {
+            free(shmemfile);
+            shmemfile = NULL;
+            if (share_reqd) {
+                PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED);
+                return PMIX_ERR_NOT_SUPPORTED;
+            } else {
+                return PMIX_SUCCESS;
+            }
+        }
+        /* ensure nobody inherits this fd */
+        pmix_fd_set_cloexec(shmemfd);
+        /* populate the shmem segment with the topology */
+        if (0 != (rc = hwloc_shmem_topology_write(pmix_hwloc_topology, shmemfd, 0,
+                                                  (void*)shmemaddr, shmemsize, 0))) {
+            unlink(shmemfile);
+            free(shmemfile);
+            shmemfile = NULL;
+            close(shmemfd);
+            shmemfd = -1;
+            if (share_reqd) {
+                PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED);
+                return PMIX_ERR_NOT_SUPPORTED;
+            } else {
+                return PMIX_SUCCESS;
+            }
+        }
+        /* store the rendezvous info */
+        kp2 = PMIX_NEW(pmix_kval_t);
+        if (NULL == kp2) {
+            return PMIX_ERR_NOMEM;
+        }
+        kp2->key = strdup(PMIX_HWLOC_SHMEM_FILE);
+        PMIX_VALUE_CREATE(kp2->value, 1);
+        PMIX_VALUE_LOAD(kp2->value, shmemfile, PMIX_STRING);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            PMIX_RELEASE(kp2);
+            return rc;
+        }
+        pmix_list_append(&pmix_server_globals.gdata, &kp2->super);
+        kp2 = PMIX_NEW(pmix_kval_t);
+        if (NULL == kp2) {
+            return PMIX_ERR_NOMEM;
+        }
+        kp2->key = strdup(PMIX_HWLOC_SHMEM_ADDR);
+        PMIX_VALUE_CREATE(kp2->value, 1);
+        PMIX_VALUE_LOAD(kp2->value, &shmemaddr, PMIX_SIZE);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            PMIX_RELEASE(kp2);
+            return rc;
+        }
+        pmix_list_append(&pmix_server_globals.gdata, &kp2->super);
+        kp2 = PMIX_NEW(pmix_kval_t);
+        if (NULL == kp2) {
+            return PMIX_ERR_NOMEM;
+        }
+        kp2->key = strdup(PMIX_HWLOC_SHMEM_SIZE);
+        PMIX_VALUE_CREATE(kp2->value, 1);
+        PMIX_VALUE_LOAD(kp2->value, &shmemsize, PMIX_SIZE);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            PMIX_RELEASE(kp2);
+            return rc;
+        }
+        pmix_list_append(&pmix_server_globals.gdata, &kp2->super);
+
+
+#endif
+    }
+
+    return PMIX_SUCCESS;
+#else  // PMIX_HAVE_HWLOC
+    return PMIX_ERR_NOT_SUPPORTED;
+#endif
+}
+
+void pmix_hwloc_cleanup(void)
+{
+#if PMIX_HAVE_HWLOC
+#if HWLOC_API_VERSION >= 0x20000
+    if (NULL != shmemfile) {
+        unlink(shmemfile);
+        free(shmemfile);
+    }
+    if (0 <= shmemfd) {
+        close(shmemfd);
+    }
+#endif
+    if (NULL != pmix_hwloc_topology && !external_topology) {
+        hwloc_topology_destroy(pmix_hwloc_topology);
+    }
+#endif
+    return;
+}
+
+#if PMIX_HAVE_HWLOC
+#if HWLOC_API_VERSION >= 0x20000
+
+static int parse_map_line(const char *line,
+                          unsigned long *beginp,
+                          unsigned long *endp,
+                          pmix_hwloc_vm_map_kind_t *kindp)
+{
+    const char *tmp = line, *next;
+    unsigned long value;
+
+    /* "beginaddr-endaddr " */
+    value = strtoull(tmp, (char **) &next, 16);
+    if (next == tmp) {
+        return PMIX_ERROR;
+    }
+
+    *beginp = (unsigned long) value;
+
+    if (*next != '-') {
+        return PMIX_ERROR;
+    }
+
+     tmp = next + 1;
+
+    value = strtoull(tmp, (char **) &next, 16);
+    if (next == tmp) {
+        return PMIX_ERROR;
+    }
+    *endp = (unsigned long) value;
+    tmp = next;
+
+    if (*next != ' ') {
+        return PMIX_ERROR;
+    }
+    tmp = next + 1;
+
+    /* look for ending absolute path */
+    next = strchr(tmp, '/');
+    if (next) {
+        *kindp = VM_MAP_FILE;
+    } else {
+        /* look for ending special tag [foo] */
+        next = strchr(tmp, '[');
+        if (next) {
+            if (!strncmp(next, "[heap]", 6)) {
+                *kindp = VM_MAP_HEAP;
+            } else if (!strncmp(next, "[stack]", 7)) {
+                *kindp = VM_MAP_STACK;
+            } else {
+                char *end;
+                if ((end = strchr(next, '\n')) != NULL) {
+                    *end = '\0';
+                }
+                *kindp = VM_MAP_OTHER;
+            }
+        } else {
+            *kindp = VM_MAP_ANONYMOUS;
+        }
+    }
+
+    return PMIX_SUCCESS;
+}
+
+#define ALIGN2MB (2*1024*1024UL)
+
+static int use_hole(unsigned long holebegin,
+                    unsigned long holesize,
+                    unsigned long *addrp,
+                    unsigned long size)
+{
+    unsigned long aligned;
+    unsigned long middle = holebegin+holesize/2;
+
+    if (holesize < size) {
+        return PMIX_ERROR;
+    }
+
+    /* try to align the middle of the hole on 64MB for POWER's 64k-page PMD */
+    #define ALIGN64MB (64*1024*1024UL)
+    aligned = (middle + ALIGN64MB) & ~(ALIGN64MB-1);
+    if (aligned + size <= holebegin + holesize) {
+        *addrp = aligned;
+        return PMIX_SUCCESS;
+    }
+
+    /* try to align the middle of the hole on 2MB for x86 PMD */
+    aligned = (middle + ALIGN2MB) & ~(ALIGN2MB-1);
+    if (aligned + size <= holebegin + holesize) {
+        *addrp = aligned;
+        return PMIX_SUCCESS;
+    }
+
+    /* just use the end of the hole */
+    *addrp = holebegin + holesize - size;
+    return PMIX_SUCCESS;
+}
+
+static int find_hole(pmix_hwloc_vm_hole_kind_t hkind,
+                     size_t *addrp, size_t size)
+{
+    unsigned long biggestbegin = 0;
+    unsigned long biggestsize = 0;
+    unsigned long prevend = 0;
+    pmix_hwloc_vm_map_kind_t prevmkind = VM_MAP_OTHER;
+    int in_libs = 0;
+    FILE *file;
+    char line[96];
+
+    file = fopen("/proc/self/maps", "r");
+    if (!file) {
+        return PMIX_ERROR;
+    }
+
+    while (fgets(line, sizeof(line), file) != NULL) {
+        unsigned long begin=0, end=0;
+        pmix_hwloc_vm_map_kind_t mkind=VM_MAP_OTHER;
+
+        if (!parse_map_line(line, &begin, &end, &mkind)) {
+            switch (hkind) {
+                case VM_HOLE_BEGIN:
+                    fclose(file);
+                    return use_hole(0, begin, addrp, size);
+
+                case VM_HOLE_AFTER_HEAP:
+                    if (prevmkind == VM_MAP_HEAP && mkind != VM_MAP_HEAP) {
+                        /* only use HEAP when there's no other HEAP after it
+                         * (there can be several of them consecutively).
+                         */
+                        fclose(file);
+                        return use_hole(prevend, begin-prevend, addrp, size);
+                    }
+                    break;
+
+                case VM_HOLE_BEFORE_STACK:
+                    if (mkind == VM_MAP_STACK) {
+                        fclose(file);
+                        return use_hole(prevend, begin-prevend, addrp, size);
+                    }
+                    break;
+
+                case VM_HOLE_IN_LIBS:
+                    /* see if we are between heap and stack */
+                    if (prevmkind == VM_MAP_HEAP) {
+                        in_libs = 1;
+                    }
+                    if (mkind == VM_MAP_STACK) {
+                        in_libs = 0;
+                    }
+                    if (!in_libs) {
+                        /* we're not in libs, ignore this entry */
+                        break;
+                    }
+                    /* we're in libs, consider this entry for searching the biggest hole below */
+                    /* fallthrough */
+
+                case VM_HOLE_BIGGEST:
+                    if (begin-prevend > biggestsize) {
+                        biggestbegin = prevend;
+                        biggestsize = begin-prevend;
+                    }
+                    break;
+
+                    default:
+                        assert(0);
+            }
+        }
+
+        while (!strchr(line, '\n')) {
+            if (!fgets(line, sizeof(line), file)) {
+                goto done;
+            }
+        }
+
+        if (mkind == VM_MAP_STACK) {
+          /* Don't go beyond the stack. Other VMAs are special (vsyscall, vvar, vdso, etc),
+           * There's no spare room there. And vsyscall is even above the userspace limit.
+           */
+          break;
+        }
+
+        prevend = end;
+        prevmkind = mkind;
+
+    }
+
+  done:
+    fclose(file);
+    if (hkind == VM_HOLE_IN_LIBS || hkind == VM_HOLE_BIGGEST) {
+        return use_hole(biggestbegin, biggestsize, addrp, size);
+    }
+
+    return PMIX_ERROR;
+}
+
+static int enough_space(const char *filename,
+                        size_t space_req,
+                        uint64_t *space_avail,
+                        bool *result)
+{
+    uint64_t avail = 0;
+    size_t fluff = (size_t)(.05 * space_req);
+    bool enough = false;
+    char *last_sep = NULL;
+    /* the target file name is passed here, but we need to check the parent
+     * directory. store it so we can extract that info later. */
+    char *target_dir = strdup(filename);
+    int rc;
+
+    if (NULL == target_dir) {
+        rc = PMIX_ERR_OUT_OF_RESOURCE;
+        goto out;
+    }
+    /* get the parent directory */
+    last_sep = strrchr(target_dir, PMIX_PATH_SEP[0]);
+    *last_sep = '\0';
+    /* now check space availability */
+    if (PMIX_SUCCESS != (rc = pmix_path_df(target_dir, &avail))) {
+        goto out;
+    }
+    /* do we have enough space? */
+    if (avail >= space_req + fluff) {
+        enough = true;
+    }
+
+out:
+    if (NULL != target_dir) {
+        free(target_dir);
+    }
+    *result = enough;
+    *space_avail = avail;
+    return rc;
+}
+#endif
+
+#endif  // PMIX_HAVE_HWLOC

opal/mca/pmix/pmix3x/pmix/src/include/pmix_globals.c

@@ -2,7 +2,7 @@
 /*
  * Copyright (c) 2014-2018 Intel, Inc.  All rights reserved.
  * Copyright (c) 2014-2017 Research Organization for Information Science
- * Copyright (c) 2014-2017 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc.  All rights reserved.
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2014-2015 Artem Y. Polyakov <artpol84@gmail.com>.
  *                         All rights reserved.
@@ -99,16 +99,19 @@ PMIX_EXPORT PMIX_CLASS_INSTANCE(pmix_cleanup_dir_t,
 static void nscon(pmix_nspace_t *p)
 {
     p->nspace = NULL;
+    p->nprocs = 0;
     p->nlocalprocs = 0;
     p->all_registered = false;
     p->version_stored = false;
     p->jobbkt = NULL;
     p->ndelivered = 0;
+    p->nfinalized = 0;
     PMIX_CONSTRUCT(&p->ranks, pmix_list_t);
     memset(&p->compat, 0, sizeof(p->compat));
     PMIX_CONSTRUCT(&p->epilog.cleanup_dirs, pmix_list_t);
     PMIX_CONSTRUCT(&p->epilog.cleanup_files, pmix_list_t);
     PMIX_CONSTRUCT(&p->epilog.ignores, pmix_list_t);
+    PMIX_CONSTRUCT(&p->setup_data, pmix_list_t);
 }
 static void nsdes(pmix_nspace_t *p)
 {
@@ -125,6 +128,7 @@ static void nsdes(pmix_nspace_t *p)
     PMIX_LIST_DESTRUCT(&p->epilog.cleanup_dirs);
     PMIX_LIST_DESTRUCT(&p->epilog.cleanup_files);
     PMIX_LIST_DESTRUCT(&p->epilog.ignores);
+    PMIX_LIST_DESTRUCT(&p->setup_data);
 }
 PMIX_EXPORT PMIX_CLASS_INSTANCE(pmix_nspace_t,
                                 pmix_list_item_t,
@@ -173,7 +177,6 @@ static void pcon(pmix_peer_t *p)
     p->proc_cnt = 0;
     p->index = 0;
     p->sd = -1;
-    p->finalized = false;
     p->send_ev_active = false;
     p->recv_ev_active = false;
     PMIX_CONSTRUCT(&p->send_queue, pmix_list_t);
@@ -183,6 +186,7 @@ static void pcon(pmix_peer_t *p)
     PMIX_CONSTRUCT(&p->epilog.cleanup_dirs, pmix_list_t);
     PMIX_CONSTRUCT(&p->epilog.cleanup_files, pmix_list_t);
     PMIX_CONSTRUCT(&p->epilog.ignores, pmix_list_t);
+
 }
 
 static void pdes(pmix_peer_t *p)
@@ -214,6 +218,9 @@ static void pdes(pmix_peer_t *p)
     PMIX_LIST_DESTRUCT(&p->epilog.cleanup_dirs);
     PMIX_LIST_DESTRUCT(&p->epilog.cleanup_files);
     PMIX_LIST_DESTRUCT(&p->epilog.ignores);
+    if (NULL != p->nptr) {
+        PMIX_RELEASE(p->nptr);
+    }
 }
 PMIX_EXPORT PMIX_CLASS_INSTANCE(pmix_peer_t,
                                 pmix_object_t,

opal/mca/pmix/pmix3x/pmix/src/include/pmix_globals.h

@@ -56,6 +56,12 @@ BEGIN_C_DECLS
 #define PMIX_PNET_SETUP_APP                 "pmix.pnet.setapp"      // (pmix_byte_object_t) blob containing info to be given to
                                                                     //      pnet framework on remote nodes
 
+#define PMIX_INFO_OP_COMPLETE    0x80000000
+#define PMIX_INFO_OP_COMPLETED(m)            \
+    ((pmix_info_t*)(m))->flags |= PMIX_INFO_OP_COMPLETE
+#define PMIX_INFO_OP_IS_COMPLETE(m)          \
+    ((m)->flags & PMIX_INFO_OP_COMPLETE)
+
 /* define an internal-only process name that has
  * a dynamically-sized nspace field to save memory */
 typedef struct {
@@ -155,11 +161,13 @@ PMIX_CLASS_DECLARATION(pmix_cleanup_dir_t);
 typedef struct {
     pmix_list_item_t super;
     char *nspace;
+    pmix_rank_t nprocs;          // num procs in this nspace
     size_t nlocalprocs;
     bool all_registered;         // all local ranks have been defined
     bool version_stored;         // the version string used by this nspace has been stored
     pmix_buffer_t *jobbkt;       // packed version of jobinfo
     size_t ndelivered;           // count of #local clients that have received the jobinfo
+    size_t nfinalized;           // count of #local clients that have finalized
     pmix_list_t ranks;           // list of pmix_rank_info_t for connection support of my clients
     /* all members of an nspace are required to have the
      * same personality, but it can differ between nspaces.
@@ -168,6 +176,8 @@ typedef struct {
     pmix_personality_t compat;
     pmix_epilog_t epilog;       // things to do upon termination of all local clients
                                 // from this nspace
+    pmix_list_t setup_data;     // list of pmix_kval_t containing info structs having blobs
+                                // for setting up the local node for this nspace/application
 } pmix_nspace_t;
 PMIX_CLASS_DECLARATION(pmix_nspace_t);
 
@@ -285,7 +295,6 @@ typedef struct {
     pmix_collect_t collect_type;    // whether or not data is to be returned at completion
     pmix_modex_cbfunc_t modexcbfunc;
     pmix_op_cbfunc_t op_cbfunc;
-    pmix_connect_cbfunc_t cnct_cbfunc;
 } pmix_server_trkr_t;
 PMIX_CLASS_DECLARATION(pmix_server_trkr_t);
 
@@ -353,7 +362,6 @@ typedef struct {
         pmix_value_cbfunc_t valuefn;
         pmix_lookup_cbfunc_t lookupfn;
         pmix_spawn_cbfunc_t spawnfn;
-        pmix_connect_cbfunc_t cnctfn;
         pmix_hdlr_reg_cbfunc_t hdlrregfn;
     } cbfunc;
     size_t errhandler_ref;
@@ -382,14 +390,6 @@ PMIX_CLASS_DECLARATION(pmix_cb_t);
 } while (0)
 
 
-#define PMIX_WAIT_FOR_COMPLETION(a)             \
-    do {                                        \
-        while ((a)) {                           \
-            usleep(10);                         \
-        }                                       \
-        PMIX_ACQUIRE_OBJECT((a));               \
-    } while (0)
-
 typedef struct {
     pmix_object_t super;
     pmix_event_t ev;
@@ -397,9 +397,28 @@ typedef struct {
     pmix_status_t status;
     pmix_proc_t source;
     pmix_data_range_t range;
+    /* For notification, we use the targets field to track
+     * any custom range of procs that are to receive the
+     * event.
+     */
     pmix_proc_t *targets;
     size_t ntargets;
+    /* When generating a notification, the originator can
+     * specify the range of procs affected by this event.
+     * For example, when creating a JOB_TERMINATED event,
+     * the RM can specify the nspace of the job that has
+     * ended, thus allowing users to provide a different
+     * callback object based on the nspace being monitored.
+     * We use the "affected" field to track these values
+     * when processing the event chain.
+     */
+    pmix_proc_t *affected;
+    size_t naffected;
+    /* track if the event generator stipulates that default
+     * event handlers are/are not to be given the event */
     bool nondefault;
+    /* carry along any other provided info so the individual
+     * handlers can look at it */
     pmix_info_t *info;
     size_t ninfo;
     pmix_buffer_t *buf;

opal/mca/pmix/pmix3x/pmix/src/include/types.h

@@ -9,7 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc.  All rights reserved.
  * Copyright (c) 2015      Mellanox Technologies, Inc.
  *                         All rights reserved.
  * $COPYRIGHT$
@@ -232,6 +232,8 @@ typedef struct event pmix_event_t;
 
 #define pmix_event_base_free(b) event_base_free(b)
 
+#define pmix_event_free(x) event_free(x)
+
 #define pmix_event_base_loopbreak(b) event_base_loopbreak(b)
 
 #define pmix_event_base_loopexit(b) event_base_loopexit(b, NULL)

opal/mca/pmix/pmix3x/pmix/src/mca/base/Makefile.am

@@ -10,7 +10,7 @@
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
 # Copyright (c) 2010-2016 Cisco Systems, Inc.  All rights reserved.
-# Copyright (c) 2017      Intel, Inc. All rights reserved.
+# Copyright (c) 2017-2018 Intel, Inc. All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -26,7 +26,7 @@ AM_CPPFLAGS = \
 
 noinst_LTLIBRARIES = libpmix_mca_base.la
 
-dist_pmixdata_DATA = help-mca-base.txt help-mca-var.txt
+dist_pmixdata_DATA = help-pmix-mca-base.txt help-pmix-mca-var.txt
 
 # Source code files
 

opal/mca/pmix/pmix3x/pmix/src/mca/base/help-pmix-mca-base.txt

@@ -11,6 +11,7 @@
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
 # Copyright (c) 2008-2014 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2018      Intel, Inc. All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow

opal/mca/pmix/pmix3x/pmix/src/mca/base/help-pmix-mca-var.txt

@@ -13,6 +13,7 @@
 # Copyright (c) 2008-2011 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2013      Los Alamos National Security, LLC. All rights
 #                         reserved.
+# Copyright (c) 2018      Intel, Inc. All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow

opal/mca/pmix/pmix3x/pmix/src/mca/base/pmix_mca_base_component_find.c

@@ -16,7 +16,7 @@
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2014-2015 Los Alamos National Security, LLC. All rights
  *                         reserved.
- * Copyright (c) 2016      Intel, Inc. All rights reserved.
+ * Copyright (c) 2016-2018 Intel, Inc. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -341,7 +341,7 @@ static int component_find_check (pmix_mca_base_framework_t *framework, char **re
         if (!found) {
             char h[MAXHOSTNAMELEN];
             gethostname(h, sizeof(h));
-            pmix_show_help("help-mca-base.txt",
+            pmix_show_help("help-pmix-mca-base.txt",
                            "find-available:not-valid", true,
                            h, framework->framework_name, requested_component_names[i]);
             return PMIX_ERR_NOT_FOUND;
@@ -375,7 +375,7 @@ int pmix_mca_base_component_parse_requested (const char *requested, bool *includ
     /* Double check to ensure that the user did not specify the negate
        character anywhere else in the value. */
     if (NULL != strstr (requested, negate)) {
-        pmix_show_help("help-mca-base.txt",
+        pmix_show_help("help-pmix-mca-base.txt",
                        "framework-param:too-many-negates",
                        true, requested_orig);
         return PMIX_ERROR;

opal/mca/pmix/pmix3x/pmix/src/mca/base/pmix_mca_base_var.c

@@ -330,7 +330,7 @@ static void process_env_list(char *env_list, char ***argv, char sep)
         if (NULL == (ptr = strchr(tokens[i], '='))) {
             value = getenv(tokens[i]);
             if (NULL == value) {
-                pmix_show_help("help-mca-var.txt", "incorrect-env-list-param",
+                pmix_show_help("help-pmix-mca-var.txt", "incorrect-env-list-param",
                                true, tokens[i], env_list);
                 break;
             }
@@ -368,7 +368,7 @@ int pmix_mca_base_var_process_env_list(char ***argv)
         if (1 == strlen(pmix_mca_base_env_list_sep)) {
             sep = pmix_mca_base_env_list_sep[0];
         } else {
-            pmix_show_help("help-mca-var.txt", "incorrect-env-list-sep",
+            pmix_show_help("help-pmix-mca-var.txt", "incorrect-env-list-sep",
                     true, pmix_mca_base_env_list_sep);
             return PMIX_SUCCESS;
         }
@@ -742,11 +742,11 @@ static int var_set_from_string (pmix_mca_base_var_t *var, char *src)
             if (var->mbv_enumerator) {
                 char *valid_values;
                 (void) var->mbv_enumerator->dump(var->mbv_enumerator, &valid_values);
-                pmix_show_help("help-mca-var.txt", "invalid-value-enum",
+                pmix_show_help("help-pmix-mca-var.txt", "invalid-value-enum",
                                true, var->mbv_full_name, src, valid_values);
                 free(valid_values);
             } else {
-                pmix_show_help("help-mca-var.txt", "invalid-value",
+                pmix_show_help("help-pmix-mca-var.txt", "invalid-value",
                                true, var->mbv_full_name, src);
             }
 
@@ -1229,7 +1229,7 @@ static int fixup_files(char **file_list, char * path, bool rel_path_search, char
         }
 
         if (NULL == tmp_file) {
-            pmix_show_help("help-mca-var.txt", "missing-param-file",
+            pmix_show_help("help-pmix-mca-var.txt", "missing-param-file",
                            true, getpid(), files[i], msg_path);
             exit_status = PMIX_ERROR;
             break;
@@ -1386,7 +1386,7 @@ static int register_variable (const char *project_name, const char *framework_na
         /* Read-only and constant variables can't be settable */
         if (scope < PMIX_MCA_BASE_VAR_SCOPE_LOCAL || (flags & PMIX_MCA_BASE_VAR_FLAG_DEFAULT_ONLY)) {
             if ((flags & PMIX_MCA_BASE_VAR_FLAG_DEFAULT_ONLY) && (flags & PMIX_MCA_BASE_VAR_FLAG_SETTABLE)) {
-                pmix_show_help("help-mca-var.txt", "invalid-flag-combination",
+                pmix_show_help("help-pmix-mca-var.txt", "invalid-flag-combination",
                                true, "PMIX_MCA_BASE_VAR_FLAG_DEFAULT_ONLY", "PMIX_MCA_BASE_VAR_FLAG_SETTABLE");
                 return PMIX_ERROR;
             }
@@ -1476,7 +1476,7 @@ static int register_variable (const char *project_name, const char *framework_na
         if (0 != compare_strings(framework_name, group->group_framework) ||
             0 != compare_strings(component_name, group->group_component) ||
             0 != compare_strings(variable_name, var->mbv_variable_name)) {
-            pmix_show_help("help-mca-var.txt", "var-name-conflict",
+            pmix_show_help("help-pmix-mca-var.txt", "var-name-conflict",
                            true, var->mbv_full_name, framework_name,
                            component_name, variable_name,
                            group->group_framework, group->group_component,
@@ -1488,7 +1488,7 @@ static int register_variable (const char *project_name, const char *framework_na
 
         if (var->mbv_type != type) {
 #if PMIX_ENABLE_DEBUG
-            pmix_show_help("help-mca-var.txt",
+            pmix_show_help("help-pmix-mca-var.txt",
                            "re-register-with-different-type",
                            true, var->mbv_full_name);
 #endif
@@ -1660,7 +1660,7 @@ static int var_set_from_env (pmix_mca_base_var_t *var, pmix_mca_base_var_t *orig
     /* we found an environment variable but this variable is default-only. print
        a warning. */
     if (PMIX_VAR_IS_DEFAULT_ONLY(original[0])) {
-        pmix_show_help("help-mca-var.txt", "default-only-param-set",
+        pmix_show_help("help-pmix-mca-var.txt", "default-only-param-set",
                        true, var_full_name);
 
         return PMIX_ERR_NOT_FOUND;
@@ -1668,7 +1668,7 @@ static int var_set_from_env (pmix_mca_base_var_t *var, pmix_mca_base_var_t *orig
 
     if (PMIX_MCA_BASE_VAR_SOURCE_OVERRIDE == original->mbv_source) {
         if (!pmix_mca_base_var_suppress_override_warning) {
-            pmix_show_help("help-mca-var.txt", "overridden-param-set",
+            pmix_show_help("help-pmix-mca-var.txt", "overridden-param-set",
                            true, var_full_name);
         }
 
@@ -1699,16 +1699,16 @@ static int var_set_from_env (pmix_mca_base_var_t *var, pmix_mca_base_var_t *orig
 
         switch (var->mbv_source) {
         case PMIX_MCA_BASE_VAR_SOURCE_ENV:
-            pmix_show_help("help-mca-var.txt", "deprecated-mca-env",
+            pmix_show_help("help-pmix-mca-var.txt", "deprecated-mca-env",
                            true, var_full_name, new_variable);
             break;
         case PMIX_MCA_BASE_VAR_SOURCE_COMMAND_LINE:
-            pmix_show_help("help-mca-var.txt", "deprecated-mca-cli",
+            pmix_show_help("help-pmix-mca-var.txt", "deprecated-mca-cli",
                            true, var_full_name, new_variable);
             break;
         case PMIX_MCA_BASE_VAR_SOURCE_FILE:
         case PMIX_MCA_BASE_VAR_SOURCE_OVERRIDE:
-            pmix_show_help("help-mca-var.txt", "deprecated-mca-file",
+            pmix_show_help("help-pmix-mca-var.txt", "deprecated-mca-file",
                            true, var_full_name, pmix_mca_base_var_source_file (var),
                            new_variable);
             break;
@@ -1747,14 +1747,14 @@ static int var_set_from_file (pmix_mca_base_var_t *var, pmix_mca_base_var_t *ori
 
         /* found it */
         if (PMIX_VAR_IS_DEFAULT_ONLY(var[0])) {
-            pmix_show_help("help-mca-var.txt", "default-only-param-set",
+            pmix_show_help("help-pmix-mca-var.txt", "default-only-param-set",
                            true, var_full_name);
 
             return PMIX_ERR_NOT_FOUND;
         }
 
         if (PMIX_MCA_BASE_VAR_FLAG_ENVIRONMENT_ONLY & original->mbv_flags) {
-            pmix_show_help("help-mca-var.txt", "environment-only-param",
+            pmix_show_help("help-pmix-mca-var.txt", "environment-only-param",
                            true, var_full_name, fv->mbvfv_value,
                            fv->mbvfv_file);
 
@@ -1763,7 +1763,7 @@ static int var_set_from_file (pmix_mca_base_var_t *var, pmix_mca_base_var_t *ori
 
         if (PMIX_MCA_BASE_VAR_SOURCE_OVERRIDE == original->mbv_source) {
             if (!pmix_mca_base_var_suppress_override_warning) {
-                pmix_show_help("help-mca-var.txt", "overridden-param-set",
+                pmix_show_help("help-pmix-mca-var.txt", "overridden-param-set",
                                true, var_full_name);
             }
 
@@ -1777,7 +1777,7 @@ static int var_set_from_file (pmix_mca_base_var_t *var, pmix_mca_base_var_t *ori
                 new_variable = original->mbv_full_name;
             }
 
-            pmix_show_help("help-mca-var.txt", "deprecated-mca-file",
+            pmix_show_help("help-pmix-mca-var.txt", "deprecated-mca-file",
                            true, var_full_name, fv->mbvfv_file,
                            new_variable);
         }
@@ -2041,7 +2041,7 @@ int pmix_mca_base_var_check_exclusive (const char *project,
         str_b = source_name(var_b);
 
         /* Print it all out */
-        pmix_show_help("help-mca-var.txt",
+        pmix_show_help("help-pmix-mca-var.txt",
                        "mutually-exclusive-vars",
                        true, var_a->mbv_full_name,
                        str_a, var_b->mbv_full_name,

opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/base.h

@@ -11,7 +11,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2012      Los Alamos National Security, Inc.  All rights reserved.
- * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc.  All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$

opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/bfrop_base_copy.c

@@ -9,7 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc.  All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
@@ -367,18 +367,23 @@ pmix_status_t pmix_bfrops_base_copy_pinfo(pmix_proc_info_t **dest,
                                           pmix_proc_info_t *src,
                                           pmix_data_type_t type)
 {
-    *dest = (pmix_proc_info_t*)malloc(sizeof(pmix_proc_info_t));
-    (void)strncpy((*dest)->proc.nspace, src->proc.nspace, PMIX_MAX_NSLEN);
-    (*dest)->proc.rank = src->proc.rank;
+    pmix_proc_info_t *p;
+
+    PMIX_PROC_INFO_CREATE(p, 1);
+    if (NULL == p) {
+        return PMIX_ERR_NOMEM;
+    }
+    memcpy(&p->proc, &src->proc, sizeof(pmix_proc_t));
     if (NULL != src->hostname) {
-        (*dest)->hostname = strdup(src->hostname);
+        p->hostname = strdup(src->hostname);
     }
     if (NULL != src->executable_name) {
-        (*dest)->executable_name = strdup(src->executable_name);
+        p->executable_name = strdup(src->executable_name);
     }
-    (*dest)->pid = src->pid;
-    (*dest)->exit_code = src->exit_code;
-    (*dest)->state = src->state;
+    memcpy(&p->pid, &src->pid, sizeof(pid_t));
+    memcpy(&p->exit_code, &src->exit_code, sizeof(int));
+    memcpy(&p->state, &src->state, sizeof(pmix_proc_state_t));
+    *dest = p;
     return PMIX_SUCCESS;
 }
 
@@ -619,7 +624,7 @@ pmix_status_t pmix_bfrops_base_copy_darray(pmix_data_array_t **dest,
             p1 = (pmix_info_t*)p->array;
             s1 = (pmix_info_t*)src->array;
             for (n=0; n < src->size; n++) {
-                PMIX_INFO_LOAD(&p1[n], s1[n].key, &s1[n].value.data.flag, s1[n].value.type);
+                PMIX_INFO_XFER(&p1[n], &s1[n]);
             }
             break;
         case PMIX_PDATA:
@@ -631,7 +636,7 @@ pmix_status_t pmix_bfrops_base_copy_darray(pmix_data_array_t **dest,
             pd = (pmix_pdata_t*)p->array;
             sd = (pmix_pdata_t*)src->array;
             for (n=0; n < src->size; n++) {
-                PMIX_PDATA_LOAD(&pd[n], &sd[n].proc, sd[n].key, &sd[n].value.data.flag, sd[n].value.type);
+                PMIX_PDATA_XFER(&pd[n], &sd[n]);
             }
             break;
         case PMIX_BUFFER:
@@ -901,7 +906,7 @@ pmix_status_t pmix_bfrops_base_copy_envar(pmix_envar_t **dest,
                                           pmix_envar_t *src,
                                           pmix_data_type_t type)
 {
-    *dest = (pmix_envar_t*)malloc(sizeof(pmix_envar_t));
+    PMIX_ENVAR_CREATE(*dest, 1);
     if (NULL == (*dest)) {
         return PMIX_ERR_NOMEM;
     }

opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/bfrop_base_fns.c

@@ -9,7 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2015-2018 Intel, Inc. All rights reserved.
+ * Copyright (c) 2015-2018 Intel, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -38,6 +38,13 @@ PMIX_EXPORT void pmix_value_load(pmix_value_t *v, const void *data,
     pmix_bfrops_base_value_load(v, data, type);
 }
 
+PMIX_EXPORT pmix_status_t pmix_value_unload(pmix_value_t *kv,
+                                            void **data,
+                                            size_t *sz)
+{
+    return pmix_bfrops_base_value_unload(kv, data, sz);
+}
+
 PMIX_EXPORT pmix_status_t pmix_value_xfer(pmix_value_t *dest,
                                           pmix_value_t *src)
 {
@@ -50,6 +57,8 @@ void pmix_bfrops_base_value_load(pmix_value_t *v, const void *data,
     pmix_byte_object_t *bo;
     pmix_proc_info_t *pi;
     pmix_envar_t *envar;
+    pmix_data_array_t *darray;
+    pmix_status_t rc;
 
     v->type = type;
     if (NULL == data) {
@@ -173,7 +182,7 @@ void pmix_bfrops_base_value_load(pmix_value_t *v, const void *data,
             memcpy(&(v->data.pinfo->exit_code), &pi->exit_code, sizeof(int));
             break;
         case PMIX_POINTER:
-            memcpy(&(v->data.ptr), data, sizeof(void*));
+            v->data.ptr = (void*)data;
             break;
         case PMIX_ENVAR:
             envar = (pmix_envar_t*)data;
@@ -185,6 +194,14 @@ void pmix_bfrops_base_value_load(pmix_value_t *v, const void *data,
             }
             v->data.envar.separator = envar->separator;
             break;
+        case PMIX_DATA_ARRAY:
+            darray = (pmix_data_array_t*)data;
+            rc = pmix_bfrops_base_copy_darray(&v->data.darray, darray, PMIX_DATA_ARRAY);
+            if (PMIX_SUCCESS != rc) {
+                PMIX_ERROR_LOG(rc);
+            }
+            break;
+
         default:
             /* silence warnings */
             break;
@@ -199,6 +216,7 @@ pmix_status_t pmix_bfrops_base_value_unload(pmix_value_t *kv,
 {
     pmix_status_t rc;
     pmix_envar_t *envar;
+    pmix_data_array_t **darray;
 
     rc = PMIX_SUCCESS;
     if (NULL == data ||
@@ -313,9 +331,14 @@ pmix_status_t pmix_bfrops_base_value_unload(pmix_value_t *kv,
             *sz = sizeof(pmix_proc_state_t);
             break;
         case PMIX_POINTER:
-            memcpy(*data, &(kv->data.ptr), sizeof(void*));
+            *data = (void*)kv->data.ptr;
             *sz = sizeof(void*);
             break;
+        case PMIX_DATA_ARRAY:
+            darray = (pmix_data_array_t**)data;
+            rc = pmix_bfrops_base_copy_darray(darray, kv->data.darray, PMIX_DATA_ARRAY);
+            *sz = sizeof(pmix_data_array_t);
+            break;
         case PMIX_ENVAR:
             PMIX_ENVAR_CREATE(envar, 1);
             if (NULL == envar) {
@@ -345,6 +368,7 @@ pmix_value_cmp_t pmix_bfrops_base_value_cmp(pmix_value_t *p,
                                             pmix_value_t *p1)
 {
     pmix_value_cmp_t rc = PMIX_VALUE1_GREATER;
+    int ret;
 
     if (p->type != p1->type) {
         return rc;
@@ -441,10 +465,10 @@ pmix_value_cmp_t pmix_bfrops_base_value_cmp(pmix_value_t *p,
                 if (NULL == p1->data.envar.envar) {
                     return PMIX_VALUE1_GREATER;
                 }
-                rc = strcmp(p->data.envar.envar, p1->data.envar.envar);
-                if (rc < 0) {
+                ret = strcmp(p->data.envar.envar, p1->data.envar.envar);
+                if (ret < 0) {
                     return PMIX_VALUE2_GREATER;
-                } else if (0 < rc) {
+                } else if (0 < ret) {
                     return PMIX_VALUE1_GREATER;
                 }
             } else if (NULL != p1->data.envar.envar) {
@@ -457,10 +481,10 @@ pmix_value_cmp_t pmix_bfrops_base_value_cmp(pmix_value_t *p,
                 if (NULL == p1->data.envar.value) {
                     return PMIX_VALUE1_GREATER;
                 }
-                rc = strcmp(p->data.envar.value, p1->data.envar.value);
-                if (rc < 0) {
+                ret = strcmp(p->data.envar.value, p1->data.envar.value);
+                if (ret < 0) {
                     return PMIX_VALUE2_GREATER;
-                } else if (0 < rc) {
+                } else if (0 < ret) {
                     return PMIX_VALUE1_GREATER;
                 }
             } else if (NULL != p1->data.envar.value) {
@@ -487,20 +511,8 @@ pmix_value_cmp_t pmix_bfrops_base_value_cmp(pmix_value_t *p,
 pmix_status_t pmix_bfrops_base_value_xfer(pmix_value_t *p,
                                           pmix_value_t *src)
 {
-    size_t n, m;
-    pmix_status_t rc;
-    char **prarray, **strarray;
-    pmix_value_t *pv, *sv;
+    size_t n;
     pmix_info_t *p1, *s1;
-    pmix_app_t *pa, *sa;
-    pmix_pdata_t *pd, *sd;
-    pmix_buffer_t *pb, *sb;
-    pmix_byte_object_t *pbo, *sbo;
-    pmix_kval_t *pk, *sk;
-    pmix_modex_data_t *pm, *sm;
-    pmix_proc_info_t *pi, *si;
-    pmix_query_t *pq, *sq;
-    pmix_envar_t *pe, *se;
 
     /* copy the right field */
     p->type = src->type;
@@ -614,411 +626,11 @@ pmix_status_t pmix_bfrops_base_value_xfer(pmix_value_t *p,
         memcpy(&p->data.state, &src->data.state, sizeof(pmix_proc_state_t));
         break;
     case PMIX_PROC_INFO:
-        PMIX_PROC_INFO_CREATE(p->data.pinfo, 1);
-        if (NULL != src->data.pinfo->hostname) {
-            p->data.pinfo->hostname = strdup(src->data.pinfo->hostname);
-        }
-        if (NULL != src->data.pinfo->executable_name) {
-            p->data.pinfo->executable_name = strdup(src->data.pinfo->executable_name);
-        }
-        memcpy(&p->data.pinfo->pid, &src->data.pinfo->pid, sizeof(pid_t));
-        memcpy(&p->data.pinfo->exit_code, &src->data.pinfo->exit_code, sizeof(int));
-        memcpy(&p->data.pinfo->state, &src->data.pinfo->state, sizeof(pmix_proc_state_t));
-        break;
+        return pmix_bfrops_base_copy_pinfo(&p->data.pinfo, src->data.pinfo, PMIX_PROC_INFO);
     case PMIX_DATA_ARRAY:
-        p->data.darray = (pmix_data_array_t*)calloc(1, sizeof(pmix_data_array_t));
-        p->data.darray->type = src->data.darray->type;
-        p->data.darray->size = src->data.darray->size;
-        if (0 == p->data.darray->size || NULL == src->data.darray->array) {
-            p->data.darray->array = NULL;
-            p->data.darray->size = 0;
-            break;
-        }
-        /* allocate space and do the copy */
-        switch (src->data.darray->type) {
-            case PMIX_UINT8:
-            case PMIX_INT8:
-            case PMIX_BYTE:
-                p->data.darray->array = (char*)malloc(src->data.darray->size);
-                if (NULL == p->data.darray->array) {
-                    return PMIX_ERR_NOMEM;
-                }
-                memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size);
-                break;
-            case PMIX_UINT16:
-            case PMIX_INT16:
-                p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(uint16_t));
-                if (NULL == p->data.darray->array) {
-                    return PMIX_ERR_NOMEM;
-                }
-                memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(uint16_t));
-                break;
-            case PMIX_UINT32:
-            case PMIX_INT32:
-                p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(uint32_t));
-                if (NULL == p->data.darray->array) {
-                    return PMIX_ERR_NOMEM;
-                }
-                memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(uint32_t));
-                break;
-            case PMIX_UINT64:
-            case PMIX_INT64:
-                p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(uint64_t));
-                if (NULL == p->data.darray->array) {
-                    return PMIX_ERR_NOMEM;
-                }
-                memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(uint64_t));
-                break;
-            case PMIX_BOOL:
-                p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(bool));
-                if (NULL == p->data.darray->array) {
-                    return PMIX_ERR_NOMEM;
-                }
-                memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(bool));
-                break;
-            case PMIX_SIZE:
-                p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(size_t));
-                if (NULL == p->data.darray->array) {
-                    return PMIX_ERR_NOMEM;
-                }
-                memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(size_t));
-                break;
-            case PMIX_PID:
-                p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(pid_t));
-                if (NULL == p->data.darray->array) {
-                    return PMIX_ERR_NOMEM;
-                }
-                memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pid_t));
-                break;
-            case PMIX_STRING:
-                p->data.darray->array = (char**)malloc(src->data.darray->size * sizeof(char*));
-                if (NULL == p->data.darray->array) {
-                    return PMIX_ERR_NOMEM;
-                }
-                prarray = (char**)p->data.darray->array;
-                strarray = (char**)src->data.darray->array;
-                for (n=0; n < src->data.darray->size; n++) {
-                    if (NULL != strarray[n]) {
-                        prarray[n] = strdup(strarray[n]);
-                    }
-                }
-                break;
-            case PMIX_INT:
-            case PMIX_UINT:
-                p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(int));
-                if (NULL == p->data.darray->array) {
-                    return PMIX_ERR_NOMEM;
-                }
-                memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(int));
-                break;
-            case PMIX_FLOAT:
-                p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(float));
-                if (NULL == p->data.darray->array) {
-                    return PMIX_ERR_NOMEM;
-                }
-                memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(float));
-                break;
-            case PMIX_DOUBLE:
-                p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(double));
-                if (NULL == p->data.darray->array) {
-                    return PMIX_ERR_NOMEM;
-                }
-                memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(double));
-                break;
-            case PMIX_TIMEVAL:
-                p->data.darray->array = (struct timeval*)malloc(src->data.darray->size * sizeof(struct timeval));
-                if (NULL == p->data.darray->array) {
-                    return PMIX_ERR_NOMEM;
-                }
-                memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(struct timeval));
-                break;
-            case PMIX_TIME:
-                p->data.darray->array = (time_t*)malloc(src->data.darray->size * sizeof(time_t));
-                if (NULL == p->data.darray->array) {
-                    return PMIX_ERR_NOMEM;
-                }
-                memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(time_t));
-                break;
-            case PMIX_STATUS:
-                p->data.darray->array = (pmix_status_t*)malloc(src->data.darray->size * sizeof(pmix_status_t));
-                if (NULL == p->data.darray->array) {
-                    return PMIX_ERR_NOMEM;
-                }
-                memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pmix_status_t));
-                break;
-            case PMIX_VALUE:
-                PMIX_VALUE_CREATE(p->data.darray->array, src->data.darray->size);
-                if (NULL == p->data.darray->array) {
-                    return PMIX_ERR_NOMEM;
-                }
-                pv = (pmix_value_t*)p->data.darray->array;
-                sv = (pmix_value_t*)src->data.darray->array;
-                for (n=0; n < src->data.darray->size; n++) {
-                    if (PMIX_SUCCESS != (rc = pmix_value_xfer(&pv[n], &sv[n]))) {
-                        PMIX_VALUE_FREE(pv, src->data.darray->size);
-                        return rc;
-                    }
-                }
-                break;
-            case PMIX_PROC:
-                PMIX_PROC_CREATE(p->data.darray->array, src->data.darray->size);
-                if (NULL == p->data.darray->array) {
-                    return PMIX_ERR_NOMEM;
-                }
-                memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pmix_proc_t));
-                break;
-            case PMIX_APP:
-                PMIX_APP_CREATE(p->data.darray->array, src->data.darray->size);
-                if (NULL == p->data.darray->array) {
-                    return PMIX_ERR_NOMEM;
-                }
-                pa = (pmix_app_t*)p->data.darray->array;
-                sa = (pmix_app_t*)src->data.darray->array;
-                for (n=0; n < src->data.darray->size; n++) {
-                    if (NULL != sa[n].cmd) {
-                        pa[n].cmd = strdup(sa[n].cmd);
-                    }
-                    if (NULL != sa[n].argv) {
-                        pa[n].argv = pmix_argv_copy(sa[n].argv);
-                    }
-                    if (NULL != sa[n].env) {
-                        pa[n].env = pmix_argv_copy(sa[n].env);
-                    }
-                    if (NULL != sa[n].cwd) {
-                        pa[n].cwd = strdup(sa[n].cwd);
-                    }
-                    pa[n].maxprocs = sa[n].maxprocs;
-                    if (0 < sa[n].ninfo && NULL != sa[n].info) {
-                        PMIX_INFO_CREATE(pa[n].info, sa[n].ninfo);
-                        if (NULL == pa[n].info) {
-                            PMIX_APP_FREE(pa, src->data.darray->size);
-                            return PMIX_ERR_NOMEM;
-                        }
-                        pa[n].ninfo = sa[n].ninfo;
-                        for (m=0; m < pa[n].ninfo; m++) {
-                            PMIX_INFO_XFER(&pa[n].info[m], &sa[n].info[m]);
-                        }
-                    }
-                }
-                break;
-            case PMIX_INFO:
-                PMIX_INFO_CREATE(p->data.darray->array, src->data.darray->size);
-                p1 = (pmix_info_t*)p->data.darray->array;
-                s1 = (pmix_info_t*)src->data.darray->array;
-                for (n=0; n < src->data.darray->size; n++) {
-                    PMIX_INFO_XFER(&p1[n], &s1[n]);
-                }
-                break;
-            case PMIX_PDATA:
-                PMIX_PDATA_CREATE(p->data.darray->array, src->data.darray->size);
-                if (NULL == p->data.darray->array) {
-                    return PMIX_ERR_NOMEM;
-                }
-                pd = (pmix_pdata_t*)p->data.darray->array;
-                sd = (pmix_pdata_t*)src->data.darray->array;
-                for (n=0; n < src->data.darray->size; n++) {
-                    PMIX_PDATA_XFER(&pd[n], &sd[n]);
-                }
-                break;
-            case PMIX_BUFFER:
-                p->data.darray->array = (pmix_buffer_t*)malloc(src->data.darray->size * sizeof(pmix_buffer_t));
-                if (NULL == p->data.darray->array) {
-                    return PMIX_ERR_NOMEM;
-                }
-                pb = (pmix_buffer_t*)p->data.darray->array;
-                sb = (pmix_buffer_t*)src->data.darray->array;
-                for (n=0; n < src->data.darray->size; n++) {
-                    PMIX_CONSTRUCT(&pb[n], pmix_buffer_t);
-                    pmix_bfrops_base_copy_payload(&pb[n], &sb[n]);
-                }
-                break;
-            case PMIX_BYTE_OBJECT:
-            case PMIX_COMPRESSED_STRING:
-                p->data.darray->array = (pmix_byte_object_t*)malloc(src->data.darray->size * sizeof(pmix_byte_object_t));
-                if (NULL == p->data.darray->array) {
-                    return PMIX_ERR_NOMEM;
-                }
-                pbo = (pmix_byte_object_t*)p->data.darray->array;
-                sbo = (pmix_byte_object_t*)src->data.darray->array;
-                for (n=0; n < src->data.darray->size; n++) {
-                    if (NULL != sbo[n].bytes && 0 < sbo[n].size) {
-                        pbo[n].size = sbo[n].size;
-                        pbo[n].bytes = (char*)malloc(pbo[n].size);
-                        memcpy(pbo[n].bytes, sbo[n].bytes, pbo[n].size);
-                    } else {
-                        pbo[n].bytes = NULL;
-                        pbo[n].size = 0;
-                    }
-                }
-                break;
-            case PMIX_KVAL:
-                p->data.darray->array = (pmix_kval_t*)calloc(src->data.darray->size , sizeof(pmix_kval_t));
-                if (NULL == p->data.darray->array) {
-                    return PMIX_ERR_NOMEM;
-                }
-                pk = (pmix_kval_t*)p->data.darray->array;
-                sk = (pmix_kval_t*)src->data.darray->array;
-                for (n=0; n < src->data.darray->size; n++) {
-                    if (NULL != sk[n].key) {
-                        pk[n].key = strdup(sk[n].key);
-                    }
-                    if (NULL != sk[n].value) {
-                        PMIX_VALUE_CREATE(pk[n].value, 1);
-                        if (NULL == pk[n].value) {
-                            free(p->data.darray->array);
-                            return PMIX_ERR_NOMEM;
-                        }
-                        if (PMIX_SUCCESS != (rc = pmix_value_xfer(pk[n].value, sk[n].value))) {
-                            return rc;
-                        }
-                    }
-                }
-                break;
-            case PMIX_MODEX:
-                PMIX_MODEX_CREATE(p->data.darray->array, src->data.darray->size);
-                if (NULL == p->data.darray->array) {
-                    return PMIX_ERR_NOMEM;
-                }
-                pm = (pmix_modex_data_t*)p->data.darray->array;
-                sm = (pmix_modex_data_t*)src->data.darray->array;
-                for (n=0; n < src->data.darray->size; n++) {
-                    memcpy(&pm[n], &sm[n], sizeof(pmix_modex_data_t));
-                    if (NULL != sm[n].blob && 0 < sm[n].size) {
-                        pm[n].blob = (uint8_t*)malloc(sm[n].size);
-                        if (NULL == pm[n].blob) {
-                            return PMIX_ERR_NOMEM;
-                        }
-                        memcpy(pm[n].blob, sm[n].blob, sm[n].size);
-                        pm[n].size = sm[n].size;
-                    } else {
-                        pm[n].blob = NULL;
-                        pm[n].size = 0;
-                    }
-                }
-                break;
-            case PMIX_PERSIST:
-                p->data.darray->array = (pmix_persistence_t*)malloc(src->data.darray->size * sizeof(pmix_persistence_t));
-                if (NULL == p->data.darray->array) {
-                    return PMIX_ERR_NOMEM;
-                }
-                memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pmix_persistence_t));
-                break;
-            case PMIX_POINTER:
-                p->data.darray->array = (char**)malloc(src->data.darray->size * sizeof(char*));
-                if (NULL == p->data.darray->array) {
-                    return PMIX_ERR_NOMEM;
-                }
-                prarray = (char**)p->data.darray->array;
-                strarray = (char**)src->data.darray->array;
-                for (n=0; n < src->data.darray->size; n++) {
-                    prarray[n] = strarray[n];
-                }
-                break;
-            case PMIX_SCOPE:
-                p->data.darray->array = (pmix_scope_t*)malloc(src->data.darray->size * sizeof(pmix_scope_t));
-                if (NULL == p->data.darray->array) {
-                    return PMIX_ERR_NOMEM;
-                }
-                memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pmix_scope_t));
-                break;
-            case PMIX_DATA_RANGE:
-                p->data.darray->array = (pmix_data_range_t*)malloc(src->data.darray->size * sizeof(pmix_data_range_t));
-                if (NULL == p->data.darray->array) {
-                    return PMIX_ERR_NOMEM;
-                }
-                memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pmix_data_range_t));
-                break;
-            case PMIX_COMMAND:
-                p->data.darray->array = (pmix_cmd_t*)malloc(src->data.darray->size * sizeof(pmix_cmd_t));
-                if (NULL == p->data.darray->array) {
-                    return PMIX_ERR_NOMEM;
-                }
-                memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pmix_cmd_t));
-                break;
-            case PMIX_INFO_DIRECTIVES:
-                p->data.darray->array = (pmix_info_directives_t*)malloc(src->data.darray->size * sizeof(pmix_info_directives_t));
-                if (NULL == p->data.darray->array) {
-                    return PMIX_ERR_NOMEM;
-                }
-                memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pmix_info_directives_t));
-                break;
-            case PMIX_PROC_INFO:
-                PMIX_PROC_INFO_CREATE(p->data.darray->array, src->data.darray->size);
-                if (NULL == p->data.darray->array) {
-                    return PMIX_ERR_NOMEM;
-                }
-                pi = (pmix_proc_info_t*)p->data.darray->array;
-                si = (pmix_proc_info_t*)src->data.darray->array;
-                for (n=0; n < src->data.darray->size; n++) {
-                    memcpy(&pi[n].proc, &si[n].proc, sizeof(pmix_proc_t));
-                    if (NULL != si[n].hostname) {
-                        pi[n].hostname = strdup(si[n].hostname);
-                    } else {
-                        pi[n].hostname = NULL;
-                    }
-                    if (NULL != si[n].executable_name) {
-                        pi[n].executable_name = strdup(si[n].executable_name);
-                    } else {
-                        pi[n].executable_name = NULL;
-                    }
-                    pi[n].pid = si[n].pid;
-                    pi[n].exit_code = si[n].exit_code;
-                    pi[n].state = si[n].state;
-                }
-                break;
-            case PMIX_DATA_ARRAY:
-                PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED);
-                return PMIX_ERR_NOT_SUPPORTED;  // don't support iterative arrays
-            case PMIX_QUERY:
-                PMIX_QUERY_CREATE(p->data.darray->array, src->data.darray->size);
-                if (NULL == p->data.darray->array) {
-                    return PMIX_ERR_NOMEM;
-                }
-                pq = (pmix_query_t*)p->data.darray->array;
-                sq = (pmix_query_t*)src->data.darray->array;
-                for (n=0; n < src->data.darray->size; n++) {
-                    if (NULL != sq[n].keys) {
-                        pq[n].keys = pmix_argv_copy(sq[n].keys);
-                    }
-                    if (NULL != sq[n].qualifiers && 0 < sq[n].nqual) {
-                        PMIX_INFO_CREATE(pq[n].qualifiers, sq[n].nqual);
-                        if (NULL == pq[n].qualifiers) {
-                            PMIX_QUERY_FREE(pq, src->data.darray->size);
-                            return PMIX_ERR_NOMEM;
-                        }
-                        for (m=0; m < sq[n].nqual; m++) {
-                            PMIX_INFO_XFER(&pq[n].qualifiers[m], &sq[n].qualifiers[m]);
-                        }
-                        pq[n].nqual = sq[n].nqual;
-                    } else {
-                        pq[n].qualifiers = NULL;
-                        pq[n].nqual = 0;
-                    }
-                }
-                break;
-            case PMIX_ENVAR:
-                PMIX_ENVAR_CREATE(p->data.darray->array, src->data.darray->size);
-                if (NULL == p->data.darray->array) {
-                    return PMIX_ERR_NOMEM;
-                }
-                pe = (pmix_envar_t*)p->data.darray->array;
-                se = (pmix_envar_t*)src->data.darray->array;
-                for (n=0; n < src->data.darray->size; n++) {
-                    if (NULL != se[n].envar) {
-                        pe[n].envar = strdup(se[n].envar);
-                    }
-                    if (NULL != se[n].value) {
-                        pe[n].value = strdup(se[n].value);
-                    }
-                    pe[n].separator = se[n].separator;
-                }
-                break;
-            default:
-                return PMIX_ERR_UNKNOWN_DATA_TYPE;
-        }
-        break;
+        return pmix_bfrops_base_copy_darray(&p->data.darray, src->data.darray, PMIX_DATA_ARRAY);
     case PMIX_POINTER:
-        memcpy(&p->data.ptr, &src->data.ptr, sizeof(void*));
+        p->data.ptr = src->data.ptr;
         break;
     case PMIX_ENVAR:
         PMIX_ENVAR_CONSTRUCT(&p->data.envar);
@@ -1048,8 +660,7 @@ pmix_status_t pmix_bfrops_base_value_xfer(pmix_value_t *p,
         break;
     /********************/
     default:
-        pmix_output(0, "XFER-PMIX-VALUE: UNSUPPORTED TYPE %d", (int)src->type);
-        assert(0);
+        pmix_output(0, "PMIX-XFER-VALUE: UNSUPPORTED TYPE %d", (int)src->type);
         return PMIX_ERROR;
     }
     return PMIX_SUCCESS;

opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/bfrop_base_frame.c

@@ -12,7 +12,7 @@
  *                         All rights reserved.
  * Copyright (c) 2012-2013 Los Alamos National Security, Inc.  All rights reserved.
  * Copyright (c) 2014-2017 Intel, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
+ * Copyright (c) 2015-2018 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
  *
@@ -90,7 +90,7 @@ static pmix_status_t pmix_bfrop_close(void)
     pmix_bfrops_globals.initialized = false;
 
     /* the components will cleanup when closed */
-    PMIX_DESTRUCT(&pmix_bfrops_globals.actives);
+    PMIX_LIST_DESTRUCT(&pmix_bfrops_globals.actives);
 
     return pmix_mca_base_framework_components_close(&pmix_bfrops_base_framework, NULL);
 }

opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/bfrop_base_pack.c

@@ -454,6 +454,7 @@ pmix_status_t pmix_bfrops_base_pack_status(pmix_buffer_t *buffer, const void *sr
     for (i = 0; i < num_vals; ++i) {
         status = (int32_t)ssrc[i];
         if (PMIX_SUCCESS != (ret = pmix_bfrops_base_pack_int32(buffer, &status, 1, PMIX_INT32))) {
+            PMIX_ERROR_LOG(ret);
             return ret;
         }
     }
@@ -1310,5 +1311,5 @@ pmix_status_t pmix_bfrops_base_pack_envar(pmix_buffer_t *buffer, const void *src
             return ret;
         }
     }
-    return pmix_bfrops_base_pack_int16(buffer, src, num_vals, PMIX_UINT16);
+    return PMIX_SUCCESS;
 }

opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/bfrop_base_print.c

@@ -10,7 +10,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2012      Los Alamos National Security, Inc.  All rights reserved.
- * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc.  All rights reserved.
  * Copyright (c) 2016      Mellanox Technologies, Inc.
  *                         All rights reserved.
  *

opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/bfrop_base_unpack.c

@@ -10,7 +10,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2012      Los Alamos National Security, Inc.  All rights reserved.
- * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc.  All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2016      Mellanox Technologies, Inc.
@@ -111,6 +111,7 @@ pmix_status_t pmix_bfrops_base_unpack(pmix_pointer_array_t *regtypes,
         }
         if (PMIX_INT32 != local_type) { /* if the length wasn't first, then error */
             *num_vals = 0;
+            PMIX_ERROR_LOG(PMIX_ERR_UNPACK_FAILURE);
             return PMIX_ERR_UNPACK_FAILURE;
         }
     }
@@ -753,13 +754,13 @@ pmix_status_t pmix_bfrops_base_unpack_val(pmix_buffer_t *buffer,
                 return ret;
             }
             break;
-        case PMIX_QUERY:
-            if (PMIX_SUCCESS != (ret = pmix_bfrops_base_unpack_query(buffer, val->data.darray, &m, PMIX_QUERY))) {
+        case PMIX_ALLOC_DIRECTIVE:
+            if (PMIX_SUCCESS != (ret = pmix_bfrops_base_unpack_alloc_directive(buffer, &val->data.adir, &m, PMIX_ALLOC_DIRECTIVE))) {
                 return ret;
             }
             break;
         case PMIX_ENVAR:
-            if (PMIX_SUCCESS != (ret = pmix_bfrops_base_unpack_envar(buffer, val->data.darray, &m, PMIX_ENVAR))) {
+            if (PMIX_SUCCESS != (ret = pmix_bfrops_base_unpack_envar(buffer, &val->data.envar, &m, PMIX_ENVAR))) {
                 return ret;
             }
             break;

opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/bfrops.h

@@ -11,7 +11,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2012      Los Alamos National Security, Inc. All rights reserved.
- * Copyright (c) 2013-2017 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2013-2018 Intel, Inc. All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2016      Mellanox Technologies, Inc.
@@ -32,7 +32,7 @@
 #define PMIX_BFROP_H_
 
 #include <src/include/pmix_config.h>
-
+#include <pmix_common.h>
 #include <src/include/types.h>
 
 #include "src/mca/mca.h"
@@ -100,11 +100,6 @@ BEGIN_C_DECLS
  * NOTE: THESE FUNCTIONS ARE NOT TO BE USED INTERNALLY -
  * USE THE MACROS INSTEAD
  */
-pmix_status_t pmix_value_xfer(pmix_value_t *kv, pmix_value_t *src);
-void pmix_value_load(pmix_value_t *v, const void *data,
-                     pmix_data_type_t type);
-pmix_status_t pmix_value_unload(pmix_value_t *kv, void **data,
-                                size_t *sz, pmix_data_type_t type);
 bool pmix_value_cmp(pmix_value_t *p, pmix_value_t *p1);
 
 

opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_dstore.c

@@ -3,6 +3,8 @@
  * Copyright (c) 2016      IBM Corporation.  All rights reserved.
  * Copyright (c) 2016-2017 Mellanox Technologies, Inc.
  *                         All rights reserved.
+ * Copyright (c) 2018      Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
  *
  * $COPYRIGHT$
  *
@@ -2269,6 +2271,10 @@ static void dstore_finalize(void)
         PMIX_RELEASE(_clients_peer->nptr);
         PMIX_RELEASE(_clients_peer);
     }
+    /* close the pshmem framework */
+    if( PMIX_SUCCESS != (rc = pmix_mca_base_framework_close(&pmix_pshmem_base_framework)) ) {
+        PMIX_ERROR_LOG(rc);
+    }
 }
 
 static pmix_status_t _dstore_store(const char *nspace,

opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_dstore_component.c

@@ -12,7 +12,7 @@
  *                         All rights reserved.
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
- * Copyright (c) 2016-2017 Intel, Inc. All rights reserved.
+ * Copyright (c) 2016-2018 Intel, Inc. All rights reserved.
  * Copyright (c) 2017      Mellanox Technologies, Inc.
  *                         All rights reserved.
  * $COPYRIGHT$
@@ -31,7 +31,7 @@
 #include <src/include/pmix_config.h>
 #include "pmix_common.h"
 
-
+#include "src/include/pmix_globals.h"
 #include "src/mca/gds/gds.h"
 #include "gds_dstore.h"
 
@@ -74,6 +74,13 @@ static int component_open(void)
 
 static int component_query(pmix_mca_base_module_t **module, int *priority)
 {
+    /* launchers cannot use the dstore */
+    if (PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) {
+        *priority = 0;
+        *module = NULL;
+        return PMIX_ERROR;
+    }
+
     *priority = 20;
     *module = (pmix_mca_base_module_t *)&pmix_ds12_module;
     return PMIX_SUCCESS;

opal/mca/pmix/pmix3x/pmix/src/mca/gds/hash/gds_hash.c

@@ -1,6 +1,8 @@
 /*
  * Copyright (c) 2015-2018 Intel, Inc.  All rights reserved.
  * Copyright (c) 2016      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2018      Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
  *
  * $COPYRIGHT$
  *
@@ -135,8 +137,11 @@ static void htdes(pmix_hash_trkr_t *p)
     if (NULL != p->nptr) {
         PMIX_RELEASE(p->nptr);
     }
+    pmix_hash_remove_data(&p->internal, PMIX_RANK_WILDCARD, NULL);
     PMIX_DESTRUCT(&p->internal);
+    pmix_hash_remove_data(&p->remote, PMIX_RANK_WILDCARD, NULL);
     PMIX_DESTRUCT(&p->remote);
+    pmix_hash_remove_data(&p->local, PMIX_RANK_WILDCARD, NULL);
     PMIX_DESTRUCT(&p->local);
 }
 static PMIX_CLASS_INSTANCE(pmix_hash_trkr_t,
@@ -327,13 +332,14 @@ static pmix_status_t store_map(pmix_hash_table_t *ht,
                 pmix_argv_free(procs);
                 return rc;
             }
-            PMIX_RELEASE(kp2);
+            PMIX_RELEASE(kp2);  // maintain acctg
         }
         pmix_argv_free(procs);
     }
 
     /* store the comma-delimited list of nodes hosting
-     * procs in this nspace */
+     * procs in this nspace in case someone using PMIx v2
+     * requests it */
     kp2 = PMIX_NEW(pmix_kval_t);
     kp2->key = strdup(PMIX_NODE_LIST);
     kp2->value = (pmix_value_t*)malloc(sizeof(pmix_value_t));
@@ -344,6 +350,7 @@ static pmix_status_t store_map(pmix_hash_table_t *ht,
         PMIX_RELEASE(kp2);
         return rc;
     }
+    PMIX_RELEASE(kp2);  // maintain acctg
 
     return PMIX_SUCCESS;
 }
@@ -397,6 +404,20 @@ pmix_status_t hash_cache_job_info(struct pmix_nspace_t *ns,
     ht = &trk->internal;
     for (n=0; n < ninfo; n++) {
         if (0 == strcmp(info[n].key, PMIX_NODE_MAP)) {
+            /* store the node map itself since that is
+             * what v3 uses */
+            kp2 = PMIX_NEW(pmix_kval_t);
+            kp2->key = strdup(PMIX_NODE_MAP);
+            kp2->value = (pmix_value_t*)malloc(sizeof(pmix_value_t));
+            kp2->value->type = PMIX_STRING;
+            kp2->value->data.string = strdup(info[n].value.data.string);
+            if (PMIX_SUCCESS != (rc = pmix_hash_store(ht, PMIX_RANK_WILDCARD, kp2))) {
+                PMIX_ERROR_LOG(rc);
+                PMIX_RELEASE(kp2);
+                return rc;
+            }
+            PMIX_RELEASE(kp2);  // maintain acctg
+
             /* parse the regex to get the argv array of node names */
             if (PMIX_SUCCESS != (rc = pmix_preg.parse_nodes(info[n].value.data.string, &nodes))) {
                 PMIX_ERROR_LOG(rc);
@@ -513,6 +534,11 @@ pmix_status_t hash_cache_job_info(struct pmix_nspace_t *ns,
                 PMIX_RELEASE(kp2);
                 goto release;
             }
+            PMIX_RELEASE(kp2);  // maintain acctg
+            /* if this is the job size, then store it */
+            if (0 == strncmp(info[n].key, PMIX_JOB_SIZE, PMIX_MAX_KEYLEN)) {
+                nptr->nprocs = info[n].value.data.uint32;
+            }
         }
     }
 
@@ -538,6 +564,7 @@ pmix_status_t hash_cache_job_info(struct pmix_nspace_t *ns,
                 PMIX_RELEASE(kp2);
                 break;
             }
+            PMIX_RELEASE(kp2);  // maintain acctg
         }
         trk->gdata_added = true;
     }
@@ -560,7 +587,6 @@ static pmix_status_t register_info(pmix_peer_t *peer,
     pmix_hash_table_t *ht;
     pmix_value_t *val, blob;
     pmix_status_t rc = PMIX_SUCCESS;
-    pmix_rank_info_t *rinfo;
     pmix_info_t *info;
     size_t ninfo, n;
     pmix_kval_t kv;
@@ -607,9 +633,9 @@ static pmix_status_t register_info(pmix_peer_t *peer,
         PMIX_VALUE_RELEASE(val);
     }
 
-    PMIX_LIST_FOREACH(rinfo, &ns->ranks, pmix_rank_info_t) {
+    for (rank=0; rank < ns->nprocs; rank++) {
         val = NULL;
-        rc = pmix_hash_fetch(ht, rinfo->pname.rank, NULL, &val);
+        rc = pmix_hash_fetch(ht, rank, NULL, &val);
         if (PMIX_SUCCESS != rc) {
             PMIX_ERROR_LOG(rc);
             if (NULL != val) {
@@ -621,7 +647,6 @@ static pmix_status_t register_info(pmix_peer_t *peer,
             return PMIX_ERR_NOT_FOUND;
         }
         PMIX_CONSTRUCT(&buf, pmix_buffer_t);
-        rank = rinfo->pname.rank;
         PMIX_BFROPS_PACK(rc, peer, &buf, &rank, 1, PMIX_PROC_RANK);
 
         info = (pmix_info_t*)val->data.darray->array;
@@ -658,7 +683,8 @@ static pmix_status_t hash_register_job_info(struct pmix_peer_t *pr,
     pmix_status_t rc;
     pmix_hash_trkr_t *trk, *t2;
 
-    if (!PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) {
+    if (!PMIX_PROC_IS_SERVER(pmix_globals.mypeer) &&
+        !PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) {
         /* this function is only available on servers */
         PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED);
         return PMIX_ERR_NOT_SUPPORTED;
@@ -758,7 +784,8 @@ static pmix_status_t hash_store_job_info(const char *nspace,
                         "[%s:%u] pmix:gds:hash store job info for nspace %s",
                         pmix_globals.myid.nspace, pmix_globals.myid.rank, nspace);
 
-    if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) {
+    if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer) &&
+        !PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) {
         /* this function is NOT available on servers */
         PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED);
         return PMIX_ERR_NOT_SUPPORTED;
@@ -913,7 +940,7 @@ static pmix_status_t hash_store_job_info(const char *nspace,
                         PMIX_DESTRUCT(&buf2);
                         return rc;
                     }
-                    PMIX_RELEASE(kp2);
+                    PMIX_RELEASE(kp2);  // maintain acctg
                 } else {
                     /* nope - so add this by itself */
                     kp2 = PMIX_NEW(pmix_kval_t);
@@ -945,7 +972,7 @@ static pmix_status_t hash_store_job_info(const char *nspace,
                         PMIX_DESTRUCT(&buf2);
                         return rc;
                     }
-                    PMIX_RELEASE(kp2);
+                    PMIX_RELEASE(kp2);  // maintain acctg
                 }
                 /* split the list of procs so we can store their
                  * individual location data */
@@ -968,7 +995,7 @@ static pmix_status_t hash_store_job_info(const char *nspace,
                         pmix_argv_free(procs);
                         return rc;
                     }
-                    PMIX_RELEASE(kp2);
+                    PMIX_RELEASE(kp2);  // maintain acctg
                 }
                 pmix_argv_free(procs);
                 PMIX_DESTRUCT(&kv);
@@ -989,7 +1016,7 @@ static pmix_status_t hash_store_job_info(const char *nspace,
                     PMIX_DESTRUCT(&buf2);
                     return rc;
                 }
-                PMIX_RELEASE(kp2);
+                PMIX_RELEASE(kp2);  // maintain acctg
             }
             /* cleanup */
             PMIX_DESTRUCT(&buf2);
@@ -1044,10 +1071,10 @@ static pmix_status_t hash_store(const pmix_proc_t *proc,
     pmix_kval_t *kp;
 
     pmix_output_verbose(2, pmix_gds_base_framework.framework_output,
-                        "[%s:%d] gds:hash:hash_store for proc [%s:%d] key %s scope %s",
+                        "[%s:%d] gds:hash:hash_store for proc [%s:%d] key %s type %s scope %s",
                         pmix_globals.myid.nspace, pmix_globals.myid.rank,
                         proc->nspace, proc->rank, kv->key,
-                        PMIx_Scope_string(scope));
+                        PMIx_Data_type_string(kv->value->type), PMIx_Scope_string(scope));
 
     if (NULL == kv->key) {
         return PMIX_ERR_BAD_PARAM;
@@ -1091,8 +1118,10 @@ static pmix_status_t hash_store(const pmix_proc_t *proc,
             }
             if (PMIX_SUCCESS != (rc = pmix_hash_store(&trk->internal, proc->rank, kp))) {
                 PMIX_ERROR_LOG(rc);
+                PMIX_RELEASE(kp);
                 return rc;
             }
+            PMIX_RELEASE(kp);  // maintain accounting
         }
     }
 
@@ -1442,7 +1471,17 @@ static pmix_status_t nspace_add(const char *nspace,
 
 static pmix_status_t nspace_del(const char *nspace)
 {
-    /* we don't need to do anything here */
+    pmix_hash_trkr_t *t;
+
+    /* find the hash table for this nspace */
+    PMIX_LIST_FOREACH(t, &myhashes, pmix_hash_trkr_t) {
+        if (0 == strcmp(nspace, t->ns)) {
+            /* release it */
+            pmix_list_remove_item(&myhashes, &t->super);
+            PMIX_RELEASE(t);
+            break;
+        }
+    }
     return PMIX_SUCCESS;
 }
 

opal/mca/pmix/pmix3x/pmix/src/mca/pif/posix_ipv4/pif_posix.c

@@ -4,7 +4,7 @@
  * Copyright (c) 2013      The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
- * Copyright (c) 2015      Intel, Inc. All rights reserved.
+ * Copyright (c) 2015-2018 Intel, Inc. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -333,7 +333,8 @@ static int if_posix_open(void)
         }
         intf->ifmtu = ifr->ifr_mtu;
 #endif
-
+        pmix_output_verbose(1, pmix_pif_base_framework.framework_output,
+                            "adding interface %s", intf->if_name);
         pmix_list_append(&pmix_if_list, &(intf->super));
     }
     free(ifconf.ifc_req);

opal/mca/pmix/pmix3x/pmix/src/mca/plog/Makefile.am

@@ -0,0 +1,44 @@
+# -*- makefile -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2012      Los Alamos National Security, Inc.  All rights reserved.
+# Copyright (c) 2013-2018 Intel, Inc. All rights reserved.
+# Copyright (c) 2016      Cisco Systems, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+AM_CPPFLAGS = $(LTDLINCL)
+
+# main library setup
+noinst_LTLIBRARIES = libmca_plog.la
+libmca_plog_la_SOURCES =
+
+# local files
+headers = plog.h
+sources =
+
+# Conditionally install the header files
+if WANT_INSTALL_HEADERS
+pmixdir = $(pmixincludedir)/$(subdir)
+nobase_pmix_HEADERS = $(headers)
+endif
+
+include base/Makefile.include
+
+libmca_plog_la_SOURCES += $(headers) $(sources)
+
+distclean-local:
+	rm -f base/static-components.h

opal/mca/pmix/pmix3x/pmix/src/mca/plog/base/Makefile.include

@@ -0,0 +1,34 @@
+# -*- makefile -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2012      Los Alamos National Security, Inc.  All rights reserved.
+# Copyright (c) 2013-2018 Intel, Inc. All rights reserved.
+# Copyright (c) 2016      Cisco Systems, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# This makefile.am does not stand on its own - it is included from
+# src/Makefile.am
+
+dist_pmixdata_DATA = base/help-pmix-plog.txt
+
+headers += \
+         base/base.h
+
+sources += \
+        base/plog_base_frame.c \
+        base/plog_base_select.c \
+        base/plog_base_stubs.c

opal/mca/pmix/pmix3x/pmix/src/mca/plog/base/base.h

@@ -0,0 +1,93 @@
+/* -*- C -*-
+ *
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2006 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2012      Los Alamos National Security, Inc.  All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+#ifndef PMIX_PLOG_BASE_H_
+#define PMIX_PLOG_BASE_H_
+
+#include <src/include/pmix_config.h>
+
+
+#ifdef HAVE_SYS_TIME_H
+#include <sys/time.h> /* for struct timeval */
+#endif
+#ifdef HAVE_STRING_H
+#include <string.h>
+#endif
+
+#include "src/class/pmix_list.h"
+#include "src/class/pmix_pointer_array.h"
+#include "src/threads/threads.h"
+#include "src/mca/mca.h"
+#include "src/mca/base/pmix_mca_base_framework.h"
+
+#include "src/mca/plog/plog.h"
+
+
+BEGIN_C_DECLS
+
+/*
+ * MCA Framework
+ */
+PMIX_EXPORT extern pmix_mca_base_framework_t pmix_plog_base_framework;
+/**
+ * PLOG select function
+ *
+ * Cycle across available components and construct the array
+ * of active modules
+ */
+PMIX_EXPORT pmix_status_t pmix_plog_base_select(void);
+
+/**
+ * Track an active component / module
+ */
+struct pmix_plog_base_active_module_t {
+    pmix_list_item_t super;
+    bool reqd;
+    bool added;
+    int pri;
+    pmix_plog_module_t *module;
+    pmix_plog_base_component_t *component;
+};
+typedef struct pmix_plog_base_active_module_t pmix_plog_base_active_module_t;
+PMIX_CLASS_DECLARATION(pmix_plog_base_active_module_t);
+
+
+/* framework globals */
+struct pmix_plog_globals_t {
+    pmix_lock_t lock;
+    pmix_pointer_array_t actives;
+    bool initialized;
+    char **channels;
+};
+typedef struct pmix_plog_globals_t pmix_plog_globals_t;
+
+PMIX_EXPORT extern pmix_plog_globals_t pmix_plog_globals;
+
+PMIX_EXPORT pmix_status_t pmix_plog_base_log(const pmix_proc_t *source,
+                                             const pmix_info_t data[], size_t ndata,
+                                             const pmix_info_t directives[], size_t ndirs,
+                                             pmix_op_cbfunc_t cbfunc, void *cbdata);
+
+END_C_DECLS
+
+#endif

opal/mca/pmix/pmix3x/pmix/src/mca/plog/base/help-pmix-plog.txt

@@ -0,0 +1,56 @@
+# -*- text -*-
+#
+# Copyright (c) 2018      Intel, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+# This is a US/English help file
+#
+[reqd-not-found]
+The plog_base_order MCA parameter included a required logging
+channel that is not available:
+
+  Channel:  %s
+
+Please update the parameter and try again.
+#
+[syslog:unrec-level]
+An unrecognized syslog level was given:
+
+  Level:  %s
+
+Please see "man syslog" for a list of defined levels. Input
+parameter strings and their corresponding syslog levels
+recognized by PMIx include:
+
+  Parameter       Level
+   err           LOG_ERR  (default)
+   alert         LOG_ALERT
+   crit          LOG_CRIT
+   emerg         LOG_EMERG
+   warn          LOG_WARNING
+   not           LOG_NOTICE
+   info          LOG_INFO
+   debug         LOG_DEBUG
+
+Please redefine the MCA parameter and try again.
+#
+[syslog:unrec-facility]
+An unsupported or unrecognized value was given for the
+syslog facility (i.e., the type of program calling syslog):
+
+  Value:  %s
+
+Please see "man syslog" for a list of defined facility values.
+PMIx currently supports only the following designations:
+
+   Parameter       Level
+    auth          LOG_AUTH
+    priv          LOG_AUTHPRIV
+    daemon        LOG_DAEMON
+    user          LOG_USER  (default)
+
+Please redefine the MCA parameter and try again.

opal/mca/pmix/pmix3x/pmix/src/mca/plog/base/plog_base_frame.c

@@ -0,0 +1,105 @@
+/* -*- Mode: C; c-basic-offset:4 ; -*- */
+/*
+ * Copyright (c) 2018      Intel, Inc. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include <src/include/pmix_config.h>
+
+#include <pmix_common.h>
+
+#ifdef HAVE_STRING_H
+#include <string.h>
+#endif
+
+#include "src/class/pmix_list.h"
+#include "src/threads/threads.h"
+#include "src/util/argv.h"
+#include "src/mca/base/base.h"
+#include "src/mca/plog/base/base.h"
+
+/*
+ * The following file was created by configure.  It contains extern
+ * statements and the definition of an array of pointers to each
+ * component's public mca_base_component_t struct.
+ */
+
+#include "src/mca/plog/base/static-components.h"
+
+/* Instantiate the global vars */
+pmix_plog_globals_t pmix_plog_globals = {{0}};
+pmix_plog_API_module_t pmix_plog = {
+    .log = pmix_plog_base_log
+};
+
+static char *order = NULL;
+static int pmix_plog_register(pmix_mca_base_register_flag_t flags)
+{
+    pmix_mca_base_var_register("pmix", "plog", "base", "order",
+                               "Comma-delimited, prioritized list of logging channels",
+                               PMIX_MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
+                               PMIX_INFO_LVL_2,
+                               PMIX_MCA_BASE_VAR_SCOPE_READONLY,
+                               &order);
+    if (NULL != order) {
+        pmix_plog_globals.channels = pmix_argv_split(order, ',');
+    }
+    return PMIX_SUCCESS;
+}
+
+static pmix_status_t pmix_plog_close(void)
+{
+    pmix_plog_base_active_module_t *active;
+    int n;
+
+    if (!pmix_plog_globals.initialized) {
+        return PMIX_SUCCESS;
+    }
+    pmix_plog_globals.initialized = false;
+
+    for (n=0; n < pmix_plog_globals.actives.size; n++) {
+        if (NULL == (active = (pmix_plog_base_active_module_t*)pmix_pointer_array_get_item(&pmix_plog_globals.actives, n))) {
+            continue;
+        }
+        if (NULL != active->module->finalize) {
+          active->module->finalize();
+        }
+        PMIX_RELEASE(active);
+        pmix_pointer_array_set_item(&pmix_plog_globals.actives, n, NULL);
+    }
+    PMIX_DESTRUCT(&pmix_plog_globals.actives);
+
+    PMIX_DESTRUCT_LOCK(&pmix_plog_globals.lock);
+
+    return pmix_mca_base_framework_components_close(&pmix_plog_base_framework, NULL);
+}
+
+static pmix_status_t pmix_plog_open(pmix_mca_base_open_flag_t flags)
+{
+    /* initialize globals */
+    pmix_plog_globals.initialized = true;
+    pmix_plog_globals.channels = NULL;
+    PMIX_CONSTRUCT(&pmix_plog_globals.actives, pmix_pointer_array_t);
+    pmix_pointer_array_init(&pmix_plog_globals.actives, 1, INT_MAX, 1);
+    PMIX_CONSTRUCT_LOCK(&pmix_plog_globals.lock);
+    pmix_plog_globals.lock.active = false;
+
+    /* Open up all available components */
+    return pmix_mca_base_framework_components_open(&pmix_plog_base_framework, flags);
+}
+
+PMIX_MCA_BASE_FRAMEWORK_DECLARE(pmix, plog, "PMIx Logging Operations",
+                                pmix_plog_register, pmix_plog_open, pmix_plog_close,
+                                mca_plog_base_static_components, 0);
+
+static void acon(pmix_plog_base_active_module_t *p)
+{
+    p->reqd = false;
+    p->added = false;
+}
+PMIX_CLASS_INSTANCE(pmix_plog_base_active_module_t,
+                    pmix_list_item_t,
+                    acon, NULL);

opal/mca/pmix/pmix3x/pmix/src/mca/plog/base/plog_base_select.c

@@ -0,0 +1,199 @@
+/*
+ * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2016-2018 Intel, Inc. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include <src/include/pmix_config.h>
+#include <pmix_common.h>
+
+#include <string.h>
+
+#include "src/mca/mca.h"
+#include "src/mca/base/base.h"
+#include "src/util/show_help.h"
+
+#include "src/mca/plog/base/base.h"
+
+static bool selected = false;
+
+/* Function for selecting a prioritized array of components
+ * from all those that are available. */
+int pmix_plog_base_select(void)
+{
+    pmix_mca_base_component_list_item_t *cli = NULL;
+    pmix_mca_base_component_t *component = NULL;
+    pmix_mca_base_module_t *module = NULL;
+    pmix_plog_module_t *nmodule;
+    pmix_plog_base_active_module_t *newmodule, *mod, *default_mod = NULL;
+    int rc, priority, n;
+    bool inserted, default_added, reqd;
+    pmix_list_t actives;
+    char *ptr;
+    size_t len;
+
+    if (selected) {
+        /* ensure we don't do this twice */
+        return PMIX_SUCCESS;
+    }
+    selected = true;
+
+    PMIX_CONSTRUCT(&actives, pmix_list_t);
+
+    /* Query all available components and ask if they have a module */
+    PMIX_LIST_FOREACH(cli, &pmix_plog_base_framework.framework_components, pmix_mca_base_component_list_item_t) {
+        component = (pmix_mca_base_component_t *) cli->cli_component;
+
+        pmix_output_verbose(5, pmix_plog_base_framework.framework_output,
+                            "mca:plog:select: checking available component %s", component->pmix_mca_component_name);
+
+        /* If there's no query function, skip it */
+        if (NULL == component->pmix_mca_query_component) {
+            pmix_output_verbose(5, pmix_plog_base_framework.framework_output,
+                                "mca:plog:select: Skipping component [%s]. It does not implement a query function",
+                                component->pmix_mca_component_name );
+            continue;
+        }
+
+        /* Query the component */
+        pmix_output_verbose(5, pmix_plog_base_framework.framework_output,
+                            "mca:plog:select: Querying component [%s]",
+                            component->pmix_mca_component_name);
+        rc = component->pmix_mca_query_component(&module, &priority);
+
+        /* If no module was returned, then skip component */
+        if (PMIX_SUCCESS != rc || NULL == module) {
+            pmix_output_verbose(5, pmix_plog_base_framework.framework_output,
+                                "mca:plog:select: Skipping component [%s]. Query failed to return a module",
+                                component->pmix_mca_component_name );
+            continue;
+        }
+
+        /* If we got a module, keep it */
+        nmodule = (pmix_plog_module_t*) module;
+        /* let it initialize */
+        if (NULL != nmodule->init && PMIX_SUCCESS != nmodule->init()) {
+            continue;
+        }
+        /* add to the list of selected modules */
+        newmodule = PMIX_NEW(pmix_plog_base_active_module_t);
+        newmodule->pri = priority;
+        newmodule->module = nmodule;
+        newmodule->component = (pmix_plog_base_component_t*)cli->cli_component;
+
+        /* maintain priority order */
+        inserted = false;
+        PMIX_LIST_FOREACH(mod, &actives, pmix_plog_base_active_module_t) {
+            if (priority > mod->pri) {
+                pmix_list_insert_pos(&actives, (pmix_list_item_t*)mod, &newmodule->super);
+                inserted = true;
+                break;
+            }
+        }
+        if (!inserted) {
+            /* must be lowest priority - add to end */
+            pmix_list_append(&actives, &newmodule->super);
+        }
+
+        /* if this is the default module, track it */
+        if (0 == strcmp(newmodule->module->name, "default")) {
+            default_mod = newmodule;
+        }
+    }
+
+    /* if they gave us a desired ordering, then impose it here */
+    if (NULL != pmix_plog_globals.channels) {
+        default_added = false;
+        for (n=0; NULL != pmix_plog_globals.channels[n]; n++) {
+            len = strlen(pmix_plog_globals.channels[n]);
+            /* check for the "req" modifier */
+            reqd = false;
+            ptr = strrchr(pmix_plog_globals.channels[n], ':');
+            if (NULL != ptr) {
+                /* get the length of the remaining string so we
+                 * can constrain our comparison of the channel
+                 * name itself */
+                len = len - strlen(ptr);
+                /* move over the ':' */
+                ++ptr;
+                /* we accept anything that starts with "req" */
+                if (0 == strncasecmp(ptr, "req", 3)) {
+                    reqd = true;
+                }
+            }
+            /* now search for this channel in our list of actives */
+            inserted = false;
+            PMIX_LIST_FOREACH(mod, &actives, pmix_plog_base_active_module_t) {
+                if (0 == strncasecmp(pmix_plog_globals.channels[n], mod->module->name, len)) {
+                    pmix_list_remove_item(&actives, &mod->super);
+                    pmix_pointer_array_add(&pmix_plog_globals.actives, mod);
+                    mod->reqd = reqd;
+                    inserted = true;
+                    break;
+                }
+            }
+            if (!inserted) {
+                /* we didn't find a supporting module - this
+                 * still might be okay because it could be something
+                 * the RM itself supports, so just insert the default
+                 * module here if it hasn't already been inserted */
+                if (!default_added) {
+                    /* if the default module isn't available and this
+                     * channel isn't optional, then there is nothing
+                     * we can do except report an error */
+                    if (NULL == default_mod && reqd) {
+                        pmix_show_help("help-pmix-plog.txt", "reqd-not-found",
+                                       true, pmix_plog_globals.channels[n]);
+                        PMIX_LIST_DESTRUCT(&actives);
+                        return PMIX_ERR_NOT_FOUND;
+                    } else if (NULL != default_mod) {
+                        pmix_pointer_array_add(&pmix_plog_globals.actives, default_mod);
+                        default_added = true;
+                        default_mod->reqd = reqd;
+                    }
+                } else if (reqd) {
+                    /* if we already added it, we still have to check the
+                     * reqd status - if any citation requires that the
+                     * default be used, then we set it, but be sure we
+                     * don't overwrite it with a "not required" if it
+                     * was already set as "required" */
+                    default_mod->reqd = reqd;
+                }
+            }
+        }
+        /* if there are any modules left over, we need to discard them */
+        PMIX_LIST_DESTRUCT(&actives);
+    } else {
+        /* insert the modules into the global array in priority order */
+        while (NULL != (mod = (pmix_plog_base_active_module_t*)pmix_list_remove_first(&actives))) {
+            pmix_pointer_array_add(&pmix_plog_globals.actives, mod);
+        }
+        PMIX_DESTRUCT(&actives);
+    }
+
+    if (4 < pmix_output_get_verbosity(pmix_plog_base_framework.framework_output)) {
+        pmix_output(0, "Final plog order");
+        /* show the prioritized order */
+        for (n=0; n < pmix_plog_globals.actives.size; n++) {
+            if (NULL != (mod = (pmix_plog_base_active_module_t*)pmix_pointer_array_get_item(&pmix_plog_globals.actives, n))) {
+                pmix_output(0, "\tplog[%d]: %s", n, mod->component->base.pmix_mca_component_name);
+            }
+        }
+    }
+
+
+    return PMIX_SUCCESS;;
+}

opal/mca/pmix/pmix3x/pmix/src/mca/plog/base/plog_base_stubs.c

@@ -0,0 +1,252 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2018      Intel, Inc. All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include <src/include/pmix_config.h>
+
+#include <pmix_common.h>
+#include "src/include/pmix_globals.h"
+
+#include "src/class/pmix_list.h"
+#include "src/util/error.h"
+#include "src/server/pmix_server_ops.h"
+
+#include "src/mca/plog/base/base.h"
+
+typedef struct {
+    pmix_object_t super;
+    pmix_lock_t lock;
+    size_t nreqs;
+    pmix_status_t status;
+    pmix_op_cbfunc_t cbfunc;
+    void *cbdata;
+} pmix_mycount_t;
+static void mycon(pmix_mycount_t *p)
+{
+    PMIX_CONSTRUCT_LOCK(&p->lock);
+    p->lock.active = false;
+    p->nreqs = 0;
+    p->status = PMIX_ERR_NOT_AVAILABLE;
+    p->cbfunc = NULL;
+    p->cbdata = NULL;
+}
+static void mydes(pmix_mycount_t *p)
+{
+    PMIX_DESTRUCT_LOCK(&p->lock);
+}
+static PMIX_CLASS_INSTANCE(pmix_mycount_t,
+                           pmix_object_t,
+                           mycon, mydes);
+
+static void localcbfunc(pmix_status_t status, void *cbdata)
+{
+    pmix_mycount_t *mycount = (pmix_mycount_t*)cbdata;
+
+    PMIX_ACQUIRE_THREAD(&mycount->lock);
+    mycount->nreqs--;
+    if (PMIX_SUCCESS != status && PMIX_SUCCESS == mycount->status) {
+        mycount->status = status;
+    }
+    if (0 == mycount->nreqs) {
+        /* execute their callback */
+        if (NULL != mycount->cbfunc) {
+            mycount->cbfunc(mycount->status, mycount->cbdata);
+        }
+        PMIX_RELEASE(mycount);
+        return;
+    }
+    PMIX_RELEASE_THREAD(&mycount->lock);
+}
+
+pmix_status_t pmix_plog_base_log(const pmix_proc_t *source,
+                                 const pmix_info_t data[], size_t ndata,
+                                 const pmix_info_t directives[], size_t ndirs,
+                                 pmix_op_cbfunc_t cbfunc, void *cbdata)
+{
+    pmix_plog_base_active_module_t *active;
+    pmix_status_t rc = PMIX_ERR_NOT_AVAILABLE;
+    size_t n, k;
+    int m;
+    bool logonce = false;
+    pmix_mycount_t *mycount;
+    pmix_list_t channels;
+    bool all_complete = true;
+
+    if (!pmix_plog_globals.initialized) {
+        return PMIX_ERR_INIT;
+    }
+
+    /* we have to serialize our way thru here as we are going
+     * to construct a list of the available modules, and those
+     * can only be on one list at a time */
+    PMIX_ACQUIRE_THREAD(&pmix_plog_globals.lock);
+
+    pmix_output_verbose(2, pmix_plog_base_framework.framework_output,
+                        "plog:log called");
+
+    /* initialize the tracker */
+    mycount = PMIX_NEW(pmix_mycount_t);
+    if (NULL == mycount) {
+        PMIX_RELEASE_THREAD(&pmix_plog_globals.lock);
+        return PMIX_ERR_NOMEM;
+    }
+    mycount->cbfunc = cbfunc;
+    mycount->cbdata = cbdata;
+    /* initialize the list of channels */
+    PMIX_CONSTRUCT(&channels, pmix_list_t);
+
+    if (NULL != directives) {
+        /* scan the directives for the PMIX_LOG_ONCE attribute
+         * which indicates we should stop with the first log
+         * channel that can successfully handle this request,
+         * and any channel directives */
+        for (n=0; n < ndirs; n++) {
+            if (0 == strncmp(directives[n].key, PMIX_LOG_ONCE, PMIX_MAX_KEYLEN)) {
+                logonce = true;
+                break;
+            }
+        }
+    }
+
+    /* scan the incoming logging requests and assemble the modules in
+     * the corresponding order - this will ensure that the one they
+     * requested first gets first shot at "log once" */
+    for (n=0; n < ndata; n++) {
+        if (PMIX_INFO_OP_IS_COMPLETE(&data[n])) {
+            continue;
+        }
+        all_complete = false;
+        for (m=0; m < pmix_plog_globals.actives.size; m++) {
+            if (NULL == (active = (pmix_plog_base_active_module_t*)pmix_pointer_array_get_item(&pmix_plog_globals.actives, m))) {
+                continue;
+            }
+            /* if this channel is included in the ones serviced by this
+             * module, then include the module */
+            if (NULL == active->module->channels) {
+                if (!active->added) {
+                    /* add this channel to the list */
+                    pmix_list_append(&channels, &active->super);
+                    /* mark it as added */
+                    active->added = true;
+                }
+            } else {
+                for (k=0; NULL != active->module->channels[k]; k++) {
+                    if (NULL != strstr(data[n].key, active->module->channels[k])) {
+                        if (!active->added) {
+                            /* add this channel to the list */
+                            pmix_list_append(&channels, &active->super);
+                            /* mark it as added */
+                            active->added = true;
+                            break;
+                        }
+                    }
+                }
+            }
+        }
+    }
+    /* reset the added marker for the next time we are called */
+    PMIX_LIST_FOREACH(active, &channels, pmix_plog_base_active_module_t) {
+        active->added = false;
+    }
+    if (all_complete) {
+        /* nothing we need do */
+        while (NULL != pmix_list_remove_first(&channels));
+        PMIX_DESTRUCT(&channels);
+        PMIX_RELEASE(mycount);
+        PMIX_RELEASE_THREAD(&pmix_plog_globals.lock);
+        return PMIX_SUCCESS;
+    }
+    PMIX_ACQUIRE_THREAD(&mycount->lock);
+    PMIX_LIST_FOREACH(active, &channels, pmix_plog_base_active_module_t) {
+        if (NULL != active->module->log) {
+            mycount->nreqs++;
+            rc = active->module->log(source, data, ndata, directives, ndirs,
+                                     localcbfunc, (void*)mycount);
+            /* The plugins are required to return:
+             *
+             * PMIX_SUCCESS - indicating that the logging operation for
+             *                that component was very quick, and therefore
+             *                done atomically. No callback will be issued
+             *
+             * PMIX_OPERATION_IN_PROGRESS - indicates that the plugin
+             *                expects to execute the desired logging request,
+             *                but must do so asynchronously. The provided
+             *                callback _must_ be executed upon completion
+             *                of the operation, indicating success or failure.
+             *
+             * PMIX_ERR_NOT_AVAILABLE - indicates that the plugin is unable
+             *                to process the request.
+             *                No callback will be issued.
+             *
+             * PMIX_ERR_TAKE_NEXT_OPTION - indicates that the plugin didn't
+             *                find any directives that it supports.
+             *                No callback will be issued.
+             *
+             * PMIX_ERR_NOT_SUPPORTED - indicates that the request cannot be
+             *                supported. The list will cease processing at
+             *                that point and return this error
+             *
+             * All other returned errors indicate that the plugin should
+             * have attempted to perform the requested operation, but determined
+             * that it could not do so. Note that this differs from the case
+             * where a plugin asynchronously attempts an operation that subsequently
+             * fails - that error would be returned in the callback function.
+             * In this case, the error indicates that the request contained
+             * an incorrect/invalid element that prevents the plugin from
+             * executing it. The first such retured error will be cached and
+             * returned to the caller upon completion of all pending operations.
+             * No callback from failed plugins shall be executed.
+             */
+            if (PMIX_SUCCESS == rc) {
+                mycount->nreqs--;
+                mycount->status = rc;
+                if (logonce) {
+                    break;
+                }
+            } else if (PMIX_ERR_NOT_AVAILABLE == rc ||
+                       PMIX_ERR_TAKE_NEXT_OPTION == rc) {
+                mycount->nreqs--;
+            } else if (PMIX_OPERATION_IN_PROGRESS == rc) {
+                /* even though the operation hasn't completed,
+                 * we still treat this as a completed request */
+                mycount->status = PMIX_SUCCESS;
+                if (logonce) {
+                    break;
+                }
+            } else {
+                /* we may have outstanding requests we need
+                 * to wait for, so mark that there was an error
+                 * for reporting purposes */
+                mycount->nreqs--;
+                mycount->status = rc;
+            }
+        }
+    }
+
+    /* cannot release the modules - just remove everything from the list */
+    while (NULL != pmix_list_remove_first(&channels));
+    PMIX_DESTRUCT(&channels);
+
+    rc = mycount->status;  // save the status as it could change when the lock is released
+    if (0 == mycount->nreqs) {
+        /* execute their callback */
+        if (NULL != mycount->cbfunc) {
+            mycount->cbfunc(mycount->status, mycount->cbdata);
+        }
+        PMIX_RELEASE_THREAD(&mycount->lock);
+        PMIX_RELEASE(mycount);
+        PMIX_RELEASE_THREAD(&pmix_plog_globals.lock);
+        return PMIX_SUCCESS;
+    }
+    PMIX_RELEASE_THREAD(&mycount->lock);
+    PMIX_RELEASE_THREAD(&pmix_plog_globals.lock);
+
+    return rc;
+}

opal/mca/pmix/pmix3x/pmix/src/mca/plog/default/Makefile.am

@@ -0,0 +1,46 @@
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
+# Copyright (c) 2017      IBM Corporation.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+sources = \
+        plog_default.h \
+        plog_default.c \
+        plog_default_component.c
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if MCA_BUILD_pmix_plog_default_DSO
+component_noinst =
+component_install = mca_plog_default.la
+else
+component_noinst = libmca_plog_default.la
+component_install =
+endif
+
+mcacomponentdir = $(pmixlibdir)
+mcacomponent_LTLIBRARIES = $(component_install)
+mca_plog_default_la_SOURCES = $(sources)
+mca_plog_default_la_LDFLAGS = -module -avoid-version
+
+noinst_LTLIBRARIES = $(component_noinst)
+libmca_plog_default_la_SOURCES =$(sources)
+libmca_plog_default_la_LDFLAGS = -module -avoid-version

opal/mca/pmix/pmix3x/pmix/src/mca/plog/default/plog_default.c

@@ -0,0 +1,150 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "pmix_config.h"
+#include "pmix_common.h"
+
+#include <string.h>
+#ifdef HAVE_SYS_TIME_H
+#include <sys/time.h>
+#endif  /* HAVE_SYS_TIME_H */
+#include <stdarg.h>
+
+#include "src/include/pmix_globals.h"
+#include "src/util/show_help.h"
+#include "src/util/error.h"
+#include "src/server/pmix_server_ops.h"
+
+#include "src/mca/plog/base/base.h"
+#include "plog_default.h"
+
+
+/* Static API's */
+static int init(void);
+static pmix_status_t mylog(const pmix_proc_t *source,
+                           const pmix_info_t data[], size_t ndata,
+                           const pmix_info_t directives[], size_t ndirs,
+                           pmix_op_cbfunc_t cbfunc, void *cbdata);
+
+/* Module def */
+pmix_plog_module_t pmix_plog_default_module = {
+    .name = "default",
+    .channels = NULL,
+    .init = init,
+    .finalize = NULL,
+    .log = mylog
+};
+
+/* local object */
+typedef struct {
+    pmix_object_t super;
+    pmix_info_t *data;
+    size_t ndata;
+    pmix_op_cbfunc_t cbfunc;
+    void *cbdata;
+} local_caddy_t;
+static void lcon(local_caddy_t *p)
+{
+    p->data = NULL;
+    p->ndata = 0;
+}
+static void ldes(local_caddy_t *p)
+{
+    if (NULL != p->data) {
+        PMIX_INFO_FREE(p->data, p->ndata);
+    }
+}
+static PMIX_CLASS_INSTANCE(local_caddy_t,
+                           pmix_object_t,
+                           lcon, ldes);
+
+
+static int init(void)
+{
+    /* we cannot operate if our host doesn't support log */
+    if (NULL == pmix_host_server.log) {
+        return PMIX_ERR_NOT_AVAILABLE;
+    }
+    return PMIX_SUCCESS;
+}
+
+static void localcbfn(pmix_status_t status, void *cbdata)
+{
+    local_caddy_t *cd = (local_caddy_t*)cbdata;
+
+    if (NULL != cd->cbfunc) {
+        cd->cbfunc(status, cd->cbdata);
+    }
+    PMIX_RELEASE(cd);
+}
+
+static pmix_status_t mylog(const pmix_proc_t *source,
+                           const pmix_info_t data[], size_t ndata,
+                           const pmix_info_t directives[], size_t ndirs,
+                           pmix_op_cbfunc_t cbfunc, void *cbdata)
+{
+    local_caddy_t *cd;
+    size_t ntodo, n;
+
+    /* if none of the prior modules performed a requested logging
+     * operation, then we will try here */
+    ntodo = 0;
+    for (n=0; n < ndata; n++) {
+        if (!PMIX_INFO_OP_IS_COMPLETE(&data[n])) {
+            ++ntodo;
+        }
+    }
+    if (0 == ntodo) {
+        return PMIX_ERR_TAKE_NEXT_OPTION;
+    }
+
+    /* send it upwards for potential handling. This might seem
+     * odd in the case where we are a gateway, but we must allow
+     * for the possibility that the host has a channel we don't
+     * directly support */
+    cd = PMIX_NEW(local_caddy_t);
+    if (NULL == cd) {
+        return PMIX_ERR_NOMEM;
+    }
+    cd->cbfunc = cbfunc;
+    cd->cbdata = cbdata;
+
+    /* separate out the ones that weren't completed */
+    PMIX_INFO_CREATE(cd->data, ntodo);
+    if (NULL == cd->data) {
+        PMIX_RELEASE(cd);
+        return PMIX_ERR_NOMEM;
+    }
+    cd->ndata = ntodo;
+    ntodo = 0;
+    for (n=0; n < ndata; n++) {
+        if (!PMIX_INFO_OP_IS_COMPLETE(&data[n])) {
+            PMIX_INFO_XFER(&cd->data[ntodo], (pmix_info_t*)&data[n]);
+            ++ntodo;
+        }
+    }
+
+    /* ask the host to log the remainder */
+    pmix_host_server.log(source, cd->data, cd->ndata,
+                         directives, ndirs,
+                         localcbfn, (void*)cd);
+
+    return PMIX_OPERATION_IN_PROGRESS;
+}

opal/mca/pmix/pmix3x/pmix/src/mca/plog/default/plog_default.h

@@ -0,0 +1,40 @@
+/* -*- C -*-
+ *
+ * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2006 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+#ifndef PLOG_DEFAULT_H
+#define PLOG_DEFAULT_H
+
+#include "pmix_config.h"
+
+#include "src/mca/plog/plog.h"
+
+BEGIN_C_DECLS
+
+/*
+ * Plog interfaces
+ */
+
+PMIX_EXPORT extern pmix_plog_base_component_t mca_plog_default_component;
+extern pmix_plog_module_t pmix_plog_default_module;
+
+END_C_DECLS
+
+#endif

opal/mca/pmix/pmix3x/pmix/src/mca/plog/default/plog_default_component.c

@@ -0,0 +1,47 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2018      Intel, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+*/
+
+/*
+ * includes
+ */
+#include "pmix_config.h"
+#include "pmix_common.h"
+
+#include "plog_default.h"
+
+
+static pmix_status_t component_query(pmix_mca_base_module_t **module,
+                                     int *priority);
+
+/*
+ * Struct of function pointers that need to be initialized
+ */
+pmix_plog_base_component_t mca_plog_default_component = {
+    .base = {
+        PMIX_PLOG_BASE_VERSION_1_0_0,
+
+        .pmix_mca_component_name = "default",
+        PMIX_MCA_BASE_MAKE_VERSION(component, PMIX_MAJOR_VERSION, PMIX_MINOR_VERSION,
+                              PMIX_RELEASE_VERSION),
+        .pmix_mca_query_component = component_query
+    },
+    .data = {
+        /* The component is checkpoint ready */
+        PMIX_MCA_BASE_METADATA_PARAM_CHECKPOINT
+    },
+};
+
+static pmix_status_t component_query(pmix_mca_base_module_t **module,
+                                     int *priority)
+{
+    *priority = 1;
+    *module = (pmix_mca_base_module_t *)&pmix_plog_default_module;
+    return PMIX_SUCCESS;
+}

opal/mca/pmix/pmix3x/pmix/src/mca/plog/plog.h

@@ -0,0 +1,102 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2007-2008 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2018 Intel, Inc. All rights reserved.
+ *
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+/**
+ * @file
+ *
+ * This interface is for use by PMIx servers to obtain network-related info
+ * such as security keys that need to be shared across applications, and to
+ * setup network support for applications prior to launch
+ *
+ * Available plugins may be defined at runtime via the typical MCA parameter
+ * syntax.
+ */
+
+#ifndef PMIX_PLOG_H
+#define PMIX_PLOG_H
+
+#include <src/include/pmix_config.h>
+#include "pmix_common.h"
+
+#include "src/class/pmix_list.h"
+#include "src/mca/mca.h"
+#include "src/mca/base/pmix_mca_base_var.h"
+#include "src/mca/base/pmix_mca_base_framework.h"
+#include "src/include/pmix_globals.h"
+
+BEGIN_C_DECLS
+
+/******    MODULE DEFINITION    ******/
+
+/**
+ * Initialize the module. Returns an error if the module cannot
+ * run, success if it can and wants to be used.
+ */
+typedef pmix_status_t (*pmix_plog_base_module_init_fn_t)(void);
+
+
+/**
+ * Finalize the module
+ */
+typedef void (*pmix_plog_base_module_fini_fn_t)(void);
+
+/**
+ * Log data to channel, if possible
+ */
+typedef pmix_status_t (*pmix_plog_base_module_log_fn_t)(const pmix_proc_t *source,
+                                                        const pmix_info_t data[], size_t ndata,
+                                                        const pmix_info_t directives[], size_t ndirs,
+                                                        pmix_op_cbfunc_t cbfunc, void *cbdata);
+
+/**
+ * Base structure for a PLOG module
+ */
+typedef struct {
+    char *name;
+    char **channels;
+    /* init/finalize */
+    pmix_plog_base_module_init_fn_t     init;
+    pmix_plog_base_module_fini_fn_t     finalize;
+    pmix_plog_base_module_log_fn_t      log;
+} pmix_plog_module_t;
+
+/**
+ * Base structure for a PLOG API
+ */
+typedef struct {
+    pmix_plog_base_module_log_fn_t      log;
+} pmix_plog_API_module_t;
+
+
+/* declare the global APIs */
+PMIX_EXPORT extern pmix_plog_API_module_t pmix_plog;
+
+/*
+ * the standard component data structure
+ */
+struct pmix_plog_base_component_t {
+    pmix_mca_base_component_t                        base;
+    pmix_mca_base_component_data_t                   data;
+};
+typedef struct pmix_plog_base_component_t pmix_plog_base_component_t;
+
+/*
+ * Macro for use in components that are of type plog
+ */
+#define PMIX_PLOG_BASE_VERSION_1_0_0 \
+    PMIX_MCA_BASE_VERSION_1_0_0("plog", 1, 0, 0)
+
+END_C_DECLS
+
+#endif

opal/mca/pmix/pmix3x/pmix/src/mca/plog/stdfd/Makefile.am

@@ -0,0 +1,46 @@
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
+# Copyright (c) 2017      IBM Corporation.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+sources = \
+        plog_stdfd.h \
+        plog_stdfd.c \
+        plog_stdfd_component.c
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if MCA_BUILD_pmix_plog_stdfd_DSO
+component_noinst =
+component_install = mca_plog_stdfd.la
+else
+component_noinst = libmca_plog_stdfd.la
+component_install =
+endif
+
+mcacomponentdir = $(pmixlibdir)
+mcacomponent_LTLIBRARIES = $(component_install)
+mca_plog_stdfd_la_SOURCES = $(sources)
+mca_plog_stdfd_la_LDFLAGS = -module -avoid-version
+
+noinst_LTLIBRARIES = $(component_noinst)
+libmca_plog_stdfd_la_SOURCES =$(sources)
+libmca_plog_stdfd_la_LDFLAGS = -module -avoid-version

opal/mca/pmix/pmix3x/pmix/src/mca/plog/stdfd/plog_stdfd.c

@@ -0,0 +1,122 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "pmix_config.h"
+#include "pmix_common.h"
+
+#include <string.h>
+#ifdef HAVE_SYS_TIME_H
+#include <sys/time.h>
+#endif  /* HAVE_SYS_TIME_H */
+#include <stdarg.h>
+
+#include "src/util/argv.h"
+#include "src/util/error.h"
+#include "src/util/name_fns.h"
+#include "src/util/show_help.h"
+#include "src/common/pmix_iof.h"
+
+#include "src/mca/plog/base/base.h"
+#include "plog_stdfd.h"
+
+
+/* Static API's */
+static int init(void);
+static void finalize(void);
+static pmix_status_t mylog(const pmix_proc_t *source,
+                           const pmix_info_t data[], size_t ndata,
+                           const pmix_info_t directives[], size_t ndirs,
+                           pmix_op_cbfunc_t cbfunc, void *cbdata);
+
+/* Module def */
+pmix_plog_module_t pmix_plog_stdfd_module = {
+    .name = "stdfd",
+    .init = init,
+    .finalize = finalize,
+    .log = mylog
+};
+
+
+static int init(void)
+{
+    char *mychannels = "stdout,stderr";
+
+    pmix_plog_stdfd_module.channels = pmix_argv_split(mychannels, ',');
+    return PMIX_SUCCESS;
+}
+
+static void finalize(void)
+{
+    pmix_argv_free(pmix_plog_stdfd_module.channels);
+}
+
+static pmix_status_t mylog(const pmix_proc_t *source,
+                           const pmix_info_t data[], size_t ndata,
+                           const pmix_info_t directives[], size_t ndirs,
+                           pmix_op_cbfunc_t cbfunc, void *cbdata)
+{
+    size_t n;
+    pmix_status_t rc;
+    pmix_byte_object_t bo;
+    pmix_iof_flags_t flags= {0};
+
+    /* if there is no data, then we don't handle it */
+    if (NULL == data || 0 == ndata) {
+        return PMIX_ERR_NOT_AVAILABLE;
+    }
+
+    /* if we are not a gateway, then we don't handle this */
+    if (!PMIX_PROC_IS_GATEWAY(pmix_globals.mypeer)) {
+        return PMIX_ERR_TAKE_NEXT_OPTION;
+    }
+
+    /* check to see if there are any relevant directives */
+    for (n=0; n < ndirs; n++) {
+        if (0 == strncmp(directives[n].key, PMIX_LOG_TIMESTAMP, PMIX_MAX_KEYLEN)) {
+            flags.timestamp = data[n].value.data.time;
+        } else if (0 == strncmp(directives[n].key, PMIX_LOG_XML_OUTPUT, PMIX_MAX_KEYLEN)) {
+            flags.xml = PMIX_INFO_TRUE(&directives[n]);
+        } else if (0 == strncmp(directives[n].key, PMIX_LOG_TAG_OUTPUT, PMIX_MAX_KEYLEN)) {
+            flags.tag = PMIX_INFO_TRUE(&directives[n]);
+        }
+    }
+
+    /* check to see if there are any stdfd entries */
+    rc = PMIX_ERR_TAKE_NEXT_OPTION;
+    for (n=0; n < ndata; n++) {
+        if (0 == strncmp(data[n].key, PMIX_LOG_STDERR, PMIX_MAX_KEYLEN)) {
+            bo.bytes = data[n].value.data.string;
+            bo.size = strlen(bo.bytes);
+            pmix_iof_write_output(source, PMIX_FWD_STDERR_CHANNEL, &bo, &flags);
+            /* flag that we did this one */
+            PMIX_INFO_OP_COMPLETED(&data[n]);
+            rc = PMIX_SUCCESS;
+        } else if (0 == strncmp(data[n].key, PMIX_LOG_STDOUT, PMIX_MAX_KEYLEN)) {
+            bo.bytes = data[n].value.data.string;
+            bo.size = strlen(bo.bytes);
+            pmix_iof_write_output(source, PMIX_FWD_STDOUT_CHANNEL, &bo, &flags);
+            /* flag that we did this one */
+            PMIX_INFO_OP_COMPLETED(&data[n]);
+            rc = PMIX_SUCCESS;
+        }
+    }
+
+    return rc;
+}

opal/mca/pmix/pmix3x/pmix/src/mca/plog/stdfd/plog_stdfd.h

@@ -0,0 +1,40 @@
+/* -*- C -*-
+ *
+ * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2006 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+#ifndef PLOG_STDFD_H
+#define PLOG_STDFD_H
+
+#include "pmix_config.h"
+
+#include "src/mca/plog/plog.h"
+
+BEGIN_C_DECLS
+
+/*
+ * Plog interfaces
+ */
+
+PMIX_EXPORT extern pmix_plog_base_component_t mca_plog_stdfd_component;
+extern pmix_plog_module_t pmix_plog_stdfd_module;
+
+END_C_DECLS
+
+#endif

opal/mca/pmix/pmix3x/pmix/src/mca/plog/stdfd/plog_stdfd_component.c

@@ -0,0 +1,47 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2018      Intel, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+*/
+
+/*
+ * includes
+ */
+#include "pmix_config.h"
+#include "pmix_common.h"
+
+#include "plog_stdfd.h"
+
+
+static pmix_status_t component_query(pmix_mca_base_module_t **module,
+                                     int *priority);
+
+/*
+ * Struct of function pointers that need to be initialized
+ */
+pmix_plog_base_component_t mca_plog_stdfd_component = {
+    .base = {
+        PMIX_PLOG_BASE_VERSION_1_0_0,
+
+        .pmix_mca_component_name = "stdfd",
+        PMIX_MCA_BASE_MAKE_VERSION(component, PMIX_MAJOR_VERSION, PMIX_MINOR_VERSION,
+                              PMIX_RELEASE_VERSION),
+        .pmix_mca_query_component = component_query,
+    },
+    .data = {
+        /* The component is checkpoint ready */
+        PMIX_MCA_BASE_METADATA_PARAM_CHECKPOINT
+    },
+};
+
+static pmix_status_t component_query(pmix_mca_base_module_t **module,
+                                     int *priority)
+{
+    *priority = 5;
+    *module = (pmix_mca_base_module_t *)&pmix_plog_stdfd_module;
+    return PMIX_SUCCESS;
+}

opal/mca/pmix/pmix3x/pmix/src/mca/plog/syslog/Makefile.am

@@ -0,0 +1,46 @@
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
+# Copyright (c) 2017      IBM Corporation.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+sources = \
+        plog_syslog.h \
+        plog_syslog.c \
+        plog_syslog_component.c
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if MCA_BUILD_pmix_plog_syslog_DSO
+component_noinst =
+component_install = mca_plog_syslog.la
+else
+component_noinst = libmca_plog_syslog.la
+component_install =
+endif
+
+mcacomponentdir = $(pmixlibdir)
+mcacomponent_LTLIBRARIES = $(component_install)
+mca_plog_syslog_la_SOURCES = $(sources)
+mca_plog_syslog_la_LDFLAGS = -module -avoid-version
+
+noinst_LTLIBRARIES = $(component_noinst)
+libmca_plog_syslog_la_SOURCES =$(sources)
+libmca_plog_syslog_la_LDFLAGS = -module -avoid-version

opal/mca/pmix/pmix3x/pmix/src/mca/plog/syslog/configure.m4

@@ -0,0 +1,30 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2017      Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2017-2018 Intel, Inc. All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# MCA_plog_syslog_CONFIG([action-if-found], [action-if-not-found])
+# -----------------------------------------------------------
+AC_DEFUN([MCA_pmix_plog_syslog_CONFIG], [
+    AC_CONFIG_FILES([src/mca/plog/syslog/Makefile])
+
+    PMIX_VAR_SCOPE_PUSH([pmix_plog_syslog_happy])
+
+    # if syslog.h is not compilable,
+    # disable this component.
+    AC_CHECK_HEADER([syslog.h],
+        [pmix_plog_syslog_happy=1],
+        [pmix_plog_syslog_happy=0])
+
+    AS_IF([test $pmix_plog_syslog_happy -eq 1],
+          [$1],
+          [$2])
+
+    PMIX_VAR_SCOPE_POP
+])dnl

opal/mca/pmix/pmix3x/pmix/src/mca/plog/syslog/plog_syslog.c

@@ -0,0 +1,227 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "pmix_config.h"
+#include "pmix_common.h"
+
+#include <string.h>
+#ifdef HAVE_TIME_H
+#include <time.h>
+#endif
+#ifdef HAVE_SYS_TIME_H
+#include <sys/time.h>
+#endif  /* HAVE_SYS_TIME_H */
+#ifdef HAVE_SYSLOG_H
+#include <syslog.h>
+#endif
+#include <stdarg.h>
+
+#include "src/util/argv.h"
+#include "src/util/error.h"
+#include "src/util/name_fns.h"
+#include "src/util/show_help.h"
+#include "src/mca/bfrops/bfrops.h"
+#include "src/server/pmix_server_ops.h"
+
+#include "src/mca/plog/base/base.h"
+#include "plog_syslog.h"
+
+
+/* Static API's */
+static pmix_status_t init(void);
+static void finalize(void);
+static pmix_status_t mylog(const pmix_proc_t *source,
+                           const pmix_info_t data[], size_t ndata,
+                           const pmix_info_t directives[], size_t ndirs,
+                           pmix_op_cbfunc_t cbfunc, void *cbdata);
+
+/* Module def */
+pmix_plog_module_t pmix_plog_syslog_module = {
+    .name = "syslog",
+    .init = init,
+    .finalize = finalize,
+    .log = mylog
+};
+
+
+static pmix_status_t init(void)
+{
+    int opts;
+    char *mychannels = "lsys,gsys,syslog,local_syslog,global_syslog";
+
+    pmix_plog_syslog_module.channels = pmix_argv_split(mychannels, ',');
+
+    opts = LOG_CONS | LOG_PID;
+    openlog("PMIx Log Report:", opts, LOG_USER);
+
+    return PMIX_SUCCESS;
+}
+
+static void finalize(void)
+{
+    closelog();
+    pmix_argv_free(pmix_plog_syslog_module.channels);
+}
+
+static pmix_status_t write_local(const pmix_proc_t *source,
+                                 time_t timestamp,
+                                 int severity, char *msg,
+                                 const pmix_info_t *data, size_t ndata);
+
+/* we only get called if we are a SERVER */
+static pmix_status_t mylog(const pmix_proc_t *source,
+                           const pmix_info_t data[], size_t ndata,
+                           const pmix_info_t directives[], size_t ndirs,
+                           pmix_op_cbfunc_t cbfunc, void *cbdata)
+{
+    size_t n;
+    int pri = mca_plog_syslog_component.level;
+    pmix_status_t rc;
+    time_t timestamp = 0;
+
+    /* if there is no data, then we don't handle it */
+    if (NULL == data || 0 == ndata) {
+        return PMIX_ERR_NOT_AVAILABLE;
+    }
+
+    /* check directives */
+    if (NULL != directives) {
+        for (n=0; n < ndirs; n++) {
+            if (0 == strncmp(directives[n].key, PMIX_LOG_SYSLOG_PRI, PMIX_MAX_KEYLEN)) {
+                pri = directives[n].value.data.integer;
+            } else if (0 == strncmp(directives[n].key, PMIX_LOG_TIMESTAMP, PMIX_MAX_KEYLEN)) {
+                timestamp = directives[n].value.data.time;
+            }
+        }
+    }
+
+    /* check to see if there are any syslog entries */
+    for (n=0; n < ndata; n++) {
+        if (0 == strncmp(data[n].key, PMIX_LOG_SYSLOG, PMIX_MAX_KEYLEN)) {
+            /* we default to using the local syslog */
+            rc = write_local(source, timestamp, pri, data[n].value.data.string, data, ndata);
+            if (PMIX_SUCCESS == rc) {
+                /* flag that we did this one */
+                PMIX_INFO_OP_COMPLETED(&data[n]);
+            }
+        } else if (0 == strncmp(data[n].key, PMIX_LOG_LOCAL_SYSLOG, PMIX_MAX_KEYLEN)) {
+            rc = write_local(source, timestamp, pri, data[n].value.data.string, data, ndata);
+            if (PMIX_SUCCESS == rc) {
+                /* flag that we did this one */
+                PMIX_INFO_OP_COMPLETED(&data[n]);
+            }
+        } else if (0 == strncmp(data[n].key, PMIX_LOG_GLOBAL_SYSLOG, PMIX_MAX_KEYLEN)) {
+            /* only do this if we are a gateway server */
+            if (PMIX_PROC_IS_GATEWAY(pmix_globals.mypeer)) {
+                rc = write_local(source, timestamp, pri, data[n].value.data.string, data, ndata);
+                if (PMIX_SUCCESS == rc) {
+                    /* flag that we did this one */
+                    PMIX_INFO_OP_COMPLETED(&data[n]);
+                }
+            }
+        }
+    }
+
+    return PMIX_SUCCESS;
+}
+
+static char* sev2str(int severity)
+{
+    switch (severity) {
+        case LOG_EMERG:
+            return "EMERGENCY";
+        case LOG_ALERT:
+            return "ALERT";
+        case LOG_CRIT:
+            return "CRITICAL";
+        case LOG_ERR:
+            return "ERROR";
+        case LOG_WARNING:
+            return "WARNING";
+        case LOG_NOTICE:
+            return "NOTICE";
+        case LOG_INFO:
+            return "INFO";
+        case LOG_DEBUG:
+            return "DEBUG";
+        default:
+            return "UNKNOWN SEVERITY";
+    }
+}
+
+static pmix_status_t write_local(const pmix_proc_t *source,
+                                 time_t timestamp,
+                                 int severity, char *msg,
+                                 const pmix_info_t *data, size_t ndata)
+{
+    char tod[48], *datastr, *tmp, *tmp2;
+    pmix_status_t rc;
+    size_t n;
+
+    pmix_output_verbose(5, pmix_plog_base_framework.framework_output,
+                           "plog:syslog:mylog function called with severity %d", severity);
+
+    if (0 < timestamp) {
+        /* If there was a message, output it */
+        (void)ctime_r(&timestamp, tod);
+        /* trim the newline */
+        tod[strlen(tod)] = '\0';
+    }
+
+    if (NULL == data) {
+        syslog(severity, "%s [%s:%d]%s PROC %s:%d REPORTS: %s",
+               tod, pmix_globals.myid.nspace, pmix_globals.myid.rank,
+               sev2str(severity),
+               source->nspace, source->rank,
+               (NULL == msg) ? "<N/A>" : msg);
+    } else {
+        /* need to print the info from the data, starting
+         * with any provided msg */
+        if (NULL == msg) {
+            datastr = strdup("\n");
+        } else {
+            if (0 > asprintf(&datastr, "%s", msg)) {
+                return PMIX_ERR_NOMEM;
+            }
+        }
+        for (n=0; n < ndata; n++) {
+            PMIX_BFROPS_PRINT(rc, pmix_globals.mypeer,
+                              &tmp, "\t", (pmix_info_t*)&data[n], PMIX_INFO);
+            if (PMIX_SUCCESS != rc) {
+                free(datastr);
+                return rc;
+            }
+            if (0 > asprintf(&tmp2, "%s\n%s", datastr, tmp)) {
+                free(datastr);
+                return PMIX_ERR_NOMEM;
+            }
+            free(datastr);
+            free(tmp);
+            datastr = tmp2;
+        }
+        /* print out the consolidated msg */
+        syslog(severity, "%s [%s:%d]%s PROC %s:%d REPORTS: %s",
+               tod, pmix_globals.myid.nspace, pmix_globals.myid.rank,
+               sev2str(severity), source->nspace, source->rank, datastr);
+        free(datastr);
+    }
+
+    return PMIX_SUCCESS;
+}

opal/mca/pmix/pmix3x/pmix/src/mca/plog/syslog/plog_syslog.h

@@ -0,0 +1,47 @@
+/* -*- C -*-
+ *
+ * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2006 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+#ifndef PLOG_SYSLOG_H
+#define PLOG_SYSLOG_H
+
+#include "pmix_config.h"
+
+#include "src/mca/plog/plog.h"
+
+BEGIN_C_DECLS
+
+/*
+ * Plog interfaces
+ */
+
+typedef struct {
+    pmix_plog_base_component_t super;
+    int console;
+    int level;
+    int facility;
+} pmix_plog_syslog_component_t;
+
+PMIX_EXPORT extern pmix_plog_syslog_component_t mca_plog_syslog_component;
+extern pmix_plog_module_t pmix_plog_syslog_module;
+
+END_C_DECLS
+
+#endif