Merge pull request #4758 from rhc54/topic/sync

Refresh ORTE PMIx support

opal/mca/pmix/base/base.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014-2017 Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow

opal/mca/pmix/base/help-pmix-base.txt

@@ -4,6 +4,7 @@
 # Copyright (c) 2016      Los Alamos National Security, LLC. All rights
 #                         reserved.
 #
+# Copyright (c) 2016-2018 Intel, Inc. All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -14,7 +15,23 @@
 #
 [pmix2-init-failed]
 PMI2_Init failed to intialize.  Return code: %d
-
+#
 [pmix2-init-returned-bad-values]
 PMI2_Init was intialized but negative values for job size and/or
 rank was returned.
+#
+[old-pmix]
+A version of PMIx was detected that is too old:
+
+  Version:      %s
+  Min version:  %s
+
+Please reconfigure against an updated version of PMIx.
+#
+[incorrect-pmix]
+An unexpected version of PMIx was loaded:
+
+  Detected:  %s
+  Expected:  %s
+
+Please check the library path and reconfigure if required.

opal/mca/pmix/ext1x/Makefile.am

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2014-2016 Intel, Inc.  All rights reserved.
+# Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
 # Copyright (c) 2015      Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2014-2015 Mellanox Technologies, Inc.
 #                         All rights reserved.

opal/mca/pmix/ext1x/configure.m4

@@ -13,7 +13,7 @@
 # Copyright (c) 2011-2013 Los Alamos National Security, LLC.
 #                         All rights reserved.
 # Copyright (c) 2010-2015 Cisco Systems, Inc.  All rights reserved.
-# Copyright (c) 2013-2017 Intel, Inc. All rights reserved.
+# Copyright (c) 2013-2018 Intel, Inc. All rights reserved.
 # Copyright (c) 2015-2017 Research Organization for Information Science
 #                         and Technology (RIST). All rights reserved.
 # Copyright (c) 2014-2015 Mellanox Technologies, Inc.

opal/mca/pmix/ext1x/pmix1x.c

@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2014-2017 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
  * Copyright (c) 2014-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2014-2015 Mellanox Technologies, Inc.
@@ -48,8 +48,13 @@
 
 static const char *pmix1_get_nspace(opal_jobid_t jobid);
 static void pmix1_register_jobid(opal_jobid_t jobid, const char *nspace);
+static bool legacy_get(void)
+{
+    return true;
+}
 
 const opal_pmix_base_module_t opal_pmix_ext1x_module = {
+    .legacy_get = legacy_get,
     /* client APIs */
     .init = pmix1_client_init,
     .finalize = pmix1_client_finalize,

opal/mca/pmix/ext1x/pmix1x.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014-2017 Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
  * Copyright (c) 2014-2015 Mellanox Technologies, Inc.
  *                         All rights reserved.
  * Copyright (c) 2016-2017 Research Organization for Information Science

opal/mca/pmix/ext1x/pmix1x_client.c

@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2014-2017 Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
  * Copyright (c) 2014-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2014-2015 Mellanox Technologies, Inc.

opal/mca/pmix/ext1x/pmix1x_component.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014-2016 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
  * Copyright (c) 2014-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2016 Cisco Systems, Inc.  All rights reserved.
@@ -21,6 +21,7 @@
 #include "opal/constants.h"
 #include "opal/class/opal_list.h"
 #include "opal/util/proc.h"
+#include "opal/util/show_help.h"
 #include "opal/mca/pmix/pmix.h"
 #include "pmix1x.h"
 
@@ -45,39 +46,48 @@ static int external_component_query(mca_base_module_t **module, int *priority);
 
 mca_pmix_ext1x_component_t mca_pmix_ext1x_component = {
     {
-    /* First, the mca_component_t struct containing meta information
+        /* First, the mca_component_t struct containing meta information
-       about the component itself */
+         * about the component itself */
 
-	.base_version = {
+        .base_version = {
-	/* Indicate that we are a pmix v1.1.0 component (which also
+           /* Indicate that we are a pmix v1.1.0 component (which also
-	   implies a specific MCA version) */
+            * implies a specific MCA version) */
 
-	    OPAL_PMIX_BASE_VERSION_2_0_0,
+            OPAL_PMIX_BASE_VERSION_2_0_0,
 
-	/* Component name and version */
+            /* Component name and version */
 
-	    .mca_component_name = "ext1x",
+            .mca_component_name = "ext1x",
-	    MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION,
+            MCA_BASE_MAKE_VERSION(component, OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION,
-				  OPAL_RELEASE_VERSION),
+                                  OPAL_RELEASE_VERSION),
 
-	/* Component open and close functions */
+            /* Component open and close functions */
 
-	    .mca_open_component = external_open,
+           .mca_open_component = external_open,
-	    .mca_close_component = external_close,
+           .mca_close_component = external_close,
-	    .mca_query_component = external_component_query,
+           .mca_query_component = external_component_query,
-	},
+        },
-	/* Next the MCA v1.0.0 component meta data */
+        /* Next the MCA v1.0.0 component meta data */
-	.base_data = {
+        .base_data = {
-	/* The component is checkpoint ready */
+            /* The component is checkpoint ready */
-	    MCA_BASE_METADATA_PARAM_CHECKPOINT
+            MCA_BASE_METADATA_PARAM_CHECKPOINT
-	}
+        }
     },
     .native_launch = false
 };
 
 static int external_open(void)
 {
+    const char *version;
+
     OBJ_CONSTRUCT(&mca_pmix_ext1x_component.jobids, opal_list_t);
+
+    version = PMIx_Get_version();
+    if (0 != strncmp(version, "1.2", 3)) {
+        opal_show_help("help-pmix-base.txt",
+                       "old-pmix", true, version, "v1.2");
+        return OPAL_ERROR;
+    }
     return OPAL_SUCCESS;
 }
 
@@ -94,12 +104,12 @@ static int external_component_query(mca_base_module_t **module, int *priority)
 
     /* see if a PMIx server is present */
     if (NULL != (t = getenv("PMIX_SERVER_URI")) ||
-	NULL != (id = getenv("PMIX_ID"))) {
+        NULL != (id = getenv("PMIX_ID"))) {
-	/* if PMIx is present, then we are a client and need to use it */
+        /* if PMIx is present, then we are a client and need to use it */
-	*priority = 100;
+        *priority = 100;
     } else {
-	/* we could be a server, so we still need to be considered */
+        /* we could be a server, so we still need to be considered */
-	*priority = 5;
+        *priority = 5;
     }
     *module = (mca_base_module_t *)&opal_pmix_ext1x_module;
     return OPAL_SUCCESS;

opal/mca/pmix/ext1x/pmix1x_server_north.c

@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2014-2016 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
  * Copyright (c) 2014-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2014-2015 Mellanox Technologies, Inc.

opal/mca/pmix/ext1x/pmix1x_server_south.c

@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2014-2017 Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
  * Copyright (c) 2014-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2014-2015 Intel, Inc.  All rights reserved.

opal/mca/pmix/ext2x/ext2x.c

@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2014-2017 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
  * Copyright (c) 2014-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2014-2015 Mellanox Technologies, Inc.
@@ -50,7 +50,7 @@
 
 /* These are functions used by both client and server to
  * access common functions in the embedded PMIx library */
-
+static bool legacy_get(void);
 static const char *ext2x_get_nspace(opal_jobid_t jobid);
 static void ext2x_register_jobid(opal_jobid_t jobid, const char *nspace);
 static void register_handler(opal_list_t *event_codes,
@@ -72,6 +72,7 @@ static void ext2x_log(opal_list_t *info,
                        opal_pmix_op_cbfunc_t cbfunc, void *cbdata);
 
 const opal_pmix_base_module_t opal_pmix_ext2x_module = {
+    .legacy_get = legacy_get,
     /* client APIs */
     .init = ext2x_client_init,
     .finalize = ext2x_client_finalize,
@@ -126,6 +127,11 @@ const opal_pmix_base_module_t opal_pmix_ext2x_module = {
     .register_jobid = ext2x_register_jobid
 };
 
+static bool legacy_get(void)
+{
+    return mca_pmix_ext2x_component.legacy_get;
+}
+
 static void opcbfunc(pmix_status_t status, void *cbdata)
 {
     ext2x_opcaddy_t *op = (ext2x_opcaddy_t*)cbdata;

opal/mca/pmix/ext2x/ext2x.h

@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2014-2017 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
  * Copyright (c) 2014-2015 Mellanox Technologies, Inc.
  *                         All rights reserved.
  * Copyright (c) 2016      Research Organization for Information Science
@@ -45,6 +45,7 @@ BEGIN_C_DECLS
 
 typedef struct {
   opal_pmix_base_component_t super;
+  bool legacy_get;
   opal_list_t jobids;
   bool native_launch;
   size_t evindex;

opal/mca/pmix/ext2x/ext2x_client.c

@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2014-2017 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
  * Copyright (c) 2014-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2014-2017 Mellanox Technologies, Inc.
@@ -400,7 +400,6 @@ int ext2x_store_local(const opal_process_name_t *proc, opal_value_t *val)
 
     PMIX_VALUE_CONSTRUCT(&kv);
     ext2x_value_load(&kv, val);
-
     /* call the library - this is a blocking call */
     rc = PMIx_Store_internal(&p, val->key, &kv);
     PMIX_VALUE_DESTRUCT(&kv);
@@ -596,10 +595,11 @@ int ext2x_get(const opal_process_name_t *proc, const char *key,
         return OPAL_ERR_NOT_INITIALIZED;
     }
 
-    if (NULL == proc) {
+    if (NULL == proc && NULL != key) {
         /* if they are asking for our jobid, then return it */
         if (0 == strcmp(key, OPAL_PMIX_JOBID)) {
             (*val) = OBJ_NEW(opal_value_t);
+            (*val)->key = strdup(key);
             (*val)->type = OPAL_UINT32;
             (*val)->data.uint32 = OPAL_PROC_MY_NAME.jobid;
             OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
@@ -608,6 +608,7 @@ int ext2x_get(const opal_process_name_t *proc, const char *key,
         /* if they are asking for our rank, return it */
         if (0 == strcmp(key, OPAL_PMIX_RANK)) {
             (*val) = OBJ_NEW(opal_value_t);
+            (*val)->key = strdup(key);
             (*val)->type = OPAL_INT;
             (*val)->data.integer = ext2x_convert_rank(my_proc.rank);
             OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
@@ -642,6 +643,9 @@ int ext2x_get(const opal_process_name_t *proc, const char *key,
     rc = PMIx_Get(&p, key, pinfo, sz, &pval);
     if (PMIX_SUCCESS == rc) {
         ival = OBJ_NEW(opal_value_t);
+        if (NULL != key) {
+            ival->key = strdup(key);
+        }
         if (OPAL_SUCCESS != (ret = ext2x_value_unload(ival, pval))) {
             rc = ext2x_convert_opalrc(ret);
         } else {
@@ -663,6 +667,9 @@ static void val_cbfunc(pmix_status_t status,
 
     OPAL_ACQUIRE_OBJECT(op);
     OBJ_CONSTRUCT(&val, opal_value_t);
+    if (NULL != op->nspace) {
+        val.key = strdup(op->nspace);
+    }
     rc = ext2x_convert_opalrc(status);
     if (PMIX_SUCCESS == status && NULL != kv) {
         rc = ext2x_value_unload(&val, kv);
@@ -702,6 +709,7 @@ int ext2x_getnb(const opal_process_name_t *proc, const char *key,
         if (0 == strcmp(key, OPAL_PMIX_JOBID)) {
             if (NULL != cbfunc) {
                 val = OBJ_NEW(opal_value_t);
+                val->key = strdup(key);
                 val->type = OPAL_UINT32;
                 val->data.uint32 = OPAL_PROC_MY_NAME.jobid;
                 cbfunc(OPAL_SUCCESS, val, cbdata);
@@ -713,6 +721,7 @@ int ext2x_getnb(const opal_process_name_t *proc, const char *key,
         if (0 == strcmp(key, OPAL_PMIX_RANK)) {
             if (NULL != cbfunc) {
                 val = OBJ_NEW(opal_value_t);
+                val->key = strdup(key);
                 val->type = OPAL_INT;
                 val->data.integer = ext2x_convert_rank(my_proc.rank);
                 cbfunc(OPAL_SUCCESS, val, cbdata);
@@ -726,7 +735,9 @@ int ext2x_getnb(const opal_process_name_t *proc, const char *key,
     op = OBJ_NEW(ext2x_opcaddy_t);
     op->valcbfunc = cbfunc;
     op->cbdata = cbdata;
-
+    if (NULL != key) {
+        op->nspace = strdup(key);
+    }
     if (NULL == proc) {
         (void)strncpy(op->p.nspace, my_proc.nspace, PMIX_MAX_NSLEN);
         op->p.rank = ext2x_convert_rank(PMIX_RANK_WILDCARD);

opal/mca/pmix/ext2x/ext2x_component.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014-2017 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
  * Copyright (c) 2014-2015 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2016 Cisco Systems, Inc.  All rights reserved.
@@ -21,6 +21,7 @@
 #include "opal/constants.h"
 #include "opal/class/opal_list.h"
 #include "opal/util/proc.h"
+#include "opal/util/show_help.h"
 #include "opal/mca/pmix/pmix.h"
 #include "ext2x.h"
 
@@ -74,6 +75,7 @@ mca_pmix_ext2x_component_t mca_pmix_ext2x_component = {
             MCA_BASE_METADATA_PARAM_CHECKPOINT
         }
     },
+    .legacy_get = true,
     .native_launch = false
 };
 
@@ -94,11 +96,23 @@ static int external_register(void)
 
 static int external_open(void)
 {
+    const char *version;
+
     mca_pmix_ext2x_component.evindex = 0;
     OBJ_CONSTRUCT(&mca_pmix_ext2x_component.jobids, opal_list_t);
     OBJ_CONSTRUCT(&mca_pmix_ext2x_component.events, opal_list_t);
     OBJ_CONSTRUCT(&mca_pmix_ext2x_component.dmdx, opal_list_t);
 
+    version = PMIx_Get_version();
+    if ('2' != version[0]) {
+        opal_show_help("help-pmix-base.txt",
+                       "incorrect-pmix", true, version, "v2.x");
+        return OPAL_ERROR;
+    }
+    if (0 == strncmp(version, "2.1", 3)) {
+        mca_pmix_ext2x_component.legacy_get = false;
+    }
+
     return OPAL_SUCCESS;
 }
 

opal/mca/pmix/pmix.h

@@ -733,6 +733,10 @@ typedef int (*opal_pmix_base_module_server_notify_event_fn_t)(int status,
                                                               opal_list_t *info,
                                                               opal_pmix_op_cbfunc_t cbfunc, void *cbdata);
 
+/* push IO to local clients */
+typedef int (*opal_pmix_base_module_server_push_io_fn_t)(const opal_process_name_t *source,
+                                                         opal_pmix_iof_channel_t channel,
+                                                         unsigned char *data, size_t nbytes);
 
 /************************************************************
  *                         TOOL APIs                        *
@@ -870,10 +874,13 @@ typedef int (*opal_pmix_base_process_monitor_fn_t)(opal_list_t *monitor,
 /* register cleanup */
 typedef int (*opal_pmix_base_register_cleanup_fn_t)(char *path, bool directory, bool ignore, bool jobscope);
 
+typedef bool (*opal_pmix_base_legacy_get_fn_t)(void);
+
 /*
  * the standard public API data structure
  */
 typedef struct {
+    opal_pmix_base_legacy_get_fn_t                          legacy_get;
     /* client APIs */
     opal_pmix_base_module_init_fn_t                         init;
     opal_pmix_base_module_fini_fn_t                         finalize;
@@ -917,6 +924,7 @@ typedef struct {
     opal_pmix_base_module_server_setup_fork_fn_t            server_setup_fork;
     opal_pmix_base_module_server_dmodex_request_fn_t        server_dmodex_request;
     opal_pmix_base_module_server_notify_event_fn_t          server_notify_event;
+    opal_pmix_base_module_server_push_io_fn_t               server_iof_push;
     /* tool APIs */
     opal_pmix_base_module_tool_init_fn_t                    tool_init;
     opal_pmix_base_module_tool_fini_fn_t                    tool_finalize;

opal/mca/pmix/pmix3x/pmix/VERSION

@@ -30,7 +30,7 @@ greek=
 # command, or with the date (if "git describe" fails) in the form of
 # "date<date>".
 
-repo_rev=git5c0b64b
+repo_rev=gitdb5d380
 
 # If tarball_version is not empty, it is used as the version string in
 # the tarball filename, regardless of all other versions listed in
@@ -44,7 +44,7 @@ tarball_version=
 
 # The date when this release was created
 
-date="Dec 11, 2017"
+date="Jan 25, 2018"
 
 # The shared library version of each of PMIx's public libraries.
 # These versions are maintained in accordance with the "Library

opal/mca/pmix/pmix3x/pmix/config/pmix_check_attributes.m4

@@ -1,6 +1,6 @@
 # -*- shell-script -*-
 # PMIx copyrights:
-# Copyright (c) 2013      Intel, Inc. All rights reserved
+# Copyright (c) 2013-2017 Intel, Inc. All rights reserved.
 #
 #########################
 #
@@ -170,6 +170,8 @@ AC_DEFUN([PMIX_CHECK_ATTRIBUTES], [
     pmix_cv___attribute__visibility=0
     pmix_cv___attribute__warn_unused_result=0
     pmix_cv___attribute__destructor=0
+    pmix_cv___attribute__optnone=0
+    pmix_cv___attribute__extension=0
   else
     AC_MSG_RESULT([yes])
 
@@ -486,6 +488,21 @@ AC_DEFUN([PMIX_CHECK_ATTRIBUTES], [
         ],
         [],
         [])
+
+    _PMIX_CHECK_SPECIFIC_ATTRIBUTE([optnone],
+        [
+        void __attribute__ ((__optnone__)) foo(void);
+        void foo(void) { return ; }
+        ],
+        [],
+        [])
+
+    _PMIX_CHECK_SPECIFIC_ATTRIBUTE([extension],
+        [
+         #define FOO __extension__ ({size_t bar; bar = 3;})
+        ],
+        [],
+        [])
   fi
 
   # Now that all the values are set, define them
@@ -536,4 +553,8 @@ AC_DEFUN([PMIX_CHECK_ATTRIBUTES], [
                      [Whether your compiler has __attribute__ weak alias or not])
   AC_DEFINE_UNQUOTED(PMIX_HAVE_ATTRIBUTE_DESTRUCTOR, [$pmix_cv___attribute__destructor],
                      [Whether your compiler has __attribute__ destructor or not])
+  AC_DEFINE_UNQUOTED(PMIX_HAVE_ATTRIBUTE_OPTNONE, [$pmix_cv___attribute__optnone],
+                     [Whether your compiler has __attribute__ optnone or not])
+  AC_DEFINE_UNQUOTED(PMIX_HAVE_ATTRIBUTE_EXTENSION, [$pmix_cv___attribute__extension],
+                     [Whether your compiler has __attribute__ extension or not])
 ])

opal/mca/pmix/pmix3x/pmix/contrib/pmix-valgrind.supp

@@ -1,6 +1,6 @@
 # -*- text -*-
 #
-# Copyright (c) 2015      Intel, Inc. All rights reserved.
+# Copyright (c) 2015-2018 Intel, Inc. All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow

opal/mca/pmix/pmix3x/pmix/etc/pmix-mca-params.conf

@@ -10,7 +10,7 @@
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
 # Copyright (c) 2006-2017 Cisco Systems, Inc.  All rights reserved
-# Copyright (c) 2017      Intel, Inc. All rights reserved.
+# Copyright (c) 2017-2018 Intel, Inc. All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -53,7 +53,7 @@
 # directory.  For example:
 
 # Change component loading path
-#   component_path = /usr/local/lib/pmix:~/my_pmix_components
+#   mca_base_component_path = /usr/local/lib/pmix:~/my_pmix_components
 
 # See "pinfo --param all all --level 9" for a full listing of PMIx
 # MCA parameters available and their default values.

opal/mca/pmix/pmix3x/pmix/examples/debuggerd.c

@@ -13,7 +13,7 @@
  *                         All rights reserved.
  * Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2011      Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2013-2017 Intel, Inc. All rights reserved.
+ * Copyright (c) 2013-2017 Intel, Inc.  All rights reserved.
  * Copyright (c) 2015      Mellanox Technologies, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
@@ -41,6 +41,7 @@ typedef struct {
 } myquery_data_t;
 
 
+static volatile bool waiting_for_debugger = true;
 static pmix_proc_t myproc;
 
 /* this is a callback function for the PMIx_Query
@@ -133,6 +134,11 @@ int main(int argc, char **argv)
     size_t nq, n;
     myquery_data_t myquery_data;
 
+fprintf(stderr, "I AM HERE\n");
+fflush(stderr);
+    sleep(10);
+    exit(0);
+
     /* init us - since we were launched by the RM, our connection info
      * will have been provided at startup. */
     if (PMIX_SUCCESS != (rc = PMIx_tool_init(&myproc, NULL, 0))) {
@@ -210,7 +216,7 @@ int main(int argc, char **argv)
     n = 0;
     fprintf(stderr, "[%s:%u] Hanging around awhile, doing debugger magic\n", myproc.nspace, myproc.rank);
     while (n < 5) {
-        usleep(10);
+        usleep(1000);
         ++n;
     }
 

opal/mca/pmix/pmix3x/pmix/include/pmix.h

@@ -567,6 +567,152 @@ PMIX_EXPORT pmix_status_t PMIx_Process_monitor_nb(const pmix_info_t *monitor, pm
         PMIX_INFO_DESTRUCT(&_in);                                           \
     } while(0)
 
+/* Request a credential from the PMIx server/SMS.
+ * Input values include:
+ *
+ * info - an array of pmix_info_t structures containing any directives the
+ *        caller may wish to pass. Typical usage might include:
+ *            PMIX_TIMEOUT - how long to wait (in seconds) for a credential
+ *                           before timing out and returning an error
+ *            PMIX_CRED_TYPE - a prioritized, comma-delimited list of desired
+ *                             credential types for use in environments where
+ *                             multiple authentication mechanisms may be
+ *                             available
+ *
+ * ninfo - number of elements in the info array
+ *
+ * cbfunc - the pmix_credential_cbfunc_t function to be called upon completion
+ *          of the request
+ *
+ * cbdata - pointer to an object to be returned when cbfunc is called
+ *
+ * Returned values:
+ * PMIX_SUCCESS - indicates that the request has been successfully communicated to
+ *                the local PMIx server. The response will be coming in the provided
+ *                callback function.
+ *
+ * Any other value indicates an appropriate error condition. The callback function
+ * will _not_ be called in such cases.
+ */
+PMIX_EXPORT pmix_status_t PMIx_Get_credential(const pmix_info_t info[], size_t ninfo,
+                                              pmix_credential_cbfunc_t cbfunc, void *cbdata);
+
+
+/* Request validation of a credential by the PMIx server/SMS
+ * Input values include:
+ *
+ * cred - pointer to a pmix_byte_object_t containing the credential
+ *
+ * info - an array of pmix_info_t structures containing any directives the
+ *        caller may wish to pass. Typical usage might include:
+ *            PMIX_TIMEOUT - how long to wait (in seconds) for validation
+ *                           before timing out and returning an error
+ *            PMIX_USERID - the expected effective userid of the credential
+ *                          to be validated
+ *            PMIX_GROUPID - the expected effective group id of the credential
+ *                          to be validated
+ *
+ * ninfo - number of elements in the info array
+ *
+ * cbfunc - the pmix_validation_cbfunc_t function to be called upon completion
+ *          of the request
+ *
+ * cbdata - pointer to an object to be returned when cbfunc is called
+ *
+ * Returned values:
+ * PMIX_SUCCESS - indicates that the request has been successfully communicated to
+ *                the local PMIx server. The response will be coming in the provided
+ *                callback function.
+ *
+ * Any other value indicates an appropriate error condition. The callback function
+ * will _not_ be called in such cases.
+ */
+PMIX_EXPORT pmix_status_t PMIx_Validate_credential(const pmix_byte_object_t *cred,
+                                                   const pmix_info_t info[], size_t ninfo,
+                                                   pmix_validation_cbfunc_t cbfunc, void *cbdata);
+
+/* Define a callback function for delivering forwarded IO to a process
+ * This function will be called whenever data becomes available, or a
+ * specified buffering size and/or time has been met. The function
+ * will be passed the following values:
+ *
+ * iofhdlr - the returned registration number of the handler being invoked.
+ *           This is required when deregistering the handler.
+ *
+ * channel - a bitmask identifying the channel the data arrived on
+ *
+ * source - the nspace/rank of the process that generated the data
+ *
+ * payload - pointer to character array containing the data. Note that
+ *           multiple strings may be included, and that the array may
+ *           _not_ be NULL terminated
+ *
+ * info - an optional array of info provided by the source containing
+ *        metadata about the payload. This could include PMIX_IOF_COMPLETE
+ *
+ * ninfo - number of elements in the optional info array
+ */
+ typedef void (*pmix_iof_cbfunc_t)(size_t iofhdlr, pmix_iof_channel_t channel,
+                                   pmix_proc_t *source, char *payload,
+                                   pmix_info_t info[], size_t ninfo);
+
+
+/* Register to receive IO forwarded from a remote process.
+ *
+ * procs - array of identifiers for sources whose IO is being
+ *         requested. Wildcard rank indicates that all procs
+ *         in the specified nspace are included in the request
+ *
+ * nprocs - number of identifiers in the procs array
+ *
+ * directives - optional array of attributes to control the
+ *              behavior of the request. For example, this
+ *              might include directives on buffering IO
+ *              before delivery, and/or directives to include
+ *              or exclude any backlogged data
+ *
+ * ndirs - number of elements in the directives array
+ *
+ * channel - bitmask of IO channels included in the request
+ *
+ * cbfunc - function to be called when relevant IO is received
+ *
+ * regcbfunc - since registration is async, this is the
+ *             function to be called when registration is
+ *             completed. The function itself will return
+ *             a non-success error if the registration cannot
+ *             be submitted - in this case, the regcbfunc
+ *             will _not_ be called.
+ *
+ * cbdata - pointer to object to be returned in regcbfunc
+ */
+PMIX_EXPORT pmix_status_t PMIx_IOF_register(const pmix_proc_t procs[], size_t nprocs,
+                                            const pmix_info_t directives[], size_t ndirs,
+                                            pmix_iof_channel_t channel, pmix_iof_cbfunc_t cbfunc,
+                                            pmix_hdlr_reg_cbfunc_t regcbfunc, void *regcbdata);
+
+/* Deregister from IO forwarded from a remote process.
+ *
+ * iofhdlr - the registration number returned from the
+ *           call to PMIx_IOF_register
+ *
+ * directives - optional array of attributes to control the
+ *              behavior of the request. For example, this
+ *              might include directives regarding what to
+ *              do with any data currently in the IO buffer
+ *              for this process
+ *
+ * cbfunc - function to be called when deregistration has
+ *          been completed. Note that any IO to be flushed
+ *          may continue to be received after deregistration
+ *          has completed.
+ *
+ * cbdata - pointer to object to be returned in cbfunc
+ */
+PMIX_EXPORT pmix_status_t PMIx_IOF_deregister(size_t iofhdlr,
+                                              const pmix_info_t directives[], size_t ndirs,
+                                              pmix_op_cbfunc_t cbfunc, void *cbdata);
+
 #if defined(c_plusplus) || defined(__cplusplus)
 }
 #endif

opal/mca/pmix/pmix3x/pmix/include/pmix_common.h.in

@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2013-2017 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2013-2018 Intel, Inc. All rights reserved.
  * Copyright (c) 2016-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2016      IBM Corporation.  All rights reserved.
@@ -147,6 +147,7 @@ typedef uint32_t pmix_rank_t;
 #define PMIX_CONNECT_RETRY_DELAY            "pmix.tool.retry"       // (uint32_t) time in seconds between connection attempts
 #define PMIX_TOOL_DO_NOT_CONNECT            "pmix.tool.nocon"       // (bool) the tool wants to use internal PMIx support, but does
                                                                     //        not want to connect to a PMIx server
+                                                                    //        from the specified processes to this tool
 
 /* identification attributes */
 #define PMIX_USERID                         "pmix.euid"             // (uint32_t) effective user id
@@ -220,7 +221,9 @@ typedef uint32_t pmix_rank_t;
 #define PMIX_LOCAL_CPUSETS                  "pmix.lcpus"            // (char*) colon-delimited cpusets of local peers within the specified nspace
 #define PMIX_PROC_URI                       "pmix.puri"             // (char*) URI containing contact info for proc
 #define PMIX_LOCALITY                       "pmix.loc"              // (uint16_t) relative locality of two procs
-#define PMIX_PARENT_ID                      "pmix.parent"           // (pmix_proc_t) process identifier of my parent process
+#define PMIX_PARENT_ID                      "pmix.parent"           // (pmix_proc_t*) identifier of the process that called PMIx_Spawn
+                                                                    //                to launch this proc's application
+
 
 /* size info */
 #define PMIX_UNIV_SIZE                      "pmix.univ.size"        // (uint32_t) #procs in this nspace
@@ -324,7 +327,7 @@ typedef uint32_t pmix_rank_t;
 #define PMIX_EVENT_WANT_TERMINATION         "pmix.evterm"           // (bool) indicates that the handler has determined that the application should be terminated
 
 
-/* attributes used to describe "spawn" attributes */
+/* attributes used to describe "spawn" directives */
 #define PMIX_PERSONALITY                    "pmix.pers"             // (char*) name of personality to use
 #define PMIX_HOST                           "pmix.host"             // (char*) comma-delimited list of hosts to use for spawned procs
 #define PMIX_HOSTFILE                       "pmix.hostfile"         // (char*) hostfile to use for spawned procs
@@ -342,9 +345,6 @@ typedef uint32_t pmix_rank_t;
 #define PMIX_PRELOAD_FILES                  "pmix.preloadfiles"     // (char*) comma-delimited list of files to pre-position
 #define PMIX_NON_PMI                        "pmix.nonpmi"           // (bool) spawned procs will not call PMIx_Init
 #define PMIX_STDIN_TGT                      "pmix.stdin"            // (uint32_t) spawned proc rank that is to receive stdin
-#define PMIX_FWD_STDIN                      "pmix.fwd.stdin"        // (bool) forward my stdin to the designated proc
-#define PMIX_FWD_STDOUT                     "pmix.fwd.stdout"       // (bool) forward stdout from spawned procs to me
-#define PMIX_FWD_STDERR                     "pmix.fwd.stderr"       // (bool) forward stderr from spawned procs to me
 #define PMIX_DEBUGGER_DAEMONS               "pmix.debugger"         // (bool) spawned app consists of debugger daemons
 #define PMIX_COSPAWN_APP                    "pmix.cospawn"          // (bool) designated app is to be spawned as a disconnected
                                                                     //        job - i.e., not part of the "comm_world" of the job
@@ -364,6 +364,11 @@ typedef uint32_t pmix_rank_t;
 #define PMIX_JOB_CONTINUOUS                 "pmix.continuous"       // (bool) application is continuous, all failed procs should
                                                                         //        be immediately restarted
 #define PMIX_MAX_RESTARTS                   "pmix.maxrestarts"      // (uint32_t) max number of times to restart a job
+#define PMIX_FWD_STDIN                      "pmix.fwd.stdin"        // (bool) forward the stdin from this process to the spawned processes
+#define PMIX_FWD_STDOUT                     "pmix.fwd.stdout"       // (bool) forward stdout from the spawned processes to this process (typically used by a tool)
+#define PMIX_FWD_STDERR                     "pmix.fwd.stderr"       // (bool) forward stderr from the spawned processes to this process (typically used by a tool)
+#define PMIX_FWD_STDDIAG                    "pmix.fwd.stddiag"      // (bool) if a diagnostic channel exists, forward any output on it
+                                                                    //        from the spawned processes to this process (typically used by a tool)
 
 
 /* connect attributes */
@@ -416,6 +421,11 @@ typedef uint32_t pmix_rank_t;
 #define PMIX_DEBUG_WAIT_FOR_NOTIFY          "pmix.dbg.notify"       // (bool) block at desired point until receiving debugger release notification
 #define PMIX_DEBUG_JOB                      "pmix.dbg.job"          // (char*) nspace of the job to be debugged - the RM/PMIx server are
 #define PMIX_DEBUG_WAITING_FOR_NOTIFY       "pmix.dbg.waiting"      // (bool) job to be debugged is waiting for a release
+#define PMIX_PREPEND_LD_PRELOAD             "pmix.prepend.preload"  // (char*) prepend the named library to any existing
+                                                                    //         LD_PRELOAD directive
+#define PMIX_APPEND_LD_PRELOAD              "pmix.append.preload"   // (char*) append the named library to any existing
+                                                                    //         LD_PRELOAD directive
+
 
 /* Resource Manager identification */
 #define PMIX_RM_NAME                        "pmix.rm.name"          // (char*) string name of the resource manager
@@ -494,6 +504,14 @@ typedef uint32_t pmix_rank_t;
 #define PMIX_MONITOR_FILE_DROPS             "pmix.monitor.fdrop"    // (uint32_t) number of file checks that can be missed before
                                                                     //            generating the event
 
+/* security attributes */
+#define PMIX_CRED_TYPE                      "pmix.sec.ctype"        //  when passed in PMIx_Get_credential, a prioritized,
+                                                                    // comma-delimited list of desired credential types for use
+                                                                    // in environments where multiple authentication mechanisms
+                                                                    // may be available. When returned in a callback function, a
+                                                                    // string identifier of the credential type
+
+
 /****    PROCESS STATE DEFINITIONS    ****/
 typedef uint8_t pmix_proc_state_t;
 #define PMIX_PROC_STATE_UNDEF                    0  /* undefined process state */
@@ -712,6 +730,8 @@ typedef uint16_t pmix_data_type_t;
 #define PMIX_ALLOC_DIRECTIVE    43
 /**** DEPRECATED ****/
 #define PMIX_INFO_ARRAY         44
+/****            ****/
+#define PMIX_IOF_CHANNEL        45
 /********************/
 
 /* define a boundary for implementers so they can add their own data types */
@@ -778,11 +798,29 @@ typedef uint8_t pmix_alloc_directive_t;
 #define PMIX_ALLOC_EXTERNAL     128
 
 
+/* define a set of bit-mask flags for specifying IO
+ * forwarding channels. These can be OR'd together
+ * to reference multiple channels */
+typedef uint16_t pmix_iof_channel_t;
+#define PMIX_FWD_NO_CHANNELS        0x00
+#define PMIX_FWD_STDIN_CHANNEL      0x01
+#define PMIX_FWD_STDOUT_CHANNEL     0x02
+#define PMIX_FWD_STDERR_CHANNEL     0x04
+#define PMIX_FWD_STDDIAG_CHANNEL    0x08
+#define PMIX_FWD_ALL_CHANNELS       0xff
+
+
 /****    PMIX BYTE OBJECT    ****/
 typedef struct pmix_byte_object {
     char *bytes;
     size_t size;
 } pmix_byte_object_t;
+#define PMIX_BYTE_OBJECT_CONSTRUCT(m)   \
+    do {                                \
+        (m)->bytes = NULL;              \
+        (m)->size = 0;                  \
+    } while(0)
+
 #define PMIX_BYTE_OBJECT_DESTRUCT(m)    \
     do {                                \
         if (NULL != (m)->bytes) {       \
@@ -1607,14 +1645,19 @@ typedef void (*pmix_notification_fn_t)(size_t evhdlr_registration_id,
                                        pmix_event_notification_cbfunc_fn_t cbfunc,
                                        void *cbdata);
 
-/* define a callback function for calls to PMIx_Register_evhdlr. The
+/* define a callback function for calls to register handlers, e.g., event
- * status indicates if the request was successful or not, evhdlr_ref is
+ * notification and IOF requests
- * an integer reference assigned to the event handler by PMIx, this reference
+ *
- * must be used to deregister the err handler. A ptr to the original
+ * status - PMIX_SUCCESS or an appropriate error constant
- * cbdata is returned. */
+ *
-typedef void (*pmix_evhdlr_reg_cbfunc_t)(pmix_status_t status,
+ * refid - reference identifier assigned to the handler by PMIx,
-                                         size_t evhdlr_ref,
+ *         used to deregister the handler
-                                         void *cbdata);
+ *
+ * cbdata - object provided to the registration call
+ */
+typedef void (*pmix_hdlr_reg_cbfunc_t)(pmix_status_t status,
+                                       size_t refid,
+                                       void *cbdata);
 
 /* define a callback function for calls to PMIx_Get_nb. The status
  * indicates if the requested data was found or not - a pointer to the
@@ -1632,6 +1675,75 @@ typedef void (*pmix_info_cbfunc_t)(pmix_status_t status,
                                    pmix_release_cbfunc_t release_fn,
                                    void *release_cbdata);
 
+/* Define a callback function to return a requested security credential.
+ * Returned values include:
+ *
+ * status - PMIX_SUCCESS if a credential could be assigned as requested, or
+ *          else an appropriate error code indicating the problem
+ *
+ * credential - pointer to an allocated pmix_byte_object_t containing the
+ *              credential (as a opaque blob) and its size. Ownership of
+ *              the credential is transferred to the receiving function - thus,
+ *              responsibility for releasing the memory lies outside the
+ *              PMIx library.
+ *
+ * info - an array of pmix_info_t structures provided by the system to pass
+ *        any additional information about the credential - e.g., the identity
+ *        of the issuing agent. The info array is owned by the PMIx library
+ *        and is not to be released or altered by the receiving party. Note that
+ *        this array is not related to the pmix_info_t structures possibly
+ *        provided in the call to PMIx_Get_credential.
+ *
+ *        Information provided by the issuing agent can subsequently be used
+ *        by the application for a variety of purposes. Examples include:
+ *            - checking identified authorizations to determine what
+ *              requests/operations are feasible as a means to steering
+ *              workflows
+ *            - compare the credential type to that of the local SMS for
+ *              compatibility
+ *
+ * ninfo - number of elements in the info array
+ *
+ * cbdata - the caller's provided void* object
+ *
+ * NOTE: the credential is opaque and therefore understandable only by
+ *       a service compatible with the issuer.
+ */
+typedef void (*pmix_credential_cbfunc_t)(pmix_status_t status,
+                                         pmix_byte_object_t *credential,
+                                         pmix_info_t info[], size_t ninfo,
+                                         void *cbdata);
+
+
+/* Define a validation callback function to indicate if a provided
+ * credential is valid, and any corresponding information regarding
+ * authorizations and other security matters
+ * Returned values include:
+ *
+ * status - PMIX_SUCCESS if the provided credential is valid. An appropriate
+ *          error code indicating the issue if the credential is rejected.
+ *
+ * info - an array of pmix_info_t structures provided by the system to pass
+ *        any additional information about the authentication - e.g., the
+ *        effective userid and group id of the certificate holder, and any
+ *        related authorizations. The info array is owned by the PMIx library
+ *        and is not to be released or altered by the receiving party. Note that
+ *        this array is not related to the pmix_info_t structures possibly
+ *        provided in the call to PMIx_Validate_credential.
+ *
+ *        The precise contents of the array will depend on the host SMS and
+ *        its associated security system. At the minimum, it is expected (but
+ *        not required) that the array will contain entries for the PMIX_USERID
+ *        and PMIX_GROUPID of the client described in the credential.
+ *
+ * ninfo - number of elements in the info array
+ *
+ * cbdata - the caller's provided void* object
+ */
+typedef void (*pmix_validation_cbfunc_t)(pmix_status_t status,
+                                         pmix_info_t info[], size_t ninfo,
+                                         void *cbdata);
+
 
 /****    COMMON SUPPORT FUNCTIONS    ****/
 /* Register an event handler to report events. Three types of events
@@ -1670,7 +1782,7 @@ typedef void (*pmix_info_cbfunc_t)(pmix_status_t status,
 PMIX_EXPORT void PMIx_Register_event_handler(pmix_status_t codes[], size_t ncodes,
                                              pmix_info_t info[], size_t ninfo,
                                              pmix_notification_fn_t evhdlr,
-                                             pmix_evhdlr_reg_cbfunc_t cbfunc,
+                                             pmix_hdlr_reg_cbfunc_t cbfunc,
                                              void *cbdata);
 
 /* Deregister an event handler
@@ -1728,6 +1840,7 @@ PMIX_EXPORT pmix_status_t PMIx_Notify_event(pmix_status_t status,
  * - pmix_info_directives_t   (PMIX_INFO_DIRECTIVES)
  * - pmix_data_type_t   (PMIX_DATA_TYPE)
  * - pmix_alloc_directive_t  (PMIX_ALLOC_DIRECTIVE)
+ * - pmix_iof_channel_t  (PMIX_IOF_CHANNEL)
  */
 PMIX_EXPORT const char* PMIx_Error_string(pmix_status_t status);
 PMIX_EXPORT const char* PMIx_Proc_state_string(pmix_proc_state_t state);
@@ -1737,6 +1850,7 @@ PMIX_EXPORT const char* PMIx_Data_range_string(pmix_data_range_t range);
 PMIX_EXPORT const char* PMIx_Info_directives_string(pmix_info_directives_t directives);
 PMIX_EXPORT const char* PMIx_Data_type_string(pmix_data_type_t type);
 PMIX_EXPORT const char* PMIx_Alloc_directive_string(pmix_alloc_directive_t directive);
+PMIX_EXPORT const char* PMIx_IOF_channel_string(pmix_iof_channel_t channel);
 
 /* Get the PMIx version string. Note that the provided string is
  * statically defined and must NOT be free'd  */

opal/mca/pmix/pmix3x/pmix/include/pmix_server.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013-2017 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2013-2018 Intel, Inc. All rights reserved.
  * Copyright (c) 2015      Artem Y. Polyakov <artpol84@gmail.com>.
  *                         All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
@@ -350,6 +350,128 @@ typedef pmix_status_t (*pmix_server_monitor_fn_t)(const pmix_proc_t *requestor,
                                                   const pmix_info_t directives[], size_t ndirs,
                                                   pmix_info_cbfunc_t cbfunc, void *cbdata);
 
+/* Request a credential from the host SMS
+ * Input values include:
+ *
+ * proc - pointer to a pmix_proc_t identifier of the process on whose behalf
+ *        the request is being made (i.e., the client originating the request)
+ *
+ * directives - an array of pmix_info_t structures containing directives pertaining
+ *              to the request. This will typically include any pmix_info_t structs
+ *              passed by the requesting client, but may also include directives
+ *              required by (or available from) the PMIx server implementation - e.g.,
+ *              the effective user and group ID's of the requesting process.
+ *
+ * ndirs - number of pmix_info_t structures in the directives array
+ *
+ * cbfunc - the pmix_credential_cbfunc_t function to be called upon completion
+ *          of the request
+ *
+ * cbdata - pointer to an object to be returned when cbfunc is called
+ *
+ * Returned values:
+ * PMIX_SUCCESS - indicates that the request is being processed by the host system
+ *                management stack. The response will be coming in the provided
+ *                callback function.
+ *
+ * Any other value indicates an appropriate error condition. The callback function
+ * will _not_ be called in such cases.
+ */
+typedef pmix_status_t (*pmix_server_get_cred_fn_t)(const pmix_proc_t *proc,
+                                                   const pmix_info_t directives[], size_t ndirs,
+                                                   pmix_credential_cbfunc_t cbfunc, void *cbdata);
+
+/* Request validation of a credential from the host SMS
+ * Input values include:
+ *
+ * proc - pointer to a pmix_proc_t identifier of the process on whose behalf
+ *        the request is being made (i.e., the client issuing the request)
+ *
+ * cred - pointer to a pmix_byte_object_t containing the provided credential
+ *
+ * directives - an array of pmix_info_t structures containing directives pertaining
+ *              to the request. This will typically include any pmix_info_t structs
+ *              passed by the requesting client, but may also include directives
+ *              used by the PMIx server implementation
+ *
+ * ndirs - number of pmix_info_t structures in the directives array
+ *
+ * cbfunc - the pmix_validation_cbfunc_t function to be called upon completion
+ *          of the request
+ *
+ * cbdata - pointer to an object to be returned when cbfunc is called
+ *
+ * Returned values:
+ * PMIX_SUCCESS - indicates that the request is being processed by the host system
+ *                management stack. The response will be coming in the provided
+ *                callback function.
+ *
+ * Any other value indicates an appropriate error condition. The callback function
+ * will _not_ be called in such cases.
+ */
+typedef pmix_status_t (*pmix_server_validate_cred_fn_t)(const pmix_proc_t *proc,
+                                                        const pmix_byte_object_t *cred,
+                                                        const pmix_info_t directives[], size_t ndirs,
+                                                        pmix_validation_cbfunc_t cbfunc, void *cbdata);
+
+/* Request the specified IO channels be forwarded from the given array of procs.
+ * The function shall return PMIX_SUCCESS once the host RM accepts the request for
+ * processing, or a PMIx error code if the request itself isn't correct or supported.
+ * The callback function shall be called when the request has been processed,
+ * returning either PMIX_SUCCESS to indicate that IO shall be forwarded as requested,
+ * or some appropriate error code if the request has been denied.
+ *
+ * procs - array of process identifiers whose IO is being requested.
+ *
+ * nprocs - size of the procs array
+ *
+ * directives - array of key-value attributes further defining the request. This
+ *              might include directives on buffering and security credentials for
+ *              access to protected channels
+ *
+ * ndirs - size of the directives array
+ *
+ * channels - bitmask identifying the channels to be forwarded
+ *
+ * cbfunc - callback function when the IO forwarding has been setup
+ *
+ * cbdata - object to be returned in cbfunc
+ *
+ * This call serves as a registration with the host RM for the given IO channels from
+ * the specified procs - the host RM is expected to ensure that this local PMIx server
+ * is on the distribution list for the channel/proc combination
+ */
+typedef pmix_status_t (*pmix_server_iof_fn_t)(const pmix_proc_t procs[], size_t nprocs,
+                                              const pmix_info_t directives[], size_t ndirs,
+                                              pmix_iof_channel_t channels,
+                                              pmix_op_cbfunc_t cbfunc, void *cbdata);
+
+/* Passes stdin to the host RM for transmission to recipients. The host RM is
+ * responsible for forwarding the data to all PMIx servers that have requested
+ * stdin from the specified source.
+ *
+ * source - pointer to the identifier of the process whose stdin is being provided
+ *
+ * bo - pointer to a byte object containing the stdin data
+ *
+ * directives - array of key-value attributes further defining the request. This
+ *              might include directives on buffering and security credentials for
+ *              access to protected channels
+ *
+ * ndirs - size of the directives array
+ *
+ * cbfunc - callback function when the IO forwarding has been setup
+ *
+ * cbdata - object to be returned in cbfunc
+ *
+ */
+
+typedef pmix_status_t (*pmix_server_stdin_fn_t)(const pmix_proc_t *source,
+                                                const pmix_byte_object_t *bo,
+                                                const pmix_info_t directives[], size_t ndirs,
+                                                pmix_op_cbfunc_t cbfunc, void *cbdata);
+
+
 typedef struct pmix_server_module_2_0_0_t {
     /* v1x interfaces */
     pmix_server_client_connected_fn_t   client_connected;
@@ -374,9 +496,14 @@ typedef struct pmix_server_module_2_0_0_t {
     pmix_server_alloc_fn_t              allocate;
     pmix_server_job_control_fn_t        job_control;
     pmix_server_monitor_fn_t            monitor;
+    /* v3x interfaces */
+    pmix_server_get_cred_fn_t           get_credential;
+    pmix_server_validate_cred_fn_t      validate_credential;
+    pmix_server_iof_fn_t                register_iof;
+    pmix_server_stdin_fn_t              push_stdin;
 } pmix_server_module_t;
 
-/****    SERVER SUPPORT INIT/FINALIZE FUNCTIONS    ****/
+/****    HOST RM FUNCTIONS FOR INTERFACE TO PMIX SERVER    ****/
 
 /* Initialize the server support library, and provide a
  * pointer to a pmix_server_module_t structure
@@ -544,6 +671,38 @@ PMIX_EXPORT pmix_status_t PMIx_server_setup_local_support(const char nspace[],
                                                           pmix_info_t info[], size_t ninfo,
                                                           pmix_op_cbfunc_t cbfunc, void *cbdata);
 
+/* Provide a function by which the host RM can pass forwarded IO
+ * to the local PMIx server for distribution to its clients. The
+ * PMIx server is responsible for determining which of its clients
+ * have actually registered for the provided data
+ *
+ * Parameters include:
+ *
+ * source - the process that provided the data being forwarded
+ *
+ * channel - the IOF channel (stdin, stdout, etc.)
+ *
+ * bo - a byte object containing the data
+ *
+ * info - an optional array of metadata describing the data, including
+ *        attributes such as PMIX_IOF_COMPLETE to indicate that the
+ *        source channel has been closed
+ *
+ * ninfo - number of elements in the info array
+ *
+ * cbfunc - a callback function to be executed once the provided data
+ *          is no longer required. The host RM is required to retain
+ *          the byte object until the callback is executed, or a
+ *          non-success status is returned by the function
+ *
+ * cbdata - object pointer to be returned in the callback function
+ */
+PMIX_EXPORT pmix_status_t PMIx_IOF_push(const pmix_proc_t *source,
+                                        pmix_iof_channel_t channel,
+                                        const pmix_byte_object_t *bo,
+                                        const pmix_info_t info[], size_t ninfo,
+                                        pmix_op_cbfunc_t cbfunc, void *cbdata);
+
 #if defined(c_plusplus) || defined(__cplusplus)
 }
 #endif

opal/mca/pmix/pmix3x/pmix/include/pmix_tool.h

@@ -98,6 +98,7 @@ PMIX_EXPORT pmix_status_t PMIx_tool_init(pmix_proc_t *proc,
  * operation. */
 PMIX_EXPORT pmix_status_t PMIx_tool_finalize(void);
 
+
 #if defined(c_plusplus) || defined(__cplusplus)
 }
 #endif

opal/mca/pmix/pmix3x/pmix/src/Makefile.am

@@ -43,6 +43,10 @@ nodist_headers =
 EXTRA_DIST =
 dist_pmixdata_DATA =
 
+# place to capture sources for backward compatibility libs
+pmi1_sources =
+pmi2_sources =
+
 libpmix_la_LIBADD = \
 	mca/base/libpmix_mca_base.la \
 	$(MCA_pmix_FRAMEWORK_LIBS) \
@@ -73,10 +77,15 @@ libpmix_la_LDFLAGS = -version-info $(libpmix_so_version)
 
 if WANT_PMI_BACKWARD
 lib_LTLIBRARIES += libpmi.la libpmi2.la
-libpmi_la_SOURCES = $(headers) $(sources)
+libpmi_la_SOURCES = $(headers) $(sources) $(pmi1_sources)
 libpmi_la_LDFLAGS = -version-info $(libpmi_so_version)
-libpmi2_la_SOURCES = $(headers) $(sources)
+libpmi_la_LIBADD = $(libpmix_la_LIBADD)
+libpmi_la_DEPENDENCIES = $(libpmi_la_LIBADD)
+
+libpmi2_la_SOURCES = $(headers) $(sources) $(pmi2_sources)
 libpmi2_la_LDFLAGS = -version-info $(libpmi2_so_version)
+libpmi2_la_LIBADD = $(libpmix_la_LIBADD)
+libpmi2_la_DEPENDENCIES = $(libpmi2_la_LIBADD)
 endif
 
 endif !PMIX_EMBEDDED_MODE

opal/mca/pmix/pmix3x/pmix/src/client/Makefile.include

@@ -23,7 +23,8 @@ sources += \
         client/pmix_client_connect.c
 
 if WANT_PMI_BACKWARD
-sources += \
+pmi1_sources += \
-        client/pmi1.c \
+        client/pmi1.c
+pmi2_sources += \
         client/pmi2.c
 endif

opal/mca/pmix/pmix3x/pmix/src/client/pmix_client.c

@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2014-2017 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
  * Copyright (c) 2014-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2014      Artem Y. Polyakov <artpol84@gmail.com>.
@@ -342,6 +342,41 @@ static void _check_for_notify(pmix_info_t info[], size_t ninfo)
     }
 }
 
+static void client_iof_handler(struct pmix_peer_t *pr,
+                               pmix_ptl_hdr_t *hdr,
+                               pmix_buffer_t *buf, void *cbdata)
+{
+    pmix_peer_t *peer = (pmix_peer_t*)pr;
+    pmix_proc_t source;
+    pmix_iof_channel_t channel;
+    pmix_byte_object_t bo;
+    int32_t cnt;
+    pmix_status_t rc;
+
+    pmix_output_verbose(2, pmix_client_globals.iof_output,
+                        "recvd IOF");
+
+    cnt = 1;
+    PMIX_BFROPS_UNPACK(rc, peer, buf, &source, &cnt, PMIX_PROC);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        return;
+    }
+    cnt = 1;
+    PMIX_BFROPS_UNPACK(rc, peer, buf, &channel, &cnt, PMIX_IOF_CHANNEL);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        return;
+    }
+    cnt = 1;
+    PMIX_BFROPS_UNPACK(rc, peer, buf, &bo, &cnt, PMIX_BYTE_OBJECT);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        return;
+    }
+    pmix_output(0, "IOF: %s", bo.bytes);
+}
+
 PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc,
                                     pmix_info_t info[], size_t ninfo)
 {
@@ -358,6 +393,7 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc,
     pmix_lock_t reglock;
     size_t n;
     bool found;
+    pmix_ptl_posted_recv_t *rcv;
 
     PMIX_ACQUIRE_THREAD(&pmix_global_lock);
 
@@ -394,6 +430,13 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc,
         PMIX_RELEASE_THREAD(&pmix_global_lock);
         return rc;
     }
+    /* setup the IO Forwarding recv */
+    rcv = PMIX_NEW(pmix_ptl_posted_recv_t);
+    rcv->tag = PMIX_PTL_TAG_IOF;
+    rcv->cbfunc = client_iof_handler;
+    /* add it to the end of the list of recvs */
+    pmix_list_append(&pmix_ptl_globals.posted_recvs, &rcv->super);
+
 
     /* setup the globals */
     PMIX_CONSTRUCT(&pmix_client_globals.pending_requests, pmix_list_t);
@@ -576,7 +619,7 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc,
     }
     PMIX_RELEASE_THREAD(&pmix_global_lock);
 
-    /* lood for a debugger attach key */
+    /* look for a debugger attach key */
     (void)strncpy(wildcard.nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN);
     wildcard.rank = PMIX_RANK_WILDCARD;
     PMIX_INFO_LOAD(&ginfo, PMIX_OPTIONAL, NULL, PMIX_BOOL);
@@ -585,6 +628,7 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc,
         /* if the value was found, then we need to wait for debugger attach here */
         /* register for the debugger release notification */
         PMIX_CONSTRUCT_LOCK(&reglock);
+        PMIX_POST_OBJECT(&reglock);
         PMIx_Register_event_handler(&code, 1, NULL, 0,
                                     notification_fn, NULL, (void*)&reglock);
         /* wait for it to arrive */

opal/mca/pmix/pmix3x/pmix/src/client/pmix_client_ops.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015-2017 Intel, Inc. All rights reserved.
+ * Copyright (c) 2015-2018 Intel, Inc. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -40,6 +40,9 @@ typedef struct {
     // verbosity for client event operations
     int event_output;
     int event_verbose;
+    // verbosity for client iof operations
+    int iof_output;
+    int iof_verbose;
     // verbosity for basic client functions
     int base_output;
     int base_verbose;

opal/mca/pmix/pmix3x/pmix/src/client/pmix_client_pub.c

@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2014-2017 Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
  * Copyright (c) 2014-2015 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2014      Artem Y. Polyakov <artpol84@gmail.com>.
@@ -564,8 +564,8 @@ static void wait_lookup_cbfunc(struct pmix_peer_t *pr,
     pmix_cb_t *cb = (pmix_cb_t*)cbdata;
     pmix_status_t rc, ret;
     int32_t cnt;
-    pmix_pdata_t *pdata = NULL;
+    pmix_pdata_t *pdata;
-    size_t ndata = 0;
+    size_t ndata;
 
     PMIX_ACQUIRE_OBJECT(cb);
 
@@ -573,6 +573,10 @@ static void wait_lookup_cbfunc(struct pmix_peer_t *pr,
                         "pmix:client recv callback activated with %d bytes",
                         (NULL == buf) ? -1 : (int)buf->bytes_used);
 
+    /* set the defaults */
+    pdata = NULL;
+    ndata = 0;
+
     if (NULL == cb->cbfunc.lookupfn) {
         /* nothing we can do with this */
         PMIX_RELEASE(cb);
@@ -589,10 +593,6 @@ static void wait_lookup_cbfunc(struct pmix_peer_t *pr,
         goto report;
     }
 
-    /* set the defaults */
-    pdata = NULL;
-    ndata = 0;
-
     /* unpack the returned status */
     cnt = 1;
     PMIX_BFROPS_UNPACK(rc, pmix_client_globals.myserver,

opal/mca/pmix/pmix3x/pmix/src/common/Makefile.include

@@ -1,7 +1,7 @@
 # -*- makefile -*-
 #
 # Copyright (c) 2015-2017 Intel, Inc. All rights reserved.
-# Copyright (c) 2016 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2016      Cisco Systems, Inc.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -14,4 +14,6 @@ sources += \
         common/pmix_strings.c \
         common/pmix_log.c \
         common/pmix_control.c \
-        common/pmix_data.c
+        common/pmix_data.c \
+        common/pmix_security.c \
+        common/pmix_iof.c

opal/mca/pmix/pmix3x/pmix/src/common/pmix_data.c

@@ -11,7 +11,7 @@
  *                         All rights reserved.
  * Copyright (c) 2007-2012 Los Alamos National Security, LLC.
  *                         All rights reserved.
- * Copyright (c) 2014-2017 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -37,18 +37,19 @@
 #include "src/mca/bfrops/bfrops.h"
 #include "src/include/pmix_globals.h"
 
-#define PMIX_EMBED_DATA_BUFFER(b, db)                   \
+#define PMIX_EMBED_DATA_BUFFER(b, db)                       \
-    do {                                                \
+    do {                                                    \
-        (b)->base_ptr = (db)->base_ptr;                 \
+        (b)->type = pmix_globals.mypeer->nptr->compat.type; \
-        (b)->pack_ptr = (db)->pack_ptr;                 \
+        (b)->base_ptr = (db)->base_ptr;                     \
-        (b)->unpack_ptr = (db)->unpack_ptr;             \
+        (b)->pack_ptr = (db)->pack_ptr;                     \
-        (b)->bytes_allocated = (db)->bytes_allocated;   \
+        (b)->unpack_ptr = (db)->unpack_ptr;                 \
-        (b)->bytes_used = (db)->bytes_used;             \
+        (b)->bytes_allocated = (db)->bytes_allocated;       \
-        (db)->base_ptr = NULL;                          \
+        (b)->bytes_used = (db)->bytes_used;                 \
-        (db)->pack_ptr = NULL;                          \
+        (db)->base_ptr = NULL;                              \
-        (db)->unpack_ptr = NULL;                        \
+        (db)->pack_ptr = NULL;                              \
-        (db)->bytes_allocated = 0;                      \
+        (db)->unpack_ptr = NULL;                            \
-        (db)->bytes_used = 0;                           \
+        (db)->bytes_allocated = 0;                          \
+        (db)->bytes_used = 0;                               \
     } while (0)
 
 #define PMIX_EXTRACT_DATA_BUFFER(b, db)                 \

opal/mca/pmix/pmix3x/pmix/src/common/pmix_iof.c

@@ -0,0 +1,175 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
+ * Copyright (c) 2016      Mellanox Technologies, Inc.
+ *                         All rights reserved.
+ * Copyright (c) 2016      IBM Corporation.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include <src/include/pmix_config.h>
+
+#include <src/include/types.h>
+#include <src/include/pmix_stdint.h>
+#include <src/include/pmix_socket_errno.h>
+
+#include <pmix.h>
+#include <pmix_common.h>
+#include <pmix_server.h>
+#include <pmix_rename.h>
+
+#include "src/threads/threads.h"
+#include "src/util/argv.h"
+#include "src/util/error.h"
+#include "src/util/output.h"
+#include "src/mca/bfrops/bfrops.h"
+#include "src/mca/ptl/ptl.h"
+
+#include "src/client/pmix_client_ops.h"
+#include "src/server/pmix_server_ops.h"
+#include "src/include/pmix_globals.h"
+
+static void msgcbfunc(struct pmix_peer_t *peer,
+                       pmix_ptl_hdr_t *hdr,
+                       pmix_buffer_t *buf, void *cbdata)
+{
+    pmix_shift_caddy_t *cd = (pmix_shift_caddy_t*)cbdata;
+    int32_t m;
+    pmix_status_t rc, status;
+
+    /* unpack the return status */
+    m=1;
+    PMIX_BFROPS_UNPACK(rc, peer, buf, &status, &m, PMIX_STATUS);
+    if (PMIX_SUCCESS == rc && PMIX_SUCCESS == status) {
+        /* store the request on our list - we are in an event, and
+         * so this is safe */
+        pmix_list_append(&pmix_globals.iof_requests, &cd->iofreq->super);
+    } else if (PMIX_SUCCESS != rc) {
+        status = rc;
+        PMIX_RELEASE(cd->iofreq);
+    }
+
+    pmix_output_verbose(2, pmix_client_globals.iof_output,
+                        "pmix:iof_register returned status %s", PMIx_Error_string(status));
+
+    if (NULL != cd->cbfunc.opcbfn) {
+        cd->cbfunc.opcbfn(status, cd->cbdata);
+    }
+    PMIX_RELEASE(cd);
+}
+
+PMIX_EXPORT pmix_status_t PMIx_IOF_register(const pmix_proc_t procs[], size_t nprocs,
+                                            const pmix_info_t directives[], size_t ndirs,
+                                            pmix_iof_channel_t channel, pmix_iof_cbfunc_t cbfunc,
+                                            pmix_hdlr_reg_cbfunc_t regcbfunc, void *regcbdata)
+{
+    pmix_shift_caddy_t *cd;
+    pmix_cmd_t cmd = PMIX_IOF_CMD;
+    pmix_buffer_t *msg;
+    pmix_status_t rc;
+
+    PMIX_ACQUIRE_THREAD(&pmix_global_lock);
+
+    pmix_output_verbose(2, pmix_client_globals.iof_output,
+                        "pmix:iof_register");
+
+    if (pmix_globals.init_cntr <= 0) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return PMIX_ERR_INIT;
+    }
+
+    /* if we are a server, we cannot do this */
+    if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return PMIX_ERR_NOT_SUPPORTED;
+    }
+
+    /* if we aren't connected, don't attempt to send */
+    if (!pmix_globals.connected) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return PMIX_ERR_UNREACH;
+    }
+    PMIX_RELEASE_THREAD(&pmix_global_lock);
+
+    /* send this request to the server */
+    cd = PMIX_NEW(pmix_shift_caddy_t);
+    if (NULL == cd) {
+        return PMIX_ERR_NOMEM;
+    }
+    cd->cbfunc.hdlrregcbfn = regcbfunc;
+    cd->cbdata = regcbdata;
+    /* setup the request item */
+    cd->iofreq = PMIX_NEW(pmix_iof_req_t);
+    if (NULL == cd->iofreq) {
+        PMIX_RELEASE(cd);
+        return PMIX_ERR_NOMEM;
+    }
+    /* retain the channels and cbfunc */
+    cd->iofreq->channels = channel;
+    cd->iofreq->cbfunc = cbfunc;
+    /* we don't need the source specifications - only the
+    * server cares as it will filter against them */
+
+    /* setup the registration cmd */
+    msg = PMIX_NEW(pmix_buffer_t);
+    if (NULL == msg) {
+        PMIX_RELEASE(cd->iofreq);
+        PMIX_RELEASE(cd);
+        return PMIX_ERR_NOMEM;
+    }
+    PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
+                     msg, &cmd, 1, PMIX_COMMAND);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        goto cleanup;
+    }
+    PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
+                     msg, &nprocs, 1, PMIX_SIZE);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        goto cleanup;
+    }
+    PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
+                     msg, procs, nprocs, PMIX_PROC);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        goto cleanup;
+    }
+    PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
+                     msg, &ndirs, 1, PMIX_SIZE);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        goto cleanup;
+    }
+    if (0 < ndirs) {
+        PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
+                         msg, directives, ndirs, PMIX_INFO);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            goto cleanup;
+        }
+    }
+    PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
+                     msg, &channel, 1, PMIX_IOF_CHANNEL);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        goto cleanup;
+    }
+
+    pmix_output_verbose(2, pmix_client_globals.iof_output,
+                        "pmix:iof_request sending to server");
+    PMIX_PTL_SEND_RECV(rc, pmix_client_globals.myserver,
+                       msg, msgcbfunc, (void*)cd);
+
+  cleanup:
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_RELEASE(msg);
+        PMIX_RELEASE(cd->iofreq);
+        PMIX_RELEASE(cd);
+    }
+    return rc;
+}

opal/mca/pmix/pmix3x/pmix/src/common/pmix_security.c

@@ -0,0 +1,420 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2014-2017 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2016      Mellanox Technologies, Inc.
+ *                         All rights reserved.
+ * Copyright (c) 2016      IBM Corporation.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include <src/include/pmix_config.h>
+
+#include <src/include/types.h>
+#include <src/include/pmix_stdint.h>
+#include <src/include/pmix_socket_errno.h>
+
+#include <pmix.h>
+#include <pmix_common.h>
+#include <pmix_server.h>
+#include <pmix_rename.h>
+
+#include "src/threads/threads.h"
+#include "src/util/argv.h"
+#include "src/util/error.h"
+#include "src/util/output.h"
+#include "src/mca/bfrops/bfrops.h"
+#include "src/mca/psec/psec.h"
+#include "src/mca/ptl/ptl.h"
+
+#include "src/client/pmix_client_ops.h"
+#include "src/server/pmix_server_ops.h"
+#include "src/include/pmix_globals.h"
+
+static void getcbfunc(struct pmix_peer_t *peer,
+                      pmix_ptl_hdr_t *hdr,
+                      pmix_buffer_t *buf, void *cbdata)
+{
+    pmix_query_caddy_t *cd = (pmix_query_caddy_t*)cbdata;
+    pmix_status_t rc, status;
+    int cnt;
+    pmix_byte_object_t cred;
+    pmix_info_t *info = NULL;
+    size_t ninfo = 0;
+
+    pmix_output_verbose(2, pmix_globals.debug_output,
+                        "pmix:security cback from server with %d bytes",
+                        (int)buf->bytes_used);
+
+    /* a zero-byte buffer indicates that this recv is being
+     * completed due to a lost connection */
+    if (PMIX_BUFFER_IS_EMPTY(buf)) {
+        /* release the caller */
+        if (NULL != cd->credcbfunc) {
+            cd->credcbfunc(PMIX_ERR_COMM_FAILURE, NULL, NULL, 0, cd->cbdata);
+        }
+        PMIX_RELEASE(cd);
+        return;
+    }
+
+    /* unpack the status */
+    cnt = 1;
+    PMIX_BFROPS_UNPACK(rc, peer, buf, &status, &cnt, PMIX_STATUS);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        goto complete;
+    }
+    if (PMIX_SUCCESS != status) {
+        goto complete;
+    }
+
+    /* unpack the credential */
+    cnt = 1;
+    PMIX_BFROPS_UNPACK(rc, peer, buf, &cred, &cnt, PMIX_BYTE_OBJECT);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        goto complete;
+    }
+
+    /* unpack any returned info */
+    cnt = 1;
+    PMIX_BFROPS_UNPACK(rc, peer, buf, &ninfo, &cnt, PMIX_SIZE);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        goto complete;
+    }
+    if (0 < ninfo) {
+        PMIX_INFO_CREATE(info, ninfo);
+        cnt = ninfo;
+        PMIX_BFROPS_UNPACK(rc, peer, buf, info, &cnt, PMIX_INFO);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            goto complete;
+        }
+    }
+
+  complete:
+    pmix_output_verbose(2, pmix_globals.debug_output,
+                        "pmix:security cback from server releasing");
+    /* release the caller */
+    if (NULL != cd->credcbfunc) {
+        cd->credcbfunc(status, &cred, info, ninfo, cd->cbdata);
+    }
+    PMIX_BYTE_OBJECT_DESTRUCT(&cred);
+    if (NULL != info) {
+        PMIX_INFO_FREE(info, ninfo);
+    }
+    PMIX_RELEASE(cd);
+}
+
+PMIX_EXPORT pmix_status_t PMIx_Get_credential(const pmix_info_t info[], size_t ninfo,
+                                              pmix_credential_cbfunc_t cbfunc, void *cbdata)
+{
+    pmix_buffer_t *msg;
+    pmix_cmd_t cmd = PMIX_GET_CREDENTIAL_CMD;
+    pmix_status_t rc;
+    pmix_query_caddy_t *cb;
+    pmix_byte_object_t cred;
+    pmix_info_t *results = NULL;
+    size_t nresults = 0;
+
+    PMIX_ACQUIRE_THREAD(&pmix_global_lock);
+
+    pmix_output_verbose(2, pmix_globals.debug_output,
+                        "pmix: Get_credential called with %d info", (int)ninfo);
+
+    if (pmix_globals.init_cntr <= 0) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return PMIX_ERR_INIT;
+    }
+
+    /* if we are the server */
+    if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        /* if the host doesn't support this operation,
+         * see if we can generate it ourselves */
+        if (NULL == pmix_host_server.get_credential) {
+            PMIX_BYTE_OBJECT_CONSTRUCT(&cred);
+            PMIX_PSEC_CREATE_CRED(rc, pmix_globals.mypeer, info, ninfo,
+                                  &results, &nresults, &cred);
+            if (PMIX_SUCCESS == rc) {
+                /* pass it back in the callback function */
+                if (NULL != cbfunc) {
+                    cbfunc(PMIX_SUCCESS, &cred, results, nresults, cbdata);
+                    if (NULL != results) {
+                        PMIX_INFO_FREE(results, nresults);
+                    }
+                    PMIX_BYTE_OBJECT_DESTRUCT(&cred);
+                }
+            }
+            return rc;
+        }
+        /* the host is available, so let them try to create it */
+        pmix_output_verbose(2, pmix_globals.debug_output,
+                            "pmix:get_credential handed to RM");
+        rc = pmix_host_server.get_credential(&pmix_globals.myid,
+                                             info, ninfo,
+                                             cbfunc, cbdata);
+        return rc;
+    }
+
+    /* if we are a client or tool and we aren't connected, see
+     * if one of our internal plugins is capable of meeting the request */
+    if (!pmix_globals.connected) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        PMIX_BYTE_OBJECT_CONSTRUCT(&cred);
+        PMIX_PSEC_CREATE_CRED(rc, pmix_globals.mypeer, info, ninfo,
+                              &results, &nresults, &cred);
+        if (PMIX_SUCCESS == rc) {
+            /* pass it back in the callback function */
+            if (NULL != cbfunc) {
+                cbfunc(PMIX_SUCCESS, &cred, results, nresults, cbdata);
+                if (NULL != results) {
+                    PMIX_INFO_FREE(results, nresults);
+                }
+                PMIX_BYTE_OBJECT_DESTRUCT(&cred);
+            }
+        }
+        return rc;
+    }
+    PMIX_RELEASE_THREAD(&pmix_global_lock);
+
+    /* if we are a client, then relay this request to the server */
+    msg = PMIX_NEW(pmix_buffer_t);
+    /* pack the cmd */
+    PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
+                     msg, &cmd, 1, PMIX_COMMAND);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_RELEASE(msg);
+        return rc;
+    }
+
+    /* pack the directives */
+    PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
+                     msg, &ninfo, 1, PMIX_SIZE);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_RELEASE(msg);
+        return rc;
+    }
+    if (0 < ninfo) {
+        PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
+                         msg, info, ninfo, PMIX_INFO);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            PMIX_RELEASE(msg);
+            return rc;
+        }
+    }
+
+    /* create a callback object as we need to pass it to the
+     * recv routine so we know which callback to use when
+     * the return message is recvd */
+    cb = PMIX_NEW(pmix_query_caddy_t);
+    cb->credcbfunc = cbfunc;
+    cb->cbdata = cbdata;
+
+    /* push the message into our event base to send to the server */
+    PMIX_PTL_SEND_RECV(rc, pmix_client_globals.myserver,
+                       msg, getcbfunc, (void*)cb);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_RELEASE(msg);
+        PMIX_RELEASE(cb);
+    }
+
+    return rc;
+}
+
+static void valid_cbfunc(struct pmix_peer_t *peer,
+                         pmix_ptl_hdr_t *hdr,
+                         pmix_buffer_t *buf, void *cbdata)
+{
+    pmix_query_caddy_t *cd = (pmix_query_caddy_t*)cbdata;
+    pmix_status_t rc, status;
+    int cnt;
+    pmix_info_t *info = NULL;
+    size_t ninfo = 0;
+
+    pmix_output_verbose(2, pmix_globals.debug_output,
+                        "pmix:security cback from server with %d bytes",
+                        (int)buf->bytes_used);
+
+    /* a zero-byte buffer indicates that this recv is being
+     * completed due to a lost connection */
+    if (PMIX_BUFFER_IS_EMPTY(buf)) {
+        /* release the caller */
+        if (NULL != cd->validcbfunc) {
+            cd->validcbfunc(PMIX_ERR_COMM_FAILURE, NULL, 0, cd->cbdata);
+        }
+        PMIX_RELEASE(cd);
+        return;
+    }
+
+    /* unpack the status */
+    cnt = 1;
+    PMIX_BFROPS_UNPACK(rc, peer, buf, &status, &cnt, PMIX_STATUS);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        goto complete;
+    }
+    if (PMIX_SUCCESS != status) {
+        goto complete;
+    }
+
+    /* unpack any returned info */
+    cnt = 1;
+    PMIX_BFROPS_UNPACK(rc, peer, buf, &ninfo, &cnt, PMIX_SIZE);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        goto complete;
+    }
+    if (0 < ninfo) {
+        PMIX_INFO_CREATE(info, ninfo);
+        cnt = ninfo;
+        PMIX_BFROPS_UNPACK(rc, peer, buf, info, &cnt, PMIX_INFO);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            goto complete;
+        }
+    }
+
+  complete:
+    pmix_output_verbose(2, pmix_globals.debug_output,
+                        "pmix:security cback from server releasing");
+    /* release the caller */
+    if (NULL != cd->validcbfunc) {
+        cd->validcbfunc(status, info, ninfo, cd->cbdata);
+    }
+    if (NULL != info) {
+        PMIX_INFO_FREE(info, ninfo);
+    }
+    PMIX_RELEASE(cd);
+}
+
+PMIX_EXPORT pmix_status_t PMIx_Validate_credential(const pmix_byte_object_t *cred,
+                                                   const pmix_info_t directives[], size_t ndirs,
+                                                   pmix_validation_cbfunc_t cbfunc, void *cbdata)
+{
+    pmix_buffer_t *msg;
+    pmix_cmd_t cmd = PMIX_VALIDATE_CRED_CMD;
+    pmix_status_t rc;
+    pmix_query_caddy_t *cb;
+    pmix_info_t *results = NULL;
+    size_t nresults = 0;
+
+    PMIX_ACQUIRE_THREAD(&pmix_global_lock);
+
+    pmix_output_verbose(2, pmix_globals.debug_output,
+                        "pmix: monitor called");
+
+    if (pmix_globals.init_cntr <= 0) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return PMIX_ERR_INIT;
+    }
+
+    /* if we are the server */
+    if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        /* if the host doesn't support this operation,
+         * see if we can validate it ourselves */
+        if (NULL == pmix_host_server.validate_credential) {
+            PMIX_PSEC_VALIDATE_CRED(rc, pmix_globals.mypeer,
+                                    directives, ndirs,
+                                    &results, &nresults, cred);
+            if (PMIX_SUCCESS == rc) {
+                /* pass it back in the callback function */
+                if (NULL != cbfunc) {
+                    cbfunc(PMIX_SUCCESS, results, nresults, cbdata);
+                    if (NULL != results) {
+                        PMIX_INFO_FREE(results, nresults);
+                    }
+                }
+            }
+            return rc;
+        }
+        /* the host is available, so let them try to validate it */
+        pmix_output_verbose(2, pmix_globals.debug_output,
+                            "pmix:get_credential handed to RM");
+        rc = pmix_host_server.validate_credential(&pmix_globals.myid, cred,
+                                                  directives, ndirs, cbfunc, cbdata);
+        return rc;
+    }
+
+    /* if we are a client or tool and we aren't connected, see
+     * if one of our internal plugins is capable of meeting the request */
+    if (!pmix_globals.connected) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        PMIX_PSEC_VALIDATE_CRED(rc, pmix_globals.mypeer,
+                                directives, ndirs,
+                                &results, &nresults, cred);
+        if (PMIX_SUCCESS == rc) {
+            /* pass it back in the callback function */
+            if (NULL != cbfunc) {
+                cbfunc(PMIX_SUCCESS, results, nresults, cbdata);
+                if (NULL != results) {
+                    PMIX_INFO_FREE(results, nresults);
+                }
+            }
+        }
+        return rc;
+    }
+    PMIX_RELEASE_THREAD(&pmix_global_lock);
+
+    /* if we are a client, then relay this request to the server */
+    msg = PMIX_NEW(pmix_buffer_t);
+    /* pack the cmd */
+    PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
+                     msg, &cmd, 1, PMIX_COMMAND);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_RELEASE(msg);
+        return rc;
+    }
+
+    /* pack the credential */
+    PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
+                     msg, cred, 1, PMIX_BYTE_OBJECT);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_RELEASE(msg);
+        return rc;
+    }
+
+    /* pack the directives */
+    PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
+                     msg, &ndirs, 1, PMIX_SIZE);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_RELEASE(msg);
+        return rc;
+    }
+    if (0 < ndirs) {
+        PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
+                         msg, directives, ndirs, PMIX_INFO);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            PMIX_RELEASE(msg);
+            return rc;
+        }
+    }
+
+    /* create a callback object as we need to pass it to the
+     * recv routine so we know which callback to use when
+     * the return message is recvd */
+    cb = PMIX_NEW(pmix_query_caddy_t);
+    cb->validcbfunc = cbfunc;
+    cb->cbdata = cbdata;
+
+    /* push the message into our event base to send to the server */
+    PMIX_PTL_SEND_RECV(rc, pmix_client_globals.myserver,
+                       msg, valid_cbfunc, (void*)cb);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_RELEASE(msg);
+        PMIX_RELEASE(cb);
+    }
+
+    return rc;
+}

opal/mca/pmix/pmix3x/pmix/src/common/pmix_strings.c

@@ -208,7 +208,48 @@ PMIX_EXPORT const char* pmix_command_string(pmix_cmd_t cmd)
             return "DEREGISTER EVENT HANDLER";
         case PMIX_QUERY_CMD:
             return "QUERY";
+        case PMIX_LOG_CMD:
+            return "LOG";
+        case PMIX_ALLOC_CMD:
+            return "ALLOCATE";
+        case PMIX_JOB_CONTROL_CMD:
+            return "JOB CONTROL";
+        case PMIX_MONITOR_CMD:
+            return "MONITOR";
+        case PMIX_IOF_CMD:
+            return "IOF";
         default:
             return "UNKNOWN";
     }
 }
+
+/* this is not a thread-safe implementation. To correctly implement this,
+ * we need to port the thread-safe data code from OPAL and use it here */
+static char answer[300];
+
+PMIX_EXPORT const char* PMIx_IOF_channel_string(pmix_iof_channel_t channel)
+{
+    size_t cnt=0;
+
+    memset(answer, 0, sizeof(answer));
+    if (PMIX_FWD_STDIN_CHANNEL & channel) {
+        strncpy(&answer[cnt], "STDIN ", strlen("STDIN "));
+        cnt += strlen("STDIN ");
+    }
+    if (PMIX_FWD_STDOUT_CHANNEL & channel) {
+        strncpy(&answer[cnt], "STDOUT ", strlen("STDOUT "));
+        cnt += strlen("STDOUT ");
+    }
+    if (PMIX_FWD_STDERR_CHANNEL & channel) {
+        strncpy(&answer[cnt], "STDERR ", strlen("STDERR "));
+        cnt += strlen("STDERR ");
+    }
+    if (PMIX_FWD_STDDIAG_CHANNEL & channel) {
+        strncpy(&answer[cnt], "STDDIAG ", strlen("STDDIAG "));
+        cnt += strlen("STDDIAG ");
+    }
+    if (0 == cnt) {
+        strncpy(&answer[cnt], "NONE", strlen("NONE"));
+    }
+    return answer;
+}

opal/mca/pmix/pmix3x/pmix/src/event/pmix_event_registration.c

@@ -41,7 +41,7 @@
     pmix_info_t *info;
     size_t ninfo;
     pmix_notification_fn_t evhdlr;
-    pmix_evhdlr_reg_cbfunc_t evregcbfn;
+    pmix_hdlr_reg_cbfunc_t evregcbfn;
     void *cbdata;
 } pmix_rshift_caddy_t;
 static void rscon(pmix_rshift_caddy_t *p)
@@ -766,7 +766,7 @@ static void reg_event_hdlr(int sd, short args, void *cbdata)
 PMIX_EXPORT void PMIx_Register_event_handler(pmix_status_t codes[], size_t ncodes,
                                              pmix_info_t info[], size_t ninfo,
                                              pmix_notification_fn_t event_hdlr,
-                                             pmix_evhdlr_reg_cbfunc_t cbfunc,
+                                             pmix_hdlr_reg_cbfunc_t cbfunc,
                                              void *cbdata)
 {
     pmix_rshift_caddy_t *cd;

opal/mca/pmix/pmix3x/pmix/src/include/pmix_config_bottom.h

@@ -337,6 +337,18 @@
 #    define __pmix_attribute_destructor__
 #endif
 
+#if PMIX_HAVE_ATTRIBUTE_OPTNONE
+#    define __pmix_attribute_optnone__    __attribute__((__optnone__))
+#else
+#    define __pmix_attribute_optnone__
+#endif
+
+#if PMIX_HAVE_ATTRIBUTE_EXTENSION
+#    define __pmix_attribute_extension__    __extension__
+#else
+#    define __pmix_attribute_extension__
+#endif
+
 /*
  * Do we have <stdint.h>?
  */

opal/mca/pmix/pmix3x/pmix/src/include/pmix_globals.c

@@ -50,7 +50,6 @@
 #include "src/class/pmix_list.h"
 #include "src/threads/threads.h"
 #include "src/util/argv.h"
-#include "src/util/error.h"
 #include "src/util/os_path.h"
 
 static void dirpath_destroy(char *path, pmix_cleanup_dir_t *cd,
@@ -167,6 +166,7 @@ PMIX_EXPORT PMIX_CLASS_INSTANCE(pmix_rank_info_t,
 static void pcon(pmix_peer_t *p)
 {
     p->proc_type = PMIX_PROC_UNDEF;
+    p->protocol = PMIX_PROTOCOL_UNDEF;
     p->finalized = false;
     p->info = NULL;
     p->proc_cnt = 0;
@@ -178,6 +178,7 @@ static void pcon(pmix_peer_t *p)
     PMIX_CONSTRUCT(&p->send_queue, pmix_list_t);
     p->send_msg = NULL;
     p->recv_msg = NULL;
+    p->commit_cnt = 0;
     PMIX_CONSTRUCT(&p->epilog.cleanup_dirs, pmix_list_t);
     PMIX_CONSTRUCT(&p->epilog.cleanup_files, pmix_list_t);
     PMIX_CONSTRUCT(&p->epilog.ignores, pmix_list_t);
@@ -217,6 +218,24 @@ PMIX_EXPORT PMIX_CLASS_INSTANCE(pmix_peer_t,
                                 pmix_object_t,
                                 pcon, pdes);
 
+static void iofreqcon(pmix_iof_req_t *p)
+{
+    p->peer = NULL;
+    memset(&p->pname, 0, sizeof(pmix_name_t));
+    p->channels = PMIX_FWD_NO_CHANNELS;
+    p->cbfunc = NULL;
+}
+static void iofreqdes(pmix_iof_req_t *p)
+{
+    if (NULL != p->peer) {
+        PMIX_RELEASE(p->peer);
+    }
+}
+PMIX_EXPORT PMIX_CLASS_INSTANCE(pmix_iof_req_t,
+                                pmix_list_item_t,
+                                iofreqcon, iofreqdes);
+
+
 static void scon(pmix_shift_caddy_t *p)
 {
     PMIX_CONSTRUCT_LOCK(&p->lock);
@@ -232,6 +251,7 @@ static void scon(pmix_shift_caddy_t *p)
     p->directives = NULL;
     p->ndirs = 0;
     p->evhdlr = NULL;
+    p->iofreq = NULL;
     p->kv = NULL;
     p->vptr = NULL;
     p->cd = NULL;
@@ -304,14 +324,18 @@ static void qcon(pmix_query_caddy_t *p)
     p->ntargets = 0;
     p->info = NULL;
     p->ninfo = 0;
+    PMIX_BYTE_OBJECT_CONSTRUCT(&p->bo);
     p->cbfunc = NULL;
     p->valcbfunc = NULL;
     p->cbdata = NULL;
     p->relcbfunc = NULL;
+    p->credcbfunc = NULL;
+    p->validcbfunc = NULL;
 }
 static void qdes(pmix_query_caddy_t *p)
 {
     PMIX_DESTRUCT_LOCK(&p->lock);
+    PMIX_BYTE_OBJECT_DESTRUCT(&p->bo);
 }
 PMIX_EXPORT PMIX_CLASS_INSTANCE(pmix_query_caddy_t,
                                 pmix_object_t,
@@ -332,7 +356,7 @@ void pmix_execute_epilog(pmix_epilog_t *epi)
         rc = stat(cf->path, &statbuf);
         if (0 != rc) {
             pmix_output_verbose(10, pmix_globals.debug_output,
-                                "File %s failed to stat: %s", cf->path, strerror(rc));
+                                "File %s failed to stat: %d", cf->path, rc);
             continue;
         }
         if (statbuf.st_uid != epi->uid ||
@@ -347,7 +371,7 @@ void pmix_execute_epilog(pmix_epilog_t *epi)
         rc = unlink(cf->path);
         if (0 != rc) {
             pmix_output_verbose(10, pmix_globals.debug_output,
-                                "File %s failed to unlink: %s", cf->path, strerror(rc));
+                                "File %s failed to unlink: %d", cf->path, rc);
         }
         pmix_list_remove_item(&epi->cleanup_files, &cf->super);
         PMIX_RELEASE(cf);
@@ -361,7 +385,7 @@ void pmix_execute_epilog(pmix_epilog_t *epi)
         rc = stat(cd->path, &statbuf);
         if (0 != rc) {
             pmix_output_verbose(10, pmix_globals.debug_output,
-                                "Directory %s failed to stat: %s", cd->path, strerror(rc));
+                                "Directory %s failed to stat: %d", cd->path, rc);
             continue;
         }
         if (statbuf.st_uid != epi->uid ||
@@ -387,12 +411,11 @@ void pmix_execute_epilog(pmix_epilog_t *epi)
 static void dirpath_destroy(char *path, pmix_cleanup_dir_t *cd, pmix_epilog_t *epi)
 {
     int rc;
-    bool is_dir = false, ignore;
+    bool is_dir = false;
     DIR *dp;
     struct dirent *ep;
     char *filenm;
     struct stat buf;
-    size_t n;
     pmix_cleanup_file_t *cf;
 
     if (NULL == path) {  /* protect against error */
@@ -427,13 +450,17 @@ static void dirpath_destroy(char *path, pmix_cleanup_dir_t *cd, pmix_epilog_t *e
          */
         filenm = pmix_os_path(false, path, ep->d_name, NULL);
 
-        /* if this path is it to be ignored, then do so */
+        /* if this path is to be ignored, then do so */
         PMIX_LIST_FOREACH(cf, &epi->ignores, pmix_cleanup_file_t) {
             if (0 == strcmp(cf->path, filenm)) {
                 free(filenm);
-                continue;
+                filenm = NULL;
+                break;
             }
         }
+        if (NULL == filenm) {
+            continue;
+        }
 
         /* Check to see if it is a directory */
         is_dir = false;
@@ -483,7 +510,6 @@ static void dirpath_destroy(char *path, pmix_cleanup_dir_t *cd, pmix_epilog_t *e
     /* Done with this directory */
     closedir(dp);
 
-  cleanup:
     /* If the directory is empty, then remove it unless we
      * were told to leave it */
     if (0 == strcmp(path, cd->path) && cd->leave_topdir) {

opal/mca/pmix/pmix3x/pmix/src/include/pmix_globals.h

@@ -10,7 +10,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2014-2018 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -31,6 +31,7 @@
 #endif
 #include PMIX_EVENT_HEADER
 
+#include <pmix.h>
 #include <pmix_common.h>
 
 #include "src/class/pmix_hash_table.h"
@@ -71,26 +72,29 @@ PMIX_CLASS_DECLARATION(pmix_namelist_t);
 typedef uint8_t pmix_cmd_t;
 
 /* define some commands */
-#define PMIX_REQ_CMD             0
+#define PMIX_REQ_CMD                 0
-#define PMIX_ABORT_CMD           1
+#define PMIX_ABORT_CMD               1
-#define PMIX_COMMIT_CMD          2
+#define PMIX_COMMIT_CMD              2
-#define PMIX_FENCENB_CMD         3
+#define PMIX_FENCENB_CMD             3
-#define PMIX_GETNB_CMD           4
+#define PMIX_GETNB_CMD               4
-#define PMIX_FINALIZE_CMD        5
+#define PMIX_FINALIZE_CMD            5
-#define PMIX_PUBLISHNB_CMD       6
+#define PMIX_PUBLISHNB_CMD           6
-#define PMIX_LOOKUPNB_CMD        7
+#define PMIX_LOOKUPNB_CMD            7
-#define PMIX_UNPUBLISHNB_CMD     8
+#define PMIX_UNPUBLISHNB_CMD         8
-#define PMIX_SPAWNNB_CMD         9
+#define PMIX_SPAWNNB_CMD             9
-#define PMIX_CONNECTNB_CMD      10
+#define PMIX_CONNECTNB_CMD          10
-#define PMIX_DISCONNECTNB_CMD   11
+#define PMIX_DISCONNECTNB_CMD       11
-#define PMIX_NOTIFY_CMD         12
+#define PMIX_NOTIFY_CMD             12
-#define PMIX_REGEVENTS_CMD      13
+#define PMIX_REGEVENTS_CMD          13
-#define PMIX_DEREGEVENTS_CMD    14
+#define PMIX_DEREGEVENTS_CMD        14
-#define PMIX_QUERY_CMD          15
+#define PMIX_QUERY_CMD              15
-#define PMIX_LOG_CMD            16
+#define PMIX_LOG_CMD                16
-#define PMIX_ALLOC_CMD          17
+#define PMIX_ALLOC_CMD              17
-#define PMIX_JOB_CONTROL_CMD    18
+#define PMIX_JOB_CONTROL_CMD        18
-#define PMIX_MONITOR_CMD        19
+#define PMIX_MONITOR_CMD            19
+#define PMIX_GET_CREDENTIAL_CMD     20
+#define PMIX_VALIDATE_CRED_CMD      21
+#define PMIX_IOF_CMD                22
 
 /* provide a "pretty-print" function for cmds */
 const char* pmix_command_string(pmix_cmd_t cmd);
@@ -202,6 +206,7 @@ typedef struct pmix_peer_t {
     pmix_nspace_t *nptr;            // point to the nspace object for this process
     pmix_rank_info_t *info;
     pmix_proc_type_t proc_type;
+    pmix_listener_protocol_t protocol;
     int proc_cnt;
     int index;                      // index into the local clients array on the server
     int sd;
@@ -213,21 +218,23 @@ typedef struct pmix_peer_t {
     pmix_list_t send_queue;         /**< list of messages to send */
     pmix_ptl_send_t *send_msg;      /**< current send in progress */
     pmix_ptl_recv_t *recv_msg;      /**< current recv in progress */
+    int commit_cnt;
     pmix_epilog_t epilog;           /**< things to be performed upon
                                          termination of this peer */
 } pmix_peer_t;
 PMIX_CLASS_DECLARATION(pmix_peer_t);
 
 
-/* define an object for moving a send
+/* tracker for IOF requests */
- * request into the server's event base
- * - instanced in pmix_server_ops.c */
 typedef struct {
     pmix_list_item_t super;
-    pmix_ptl_hdr_t hdr;
     pmix_peer_t *peer;
-} pmix_server_caddy_t;
+    pmix_name_t pname;
-PMIX_CLASS_DECLARATION(pmix_server_caddy_t);
+    pmix_iof_channel_t channels;
+    pmix_iof_cbfunc_t cbfunc;
+} pmix_iof_req_t;
+PMIX_CLASS_DECLARATION(pmix_iof_req_t);
+
 
 /* caddy for query requests */
 typedef struct {
@@ -241,9 +248,12 @@ typedef struct {
     size_t ntargets;
     pmix_info_t *info;
     size_t ninfo;
+    pmix_byte_object_t bo;
     pmix_info_cbfunc_t cbfunc;
     pmix_value_cbfunc_t valcbfunc;
     pmix_release_cbfunc_t relcbfunc;
+    pmix_credential_cbfunc_t credcbfunc;
+    pmix_validation_cbfunc_t validcbfunc;
     void *cbdata;
 } pmix_query_caddy_t;
 PMIX_CLASS_DECLARATION(pmix_query_caddy_t);
@@ -273,6 +283,20 @@ typedef struct {
 } pmix_server_trkr_t;
 PMIX_CLASS_DECLARATION(pmix_server_trkr_t);
 
+/* define an object for moving a send
+ * request into the server's event base and
+ * dealing with some request timeouts
+ * - instanced in pmix_server_ops.c */
+typedef struct {
+    pmix_list_item_t super;
+    pmix_event_t ev;
+    bool event_active;
+    pmix_server_trkr_t *trk;
+    pmix_ptl_hdr_t hdr;
+    pmix_peer_t *peer;
+} pmix_server_caddy_t;
+PMIX_CLASS_DECLARATION(pmix_server_caddy_t);
+
 /****    THREAD-RELATED    ****/
  /* define a caddy for thread-shifting operations */
  typedef struct {
@@ -291,6 +315,7 @@ PMIX_CLASS_DECLARATION(pmix_server_trkr_t);
     pmix_info_t *directives;
     size_t ndirs;
     pmix_notification_fn_t evhdlr;
+    pmix_iof_req_t *iofreq;
     pmix_kval_t *kv;
     pmix_value_t *vptr;
     pmix_server_caddy_t *cd;
@@ -298,9 +323,8 @@ PMIX_CLASS_DECLARATION(pmix_server_trkr_t);
     bool enviro;
     union {
        pmix_release_cbfunc_t relfn;
-       pmix_evhdlr_reg_cbfunc_t evregcbfn;
+       pmix_hdlr_reg_cbfunc_t hdlrregcbfn;
        pmix_op_cbfunc_t opcbfn;
-       pmix_evhdlr_reg_cbfunc_t errregcbfn;
     } cbfunc;
     void *cbdata;
     size_t ref;
@@ -324,7 +348,7 @@ typedef struct {
         pmix_lookup_cbfunc_t lookupfn;
         pmix_spawn_cbfunc_t spawnfn;
         pmix_connect_cbfunc_t cnctfn;
-        pmix_evhdlr_reg_cbfunc_t errregfn;
+        pmix_hdlr_reg_cbfunc_t hdlrregfn;
     } cbfunc;
     size_t errhandler_ref;
     void *cbdata;
@@ -398,6 +422,7 @@ typedef struct {
     bool commits_pending;
     struct timeval event_window;
     pmix_list_t cached_events;          // events waiting in the window prior to processing
+    pmix_list_t iof_requests;           // list of pmix_iof_req_t IOF requests
     pmix_ring_buffer_t notifications;   // ring buffer of pending notifications
     /* processes also need a place where they can store
      * their own internal data - e.g., data provided by

opal/mca/pmix/pmix3x/pmix/src/mca/base/pmix_mca_base_cmd_line.c

@@ -231,7 +231,9 @@ void pmix_mca_base_cmd_line_wrap_args(char **args)
                 return;
             }
             i += 2;
-            asprintf(&tstr, "\"%s\"", args[i]);
+            if (0 > asprintf(&tstr, "\"%s\"", args[i])) {
+                return;
+            }
             free(args[i]);
             args[i] = tstr;
         }

opal/mca/pmix/pmix3x/pmix/src/mca/base/pmix_mca_base_var.c

@@ -13,7 +13,7 @@
  * Copyright (c) 2008-2015 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2012-2015 Los Alamos National Security, LLC. All rights
  *                         reserved.
- * Copyright (c) 2014-2017 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
@@ -558,7 +558,7 @@ int pmix_mca_base_var_cache_files(bool rel_path_search)
     if (NULL != pmix_mca_base_var_file_prefix) {
        resolve_relative_paths(&pmix_mca_base_var_file_prefix, pmix_mca_base_param_file_path, rel_path_search, &pmix_mca_base_var_files, PMIX_ENV_SEP);
     }
-    read_files (pmix_mca_base_var_files, &pmix_mca_base_var_file_values, PMIX_ENV_SEP);
+    read_files (pmix_mca_base_var_files, &pmix_mca_base_var_file_values, ',');
 
     if (NULL != pmix_mca_base_envar_file_prefix) {
        resolve_relative_paths(&pmix_mca_base_envar_file_prefix, pmix_mca_base_param_file_path, rel_path_search, &pmix_mca_base_envar_files, ',');

opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/base.h

@@ -379,6 +379,8 @@ PMIX_EXPORT pmix_status_t pmix_bfrops_base_pack_val(pmix_buffer_t *buffer,
                                                     pmix_value_t *p);
 PMIX_EXPORT pmix_status_t pmix_bfrops_base_pack_alloc_directive(pmix_buffer_t *buffer, const void *src,
                                                                 int32_t num_vals, pmix_data_type_t type);
+PMIX_EXPORT pmix_status_t pmix_bfrops_base_pack_iof_channel(pmix_buffer_t *buffer, const void *src,
+                                                            int32_t num_vals, pmix_data_type_t type);
 
 /*
 * "Standard" unpack functions
@@ -466,6 +468,8 @@ PMIX_EXPORT pmix_status_t pmix_bfrops_base_unpack_query(pmix_buffer_t *buffer, v
                                                         int32_t *num_vals, pmix_data_type_t type);
 PMIX_EXPORT pmix_status_t pmix_bfrops_base_unpack_alloc_directive(pmix_buffer_t *buffer, void *dest,
                                                                   int32_t *num_vals, pmix_data_type_t type);
+PMIX_EXPORT pmix_status_t pmix_bfrops_base_unpack_iof_channel(pmix_buffer_t *buffer, void *dest,
+                                                              int32_t *num_vals, pmix_data_type_t type);
 /**** DEPRECATED ****/
 PMIX_EXPORT pmix_status_t pmix_bfrops_base_unpack_array(pmix_buffer_t *buffer, void *dest,
                                                         int32_t *num_vals, pmix_data_type_t type);
@@ -637,6 +641,9 @@ PMIX_EXPORT pmix_status_t pmix_bfrops_base_print_rank(char **output, char *prefi
 PMIX_EXPORT pmix_status_t pmix_bfrops_base_print_alloc_directive(char **output, char *prefix,
                                                                  pmix_alloc_directive_t *src,
                                                                  pmix_data_type_t type);
+PMIX_EXPORT pmix_status_t pmix_bfrops_base_print_iof_channel(char **output, char *prefix,
+                                                            pmix_iof_channel_t *src,
+                                                            pmix_data_type_t type);
 
 /*
  * Common helper functions

opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/bfrop_base_copy.c

@@ -113,6 +113,7 @@ pmix_status_t pmix_bfrops_base_std_copy(void **dest, void *src,
 
     case PMIX_INT16:
     case PMIX_UINT16:
+    case PMIX_IOF_CHANNEL:
         datasize = 2;
         break;
 

opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/bfrop_base_pack.c

@@ -9,7 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2015-2017 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2015-2018 Intel, Inc. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -1270,3 +1270,9 @@ pmix_status_t pmix_bfrops_base_pack_array(pmix_buffer_t *buffer, const void *src
 
     return PMIX_SUCCESS;
 }
+
+PMIX_EXPORT pmix_status_t pmix_bfrops_base_pack_iof_channel(pmix_buffer_t *buffer, const void *src,
+                                                            int32_t num_vals, pmix_data_type_t type)
+{
+    return pmix_bfrops_base_pack_int16(buffer, src, num_vals, PMIX_UINT16);
+}

opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/bfrop_base_print.c

@@ -1636,6 +1636,35 @@ pmix_status_t pmix_bfrops_base_print_alloc_directive(char **output, char *prefix
     }
 }
 
+pmix_status_t pmix_bfrops_base_print_iof_channel(char **output, char *prefix,
+                                                 pmix_iof_channel_t *src,
+                                                 pmix_data_type_t type)
+{
+    char *prefx;
+    int ret;
+
+    /* deal with NULL prefix */
+    if (NULL == prefix) {
+        if (0 > asprintf(&prefx, " ")) {
+            return PMIX_ERR_NOMEM;
+        }
+    } else {
+        prefx = prefix;
+    }
+
+    ret = asprintf(output, "%sData type: PMIX_IOF_CHANNEL\tValue: %s",
+                   prefx, PMIx_IOF_channel_string(*src));
+    if (prefx != prefix) {
+        free(prefx);
+    }
+
+    if (0 > ret) {
+        return PMIX_ERR_OUT_OF_RESOURCE;
+    } else {
+        return PMIX_SUCCESS;
+    }
+}
+
 
 /**** DEPRECATED ****/
 pmix_status_t pmix_bfrops_base_print_array(char **output, char *prefix,

opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/bfrop_base_unpack.c

@@ -1603,6 +1603,11 @@ pmix_status_t pmix_bfrops_base_unpack_alloc_directive(pmix_buffer_t *buffer, voi
     return pmix_bfrops_base_unpack_byte(buffer, dest, num_vals, PMIX_UINT8);
 }
 
+pmix_status_t pmix_bfrops_base_unpack_iof_channel(pmix_buffer_t *buffer, void *dest,
+                                                  int32_t *num_vals, pmix_data_type_t type)
+{
+    return pmix_bfrops_base_unpack_int16(buffer, dest, num_vals, PMIX_UINT16);
+}
 
 /**** DEPRECATED ****/
 pmix_status_t pmix_bfrops_base_unpack_array(pmix_buffer_t *buffer, void *dest,

opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/v3/Makefile.am

@@ -0,0 +1,50 @@
+# -*- makefile -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2012      Los Alamos National Security, Inc.  All rights reserved.
+# Copyright (c) 2013-2017 Intel, Inc. All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+headers = bfrop_pmix3.h
+sources = \
+        bfrop_pmix3_component.c \
+        bfrop_pmix3.c
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if MCA_BUILD_pmix_bfrops_v3_DSO
+lib =
+lib_sources =
+component = mca_bfrops_v3.la
+component_sources = $(headers) $(sources)
+else
+lib = libmca_bfrops_v3.la
+lib_sources = $(headers) $(sources)
+component =
+component_sources =
+endif
+
+mcacomponentdir = $(pmixlibdir)
+mcacomponent_LTLIBRARIES = $(component)
+mca_bfrops_v3_la_SOURCES = $(component_sources)
+mca_bfrops_v3_la_LDFLAGS = -module -avoid-version
+
+noinst_LTLIBRARIES = $(lib)
+libmca_bfrops_v3_la_SOURCES = $(lib_sources)
+libmca_bfrops_v3_la_LDFLAGS = -module -avoid-version

opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/v3/bfrop_pmix3.c

@@ -0,0 +1,456 @@
+/*
+ * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2011 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2010-2011 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2011-2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2011-2013 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2013-2017 Intel, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+
+#include <src/include/pmix_config.h>
+
+#include "src/mca/bfrops/base/base.h"
+#include "bfrop_pmix3.h"
+
+static pmix_status_t init(void);
+static void finalize(void);
+static pmix_status_t pmix3_pack(pmix_buffer_t *buffer,
+                                const void *src, int num_vals,
+                                pmix_data_type_t type);
+static pmix_status_t pmix3_unpack(pmix_buffer_t *buffer, void *dest,
+                                  int32_t *num_vals, pmix_data_type_t type);
+static pmix_status_t pmix3_copy(void **dest, void *src,
+                                pmix_data_type_t type);
+static pmix_status_t pmix3_print(char **output, char *prefix,
+                                 void *src, pmix_data_type_t type);
+static pmix_status_t register_type(const char *name,
+                                   pmix_data_type_t type,
+                                   pmix_bfrop_pack_fn_t pack,
+                                   pmix_bfrop_unpack_fn_t unpack,
+                                   pmix_bfrop_copy_fn_t copy,
+                                   pmix_bfrop_print_fn_t print);
+static const char* data_type_string(pmix_data_type_t type);
+
+pmix_bfrops_module_t pmix_bfrops_pmix3_module = {
+    .version = "v3",
+    .init = init,
+    .finalize = finalize,
+    .pack = pmix3_pack,
+    .unpack = pmix3_unpack,
+    .copy = pmix3_copy,
+    .print = pmix3_print,
+    .copy_payload = pmix_bfrops_base_copy_payload,
+    .value_xfer = pmix_bfrops_base_value_xfer,
+    .value_load = pmix_bfrops_base_value_load,
+    .value_unload = pmix_bfrops_base_value_unload,
+    .value_cmp = pmix_bfrops_base_value_cmp,
+    .register_type = register_type,
+    .data_type_string = data_type_string
+};
+
+static pmix_status_t init(void)
+{
+    /* some standard types don't require anything special */
+    PMIX_REGISTER_TYPE("PMIX_BOOL", PMIX_BOOL,
+                       pmix_bfrops_base_pack_bool,
+                       pmix_bfrops_base_unpack_bool,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_bool,
+                       &mca_bfrops_v3_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_BYTE", PMIX_BYTE,
+                       pmix_bfrops_base_pack_byte,
+                       pmix_bfrops_base_unpack_byte,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_byte,
+                       &mca_bfrops_v3_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_STRING", PMIX_STRING,
+                       pmix_bfrops_base_pack_string,
+                       pmix_bfrops_base_unpack_string,
+                       pmix_bfrops_base_copy_string,
+                       pmix_bfrops_base_print_string,
+                       &mca_bfrops_v3_component.types);
+
+    /* Register the rest of the standard generic types to point to internal functions */
+    PMIX_REGISTER_TYPE("PMIX_SIZE", PMIX_SIZE,
+                       pmix_bfrops_base_pack_sizet,
+                       pmix_bfrops_base_unpack_sizet,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_size,
+                       &mca_bfrops_v3_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_PID", PMIX_PID,
+                       pmix_bfrops_base_pack_pid,
+                       pmix_bfrops_base_unpack_pid,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_pid,
+                       &mca_bfrops_v3_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_INT", PMIX_INT,
+                       pmix_bfrops_base_pack_int,
+                       pmix_bfrops_base_unpack_int,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_int,
+                       &mca_bfrops_v3_component.types);
+
+    /* Register all the standard fixed types to point to base functions */
+    PMIX_REGISTER_TYPE("PMIX_INT8", PMIX_INT8,
+                       pmix_bfrops_base_pack_byte,
+                       pmix_bfrops_base_unpack_byte,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_int8,
+                       &mca_bfrops_v3_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_INT16", PMIX_INT16,
+                       pmix_bfrops_base_pack_int16,
+                       pmix_bfrops_base_unpack_int16,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_int16,
+                       &mca_bfrops_v3_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_INT32", PMIX_INT32,
+                       pmix_bfrops_base_pack_int32,
+                       pmix_bfrops_base_unpack_int32,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_int32,
+                       &mca_bfrops_v3_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_INT64", PMIX_INT64,
+                       pmix_bfrops_base_pack_int64,
+                       pmix_bfrops_base_unpack_int64,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_int64,
+                       &mca_bfrops_v3_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_UINT", PMIX_UINT,
+                       pmix_bfrops_base_pack_int,
+                       pmix_bfrops_base_unpack_int,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_uint,
+                       &mca_bfrops_v3_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_UINT8", PMIX_UINT8,
+                       pmix_bfrops_base_pack_byte,
+                       pmix_bfrops_base_unpack_byte,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_uint8,
+                       &mca_bfrops_v3_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_UINT16", PMIX_UINT16,
+                       pmix_bfrops_base_pack_int16,
+                       pmix_bfrops_base_unpack_int16,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_uint16,
+                       &mca_bfrops_v3_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_UINT32", PMIX_UINT32,
+                       pmix_bfrops_base_pack_int32,
+                       pmix_bfrops_base_unpack_int32,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_uint32,
+                       &mca_bfrops_v3_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_UINT64", PMIX_UINT64,
+                       pmix_bfrops_base_pack_int64,
+                       pmix_bfrops_base_unpack_int64,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_uint64,
+                       &mca_bfrops_v3_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_FLOAT", PMIX_FLOAT,
+                       pmix_bfrops_base_pack_float,
+                       pmix_bfrops_base_unpack_float,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_float,
+                       &mca_bfrops_v3_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_DOUBLE", PMIX_DOUBLE,
+                       pmix_bfrops_base_pack_double,
+                       pmix_bfrops_base_unpack_double,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_double,
+                       &mca_bfrops_v3_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_TIMEVAL", PMIX_TIMEVAL,
+                       pmix_bfrops_base_pack_timeval,
+                       pmix_bfrops_base_unpack_timeval,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_timeval,
+                       &mca_bfrops_v3_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_TIME", PMIX_TIME,
+                       pmix_bfrops_base_pack_time,
+                       pmix_bfrops_base_unpack_time,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_time,
+                       &mca_bfrops_v3_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_STATUS", PMIX_STATUS,
+                       pmix_bfrops_base_pack_status,
+                       pmix_bfrops_base_unpack_status,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_status,
+                       &mca_bfrops_v3_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_VALUE", PMIX_VALUE,
+                       pmix_bfrops_base_pack_value,
+                       pmix_bfrops_base_unpack_value,
+                       pmix_bfrops_base_copy_value,
+                       pmix_bfrops_base_print_value,
+                       &mca_bfrops_v3_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_PROC", PMIX_PROC,
+                       pmix_bfrops_base_pack_proc,
+                       pmix_bfrops_base_unpack_proc,
+                       pmix_bfrops_base_copy_proc,
+                       pmix_bfrops_base_print_proc,
+                       &mca_bfrops_v3_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_APP", PMIX_APP,
+                       pmix_bfrops_base_pack_app,
+                       pmix_bfrops_base_unpack_app,
+                       pmix_bfrops_base_copy_app,
+                       pmix_bfrops_base_print_app,
+                       &mca_bfrops_v3_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_INFO", PMIX_INFO,
+                       pmix_bfrops_base_pack_info,
+                       pmix_bfrops_base_unpack_info,
+                       pmix_bfrops_base_copy_info,
+                       pmix_bfrops_base_print_info,
+                       &mca_bfrops_v3_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_PDATA", PMIX_PDATA,
+                       pmix_bfrops_base_pack_pdata,
+                       pmix_bfrops_base_unpack_pdata,
+                       pmix_bfrops_base_copy_pdata,
+                       pmix_bfrops_base_print_pdata,
+                       &mca_bfrops_v3_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_BUFFER", PMIX_BUFFER,
+                       pmix_bfrops_base_pack_buf,
+                       pmix_bfrops_base_unpack_buf,
+                       pmix_bfrops_base_copy_buf,
+                       pmix_bfrops_base_print_buf,
+                       &mca_bfrops_v3_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_BYTE_OBJECT", PMIX_BYTE_OBJECT,
+                       pmix_bfrops_base_pack_bo,
+                       pmix_bfrops_base_unpack_bo,
+                       pmix_bfrops_base_copy_bo,
+                       pmix_bfrops_base_print_bo,
+                       &mca_bfrops_v3_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_KVAL", PMIX_KVAL,
+                       pmix_bfrops_base_pack_kval,
+                       pmix_bfrops_base_unpack_kval,
+                       pmix_bfrops_base_copy_kval,
+                       pmix_bfrops_base_print_kval,
+                       &mca_bfrops_v3_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_MODEX", PMIX_MODEX,
+                       pmix_bfrops_base_pack_modex,
+                       pmix_bfrops_base_unpack_modex,
+                       pmix_bfrops_base_copy_modex,
+                       pmix_bfrops_base_print_modex,
+                       &mca_bfrops_v3_component.types);
+
+    /* these are fixed-sized values and can be done by base */
+    PMIX_REGISTER_TYPE("PMIX_PERSIST", PMIX_PERSIST,
+                       pmix_bfrops_base_pack_persist,
+                       pmix_bfrops_base_unpack_persist,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_persist,
+                       &mca_bfrops_v3_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_POINTER", PMIX_POINTER,
+                       pmix_bfrops_base_pack_ptr,
+                       pmix_bfrops_base_unpack_ptr,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_ptr,
+                       &mca_bfrops_v3_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_SCOPE", PMIX_SCOPE,
+                       pmix_bfrops_base_pack_scope,
+                       pmix_bfrops_base_unpack_scope,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_std_copy,
+                       &mca_bfrops_v3_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_DATA_RANGE", PMIX_DATA_RANGE,
+                       pmix_bfrops_base_pack_range,
+                       pmix_bfrops_base_unpack_range,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_ptr,
+                       &mca_bfrops_v3_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_COMMAND", PMIX_COMMAND,
+                       pmix_bfrops_base_pack_cmd,
+                       pmix_bfrops_base_unpack_cmd,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_cmd,
+                       &mca_bfrops_v3_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_INFO_DIRECTIVES", PMIX_INFO_DIRECTIVES,
+                       pmix_bfrops_base_pack_info_directives,
+                       pmix_bfrops_base_unpack_info_directives,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_info_directives,
+                       &mca_bfrops_v3_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_DATA_TYPE", PMIX_DATA_TYPE,
+                       pmix_bfrops_base_pack_datatype,
+                       pmix_bfrops_base_unpack_datatype,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_datatype,
+                       &mca_bfrops_v3_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_PROC_STATE", PMIX_PROC_STATE,
+                       pmix_bfrops_base_pack_pstate,
+                       pmix_bfrops_base_unpack_pstate,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_pstate,
+                       &mca_bfrops_v3_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_PROC_INFO", PMIX_PROC_INFO,
+                       pmix_bfrops_base_pack_pinfo,
+                       pmix_bfrops_base_unpack_pinfo,
+                       pmix_bfrops_base_copy_pinfo,
+                       pmix_bfrops_base_print_pinfo,
+                       &mca_bfrops_v3_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_DATA_ARRAY", PMIX_DATA_ARRAY,
+                       pmix_bfrops_base_pack_darray,
+                       pmix_bfrops_base_unpack_darray,
+                       pmix_bfrops_base_copy_darray,
+                       pmix_bfrops_base_print_darray,
+                       &mca_bfrops_v3_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_PROC_RANK", PMIX_PROC_RANK,
+                       pmix_bfrops_base_pack_rank,
+                       pmix_bfrops_base_unpack_rank,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_rank,
+                       &mca_bfrops_v3_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_QUERY", PMIX_QUERY,
+                       pmix_bfrops_base_pack_query,
+                       pmix_bfrops_base_unpack_query,
+                       pmix_bfrops_base_copy_query,
+                       pmix_bfrops_base_print_query,
+                       &mca_bfrops_v3_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_COMPRESSED_STRING",
+                       PMIX_COMPRESSED_STRING,
+                       pmix_bfrops_base_pack_bo,
+                       pmix_bfrops_base_unpack_bo,
+                       pmix_bfrops_base_copy_bo,
+                       pmix_bfrops_base_print_bo,
+                       &mca_bfrops_v3_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_ALLOC_DIRECTIVE",
+                       PMIX_ALLOC_DIRECTIVE,
+                       pmix_bfrops_base_pack_alloc_directive,
+                       pmix_bfrops_base_unpack_alloc_directive,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_alloc_directive,
+                       &mca_bfrops_v3_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_IOF_CHANNEL",
+                       PMIX_IOF_CHANNEL,
+                       pmix_bfrops_base_pack_iof_channel,
+                       pmix_bfrops_base_unpack_iof_channel,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_iof_channel,
+                       &mca_bfrops_v3_component.types);
+
+    /**** DEPRECATED ****/
+    PMIX_REGISTER_TYPE("PMIX_INFO_ARRAY", PMIX_INFO_ARRAY,
+                       pmix_bfrops_base_pack_array,
+                       pmix_bfrops_base_unpack_array,
+                       pmix_bfrops_base_copy_array,
+                       pmix_bfrops_base_print_array,
+                       &mca_bfrops_v3_component.types);
+    /********************/
+
+
+    return PMIX_SUCCESS;
+}
+
+static void finalize(void)
+{
+    int n;
+    pmix_bfrop_type_info_t *info;
+
+    for (n=0; n < mca_bfrops_v3_component.types.size; n++) {
+        if (NULL != (info = (pmix_bfrop_type_info_t*)pmix_pointer_array_get_item(&mca_bfrops_v3_component.types, n))) {
+            PMIX_RELEASE(info);
+            pmix_pointer_array_set_item(&mca_bfrops_v3_component.types, n, NULL);
+        }
+    }
+}
+
+static pmix_status_t pmix3_pack(pmix_buffer_t *buffer,
+                                const void *src, int num_vals,
+                                pmix_data_type_t type)
+{
+    /* kick the process off by passing this in to the base */
+    return pmix_bfrops_base_pack(&mca_bfrops_v3_component.types,
+                                 buffer, src, num_vals, type);
+}
+
+static pmix_status_t pmix3_unpack(pmix_buffer_t *buffer, void *dest,
+                                  int32_t *num_vals, pmix_data_type_t type)
+{
+     /* kick the process off by passing this in to the base */
+    return pmix_bfrops_base_unpack(&mca_bfrops_v3_component.types,
+                                   buffer, dest, num_vals, type);
+}
+
+static pmix_status_t pmix3_copy(void **dest, void *src,
+                                pmix_data_type_t type)
+{
+    return pmix_bfrops_base_copy(&mca_bfrops_v3_component.types,
+                                 dest, src, type);
+}
+
+static pmix_status_t pmix3_print(char **output, char *prefix,
+                                 void *src, pmix_data_type_t type)
+{
+    return pmix_bfrops_base_print(&mca_bfrops_v3_component.types,
+                                  output, prefix, src, type);
+}
+
+static pmix_status_t register_type(const char *name, pmix_data_type_t type,
+                                   pmix_bfrop_pack_fn_t pack,
+                                   pmix_bfrop_unpack_fn_t unpack,
+                                   pmix_bfrop_copy_fn_t copy,
+                                   pmix_bfrop_print_fn_t print)
+{
+    PMIX_REGISTER_TYPE(name, type,
+                       pack, unpack,
+                       copy, print,
+                       &mca_bfrops_v3_component.types);
+    return PMIX_SUCCESS;
+}
+
+static const char* data_type_string(pmix_data_type_t type)
+{
+    return pmix_bfrops_base_data_type_string(&mca_bfrops_v3_component.types, type);
+}

opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/v3/bfrop_pmix3.h

@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2006 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2016-2017 Intel, Inc. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef PMIX_BFROPS_PMIX3_H
+#define PMIX_BFROPS_PMIX3_H
+
+#include "src/mca/bfrops/bfrops.h"
+
+BEGIN_C_DECLS
+
+/* the component must be visible data for the linker to find it */
+ PMIX_EXPORT extern pmix_bfrops_base_component_t mca_bfrops_v3_component;
+
+extern pmix_bfrops_module_t pmix_bfrops_pmix3_module;
+
+END_C_DECLS
+
+#endif /* PMIX_BFROPS_PMIX3_H */

opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/v3/bfrop_pmix3_component.c

@@ -0,0 +1,99 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennbfropsee and The University
+ *                         of Tennbfropsee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2016-2017 Intel, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ * These symbols are in a file by themselves to provide nice linker
+ * semantics.  Since linkers generally pull in symbols by object
+ * files, keeping these symbols as the only symbols in this file
+ * prevents utility programs such as "ompi_info" from having to import
+ * entire components just to query their version and parameters.
+ */
+
+#include <src/include/pmix_config.h>
+#include <pmix_common.h>
+#include "src/include/types.h"
+#include "src/include/pmix_globals.h"
+
+#include "src/util/error.h"
+#include "src/server/pmix_server_ops.h"
+#include "src/mca/bfrops/base/base.h"
+#include "bfrop_pmix3.h"
+
+extern pmix_bfrops_module_t pmix_bfrops_pmix3_module;
+
+static pmix_status_t component_open(void);
+static pmix_status_t component_query(pmix_mca_base_module_t **module, int *priority);
+static pmix_status_t component_close(void);
+static pmix_bfrops_module_t* assign_module(void);
+
+/*
+ * Instantiate the public struct with all of our public information
+ * and pointers to our public functions in it
+ */
+pmix_bfrops_base_component_t mca_bfrops_v3_component = {
+    .base = {
+        PMIX_BFROPS_BASE_VERSION_1_0_0,
+
+        /* Component name and version */
+        .pmix_mca_component_name = "v3",
+        PMIX_MCA_BASE_MAKE_VERSION(component, PMIX_MAJOR_VERSION, PMIX_MINOR_VERSION,
+                                   PMIX_RELEASE_VERSION),
+
+        /* Component open and close functions */
+        .pmix_mca_open_component = component_open,
+        .pmix_mca_close_component = component_close,
+        .pmix_mca_query_component = component_query,
+    },
+    .priority = 40,
+    .assign_module = assign_module
+};
+
+
+pmix_status_t component_open(void)
+{
+    /* setup the types array */
+    PMIX_CONSTRUCT(&mca_bfrops_v3_component.types, pmix_pointer_array_t);
+    pmix_pointer_array_init(&mca_bfrops_v3_component.types, 32, INT_MAX, 16);
+
+    return PMIX_SUCCESS;
+}
+
+
+pmix_status_t component_query(pmix_mca_base_module_t **module, int *priority)
+{
+
+    *priority = mca_bfrops_v3_component.priority;
+    *module = (pmix_mca_base_module_t *)&pmix_bfrops_pmix3_module;
+    return PMIX_SUCCESS;
+}
+
+
+pmix_status_t component_close(void)
+{
+    PMIX_DESTRUCT(&mca_bfrops_v3_component.types);
+    return PMIX_SUCCESS;
+}
+
+static pmix_bfrops_module_t* assign_module(void)
+{
+    pmix_output_verbose(10, pmix_bfrops_base_framework.framework_output,
+                        "bfrops:pmix3x assigning module");
+    return &pmix_bfrops_pmix3_module;
+}

opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_dstore.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015-2017 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2015-2018 Intel, Inc. All rights reserved.
  * Copyright (c) 2016      IBM Corporation.  All rights reserved.
  * Copyright (c) 2016-2017 Mellanox Technologies, Inc.
  *                         All rights reserved.
@@ -61,43 +61,132 @@
 #define ESH_MIN_KEY_LEN             (sizeof(ESH_REGION_INVALIDATED))
 
 #define ESH_KV_SIZE(addr)                                   \
-__extension__ ({                                            \
+__pmix_attribute_extension__ ({                             \
+    size_t sz;                                              \
+    if (PMIX_PROC_IS_V1(_client_peer())) {                  \
+        sz = ESH_KV_SIZE_V12(addr);                         \
+    } else {                                                \
+        sz = ESH_KV_SIZE_V20(addr);                         \
+    }                                                       \
+    sz;                                                     \
+})
+
+#define ESH_KNAME_PTR(addr)                                 \
+__pmix_attribute_extension__ ({                             \
+    char *name_ptr;                                         \
+    if (PMIX_PROC_IS_V1(_client_peer())) {                  \
+        name_ptr = ESH_KNAME_PTR_V12(addr);                 \
+    } else {                                                \
+        name_ptr = ESH_KNAME_PTR_V20(addr);                 \
+    }                                                       \
+    name_ptr;                                               \
+})
+
+#define ESH_KNAME_LEN(key)                                  \
+__pmix_attribute_extension__ ({                             \
+    size_t len;                                             \
+    if (PMIX_PROC_IS_V1(_client_peer())) {                  \
+        len = ESH_KNAME_LEN_V12(key);                       \
+    } else {                                                \
+        len = ESH_KNAME_LEN_V20(key);                       \
+    }                                                       \
+    len;                                                    \
+})
+
+#define ESH_DATA_PTR(addr)                                  \
+__pmix_attribute_extension__ ({                             \
+    uint8_t *data_ptr;                                      \
+    if (PMIX_PROC_IS_V1(_client_peer())) {                  \
+        data_ptr = ESH_DATA_PTR_V12(addr);                  \
+    } else {                                                \
+        data_ptr = ESH_DATA_PTR_V20(addr);                  \
+    }                                                       \
+    data_ptr;                                               \
+})
+
+#define ESH_DATA_SIZE(addr, data_ptr)                       \
+__pmix_attribute_extension__ ({                             \
+    size_t sz;                                              \
+    if (PMIX_PROC_IS_V1(_client_peer())) {                  \
+        sz = ESH_DATA_SIZE_V12(addr);                       \
+    } else {                                                \
+        sz = ESH_DATA_SIZE_V20(addr, data_ptr);             \
+    }                                                       \
+    sz;                                                     \
+})
+
+#define ESH_KEY_SIZE(key, size)                             \
+__pmix_attribute_extension__ ({                             \
+    size_t len;                                             \
+    if (PMIX_PROC_IS_V1(_client_peer())) {                  \
+        len = ESH_KEY_SIZE_V12(key, size);                  \
+    } else {                                                \
+        len = ESH_KEY_SIZE_V20(key, size);                  \
+    }                                                       \
+    len;                                                    \
+})
+
+#define EXT_SLOT_SIZE()                                     \
+__pmix_attribute_extension__ ({                             \
+    size_t sz;                                              \
+    if (PMIX_PROC_IS_V1(_client_peer())) {                  \
+        sz = EXT_SLOT_SIZE_V12();                           \
+    } else {                                                \
+        sz = EXT_SLOT_SIZE_V20();                           \
+    }                                                       \
+    sz;                                                     \
+})
+
+#define ESH_PUT_KEY(addr, key, buffer, size)                \
+__pmix_attribute_extension__ ({                             \
+    if (PMIX_PROC_IS_V1(_client_peer())) {                  \
+        ESH_PUT_KEY_V12(addr, key, buffer, size);           \
+    } else {                                                \
+        ESH_PUT_KEY_V20(addr, key, buffer, size);           \
+    }                                                       \
+})
+
+/* PMIx v2.x dstore specific macro */
+#define ESH_KV_SIZE_V20(addr)                               \
+__pmix_attribute_extension__ ({                             \
     size_t sz;                                              \
     memcpy(&sz, addr, sizeof(size_t));                      \
     sz;                                                     \
 })
 
-#define ESH_KNAME_PTR(addr)                                 \
+#define ESH_KNAME_PTR_V20(addr)                             \
-__extension__ ({                                            \
+__pmix_attribute_extension__ ({                             \
     char *name_ptr = (char *)addr + sizeof(size_t);         \
     name_ptr;                                               \
 })
 
-#define ESH_KNAME_LEN(key)                                  \
+#define ESH_KNAME_LEN_V20(key)                              \
-__extension__ ({                                            \
+__pmix_attribute_extension__ ({                             \
     size_t kname_len = strlen(key) + 1;                     \
     size_t len = (kname_len < ESH_MIN_KEY_LEN) ?            \
     ESH_MIN_KEY_LEN : kname_len;                            \
     len;                                                    \
 })
 
-#define ESH_DATA_PTR(addr)                                  \
+#define ESH_DATA_PTR_V20(addr)                              \
-__extension__ ({                                            \
+__pmix_attribute_extension__ ({                             \
-    size_t kname_len = ESH_KNAME_LEN(ESH_KNAME_PTR(addr));  \
+    size_t kname_len =                                      \
+        ESH_KNAME_LEN_V20(ESH_KNAME_PTR_V20(addr));         \
     uint8_t *data_ptr = addr + sizeof(size_t) + kname_len;  \
     data_ptr;                                               \
 })
 
-#define ESH_DATA_SIZE(addr, data_ptr)                       \
+#define ESH_DATA_SIZE_V20(addr, data_ptr)                   \
-__extension__ ({                                            \
+__pmix_attribute_extension__ ({                             \
-    size_t sz = ESH_KV_SIZE(addr);                          \
+    size_t sz = ESH_KV_SIZE_V20(addr);                      \
     size_t data_size = sz - (data_ptr - addr);              \
     data_size;                                              \
 })
 
-#define ESH_KEY_SIZE(key, size)                             \
+#define ESH_KEY_SIZE_V20(key, size)                         \
-__extension__ ({                                            \
+__pmix_attribute_extension__ ({                             \
-    size_t len = sizeof(size_t) + ESH_KNAME_LEN(key) + size;\
+    size_t len =                                            \
+        sizeof(size_t) + ESH_KNAME_LEN_V20(key) + size;     \
     len;                                                    \
 })
 
@@ -105,24 +194,91 @@ __extension__ ({                                            \
  * new data were added for the same process during
  * next commit
  */
-#define EXT_SLOT_SIZE()                                     \
+#define EXT_SLOT_SIZE_V20()                                 \
-    (ESH_KEY_SIZE(ESH_REGION_EXTENSION, sizeof(size_t)))
+    (ESH_KEY_SIZE_V20(ESH_REGION_EXTENSION, sizeof(size_t)))
 
 
-#define ESH_PUT_KEY(addr, key, buffer, size)                \
+#define ESH_PUT_KEY_V20(addr, key, buffer, size)            \
-__extension__ ({                                            \
+__pmix_attribute_extension__ ({                             \
-    size_t sz = ESH_KEY_SIZE(key, size);                    \
+    size_t sz = ESH_KEY_SIZE_V20(key, size);                \
     memcpy(addr, &sz, sizeof(size_t));                      \
-    memset(addr + sizeof(size_t), 0, ESH_KNAME_LEN(key));   \
+    memset(addr + sizeof(size_t), 0,                        \
+        ESH_KNAME_LEN_V20(key));                            \
     strncpy((char *)addr + sizeof(size_t),                  \
-            key, ESH_KNAME_LEN(key));                       \
+            key, ESH_KNAME_LEN_V20(key));                   \
-    memcpy(addr + sizeof(size_t) + ESH_KNAME_LEN(key),      \
+    memcpy(addr + sizeof(size_t) + ESH_KNAME_LEN_V20(key),  \
+            buffer, size);                                  \
+})
+
+/* PMIx v1.2 dstore specific macro */
+#define ESH_KEY_SIZE_V12(key, size)                         \
+__pmix_attribute_extension__ ({                             \
+    size_t len = strlen(key) + 1 + sizeof(size_t) + size;   \
+    len;                                                    \
+})
+
+/* in ext slot new offset will be stored in case if
+ * new data were added for the same process during
+ * next commit
+ */
+#define EXT_SLOT_SIZE_V12()                                 \
+    (ESH_KEY_SIZE_V12(ESH_REGION_EXTENSION, sizeof(size_t)))
+
+#define ESH_KV_SIZE_V12(addr)                               \
+__pmix_attribute_extension__ ({                             \
+    size_t sz;                                              \
+    memcpy(&sz, addr +                                      \
+        ESH_KNAME_LEN_V12(ESH_KNAME_PTR_V12(addr)),         \
+        sizeof(size_t));                                    \
+    sz += ESH_KNAME_LEN_V12(ESH_KNAME_PTR_V12(addr)) +      \
+        sizeof(size_t);                                     \
+    sz;                                                     \
+})
+
+#define ESH_KNAME_PTR_V12(addr)                             \
+__pmix_attribute_extension__ ({                             \
+    char *name_ptr = (char *)addr;                          \
+    name_ptr;                                               \
+})
+
+#define ESH_KNAME_LEN_V12(key)                              \
+__pmix_attribute_extension__ ({                             \
+    size_t len = strlen((char*)key) + 1;                    \
+    len;                                                    \
+})
+
+#define ESH_DATA_PTR_V12(addr)                              \
+__pmix_attribute_extension__ ({                             \
+    uint8_t *data_ptr =                                     \
+        addr +                                              \
+        sizeof(size_t) +                                    \
+        ESH_KNAME_LEN_V12(ESH_KNAME_PTR_V12(addr));         \
+    data_ptr;                                               \
+})
+
+#define ESH_DATA_SIZE_V12(addr)                             \
+__pmix_attribute_extension__ ({                             \
+    size_t data_size;                                       \
+    memcpy(&data_size,                                      \
+        addr + ESH_KNAME_LEN_V12(ESH_KNAME_PTR_V12(addr)),  \
+        sizeof(size_t));                                    \
+    data_size;                                              \
+})
+
+#define ESH_PUT_KEY_V12(addr, key, buffer, size)            \
+__pmix_attribute_extension__ ({                             \
+    size_t sz = size;                                       \
+    memset(addr, 0, ESH_KNAME_LEN_V12(key));                \
+    strncpy((char *)addr, key, ESH_KNAME_LEN_V12(key));     \
+    memcpy(addr + ESH_KNAME_LEN_V12(key), &sz,              \
+        sizeof(size_t));                                    \
+    memcpy(addr + ESH_KNAME_LEN_V12(key) + sizeof(size_t),  \
             buffer, size);                                  \
 })
 
 #ifdef ESH_PTHREAD_LOCK
 #define _ESH_LOCK(rwlock, func)                             \
-__extension__ ({                                            \
+__pmix_attribute_extension__ ({                             \
     pmix_status_t ret = PMIX_SUCCESS;                       \
     int rc;                                                 \
     rc = pthread_rwlock_##func(rwlock);                     \
@@ -150,7 +306,7 @@ __extension__ ({                                            \
 
 #ifdef ESH_FCNTL_LOCK
 #define _ESH_LOCK(lockfd, operation)                        \
-__extension__ ({                                            \
+__pmix_attribute_extension__ ({                             \
     pmix_status_t ret = PMIX_SUCCESS;                       \
     int i;                                                  \
     struct flock fl = {0};                                  \
@@ -2839,44 +2995,35 @@ static pmix_status_t dstore_store_modex(struct pmix_nspace_t *nspace,
                                       pmix_byte_object_t *bo)
 {
     pmix_nspace_t *ns = (pmix_nspace_t*)nspace;
-    pmix_server_caddy_t *scd;
     pmix_status_t rc = PMIX_SUCCESS;
     int32_t cnt;
     pmix_buffer_t pbkt;
     pmix_proc_t proc;
     pmix_kval_t *kv;
-    pmix_peer_t *peer;
 
     pmix_output_verbose(2, pmix_gds_base_framework.framework_output,
                         "[%s:%d] gds:dstore:store_modex for nspace %s",
                         pmix_globals.myid.nspace, pmix_globals.myid.rank,
                         ns->nspace);
 
+    /* NOTE: THE BYTE OBJECT DELIVERED HERE WAS CONSTRUCTED
+     * BY A SERVER, AND IS THEREFORE PACKED USING THE SERVER'S
+     * PEER OBJECT (WHICH IS REQUIRED TO BE THE SAME AS OUR OWN) */
+
     /* this is data returned via the PMIx_Fence call when
      * data collection was requested, so it only contains
      * REMOTE/GLOBAL data. The byte object contains
      * the rank followed by pmix_kval_t's. The list of callbacks
      * contains all local participants. */
-    peer = NULL;
-    PMIX_LIST_FOREACH(scd, cbs, pmix_server_caddy_t) {
-        if (scd->peer->nptr == ns) {
-            peer = scd->peer;
-            break;
-        }
-    }
-    if (NULL == peer) {
-        /* we can ignore this one */
-        return PMIX_SUCCESS;
-    }
 
     /* setup the byte object for unpacking */
     PMIX_CONSTRUCT(&pbkt, pmix_buffer_t);
     /* the next step unfortunately NULLs the byte object's
      * entries, so we need to ensure we restore them! */
-    PMIX_LOAD_BUFFER(peer, &pbkt, bo->bytes, bo->size);
+    PMIX_LOAD_BUFFER(pmix_globals.mypeer, &pbkt, bo->bytes, bo->size);
     /* unload the proc that provided this data */
     cnt = 1;
-    PMIX_BFROPS_UNPACK(rc, peer, &pbkt, &proc, &cnt, PMIX_PROC);
+    PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, &pbkt, &proc, &cnt, PMIX_PROC);
     if (PMIX_SUCCESS != rc) {
         PMIX_ERROR_LOG(rc);
         bo->bytes = pbkt.base_ptr;
@@ -2896,7 +3043,7 @@ static pmix_status_t dstore_store_modex(struct pmix_nspace_t *nspace,
     /* unpack the remaining values until we hit the end of the buffer */
     cnt = 1;
     kv = PMIX_NEW(pmix_kval_t);
-    PMIX_BFROPS_UNPACK(rc, peer, &pbkt, kv, &cnt, PMIX_KVAL);
+    PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, &pbkt, kv, &cnt, PMIX_KVAL);
     while (PMIX_SUCCESS == rc) {
         /* store this in the hash table */
         PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, &proc, PMIX_REMOTE, kv);
@@ -2915,7 +3062,7 @@ static pmix_status_t dstore_store_modex(struct pmix_nspace_t *nspace,
         /* continue along */
         kv = PMIX_NEW(pmix_kval_t);
         cnt = 1;
-        PMIX_BFROPS_UNPACK(rc, peer, &pbkt, kv, &cnt, PMIX_KVAL);
+        PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, &pbkt, kv, &cnt, PMIX_KVAL);
     }
     PMIX_RELEASE(kv);  // maintain accounting
     if (PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER != rc) {
@@ -3086,7 +3233,7 @@ static void _client_compat_save(pmix_peer_t *peer)
 static inline pmix_peer_t * _client_peer(void)
 {
     if (NULL == _clients_peer) {
-        return pmix_client_globals.myserver;
+        return pmix_globals.mypeer;
     }
     return _clients_peer;
 }

opal/mca/pmix/pmix3x/pmix/src/mca/psec/base/psec_base_select.c

@@ -74,12 +74,21 @@ int pmix_psec_base_select(void)
         if (PMIX_SUCCESS != rc || NULL == module) {
             pmix_output_verbose(5, pmix_psec_base_framework.framework_output,
                                 "mca:psec:select: Skipping component [%s]. Query failed to return a module",
-                                component->pmix_mca_component_name );
+                                component->pmix_mca_component_name);
+            continue;
+        }
+        nmodule = (pmix_psec_module_t*) module;
+
+        /* give the module a chance to init */
+        if (NULL != nmodule->init && PMIX_SUCCESS != nmodule->init()) {
+            /* failed to init, so skip it */
+            pmix_output_verbose(5, pmix_psec_base_framework.framework_output,
+                                "mca:psec:select: Skipping component [%s]. Failed to init",
+                                component->pmix_mca_component_name);
             continue;
         }
 
         /* If we got a module, keep it */
-        nmodule = (pmix_psec_module_t*) module;
         /* add to the list of selected modules */
         newmodule = PMIX_NEW(pmix_psec_base_active_module_t);
         newmodule->pri = priority;

opal/mca/pmix/pmix3x/pmix/src/mca/psec/munge/Makefile.am

@@ -19,6 +19,8 @@
 # $HEADER$
 #
 
+AM_CPPFLAGS = $(psec_munge_CPPFLAGS)
+
 headers = psec_munge.h
 sources = \
         psec_munge_component.c \
@@ -43,8 +45,10 @@ endif
 mcacomponentdir = $(pmixlibdir)
 mcacomponent_LTLIBRARIES = $(component)
 mca_psec_munge_la_SOURCES = $(component_sources)
-mca_psec_munge_la_LDFLAGS = -module -avoid-version
+mca_psec_munge_la_LDFLAGS = -module -avoid-version $(psec_munge_LDFLAGS)
+mca_psec_munge_la_LIBADD = $(psec_munge_LIBS)
 
 noinst_LTLIBRARIES = $(lib)
 libmca_psec_munge_la_SOURCES = $(lib_sources)
-libmca_psec_munge_la_LDFLAGS = -module -avoid-version
+libmca_psec_munge_la_LDFLAGS = -module -avoid-version $(psec_munge_LDFLAGS)
+libmca_psec_munge_la_LIBADD = $(psec_munge_LIBS)

opal/mca/pmix/pmix3x/pmix/src/mca/psec/munge/psec_munge.c

@@ -25,27 +25,30 @@
 #endif
 #include <munge.h>
 
+#include "src/threads/threads.h"
 #include "src/mca/psec/psec.h"
 #include "psec_munge.h"
 
 static pmix_status_t munge_init(void);
 static void munge_finalize(void);
-static pmix_status_t create_cred(pmix_listener_protocol_t protocol,
+static pmix_status_t create_cred(struct pmix_peer_t *peer,
-                                 char **cred, size_t *len);
+                                 const pmix_info_t directives[], size_t ndirs,
-static pmix_status_t validate_cred(int sd, uid_t uid, gid_t gid,
+                                 pmix_info_t **info, size_t *ninfo,
-                                   pmix_listener_protocol_t protocol,
+                                 pmix_byte_object_t *cred);
-                                   char *cred, size_t len);
+static pmix_status_t validate_cred(struct pmix_peer_t *peer,
+                                   const pmix_info_t directives[], size_t ndirs,
+                                   pmix_info_t **info, size_t *ninfo,
+                                   const pmix_byte_object_t *cred);
 
 pmix_psec_module_t pmix_munge_module = {
-    "munge",
+    .name = "munge",
-    munge_init,
+    .init = munge_init,
-    munge_finalize,
+    .finalize = munge_finalize,
-    create_cred,
+    .create_cred = create_cred,
-    NULL,
+    .validate_cred = validate_cred
-    validate_cred,
-    NULL
 };
 
+static pmix_lock_t lock;
 static char *mycred = NULL;
 static bool initialized = false;
 static bool refresh = false;
@@ -57,6 +60,9 @@ static pmix_status_t munge_init(void)
     pmix_output_verbose(2, pmix_globals.debug_output,
                         "psec: munge init");
 
+    PMIX_CONSTRUCT_LOCK(&lock);
+    lock.active = false;
+
     /* attempt to get a credential as a way of checking that
      * the munge server is available - cache the credential
      * for later use */
@@ -67,6 +73,7 @@ static pmix_status_t munge_init(void)
                             munge_strerror(rc));
         return PMIX_ERR_SERVER_NOT_AVAIL;
     }
+
     initialized = true;
 
     return PMIX_SUCCESS;
@@ -74,6 +81,8 @@ static pmix_status_t munge_init(void)
 
 static void munge_finalize(void)
 {
+    PMIX_ACQUIRE_THREAD(&lock);
+
     pmix_output_verbose(2, pmix_globals.debug_output,
                         "psec: munge finalize");
     if (initialized) {
@@ -82,21 +91,57 @@ static void munge_finalize(void)
             mycred = NULL;
         }
     }
+    PMIX_RELEASE_THREAD(&lock);
+    PMIX_DESTRUCT_LOCK(&lock);
 }
 
-static pmix_status_t create_cred(pmix_listener_protocol_t protocol,
+static pmix_status_t create_cred(struct pmix_peer_t *peer,
-                                 char **cred, size_t *len)
+                                 const pmix_info_t directives[], size_t ndirs,
+                                 pmix_info_t **info, size_t *ninfo,
+                                 pmix_byte_object_t *cred)
 {
     int rc;
+    bool takeus;
+    char **types;
+    size_t n, m;
+
+    PMIX_ACQUIRE_THREAD(&lock);
 
     pmix_output_verbose(2, pmix_globals.debug_output,
                         "psec: munge create_cred");
 
+    /* ensure initialization */
+    PMIX_BYTE_OBJECT_CONSTRUCT(cred);
+
+    /* if we are responding to a local request to create a credential,
+     * then see if they specified a mechanism */
+    if (NULL != directives && 0 < ndirs) {
+        for (n=0; n < ndirs; n++) {
+            if (0 == strncmp(directives[n].key, PMIX_CRED_TYPE, PMIX_MAX_KEYLEN)) {
+                /* split the specified string */
+                types = pmix_argv_split(directives[n].value.data.string, ',');
+                takeus = false;
+                for (m=0; NULL != types[m]; m++) {
+                    if (0 == strcmp(types[m], "munge")) {
+                        /* it's us! */
+                        takeus = true;
+                        break;
+                    }
+                }
+                pmix_argv_free(types);
+                if (!takeus) {
+                    PMIX_RELEASE_THREAD(&lock);
+                    return PMIX_ERR_NOT_SUPPORTED;
+                }
+            }
+        }
+    }
+
     if (initialized) {
         if (!refresh) {
             refresh = true;
-            *cred = strdup(mycred);
+            cred->bytes = strdup(mycred);
-            *len = strlen(mycred) + 1;
+            cred->size = strlen(mycred) + 1;
         } else {
             /* munge does not allow reuse of a credential, so we have to
              * refresh it for every use */
@@ -107,28 +152,70 @@ static pmix_status_t create_cred(pmix_listener_protocol_t protocol,
                 pmix_output_verbose(2, pmix_globals.debug_output,
                                     "psec: munge failed to create credential: %s",
                                     munge_strerror(rc));
-                return NULL;
+                PMIX_RELEASE_THREAD(&lock);
+                return PMIX_ERR_NOT_SUPPORTED;
             }
-            *cred = strdup(mycred);
+            cred->bytes = strdup(mycred);
-            *len = strlen(mycred) + 1;
+            cred->size = strlen(mycred) + 1;
         }
     }
+    if (NULL != info) {
+        /* mark that this came from us */
+        PMIX_INFO_CREATE(*info, 1);
+        if (NULL == *info) {
+            PMIX_RELEASE_THREAD(&lock);
+            return PMIX_ERR_NOMEM;
+        }
+        *ninfo = 1;
+        PMIX_INFO_LOAD(info[0], PMIX_CRED_TYPE, "munge", PMIX_STRING);
+    }
+    PMIX_RELEASE_THREAD(&lock);
     return PMIX_SUCCESS;
 }
 
-static pmix_status_t validate_cred(int sd, uid_t uid, gid_t gid,
+static pmix_status_t validate_cred(struct pmix_peer_t *peer,
-                                   pmix_listener_protocol_t protocol,
+                                   const pmix_info_t directives[], size_t ndirs,
-                                   char *cred, size_t len)
+                                   pmix_info_t **info, size_t *ninfo,
+                                   const pmix_byte_object_t *cred)
 {
+    pmix_peer_t *pr = (pmix_peer_t*)peer;
     uid_t euid;
     gid_t egid;
     munge_err_t rc;
+    bool takeus;
+    char **types;
+    size_t n, m;
+    uint32_t u32;
 
     pmix_output_verbose(2, pmix_globals.debug_output,
-                        "psec: munge validate_cred %s", cred ? cred : "NULL");
+                        "psec: munge validate_cred %s",
+                        (NULL == cred) ? "NULL" : "NON-NULL");
+
+    /* if we are responding to a local request to validate a credential,
+     * then see if they specified a mechanism */
+    if (NULL != directives && 0 < ndirs) {
+        for (n=0; n < ndirs; n++) {
+            if (0 == strncmp(directives[n].key, PMIX_CRED_TYPE, PMIX_MAX_KEYLEN)) {
+                /* split the specified string */
+                types = pmix_argv_split(directives[n].value.data.string, ',');
+                takeus = false;
+                for (m=0; NULL != types[m]; m++) {
+                    if (0 == strcmp(types[m], "munge")) {
+                        /* it's us! */
+                        takeus = true;
+                        break;
+                    }
+                }
+                pmix_argv_free(types);
+                if (!takeus) {
+                    return PMIX_ERR_NOT_SUPPORTED;
+                }
+            }
+        }
+    }
 
     /* parse the inbound string */
-    if (EMUNGE_SUCCESS != (rc = munge_decode(cred, NULL, NULL, NULL, &euid, &egid))) {
+    if (EMUNGE_SUCCESS != (rc = munge_decode(cred->bytes, NULL, NULL, NULL, &euid, &egid))) {
         pmix_output_verbose(2, pmix_globals.debug_output,
                             "psec: munge failed to decode credential: %s",
                             munge_strerror(rc));
@@ -136,16 +223,31 @@ static pmix_status_t validate_cred(int sd, uid_t uid, gid_t gid,
     }
 
     /* check uid */
-    if (euid != uid) {
+    if (euid != pr->info->uid) {
         return PMIX_ERR_INVALID_CRED;
     }
 
     /* check guid */
-    if (egid != gid) {
+    if (egid != pr->info->gid) {
         return PMIX_ERR_INVALID_CRED;
     }
 
     pmix_output_verbose(2, pmix_globals.debug_output,
                         "psec: munge credential valid");
+    if (NULL != info) {
+        PMIX_INFO_CREATE(*info, 3);
+        if (NULL == *info) {
+            return PMIX_ERR_NOMEM;
+        }
+        *ninfo = 3;
+        /* mark that this came from us */
+        PMIX_INFO_LOAD(info[0], PMIX_CRED_TYPE, "munge", PMIX_STRING);
+        /* provide the uid it contained */
+        u32 = euid;
+        PMIX_INFO_LOAD(info[1], PMIX_USERID, &u32, PMIX_UINT32);
+        /* provide the gid it contained */
+        u32 = egid;
+        PMIX_INFO_LOAD(info[2], PMIX_GRPID, &u32, PMIX_UINT32);
+    }
     return PMIX_SUCCESS;
 }

opal/mca/pmix/pmix3x/pmix/src/mca/psec/native/psec_native.c

@@ -1,7 +1,8 @@
 /*
- * Copyright (c) 2015-2017 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2015-2018 Intel, Inc. All rights reserved.
  * Copyright (c) 2016      IBM Corporation.  All rights reserved.
- *
+ * Copyright (c) 2017      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -29,11 +30,14 @@
 
 static pmix_status_t native_init(void);
 static void native_finalize(void);
-static pmix_status_t create_cred(pmix_listener_protocol_t protocol,
+static pmix_status_t create_cred(struct pmix_peer_t *peer,
-                                 char **cred, size_t *len);
+                                 const pmix_info_t directives[], size_t ndirs,
-static pmix_status_t validate_cred(int sd, uid_t uid, gid_t gid,
+                                 pmix_info_t **info, size_t *ninfo,
-                                   pmix_listener_protocol_t protocol,
+                                 pmix_byte_object_t *cred);
-                                   char *cred, size_t len);
+static pmix_status_t validate_cred(struct pmix_peer_t *peer,
+                                   const pmix_info_t directives[], size_t ndirs,
+                                   pmix_info_t **info, size_t *ninfo,
+                                   const pmix_byte_object_t *cred);
 
 pmix_psec_module_t pmix_native_module = {
     .name = "native",
@@ -56,21 +60,55 @@ static void native_finalize(void)
                         "psec: native finalize");
 }
 
-static pmix_status_t create_cred(pmix_listener_protocol_t protocol,
+static pmix_status_t create_cred(struct pmix_peer_t *peer,
-                                 char **cred, size_t *len)
+                                 const pmix_info_t directives[], size_t ndirs,
+                                 pmix_info_t **info, size_t *ninfo,
+                                 pmix_byte_object_t *cred)
 {
+    pmix_peer_t *pr = (pmix_peer_t*)peer;
+    char **types;
+    size_t n, m;
+    bool takeus;
     uid_t euid;
     gid_t egid;
     char *tmp, *ptr;
 
-    if (PMIX_PROTOCOL_V1 == protocol ||
+    /* ensure initialization */
-        PMIX_PROTOCOL_V3 == protocol) {
+    PMIX_BYTE_OBJECT_CONSTRUCT(cred);
-        /* these are usock protocols - nothing to do */
+
-        *cred = NULL;
+    /* we may be responding to a local request for a credential, so
-        *len = 0;
+     * see if they specified a mechanism */
-        return PMIX_SUCCESS;
+    if (NULL != directives && 0 < ndirs) {
+        /* cycle across the provided info and see if they specified
+         * any desired credential types */
+        takeus = true;
+        for (n=0; n < ndirs; n++) {
+            if (0 == strncmp(directives[n].key, PMIX_CRED_TYPE, PMIX_MAX_KEYLEN)) {
+                /* see if we are included */
+                types = pmix_argv_split(directives[n].value.data.string, ',');
+                /* start by assuming they don't want us */
+                takeus = false;
+                for (m=0; NULL != types[m]; m++) {
+                    if (0 == strcmp(types[m], "native")) {
+                        /* it's us! */
+                        takeus = true;
+                        break;
+                    }
+                }
+                pmix_argv_free(types);
+                break;
+            }
+        }
+        if (!takeus) {
+            PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED);
+            return PMIX_ERR_NOT_SUPPORTED;
+        }
     }
-    if (PMIX_PROTOCOL_V2 == protocol) {
+
+    if (PMIX_PROTOCOL_V1 == pr->protocol) {
+        /* usock protocol - nothing to do */
+        goto complete;
+    } else if (PMIX_PROTOCOL_V2 == pr->protocol) {
         /* tcp protocol - need to provide our effective
          * uid and gid for validation on remote end */
         tmp = (char*)malloc(sizeof(uid_t) + sizeof(gid_t));
@@ -82,19 +120,35 @@ static pmix_status_t create_cred(pmix_listener_protocol_t protocol,
         ptr = tmp + sizeof(uid_t);
         egid = getegid();
         memcpy(ptr, &egid, sizeof(gid_t));
-        *cred = tmp;
+        cred->bytes = tmp;
-        *len = sizeof(uid_t) + sizeof(gid_t);
+        cred->size = sizeof(uid_t) + sizeof(gid_t);
-        return PMIX_SUCCESS;
+        goto complete;
+    } else {
+        /* unrecognized protocol */
+        PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED);
+        return PMIX_ERR_NOT_SUPPORTED;
     }
 
-    /* unrecognized protocol */
+  complete:
-    return PMIX_ERR_NOT_SUPPORTED;
+    if (NULL != info) {
+        /* mark that this came from us */
+        PMIX_INFO_CREATE(*info, 1);
+        if (NULL == *info) {
+            return PMIX_ERR_NOMEM;
+        }
+        *ninfo = 1;
+        PMIX_INFO_LOAD(info[0], PMIX_CRED_TYPE, "native", PMIX_STRING);
+    }
+    return PMIX_SUCCESS;
 }
 
-static pmix_status_t validate_cred(int sd, uid_t uid, gid_t gid,
+static pmix_status_t validate_cred(struct pmix_peer_t *peer,
-                                   pmix_listener_protocol_t protocol,
+                                   const pmix_info_t directives[], size_t ndirs,
-                                   char *cred, size_t len)
+                                   pmix_info_t **info, size_t *ninfo,
+                                   const pmix_byte_object_t *cred)
 {
+    pmix_peer_t *pr = (pmix_peer_t*)peer;
+
 #if defined(SO_PEERCRED)
 #ifdef HAVE_STRUCT_SOCKPEERCRED_UID
 #define HAVE_STRUCT_UCRED_UID
@@ -108,18 +162,22 @@ static pmix_status_t validate_cred(int sd, uid_t uid, gid_t gid,
     gid_t egid;
     char *ptr;
     size_t ln;
+    bool takeus;
+    char **types;
+    size_t n, m;
+    uint32_t u32;
 
     pmix_output_verbose(2, pmix_globals.debug_output,
-                        "psec: native validate_cred %s", cred ? cred : "NULL");
+                        "psec: native validate_cred %s",
+                        (NULL == cred) ? "NULL" : "NON-NULL");
 
-    if (PMIX_PROTOCOL_V1 == protocol ||
+    if (PMIX_PROTOCOL_V1 == pr->protocol) {
-        PMIX_PROTOCOL_V3 == protocol) {
+        /* usock protocol - get the remote side's uid/gid */
-        /* these are usock protocols - get the remote side's uid/gid */
 #if defined(SO_PEERCRED) && (defined(HAVE_STRUCT_UCRED_UID) || defined(HAVE_STRUCT_UCRED_CR_UID))
         /* Ignore received 'cred' and validate ucred for socket instead. */
         pmix_output_verbose(2, pmix_globals.debug_output,
-                            "psec:native checking getsockopt on socket %d for peer credentials", sd);
+                            "psec:native checking getsockopt on socket %d for peer credentials", pr->sd);
-        if (getsockopt (sd, SOL_SOCKET, SO_PEERCRED, &ucred, &crlen) < 0) {
+        if (getsockopt(pr->sd, SOL_SOCKET, SO_PEERCRED, &ucred, &crlen) < 0) {
             pmix_output_verbose(2, pmix_globals.debug_output,
                                 "psec: getsockopt SO_PEERCRED failed: %s",
                                 strerror (pmix_socket_errno));
@@ -135,8 +193,8 @@ static pmix_status_t validate_cred(int sd, uid_t uid, gid_t gid,
 
 #elif defined(HAVE_GETPEEREID)
         pmix_output_verbose(2, pmix_globals.debug_output,
-                            "psec:native checking getpeereid on socket %d for peer credentials", sd);
+                            "psec:native checking getpeereid on socket %d for peer credentials", pr->sd);
-        if (0 != getpeereid(sd, &euid, &egid)) {
+        if (0 != getpeereid(pr->sd, &euid, &egid)) {
             pmix_output_verbose(2, pmix_globals.debug_output,
                                 "psec: getsockopt getpeereid failed: %s",
                                 strerror (pmix_socket_errno));
@@ -145,41 +203,20 @@ static pmix_status_t validate_cred(int sd, uid_t uid, gid_t gid,
 #else
         return PMIX_ERR_NOT_SUPPORTED;
 #endif
-
+    } else if (PMIX_PROTOCOL_V2 == pr->protocol) {
-        /* check uid */
-        if (euid != uid) {
-            pmix_output_verbose(2, pmix_globals.debug_output,
-                                "psec: socket cred contains invalid uid %u", euid);
-            return PMIX_ERR_INVALID_CRED;
-        }
-
-        /* check gid */
-        if (egid != gid) {
-            pmix_output_verbose(2, pmix_globals.debug_output,
-                                "psec: socket cred contains invalid gid %u", egid);
-            return PMIX_ERR_INVALID_CRED;
-        }
-
-        pmix_output_verbose(2, pmix_globals.debug_output,
-                            "psec: native credential %u:%u valid",
-                            euid, egid);
-        return PMIX_SUCCESS;
-    }
-
-    if (PMIX_PROTOCOL_V2 == protocol) {
         /* this is a tcp protocol, so the cred is actually the uid/gid
          * passed upwards from the client */
         if (NULL == cred) {
             /* not allowed */
             return PMIX_ERR_INVALID_CRED;
         }
-        ln = len;
+        ln = cred->size;
         euid = 0;
         egid = 0;
         if (sizeof(uid_t) <= ln) {
-            memcpy(&euid, cred, sizeof(uid_t));
+            memcpy(&euid, cred->bytes, sizeof(uid_t));
             ln -= sizeof(uid_t);
-            ptr = cred + sizeof(uid_t);
+            ptr = cred->bytes + sizeof(uid_t);
         } else {
             return PMIX_ERR_INVALID_CRED;
         }
@@ -188,26 +225,64 @@ static pmix_status_t validate_cred(int sd, uid_t uid, gid_t gid,
         } else {
             return PMIX_ERR_INVALID_CRED;
         }
-        /* check uid */
+    } else if (PMIX_PROTOCOL_UNDEF != pr->protocol) {
-        if (euid != uid) {
+        /* don't recognize the protocol */
-            pmix_output_verbose(2, pmix_globals.debug_output,
+        return PMIX_ERR_NOT_SUPPORTED;
-                                "psec: socket cred contains invalid uid %u", euid);
-            return PMIX_ERR_INVALID_CRED;
-        }
-
-        /* check gid */
-        if (egid != gid) {
-            pmix_output_verbose(2, pmix_globals.debug_output,
-                                "psec: socket cred contains invalid gid %u", egid);
-            return PMIX_ERR_INVALID_CRED;
-        }
-
-        pmix_output_verbose(2, pmix_globals.debug_output,
-                            "psec: native credential %u:%u valid",
-                            euid, egid);
-        return PMIX_SUCCESS;
     }
 
-    /* don't recognize the protocol */
+    /* if we are responding to a local request to validate a credential,
-    return PMIX_ERR_NOT_SUPPORTED;
+     * then see if they specified a mechanism */
+    if (NULL != directives && 0 < ndirs) {
+        for (n=0; n < ndirs; n++) {
+            if (0 == strncmp(directives[n].key, PMIX_CRED_TYPE, PMIX_MAX_KEYLEN)) {
+                /* split the specified string */
+                types = pmix_argv_split(directives[n].value.data.string, ',');
+                takeus = false;
+                for (m=0; NULL != types[m]; m++) {
+                    if (0 == strcmp(types[m], "native")) {
+                        /* it's us! */
+                        takeus = true;
+                        break;
+                    }
+                }
+                pmix_argv_free(types);
+                if (!takeus) {
+                    return PMIX_ERR_NOT_SUPPORTED;
+                }
+            }
+        }
+    }
+
+    /* check uid */
+    if (euid != pr->info->uid) {
+        pmix_output_verbose(2, pmix_globals.debug_output,
+                            "psec: socket cred contains invalid uid %u", euid);
+        return PMIX_ERR_INVALID_CRED;
+    }
+
+    /* check gid */
+    if (egid != pr->info->gid) {
+        pmix_output_verbose(2, pmix_globals.debug_output,
+                            "psec: socket cred contains invalid gid %u", egid);
+        return PMIX_ERR_INVALID_CRED;
+    }
+
+    /* validated - mark that we did it */
+    if (NULL != info) {
+        PMIX_INFO_CREATE(*info, 3);
+        if (NULL == *info) {
+            return PMIX_ERR_NOMEM;
+        }
+        *ninfo = 3;
+        /* mark that this came from us */
+        PMIX_INFO_LOAD(info[0], PMIX_CRED_TYPE, "munge", PMIX_STRING);
+        /* provide the uid it contained */
+        u32 = euid;
+        PMIX_INFO_LOAD(info[1], PMIX_USERID, &u32, PMIX_UINT32);
+        /* provide the gid it contained */
+        u32 = egid;
+        PMIX_INFO_LOAD(info[2], PMIX_GRPID, &u32, PMIX_UINT32);
+    }
+    return PMIX_SUCCESS;
+
 }

opal/mca/pmix/pmix3x/pmix/src/mca/psec/none/psec_none.c

@@ -1,6 +1,8 @@
 /*
  * Copyright (c) 2015-2017 Intel, Inc. All rights reserved.
  * Copyright (c) 2016      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2017      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
  *
  * $COPYRIGHT$
  *
@@ -29,14 +31,20 @@
 
 static pmix_status_t none_init(void);
 static void none_finalize(void);
-static pmix_status_t validate_cred(int sd, uid_t uid, gid_t gid,
+static pmix_status_t create_cred(struct pmix_peer_t *peer,
-                                   pmix_listener_protocol_t protocol,
+                                 const pmix_info_t directives[], size_t ndirs,
-                                   char *cred, size_t len);
+                                 pmix_info_t **info, size_t *ninfo,
+                                 pmix_byte_object_t *cred);
+static pmix_status_t validate_cred(struct pmix_peer_t *peer,
+                                   const pmix_info_t directives[], size_t ndirs,
+                                   pmix_info_t **info, size_t *ninfo,
+                                   const pmix_byte_object_t *cred);
 
 pmix_psec_module_t pmix_none_module = {
     .name = "none",
     .init = none_init,
     .finalize = none_finalize,
+    .create_cred = create_cred,
     .validate_cred = validate_cred
 };
 
@@ -53,11 +61,61 @@ static void none_finalize(void)
                         "psec: none finalize");
 }
 
-static pmix_status_t validate_cred(int sd, uid_t uid, gid_t gid,
+static pmix_status_t create_cred(struct pmix_peer_t *peer,
-                                   pmix_listener_protocol_t protocol,
+                                 const pmix_info_t directives[], size_t ndirs,
-                                   char *cred, size_t len)
+                                 pmix_info_t **info, size_t *ninfo,
+                                 pmix_byte_object_t *cred)
 {
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    /* ensure initialization */
-                        "psec: none always reports valid");
+    PMIX_BYTE_OBJECT_CONSTRUCT(cred);
+
+    return PMIX_SUCCESS;
+}
+
+static pmix_status_t validate_cred(struct pmix_peer_t *peer,
+                                   const pmix_info_t directives[], size_t ndirs,
+                                   pmix_info_t **info, size_t *ninfo,
+                                   const pmix_byte_object_t *cred)
+{
+    size_t n, m;
+    char **types;
+    bool takeus;
+
+    pmix_output_verbose(2, pmix_globals.debug_output,
+                        "psec: none always reports valid");
+
+    /* if we are responding to a local request to validate a credential,
+     * then see if they specified a mechanism */
+    if (NULL != directives && 0 < ndirs) {
+        for (n=0; n < ndirs; n++) {
+            if (0 == strncmp(directives[n].key, PMIX_CRED_TYPE, PMIX_MAX_KEYLEN)) {
+                /* split the specified string */
+                types = pmix_argv_split(directives[n].value.data.string, ',');
+                takeus = false;
+                for (m=0; NULL != types[m]; m++) {
+                    if (0 == strcmp(types[m], "none")) {
+                        /* it's us! */
+                        takeus = true;
+                        break;
+                    }
+                }
+                pmix_argv_free(types);
+                if (!takeus) {
+                    return PMIX_ERR_NOT_SUPPORTED;
+                }
+            }
+        }
+    }
+
+    /* mark that this came from us */
+    if (NULL != info) {
+        /* mark that this came from us */
+        PMIX_INFO_CREATE(*info, 1);
+        if (NULL == *info) {
+            return PMIX_ERR_NOMEM;
+        }
+        *ninfo = 1;
+        PMIX_INFO_LOAD(info[0], PMIX_CRED_TYPE, "none", PMIX_STRING);
+    }
     return PMIX_SUCCESS;
 }

opal/mca/pmix/pmix3x/pmix/src/mca/psec/none/psec_none_component.c

@@ -12,7 +12,7 @@
  *                         All rights reserved.
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
- * Copyright (c) 2016      Intel, Inc. All rights reserved.
+ * Copyright (c) 2016-2017 Intel, Inc. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -29,7 +29,7 @@
 #include <src/include/pmix_config.h>
 #include "pmix_common.h"
 
-
+#include "src/mca/base/pmix_mca_base_var.h"
 #include "src/mca/psec/psec.h"
 #include "psec_none.h"
 
@@ -68,7 +68,21 @@ pmix_psec_base_component_t mca_psec_none_component = {
 
 static int component_open(void)
 {
-    return PMIX_SUCCESS;
+    int index;
+    const pmix_mca_base_var_storage_t *value=NULL;
+
+    /* we only allow ourselves to be considered IF the user
+     * specifically requested so */
+    if (0 > (index = pmix_mca_base_var_find("pmix", "psec", NULL, NULL))) {
+        return PMIX_ERROR;
+    }
+    pmix_mca_base_var_get_value (index, &value, NULL, NULL);
+    if (NULL != value && NULL != value->stringval && '\0' != value->stringval[0]) {
+        if (NULL != strstr(value->stringval, "none")) {
+            return PMIX_SUCCESS;
+        }
+    }
+    return PMIX_ERROR;
 }
 
 

opal/mca/pmix/pmix3x/pmix/src/mca/psec/psec.h

@@ -37,6 +37,9 @@
 
 BEGIN_C_DECLS
 
+/* forward declaration */
+struct pmix_peer_t;
+
 /******    MODULE DEFINITION    ******/
 
 /**
@@ -55,10 +58,16 @@ typedef void (*pmix_psec_base_module_fini_fn_t)(void);
 /**
  * Create and return a credential for this client - this
  * could be a string or a byte array, which is why we must
- * also return the length
+ * also return the length. The directives contain info on
+ * desired credential type, or other directives used by
+ * the credential agent. The returned info array typically
+ * contains the name of the agent issuing the credential,
+ * plus any other info the agent chooses to return.
  */
-typedef pmix_status_t (*pmix_psec_base_module_create_cred_fn_t)(pmix_listener_protocol_t protocol,
+typedef pmix_status_t (*pmix_psec_base_module_create_cred_fn_t)(struct pmix_peer_t *peer,
-                                                                char **cred, size_t *len);
+                                                                const pmix_info_t directives[], size_t ndirs,
+                                                                pmix_info_t **info, size_t *ninfo,
+                                                                pmix_byte_object_t *cred);
 
 /**
  * Perform the client-side handshake. Note that it is not required
@@ -71,11 +80,18 @@ typedef pmix_status_t (*pmix_psec_base_module_client_hndshk_fn_t)(int sd);
 /****    SERVER-SIDE FUNCTIONS    ****/
 /**
  * Validate a client's credential - the credential could be a string
- * or an array of bytes, which is why we include the length
+ * or an array of bytes, which is why we include the length. The directives
+ * contain information provided by the requestor to aid in the validation
+ * process - e.g., the uid/gid of the process seeking validation, or
+ * the name of the agent being asked to perform the validation. The
+ * returned info array typically contains the name of the agent that
+ * actually performed the validation, plus any other info the agent
+ * chooses to return (e.g., the uid/gid contained in the credential)
  */
-typedef pmix_status_t (*pmix_psec_base_module_validate_cred_fn_t)(int sd, uid_t uid, gid_t gid,
+typedef pmix_status_t (*pmix_psec_base_module_validate_cred_fn_t)(struct pmix_peer_t *peer,
-                                                                  pmix_listener_protocol_t protocol,
+                                                                  const pmix_info_t directives[], size_t ndirs,
-                                                                  char *cred, size_t len);
+                                                                  pmix_info_t **info, size_t *ninfo,
+                                                                  const pmix_byte_object_t *cred);
 
 /**
  * Perform the server-side handshake. Note that it is not required
@@ -112,21 +128,25 @@ PMIX_EXPORT pmix_psec_module_t* pmix_psec_base_assign_module(const char *options
 
 /* MACROS FOR EXECUTING PSEC FUNCTIONS */
 
-#define PMIX_PSEC_CREATE_CRED(r, p, pr, c, l)       \
+#define PMIX_PSEC_CREATE_CRED(r, p, d, nd, in, nin, c)                      \
-    (r) = (p)->nptr->compat.psec->create_cred(pr, c, l)
+    (r) = (p)->nptr->compat.psec->create_cred((struct pmix_peer_t*)(p),     \
+                                              (d), (nd), (in), (nin), c)
 
 #define PMIX_PSEC_CLIENT_HANDSHAKE(r, p, sd) \
     (r) = (p)->nptr->compat.psec->client_handshake(sd)
 
-#define PMIX_PSEC_VALIDATE_CRED(r, p, pr, c, l)     \
+#define PMIX_PSEC_VALIDATE_CRED(r, p, d, nd, in, nin, c)                    \
-    (r) = (p)->nptr->compat.psec->validate_cred((p)->sd, (p)->info->uid, (p)->info->gid, pr, c, l)
+    (r) = (p)->nptr->compat.psec->validate_cred((struct pmix_peer_t*)(p),   \
+                                                (d), (nd),                  \
+                                                (in), (nin), c)
 
-#define PMIX_PSEC_VALIDATE_CONNECTION(r, p, pr, c, l)                                                       \
+#define PMIX_PSEC_VALIDATE_CONNECTION(r, p, d, nd, in, nin, c)                                              \
     do {                                                                                                    \
         int _r;                                                                                             \
         /* if a credential is available, then check it */                                                   \
         if (NULL != (p)->nptr->compat.psec->validate_cred) {                                                \
-            _r = (p)->nptr->compat.psec->validate_cred((p)->sd, (p)->info->uid, (p)->info->gid, pr, c, l);  \
+            _r = (p)->nptr->compat.psec->validate_cred((struct pmix_peer_t*)(p),                            \
+                                                       (d), (nd), (in), (nin), c);                          \
             if (PMIX_SUCCESS != _r) {                                                                       \
                 pmix_output_verbose(2, pmix_globals.debug_output,                                           \
                                     "validation of credential failed: %s",                                  \

opal/mca/pmix/pmix3x/pmix/src/mca/ptl/base/ptl_base_sendrecv.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014-2017 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
  * Copyright (c) 2014      Artem Y. Polyakov <artpol84@gmail.com>.
  *                         All rights reserved.
  * Copyright (c) 2015-2017 Research Organization for Information Science
@@ -58,7 +58,6 @@ void pmix_ptl_base_lost_connection(pmix_peer_t *peer, pmix_status_t err)
 {
     pmix_server_trkr_t *trk;
     pmix_server_caddy_t *rinfo, *rnext;
-    pmix_trkr_caddy_t *tcd;
     pmix_regevents_info_t *reginfoptr, *regnext;
     pmix_peer_events_info_t *pr, *pnext;
     pmix_rank_info_t *info, *pinfo;
@@ -102,13 +101,21 @@ void pmix_ptl_base_lost_connection(pmix_peer_t *peer, pmix_status_t err)
                 /* remove it from the list */
                 pmix_list_remove_item(&trk->local_cbs, &rinfo->super);
                 PMIX_RELEASE(rinfo);
-                /* check for completion */
+                /* we need to let the other participants know that this
-                if (pmix_list_get_size(&trk->local_cbs) == trk->nlocal) {
+                 * proc has disappeared as otherwise the collective will never
-                    /* complete, so now we need to process it
+                 * complete */
-                     * we don't want to block someone
+                if (PMIX_FENCENB_CMD == trk->type) {
-                     * here, so kick any completed trackers into a
+                    if (NULL != trk->modexcbfunc) {
-                     * new event for processing */
+                        trk->modexcbfunc(PMIX_ERR_LOST_CONNECTION_TO_CLIENT, NULL, 0, trk, NULL, NULL);
-                    PMIX_EXECUTE_COLLECTIVE(tcd, trk, pmix_server_execute_collective);
+                    }
+                } else if (PMIX_CONNECTNB_CMD == trk->type) {
+                    if (NULL != trk->cnct_cbfunc) {
+                        trk->cnct_cbfunc(PMIX_ERR_LOST_CONNECTION_TO_CLIENT, NULL, PMIX_RANK_WILDCARD, trk);
+                    }
+                } else if (PMIX_DISCONNECTNB_CMD == trk->type) {
+                    if (NULL != trk->op_cbfunc) {
+                        trk->op_cbfunc(PMIX_ERR_LOST_CONNECTION_TO_CLIENT, trk);
+                    }
                 }
             }
         }

opal/mca/pmix/pmix3x/pmix/src/mca/ptl/ptl_types.h

@@ -89,6 +89,7 @@ typedef uint32_t pmix_ptl_tag_t;
  * within the system */
 #define PMIX_PTL_TAG_NOTIFY           0
 #define PMIX_PTL_TAG_HEARTBEAT        1
+#define PMIX_PTL_TAG_IOF              2
 
 /* define the start of dynamic tags that are
  * assigned for send/recv operations */
@@ -176,9 +177,9 @@ PMIX_CLASS_DECLARATION(pmix_ptl_queue_t);
 
 /* define listener protocol types */
 typedef uint16_t pmix_listener_protocol_t;
-#define PMIX_PROTOCOL_V1        0       // legacy usock
+#define PMIX_PROTOCOL_UNDEF     0
-#define PMIX_PROTOCOL_V2        1       // tcp
+#define PMIX_PROTOCOL_V1        1       // legacy usock
-#define PMIX_PROTOCOL_V3        2       // updated usock
+#define PMIX_PROTOCOL_V2        2       // tcp
 
 /* connection support */
 typedef struct {

opal/mca/pmix/pmix3x/pmix/src/mca/ptl/tcp/ptl_tcp.c

@@ -13,7 +13,7 @@
  * Copyright (c) 2011-2014 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2011-2013 Los Alamos National Security, LLC.  All rights
  *                         reserved.
- * Copyright (c) 2013-2017 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2013-2018 Intel, Inc. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -163,6 +163,8 @@ static pmix_status_t connect_to_peer(struct pmix_peer_t *peer,
         }
         /* the server will be using the same bfrops as us */
         pmix_client_globals.myserver->nptr->compat.bfrops = pmix_globals.mypeer->nptr->compat.bfrops;
+        /* mark that we are using the V2 protocol */
+        pmix_globals.mypeer->protocol = PMIX_PROTOCOL_V2;
 
         /* the URI consists of the following elements:
         *    - server nspace.rank
@@ -230,6 +232,8 @@ static pmix_status_t connect_to_peer(struct pmix_peer_t *peer,
             }
         }
     }
+    /* mark that we are using the V2 protocol */
+    pmix_globals.mypeer->protocol = PMIX_PROTOCOL_V2;
     gethostname(myhost, sizeof(myhost));
     /* if we were given a URI via MCA param, then look no further */
     if (NULL != mca_ptl_tcp_component.super.uri) {
@@ -528,14 +532,17 @@ static pmix_status_t parse_uri_file(char *filename,
         pmix_client_globals.myserver->proc_type = PMIX_PROC_SERVER | PMIX_PROC_V20;
         pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
                             "V20 SERVER DETECTED");
+        pmix_client_globals.myserver->protocol = PMIX_PROTOCOL_V2;
     } else if (0 == strncmp(p2, "v2.1", strlen("v2.1")) ||
                0 == strncmp(p2, "2.1", strlen("2.1"))) {
         pmix_client_globals.myserver->proc_type = PMIX_PROC_SERVER | PMIX_PROC_V21;
+        pmix_client_globals.myserver->protocol = PMIX_PROTOCOL_V2;
         pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
                             "V21 SERVER DETECTED");
     } else if (0 == strncmp(p2, "3", strlen("3")) ||
                0 == strncmp(p2, "v3", strlen("v3"))) {
         pmix_client_globals.myserver->proc_type = PMIX_PROC_SERVER | PMIX_PROC_V3;
+        pmix_client_globals.myserver->protocol = PMIX_PROTOCOL_V2;
         pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
                             "V3 SERVER DETECTED");
     } else {
@@ -661,7 +668,7 @@ static pmix_status_t try_connect(int *sd)
   retry:
     /* establish the connection */
     if (PMIX_SUCCESS != (rc = pmix_ptl_base_connect(&mca_ptl_tcp_component.connection, len, sd))) {
-        PMIX_ERROR_LOG(rc);
+        /* do not error log - might just be a stale connection point */
         return rc;
     }
 
@@ -692,8 +699,8 @@ static pmix_status_t send_connect_ack(int sd)
 {
     char *msg;
     pmix_ptl_hdr_t hdr;
-    size_t sdsize=0, csize=0, len;
+    size_t sdsize=0, csize=0;
-    char *cred = NULL;
+    pmix_byte_object_t cred;
     char *sec, *bfrops, *gds;
     pmix_bfrop_buffer_type_t bftype;
     pmix_status_t rc;
@@ -720,9 +727,11 @@ static pmix_status_t send_connect_ack(int sd)
      * local PMIx server, if known. Now use that module to
      * get a credential, if the security system provides one. Not
      * every psec module will do so, thus we must first check */
-    PMIX_PSEC_CREATE_CRED(rc, pmix_client_globals.myserver,
+    PMIX_BYTE_OBJECT_CONSTRUCT(&cred);
-                          PMIX_PROTOCOL_V2, &cred, &len);
+    PMIX_PSEC_CREATE_CRED(rc, pmix_globals.mypeer,
+                          NULL, 0, NULL, 0, &cred);
     if (PMIX_SUCCESS != rc) {
+        pmix_output(0, "OUCH: %d", __LINE__);
         return rc;
     }
 
@@ -754,14 +763,12 @@ static pmix_status_t send_connect_ack(int sd)
     /* set the number of bytes to be read beyond the header */
     hdr.nbytes = sdsize + strlen(PMIX_VERSION) + 1 + strlen(sec) + 1 \
                 + strlen(bfrops) + 1 + sizeof(bftype) \
-                + strlen(gds) + 1 + sizeof(uint32_t) + len;  // must NULL terminate the strings!
+                + strlen(gds) + 1 + sizeof(uint32_t) + cred.size;  // must NULL terminate the strings!
 
     /* create a space for our message */
     sdsize = (sizeof(hdr) + hdr.nbytes);
     if (NULL == (msg = (char*)malloc(sdsize))) {
-        if (NULL != cred) {
+        PMIX_BYTE_OBJECT_DESTRUCT(&cred);
-            free(cred);
-        }
         free(sec);
         return PMIX_ERR_OUT_OF_RESOURCE;
     }
@@ -779,14 +786,15 @@ static pmix_status_t send_connect_ack(int sd)
     /* load the length of the credential - we put this in uint32_t
      * format as that is a fixed size, and convert to network
      * byte order for heterogeneity */
-    u32 = htonl((uint32_t)len);
+    u32 = htonl((uint32_t)cred.size);
     memcpy(msg+csize, &u32, sizeof(uint32_t));
     csize += sizeof(uint32_t);
     /* load the credential */
     if (0 < u32) {
-        memcpy(msg+csize, cred, len);
+        memcpy(msg+csize, cred.bytes, cred.size);
-        csize += len;
+        csize += cred.size;
     }
+    PMIX_BYTE_OBJECT_DESTRUCT(&cred);
 
     /* load our process type - this is a single byte,
      * so no worry about heterogeneity here */
@@ -833,15 +841,9 @@ static pmix_status_t send_connect_ack(int sd)
     /* send the entire message across */
     if (PMIX_SUCCESS != pmix_ptl_base_send_blocking(sd, msg, sdsize)) {
         free(msg);
-        if (NULL != cred) {
-            free(cred);
-        }
         return PMIX_ERR_UNREACH;
     }
     free(msg);
-    if (NULL != cred) {
-        free(cred);
-    }
     return PMIX_SUCCESS;
 }
 

opal/mca/pmix/pmix3x/pmix/src/mca/ptl/tcp/ptl_tcp_component.c

@@ -823,6 +823,7 @@ static void connection_handler(int sd, short args, void *cbdata)
     pmix_proc_t proc;
     pmix_info_t ginfo;
     pmix_proc_type_t proc_type;
+    pmix_byte_object_t cred;
 
     /* acquire the object */
     PMIX_ACQUIRE_OBJECT(pnd);
@@ -1156,6 +1157,8 @@ static void connection_handler(int sd, short args, void *cbdata)
     }
     /* mark that this peer is a client of the given type */
     peer->proc_type = PMIX_PROC_CLIENT | proc_type;
+    /* save the protocol */
+    peer->protocol = pnd->protocol;
     /* add in the nspace pointer */
     PMIX_RETAIN(nptr);
     peer->nptr = nptr;
@@ -1225,9 +1228,9 @@ static void connection_handler(int sd, short args, void *cbdata)
     peer->nptr->compat.ptl = &pmix_ptl_tcp_module;
 
     /* validate the connection */
-    PMIX_PSEC_VALIDATE_CONNECTION(rc, peer,
+    cred.bytes = pnd->cred;
-                                  PMIX_PROTOCOL_V2,
+    cred.size = pnd->len;
-                                  pnd->cred, pnd->len);
+    PMIX_PSEC_VALIDATE_CONNECTION(rc, peer, NULL, 0, NULL, NULL, &cred);
     if (PMIX_SUCCESS != rc) {
         pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
                             "validation of client connection failed");
@@ -1313,6 +1316,7 @@ static void process_cbfunc(int sd, short args, void *cbdata)
     int rc;
     uint32_t u32;
     pmix_info_t ginfo;
+    pmix_byte_object_t cred;
 
     /* acquire the object */
     PMIX_ACQUIRE_OBJECT(cd);
@@ -1400,6 +1404,8 @@ static void process_cbfunc(int sd, short args, void *cbdata)
     }
     /* mark the peer proc type */
     peer->proc_type = PMIX_PROC_TOOL | pnd->proc_type;
+    /* save the protocol */
+    peer->protocol = pnd->protocol;
     /* add in the nspace pointer */
     PMIX_RETAIN(nptr);
     peer->nptr = nptr;
@@ -1450,9 +1456,9 @@ static void process_cbfunc(int sd, short args, void *cbdata)
     }
 
     /* validate the connection */
-    PMIX_PSEC_VALIDATE_CONNECTION(rc, peer,
+    cred.bytes = pnd->cred;
-                                  PMIX_PROTOCOL_V2,
+    cred.size = pnd->len;
-                                  pnd->cred, pnd->len);
+    PMIX_PSEC_VALIDATE_CONNECTION(rc, peer, NULL, 0, NULL, NULL, &cred);
     if (PMIX_SUCCESS != rc) {
         pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
                             "validation of tool credentials failed: %s",

opal/mca/pmix/pmix3x/pmix/src/mca/ptl/usock/ptl_usock.c

@@ -132,6 +132,8 @@ static pmix_status_t connect_to_peer(struct pmix_peer_t *peer,
     }
     /* the server will be using the same bfrops as us */
     pmix_client_globals.myserver->nptr->compat.bfrops = pmix_globals.mypeer->nptr->compat.bfrops;
+    /* mark that we are using the V1 protocol */
+    pmix_globals.mypeer->protocol = PMIX_PROTOCOL_V1;
 
     uri = pmix_argv_split(evar, ':');
     if (3 != pmix_argv_count(uri)) {
@@ -273,8 +275,8 @@ static pmix_status_t send_connect_ack(int sd)
 {
     char *msg;
     pmix_usock_hdr_t hdr;
-    size_t sdsize=0, csize=0, len;
+    size_t sdsize=0, csize=0;
-    char *cred = NULL;
+    pmix_byte_object_t cred;
     pmix_status_t rc;
     char *sec, *bfrops, *gds;
     pmix_bfrop_buffer_type_t bftype;
@@ -292,8 +294,9 @@ static pmix_status_t send_connect_ack(int sd)
 
     /* get a credential, if the security system provides one. Not
      * every SPC will do so, thus we must first check */
-    PMIX_PSEC_CREATE_CRED(rc, pmix_client_globals.myserver,
+    PMIX_BYTE_OBJECT_CONSTRUCT(&cred);
-                          PMIX_PROTOCOL_V1, &cred, &len);
+    PMIX_PSEC_CREATE_CRED(rc, pmix_globals.mypeer,
+                          NULL, 0, NULL, 0, &cred);
     if (PMIX_SUCCESS != rc) {
         return rc;
     }
@@ -312,7 +315,7 @@ static pmix_status_t send_connect_ack(int sd)
 
     /* set the number of bytes to be read beyond the header */
     hdr.nbytes = sdsize + (strlen(PMIX_VERSION) + 1) + \
-                (sizeof(size_t) + len) + \
+                (sizeof(size_t) + cred.size) + \
                 (strlen(sec) + 1) + \
                 (strlen(bfrops) + 1) + sizeof(bftype) + \
                 (strlen(gds) + 1);  // must NULL terminate the strings!
@@ -320,9 +323,7 @@ static pmix_status_t send_connect_ack(int sd)
     /* create a space for our message */
     sdsize = (sizeof(hdr) + hdr.nbytes);
     if (NULL == (msg = (char*)malloc(sdsize))) {
-        if (NULL != cred) {
+        PMIX_BYTE_OBJECT_DESTRUCT(&cred);
-            free(cred);
-        }
         return PMIX_ERR_OUT_OF_RESOURCE;
     }
     memset(msg, 0, sdsize);
@@ -343,12 +344,13 @@ static pmix_status_t send_connect_ack(int sd)
     csize += strlen(PMIX_VERSION)+1;
 
     /* pass the size of the credential */
-    memcpy(msg+csize, &len, sizeof(size_t));
+    memcpy(msg+csize, &cred.size, sizeof(size_t));
     csize += sizeof(size_t);
-    if (0 < len) {
+    if (0 < cred.size) {
-        memcpy(msg+csize, cred, len);
+        memcpy(msg+csize, cred.bytes, cred.size);
-        csize += len;
+        csize += cred.size;
     }
+    PMIX_BYTE_OBJECT_DESTRUCT(&cred);
 
     /* pass our active sec module */
     memcpy(msg+csize, sec, strlen(sec));
@@ -368,15 +370,9 @@ static pmix_status_t send_connect_ack(int sd)
     /* send the entire msg across */
     if (PMIX_SUCCESS != pmix_ptl_base_send_blocking(sd, msg, sdsize)) {
         free(msg);
-        if (NULL != cred) {
-            free(cred);
-        }
         return PMIX_ERR_UNREACH;
     }
     free(msg);
-    if (NULL != cred) {
-        free(cred);
-    }
     return PMIX_SUCCESS;
 }
 

opal/mca/pmix/pmix3x/pmix/src/mca/ptl/usock/ptl_usock_component.c

@@ -337,7 +337,7 @@ static void listener_cb(int incoming_sd, void *cbdata)
 static void connection_handler(int sd, short args, void *cbdata)
 {
     pmix_pending_connection_t *pnd = (pmix_pending_connection_t*)cbdata;
-    char *msg, *ptr, *nspace, *version, *cred, *sec, *bfrops, *gds;
+    char *msg, *ptr, *nspace, *version, *sec, *bfrops, *gds;
     pmix_status_t rc;
     unsigned int rank;
     pmix_usock_hdr_t hdr;
@@ -352,7 +352,7 @@ static void connection_handler(int sd, short args, void *cbdata)
     int major, minor, rel;
     unsigned int msglen;
     pmix_info_t ginfo;
-    size_t credlen;
+    pmix_byte_object_t cred;
 
     /* acquire the object */
     PMIX_ACQUIRE_OBJECT(pnd);
@@ -452,11 +452,13 @@ static void connection_handler(int sd, short args, void *cbdata)
 
     /* get any provided credential */
     if (1 == major) {
+        /* credential is always a string */
         PMIX_STRNLEN(msglen, ptr, len);
         if (msglen < len) {
-            cred = ptr;
+            cred.bytes = ptr;
-            ptr += strlen(cred) + 1;
+            cred.size = strlen(ptr);
-            len -= strlen(cred) + 1;
+            ptr += cred.size + 1;
+            len -= cred.size + 1;
         } else {
             free(msg);
             CLOSE_THE_SOCKET(pnd->sd);
@@ -464,8 +466,9 @@ static void connection_handler(int sd, short args, void *cbdata)
             return;
         }
     } else {
+        /* credential could be something else */
         if (sizeof(size_t) < len) {
-            memcpy(&credlen, ptr, sizeof(size_t));
+            memcpy(&cred.size, ptr, sizeof(size_t));
             ptr += sizeof(size_t);
             len -= sizeof(size_t);
         } else {
@@ -474,10 +477,10 @@ static void connection_handler(int sd, short args, void *cbdata)
             PMIX_RELEASE(pnd);
             return;
         }
-        if (0 < credlen) {
+        if (0 < cred.size) {
-            cred = ptr;
+            cred.bytes = ptr;
-            ptr += credlen;
+            ptr += cred.size;
-            len -= credlen;
+            len -= cred.size;
         }
     }
 
@@ -596,6 +599,8 @@ static void connection_handler(int sd, short args, void *cbdata)
         PMIX_RELEASE(pnd);
         return;
     }
+    /* save the protocol */
+    psave->protocol = pnd->protocol;
     /* add the nspace tracker */
     PMIX_RETAIN(nptr);
     psave->nptr = nptr;
@@ -669,13 +674,7 @@ static void connection_handler(int sd, short args, void *cbdata)
     nptr->compat.ptl = &pmix_ptl_usock_module;
 
     /* validate the connection */
-    if (NULL == cred) {
+    PMIX_PSEC_VALIDATE_CONNECTION(rc, psave, NULL, 0, NULL, 0, &cred);
-        len = 0;
-    } else {
-        len = strlen(cred);
-    }
-    PMIX_PSEC_VALIDATE_CONNECTION(rc, psave,
-                                  PMIX_PROTOCOL_V1, cred, len);
     /* now done with the msg */
     free(msg);
 
@@ -738,9 +737,6 @@ static void connection_handler(int sd, short args, void *cbdata)
     return;
 
   error:
-    if (NULL != cred) {
-        free(cred);
-    }
     /* send an error reply to the client */
     if (PMIX_SUCCESS != pmix_ptl_base_send_blocking(pnd->sd, (char*)&rc, sizeof(int))) {
         PMIX_ERROR_LOG(rc);

opal/mca/pmix/pmix3x/pmix/src/runtime/Makefile.include

@@ -12,7 +12,7 @@
 #                         All rights reserved.
 # Copyright (c) 2012      Los Alamos National Security, LLC.
 #                         All rights reserved.
-# Copyright (c) 2014-2016 Intel, Inc. All rights reserved
+# Copyright (c) 2014-2017 Intel, Inc. All rights reserved.
 # Copyright (c) 2014      Cisco Systems, Inc.  All rights reserved.
 # $COPYRIGHT$
 #
@@ -29,7 +29,7 @@ headers += \
         runtime/pmix_rte.h \
         runtime/pmix_progress_threads.h
 
-libpmix_la_SOURCES += \
+sources += \
         runtime/pmix_finalize.c \
         runtime/pmix_init.c \
         runtime/pmix_params.c \

opal/mca/pmix/pmix3x/pmix/src/runtime/pmix_finalize.c

@@ -12,7 +12,7 @@
  * Copyright (c) 2008-2015 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2010-2015 Los Alamos National Security, LLC.
  *                         All rights reserved.
- * Copyright (c) 2013-2017 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2013-2018 Intel, Inc. All rights reserved.
  * Copyright (c) 2016      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
@@ -115,6 +115,7 @@ void pmix_rte_finalize(void)
     PMIX_DESTRUCT(&pmix_globals.events);
     PMIX_LIST_DESTRUCT(&pmix_globals.cached_events);
     PMIX_DESTRUCT(&pmix_globals.notifications);
+    PMIX_LIST_DESTRUCT(&pmix_globals.iof_requests);
 
     /* now safe to release the event base */
     if (!pmix_globals.external_evbase) {

opal/mca/pmix/pmix3x/pmix/src/runtime/pmix_init.c

@@ -15,7 +15,7 @@
  * Copyright (c) 2009      Oak Ridge National Labs.  All rights reserved.
  * Copyright (c) 2010-2015 Los Alamos National Security, LLC.
  *                         All rights reserved.
- * Copyright (c) 2013-2017 Intel, Inc. All rights reserved.
+ * Copyright (c) 2013-2018 Intel, Inc. All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
@@ -159,6 +159,8 @@ int pmix_rte_init(pmix_proc_type_t type,
     /* construct the global notification ring buffer */
     PMIX_CONSTRUCT(&pmix_globals.notifications, pmix_ring_buffer_t);
     pmix_ring_buffer_init(&pmix_globals.notifications, 256);
+    /* and setup the iof request tracking list */
+    PMIX_CONSTRUCT(&pmix_globals.iof_requests, pmix_list_t);
 
     /* Setup client verbosities as all procs are allowed to
      * access client APIs */
@@ -198,6 +200,12 @@ int pmix_rte_init(pmix_proc_type_t type,
         pmix_output_set_verbosity(pmix_client_globals.event_output,
                                   pmix_client_globals.event_verbose);
     }
+    if (0 < pmix_client_globals.iof_verbose) {
+        /* set default output */
+        pmix_client_globals.iof_output = pmix_output_open(NULL);
+        pmix_output_set_verbosity(pmix_client_globals.iof_output,
+                                  pmix_client_globals.iof_verbose);
+    }
 
     /* get our effective id's */
     pmix_globals.uid = geteuid();

opal/mca/pmix/pmix3x/pmix/src/runtime/pmix_params.c

@@ -21,7 +21,7 @@
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2015      Mellanox Technologies, Inc.
  *                         All rights reserved.
- * Copyright (c) 2016-2017 Intel, Inc. All rights reserved.
+ * Copyright (c) 2016-2018 Intel, Inc. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -140,11 +140,17 @@ pmix_status_t pmix_register_params(void)
                                   &pmix_client_globals.spawn_verbose);
 
     (void) pmix_mca_base_var_register ("pmix", "pmix", "client", "event_verbose",
-                                  "Verbosity for eventt spawn operations",
+                                  "Verbosity for client spawn operations",
                                   PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
                                   PMIX_INFO_LVL_1, PMIX_MCA_BASE_VAR_SCOPE_ALL,
                                   &pmix_client_globals.event_verbose);
 
+    (void) pmix_mca_base_var_register ("pmix", "pmix", "client", "iof_verbose",
+                                  "Verbosity for client iof operations",
+                                  PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                  PMIX_INFO_LVL_1, PMIX_MCA_BASE_VAR_SCOPE_ALL,
+                                  &pmix_client_globals.iof_verbose);
+
     (void) pmix_mca_base_var_register ("pmix", "pmix", "client", "base_verbose",
                                   "Verbosity for basic client operations",
                                   PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
@@ -188,6 +194,12 @@ pmix_status_t pmix_register_params(void)
                                   PMIX_INFO_LVL_1, PMIX_MCA_BASE_VAR_SCOPE_ALL,
                                   &pmix_server_globals.event_verbose);
 
+    (void) pmix_mca_base_var_register ("pmix", "pmix", "server", "iof_verbose",
+                                  "Verbosity for server iof operations",
+                                  PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                  PMIX_INFO_LVL_1, PMIX_MCA_BASE_VAR_SCOPE_ALL,
+                                  &pmix_server_globals.iof_verbose);
+
     (void) pmix_mca_base_var_register ("pmix", "pmix", "server", "base_verbose",
                                   "Verbosity for basic server operations",
                                   PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,

opal/mca/pmix/pmix3x/pmix/src/server/pmix_server.c

@@ -170,6 +170,12 @@ PMIX_EXPORT pmix_status_t PMIx_server_init(pmix_server_module_t *module,
         pmix_output_set_verbosity(pmix_server_globals.event_output,
                                   pmix_server_globals.event_verbose);
     }
+    if (0 < pmix_server_globals.iof_verbose) {
+        /* set default output */
+        pmix_server_globals.iof_output = pmix_output_open(NULL);
+        pmix_output_set_verbosity(pmix_server_globals.iof_output,
+                                  pmix_server_globals.iof_verbose);
+    }
     /* setup the base verbosity */
     if (0 < pmix_server_globals.base_verbose) {
         /* set default output */
@@ -1437,6 +1443,99 @@ pmix_status_t PMIx_server_setup_local_support(const char nspace[],
     return PMIX_SUCCESS;
 }
 
+static void _iofpush(int sd, short args, void *cbdata)
+{
+    pmix_setup_caddy_t *cd = (pmix_setup_caddy_t*)cbdata;
+    pmix_iof_req_t *req;
+    pmix_status_t rc;
+    pmix_buffer_t *msg;
+
+    pmix_output_verbose(2, pmix_server_globals.iof_output,
+                        "PMIX:SERVER pushing IOF");
+
+    /* cycle across our list of IOF requestors and see who wants
+     * this channel from this source */
+    PMIX_LIST_FOREACH(req, &pmix_globals.iof_requests, pmix_iof_req_t) {
+        /* if the channel wasn't included, then ignore it */
+        if (!(cd->channels & req->channels)) {
+            continue;
+        }
+        /* if the source matches the request, then forward this along */
+        if (0 != strncmp(cd->procs->nspace, req->pname.nspace, PMIX_MAX_NSLEN) ||
+            (PMIX_RANK_WILDCARD != req->pname.rank && cd->procs->rank != req->pname.rank)) {
+            continue;
+        }
+        /* setup the msg */
+        if (NULL == (msg = PMIX_NEW(pmix_buffer_t))) {
+            PMIX_ERROR_LOG(PMIX_ERR_OUT_OF_RESOURCE);
+            rc = PMIX_ERR_OUT_OF_RESOURCE;
+            break;
+        }
+        /* provide the source */
+        PMIX_BFROPS_PACK(rc, req->peer, msg, cd->procs, 1, PMIX_PROC);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            PMIX_RELEASE(msg);
+            break;
+        }
+        /* provide the channel */
+        PMIX_BFROPS_PACK(rc, req->peer, msg, &cd->channels, 1, PMIX_IOF_CHANNEL);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            PMIX_RELEASE(msg);
+            break;
+        }
+        /* pack the data */
+        PMIX_BFROPS_PACK(rc, req->peer, msg, cd->bo, 1, PMIX_BYTE_OBJECT);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            PMIX_RELEASE(msg);
+            break;
+        }
+        /* send it to the requestor */
+        PMIX_PTL_SEND_ONEWAY(rc, req->peer, msg, PMIX_PTL_TAG_IOF);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            PMIX_RELEASE(msg);
+        }
+    }
+
+    if (NULL != cd->opcbfunc) {
+        cd->opcbfunc(rc, cd->cbdata);
+    }
+    PMIX_RELEASE(cd);
+}
+
+pmix_status_t PMIx_IOF_push(const pmix_proc_t *source, pmix_iof_channel_t channel,
+                            const pmix_byte_object_t *bo,
+                            const pmix_info_t info[], size_t ninfo,
+                            pmix_op_cbfunc_t cbfunc, void *cbdata)
+{
+    pmix_setup_caddy_t *cd;
+
+    PMIX_ACQUIRE_THREAD(&pmix_global_lock);
+    if (pmix_globals.init_cntr <= 0) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return PMIX_ERR_INIT;
+    }
+    PMIX_RELEASE_THREAD(&pmix_global_lock);
+
+    /* need to threadshift this request */
+    cd = PMIX_NEW(pmix_setup_caddy_t);
+    if (NULL == cd) {
+        return PMIX_ERR_NOMEM;
+    }
+    cd->procs = (pmix_proc_t*)source;
+    cd->channels = channel;
+    cd->bo = (pmix_byte_object_t*)bo;
+    cd->info = (pmix_info_t*)info;
+    cd->ninfo = ninfo;
+    cd->opcbfunc = cbfunc;
+    cd->cbdata = cbdata;
+    PMIX_THREADSHIFT(cd, _iofpush);
+
+    return PMIX_SUCCESS;
+}
 
 /****    THE FOLLOWING CALLBACK FUNCTIONS ARE USED BY THE HOST SERVER    ****
  ****    THEY THEREFORE CAN OCCUR IN EITHER THE HOST SERVER'S THREAD     ****
@@ -2240,6 +2339,177 @@ static void query_cbfunc(pmix_status_t status,
     PMIX_RELEASE(cd);
 }
 
+static void cred_cbfunc(pmix_status_t status,
+                        pmix_byte_object_t *credential,
+                        pmix_info_t info[], size_t ninfo,
+                        void *cbdata)
+{
+    pmix_query_caddy_t *qcd = (pmix_query_caddy_t*)cbdata;
+    pmix_server_caddy_t *cd = (pmix_server_caddy_t*)qcd->cbdata;
+    pmix_buffer_t *reply;
+    pmix_status_t rc;
+
+    pmix_output_verbose(2, pmix_globals.debug_output,
+                        "pmix:get credential callback with status %d", status);
+
+    reply = PMIX_NEW(pmix_buffer_t);
+    if (NULL == reply) {
+        PMIX_ERROR_LOG(PMIX_ERR_NOMEM);
+        PMIX_RELEASE(cd);
+        return;
+    }
+
+    /* pack the status */
+    PMIX_BFROPS_PACK(rc, cd->peer, reply, &status, 1, PMIX_STATUS);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        goto complete;
+    }
+
+    if (PMIX_SUCCESS == status) {
+        /* pack the returned credential */
+        PMIX_BFROPS_PACK(rc, cd->peer, reply, credential, 1, PMIX_BYTE_OBJECT);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            goto complete;
+        }
+
+        /* pack any returned data */
+        PMIX_BFROPS_PACK(rc, cd->peer, reply, &ninfo, 1, PMIX_SIZE);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            goto complete;
+        }
+        if (0 < ninfo) {
+            PMIX_BFROPS_PACK(rc, cd->peer, reply, info, ninfo, PMIX_INFO);
+            if (PMIX_SUCCESS != rc) {
+                PMIX_ERROR_LOG(rc);
+            }
+        }
+    }
+
+  complete:
+    // send reply
+    PMIX_SERVER_QUEUE_REPLY(cd->peer, cd->hdr.tag, reply);
+    // cleanup
+    if (NULL != qcd->info) {
+        PMIX_INFO_FREE(qcd->info, qcd->ninfo);
+    }
+    PMIX_RELEASE(qcd);
+    PMIX_RELEASE(cd);
+}
+
+static void validate_cbfunc(pmix_status_t status,
+                            pmix_info_t info[], size_t ninfo,
+                            void *cbdata)
+{
+    pmix_query_caddy_t *qcd = (pmix_query_caddy_t*)cbdata;
+    pmix_server_caddy_t *cd = (pmix_server_caddy_t*)qcd->cbdata;
+    pmix_buffer_t *reply;
+    pmix_status_t rc;
+
+    pmix_output_verbose(2, pmix_globals.debug_output,
+                        "pmix:validate credential callback with status %d", status);
+
+    reply = PMIX_NEW(pmix_buffer_t);
+    if (NULL == reply) {
+        PMIX_ERROR_LOG(PMIX_ERR_NOMEM);
+        PMIX_RELEASE(cd);
+        return;
+    }
+    PMIX_BFROPS_PACK(rc, cd->peer, reply, &status, 1, PMIX_STATUS);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        goto complete;
+    }
+    /* pack any returned data */
+    PMIX_BFROPS_PACK(rc, cd->peer, reply, &ninfo, 1, PMIX_SIZE);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        goto complete;
+    }
+    if (0 < ninfo) {
+        PMIX_BFROPS_PACK(rc, cd->peer, reply, info, ninfo, PMIX_INFO);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+        }
+    }
+
+  complete:
+    // send reply
+    PMIX_SERVER_QUEUE_REPLY(cd->peer, cd->hdr.tag, reply);
+    // cleanup
+    if (NULL != qcd->info) {
+        PMIX_INFO_FREE(qcd->info, qcd->ninfo);
+    }
+    PMIX_RELEASE(qcd);
+    PMIX_RELEASE(cd);
+}
+
+
+static void _iofreg(int sd, short args, void *cbdata)
+{
+    pmix_setup_caddy_t *cd = (pmix_setup_caddy_t*)cbdata;
+    pmix_server_caddy_t *scd = (pmix_server_caddy_t*)cd->cbdata;
+    pmix_buffer_t *reply;
+    pmix_status_t rc;
+
+    PMIX_ACQUIRE_OBJECT(cd);
+
+    /* setup the reply to the requestor */
+    reply = PMIX_NEW(pmix_buffer_t);
+    if (NULL == reply) {
+        PMIX_ERROR_LOG(PMIX_ERR_NOMEM);
+        rc = PMIX_ERR_NOMEM;
+        goto cleanup;
+    }
+    /* start with the status */
+    PMIX_BFROPS_PACK(rc, scd->peer, reply, &cd->status, 1, PMIX_STATUS);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_RELEASE(reply);
+        goto cleanup;
+    }
+
+    /* was the request a success? */
+    if (PMIX_SUCCESS != cd->status) {
+        /* find and remove the tracker(s) */
+    }
+
+    pmix_output_verbose(2, pmix_server_globals.iof_output,
+                        "server:_iofreg reply being sent to %s:%u",
+                        scd->peer->info->pname.nspace, scd->peer->info->pname.rank);
+    PMIX_SERVER_QUEUE_REPLY(scd->peer, scd->hdr.tag, reply);
+
+  cleanup:
+    /* release the cached info */
+    if (NULL != cd->procs) {
+        PMIX_PROC_FREE(cd->procs, cd->nprocs);
+    }
+    PMIX_INFO_FREE(cd->info, cd->ninfo);
+    /* we are done */
+    PMIX_RELEASE(cd);
+}
+
+static void iof_cbfunc(pmix_status_t status,
+                       void *cbdata)
+{
+    pmix_setup_caddy_t *cd = (pmix_setup_caddy_t*)cbdata;
+
+    pmix_output_verbose(2, pmix_server_globals.iof_output,
+                        "server:iof_cbfunc called with status %d",
+                        status);
+
+    if (NULL == cd) {
+        /* nothing to do */
+        return;
+    }
+    cd->status = status;
+
+    /* need to thread-shift this callback as it accesses global data */
+    PMIX_THREADSHIFT(cd, _iofreg);
+}
+
 /* the switchyard is the primary message handling function. It's purpose
  * is to take incoming commands (packed into a buffer), unpack them,
  * and then call the corresponding host server's function to execute
@@ -2479,6 +2749,24 @@ static pmix_status_t server_switchyard(pmix_peer_t *peer, uint32_t tag,
         return rc;
     }
 
+    if (PMIX_GET_CREDENTIAL_CMD == cmd) {
+        PMIX_GDS_CADDY(cd, peer, tag);
+        rc = pmix_server_get_credential(peer, buf, cred_cbfunc, cd);
+        return rc;
+    }
+
+    if (PMIX_VALIDATE_CRED_CMD == cmd) {
+        PMIX_GDS_CADDY(cd, peer, tag);
+        rc = pmix_server_validate_credential(peer, buf, validate_cbfunc, cd);
+        return rc;
+    }
+
+    if (PMIX_IOF_CMD == cmd) {
+        PMIX_GDS_CADDY(cd, peer, tag);
+        rc = pmix_server_iofreg(peer, buf, iof_cbfunc, cd);
+        return rc;
+    }
+
     return PMIX_ERR_NOT_SUPPORTED;
 }
 

opal/mca/pmix/pmix3x/pmix/src/server/pmix_server_get.c

@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2014-2017 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
  * Copyright (c) 2014-2015 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2014-2015 Artem Y. Polyakov <artpol84@gmail.com>.
@@ -91,7 +91,10 @@ static pmix_status_t create_local_tracker(char nspace[], pmix_rank_t rank,
                                           pmix_info_t info[], size_t ninfo,
                                           pmix_modex_cbfunc_t cbfunc,
                                           void *cbdata,
-                                          pmix_dmdx_local_t **lcd);
+                                          pmix_dmdx_local_t **lcd,
+                                          pmix_dmdx_request_t **rq);
+
+static void get_timeout(int sd, short args, void *cbdata);
 
 
 /* declare a function whose sole purpose is to
@@ -120,8 +123,10 @@ pmix_status_t pmix_server_get(pmix_buffer_t *buf,
     pmix_info_t *info=NULL;
     size_t ninfo=0;
     pmix_dmdx_local_t *lcd;
+    pmix_dmdx_request_t *req;
     bool local;
     bool localonly = false;
+    struct timeval tv = {0, 0};
     pmix_buffer_t pbkt, pkt;
     pmix_byte_object_t bo;
     pmix_cb_t cb;
@@ -175,10 +180,12 @@ pmix_status_t pmix_server_get(pmix_buffer_t *buf,
 
     /* search for directives we can deal with here */
     for (n=0; n < ninfo; n++) {
-        if (0 == strcmp(info[n].key, PMIX_IMMEDIATE)) {
+        if (0 == strncmp(info[n].key, PMIX_IMMEDIATE, PMIX_MAX_KEYLEN)) {
             /* just check our own data - don't wait
              * or request it from someone else */
             localonly = PMIX_INFO_TRUE(&info[n]);
+        } else if (0 == strncmp(info[n].key, PMIX_TIMEOUT, PMIX_MAX_KEYLEN)) {
+            tv.tv_sec = info[n].value.data.uint32;
         }
     }
 
@@ -234,7 +241,7 @@ pmix_status_t pmix_server_get(pmix_buffer_t *buf,
          * back when we receive it */
         rc = create_local_tracker(nspace, rank,
                                   info, ninfo,
-                                  cbfunc, cbdata, &lcd);
+                                  cbfunc, cbdata, &lcd, &req);
         if (PMIX_ERR_NOMEM == rc) {
             PMIX_INFO_FREE(info, ninfo);
             return rc;
@@ -256,6 +263,13 @@ pmix_status_t pmix_server_get(pmix_buffer_t *buf,
             pmix_host_server.direct_modex(&lcd->proc, info, ninfo, dmdx_cbfunc, lcd);
         }
 
+        /* if they specified a timeout, set it up now */
+        if (0 < tv.tv_sec) {
+            pmix_event_evtimer_set(pmix_globals.evbase, &req->ev,
+                                   get_timeout, req);
+            pmix_event_evtimer_add(&req->ev, &tv);
+            req->event_active = true;
+        }
         return PMIX_SUCCESS;
     }
 
@@ -336,7 +350,7 @@ pmix_status_t pmix_server_get(pmix_buffer_t *buf,
         /* we cannot do anything further, so just track this request
          * for now */
         rc = create_local_tracker(nspace, rank, info, ninfo,
-                                  cbfunc, cbdata, &lcd);
+                                  cbfunc, cbdata, &lcd, &req);
         if (PMIX_ERR_NOMEM == rc) {
             PMIX_INFO_FREE(info, ninfo);
         }
@@ -344,6 +358,13 @@ pmix_status_t pmix_server_get(pmix_buffer_t *buf,
                             "%s:%d TRACKER CREATED - WAITING",
                             pmix_globals.myid.nspace,
                             pmix_globals.myid.rank);
+        /* if they specified a timeout, set it up now */
+        if (0 < tv.tv_sec) {
+            pmix_event_evtimer_set(pmix_globals.evbase, &req->ev,
+                                   get_timeout, req);
+            pmix_event_evtimer_add(&req->ev, &tv);
+            req->event_active = true;
+        }
         return rc;
     }
 
@@ -373,7 +394,7 @@ pmix_status_t pmix_server_get(pmix_buffer_t *buf,
     /* Check to see if we already have a pending request for the data - if
      * we do, then we can just wait for it to arrive */
     rc = create_local_tracker(nspace, rank, info, ninfo,
-                              cbfunc, cbdata, &lcd);
+                              cbfunc, cbdata, &lcd, &req);
     if (PMIX_SUCCESS == rc) {
        /* we are already waiting for the data - nothing more
         * for us to do as the function added the new request
@@ -393,6 +414,13 @@ pmix_status_t pmix_server_get(pmix_buffer_t *buf,
      * if this is one, then we have nothing further to do - we will
      * fulfill the request once the process commits its data */
     if (local) {
+        /* if they specified a timeout, set it up now */
+        if (0 < tv.tv_sec) {
+            pmix_event_evtimer_set(pmix_globals.evbase, &req->ev,
+                                   get_timeout, req);
+            pmix_event_evtimer_add(&req->ev, &tv);
+            req->event_active = true;
+        }
         return PMIX_SUCCESS;
     }
 
@@ -401,6 +429,13 @@ pmix_status_t pmix_server_get(pmix_buffer_t *buf,
      * whomever is hosting the target process */
     if (NULL != pmix_host_server.direct_modex) {
         rc = pmix_host_server.direct_modex(&lcd->proc, info, ninfo, dmdx_cbfunc, lcd);
+        /* if they specified a timeout, set it up now */
+        if (0 < tv.tv_sec) {
+            pmix_event_evtimer_set(pmix_globals.evbase, &req->ev,
+                                   get_timeout, req);
+            pmix_event_evtimer_add(&req->ev, &tv);
+            req->event_active = true;
+        }
     } else {
         pmix_output_verbose(2, pmix_server_globals.get_output,
                             "%s:%d NO SERVER SUPPORT",
@@ -421,7 +456,8 @@ static pmix_status_t create_local_tracker(char nspace[], pmix_rank_t rank,
                                           pmix_info_t info[], size_t ninfo,
                                           pmix_modex_cbfunc_t cbfunc,
                                           void *cbdata,
-                                          pmix_dmdx_local_t **ld)
+                                          pmix_dmdx_local_t **ld,
+                                          pmix_dmdx_request_t **rq)
 {
     pmix_dmdx_local_t *lcd, *cd;
     pmix_dmdx_request_t *req;
@@ -429,6 +465,7 @@ static pmix_status_t create_local_tracker(char nspace[], pmix_rank_t rank,
 
     /* define default */
     *ld = NULL;
+    *rq = NULL;
 
     /* see if we already have an existing request for data
      * from this namespace/rank */
@@ -464,10 +501,17 @@ static pmix_status_t create_local_tracker(char nspace[], pmix_rank_t rank,
     /* track this specific requestor so we return the
      * data to them */
     req = PMIX_NEW(pmix_dmdx_request_t);
+    if (NULL == req) {
+        *ld = lcd;
+        return PMIX_ERR_NOMEM;
+    }
+    PMIX_RETAIN(lcd);
+    req->lcd = lcd;
     req->cbfunc = cbfunc;
     req->cbdata = cbdata;
     pmix_list_append(&lcd->loc_reqs, &req->super);
     *ld = lcd;
+    *rq = req;
     return rc;
 }
 
@@ -550,35 +594,22 @@ static pmix_status_t _satisfy_request(pmix_nspace_t *nptr, pmix_rank_t rank,
             *local = true;
         }
         if (PMIX_RANK_WILDCARD != rank) {
+            peer = NULL;
             /* see if the requested rank is local */
             PMIX_LIST_FOREACH(iptr, &nptr->ranks, pmix_rank_info_t) {
                 if (rank == iptr->pname.rank) {
                     scope = PMIX_LOCAL;
+                    if (0 <= iptr->peerid) {
+                        peer = (pmix_peer_t*)pmix_pointer_array_get_item(&pmix_server_globals.clients, iptr->peerid);
+                    }
+                    if (NULL == peer) {
+                        /* this rank has not connected yet, so this request needs to be held */
+                        return PMIX_ERR_NOT_FOUND;
+                    }
                     break;
                 }
             }
-            if (PMIX_LOCAL == scope) {
+            if (PMIX_LOCAL != scope)  {
-                /* must have found a local rank
-                 * we need the personality module for a client from this
-                 * nspace, but it doesn't matter which one as they all
-                 * must use the same GDS module. We don't know the GDS
-                 * module, however, until _after_ the first local client
-                 * connects to us. Since the nspace of the requestor may
-                 * not match the nspace of the proc whose info is being
-                 * requested, we cannot be sure this will have occurred.
-                 * So we have to loop again to see if someone has connected */
-                peer = NULL;
-                PMIX_LIST_FOREACH(iptr, &nptr->ranks, pmix_rank_info_t) {
-                    if (0 <= iptr->peerid) {
-                        peer = (pmix_peer_t*)pmix_pointer_array_get_item(&pmix_server_globals.clients, iptr->peerid);
-                        break;
-                    }
-                }
-                if (NULL == peer) {
-                    /* nobody has connected yet, so this request needs to be held */
-                    return PMIX_ERR_NOT_FOUND;
-                }
-            } else {
                 /* this must be a remote rank */
                 if (local) {
                     *local = false;
@@ -658,6 +689,9 @@ static pmix_status_t _satisfy_request(pmix_nspace_t *nptr, pmix_rank_t rank,
 
     /* retrieve the data for the specific rank they are asking about */
     if (PMIX_RANK_WILDCARD != rank) {
+        if (!peer->commit_cnt) {
+            return PMIX_ERR_NOT_FOUND;
+        }
         proc.rank = rank;
         PMIX_CONSTRUCT(&cb, pmix_cb_t);
         /* this is a local request, so give the gds the option
@@ -942,3 +976,18 @@ static void dmdx_cbfunc(pmix_status_t status,
                         caddy->lcd->proc.nspace, caddy->lcd->proc.rank);
     PMIX_THREADSHIFT(caddy, _process_dmdx_reply);
 }
+
+static void get_timeout(int sd, short args, void *cbdata)
+{
+    pmix_dmdx_request_t *req = (pmix_dmdx_request_t*)cbdata;
+
+    pmix_output_verbose(2, pmix_server_globals.get_output,
+                        "ALERT: get timeout fired");
+    /* execute the provided callback function with the error */
+    if (NULL != req->cbfunc) {
+        req->cbfunc(PMIX_ERR_TIMEOUT, NULL, 0, req->cbdata, NULL, NULL);
+    }
+    req->event_active = false;
+    pmix_list_remove_item(&req->lcd->loc_reqs, &req->super);
+    PMIX_RELEASE(req);
+}

opal/mca/pmix/pmix3x/pmix/src/server/pmix_server_ops.c

@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2014-2017 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
  * Copyright (c) 2014-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2014-2015 Artem Y. Polyakov <artpol84@gmail.com>.
@@ -226,6 +226,9 @@ pmix_status_t pmix_server_commit(pmix_peer_t *peer, pmix_buffer_t *buf)
     /* mark us as having successfully received a blob from this proc */
     info->modex_recvd = true;
 
+    /* update the commit counter */
+    peer->commit_cnt++;
+
     /* see if anyone remote is waiting on this data - could be more than one */
     PMIX_LIST_FOREACH_SAFE(dcd, dcdnext, &pmix_server_globals.remote_pnd, pmix_dmdx_remote_t) {
         if (0 != strncmp(dcd->cd->proc.nspace, nptr->nspace, PMIX_MAX_NSLEN)) {
@@ -449,6 +452,24 @@ static pmix_server_trkr_t* new_tracker(pmix_proc_t *procs,
     return trk;
 }
 
+static void fence_timeout(int sd, short args, void *cbdata)
+{
+    pmix_server_caddy_t *cd = (pmix_server_caddy_t*)cbdata;
+
+    pmix_output_verbose(2, pmix_server_globals.fence_output,
+                        "ALERT: fence timeout fired");
+
+    /* execute the provided callback function with the error */
+    if (NULL != cd->trk->modexcbfunc) {
+        cd->trk->modexcbfunc(PMIX_ERR_TIMEOUT, NULL, 0, cd->trk, NULL, NULL);
+        return;  // the cbfunc will have cleaned up the tracker
+    }
+    cd->event_active = false;
+    /* remove it from the list */
+    pmix_list_remove_item(&cd->trk->local_cbs, &cd->super);
+    PMIX_RELEASE(cd);
+}
+
 pmix_status_t pmix_server_fence(pmix_server_caddy_t *cd,
                                 pmix_buffer_t *buf,
                                 pmix_modex_cbfunc_t modexcbfunc,
@@ -469,6 +490,7 @@ pmix_status_t pmix_server_fence(pmix_server_caddy_t *cd,
     pmix_byte_object_t bo;
     pmix_info_t *info = NULL;
     size_t ninfo=0, n;
+    struct timeval tv = {0, 0};
 
     pmix_output_verbose(2, pmix_server_globals.fence_output,
                         "recvd FENCE");
@@ -523,12 +545,13 @@ pmix_status_t pmix_server_fence(pmix_server_caddy_t *cd,
         if (PMIX_SUCCESS != rc) {
             goto cleanup;
         }
-        /* see if we are to collect data - we don't internally care
+        /* see if we are to collect data or enforce a timeout - we don't internally care
          * about any other directives */
         for (n=0; n < ninfo; n++) {
             if (0 == strcmp(info[n].key, PMIX_COLLECT_DATA)) {
                 collect_data = true;
-                break;
+            } else if (0 == strncmp(info[n].key, PMIX_TIMEOUT, PMIX_MAX_KEYLEN)) {
+                tv.tv_sec = info[n].value.data.uint32;
             }
         }
     }
@@ -585,6 +608,15 @@ pmix_status_t pmix_server_fence(pmix_server_caddy_t *cd,
     /* add this contributor to the tracker so they get
      * notified when we are done */
     pmix_list_append(&trk->local_cbs, &cd->super);
+    /* if a timeout was specified, set it */
+    if (0 < tv.tv_sec) {
+        PMIX_RETAIN(trk);
+        cd->trk = trk;
+        pmix_event_evtimer_set(pmix_globals.evbase, &cd->ev,
+                               fence_timeout, cd);
+        pmix_event_evtimer_add(&cd->ev, &tv);
+        cd->event_active = true;
+    }
 
     /* if all local contributions have been received,
      * let the local host's server know that we are at the
@@ -993,7 +1025,22 @@ static void spcbfunc(pmix_status_t status,
                      char nspace[], void *cbdata)
 {
     pmix_setup_caddy_t *cd = (pmix_setup_caddy_t*)cbdata;
+    pmix_iof_req_t *req;
 
+    /* if it was successful, and there are IOF requests, then
+     * register them now */
+    if (PMIX_SUCCESS == status && PMIX_FWD_NO_CHANNELS != cd->channels) {
+         /* record the request */
+        req = PMIX_NEW(pmix_iof_req_t);
+        if (NULL != req) {
+            PMIX_RETAIN(cd->peer);
+            req->peer = cd->peer;
+            req->pname.nspace = strdup(nspace);
+            req->pname.rank = PMIX_RANK_WILDCARD;
+            req->channels = cd->channels;
+            pmix_list_append(&pmix_globals.iof_requests, &req->super);
+        }
+    }
     /* cleanup the caddy */
     if (NULL != cd->info) {
         PMIX_INFO_FREE(cd->info, cd->ninfo);
@@ -1016,7 +1063,8 @@ pmix_status_t pmix_server_spawn(pmix_peer_t *peer,
     int32_t cnt;
     pmix_status_t rc;
     pmix_proc_t proc;
-    size_t ninfo;
+    size_t ninfo, n;
+    bool stdout_found = false, stderr_found = false, stddiag_found = false;
 
     pmix_output_verbose(2, pmix_server_globals.spawn_output,
                         "recvd SPAWN");
@@ -1031,6 +1079,8 @@ pmix_status_t pmix_server_spawn(pmix_peer_t *peer,
     if (NULL == cd) {
         return PMIX_ERR_NOMEM;
     }
+    PMIX_RETAIN(peer);
+    cd->peer = peer;
     cd->spcbfunc = cbfunc;
     cd->cbdata = cbdata;
 
@@ -1059,10 +1109,48 @@ pmix_status_t pmix_server_spawn(pmix_peer_t *peer,
             PMIX_ERROR_LOG(rc);
             goto cleanup;
         }
+        /* run a quick check of the directives to see if any IOF
+         * requests were included so we can set that up now - helps
+         * to catch any early output */
+        cd->channels = PMIX_FWD_NO_CHANNELS;
+        for (n=0; n < cd->ninfo; n++) {
+            if (0 == strncmp(cd->info[n].key, PMIX_FWD_STDIN, PMIX_MAX_KEYLEN)) {
+                stdout_found = true;
+                if (PMIX_INFO_TRUE(&cd->info[n])) {
+                    cd->channels |= PMIX_FWD_STDIN_CHANNEL;
+                }
+            } else if (0 == strncmp(cd->info[n].key, PMIX_FWD_STDOUT, PMIX_MAX_KEYLEN)) {
+                if (PMIX_INFO_TRUE(&cd->info[n])) {
+                    cd->channels |= PMIX_FWD_STDOUT_CHANNEL;
+                }
+            } else if (0 == strncmp(cd->info[n].key, PMIX_FWD_STDERR, PMIX_MAX_KEYLEN)) {
+                stderr_found = true;
+                if (PMIX_INFO_TRUE(&cd->info[n])) {
+                    cd->channels |= PMIX_FWD_STDERR_CHANNEL;
+                }
+            } else if (0 == strncmp(cd->info[n].key, PMIX_FWD_STDDIAG, PMIX_MAX_KEYLEN)) {
+                stddiag_found = true;
+                if (PMIX_INFO_TRUE(&cd->info[n])) {
+                    cd->channels |= PMIX_FWD_STDDIAG_CHANNEL;
+                }
+            }
+        }
+        /* we will construct any required iof request tracker upon completion of the spawn */
     }
     /* add the directive to the end */
     if (PMIX_PROC_IS_TOOL(peer)) {
         PMIX_INFO_LOAD(&cd->info[ninfo], PMIX_REQUESTOR_IS_TOOL, NULL, PMIX_BOOL);
+        /* if the requestor is a tool, we default to forwarding all
+         * output IO channels */
+        if (!stdout_found) {
+            cd->channels |= PMIX_FWD_STDOUT_CHANNEL;
+        }
+        if (!stderr_found) {
+            cd->channels |= PMIX_FWD_STDERR_CHANNEL;
+        }
+        if (!stddiag_found) {
+            cd->channels |= PMIX_FWD_STDDIAG_CHANNEL;
+        }
     } else {
         PMIX_INFO_LOAD(&cd->info[ninfo], PMIX_REQUESTOR_IS_CLIENT, NULL, PMIX_BOOL);
     }
@@ -1218,6 +1306,24 @@ pmix_status_t pmix_server_disconnect(pmix_server_caddy_t *cd,
     return rc;
 }
 
+static void connect_timeout(int sd, short args, void *cbdata)
+{
+    pmix_server_caddy_t *cd = (pmix_server_caddy_t*)cbdata;
+
+    pmix_output_verbose(2, pmix_server_globals.connect_output,
+                        "ALERT: connect timeout fired");
+
+    /* execute the provided callback function with the error */
+    if (NULL != cd->trk->cnct_cbfunc) {
+        cd->trk->cnct_cbfunc(PMIX_ERR_TIMEOUT, NULL, PMIX_RANK_UNDEF, cd->trk);
+        return;  // the cbfunc will have cleaned up the tracker
+    }
+    cd->event_active = false;
+    /* remove it from the list */
+    pmix_list_remove_item(&cd->trk->local_cbs, &cd->super);
+    PMIX_RELEASE(cd);
+}
+
 pmix_status_t pmix_server_connect(pmix_server_caddy_t *cd,
                                   pmix_buffer_t *buf,
                                   pmix_connect_cbfunc_t cbfunc)
@@ -1226,8 +1332,9 @@ pmix_status_t pmix_server_connect(pmix_server_caddy_t *cd,
     pmix_status_t rc;
     pmix_proc_t *procs = NULL;
     pmix_info_t *info = NULL;
-    size_t nprocs, ninfo;
+    size_t nprocs, ninfo, n;
     pmix_server_trkr_t *trk;
+    struct timeval tv = {0, 0};
 
     pmix_output_verbose(2, pmix_server_globals.connect_output,
                         "recvd CONNECT from peer %s:%d",
@@ -1287,6 +1394,13 @@ pmix_status_t pmix_server_connect(pmix_server_caddy_t *cd,
         if (PMIX_SUCCESS != rc) {
             goto cleanup;
         }
+        /* check for a timeout */
+        for (n=0; n < ninfo; n++) {
+            if (0 == strncmp(info[n].key, PMIX_TIMEOUT, PMIX_MAX_KEYLEN)) {
+                tv.tv_sec = info[n].value.data.uint32;
+                break;
+            }
+        }
     }
 
     /* find/create the local tracker for this operation */
@@ -1318,6 +1432,16 @@ pmix_status_t pmix_server_connect(pmix_server_caddy_t *cd,
      * notified when we are done */
     PMIX_RETAIN(cd);  // prevent the caddy from being released when we return
     pmix_list_append(&trk->local_cbs, &cd->super);
+    /* if a timeout was specified, set it */
+    if (0 < tv.tv_sec) {
+        PMIX_RETAIN(trk);
+        cd->trk = trk;
+        pmix_event_evtimer_set(pmix_globals.evbase, &cd->ev,
+                               connect_timeout, cd);
+        pmix_event_evtimer_add(&cd->ev, &tv);
+        cd->event_active = true;
+    }
+
     /* if all local contributions have been received,
      * let the local host's server know that we are at the
      * "fence" point - they will callback once the [dis]connect
@@ -2016,12 +2140,11 @@ pmix_status_t pmix_server_job_ctrl(pmix_peer_t *peer,
     pmix_query_caddy_t *cd;
     pmix_proc_t proc;
     size_t n;
-    bool recurse, leave_topdir, duplicate;
+    bool recurse = false, leave_topdir = false, duplicate;
     pmix_list_t cachedirs, cachefiles;
-    pmix_epilog_t *epi;
+    pmix_epilog_t *epi = NULL;
     pmix_cleanup_file_t *cf, *cf2;
     pmix_cleanup_dir_t *cdir, *cdir2;
-    struct stat statbuf;
 
     pmix_output_verbose(2, pmix_server_globals.base_output,
                         "recvd job control request from client");
@@ -2064,8 +2187,6 @@ pmix_status_t pmix_server_job_ctrl(pmix_peer_t *peer,
                 epi = &peer->epilog;
             }
         }
-    } else {
-        epi = NULL;  // do not allow epilog requests
     }
 
     /* unpack the number of info objects */
@@ -2250,7 +2371,7 @@ pmix_status_t pmix_server_job_ctrl(pmix_peer_t *peer,
             }
         }
         PMIX_DESTRUCT(&cachefiles);
-        if (cnt == cd->ninfo) {
+        if (cnt == (int)cd->ninfo) {
             /* nothing more to do */
             if (NULL != cbfunc) {
                 cbfunc(PMIX_SUCCESS, NULL, 0, cd, NULL, NULL);
@@ -2354,6 +2475,253 @@ pmix_status_t pmix_server_monitor(pmix_peer_t *peer,
     return rc;
 }
 
+pmix_status_t pmix_server_get_credential(pmix_peer_t *peer,
+                                         pmix_buffer_t *buf,
+                                         pmix_credential_cbfunc_t cbfunc,
+                                         void *cbdata)
+{
+    int32_t cnt;
+    pmix_status_t rc;
+    pmix_query_caddy_t *cd;
+    pmix_proc_t proc;
+
+    pmix_output_verbose(2, pmix_globals.debug_output,
+                        "recvd get credential request from client");
+
+    if (NULL == pmix_host_server.get_credential) {
+        return PMIX_ERR_NOT_SUPPORTED;
+    }
+
+    cd = PMIX_NEW(pmix_query_caddy_t);
+    if (NULL == cd) {
+        return PMIX_ERR_NOMEM;
+    }
+    cd->cbdata = cbdata;
+
+    /* unpack the number of directives */
+    cnt = 1;
+    PMIX_BFROPS_UNPACK(rc, peer, buf, &cd->ninfo, &cnt, PMIX_SIZE);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        goto exit;
+    }
+    /* unpack the directives */
+    if (0 < cd->ninfo) {
+        PMIX_INFO_CREATE(cd->info, cd->ninfo);
+        cnt = cd->ninfo;
+        PMIX_BFROPS_UNPACK(rc, peer, buf, cd->info, &cnt, PMIX_INFO);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            goto exit;
+        }
+    }
+
+    /* setup the requesting peer name */
+    (void)strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN);
+    proc.rank = peer->info->pname.rank;
+
+    /* ask the host to execute the request */
+    if (PMIX_SUCCESS != (rc = pmix_host_server.get_credential(&proc, cd->info, cd->ninfo,
+                                                              cbfunc, cd))) {
+        goto exit;
+    }
+    return PMIX_SUCCESS;
+
+  exit:
+    PMIX_RELEASE(cd);
+    return rc;
+}
+
+pmix_status_t pmix_server_validate_credential(pmix_peer_t *peer,
+                                              pmix_buffer_t *buf,
+                                              pmix_validation_cbfunc_t cbfunc,
+                                              void *cbdata)
+{
+    int32_t cnt;
+    pmix_status_t rc;
+    pmix_query_caddy_t *cd;
+    pmix_proc_t proc;
+
+    pmix_output_verbose(2, pmix_globals.debug_output,
+                        "recvd validate credential request from client");
+
+    if (NULL == pmix_host_server.validate_credential) {
+        return PMIX_ERR_NOT_SUPPORTED;
+    }
+
+    cd = PMIX_NEW(pmix_query_caddy_t);
+    if (NULL == cd) {
+        return PMIX_ERR_NOMEM;
+    }
+    cd->cbdata = cbdata;
+
+    /* unpack the credential */
+    cnt = 1;
+    PMIX_BFROPS_UNPACK(rc, peer, buf, &cd->bo, &cnt, PMIX_BYTE_OBJECT);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        goto exit;
+    }
+
+    /* unpack the number of directives */
+    cnt = 1;
+    PMIX_BFROPS_UNPACK(rc, peer, buf, &cd->ninfo, &cnt, PMIX_SIZE);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        goto exit;
+    }
+    /* unpack the directives */
+    if (0 < cd->ninfo) {
+        PMIX_INFO_CREATE(cd->info, cd->ninfo);
+        cnt = cd->ninfo;
+        PMIX_BFROPS_UNPACK(rc, peer, buf, cd->info, &cnt, PMIX_INFO);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            goto exit;
+        }
+    }
+
+    /* setup the requesting peer name */
+    (void)strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN);
+    proc.rank = peer->info->pname.rank;
+
+    /* ask the host to execute the request */
+    if (PMIX_SUCCESS != (rc = pmix_host_server.validate_credential(&proc, &cd->bo,
+                                                                   cd->info, cd->ninfo,
+                                                                   cbfunc, cd))) {
+        goto exit;
+    }
+    return PMIX_SUCCESS;
+
+  exit:
+    PMIX_RELEASE(cd);
+    return rc;
+}
+
+pmix_status_t pmix_server_iofreg(pmix_peer_t *peer,
+                                 pmix_buffer_t *buf,
+                                 pmix_op_cbfunc_t cbfunc,
+                                 void *cbdata)
+{
+    int32_t cnt;
+    pmix_status_t rc;
+    pmix_setup_caddy_t *cd;
+    pmix_iof_req_t *req;
+    bool notify, match;
+    size_t n;
+
+    pmix_output_verbose(2, pmix_server_globals.iof_output,
+                        "recvd register IOF request from client");
+
+    if (NULL == pmix_host_server.register_iof) {
+        return PMIX_ERR_NOT_SUPPORTED;
+    }
+
+    cd = PMIX_NEW(pmix_setup_caddy_t);
+    if (NULL == cd) {
+        return PMIX_ERR_NOMEM;
+    }
+    cd->cbdata = cbdata;  // this is the pmix_server_caddy_t
+
+    /* unpack the number of procs */
+    cnt = 1;
+    PMIX_BFROPS_UNPACK(rc, peer, buf, &cd->nprocs, &cnt, PMIX_SIZE);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        goto exit;
+    }
+    /* unpack the procs */
+    if (0 < cd->nprocs) {
+        PMIX_PROC_CREATE(cd->procs, cd->nprocs);
+        cnt = cd->nprocs;
+        PMIX_BFROPS_UNPACK(rc, peer, buf, cd->procs, &cnt, PMIX_PROC);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            goto exit;
+        }
+    }
+
+    /* unpack the number of directives */
+    cnt = 1;
+    PMIX_BFROPS_UNPACK(rc, peer, buf, &cd->ninfo, &cnt, PMIX_SIZE);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        goto exit;
+    }
+    /* unpack the directives */
+    if (0 < cd->ninfo) {
+        PMIX_INFO_CREATE(cd->info, cd->ninfo);
+        cnt = cd->ninfo;
+        PMIX_BFROPS_UNPACK(rc, peer, buf, cd->info, &cnt, PMIX_INFO);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            goto exit;
+        }
+    }
+
+    /* unpack the channels */
+    cnt = 1;
+    PMIX_BFROPS_UNPACK(rc, peer, buf, &cd->channels, &cnt, PMIX_IOF_CHANNEL);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        goto exit;
+    }
+
+    /* check to see if we have already registered this source/channel combination */
+    notify = false;
+    for (n=0; n < cd->nprocs; n++) {
+        match = false;
+        PMIX_LIST_FOREACH(req, &pmix_globals.iof_requests, pmix_iof_req_t) {
+            /* is this request from the same peer? */
+            if (peer != req->peer) {
+                continue;
+            }
+            /* do we already have this source for this peer? */
+            if (0 == strncmp(cd->procs[n].nspace, req->pname.nspace, PMIX_MAX_NSLEN) &&
+                (PMIX_RANK_WILDCARD == req->pname.rank || cd->procs[n].rank == req->pname.rank)) {
+                match = true;
+                if ((req->channels & cd->channels) != cd->channels) {
+                    /* this is a channel update */
+                    req->channels |= cd->channels;
+                    /* we need to notify the host */
+                    notify = true;
+                }
+                break;
+            }
+        }
+        /* if we didn't find the matching entry, then add it */
+        if (!match) {
+            /* record the request */
+            req = PMIX_NEW(pmix_iof_req_t);
+            if (NULL == req) {
+                rc = PMIX_ERR_NOMEM;
+                goto exit;
+            }
+            PMIX_RETAIN(peer);
+            req->peer = peer;
+            req->pname.nspace = strdup(cd->procs[n].nspace);
+            req->pname.rank = cd->procs[n].rank;
+            req->channels = cd->channels;
+            pmix_list_append(&pmix_globals.iof_requests, &req->super);
+        }
+    }
+
+    if (notify) {
+        /* ask the host to execute the request */
+        if (PMIX_SUCCESS != (rc = pmix_host_server.register_iof(cd->procs, cd->nprocs,
+                                                                cd->info, cd->ninfo,
+                                                                cd->channels,
+                                                                cbfunc, cd))) {
+            goto exit;
+        }
+    }
+    return PMIX_SUCCESS;
+
+  exit:
+    PMIX_RELEASE(cd);
+    return rc;
+}
+
 /*****    INSTANCE SERVER LIBRARY CLASSES    *****/
 static void tcon(pmix_server_trkr_t *t)
 {
@@ -2391,10 +2759,19 @@ PMIX_CLASS_INSTANCE(pmix_server_trkr_t,
 
 static void cdcon(pmix_server_caddy_t *cd)
 {
+    memset(&cd->ev, 0, sizeof(pmix_event_t));
+    cd->event_active = false;
+    cd->trk = NULL;
     cd->peer = NULL;
 }
 static void cddes(pmix_server_caddy_t *cd)
 {
+    if (cd->event_active) {
+        pmix_event_del(&cd->ev);
+    }
+    if (NULL != cd->trk) {
+        PMIX_RELEASE(cd->trk);
+    }
     if (NULL != cd->peer) {
         PMIX_RELEASE(cd->peer);
     }
@@ -2406,6 +2783,7 @@ PMIX_CLASS_INSTANCE(pmix_server_caddy_t,
 
 static void scadcon(pmix_setup_caddy_t *p)
 {
+    p->peer = NULL;
     memset(&p->proc, 0, sizeof(pmix_proc_t));
     PMIX_CONSTRUCT_LOCK(&p->lock);
     p->nspace = NULL;
@@ -2418,6 +2796,8 @@ static void scadcon(pmix_setup_caddy_t *p)
     p->info = NULL;
     p->ninfo = 0;
     p->keys = NULL;
+    p->channels = PMIX_FWD_NO_CHANNELS;
+    p->bo = NULL;
     p->cbfunc = NULL;
     p->opcbfunc = NULL;
     p->setupcbfunc = NULL;
@@ -2427,6 +2807,9 @@ static void scadcon(pmix_setup_caddy_t *p)
 }
 static void scaddes(pmix_setup_caddy_t *p)
 {
+    if (NULL != p->peer) {
+        PMIX_RELEASE(p->peer);
+    }
     PMIX_DESTRUCT_LOCK(&p->lock);
 }
 PMIX_EXPORT PMIX_CLASS_INSTANCE(pmix_setup_caddy_t,
@@ -2478,9 +2861,24 @@ PMIX_CLASS_INSTANCE(pmix_dmdx_remote_t,
                     pmix_list_item_t,
                     dmcon, dmdes);
 
+static void dmrqcon(pmix_dmdx_request_t *p)
+{
+    memset(&p->ev, 0, sizeof(pmix_event_t));
+    p->event_active = false;
+    p->lcd = NULL;
+}
+static void dmrqdes(pmix_dmdx_request_t *p)
+{
+    if (p->event_active) {
+        pmix_event_del(&p->ev);
+    }
+    if (NULL != p->lcd) {
+        PMIX_RELEASE(p->lcd);
+    }
+}
 PMIX_CLASS_INSTANCE(pmix_dmdx_request_t,
                     pmix_list_item_t,
-                    NULL, NULL);
+                    dmrqcon, dmrqdes);
 
 static void lmcon(pmix_dmdx_local_t *p)
 {

opal/mca/pmix/pmix3x/pmix/src/server/pmix_server_ops.h

@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2015-2017 Intel, Inc. All rights reserved.
+ * Copyright (c) 2015-2018 Intel, Inc. All rights reserved.
  * Copyright (c) 2015      Artem Y. Polyakov <artpol84@gmail.com>.
  *                         All rights reserved.
  * Copyright (c) 2015      Mellanox Technologies, Inc.
@@ -35,6 +35,7 @@ typedef struct {
     pmix_object_t super;
     pmix_event_t ev;
     pmix_lock_t lock;
+    pmix_peer_t *peer;
     char *nspace;
     pmix_status_t status;
     pmix_status_t *codes;
@@ -51,6 +52,8 @@ typedef struct {
     char **keys;
     pmix_app_t *apps;
     size_t napps;
+    pmix_iof_channel_t channels;
+    pmix_byte_object_t *bo;
     pmix_op_cbfunc_t opcbfunc;
     pmix_dmodex_response_fn_t cbfunc;
     pmix_setup_application_cbfunc_t setupcbfunc;
@@ -66,13 +69,6 @@ typedef struct {
 } pmix_dmdx_remote_t;
 PMIX_CLASS_DECLARATION(pmix_dmdx_remote_t);
 
-typedef struct {
-    pmix_list_item_t super;
-    pmix_modex_cbfunc_t cbfunc;     // cbfunc to be executed when data is available
-    void *cbdata;
-} pmix_dmdx_request_t;
-PMIX_CLASS_DECLARATION(pmix_dmdx_request_t);
-
 typedef struct {
     pmix_list_item_t super;
     pmix_proc_t proc;               // id of proc whose data is being requested
@@ -83,6 +79,16 @@ typedef struct {
 } pmix_dmdx_local_t;
 PMIX_CLASS_DECLARATION(pmix_dmdx_local_t);
 
+typedef struct {
+    pmix_list_item_t super;
+    pmix_event_t ev;
+    bool event_active;
+    pmix_dmdx_local_t *lcd;
+    pmix_modex_cbfunc_t cbfunc;     // cbfunc to be executed when data is available
+    void *cbdata;
+} pmix_dmdx_request_t;
+PMIX_CLASS_DECLARATION(pmix_dmdx_request_t);
+
 /* event/error registration book keeping */
 typedef struct {
     pmix_list_item_t super;
@@ -93,7 +99,7 @@ PMIX_CLASS_DECLARATION(pmix_peer_events_info_t);
 
 typedef struct {
     pmix_list_item_t super;
-    pmix_list_t peers;              // list of pmix_prevents_info_t
+    pmix_list_t peers;              // list of pmix_peer_events_info_t
     int code;
 } pmix_regevents_info_t;
 PMIX_CLASS_DECLARATION(pmix_regevents_info_t);
@@ -125,6 +131,9 @@ typedef struct {
     // verbosity for server event operations
     int event_output;
     int event_verbose;
+    // verbosity for server iof operations
+    int iof_output;
+    int iof_verbose;
     // verbosity for basic server functions
     int base_output;
     int base_verbose;
@@ -243,6 +252,21 @@ pmix_status_t pmix_server_monitor(pmix_peer_t *peer,
                                   pmix_info_cbfunc_t cbfunc,
                                   void *cbdata);
 
+pmix_status_t pmix_server_get_credential(pmix_peer_t *peer,
+                                         pmix_buffer_t *buf,
+                                         pmix_credential_cbfunc_t cbfunc,
+                                         void *cbdata);
+
+pmix_status_t pmix_server_validate_credential(pmix_peer_t *peer,
+                                              pmix_buffer_t *buf,
+                                              pmix_validation_cbfunc_t cbfunc,
+                                              void *cbdata);
+
+pmix_status_t pmix_server_iofreg(pmix_peer_t *peer,
+                                 pmix_buffer_t *buf,
+                                 pmix_op_cbfunc_t cbfunc,
+                                 void *cbdata);
+
 pmix_status_t pmix_server_event_recvd_from_client(pmix_peer_t *peer,
                                                   pmix_buffer_t *buf,
                                                   pmix_op_cbfunc_t cbfunc,

opal/mca/pmix/pmix3x/pmix/src/threads/Makefile.include

@@ -32,7 +32,7 @@ headers += \
         threads/wait_sync.h \
 	threads/thread_usage.h
 
-libpmix_la_SOURCES += \
+sources += \
         threads/mutex.c \
         threads/thread.c \
         threads/wait_sync.c

opal/mca/pmix/pmix3x/pmix/src/tool/pmix_tool.c

@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2014-2017 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
  * Copyright (c) 2014-2016 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2014      Artem Y. Polyakov <artpol84@gmail.com>.
@@ -69,7 +69,7 @@ extern pmix_client_globals_t pmix_client_globals;
 #include "src/runtime/pmix_rte.h"
 #include "src/mca/bfrops/base/base.h"
 #include "src/mca/gds/base/base.h"
-#include "src/mca/ptl/ptl.h"
+#include "src/mca/ptl/base/base.h"
 #include "src/mca/psec/psec.h"
 #include "src/include/pmix_globals.h"
 
@@ -189,6 +189,53 @@ static void pmix_tool_notify_recv(struct pmix_peer_t *peer,
 }
 
 
+static void tool_iof_handler(struct pmix_peer_t *pr,
+                             pmix_ptl_hdr_t *hdr,
+                             pmix_buffer_t *buf, void *cbdata)
+{
+    pmix_peer_t *peer = (pmix_peer_t*)pr;
+    pmix_proc_t source;
+    pmix_iof_channel_t channel;
+    pmix_byte_object_t bo;
+    int32_t cnt;
+    pmix_status_t rc;
+
+    pmix_output_verbose(2, pmix_client_globals.iof_output,
+                        "recvd IOF");
+
+    /* if the buffer is empty, they are simply closing the channel */
+    if (0 == buf->bytes_used) {
+        return;
+    }
+
+    cnt = 1;
+    PMIX_BFROPS_UNPACK(rc, peer, buf, &source, &cnt, PMIX_PROC);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        return;
+    }
+    cnt = 1;
+    PMIX_BFROPS_UNPACK(rc, peer, buf, &channel, &cnt, PMIX_IOF_CHANNEL);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        return;
+    }
+    cnt = 1;
+    PMIX_BFROPS_UNPACK(rc, peer, buf, &bo, &cnt, PMIX_BYTE_OBJECT);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        return;
+    }
+    if (NULL != bo.bytes && 0 < bo.size) {
+        if (channel & PMIX_FWD_STDOUT_CHANNEL) {
+            write(fileno(stdout), bo.bytes, bo.size);
+        } else {
+            fprintf(stderr, "%s", bo.bytes);
+        }
+    }
+    PMIX_BYTE_OBJECT_DESTRUCT(&bo);
+}
+
 PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc,
                                pmix_info_t info[], size_t ninfo)
 {
@@ -200,6 +247,8 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc,
     bool rank_given = false;
     pmix_info_t ginfo;
     size_t n;
+    pmix_ptl_posted_recv_t *rcv;
+    pmix_proc_t wildcard;
 
     PMIX_ACQUIRE_THREAD(&pmix_global_lock);
 
@@ -237,6 +286,13 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc,
         PMIX_RELEASE_THREAD(&pmix_global_lock);
         return rc;
     }
+    /* setup the IO Forwarding recv */
+    rcv = PMIX_NEW(pmix_ptl_posted_recv_t);
+    rcv->tag = PMIX_PTL_TAG_IOF;
+    rcv->cbfunc = tool_iof_handler;
+    /* add it to the end of the list of recvs */
+    pmix_list_append(&pmix_ptl_globals.posted_recvs, &rcv->super);
+
 
     PMIX_CONSTRUCT(&pmix_client_globals.pending_requests, pmix_list_t);
     pmix_client_globals.myserver = PMIX_NEW(pmix_peer_t);
@@ -323,9 +379,15 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc,
             return rc;
         }
     }
-    /* Success, so copy the nspace and rank */
+    /* Success, so copy the nspace and rank to the proc struct they gave us */
     (void)strncpy(proc->nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN);
     proc->rank = pmix_globals.myid.rank;
+    /* and into our own peer object */
+    if (NULL == pmix_globals.mypeer->nptr->nspace) {
+        pmix_globals.mypeer->nptr->nspace = strdup(proc->nspace);
+    }
+    (void)strncpy(pmix_globals.mypeer->info->pname.nspace, proc->nspace, PMIX_MAX_NSLEN);
+    pmix_globals.mypeer->info->pname.rank = proc->rank;
 
     /* increment our init reference counter */
     pmix_globals.init_cntr++;
@@ -335,6 +397,8 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc,
      * datastore with typical job-related info. No point
      * in having the server generate these as we are
      * obviously a singleton, and so the values are well-known */
+    (void)strncpy(wildcard.nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN);
+    wildcard.rank = pmix_globals.myid.rank;
 
     /* the jobid is just our nspace */
     kptr = PMIX_NEW(pmix_kval_t);
@@ -343,7 +407,7 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc,
     kptr->value->type = PMIX_STRING;
     kptr->value->data.string = strdup(pmix_globals.myid.nspace);
     PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer,
-                      &pmix_globals.myid,
+                      &wildcard,
                       PMIX_INTERNAL, kptr);
     if (PMIX_SUCCESS != rc) {
         PMIX_ERROR_LOG(rc);
@@ -375,7 +439,7 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc,
     kptr->value->type = PMIX_UINT32;
     kptr->value->data.uint32 = 0;
     PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer,
-                      &pmix_globals.myid,
+                      &wildcard,
                       PMIX_INTERNAL, kptr);
     if (PMIX_SUCCESS != rc) {
         PMIX_ERROR_LOG(rc);
@@ -391,7 +455,7 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc,
     kptr->value->type = PMIX_UINT32;
     kptr->value->data.uint32 = 1;
     PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer,
-                      &pmix_globals.myid,
+                      &wildcard,
                       PMIX_INTERNAL, kptr);
     if (PMIX_SUCCESS != rc) {
         PMIX_ERROR_LOG(rc);
@@ -407,7 +471,7 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc,
     kptr->value->type = PMIX_STRING;
     kptr->value->data.string = strdup("0");
     PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer,
-                      &pmix_globals.myid,
+                      &wildcard,
                       PMIX_INTERNAL, kptr);
     if (PMIX_SUCCESS != rc) {
         PMIX_ERROR_LOG(rc);
@@ -423,7 +487,7 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc,
     kptr->value->type = PMIX_UINT32;
     kptr->value->data.uint32 = 0;
     PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer,
-                      &pmix_globals.myid,
+                      &wildcard,
                       PMIX_INTERNAL, kptr);
     if (PMIX_SUCCESS != rc) {
         PMIX_ERROR_LOG(rc);
@@ -439,7 +503,7 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc,
     kptr->value->type = PMIX_UINT32;
     kptr->value->data.uint32 = 1;
     PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer,
-                      &pmix_globals.myid,
+                      &wildcard,
                       PMIX_INTERNAL, kptr);
     if (PMIX_SUCCESS != rc) {
         PMIX_ERROR_LOG(rc);
@@ -455,7 +519,7 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc,
     kptr->value->type = PMIX_UINT32;
     kptr->value->data.uint32 = 1;
     PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer,
-                      &pmix_globals.myid,
+                      &wildcard,
                       PMIX_INTERNAL, kptr);
     if (PMIX_SUCCESS != rc) {
         PMIX_ERROR_LOG(rc);
@@ -471,7 +535,7 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc,
     kptr->value->type = PMIX_UINT32;
     kptr->value->data.uint32 = 1;
     PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer,
-                      &pmix_globals.myid,
+                      &wildcard,
                       PMIX_INTERNAL, kptr);
     if (PMIX_SUCCESS != rc) {
         PMIX_ERROR_LOG(rc);
@@ -488,7 +552,7 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc,
     kptr->value->type = PMIX_UINT32;
     kptr->value->data.uint32 = 1;
     PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer,
-                      &pmix_globals.myid,
+                      &wildcard,
                       PMIX_INTERNAL, kptr);
     if (PMIX_SUCCESS != rc) {
         PMIX_ERROR_LOG(rc);
@@ -611,7 +675,7 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc,
      kptr->value->type = PMIX_STRING;
      kptr->value->data.string = strdup(hostname);
      PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer,
-                       &pmix_globals.myid,
+                       &wildcard,
                        PMIX_INTERNAL, kptr);
      if (PMIX_SUCCESS != rc) {
         PMIX_ERROR_LOG(rc);
@@ -628,7 +692,7 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc,
     kptr->value->type = PMIX_STRING;
     kptr->value->data.string = strdup("0");
     PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer,
-                      &pmix_globals.myid,
+                      &wildcard,
                       PMIX_INTERNAL, kptr);
     if (PMIX_SUCCESS != rc) {
         PMIX_ERROR_LOG(rc);

opal/mca/pmix/pmix3x/pmix/test/Makefile.am

@@ -51,13 +51,13 @@ pmi_client_SOURCES = $(headers) \
         pmi_client.c
 pmi_client_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS)
 pmi_client_LDADD = \
-    $(top_builddir)/src/libpmix.la
+    $(top_builddir)/src/libpmi.la
 
 pmi2_client_SOURCES = $(headers) \
         pmi2_client.c
 pmi2_client_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS)
 pmi2_client_LDADD = \
-    $(top_builddir)/src/libpmix.la
+    $(top_builddir)/src/libpmi2.la
 endif
 
 pmix_client_SOURCES = $(headers) \

opal/mca/pmix/pmix3x/pmix/test/pmi2_client.c

@@ -27,17 +27,17 @@ static int _verbose = 1;
 static void log_fatal(const char *format, ...)
 {
     va_list arglist;
-    char **output = NULL;
+    char *output = NULL;
 
     va_start(arglist, format);
     if (_verbose > 0) {
-        if (0 > vasprintf(output, format, arglist) ||
+        if (0 > vasprintf(&output, format, arglist) ||
-            NULL == output || NULL == *output) {
+            NULL == output) {
             va_end(arglist);
             return;
         }
-        fprintf(stderr, "FATAL: %s", *output);
+        fprintf(stderr, "FATAL: %s", output);
-        free(*output);
+        free(output);
     }
     va_end(arglist);
 }
@@ -45,17 +45,17 @@ static void log_fatal(const char *format, ...)
 static void log_error(const char *format, ...)
 {
     va_list arglist;
-    char **output = NULL;
+    char *output = NULL;
 
     va_start(arglist, format);
     if (_verbose > 0) {
-        if (0 > vasprintf(output, format, arglist) ||
+        if (0 > vasprintf(&output, format, arglist) ||
-            NULL == output || NULL == *output) {
+            NULL == output) {
             va_end(arglist);
             return;
         }
-        fprintf(stderr, "ERROR: %s", *output);
+        fprintf(stderr, "ERROR: %s", output);
-        free(*output);
+        free(output);
     }
     va_end(arglist);
 }
@@ -63,17 +63,17 @@ static void log_error(const char *format, ...)
 static void log_info(const char *format, ...)
 {
     va_list arglist;
-    char **output = NULL;
+    char *output = NULL;
 
     va_start(arglist, format);
     if (_verbose > 0) {
-        if (0 > vasprintf(output, format, arglist) ||
+        if (0 > vasprintf(&output, format, arglist) ||
-            NULL == output || NULL == *output) {
+            NULL == output) {
             va_end(arglist);
             return;
         }
-        fprintf(stderr, "INFO: %s", *output);
+        fprintf(stderr, "INFO: %s", output);
-        free(*output);
+        free(output);
     }
     va_end(arglist);
 }

opal/mca/pmix/pmix3x/pmix/test/pmi_client.c

@@ -27,17 +27,17 @@ static int _verbose = 1;
 static void log_fatal(const char *format, ...)
 {
     va_list arglist;
-    char **output = NULL;
+    char *output = NULL;
 
     va_start(arglist, format);
     if (_verbose > 0) {
-        if (0 > vasprintf(output, format, arglist) ||
+        if (0 > vasprintf(&output, format, arglist) ||
-            NULL == output || NULL == *output) {
+            NULL == output) {
             va_end(arglist);
             return;
         }
-        fprintf(stderr, "FATAL: %s", *output);
+        fprintf(stderr, "FATAL: %s", output);
-        free(*output);
+        free(output);
     }
     va_end(arglist);
 }
@@ -45,17 +45,17 @@ static void log_fatal(const char *format, ...)
 static void log_error(const char *format, ...)
 {
     va_list arglist;
-    char **output = NULL;
+    char *output = NULL;
 
     va_start(arglist, format);
     if (_verbose > 0) {
-        if (0 > vasprintf(output, format, arglist) ||
+        if (0 > vasprintf(&output, format, arglist) ||
-            NULL == output || NULL == *output) {
+            NULL == output) {
             va_end(arglist);
             return;
         }
-        fprintf(stderr, "ERROR: %s", *output);
+        fprintf(stderr, "ERROR: %s", output);
-        free(*output);
+        free(output);
     }
     va_end(arglist);
 }
@@ -63,17 +63,17 @@ static void log_error(const char *format, ...)
 static void log_info(const char *format, ...)
 {
     va_list arglist;
-    char **output = NULL;
+    char *output = NULL;
 
     va_start(arglist, format);
     if (_verbose > 0) {
-        if (0 > vasprintf(output, format, arglist) ||
+        if (0 > vasprintf(&output, format, arglist) ||
-            NULL == output || NULL == *output) {
+            NULL == output) {
             va_end(arglist);
             return;
         }
-        fprintf(stderr, "INFO: %s", *output);
+        fprintf(stderr, "INFO: %s", output);
-        free(*output);
+        free(output);
     }
     va_end(arglist);
 }

opal/mca/pmix/pmix3x/pmix/test/simple/Makefile.am

@@ -11,7 +11,7 @@
 #                         All rights reserved.
 # Copyright (c) 2006-2010 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2012-2013 Los Alamos National Security, Inc.  All rights reserved.
-# Copyright (c) 2013-2017 Intel, Inc. All rights reserved.
+# Copyright (c) 2013-2018 Intel, Inc. All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -21,7 +21,7 @@
 
 AM_CPPFLAGS = -I$(top_builddir)/src -I$(top_builddir)/src/include -I$(top_builddir)/include -I$(top_builddir)/include/pmix
 
-noinst_PROGRAMS = simptest simpclient simppub simpdyn simpft simpdmodex test_pmix simptool simpdie simplegacy
+noinst_PROGRAMS = simptest simpclient simppub simpdyn simpft simpdmodex test_pmix simptool simpdie simplegacy simptimeout
 
 simptest_SOURCES = \
         simptest.c
@@ -82,3 +82,9 @@ simplegacy_SOURCES = \
 simplegacy_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS)
 simplegacy_LDADD = \
     $(top_builddir)/src/libpmix.la
+
+simptimeout_SOURCES = \
+        simptimeout.c
+simptimeout_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS)
+simptimeout_LDADD = \
+    $(top_builddir)/src/libpmix.la

opal/mca/pmix/pmix3x/pmix/test/simple/simptest.c

@@ -13,7 +13,7 @@
  *                         All rights reserved.
  * Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2011      Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2013-2017 Intel, Inc. All rights reserved.
+ * Copyright (c) 2013-2018 Intel, Inc. All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2016      IBM Corporation.  All rights reserved.
@@ -161,6 +161,7 @@ PMIX_CLASS_INSTANCE(myxfer_t,
 
 typedef struct {
     pmix_list_item_t super;
+    int exit_code;
     pid_t pid;
 } wait_tracker_t;
 PMIX_CLASS_INSTANCE(wait_tracker_t,
@@ -168,9 +169,11 @@ PMIX_CLASS_INSTANCE(wait_tracker_t,
                     NULL, NULL);
 
 static volatile int wakeup;
+static int exit_code = 0;
 static pmix_list_t pubdata;
 static pmix_event_t handler;
 static pmix_list_t children;
+static bool istimeouttest = false;
 
 static void set_namespace(int nprocs, char *ranks, char *nspace,
                           pmix_op_cbfunc_t cbfunc, myxfer_t *x);
@@ -283,6 +286,10 @@ int main(int argc, char **argv)
         } else if (0 == strcmp("-e", argv[n]) &&
                    NULL != argv[n+1]) {
             executable = strdup(argv[n+1]);
+            /* check for timeout test */
+            if (NULL != strstr(executable, "simptimeout")) {
+                istimeouttest = true;
+            }
             for (k=n+2; NULL != argv[k]; k++) {
                 pmix_argv_append_nosize(&client_argv, argv[k]);
             }
@@ -407,6 +414,9 @@ int main(int argc, char **argv)
             } else {
                 pmix_setenv("PMIX_MCA_ptl", "usock", true, &client_env);
             }
+        } else if (!usock) {
+            /* don't disable usock => enable it on client */
+            pmix_setenv("PMIX_MCA_ptl", "usock", true, &client_env);
         }
         x = PMIX_NEW(myxfer_t);
         if (PMIX_SUCCESS != (rc = PMIx_server_register_client(&proc, myuid, mygid,
@@ -447,6 +457,15 @@ int main(int argc, char **argv)
         nanosleep(&ts, NULL);
     }
 
+    /* see if anyone exited with non-zero status */
+    n=0;
+    PMIX_LIST_FOREACH(child, &children, wait_tracker_t) {
+        if (0 != child->exit_code) {
+            fprintf(stderr, "Child %d exited with status %d - test FAILED\n", n, child->exit_code);
+            goto done;
+        }
+        ++n;
+    }
     /* try notifying ourselves */
     ninfo = 3;
     PMIX_INFO_CREATE(info, ninfo);
@@ -463,20 +482,29 @@ int main(int argc, char **argv)
     }
     PMIX_INFO_FREE(info, ninfo);
 
+  done:
     /* deregister the event handlers */
     PMIx_Deregister_event_handler(0, NULL, NULL);
 
     /* release any pub data */
     PMIX_LIST_DESTRUCT(&pubdata);
 
+    /* release the child tracker */
+    PMIX_LIST_DESTRUCT(&children);
+
     /* finalize the server library */
     if (PMIX_SUCCESS != (rc = PMIx_server_finalize())) {
         fprintf(stderr, "Finalize failed with error %d\n", rc);
+        exit_code = rc;
     }
 
-    fprintf(stderr, "Test finished OK!\n");
+    if (0 == exit_code) {
+        fprintf(stderr, "Test finished OK!\n");
+    } else {
+        fprintf(stderr, "TEST FAILED WITH ERROR %d\n", exit_code);
+    }
 
-    return rc;
+    return exit_code;
 }
 
 static void set_namespace(int nprocs, char *ranks, char *nspace,
@@ -641,6 +669,11 @@ static pmix_status_t dmodex_fn(const pmix_proc_t *proc,
 {
     pmix_output(0, "SERVER: DMODEX");
 
+    /* if this is a timeout test, then do nothing */
+    if (istimeouttest) {
+        return PMIX_SUCCESS;
+    }
+
     /* we don't have any data for remote procs as this
      * test only runs one server - so report accordingly */
     if (NULL != cbfunc) {
@@ -936,7 +969,11 @@ static void wait_signal_callback(int fd, short event, void *arg)
         /* we are already in an event, so it is safe to access the list */
         PMIX_LIST_FOREACH(t2, &children, wait_tracker_t) {
             if (pid == t2->pid) {
+                t2->exit_code = status;
                 /* found it! */
+                if (0 != status && 0 == exit_code) {
+                    exit_code = status;
+                }
                 --wakeup;
                 break;
             }

opal/mca/pmix/pmix3x/pmix/test/simple/simptimeout.c

@@ -0,0 +1,168 @@
+/*
+ * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2011 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2013 Los Alamos National Security, LLC.
+ *                         All rights reserved.
+ * Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2011      Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2013-2018 Intel, Inc. All rights reserved.
+ * Copyright (c) 2015      Mellanox Technologies, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+
+#include <src/include/pmix_config.h>
+#include <pmix.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <time.h>
+
+#include "src/class/pmix_object.h"
+#include "src/util/output.h"
+#include "src/util/printf.h"
+
+#define MAXCNT 1
+
+static volatile bool completed = false;
+static pmix_proc_t myproc;
+
+static void notification_fn(size_t evhdlr_registration_id,
+                            pmix_status_t status,
+                            const pmix_proc_t *source,
+                            pmix_info_t info[], size_t ninfo,
+                            pmix_info_t results[], size_t nresults,
+                            pmix_event_notification_cbfunc_fn_t cbfunc,
+                            void *cbdata)
+{
+    pmix_output(0, "Client %s:%d NOTIFIED with status %s", myproc.nspace, myproc.rank, PMIx_Error_string(status));
+    if (NULL != cbfunc) {
+        cbfunc(PMIX_SUCCESS, NULL, 0, NULL, NULL, cbdata);
+    }
+    completed = true;
+}
+
+static void errhandler_reg_callbk(pmix_status_t status,
+                                  size_t errhandler_ref,
+                                  void *cbdata)
+{
+    volatile bool *active = (volatile bool*)cbdata;
+
+    pmix_output(0, "Client: ERRHANDLER REGISTRATION CALLBACK CALLED WITH STATUS %d, ref=%lu",
+                status, (unsigned long)errhandler_ref);
+    *active = false;
+}
+
+
+int main(int argc, char **argv)
+{
+    int rc;
+    pmix_value_t value;
+    pmix_value_t *val = &value;
+    pmix_proc_t proc, newproc;
+    uint32_t nprocs, n;
+    volatile bool active;
+    pmix_info_t info;
+
+    /* init us */
+    if (PMIX_SUCCESS != (rc = PMIx_Init(&myproc, NULL, 0))) {
+        pmix_output(0, "Client ns %s rank %d: PMIx_Init failed: %s",
+                    myproc.nspace, myproc.rank, PMIx_Error_string(rc));
+        exit(rc);
+    }
+    pmix_output(0, "Client ns %s rank %d: Running", myproc.nspace, myproc.rank);
+
+    /* test something */
+    (void)strncpy(proc.nspace, myproc.nspace, PMIX_MAX_NSLEN);
+    proc.rank = PMIX_RANK_WILDCARD;
+    if (PMIX_SUCCESS != (rc = PMIx_Get(&proc, PMIX_JOB_SIZE, NULL, 0, &val))) {
+        pmix_output(0, "Client ns %s rank %d: PMIx_Get failed: %s",
+                    myproc.nspace, myproc.rank, PMIx_Error_string(rc));
+        exit(rc);
+    }
+    PMIX_VALUE_RELEASE(val);
+
+    /* register our errhandler */
+    active = true;
+    PMIx_Register_event_handler(NULL, 0, NULL, 0,
+                                notification_fn, errhandler_reg_callbk, (void*)&active);
+    while (active) {
+        usleep(10);
+    }
+
+    /* get our universe size */
+    (void)strncpy(proc.nspace, myproc.nspace, PMIX_MAX_NSLEN);
+    proc.rank = PMIX_RANK_WILDCARD;
+    if (PMIX_SUCCESS != (rc = PMIx_Get(&proc, PMIX_UNIV_SIZE, NULL, 0, &val))) {
+        pmix_output(0, "Client ns %s rank %d: PMIx_Get universe size failed: %s",
+                    myproc.nspace, myproc.rank, PMIx_Error_string(rc));
+        goto done;
+    }
+    nprocs = val->data.uint32;
+    PMIX_VALUE_RELEASE(val);
+    pmix_output(0, "Client %s:%d universe size %d", myproc.nspace, myproc.rank, nprocs);
+
+    /* if we are rank=0, then do a fence with timeout */
+    if (0 == myproc.rank) {
+        PMIX_INFO_CONSTRUCT(&info);
+        n = 1;
+        PMIX_INFO_LOAD(&info, PMIX_TIMEOUT, &n, PMIX_UINT32);
+        PMIX_PROC_CONSTRUCT(&proc);
+        (void)strncpy(proc.nspace, myproc.nspace, PMIX_MAX_NSLEN);
+        proc.rank = PMIX_RANK_WILDCARD;
+        pmix_output(0, "TEST FENCE TIMEOUT");
+        if (PMIX_ERR_TIMEOUT != (rc = PMIx_Fence(&proc, 1, &info, 1))) {
+            pmix_output(0, "Client ns %s rank %d: PMIx_Fence did not timeout: %s",
+                        myproc.nspace, myproc.rank, PMIx_Error_string(rc));
+            goto done;
+        }
+        pmix_output(0, "FENCE TIMEOUT SUCCEEDED");
+
+        /* check timeout on connect */
+        pmix_output(0, "TEST CONNECT TIMEOUT");
+        if (PMIX_ERR_TIMEOUT != (rc = PMIx_Connect(&proc, 1, &info, 1, newproc.nspace, &newproc.rank))) {
+            pmix_output(0, "Client ns %s rank %d: PMIx_Connect did not timeout: %s",
+                        myproc.nspace, myproc.rank, PMIx_Error_string(rc));
+            goto done;
+        }
+        pmix_output(0, "CONNECT TIMEOUT SUCCEEDED");
+
+        /* check timeout on Get */
+        proc.rank = 1;
+        pmix_output(0, "TEST GET TIMEOUT");
+        if (PMIX_ERR_TIMEOUT == (rc = PMIx_Get(&proc, "1234", &info, 1, &val))) {
+            pmix_output(0, "Client ns %s rank %d: PMIx_Get did not timeout: %s",
+                        myproc.nspace, myproc.rank, PMIx_Error_string(rc));
+            goto done;
+        }
+        pmix_output(0, "GET TIMEOUT SUCCEEDED");
+
+    } else {
+        sleep(5);
+    }
+
+ done:
+    /* finalize us */
+    pmix_output(0, "Client ns %s rank %d: Finalizing", myproc.nspace, myproc.rank);
+    if (PMIX_SUCCESS != (rc = PMIx_Finalize(NULL, 0))) {
+        fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize failed: %s\n",
+                myproc.nspace, myproc.rank, PMIx_Error_string(rc));
+    } else {
+        fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize successfully completed\n", myproc.nspace, myproc.rank);
+    }
+    fflush(stderr);
+    return(rc);
+}

opal/mca/pmix/pmix3x/pmix3x.c

@@ -53,7 +53,7 @@
 
 /* These are functions used by both client and server to
  * access common functions in the embedded PMIx library */
-
+static bool legacy_get(void);
 static const char *pmix3x_get_nspace(opal_jobid_t jobid);
 static void pmix3x_register_jobid(opal_jobid_t jobid, const char *nspace);
 static void register_handler(opal_list_t *event_codes,
@@ -77,6 +77,7 @@ static void pmix3x_log(opal_list_t *info,
 static int pmix3x_register_cleanup(char *path, bool directory, bool ignore, bool jobscope);
 
 const opal_pmix_base_module_t opal_pmix_pmix3x_module = {
+    .legacy_get = legacy_get,
     /* client APIs */
     .init = pmix3x_client_init,
     .finalize = pmix3x_client_finalize,
@@ -119,6 +120,7 @@ const opal_pmix_base_module_t opal_pmix_pmix3x_module = {
     .server_setup_fork = pmix3x_server_setup_fork,
     .server_dmodex_request = pmix3x_server_dmodex,
     .server_notify_event = pmix3x_server_notify_event,
+    .server_iof_push = pmix3x_server_iof_push,
     /* tool APIs */
     .tool_init = pmix3x_tool_init,
     .tool_finalize = pmix3x_tool_fini,
@@ -132,6 +134,11 @@ const opal_pmix_base_module_t opal_pmix_pmix3x_module = {
     .register_jobid = pmix3x_register_jobid
 };
 
+static bool legacy_get(void)
+{
+    return false;
+}
+
 static void opcbfunc(pmix_status_t status, void *cbdata)
 {
     pmix3x_opcaddy_t *op = (pmix3x_opcaddy_t*)cbdata;

opal/mca/pmix/pmix3x/pmix3x.h

@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2014-2017 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
  * Copyright (c) 2014-2015 Mellanox Technologies, Inc.
  *                         All rights reserved.
  * Copyright (c) 2016      Research Organization for Information Science
@@ -297,6 +297,9 @@ OPAL_MODULE_DECLSPEC int pmix3x_server_notify_event(int status,
                                                     opal_list_t *info,
                                                     opal_pmix_op_cbfunc_t cbfunc, void *cbdata);
 
+OPAL_MODULE_DECLSPEC int pmix3x_server_iof_push(const opal_process_name_t *source,
+                                                opal_pmix_iof_channel_t channel,
+                                                unsigned char *data, size_t nbytes);
 
 /****  COMPONENT UTILITY FUNCTIONS  ****/
 OPAL_MODULE_DECLSPEC int opal_pmix_pmix3x_check_evars(void);

opal/mca/pmix/pmix3x/pmix3x_client.c

@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2014-2017 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
  * Copyright (c) 2014-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2014-2017 Mellanox Technologies, Inc.
@@ -604,6 +604,7 @@ int pmix3x_get(const opal_process_name_t *proc, const char *key,
         /* if they are asking for our jobid, then return it */
         if (0 == strcmp(key, OPAL_PMIX_JOBID)) {
             (*val) = OBJ_NEW(opal_value_t);
+            (*val)->key = strdup(key);
             (*val)->type = OPAL_UINT32;
             (*val)->data.uint32 = OPAL_PROC_MY_NAME.jobid;
             OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
@@ -612,6 +613,7 @@ int pmix3x_get(const opal_process_name_t *proc, const char *key,
         /* if they are asking for our rank, return it */
         if (0 == strcmp(key, OPAL_PMIX_RANK)) {
             (*val) = OBJ_NEW(opal_value_t);
+            (*val)->key = strdup(key);
             (*val)->type = OPAL_INT;
             (*val)->data.integer = pmix3x_convert_rank(mca_pmix_pmix3x_component.myproc.rank);
             OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
@@ -646,6 +648,9 @@ int pmix3x_get(const opal_process_name_t *proc, const char *key,
     rc = PMIx_Get(&p, key, pinfo, sz, &pval);
     if (PMIX_SUCCESS == rc) {
         ival = OBJ_NEW(opal_value_t);
+        if (NULL != key) {
+            ival->key = strdup(key);
+        }
         if (OPAL_SUCCESS != (ret = pmix3x_value_unload(ival, pval))) {
             rc = pmix3x_convert_opalrc(ret);
         } else {
@@ -667,6 +672,9 @@ static void val_cbfunc(pmix_status_t status,
 
     OPAL_ACQUIRE_OBJECT(op);
     OBJ_CONSTRUCT(&val, opal_value_t);
+    if (NULL != op->nspace) {
+        val.key = strdup(op->nspace);
+    }
     rc = pmix3x_convert_opalrc(status);
     if (PMIX_SUCCESS == status && NULL != kv) {
         rc = pmix3x_value_unload(&val, kv);
@@ -706,6 +714,7 @@ int pmix3x_getnb(const opal_process_name_t *proc, const char *key,
         if (0 == strcmp(key, OPAL_PMIX_JOBID)) {
             if (NULL != cbfunc) {
                 val = OBJ_NEW(opal_value_t);
+                val->key = strdup(key);
                 val->type = OPAL_UINT32;
                 val->data.uint32 = OPAL_PROC_MY_NAME.jobid;
                 cbfunc(OPAL_SUCCESS, val, cbdata);
@@ -717,6 +726,7 @@ int pmix3x_getnb(const opal_process_name_t *proc, const char *key,
         if (0 == strcmp(key, OPAL_PMIX_RANK)) {
             if (NULL != cbfunc) {
                 val = OBJ_NEW(opal_value_t);
+                val->key = strdup(key);
                 val->type = OPAL_INT;
                 val->data.integer = pmix3x_convert_rank(mca_pmix_pmix3x_component.myproc.rank);
                 cbfunc(OPAL_SUCCESS, val, cbdata);
@@ -730,7 +740,9 @@ int pmix3x_getnb(const opal_process_name_t *proc, const char *key,
     op = OBJ_NEW(pmix3x_opcaddy_t);
     op->valcbfunc = cbfunc;
     op->cbdata = cbdata;
-
+    if (NULL != key) {
+        op->nspace = strdup(key);
+    }
     if (NULL == proc) {
         (void)strncpy(op->p.nspace, mca_pmix_pmix3x_component.myproc.nspace, PMIX_MAX_NSLEN);
         op->p.rank = pmix3x_convert_rank(PMIX_RANK_WILDCARD);

opal/mca/pmix/pmix3x/pmix3x_component.c

@@ -1,8 +1,8 @@
 /*
- * Copyright (c) 2014-2017 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
  * Copyright (c) 2014-2015 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2016 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2016      Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -21,6 +21,7 @@
 #include "opal/constants.h"
 #include "opal/class/opal_list.h"
 #include "opal/util/proc.h"
+#include "opal/util/show_help.h"
 #include "opal/mca/pmix/pmix.h"
 #include "pmix3x.h"
 
@@ -94,11 +95,19 @@ static int external_register(void)
 
 static int external_open(void)
 {
+    const char *version;
+
     mca_pmix_pmix3x_component.evindex = 0;
     OBJ_CONSTRUCT(&mca_pmix_pmix3x_component.jobids, opal_list_t);
     OBJ_CONSTRUCT(&mca_pmix_pmix3x_component.events, opal_list_t);
     OBJ_CONSTRUCT(&mca_pmix_pmix3x_component.dmdx, opal_list_t);
 
+    version = PMIx_Get_version();
+    if ('3' != version[0]) {
+        opal_show_help("help-pmix-base.txt",
+                       "incorrect-pmix", true, version, "v3.x");
+        return OPAL_ERROR;
+    }
     return OPAL_SUCCESS;
 }
 

opal/mca/pmix/pmix3x/pmix3x_server_south.c

@@ -86,6 +86,7 @@ static void lkcbfunc(pmix_status_t status, void *cbdata)
     opal_pmix_lock_t *lk = (opal_pmix_lock_t*)cbdata;
 
     OPAL_POST_OBJECT(lk);
+    lk->status = pmix3x_convert_rc(status);
     OPAL_PMIX_WAKEUP_THREAD(lk);
 }
 
@@ -567,3 +568,70 @@ int pmix3x_server_notify_event(int status,
     }
     return pmix3x_convert_rc(rc);
 }
+
+int pmix3x_server_iof_push(const opal_process_name_t *source,
+                           opal_pmix_iof_channel_t channel,
+                           unsigned char *data, size_t nbytes)
+{
+    pmix3x_opcaddy_t *op;
+    pmix_byte_object_t bo;
+    pmix_iof_channel_t pchan;
+    opal_pmix_lock_t lock;
+    pmix_status_t rc;
+    int ret;
+
+    opal_output_verbose(2, opal_pmix_base_framework.framework_output,
+                        "%s IOF push from %s with %d bytes",
+                        OPAL_NAME_PRINT(OPAL_PROC_MY_NAME),
+                        OPAL_NAME_PRINT(*source), (int)nbytes);
+
+    OPAL_PMIX_ACQUIRE_THREAD(&opal_pmix_base.lock);
+    if (0 >= opal_pmix_base.initialized) {
+        OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
+        return OPAL_ERR_NOT_INITIALIZED;
+    }
+    OPAL_PMIX_RELEASE_THREAD(&opal_pmix_base.lock);
+
+    /* setup the caddy */
+    op = OBJ_NEW(pmix3x_opcaddy_t);
+    /* convert the source */
+    (void)opal_snprintf_jobid(op->p.nspace, PMIX_MAX_NSLEN, source->jobid);
+    op->p.rank = pmix3x_convert_opalrank(source->vpid);
+    /* convert the channel */
+    pchan = 0;
+    if (OPAL_PMIX_FWD_STDIN_CHANNEL & channel) {
+        pchan |= PMIX_FWD_STDIN_CHANNEL;
+    }
+    if (OPAL_PMIX_FWD_STDOUT_CHANNEL & channel) {
+        pchan |= PMIX_FWD_STDOUT_CHANNEL;
+    }
+    if (OPAL_PMIX_FWD_STDERR_CHANNEL & channel) {
+        pchan |= PMIX_FWD_STDERR_CHANNEL;
+    }
+    if (OPAL_PMIX_FWD_STDDIAG_CHANNEL & channel) {
+        pchan |= PMIX_FWD_STDDIAG_CHANNEL;
+    }
+
+    /* setup the byte object */
+    PMIX_BYTE_OBJECT_CONSTRUCT(&bo);
+    if (0 < nbytes) {
+        bo.bytes = (char*)data;
+    }
+    bo.size = nbytes;
+
+    /* push the IO */
+    OPAL_PMIX_CONSTRUCT_LOCK(&lock);
+    rc = PMIx_IOF_push(&op->p, pchan, &bo, NULL, 0, lkcbfunc, (void*)&lock);
+    if (PMIX_SUCCESS != rc) {
+        ret = pmix3x_convert_rc(rc);
+    } else {
+        /* wait for completion */
+        OPAL_PMIX_WAIT_THREAD(&lock);
+        ret = lock.status;
+        OPAL_PMIX_DESTRUCT_LOCK(&lock);
+    }
+    /* cleanup */
+    OBJ_RELEASE(op);
+
+    return ret;
+}

opal/mca/pmix/pmix_server.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014-2017 Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -243,6 +243,11 @@ typedef int (*opal_pmix_server_job_control_fn_t)(const opal_process_name_t *requ
 
 /* we do not provide a monitoring capability */
 
+/* Entry point for pushing forwarded IO to clients/tools */
+typedef int (*opal_pmix_server_iof_fn_t)(const opal_process_name_t *source,
+                                         opal_pmix_iof_channel_t channel,
+                                         unsigned char *data, size_t nbytes);
+
 typedef struct opal_pmix_server_module_1_0_0_t {
     opal_pmix_server_client_connected_fn_t      client_connected;
     opal_pmix_server_client_finalized_fn_t      client_finalized;

opal/mca/pmix/pmix_types.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014-2017 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
  * Copyright (c) 2016      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
@@ -443,6 +443,15 @@ typedef enum {
     OPAL_PMIX_ALLOC_REAQCUIRE
 } opal_pmix_alloc_directive_t;
 
+/* define a set of bit-mask flags for specifying IO
+ * forwarding channels. These can be OR'd together
+ * to reference multiple channels */
+typedef uint16_t opal_pmix_iof_channel_t;
+#define OPAL_PMIX_FWD_STDIN_CHANNEL      0x01
+#define OPAL_PMIX_FWD_STDOUT_CHANNEL     0x02
+#define OPAL_PMIX_FWD_STDERR_CHANNEL     0x04
+#define OPAL_PMIX_FWD_STDDIAG_CHANNEL    0x08
+
 
 /****    PMIX INFO STRUCT    ****/
 

orte/mca/ess/alps/Makefile.am

@@ -10,6 +10,7 @@
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
 # Copyright (c) 2008-2010 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2017-2018 Intel, Inc. All rights reserved.
 # Copyright (c) 2017      IBM Corporation.  All rights reserved.
 # $COPYRIGHT$
 #
@@ -50,4 +51,3 @@ libmca_ess_alps_la_SOURCES =$(sources)
 libmca_ess_alps_la_CPPFLAGS = $(ess_alps_CPPFLAGS)
 libmca_ess_alps_la_LDFLAGS = -module -avoid-version  $(ess_alps_LDFLAGS)
 libmca_ess_alps_la_LIBADD = $(ess_alps_LIBS)
-

orte/mca/ess/base/ess_base_std_app.c

@@ -45,7 +45,6 @@
 #include "opal/util/output.h"
 #include "opal/util/proc.h"
 #include "opal/runtime/opal.h"
-#include "opal/runtime/opal_cr.h"
 
 #include "orte/mca/rml/base/base.h"
 #include "orte/mca/routed/base/base.h"
@@ -58,17 +57,12 @@
 #include "orte/mca/odls/odls_types.h"
 #include "orte/mca/filem/base/base.h"
 #include "orte/mca/errmgr/base/base.h"
-#if OPAL_ENABLE_FT_CR == 1
-#include "orte/mca/snapc/base/base.h"
-#include "orte/mca/sstore/base/base.h"
-#endif
 #include "orte/mca/state/base/base.h"
 #include "orte/util/proc_info.h"
 #include "orte/util/session_dir.h"
 #include "orte/util/name_fns.h"
 #include "orte/util/show_help.h"
 
-#include "orte/runtime/orte_cr.h"
 #include "orte/runtime/orte_globals.h"
 #include "orte/runtime/orte_wait.h"
 
@@ -285,44 +279,6 @@ int orte_ess_base_app_setup(bool db_restrict_local)
         goto error;
     }
 
-#if OPAL_ENABLE_FT_CR == 1
-    /*
-     * Setup the SnapC
-     */
-    if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_snapc_base_framework, 0))) {
-        ORTE_ERROR_LOG(ret);
-        error = "orte_snapc_base_open";
-        goto error;
-    }
-    if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_sstore_base_framework, 0))) {
-        ORTE_ERROR_LOG(ret);
-        error = "orte_sstore_base_open";
-        goto error;
-    }
-    if (ORTE_SUCCESS != (ret = orte_snapc_base_select(ORTE_PROC_IS_HNP, ORTE_PROC_IS_APP))) {
-        ORTE_ERROR_LOG(ret);
-        error = "orte_snapc_base_select";
-        goto error;
-    }
-    if (ORTE_SUCCESS != (ret = orte_sstore_base_select())) {
-        ORTE_ERROR_LOG(ret);
-        error = "orte_sstore_base_select";
-        goto error;
-    }
-    /* apps need the OPAL CR stuff */
-    opal_cr_set_enabled(true);
-#else
-    opal_cr_set_enabled(false);
-#endif
-    /* Initalize the CR setup
-     * Note: Always do this, even in non-FT builds.
-     * If we don't some user level tools may hang.
-     */
-    if (ORTE_SUCCESS != (ret = orte_cr_init())) {
-        ORTE_ERROR_LOG(ret);
-        error = "orte_cr_init";
-        goto error;
-    }
     /* open the distributed file system */
     if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_dfs_base_framework, 0))) {
         ORTE_ERROR_LOG(ret);
@@ -344,13 +300,6 @@ int orte_ess_base_app_setup(bool db_restrict_local)
 
 int orte_ess_base_app_finalize(void)
 {
-    orte_cr_finalize();
-
-#if OPAL_ENABLE_FT_CR == 1
-    (void) mca_base_framework_close(&orte_snapc_base_framework);
-    (void) mca_base_framework_close(&orte_sstore_base_framework);
-#endif
-
     /* release the conduits */
     orte_rml.close_conduit(orte_mgmt_conduit);
     orte_rml.close_conduit(orte_coll_conduit);
@@ -414,8 +363,7 @@ void orte_ess_base_app_abort(int status, bool report)
      * clean environment. Taken from orte_finalize():
      * - Assume errmgr cleans up child processes before we exit.
      */
-    /* CRS cleanup since it may have a named pipe and thread active */
+
-    orte_cr_finalize();
     /* If we were asked to report this termination, do so.
      * Since singletons don't start an HNP unless necessary, and
      * direct-launched procs don't have daemons at all, only send

orte/mca/ess/base/ess_base_std_orted.c

@@ -38,11 +38,11 @@
 #include "opal/dss/dss.h"
 #include "opal/mca/event/event.h"
 #include "opal/runtime/opal.h"
-#include "opal/runtime/opal_cr.h"
 #include "opal/mca/hwloc/base/base.h"
 #include "opal/mca/pmix/base/base.h"
 #include "opal/mca/pstat/base/base.h"
 #include "opal/util/arch.h"
+#include "opal/util/opal_environ.h"
 #include "opal/util/os_path.h"
 #include "opal/util/proc.h"
 
@@ -61,10 +61,6 @@
 #include "orte/mca/regx/base/base.h"
 #include "orte/mca/errmgr/errmgr.h"
 #include "orte/mca/rmaps/base/base.h"
-#if OPAL_ENABLE_FT_CR == 1
-#include "orte/mca/snapc/base/base.h"
-#include "orte/mca/sstore/base/base.h"
-#endif
 #include "orte/mca/filem/base/base.h"
 #include "orte/util/proc_info.h"
 #include "orte/util/session_dir.h"
@@ -73,7 +69,6 @@
 #include "orte/mca/errmgr/base/base.h"
 #include "orte/mca/state/base/base.h"
 #include "orte/mca/state/state.h"
-#include "orte/runtime/orte_cr.h"
 #include "orte/runtime/orte_wait.h"
 #include "orte/runtime/orte_globals.h"
 #include "orte/runtime/orte_quit.h"
@@ -626,46 +621,6 @@ int orte_ess_base_orted_setup(void)
         goto error;
     }
 
-#if OPAL_ENABLE_FT_CR == 1
-    /*
-     * Setup the SnapC
-     */
-    if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_snapc_base_framework, 0))) {
-        ORTE_ERROR_LOG(ret);
-        error = "orte_snapc_base_open";
-        goto error;
-    }
-    if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_sstore_base_framework, 0))) {
-        ORTE_ERROR_LOG(ret);
-        error = "orte_sstore_base_open";
-        goto error;
-    }
-    if (ORTE_SUCCESS != (ret = orte_snapc_base_select(!ORTE_PROC_IS_HNP, ORTE_PROC_IS_DAEMON))) {
-        ORTE_ERROR_LOG(ret);
-        error = "orte_snapc_base_select";
-        goto error;
-    }
-    if (ORTE_SUCCESS != (ret = orte_sstore_base_select())) {
-        ORTE_ERROR_LOG(ret);
-        error = "orte_sstore_base_select";
-        goto error;
-    }
-
-    /* For daemons, ORTE doesn't need the OPAL CR stuff */
-    opal_cr_set_enabled(false);
-#else
-    opal_cr_set_enabled(false);
-#endif
-    /*
-     * Initalize the CR setup
-     * Note: Always do this, even in non-FT builds.
-     * If we don't some user level tools may hang.
-     */
-    if (ORTE_SUCCESS != (ret = orte_cr_init())) {
-        ORTE_ERROR_LOG(ret);
-        error = "orte_cr_init";
-        goto error;
-    }
     /* setup the DFS framework */
     if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_dfs_base_framework, 0))) {
         ORTE_ERROR_LOG(ret);
@@ -680,7 +635,7 @@ int orte_ess_base_orted_setup(void)
 
     return ORTE_SUCCESS;
 
- error:
+  error:
     orte_show_help("help-orte-runtime.txt",
                    "orte_init:startup:internal-failure",
                    true, error, ORTE_ERROR_NAME(ret), ret);

orte/mca/ess/base/ess_base_std_tool.c

@@ -11,7 +11,7 @@
  *                         All rights reserved.
  * Copyright (c) 2011-2013 Los Alamos National Security, LLC.
  *                         All rights reserved.
- * Copyright (c) 2013-2017 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2013-2018 Intel, Inc. All rights reserved.
  * Copyright (c) 2014      Hochschule Esslingen.  All rights reserved.
  *
  * Copyright (c) 2015      Cisco Systems, Inc.  All rights reserved.
@@ -37,25 +37,24 @@
 #include "opal/mca/event/event.h"
 #include "opal/mca/pmix/base/base.h"
 #include "opal/runtime/opal.h"
-#include "opal/runtime/opal_cr.h"
 #include "opal/runtime/opal_progress_threads.h"
 #include "opal/util/arch.h"
+#include "opal/util/opal_environ.h"
 #include "opal/util/argv.h"
 #include "opal/util/proc.h"
 
+#include "orte/mca/iof/base/base.h"
 #include "orte/mca/oob/base/base.h"
 #include "orte/mca/plm/base/base.h"
 #include "orte/mca/rml/base/base.h"
 #include "orte/mca/rml/base/rml_contact.h"
 #include "orte/mca/routed/base/base.h"
 #include "orte/mca/errmgr/base/base.h"
-#include "orte/mca/iof/base/base.h"
 #include "orte/mca/state/base/base.h"
 #include "orte/util/proc_info.h"
 #include "orte/util/session_dir.h"
 #include "orte/util/show_help.h"
 
-#include "orte/runtime/orte_cr.h"
 #include "orte/runtime/orte_globals.h"
 #include "orte/runtime/orte_wait.h"
 
@@ -96,8 +95,6 @@ int orte_ess_base_tool_setup(opal_list_t *flags)
     int ret;
     char *error = NULL;
     opal_list_t transports;
-    orte_jobid_t jobid;
-    orte_vpid_t vpid;
     opal_list_t info;
     opal_value_t *kv, *knext, val;
     opal_pmix_query_t *q;
@@ -123,65 +120,17 @@ int orte_ess_base_tool_setup(opal_list_t *flags)
     }
     if (NULL == opal_pmix.tool_init) {
         /* we no longer support non-pmix tools */
+        orte_show_help("help-ess-base.txt",
+                       "legacy-tool", true);
+        ret = ORTE_ERR_SILENT;
         error = "opal_pmix.tool_init";
-        ret = ORTE_ERR_NOT_SUPPORTED;
         goto error;
     }
     /* set the event base for the pmix component code */
     opal_pmix_base_set_evbase(orte_event_base);
 
-    /* we have to define our name here */
+    /* initialize */
-    if (NULL != orte_ess_base_jobid &&
-        NULL != orte_ess_base_vpid) {
-        opal_output_verbose(2, orte_ess_base_framework.framework_output,
-                            "ess:tool:obtaining name from environment");
-        if (ORTE_SUCCESS != (ret = orte_util_convert_string_to_jobid(&jobid, orte_ess_base_jobid))) {
-            return(ret);
-        }
-        ORTE_PROC_MY_NAME->jobid = jobid;
-        if (ORTE_SUCCESS != (ret = orte_util_convert_string_to_vpid(&vpid, orte_ess_base_vpid))) {
-            return(ret);
-        }
-        ORTE_PROC_MY_NAME->vpid = vpid;
-    } else {
-        /* If we are a tool with no name, then define it here */
-        uint16_t jobfam;
-        uint32_t hash32;
-        uint32_t bias;
-
-        opal_output_verbose(2, orte_ess_base_framework.framework_output,
-                            "ess:tool:computing name");
-        /* hash the nodename */
-        OPAL_HASH_STR(orte_process_info.nodename, hash32);
-        bias = (uint32_t)orte_process_info.pid;
-        /* fold in the bias */
-        hash32 = hash32 ^ bias;
-
-        /* now compress to 16-bits */
-        jobfam = (uint16_t)(((0x0000ffff & (0xffff0000 & hash32) >> 16)) ^ (0x0000ffff & hash32));
-
-        /* set the name */
-        ORTE_PROC_MY_NAME->jobid = 0xffff0000 & ((uint32_t)jobfam << 16);
-        ORTE_PROC_MY_NAME->vpid = 0;
-    }
-    /* my name is set, xfer it to the OPAL layer */
-    orte_process_info.super.proc_name = *(opal_process_name_t*)ORTE_PROC_MY_NAME;
-
-    /* initialize - PMIx may set our name here if we attach to
-     * a PMIx server */
     OBJ_CONSTRUCT(&info, opal_list_t);
-    /* pass our name so the PMIx layer can use it */
-    kv = OBJ_NEW(opal_value_t);
-    kv->key = strdup(OPAL_PMIX_TOOL_NSPACE);
-    orte_util_convert_jobid_to_string(&kv->data.string, ORTE_PROC_MY_NAME->jobid);
-    kv->type = OPAL_STRING;
-    opal_list_append(&info, &kv->super);
-    /* ditto for our rank */
-    kv = OBJ_NEW(opal_value_t);
-    kv->key = strdup(OPAL_PMIX_TOOL_RANK);
-    kv->data.name.vpid = ORTE_PROC_MY_NAME->vpid;
-    kv->type = OPAL_VPID;
-    opal_list_append(&info, &kv->super);
     if (NULL != flags) {
         /* pass along any directives */
         OPAL_LIST_FOREACH_SAFE(kv, knext, flags, opal_value_t) {
@@ -196,9 +145,9 @@ int orte_ess_base_tool_setup(opal_list_t *flags)
         goto error;
     }
     OPAL_LIST_DESTRUCT(&info);
+    /* the PMIx server set our name - record it here */
     ORTE_PROC_MY_NAME->jobid = OPAL_PROC_MY_NAME.jobid;
     ORTE_PROC_MY_NAME->vpid = OPAL_PROC_MY_NAME.vpid;
-
     orte_process_info.super.proc_hostname = strdup(orte_process_info.nodename);
     orte_process_info.super.proc_flags = OPAL_PROC_ALL_LOCAL;
     orte_process_info.super.proc_arch = opal_local_arch;
@@ -294,7 +243,7 @@ int orte_ess_base_tool_setup(opal_list_t *flags)
     }
 
     /* setup I/O forwarding system - must come after we init routes */
-    if (NULL != orte_process_info.my_hnp_uri) {
+    if (NULL != orte_process_info.my_hnp_uri && NULL == opal_pmix.server_iof_push) {
         /* extract the name */
         if (ORTE_SUCCESS != orte_rml_base_parse_uris(orte_process_info.my_hnp_uri, ORTE_PROC_MY_HNP, NULL)) {
             orte_show_help("help-orte-top.txt", "orte-top:hnp-uri-bad", true, orte_process_info.my_hnp_uri);
@@ -374,13 +323,14 @@ int orte_ess_base_tool_finalize(void)
      * a very small subset of orte_init - ensure that
      * I only back those elements out
      */
-    if (NULL != orte_process_info.my_hnp_uri) {
+    if (NULL != orte_process_info.my_hnp_uri && NULL == opal_pmix.server_iof_push) {
         (void) mca_base_framework_close(&orte_iof_base_framework);
     }
     (void) mca_base_framework_close(&orte_routed_base_framework);
     (void) mca_base_framework_close(&orte_rml_base_framework);
     (void) mca_base_framework_close(&orte_errmgr_base_framework);
 
+    opal_pmix.finalize();
     (void) mca_base_framework_close(&opal_pmix_base_framework);
 
     return ORTE_SUCCESS;

orte/mca/ess/base/help-ess-base.txt

@@ -10,7 +10,7 @@
 #                         University of Stuttgart.  All rights reserved.
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
-# Copyright (c) 2017      Intel, Inc. All rights reserved.
+# Copyright (c) 2017-2018 Intel, Inc. All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -89,3 +89,9 @@ when OMPI was not configured --with-alps and we weren't able
 to discover an ALPS installation in the usual places.
 
 Please configure as appropriate and try again.
+#
+[legacy-tool]
+We no longer support non-PMIx-based tools, and require a
+minimum level of PMIx v2.0.
+
+Please update the tool and/or the PMIx version you are using.

orte/mca/ess/env/Makefile.am

@@ -11,6 +11,7 @@
 #                         All rights reserved.
 # Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2017      IBM Corporation.  All rights reserved.
+# Copyright (c) 2017-2018 Intel, Inc. All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow

orte/mca/ess/env/ess_env_module.c

@@ -56,9 +56,6 @@
 #include "orte/mca/plm/base/base.h"
 
 #include "orte/mca/rmaps/base/base.h"
-#if OPAL_ENABLE_FT_CR == 1
-#include "orte/mca/snapc/base/base.h"
-#endif
 #include "orte/mca/filem/base/base.h"
 #include "orte/util/proc_info.h"
 #include "orte/util/session_dir.h"
@@ -68,7 +65,6 @@
 #include "orte/runtime/orte_wait.h"
 #include "orte/runtime/orte_globals.h"
 
-#include "orte/runtime/orte_cr.h"
 #include "orte/mca/ess/ess.h"
 #include "orte/mca/ess/base/base.h"
 #include "orte/mca/ess/env/ess_env.h"
@@ -78,19 +74,11 @@ static int env_set_name(void);
 static int rte_init(void);
 static int rte_finalize(void);
 
-#if OPAL_ENABLE_FT_CR == 1
-static int rte_ft_event(int state);
-#endif
-
 orte_ess_base_module_t orte_ess_env_module = {
     rte_init,
     rte_finalize,
     orte_ess_base_app_abort,
-#if OPAL_ENABLE_FT_CR == 1
-    rte_ft_event
-#else
     NULL
-#endif
 };
 
 static int rte_init(void)
@@ -175,201 +163,3 @@ static int env_set_name(void)
 
     return ORTE_SUCCESS;
 }
-
-#if OPAL_ENABLE_FT_CR == 1
-static int rte_ft_event(int state)
-{
-    int ret, exit_status = ORTE_SUCCESS;
-    orte_proc_type_t svtype;
-
-    /******** Checkpoint Prep ********/
-    if(OPAL_CRS_CHECKPOINT == state) {
-        /*
-         * Notify SnapC
-         */
-        if( ORTE_SUCCESS != (ret = orte_snapc.ft_event(OPAL_CRS_CHECKPOINT))) {
-            ORTE_ERROR_LOG(ret);
-            return ret;
-        }
-
-        /*
-         * Notify Routed
-         */
-        if( ORTE_SUCCESS != (ret = orte_routed.ft_event(OPAL_CRS_CHECKPOINT))) {
-            ORTE_ERROR_LOG(ret);
-            return ret;
-        }
-
-        /*
-         * Notify RML -> OOB
-         */
-        if( ORTE_SUCCESS != (ret = orte_rml.ft_event(OPAL_CRS_CHECKPOINT))) {
-            ORTE_ERROR_LOG(ret);
-            return ret;
-        }
-    }
-    /******** Continue Recovery ********/
-    else if (OPAL_CRS_CONTINUE == state ) {
-        OPAL_OUTPUT_VERBOSE((1, orte_ess_base_framework.framework_output,
-                             "ess:env ft_event(%2d) - %s is Continuing",
-                             state, ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
-
-        /*
-         * Notify RML -> OOB
-         */
-        if( ORTE_SUCCESS != (ret = orte_rml.ft_event(OPAL_CRS_CONTINUE))) {
-            ORTE_ERROR_LOG(ret);
-            return ret;
-        }
-
-        /*
-         * Notify Routed
-         */
-        if( ORTE_SUCCESS != (ret = orte_routed.ft_event(OPAL_CRS_CONTINUE))) {
-            ORTE_ERROR_LOG(ret);
-            return ret;
-        }
-
-        /*
-         * Notify SnapC
-         */
-        if( ORTE_SUCCESS != (ret = orte_snapc.ft_event(OPAL_CRS_CONTINUE))) {
-            ORTE_ERROR_LOG(ret);
-            return ret;
-        }
-
-        if (opal_cr_continue_like_restart) {
-            /*
-             * Barrier to make all processes have been successfully restarted before
-             * we try to remove some restart only files.
-             */
-            opal_pmix.fence(NULL, 0);
-
-            if( orte_cr_flush_restart_files ) {
-                OPAL_OUTPUT_VERBOSE((1, orte_ess_base_framework.framework_output,
-                                     "ess:env ft_event(%2d): %s "
-                                     "Cleanup restart files...",
-                                     state, ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
-                opal_crs_base_cleanup_flush();
-            }
-        }
-    }
-    /******** Restart Recovery ********/
-    else if (OPAL_CRS_RESTART == state ) {
-        OPAL_OUTPUT_VERBOSE((1, orte_ess_base_framework.framework_output,
-                             "ess:env ft_event(%2d) - %s is Restarting",
-                             state, ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
-
-        /*
-         * This should follow the ess init() function
-         */
-
-        /*
-         * - Reset Contact information
-         */
-        if( ORTE_SUCCESS != (ret = env_set_name() ) ) {
-            exit_status = ret;
-        }
-
-        /*
-         * Notify RML -> OOB
-         */
-        if( ORTE_SUCCESS != (ret = orte_rml.ft_event(OPAL_CRS_RESTART))) {
-            ORTE_ERROR_LOG(ret);
-            return ret;
-        }
-
-        /*
-         * Restart the routed framework
-         * JJH: Lie to the finalize function so it does not try to contact the daemon.
-         */
-        svtype = orte_process_info.proc_type;
-        orte_process_info.proc_type = ORTE_PROC_TOOL;
-        if (ORTE_SUCCESS != (ret = orte_routed.finalize()) ) {
-            ORTE_ERROR_LOG(ret);
-            return ret;
-        }
-        orte_process_info.proc_type = svtype;
-        if (ORTE_SUCCESS != (ret = orte_routed.initialize()) ) {
-            ORTE_ERROR_LOG(ret);
-            return ret;
-        }
-
-        /*
-         * Restart the PLM - Does nothing at the moment, but included for completeness
-         */
-        if (ORTE_SUCCESS != (ret = orte_plm.finalize())) {
-            ORTE_ERROR_LOG(ret);
-            return ret;
-        }
-
-        if (ORTE_SUCCESS != (ret = orte_plm.init())) {
-            ORTE_ERROR_LOG(ret);
-            return ret;
-        }
-
-        /*
-         * RML - Enable communications
-         */
-        if (ORTE_SUCCESS != (ret = orte_rml.enable_comm())) {
-            ORTE_ERROR_LOG(ret);
-            return ret;
-        }
-
-        /*
-         * Notify Routed
-         */
-        if( ORTE_SUCCESS != (ret = orte_routed.ft_event(OPAL_CRS_RESTART))) {
-            ORTE_ERROR_LOG(ret);
-            return ret;
-        }
-
-        /*
-         * Barrier to make all processes have been successfully restarted before
-         * we try to remove some restart only files.
-         */
-        opal_pmix.fence(NULL, 0);
-
-        if( orte_cr_flush_restart_files ) {
-            OPAL_OUTPUT_VERBOSE((1, orte_ess_base_framework.framework_output,
-                                 "ess:env ft_event(%2d): %s "
-                                 "Cleanup restart files...",
-                                 state, ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
-
-            opal_crs_base_cleanup_flush();
-        }
-
-        /*
-         * Session directory re-init
-         */
-        if (orte_create_session_dirs) {
-            if (ORTE_SUCCESS != (ret = orte_session_dir(true,
-                                                        orte_process_info.tmpdir_base,
-                                                        orte_process_info.nodename,
-                                                        NULL, /* Batch ID -- Not used */
-                                                        ORTE_PROC_MY_NAME))) {
-                exit_status = ret;
-            }
-
-            opal_output_set_output_file_info(orte_process_info.proc_session_dir,
-                                             "output-", NULL, NULL);
-        }
-
-        /*
-         * Notify SnapC
-         */
-        if( ORTE_SUCCESS != (ret = orte_snapc.ft_event(OPAL_CRS_RESTART))) {
-            ORTE_ERROR_LOG(ret);
-            return ret;
-        }
-    }
-    else if (OPAL_CRS_TERM == state ) {
-        /* Nothing */
-    }
-    else {
-        /* Error state = Nothing */
-    }
-
-    return exit_status;
-}
-#endif

orte/mca/ess/hnp/ess_hnp_module.c

@@ -42,13 +42,13 @@
 #include "opal/class/opal_list.h"
 #include "opal/mca/event/event.h"
 #include "opal/runtime/opal.h"
-#include "opal/runtime/opal_cr.h"
 
 #include "opal/util/arch.h"
 #include "opal/util/argv.h"
 #include "opal/util/if.h"
 #include "opal/util/os_path.h"
 #include "opal/util/output.h"
+#include "opal/util/opal_environ.h"
 #include "opal/util/malloc.h"
 #include "opal/util/basename.h"
 #include "opal/util/fd.h"
@@ -72,10 +72,6 @@
 #include "orte/mca/plm/plm.h"
 #include "orte/mca/odls/base/base.h"
 #include "orte/mca/rmaps/base/base.h"
-#if OPAL_ENABLE_FT_CR == 1
-#include "orte/mca/snapc/base/base.h"
-#include "orte/mca/sstore/base/base.h"
-#endif
 #include "orte/mca/filem/base/base.h"
 #include "orte/mca/state/base/base.h"
 #include "orte/mca/state/state.h"
@@ -95,7 +91,6 @@
 #include "orte/runtime/orte_wait.h"
 #include "orte/runtime/orte_globals.h"
 #include "orte/runtime/orte_quit.h"
-#include "orte/runtime/orte_cr.h"
 #include "orte/runtime/orte_locks.h"
 
 #include "orte/mca/ess/ess.h"
@@ -150,6 +145,7 @@ static int rte_init(void)
     orte_topology_t *t;
     opal_list_t transports;
     orte_ess_base_signal_t *sig;
+    opal_value_t val;
 
     /* run the prolog */
     if (ORTE_SUCCESS != (ret = orte_ess_base_std_prolog())) {
@@ -473,6 +469,22 @@ static int rte_init(void)
     proc->pid = orte_process_info.pid;
     orte_oob_base_get_addr(&proc->rml_uri);
     orte_process_info.my_hnp_uri = strdup(proc->rml_uri);
+    /* store it in the local PMIx repo for later retrieval */
+    OBJ_CONSTRUCT(&val, opal_value_t);
+    val.key = OPAL_PMIX_PROC_URI;
+    val.type = OPAL_STRING;
+    val.data.string = proc->rml_uri;
+    if (OPAL_SUCCESS != (ret = opal_pmix.store_local(ORTE_PROC_MY_NAME, &val))) {
+        ORTE_ERROR_LOG(ret);
+        val.key = NULL;
+        val.data.string = NULL;
+        OBJ_DESTRUCT(&val);
+        error = "store uri";
+        goto error;
+    }
+    val.key = NULL;
+    val.data.string = NULL;
+    OBJ_DESTRUCT(&val);
     /* we are also officially a daemon, so better update that field too */
     orte_process_info.my_daemon_uri = strdup(proc->rml_uri);
     proc->state = ORTE_PROC_STATE_RUNNING;
@@ -684,46 +696,7 @@ static int rte_init(void)
         error = "orte_filem_base_select";
         goto error;
     }
-#if OPAL_ENABLE_FT_CR == 1
-    /*
-     * Setup the SnapC
-     */
-    if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_snapc_base_framework, 0))) {
-        ORTE_ERROR_LOG(ret);
-        error = "orte_snapc_base_open";
-        goto error;
-    }
-    if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_sstore_base_framework, 0))) {
-        ORTE_ERROR_LOG(ret);
-        error = "orte_sstore_base_open";
-        goto error;
-    }
-    if (ORTE_SUCCESS != (ret = orte_snapc_base_select(ORTE_PROC_IS_HNP, ORTE_PROC_IS_APP))) {
-        ORTE_ERROR_LOG(ret);
-        error = "orte_snapc_base_select";
-        goto error;
-    }
-    if (ORTE_SUCCESS != (ret = orte_sstore_base_select())) {
-        ORTE_ERROR_LOG(ret);
-        error = "orte_sstore_base_select";
-        goto error;
-    }
 
-    /* For HNP, ORTE doesn't need the OPAL CR stuff */
-    opal_cr_set_enabled(false);
-#else
-    opal_cr_set_enabled(false);
-#endif
-    /*
-     * Initalize the CR setup
-     * Note: Always do this, even in non-FT builds.
-     * If we don't some user level tools may hang.
-     */
-    if (ORTE_SUCCESS != (ret = orte_cr_init())) {
-        ORTE_ERROR_LOG(ret);
-        error = "orte_cr_init";
-        goto error;
-    }
     /* setup the dfs framework */
     if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_dfs_base_framework, 0))) {
         ORTE_ERROR_LOG(ret);
@@ -773,7 +746,7 @@ static int rte_init(void)
     opal_progress_set_yield_when_idle(false);
     return ORTE_SUCCESS;
 
- error:
+  error:
     if (ORTE_ERR_SILENT != ret && !orte_report_silent_errors) {
         orte_show_help("help-orte-runtime.txt",
                        "orte_init:startup:internal-failure",
@@ -898,8 +871,6 @@ static void rte_abort(int status, bool report)
      * - Assume errmgr cleans up child processes before we exit.
      */
 
-    /* CRS cleanup since it may have a named pipe and thread active */
-    orte_cr_finalize();
     /* ensure we scrub the session directory tree */
     orte_session_dir_cleanup(ORTE_JOBID_WILDCARD);
     /* - Clean out the global structures

orte/mca/ess/lsf/Makefile.am

@@ -11,6 +11,7 @@
 #                         All rights reserved.
 # Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2017      IBM Corporation.  All rights reserved.
+# Copyright (c) 2017-2018 Intel, Inc. All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow

orte/mca/ess/pmi/Makefile.am

@@ -2,7 +2,7 @@
 # Copyright (c) 2011      Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2013      Los Alamos National Security, LLC.
 #                         All rights reserved.
-# Copyright (c) 2014      Intel, Inc. All rights reserved
+# Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
 # Copyright (c) 2017      IBM Corporation.  All rights reserved.
 # $COPYRIGHT$
 #

orte/mca/ess/singleton/Makefile.am

@@ -10,6 +10,7 @@
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
 # Copyright (c) 2017      IBM Corporation.  All rights reserved.
+# Copyright (c) 2017-2018 Intel, Inc. All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow

orte/mca/ess/slurm/Makefile.am

@@ -10,6 +10,7 @@
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
 # Copyright (c) 2017      IBM Corporation.  All rights reserved.
+# Copyright (c) 2017-2018 Intel, Inc. All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow

orte/mca/ess/tm/Makefile.am

@@ -10,6 +10,7 @@
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
 # Copyright (c) 2017      IBM Corporation.  All rights reserved.
+# Copyright (c) 2017-2018 Intel, Inc. All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow

orte/mca/ess/tool/Makefile.am

@@ -11,6 +11,7 @@
 #                         All rights reserved.
 # Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2017      IBM Corporation.  All rights reserved.
+# Copyright (c) 2017-2018 Intel, Inc. All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow