Update to latest PMIx master

opal/mca/pmix/pmix3x/pmix/VERSION

@@ -30,7 +30,7 @@ greek=
 # command, or with the date (if "git describe" fails) in the form of
 # "date<date>".
 
-repo_rev=gitd2aa31f
+repo_rev=git967f3e3
 
 # If tarball_version is not empty, it is used as the version string in
 # the tarball filename, regardless of all other versions listed in
@@ -44,7 +44,7 @@ tarball_version=
 
 # The date when this release was created
 
-date="Oct 13, 2016"
+date="Oct 20, 2016"
 
 # The shared library version of each of PMIx's public libraries.
 # These versions are maintained in accordance with the "Library

opal/mca/pmix/pmix3x/pmix/config/pmix.m4

@@ -17,7 +17,7 @@ dnl Copyright (c) 2009      Los Alamos National Security, LLC.  All rights
 dnl                         reserved.
 dnl Copyright (c) 2009-2011 Oak Ridge National Labs.  All rights reserved.
 dnl Copyright (c) 2011-2013 NVIDIA Corporation.  All rights reserved.
-dnl Copyright (c) 2013-2016 Intel, Inc. All rights reserved.
+dnl Copyright (c) 2013-2016 Intel, Inc.  All rights reserved.
 dnl Copyright (c) 2015-2016 Research Organization for Information Science
 dnl                         and Technology (RIST). All rights reserved.
 dnl Copyright (c) 2016      Mellanox Technologies, Inc.
@@ -312,14 +312,14 @@ AC_DEFUN([PMIX_SETUP_CORE],[
     pmix_show_title "Header file tests"
 
     AC_CHECK_HEADERS([arpa/inet.h \
-                      fcntl.h inttypes.h libgen.h \
+                      fcntl.h ifaddrs.h inttypes.h libgen.h \
-                      netinet/in.h \
+                      net/if.h net/uio.h netinet/in.h \
                       stdint.h stddef.h \
                       stdlib.h string.h strings.h \
-                      sys/param.h \
+                      sys/ioctl.h sys/param.h \
-                      sys/select.h sys/socket.h \
+                      sys/select.h sys/socket.h sys/sockio.h \
                       stdarg.h sys/stat.h sys/time.h \
-                      sys/types.h sys/un.h sys/uio.h net/uio.h \
+                      sys/types.h sys/un.h sys/uio.h \
                       sys/wait.h syslog.h \
                       time.h unistd.h dirent.h \
                       crt_externs.h signal.h \

opal/mca/pmix/pmix3x/pmix/config/pmix_check_os_flavors.m4

@@ -0,0 +1,69 @@
+dnl -*- shell-script -*-
+dnl
+dnl Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
+dnl Copyright (c) 2014-2016 Intel, Inc. All rights reserved.
+dnl Copyright (c) 2014      Research Organization for Information Science
+dnl                         and Technology (RIST). All rights reserved.
+dnl
+dnl $COPYRIGHT$
+dnl
+dnl Additional copyrights may follow
+dnl
+dnl $HEADER$
+dnl
+
+# PMIX_CHECK_OS_FLAVOR_SPECIFIC()
+# ----------------------------------------------------
+# Helper macro from PMIX-CHECK-OS-FLAVORS(), below.
+# $1 = macro to look for
+# $2 = suffix of env variable to set with results
+AC_DEFUN([PMIX_CHECK_OS_FLAVOR_SPECIFIC],
+[
+    AC_MSG_CHECKING([$1])
+    AC_COMPILE_IFELSE([AC_LANG_PROGRAM(
+     [[#ifndef $1
+      error: this isnt $1
+      #endif
+     ]])],
+                      [pmix_found_$2=yes],
+                      [pmix_found_$2=no])
+    AC_MSG_RESULT([$pmix_found_$2])
+])dnl
+
+# PMIX_CHECK_OS_FLAVORS()
+# ----------------------------------------------------
+# Try to figure out the various OS flavors out there.
+#
+AC_DEFUN([PMIX_CHECK_OS_FLAVORS],
+[
+    PMIX_CHECK_OS_FLAVOR_SPECIFIC([__NetBSD__], [netbsd])
+    PMIX_CHECK_OS_FLAVOR_SPECIFIC([__FreeBSD__], [freebsd])
+    PMIX_CHECK_OS_FLAVOR_SPECIFIC([__OpenBSD__], [openbsd])
+    PMIX_CHECK_OS_FLAVOR_SPECIFIC([__DragonFly__], [dragonfly])
+    PMIX_CHECK_OS_FLAVOR_SPECIFIC([__386BSD__], [386bsd])
+    PMIX_CHECK_OS_FLAVOR_SPECIFIC([__bsdi__], [bsdi])
+    PMIX_CHECK_OS_FLAVOR_SPECIFIC([__APPLE__], [apple])
+    PMIX_CHECK_OS_FLAVOR_SPECIFIC([__linux__], [linux])
+    PMIX_CHECK_OS_FLAVOR_SPECIFIC([__sun__], [sun])
+    AS_IF([test "$pmix_found_sun" = "no"],
+          PMIX_CHECK_OS_FLAVOR_SPECIFIC([__sun], [sun]))
+
+    AS_IF([test "$pmix_found_sun" = "yes"],
+          [pmix_have_solaris=1
+           CFLAGS="$CFLAGS -D_REENTRANT"
+           CPPFLAGS="$CPPFLAGS -D_REENTRANT"],
+          [pmix_have_solaris=0])
+    AC_DEFINE_UNQUOTED([PMIX_HAVE_SOLARIS],
+                       [$pmix_have_solaris],
+                       [Whether or not we have solaris])
+
+    # check for sockaddr_in (a good sign we have TCP)
+    AC_CHECK_HEADERS([netdb.h netinet/in.h netinet/tcp.h])
+    AC_CHECK_TYPES([struct sockaddr_in],
+                   [pmix_found_sockaddr=yes],
+                   [pmix_found_sockaddr=no],
+                   [AC_INCLUDES_DEFAULT
+#ifdef HAVE_NETINET_IN_H
+#include <netinet/in.h>
+#endif])
+])dnl

opal/mca/pmix/pmix3x/pmix/config/pmix_setup_libevent.m4

@@ -116,10 +116,6 @@ AC_DEFUN([_PMIX_LIBEVENT_EXTERNAL],[
                   AC_MSG_WARN([PMIx requires libevent to be compiled with])
                   AC_MSG_WARN([thread support enabled])
                   AC_MSG_ERROR([Cannot continue])])
-    # Chck if this libevent has the symbol
-    # "libevent_global_shutdown", which will only exist in
-    # libevent version 2.1.1+
-    AC_CHECK_FUNCS([libevent_global_shutdown],[], [])
 
     # Set output variables
     PMIX_EVENT_HEADER="<event.h>"

opal/mca/pmix/pmix3x/pmix/examples/fault.c

@@ -49,15 +49,15 @@ static void notification_fn(size_t evhdlr_registration_id,
 static void op_callbk(pmix_status_t status,
                       void *cbdata)
 {
-    fprintf(stderr, "client: OP CALLBACK CALLED WITH STATUS %d", status);
+    fprintf(stderr, "Client %s:%d OP CALLBACK CALLED WITH STATUS %d\n", myproc.nspace, myproc.rank, status);
 }
 
 static void errhandler_reg_callbk(pmix_status_t status,
                                   size_t errhandler_ref,
                                   void *cbdata)
 {
-    fprintf(stderr, "client: ERRHANDLER REGISTRATION CALLBACK CALLED WITH STATUS %d, ref=%lu",
+    fprintf(stderr, "Client %s:%d ERRHANDLER REGISTRATION CALLBACK CALLED WITH STATUS %d, ref=%lu\n",
-                status, (unsigned long)errhandler_ref);
+               myproc.nspace, myproc.rank, status, (unsigned long)errhandler_ref);
 }
 
 int main(int argc, char **argv)
@@ -76,7 +76,9 @@ int main(int argc, char **argv)
     fprintf(stderr, "Client ns %s rank %d: Running\n", myproc.nspace, myproc.rank);
 
     /* get our universe size */
-    if (PMIX_SUCCESS != (rc = PMIx_Get(&myproc, PMIX_UNIV_SIZE, NULL, 0, &val))) {
+    memcpy(proc.nspace, myproc.nspace, PMIX_MAX_NSLEN);
+    proc.rank = PMIX_RANK_WILDCARD;
+    if (PMIX_SUCCESS != (rc = PMIx_Get(&proc, PMIX_UNIV_SIZE, NULL, 0, &val))) {
         fprintf(stderr, "Client ns %s rank %d: PMIx_Get universe size failed: %d\n", myproc.nspace, myproc.rank, rc);
         goto done;
     }
@@ -100,9 +102,9 @@ int main(int argc, char **argv)
 
     /* rank=0 calls abort */
     if (0 == myproc.rank) {
-        PMIx_Abort(PMIX_ERR_OUT_OF_RESOURCE, "Eat rocks",
+        sleep(2);
-                   &proc, 1);
+        fprintf(stderr, "Client ns %s rank %d: exiting with error\n", myproc.nspace, myproc.rank);
-        fprintf(stderr, "Client ns %s rank %d: Abort called\n", myproc.nspace, myproc.rank);
+        exit(1);
     }
     /* everyone simply waits */
     while (!completed) {

opal/mca/pmix/pmix3x/pmix/include/pmix_version.h.in

@@ -1,7 +1,7 @@
 /*
  * Copyright (c) 2016      Mellanox Technologies, Inc.
  *                         All rights reserved.
- * Copyright (c) 2016      Intel, Inc. All rights reserved.
+ * Copyright (c) 2016      Intel, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow

opal/mca/pmix/pmix3x/pmix/src/client/pmix_client_get.c

@@ -571,10 +571,12 @@ static void _getnbfn(int fd, short flags, void *cbdata)
 
     /* otherwise, the data must be something they "put" */
 #if defined(PMIX_ENABLE_DSTORE) && (PMIX_ENABLE_DSTORE == 1)
-    if (PMIX_SUCCESS == (rc = pmix_dstore_fetch(nptr->nspace, cb->rank, cb->key, &val))) {
+    rc = pmix_dstore_fetch(nptr->nspace, cb->rank, cb->key, &val);
 #else
-    if (PMIX_SUCCESS == (rc = pmix_hash_fetch(&nptr->modex, cb->rank, cb->key, &val))) {
+    rc = pmix_hash_fetch(&nptr->modex, cb->rank, cb->key, &val);
 #endif /* PMIX_ENABLE_DSTORE */
+
+    if ( PMIX_SUCCESS == rc ) {
         pmix_output_verbose(2, pmix_globals.debug_output,
                             "pmix_get[%d]: value retrieved from dstore", __LINE__);
         /* found it - we are in an event, so we can
@@ -607,6 +609,15 @@ static void _getnbfn(int fd, short flags, void *cbdata)
                             "Unable to locally satisfy request for key=%s for rank = %d, namespace = %s",
                             cb->key, cb->rank, cb->nspace);
         cb->checked = true; // flag that we are going to check this again
+    } else if (PMIX_ERR_PROC_ENTRY_NOT_FOUND != rc) {
+        /* errors are fatal */
+        cb->value_cbfunc(rc, NULL, cb->cbdata);
+        /* protect the data */
+        cb->procs = NULL;
+        cb->key = NULL;
+        cb->info = NULL;
+        PMIX_RELEASE(cb);
+        return;
     }
 
   request:

opal/mca/pmix/pmix3x/pmix/src/dstore/pmix_dstore.c

@@ -1,7 +1,7 @@
 /*
  * Copyright (c) 2016      Mellanox Technologies, Inc.
  *                         All rights reserved.
- * Copyright (c) 2016      Intel, Inc. All rights reserved.
+ * Copyright (c) 2016      Intel, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow

opal/mca/pmix/pmix3x/pmix/src/dstore/pmix_dstore.h

@@ -25,6 +25,16 @@ BEGIN_C_DECLS
 int pmix_dstore_init(pmix_info_t info[], size_t ninfo);
 void pmix_dstore_finalize(void);
 int pmix_dstore_store(const char *nspace, pmix_rank_t rank, pmix_kval_t *kv);
+
+/*
+ * Return codes:
+ * - PMIX_ERR_BAD_PARAM - bad parameters - can't proceed.
+ * - PMIX_ERR_FATAL - fatal error
+ * - PMIX_ERR_NOT_FOUND - we have the BLOB for the process but the
+ *   requested key wasn't found there
+ * - PMIX_ERR_PROC_ENTRY_NOT_FOUND - the BLOB for the process wasn't
+ *   found - need to request it from the server.
+ */
 int pmix_dstore_fetch(const char *nspace, pmix_rank_t rank,
                       const char *key, pmix_value_t **kvs);
 int pmix_dstore_patch_env(const char *nspace, char ***env);

opal/mca/pmix/pmix3x/pmix/src/dstore/pmix_esh.c

@@ -3,7 +3,7 @@
  *                         All rights reserved.
  * Copyright (c) 2016      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2016      Intel, Inc. All rights reserved.
+ * Copyright (c) 2016      Intel, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -220,36 +220,55 @@ static inline int _esh_dir_del(const char *path)
     DIR *dir;
     struct dirent *d_ptr;
     struct stat st;
+    pmix_status_t rc = PMIX_SUCCESS;
 
     char name[PMIX_PATH_MAX];
 
     dir = opendir(path);
-    if (dir != NULL) {
+    if (NULL == dir) {
-        while (NULL != (d_ptr = readdir(dir))) {
+        rc = PMIX_ERR_BAD_PARAM;
-            snprintf(name, PMIX_PATH_MAX, "%s/%s", path, d_ptr->d_name);
+        PMIX_ERROR_LOG(rc);
+        return rc;
+    }
 
-            if (lstat(name, &st) < 0){
+    while (NULL != (d_ptr = readdir(dir))) {
-                PMIX_ERROR_LOG(PMIX_ERROR);
+        snprintf(name, PMIX_PATH_MAX, "%s/%s", path, d_ptr->d_name);
-                return PMIX_ERROR;
+        if ( 0 > lstat(name, &st) ){
-            }
+            /* No fatal error here - just log this event
+             * we will hit the error later at rmdir. Keep trying ...
+             */
+            PMIX_ERROR_LOG(PMIX_ERR_NOT_FOUND);
+            continue;
+        }
 
-            if(S_ISDIR(st.st_mode)) {
+        if(S_ISDIR(st.st_mode)) {
-                if(strcmp(d_ptr->d_name, ".") && strcmp(d_ptr->d_name, "..")) {
+            if(strcmp(d_ptr->d_name, ".") && strcmp(d_ptr->d_name, "..")) {
-                    _esh_dir_del(name);
+                rc = _esh_dir_del(name);
+                if( PMIX_SUCCESS != rc ){
+                    /* No fatal error here - just log this event
+                     * we will hit the error later at rmdir. Keep trying ...
+                     */
+                    PMIX_ERROR_LOG(rc);
                 }
             }
-            else {
+        }
-                unlink(name);
+        else {
+            if( 0 > unlink(name) ){
+                /* No fatal error here - just log this event
+                 * we will hit the error later at rmdir. Keep trying ...
+                 */
+                PMIX_ERROR_LOG(PMIX_ERR_NO_PERMISSIONS);
             }
         }
-        closedir(dir);
-    }
-    else {
-        PMIX_ERROR_LOG(PMIX_ERROR);
-        return PMIX_ERROR;
     }
+    closedir(dir);
 
-    return rmdir(path);
+    /* remove the top dir */
+    if( 0 > rmdir(path) ){
+        rc = PMIX_ERR_NO_PERMISSIONS;
+        PMIX_ERROR_LOG(rc);
+    }
+    return rc;
 }
 
 static inline int _esh_tbls_init(void)
@@ -257,6 +276,12 @@ static inline int _esh_tbls_init(void)
     pmix_status_t rc = PMIX_SUCCESS;
     size_t idx;
 
+    /* initial settings */
+    _ns_track_array = NULL;
+    _session_array = NULL;
+    _ns_map_array = NULL;
+
+    /* Setup namespace tracking array */
     if (NULL == (_ns_track_array = PMIX_NEW(pmix_value_array_t))) {
         rc = PMIX_ERR_OUT_OF_RESOURCE;
         PMIX_ERROR_LOG(rc);
@@ -267,34 +292,53 @@ static inline int _esh_tbls_init(void)
         goto err_exit;
     }
 
+    /* Setup sessions table */
     if (NULL == (_session_array = PMIX_NEW(pmix_value_array_t))){
-            rc = PMIX_ERR_OUT_OF_RESOURCE;
+        rc = PMIX_ERR_OUT_OF_RESOURCE;
+        PMIX_ERROR_LOG(rc);
+        goto err_exit;
     }
     if (PMIX_SUCCESS != (rc = pmix_value_array_init(_session_array, sizeof(session_t)))) {
+        PMIX_ERROR_LOG(rc);
         goto err_exit;
     }
     if (PMIX_SUCCESS != (rc = pmix_value_array_set_size(_session_array, ESH_INIT_SESSION_TBL_SIZE))) {
+        PMIX_ERROR_LOG(rc);
         goto err_exit;
     }
     for (idx = 0; idx < ESH_INIT_SESSION_TBL_SIZE; idx++) {
         memset(pmix_value_array_get_item(_session_array, idx), 0, sizeof(session_t));
     }
 
+    /* Setup namespace map array */
     if (NULL == (_ns_map_array = PMIX_NEW(pmix_value_array_t))) {
         rc = PMIX_ERR_OUT_OF_RESOURCE;
+        PMIX_ERROR_LOG(rc);
+        goto err_exit;
     }
     if (PMIX_SUCCESS != (rc = pmix_value_array_init(_ns_map_array, sizeof(ns_map_t)))) {
+        PMIX_ERROR_LOG(rc);
         goto err_exit;
     }
     if (PMIX_SUCCESS != (rc = pmix_value_array_set_size(_ns_map_array, ESH_INIT_NS_MAP_TBL_SIZE))) {
+        PMIX_ERROR_LOG(rc);
         goto err_exit;
     }
     for (idx = 0; idx < ESH_INIT_NS_MAP_TBL_SIZE; idx++) {
         _esh_session_map_clean(pmix_value_array_get_item(_ns_map_array, idx));
     }
 
-
+    return PMIX_SUCCESS;
 err_exit:
+    if (NULL != _ns_track_array) {
+        PMIX_RELEASE(_ns_track_array);
+    }
+    if (NULL != _session_array) {
+        PMIX_RELEASE(_session_array);
+    }
+    if (NULL != _ns_map_array) {
+        PMIX_RELEASE(_ns_map_array);
+    }
     return rc;
 }
 
@@ -400,7 +444,8 @@ static inline int _esh_jobuid_tbl_search(uid_t jobuid, size_t *tbl_idx)
             return PMIX_SUCCESS;
         }
     }
-    return PMIX_ERROR;
+
+    return PMIX_ERR_NOT_FOUND;
 }
 
 static inline int _esh_session_tbl_add(size_t *tbl_idx)
@@ -409,6 +454,7 @@ static inline int _esh_session_tbl_add(size_t *tbl_idx)
     size_t size = pmix_value_array_get_size(_session_array);
     session_t *s_tbl = PMIX_VALUE_ARRAY_GET_BASE(_session_array, session_t);
     session_t *new_sesion;
+    pmix_status_t rc = PMIX_SUCCESS;
 
     for(idx = 0; idx < size; idx ++) {
         if (0 == s_tbl[idx].in_use) {
@@ -419,7 +465,9 @@ static inline int _esh_session_tbl_add(size_t *tbl_idx)
     }
 
     if (NULL == (new_sesion = pmix_value_array_get_item(_session_array, idx))) {
-        return PMIX_ERR_OUT_OF_RESOURCE;
+        rc = PMIX_ERR_OUT_OF_RESOURCE;
+        PMIX_ERROR_LOG(rc);
+        return rc;
     }
     s_tbl[idx].in_use = 1;
     *tbl_idx = idx;
@@ -467,10 +515,12 @@ static inline int _esh_session_init(size_t idx, ns_map_data_t *m, size_t jobuid,
     struct stat st = {0};
     seg_desc_t *seg = NULL;
     session_t *s = &(PMIX_VALUE_ARRAY_GET_ITEM(_session_array, session_t, idx));
+    pmix_status_t rc = PMIX_SUCCESS;
 
     if (NULL == s) {
-        PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
+        rc = PMIX_ERR_BAD_PARAM;
-        return PMIX_ERR_BAD_PARAM;
+        PMIX_ERROR_LOG(rc);
+        return rc;
     }
 
     s->jobuid = jobuid;
@@ -480,8 +530,9 @@ static inline int _esh_session_init(size_t idx, ns_map_data_t *m, size_t jobuid,
     * This situation is quite often, especially in case of direct modex when clients might ask for data
     * simultaneously.*/
     if(0 > asprintf(&s->lockfile, "%s/dstore_sm.lock", s->nspace_path)) {
-        PMIX_ERROR_LOG(PMIX_ERR_OUT_OF_RESOURCE);
+        rc = PMIX_ERR_OUT_OF_RESOURCE;
-        return PMIX_ERR_OUT_OF_RESOURCE;
+        PMIX_ERROR_LOG(rc);
+        return rc;
     }
     PMIX_OUTPUT_VERBOSE((10, pmix_globals.debug_output,
         "%s:%d:%s _lockfile_name: %s", __FILE__, __LINE__, __func__, s->lockfile));
@@ -489,8 +540,9 @@ static inline int _esh_session_init(size_t idx, ns_map_data_t *m, size_t jobuid,
     if (PMIX_PROC_SERVER == pmix_globals.proc_type) {
         if (stat(s->nspace_path, &st) == -1){
             if (0 != mkdir(s->nspace_path, 0770)) {
-                PMIX_ERROR_LOG(PMIX_ERROR);
+                rc = PMIX_ERROR;
-                return PMIX_ERROR;
+                PMIX_ERROR_LOG(rc);
+                return rc;
             }
         }
         s->lockfd = open(s->lockfile, O_CREAT | O_RDWR | O_EXCL, 0600);
@@ -501,41 +553,53 @@ static inline int _esh_session_init(size_t idx, ns_map_data_t *m, size_t jobuid,
             unlink(s->lockfile);
             s->lockfd = open(s->lockfile, O_CREAT | O_RDWR, 0600);
             if (s->lockfd < 0) {
-                PMIX_ERROR_LOG(PMIX_ERROR);
+                rc = PMIX_ERROR;
-                return PMIX_ERROR;
+                PMIX_ERROR_LOG(rc);
+                return rc;
             }
         }
         if (setjobuid > 0){
-            if (chown(s->nspace_path, (uid_t) jobuid, (gid_t) -1) < 0){
+            if (0 > chown(s->nspace_path, (uid_t) jobuid, (gid_t) -1)){
-                PMIX_ERROR_LOG(PMIX_ERROR);
+                rc = PMIX_ERROR;
-                return PMIX_ERROR;
+                PMIX_ERROR_LOG(rc);
+                return rc;
             }
-            if (chown(s->lockfile, (uid_t) jobuid, (gid_t) -1) < 0) {
+            if (0 > chown(s->lockfile, (uid_t) jobuid, (gid_t) -1)) {
-                PMIX_ERROR_LOG(PMIX_ERROR);
+                rc = PMIX_ERROR;
-                return PMIX_ERROR;
+                PMIX_ERROR_LOG(rc);
+                return rc;
             }
-            if (0 != chmod(s->lockfile, S_IRUSR | S_IWGRP | S_IRGRP)) {
+            if (0 > chmod(s->lockfile, S_IRUSR | S_IWGRP | S_IRGRP)) {
-                PMIX_ERROR_LOG(PMIX_ERROR);
+                rc = PMIX_ERROR;
-                return PMIX_ERROR;
+                PMIX_ERROR_LOG(rc);
+                return rc;
             }
         }
         seg = _create_new_segment(INITIAL_SEGMENT, m, 0);
+        if( NULL == seg ){
+            rc = PMIX_ERR_OUT_OF_RESOURCE;
+            PMIX_ERROR_LOG(rc);
+            return rc;
+        }
     }
     else {
         s->lockfd = open(s->lockfile, O_RDONLY);
         if (-1 == s->lockfd) {
-            PMIX_ERROR_LOG(PMIX_ERROR);
+            rc = PMIX_ERROR;
-            return PMIX_ERROR;
+            PMIX_ERROR_LOG(rc);
+            return rc;
         }
         seg = _attach_new_segment(INITIAL_SEGMENT, m, 0);
-    }
+        if( NULL == seg ){
-    if (NULL != seg) {
+            rc = PMIX_ERR_OUT_OF_RESOURCE;
-        s->sm_seg_first = seg;
+            PMIX_ERROR_LOG(rc);
-        s->sm_seg_last = s->sm_seg_first;
+            return rc;
-        return PMIX_SUCCESS;
+        }
     }
 
-    return PMIX_ERR_OUT_OF_RESOURCE;
+    s->sm_seg_first = seg;
+    s->sm_seg_last = s->sm_seg_first;
+    return PMIX_SUCCESS;
 }
 
 static inline void _esh_session_release(session_t *s)
@@ -564,7 +628,7 @@ static inline void _esh_session_release(session_t *s)
 
 int _esh_init(pmix_info_t info[], size_t ninfo)
 {
-    int rc;
+    pmix_status_t rc;
     size_t n;
     char *dstor_tmpdir = NULL;
     size_t tbl_idx;
@@ -578,6 +642,7 @@ int _esh_init(pmix_info_t info[], size_t ninfo)
     _setjobuid = 0;
 
     if (PMIX_SUCCESS != (rc = _esh_tbls_init())) {
+        PMIX_ERROR_LOG(rc);
         goto err_exit;
     }
 
@@ -617,13 +682,20 @@ int _esh_init(pmix_info_t info[], size_t ninfo)
                      *
                      * PMIX_DSTPATH has higher priority than PMIX_SERVER_TMPDIR
                      */
-                    if (NULL != dstor_tmpdir) {
+                    if( PMIX_STRING != info[n].value.type ){
-                        free(dstor_tmpdir);
+                        rc = PMIX_ERR_BAD_PARAM;
+                        PMIX_ERROR_LOG(rc);
+                        goto err_exit;
                     }
-                    dstor_tmpdir = strdup((char*)info[n].value.data.string);
+                    dstor_tmpdir = (char*)info[n].value.data.string;
                     continue;
                 }
                 if (0 == strcmp(PMIX_SERVER_TMPDIR, info[n].key)) {
+                    if( PMIX_STRING != info[n].value.type ){
+                        rc = PMIX_ERR_BAD_PARAM;
+                        PMIX_ERROR_LOG(rc);
+                        goto err_exit;
+                    }
                     if (NULL == dstor_tmpdir) {
                         dstor_tmpdir = (char*)info[n].value.data.string;
                     }
@@ -642,21 +714,24 @@ int _esh_init(pmix_info_t info[], size_t ninfo)
             }
         }
 
-        asprintf(&_base_path, "%s/pmix_dstor_%d", dstor_tmpdir, getpid());
+        rc = asprintf(&_base_path, "%s/pmix_dstor_%d", dstor_tmpdir, getpid());
-        if (NULL == _base_path) {
+        if ((0 > rc) || (NULL == _base_path)) {
-            PMIX_ERROR_LOG(PMIX_ERR_OUT_OF_RESOURCE);
+            rc = PMIX_ERR_OUT_OF_RESOURCE;
-            return PMIX_ERR_OUT_OF_RESOURCE;
+            PMIX_ERROR_LOG(rc);
+            goto err_exit;
         }
 
-        if (stat(_base_path, &st) == -1){
+        if (0 > stat(_base_path, &st)){
-            if (0 != mkdir(_base_path, 0770)) {
+            if (0 > mkdir(_base_path, 0770)) {
-                PMIX_ERROR_LOG(PMIX_ERROR);
+                rc = PMIX_ERR_NO_PERMISSIONS;
+                PMIX_ERROR_LOG(rc);
                 goto err_exit;
             }
         }
         if (_setjobuid > 0) {
             if (chown(_base_path, (uid_t) _jobuid, (gid_t) -1) < 0){
-                PMIX_ERROR_LOG(PMIX_ERROR);
+                rc = PMIX_ERR_NO_PERMISSIONS;
+                PMIX_ERROR_LOG(rc);
                 goto err_exit;
             }
         }
@@ -687,23 +762,24 @@ int _esh_init(pmix_info_t info[], size_t ninfo)
     ns_map = _esh_session_map(pmix_globals.myid.nspace, tbl_idx);
     if (NULL == ns_map) {
         rc = PMIX_ERR_OUT_OF_RESOURCE;
+        PMIX_ERROR_LOG(rc);
         goto err_exit;
     }
 
     if (PMIX_SUCCESS != (rc =_esh_session_init(tbl_idx, ns_map, _jobuid, _setjobuid))) {
+        PMIX_ERROR_LOG(rc);
         goto err_exit;
     }
 
     return PMIX_SUCCESS;
 err_exit:
-    pmix_output(0, "Cannot initialize dstore/esh component");
     return rc;
 }
 
 int _esh_finalize(void)
 {
     struct stat st = {0};
-    pmix_status_t rc;
+    pmix_status_t rc = PMIX_SUCCESS;
 
     PMIX_OUTPUT_VERBOSE((10, pmix_globals.debug_output,
                          "%s:%d:%s", __FILE__, __LINE__, __func__));
@@ -718,7 +794,7 @@ int _esh_finalize(void)
         if(PMIX_PROC_SERVER == pmix_globals.proc_type) {
             if (lstat(_base_path, &st) >= 0){
                 if (PMIX_SUCCESS != (rc = _esh_dir_del(_base_path))) {
-                    PMIX_ERROR_LOG(PMIX_ERROR);
+                    PMIX_ERROR_LOG(rc);
                 }
             }
         }
@@ -726,12 +802,12 @@ int _esh_finalize(void)
         _base_path = NULL;
     }
 
-    return PMIX_SUCCESS;
+    return rc;
 }
 
 int _esh_store(const char *nspace, pmix_rank_t rank, pmix_kval_t *kv)
 {
-    int rc = PMIX_ERROR, lock_rc;
+    pmix_status_t rc = PMIX_SUCCESS, tmp_rc;
     ns_track_elem_t *elem;
     pmix_buffer_t pbkt, xfer;
     ns_seg_info_t ns_info;
@@ -745,13 +821,15 @@ int _esh_store(const char *nspace, pmix_rank_t rank, pmix_kval_t *kv)
                          "%s:%d:%s: for %s:%u",
                          __FILE__, __LINE__, __func__, nspace, rank));
 
-    if (NULL == (ns_map =_esh_session_map_search(nspace))) {
+    if (NULL == (ns_map = _esh_session_map_search(nspace))) {
-        PMIX_ERROR_LOG(PMIX_ERROR);
+        rc = PMIX_ERROR;
-        return PMIX_ERROR;
+        PMIX_ERROR_LOG(rc);
+        return rc;
     }
 
     /* set exclusive lock */
     if (PMIX_SUCCESS != (rc = _ESH_WRLOCK(_ESH_SESSION_lockfd(ns_map->tbl_idx)))) {
+        PMIX_ERROR_LOG(rc);
         return rc;
     }
 
@@ -767,6 +845,7 @@ int _esh_store(const char *nspace, pmix_rank_t rank, pmix_kval_t *kv)
 
     elem = _get_track_elem_for_namespace(ns_map);
     if (NULL == elem) {
+        rc = PMIX_ERR_OUT_OF_RESOURCE;
         PMIX_ERROR_LOG(rc);
         goto err_exit;
     }
@@ -792,6 +871,7 @@ int _esh_store(const char *nspace, pmix_rank_t rank, pmix_kval_t *kv)
         /* put ns's shared segments info to the global meta segment. */
         rc = _put_ns_info_to_initial_segment(ns_map, &elem->meta_seg->seg_info, &elem->data_seg->seg_info);
         if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
             goto err_exit;
         }
     }
@@ -807,28 +887,36 @@ int _esh_store(const char *nspace, pmix_rank_t rank, pmix_kval_t *kv)
     xfer.bytes_used = 0;
 
     rc = _store_data_for_rank(elem, rank, &pbkt);
-
     PMIX_DESTRUCT(&xfer);
     PMIX_DESTRUCT(&pbkt);
 
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        goto err_exit;
+    }
+
     /* unset lock */
-    if (PMIX_SUCCESS != (lock_rc = _ESH_UNLOCK(_ESH_SESSION_lockfd(ns_map->tbl_idx)))) {
+    if (PMIX_SUCCESS != (rc = _ESH_UNLOCK(_ESH_SESSION_lockfd(ns_map->tbl_idx)))) {
-        return lock_rc;
+        PMIX_ERROR_LOG(rc);
     }
     return rc;
 
 err_exit:
     /* unset lock */
-    if (PMIX_SUCCESS != (lock_rc = _ESH_UNLOCK(_ESH_SESSION_lockfd(ns_map->tbl_idx)))) {
+    if (PMIX_SUCCESS != (tmp_rc = _ESH_UNLOCK(_ESH_SESSION_lockfd(ns_map->tbl_idx)))) {
-        return lock_rc;
+        PMIX_ERROR_LOG(tmp_rc);
     }
     return rc;
 }
 
+/*
+ * See return codes description for the corresponding function
+ * in pmix_dstore.h
+ */
 int _esh_fetch(const char *nspace, pmix_rank_t rank, const char *key, pmix_value_t **kvs)
 {
     ns_seg_info_t *ns_info = NULL;
-    int rc = PMIX_ERROR, lock_rc;
+    pmix_status_t rc = PMIX_ERROR, lock_rc;
     ns_track_elem_t *elem;
     rank_meta_info *rinfo = NULL;
     size_t kval_cnt;
@@ -839,20 +927,28 @@ int _esh_fetch(const char *nspace, pmix_rank_t rank, const char *key, pmix_value
     uint32_t nprocs;
     pmix_rank_t cur_rank;
     ns_map_data_t *ns_map = NULL;
+    bool all_ranks_found = true;
+    bool key_found = false;
 
     if (NULL == key) {
         PMIX_OUTPUT_VERBOSE((7, pmix_globals.debug_output,
                              "dstore: Does not support passed parameters"));
-        return PMIX_ERROR;
+        rc = PMIX_ERR_BAD_PARAM;
+        PMIX_ERROR_LOG(rc);
+        return rc;
     }
 
     PMIX_OUTPUT_VERBOSE((10, pmix_globals.debug_output,
                          "%s:%d:%s: for %s:%u look for key %s",
                          __FILE__, __LINE__, __func__, nspace, rank, key));
 
-    if (NULL == (ns_map =_esh_session_map_search(nspace))) {
+    if (NULL == (ns_map = _esh_session_map_search(nspace))) {
-        PMIX_ERROR_LOG(PMIX_ERROR);
+        /* This call is issued from the the client.
-        return PMIX_ERROR;
+         * client must have the session, otherwise the error is fatal.
+         */
+        rc = PMIX_ERR_FATAL;
+        PMIX_ERROR_LOG(rc);
+        return rc;
     }
 
     if (kvs) {
@@ -867,8 +963,11 @@ int _esh_fetch(const char *nspace, pmix_rank_t rank, const char *key, pmix_value
         cur_rank = rank;
     }
 
-    /* set shared lock */
+    /* grab shared lock */
     if (PMIX_SUCCESS != (lock_rc = _ESH_RDLOCK(_ESH_SESSION_lockfd(ns_map->tbl_idx)))) {
+        /* Something wrong with the lock. The error is fatal */
+        rc = PMIX_ERR_FATAL;
+        PMIX_ERROR_LOG(lock_rc);
         return lock_rc;
     }
 
@@ -887,34 +986,35 @@ int _esh_fetch(const char *nspace, pmix_rank_t rank, const char *key, pmix_value
     /* first update local information about initial segments. they can be extended, so then we need to attach to new segments. */
     _update_initial_segment_info(ns_map);
 
-    rc = PMIX_ERROR;
-    /* get information about shared segments per this namespace from the initial segment. */
-
-    //pmix_output(0, "%s:%d:%s: ns %s %s",__FILE__, __LINE__, __func__, nspace, ns_map->name);
-
     ns_info = _get_ns_info_from_initial_segment(ns_map);
     if (NULL == ns_info) {
         /* no data for this namespace is found in the shared memory. */
         PMIX_OUTPUT_VERBOSE((7, pmix_globals.debug_output,
                     "%s:%d:%s:  no data for ns %s is found in the shared memory.",
                     __FILE__, __LINE__, __func__, ns_map->name));
+        rc = PMIX_ERR_PROC_ENTRY_NOT_FOUND;
         goto done;
     }
 
     /* get ns_track_elem_t object for the target namespace from the local track list. */
     elem = _get_track_elem_for_namespace(ns_map);
     if (NULL == elem) {
+        /* Shouldn't happen! */
+        rc = PMIX_ERR_FATAL;
+        PMIX_ERROR_LOG(rc);
         goto done;
     }
+
     /* need to update tracker:
      * attach to shared memory regions for this namespace and store its info locally
      * to operate with address and detach/unlink afterwards. */
     rc = _update_ns_elem(elem, ns_info);
     if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
         goto done;
     }
 
-    /* now we know info about meta segment for this namespace. */
+    /* Now we have the data from meta segment for this namespace. */
     meta_seg = elem->meta_seg;
     data_seg = elem->data_seg;
 
@@ -922,34 +1022,38 @@ int _esh_fetch(const char *nspace, pmix_rank_t rank, const char *key, pmix_value
         if (PMIX_RANK_UNDEF == rank) {
             cur_rank++;
         }
-        /* Then we look for the rank meta info in the shared meta segment. */
+        /* Get the rank meta info in the shared meta segment. */
         rinfo = _get_rank_meta_info(cur_rank, meta_seg);
         if (NULL == rinfo) {
             PMIX_OUTPUT_VERBOSE((7, pmix_globals.debug_output,
                         "%s:%d:%s:  no data for this rank is found in the shared memory. rank %u",
                         __FILE__, __LINE__, __func__, cur_rank));
-            rc = PMIX_ERR_PROC_ENTRY_NOT_FOUND;
+            all_ranks_found = false;
             continue;
         }
         addr = _get_data_region_by_offset(data_seg, rinfo->offset);
         if (NULL == addr) {
-            PMIX_ERROR_LOG(PMIX_ERROR);
+            /* This means that meta-info is broken - error is fatal */
-            rc = PMIX_ERR_PROC_ENTRY_NOT_FOUND;
+            rc = PMIX_ERR_FATAL;
-            continue;
+            PMIX_ERROR_LOG(rc);
+            goto done;
         }
         kval_cnt = rinfo->count;
-        /* TODO: probably PMIX_ERR_NOT_FOUND is a better way but
-         * setting to one initiates wrong next logic for unknown reason */
-        rc = PMIX_ERROR;
 
+        rc = PMIX_SUCCESS;
         while (0 < kval_cnt) {
             /* data is stored in the following format:
-             * key[PMIX_MAX_KEYLEN+1]
+             * key_val_pair {
-             * size_t size
+             *     char key[PMIX_MAX_KEYLEN+1];
-             * byte buffer containing pmix_value, should be loaded to pmix_buffer_t and unpacked.
+             *     size_t size;
-             * next kval pair
+             *     byte_t byte[size]; // should be loaded to pmix_buffer_t and unpacked.
-             * .....
+             * };
-             * EXTENSION slot which has key = EXTENSION_SLOT and a size_t value for offset to next data address for this process.
+             * segment_format {
+             *     key_val_pair kv_array[n];
+             *     EXTENSION slot;
+             * }
+             * EXTENSION slot which has key = EXTENSION_SLOT and a size_t value for offset
+             * to next data address for this process.
              */
             if (0 == strncmp((const char *)addr, ESH_REGION_INVALIDATED, PMIX_MAX_KEYLEN+1)) {
                 PMIX_OUTPUT_VERBOSE((10, pmix_globals.debug_output,
@@ -970,6 +1074,8 @@ int _esh_fetch(const char *nspace, pmix_rank_t rank, const char *key, pmix_value
                     /* go to next item, updating address */
                     addr = _get_data_region_by_offset(data_seg, offset);
                     if (NULL == addr) {
+                        /* This shouldn't happen - error is fatal */
+                        rc = PMIX_ERR_FATAL;
                         PMIX_ERROR_LOG(rc);
                         goto done;
                     }
@@ -995,13 +1101,17 @@ int _esh_fetch(const char *nspace, pmix_rank_t rank, const char *key, pmix_value
                 PMIX_VALUE_CONSTRUCT(&val);
                 if (PMIX_SUCCESS != (rc = pmix_bfrop.unpack(&buffer, &val, &cnt, PMIX_VALUE))) {
                     PMIX_ERROR_LOG(rc);
-                } else if (PMIX_SUCCESS != (rc = pmix_bfrop.copy((void**)kvs, &val, PMIX_VALUE))) {
+                    goto done;
+                }
+                if (PMIX_SUCCESS != (rc = pmix_bfrop.copy((void**)kvs, &val, PMIX_VALUE))) {
                     PMIX_ERROR_LOG(rc);
+                    goto done;
                 }
                 PMIX_VALUE_DESTRUCT(&val);
                 buffer.base_ptr = NULL;
                 buffer.bytes_used = 0;
                 PMIX_DESTRUCT(&buffer);
+                key_found = true;
                 goto done;
             } else {
                 char ckey[PMIX_MAX_KEYLEN+1] = {0};
@@ -1020,43 +1130,62 @@ int _esh_fetch(const char *nspace, pmix_rank_t rank, const char *key, pmix_value
 done:
     /* unset lock */
     if (PMIX_SUCCESS != (lock_rc = _ESH_UNLOCK(_ESH_SESSION_lockfd(ns_map->tbl_idx)))) {
-        return lock_rc;
+        PMIX_ERROR_LOG(lock_rc);
     }
+
+    if( rc != PMIX_SUCCESS ){
+        return rc;
+    }
+
+    if( key_found ){
+        /* the key is found - nothing to do */
+        return PMIX_SUCCESS;
+    }
+
+    if( !all_ranks_found ){
+        /* Not all ranks was found - need to request
+         * all of them and search again
+         */
+        rc = PMIX_ERR_PROC_ENTRY_NOT_FOUND;
+        return rc;
+    }
+    rc = PMIX_ERR_NOT_FOUND;
     return rc;
 }
 
 static int _esh_patch_env(const char *nspace, char ***env)
 {
-    pmix_status_t rc;
+    pmix_status_t rc = PMIX_SUCCESS;
     ns_map_data_t *ns_map = NULL;
 
     if (NULL == _esh_session_map_search) {
-        PMIX_ERROR_LOG(PMIX_ERROR);
+        rc = PMIX_ERR_NOT_AVAILABLE;
-        return PMIX_ERROR;
+        PMIX_ERROR_LOG(rc);
+        return rc;
     }
 
-    if (NULL == (ns_map =_esh_session_map_search(nspace))) {
+    if (NULL == (ns_map = _esh_session_map_search(nspace))) {
-        PMIX_ERROR_LOG(PMIX_ERROR);
+        rc = PMIX_ERR_NOT_AVAILABLE;
-        return PMIX_ERROR;
+        PMIX_ERROR_LOG(rc);
+        return rc;
     }
 
     if ((NULL == _base_path) || (strlen(_base_path) == 0)){
-        PMIX_ERROR_LOG(PMIX_ERROR);
+        rc = PMIX_ERR_NOT_AVAILABLE;
-        return PMIX_ERROR;
+        PMIX_ERROR_LOG(rc);
+        return rc;
     }
 
     if(PMIX_SUCCESS != (rc = pmix_setenv(PMIX_DSTORE_ESH_BASE_PATH,
                                         _ESH_SESSION_path(ns_map->tbl_idx), true, env))){
         PMIX_ERROR_LOG(rc);
-        return rc;
     }
-
+    return rc;
-    return PMIX_SUCCESS;
 }
 
 static int _esh_nspace_add(const char *nspace, pmix_info_t info[], size_t ninfo)
 {
-    int rc;
+    pmix_status_t rc;
     size_t tbl_idx;
     uid_t jobuid = _jobuid;
     char setjobuid = _setjobuid;
@@ -1077,21 +1206,28 @@ static int _esh_nspace_add(const char *nspace, pmix_info_t info[], size_t ninfo)
 
         rc = _esh_session_tbl_add(&tbl_idx);
         if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
             return rc;
         }
         ns_map = _esh_session_map(nspace, tbl_idx);
         if (NULL == ns_map) {
-            return PMIX_ERROR;
+            rc = PMIX_ERROR;
+            PMIX_ERROR_LOG(rc);
+            return rc;
         }
 
         if (PMIX_SUCCESS != (rc =_esh_session_init(tbl_idx, ns_map, jobuid, setjobuid))) {
+            rc = PMIX_ERROR;
+            PMIX_ERROR_LOG(rc);
             return rc;
         }
     }
     else {
         ns_map = _esh_session_map(nspace, tbl_idx);
         if (NULL == ns_map) {
-            return PMIX_ERROR;
+            rc = PMIX_ERROR;
+            PMIX_ERROR_LOG(rc);
+            return rc;
         }
     }
 
@@ -1111,9 +1247,10 @@ static int _esh_nspace_del(const char *nspace)
     PMIX_OUTPUT_VERBOSE((10, pmix_globals.debug_output,
         "%s:%d:%s delete nspace `%s`", __FILE__, __LINE__, __func__, nspace));
 
-    if (NULL == (ns_map_data =_esh_session_map_search(nspace))) {
+    if (NULL == (ns_map_data = _esh_session_map_search(nspace))) {
-        PMIX_ERROR_LOG(PMIX_ERROR);
+        rc = PMIX_ERR_NOT_AVAILABLE;
-        return PMIX_ERROR;
+        PMIX_ERROR_LOG(rc);
+        return rc;
     }
 
     size = pmix_value_array_get_size(_ns_map_array);
@@ -1134,8 +1271,9 @@ static int _esh_nspace_del(const char *nspace)
     if(ns_map_data->track_idx >= 0) {
         trk = pmix_value_array_get_item(_ns_track_array, ns_map_data->track_idx);
         if((ns_map_data->track_idx + 1) > (int)pmix_value_array_get_size(_ns_track_array)) {
-            PMIX_ERROR_LOG(PMIX_ERROR);
+            rc = PMIX_ERR_VALUE_OUT_OF_BOUNDS;
-            return PMIX_ERROR;
+            PMIX_ERROR_LOG(rc);
+            goto exit;
         }
         PMIX_DESTRUCT(trk);
     }
@@ -1149,7 +1287,7 @@ static int _esh_nspace_del(const char *nspace)
             "%s:%d:%s delete session for jobuid: %d", __FILE__, __LINE__, __func__, session_tbl[ns_map_data->tbl_idx].jobuid));
         _esh_session_release(&session_tbl[ns_map_data->tbl_idx]);
      }
-
+exit:
     return rc;
 }
 
@@ -1226,7 +1364,7 @@ static int _pmix_getpagesize(void)
 
 static seg_desc_t *_create_new_segment(segment_type type, const ns_map_data_t *ns_map, uint32_t id)
 {
-    int rc;
+    pmix_status_t rc;
     char file_name[PMIX_PATH_MAX];
     size_t size;
     seg_desc_t *new_seg = NULL;
@@ -1261,37 +1399,37 @@ static seg_desc_t *_create_new_segment(segment_type type, const ns_map_data_t *n
         new_seg->next = NULL;
         new_seg->type = type;
         rc = pmix_sm_segment_create(&new_seg->seg_info, file_name, size);
-        if (PMIX_SUCCESS == rc) {
+        if (PMIX_SUCCESS != rc) {
-            memset(new_seg->seg_info.seg_base_addr, 0, size);
-        } else {
-            free(new_seg);
-            new_seg = NULL;
             PMIX_ERROR_LOG(rc);
+            goto err_exit;
         }
+        memset(new_seg->seg_info.seg_base_addr, 0, size);
 
         if (_setjobuid > 0){
-            rc = PMIX_SUCCESS;
+            rc = PMIX_ERR_PERM;
-            if (chown(file_name, (uid_t) _jobuid, (gid_t) -1) < 0){
+            if (0 > chown(file_name, (uid_t) _jobuid, (gid_t) -1)){
-                PMIX_ERROR_LOG(PMIX_ERROR);
+                PMIX_ERROR_LOG(rc);
-                rc = PMIX_ERROR;
+                goto err_exit;
             }
             /* set the mode as required */
-            if (0 != chmod(file_name, S_IRUSR | S_IRGRP | S_IWGRP )) {
+            if (0 > chmod(file_name, S_IRUSR | S_IRGRP | S_IWGRP )) {
-                PMIX_ERROR_LOG(PMIX_ERROR);
+                PMIX_ERROR_LOG(rc);
-                rc = PMIX_ERROR;
+                goto err_exit;
-            }
-            if (rc != PMIX_SUCCESS) {
-                free(new_seg);
-                new_seg = NULL;
             }
         }
     }
     return new_seg;
+
+err_exit:
+    if( NULL != new_seg ){
+        free(new_seg);
+    }
+    return NULL;
 }
 
 static seg_desc_t *_attach_new_segment(segment_type type, const ns_map_data_t *ns_map, uint32_t id)
 {
-    int rc;
+    pmix_status_t rc;
     seg_desc_t *new_seg = NULL;
     new_seg = (seg_desc_t*)malloc(sizeof(seg_desc_t));
     new_seg->id = id;
@@ -1337,14 +1475,16 @@ static int _update_ns_elem(ns_track_elem_t *ns_elem, ns_seg_info_t *info)
     seg_desc_t *seg, *tmp = NULL;
     size_t i, offs;
     ns_map_data_t *ns_map = NULL;
+    pmix_status_t rc;
 
     PMIX_OUTPUT_VERBOSE((10, pmix_globals.debug_output,
                          "%s:%d:%s",
                          __FILE__, __LINE__, __func__));
 
-    if (NULL == (ns_map =_esh_session_map_search(info->ns_map.name))) {
+    if (NULL == (ns_map = _esh_session_map_search(info->ns_map.name))) {
-        PMIX_ERROR_LOG(PMIX_ERROR);
+        rc = PMIX_ERR_NOT_AVAILABLE;
-        return PMIX_ERROR;
+        PMIX_ERROR_LOG(rc);
+        return rc;
     }
 
     tmp = ns_elem->meta_seg;
@@ -1358,12 +1498,20 @@ static int _update_ns_elem(ns_track_elem_t *ns_elem, ns_seg_info_t *info)
     for (i = ns_elem->num_meta_seg; i < info->num_meta_seg; i++) {
         if (PMIX_PROC_SERVER == pmix_globals.proc_type) {
             seg = _create_new_segment(NS_META_SEGMENT, &info->ns_map, i);
+            if (NULL == seg) {
+                rc = PMIX_ERR_OUT_OF_RESOURCE;
+                PMIX_ERROR_LOG(rc);
+                return rc;
+            }
         } else {
             seg = _attach_new_segment(NS_META_SEGMENT, &info->ns_map, i);
+            if (NULL == seg) {
+                rc = PMIX_ERR_NOT_AVAILABLE;
+                PMIX_ERROR_LOG(rc);
+                return rc;
+            }
         }
-        if (NULL == seg) {
+
-            return PMIX_ERROR;
-        }
         if (NULL == tmp) {
             ns_elem->meta_seg = seg;
         } else {
@@ -1383,16 +1531,22 @@ static int _update_ns_elem(ns_track_elem_t *ns_elem, ns_seg_info_t *info)
     for (i = ns_elem->num_data_seg; i < info->num_data_seg; i++) {
         if (PMIX_PROC_SERVER == pmix_globals.proc_type) {
             seg = _create_new_segment(NS_DATA_SEGMENT, &info->ns_map, i);
-            if (seg) {
+            if (NULL == seg) {
-                offs = sizeof(size_t);//shift on offset field itself
+                rc = PMIX_ERR_OUT_OF_RESOURCE;
-                memcpy(seg->seg_info.seg_base_addr, &offs, sizeof(size_t));
+                PMIX_ERROR_LOG(rc);
+                return rc;
             }
+            offs = sizeof(size_t);//shift on offset field itself
+            memcpy(seg->seg_info.seg_base_addr, &offs, sizeof(size_t));
         } else {
             seg = _attach_new_segment(NS_DATA_SEGMENT, &info->ns_map, i);
+            if (NULL == seg) {
+                rc = PMIX_ERR_NOT_AVAILABLE;
+                PMIX_ERROR_LOG(rc);
+                return rc;
+            }
         }
-        if (NULL == seg) {
+
-            return PMIX_ERROR;
-        }
         if (NULL == tmp) {
             ns_elem->data_seg = seg;
         } else {
@@ -1430,6 +1584,7 @@ static int _put_ns_info_to_initial_segment(const ns_map_data_t *ns_map, pmix_sm_
     size_t num_elems;
     num_elems = *((size_t*)(_ESH_SESSION_sm_seg_last(ns_map->tbl_idx)->seg_info.seg_base_addr));
     seg_desc_t *last_seg = _ESH_SESSION_sm_seg_last(ns_map->tbl_idx);
+    pmix_status_t rc;
 
     PMIX_OUTPUT_VERBOSE((10, pmix_globals.debug_output,
                          "%s:%d:%s", __FILE__, __LINE__, __func__));
@@ -1437,8 +1592,9 @@ static int _put_ns_info_to_initial_segment(const ns_map_data_t *ns_map, pmix_sm_
     if (_max_ns_num == num_elems) {
         num_elems = 0;
         if (NULL == (last_seg = extend_segment(last_seg, ns_map))) {
-            PMIX_ERROR_LOG(PMIX_ERROR);
+            rc = PMIX_ERROR;
-            return PMIX_ERROR;
+            PMIX_ERROR_LOG(rc);
+            return rc;
         }
         /* mark previous segment as full */
         size_t full = 1;
@@ -1481,7 +1637,7 @@ static void _update_initial_segment_info(const ns_map_data_t *ns_map)
 /* this function will be used by clients to get ns data from the initial segment and add them to the tracker list */
 static ns_seg_info_t *_get_ns_info_from_initial_segment(const ns_map_data_t *ns_map)
 {
-    int rc;
+    pmix_status_t rc;
     size_t i;
     seg_desc_t *tmp;
     ns_seg_info_t *elem, *cur_elem;
@@ -1708,8 +1864,8 @@ static uint8_t *_get_data_region_by_offset(seg_desc_t *segdesc, size_t offset)
             dataaddr = tmp->seg_info.seg_base_addr + rel_offset;
         }
         tmp = tmp->next;
-    }
+    } while (NULL != tmp && NULL == dataaddr);
-    while (NULL != tmp && NULL == dataaddr);
+
     return dataaddr;
 }
 
@@ -1739,6 +1895,7 @@ static int put_empty_ext_slot(seg_desc_t *dataseg)
     global_offset = get_free_offset(dataseg);
     rel_offset = global_offset % _data_segment_size;
     if (rel_offset + EXT_SLOT_SIZE > _data_segment_size) {
+        PMIX_ERROR_LOG(PMIX_ERROR);
         return PMIX_ERROR;
     }
     addr = _get_data_region_by_offset(dataseg, global_offset);
@@ -1826,7 +1983,7 @@ static int pmix_sm_store(ns_track_elem_t *ns_info, pmix_rank_t rank, pmix_kval_t
 {
     size_t offset, size, kval_cnt;
     pmix_buffer_t *buffer;
-    int rc;
+    pmix_status_t rc;
     seg_desc_t *datadesc;
     uint8_t *addr;
 
@@ -2020,7 +2177,7 @@ static int pmix_sm_store(ns_track_elem_t *ns_info, pmix_rank_t rank, pmix_kval_t
 
 static int _store_data_for_rank(ns_track_elem_t *ns_info, pmix_rank_t rank, pmix_buffer_t *buf)
 {
-    int rc;
+    pmix_status_t rc;
     int32_t cnt;
 
     pmix_buffer_t *bptr;
@@ -2038,8 +2195,9 @@ static int _store_data_for_rank(ns_track_elem_t *ns_info, pmix_rank_t rank, pmix
     datadesc = ns_info->data_seg;
 
     if (NULL == datadesc || NULL == metadesc) {
-        PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
+        rc = PMIX_ERR_BAD_PARAM;
-        return PMIX_ERROR;
+        PMIX_ERROR_LOG(rc);
+        return rc;
     }
 
     num_elems = *((size_t*)(metadesc->seg_info.seg_base_addr));
@@ -2067,6 +2225,7 @@ static int _store_data_for_rank(ns_track_elem_t *ns_info, pmix_rank_t rank, pmix
                                 "pmix: unpacked key %s", kp->key);
             if (PMIX_SUCCESS != (rc = pmix_sm_store(ns_info, rank, kp, &rinfo, data_exist))) {
                 PMIX_ERROR_LOG(rc);
+                return rc;
             }
             PMIX_RELEASE(kp); // maintain acctg - hash_store does a retain
             cnt = 1;
@@ -2082,6 +2241,7 @@ static int _store_data_for_rank(ns_track_elem_t *ns_info, pmix_rank_t rank, pmix
     }
     if (PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER != rc) {
         PMIX_ERROR_LOG(rc);
+        /* TODO: should we error-exit here? */
     } else {
         rc = PMIX_SUCCESS;
     }

opal/mca/pmix/pmix3x/pmix/src/dstore/pmix_esh.h

@@ -1,7 +1,7 @@
 /*
  * Copyright (c) 2015-2016 Mellanox Technologies, Inc.
  *                         All rights reserved.
- * Copyright (c) 2016      Intel, Inc. All rights reserved.
+ * Copyright (c) 2016      Intel, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow

opal/mca/pmix/pmix3x/pmix/src/include/types.h

@@ -9,7 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2014-2015 Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2016 Intel, Inc.  All rights reserved.
  * Copyright (c) 2015      Mellanox Technologies, Inc.
  *                         All rights reserved.
  * $COPYRIGHT$
@@ -254,8 +254,4 @@ typedef struct event pmix_event_t;
 
 #define pmix_event_loop(b, fg) event_base_loop((b), (fg))
 
-#ifdef HAVE_LIBEVENT_GLOBAL_SHUTDOWN
-#define pmix_libevent_global_shutdown() libevent_global_shutdown()
-#endif
-
 #endif /* PMIX_TYPES_H */

opal/mca/pmix/pmix3x/pmix/src/mca/pdl/pdlopen/configure.m4

@@ -4,7 +4,7 @@
 # Copyright (c) 2016      Research Organization for Information Science
 #                         and Technology (RIST). All rights reserved.
 #
-# Copyright (c) 2016      Intel, Inc. All rights reserved.
+# Copyright (c) 2016      Intel, Inc.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow

opal/mca/pmix/pmix3x/pmix/src/mca/pif/Makefile.am

@@ -0,0 +1,28 @@
+#
+# Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2016      Intel, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# main library setup
+noinst_LTLIBRARIES = libmca_pif.la
+libmca_pif_la_SOURCES =
+
+# local files
+headers = pif.h
+libmca_pif_la_SOURCES += $(headers)
+
+# Conditionally install the header files
+if WANT_INSTALL_HEADERS
+pmixdir = $(pmixincludedir)/pmix/mca/pif
+nobase_pmix_HEADERS = $(headers)
+endif
+
+include base/Makefile.am
+
+distclean-local:
+	rm -f base/static-components.h

opal/mca/pmix/pmix3x/pmix/src/mca/pif/base/Makefile.am

@@ -0,0 +1,15 @@
+#
+# Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2016      Intel, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+headers += \
+        base/base.h
+
+libmca_pif_la_SOURCES += \
+        base/pif_base_components.c

opal/mca/pmix/pmix3x/pmix/src/mca/pif/base/base.h

@@ -0,0 +1,30 @@
+/*
+ * Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2016      Intel, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+
+#ifndef PMIX_PIF_BASE_H
+#define PMIX_PIF_BASE_H
+
+#include "pmix_config.h"
+
+#include "src/mca/base/pmix_mca_base_framework.h"
+
+#include "src/mca/pif/pif.h"
+
+BEGIN_C_DECLS
+
+/*
+ * Framework declaration
+ */
+PMIX_EXPORT extern pmix_mca_base_framework_t pmix_pif_base_framework;
+
+END_C_DECLS
+
+#endif /* PMIX_BASE_PIF_H */

opal/mca/pmix/pmix3x/pmix/src/mca/pif/base/pif_base_components.c

@@ -0,0 +1,104 @@
+/*
+ * Copyright (c) 2010-2013 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2016 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+
+#include "pmix_config.h"
+
+#include "pmix_common.h"
+#include "src/util/output.h"
+#include "src/mca/mca.h"
+#include "src/mca/pif/pif.h"
+#include "src/mca/pif/base/base.h"
+#include "src/mca/pif/base/static-components.h"
+
+/* instantiate the global list of interfaces */
+pmix_list_t pmix_if_list = {{0}};
+bool pmix_if_do_not_resolve = false;
+bool pmix_if_retain_loopback = false;
+
+static int pmix_pif_base_register (pmix_mca_base_register_flag_t flags);
+static int pmix_pif_base_open (pmix_mca_base_open_flag_t flags);
+static int pmix_pif_base_close(void);
+static void pmix_pif_construct(pmix_pif_t *obj);
+
+static bool frameopen = false;
+
+/* instance the pmix_pif_t object */
+PMIX_CLASS_INSTANCE(pmix_pif_t, pmix_list_item_t, pmix_pif_construct, NULL);
+
+PMIX_MCA_BASE_FRAMEWORK_DECLARE(pmix, pif, NULL, pmix_pif_base_register, pmix_pif_base_open, pmix_pif_base_close,
+                                mca_pif_base_static_components, 0);
+
+static int pmix_pif_base_register (pmix_mca_base_register_flag_t flags)
+{
+    pmix_if_do_not_resolve = false;
+    (void) pmix_mca_base_framework_var_register (&pmix_pif_base_framework, "do_not_resolve",
+                                                 "If nonzero, do not attempt to resolve interfaces",
+                                                 PMIX_MCA_BASE_VAR_TYPE_BOOL, NULL, 0, PMIX_MCA_BASE_VAR_FLAG_SETTABLE,
+                                                 PMIX_INFO_LVL_9, PMIX_MCA_BASE_VAR_SCOPE_ALL_EQ,
+                                                 &pmix_if_do_not_resolve);
+
+    pmix_if_retain_loopback = false;
+    (void) pmix_mca_base_framework_var_register (&pmix_pif_base_framework, "retain_loopback",
+                                                 "If nonzero, retain loopback interfaces",
+                                                 PMIX_MCA_BASE_VAR_TYPE_BOOL, NULL, 0, PMIX_MCA_BASE_VAR_FLAG_SETTABLE,
+                                                 PMIX_INFO_LVL_9, PMIX_MCA_BASE_VAR_SCOPE_ALL_EQ,
+                                                 &pmix_if_retain_loopback);
+
+    return PMIX_SUCCESS;
+}
+
+
+static int pmix_pif_base_open (pmix_mca_base_open_flag_t flags)
+{
+    if (frameopen) {
+        return PMIX_SUCCESS;
+    }
+    frameopen = true;
+
+    /* setup the global list */
+    PMIX_CONSTRUCT(&pmix_if_list, pmix_list_t);
+
+    return pmix_mca_base_framework_components_open(&pmix_pif_base_framework, flags);
+}
+
+
+static int pmix_pif_base_close(void)
+{
+    pmix_list_item_t *item;
+
+    if (!frameopen) {
+        return PMIX_SUCCESS;
+    }
+
+    while (NULL != (item = pmix_list_remove_first(&pmix_if_list))) {
+        PMIX_RELEASE(item);
+    }
+    PMIX_DESTRUCT(&pmix_if_list);
+
+    return pmix_mca_base_framework_components_close(&pmix_pif_base_framework, NULL);
+}
+
+static void pmix_pif_construct(pmix_pif_t *obj)
+{
+    memset(obj->if_name, 0, sizeof(obj->if_name));
+    obj->if_index = -1;
+    obj->if_kernel_index = (uint16_t) -1;
+    obj->af_family = PF_UNSPEC;
+    obj->if_flags = 0;
+    obj->if_speed = 0;
+    memset(&obj->if_addr, 0, sizeof(obj->if_addr));
+    obj->if_mask = 0;
+    obj->if_bandwidth = 0;
+    memset(obj->if_mac, 0, sizeof(obj->if_mac));
+    obj->ifmtu = 0;
+}

opal/mca/pmix/pmix3x/pmix/src/mca/pif/bsdx_ipv4/Makefile.am

@@ -0,0 +1,13 @@
+#
+# Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2016      Intel, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+noinst_LTLIBRARIES = libmca_pif_bsdx_ipv4.la
+
+libmca_pif_bsdx_ipv4_la_SOURCES = pif_bsdx.c

opal/mca/pmix/pmix3x/pmix/src/mca/pif/bsdx_ipv4/configure.m4

@@ -0,0 +1,46 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2015      Research Organization for Information Science
+#                         and Technology (RIST). All rights reserved.
+# Copyright (c) 2016      Intel, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+AC_DEFUN([MCA_pmix_pif_bsdx_ipv4_COMPILE_MODE], [
+    AC_MSG_CHECKING([for MCA component $1:$2 compile mode])
+    $3="static"
+    AC_MSG_RESULT([$$3])
+])
+
+# MCA_pif_config_CONFIG(action-if-can-compile,
+#                      [action-if-cant-compile])
+# ------------------------------------------------
+AC_DEFUN([MCA_pmix_pif_bsdx_ipv4_CONFIG], [
+    AC_CONFIG_FILES([src/mca/pif/bsdx_ipv4/Makefile])
+
+    AC_REQUIRE([PMIX_CHECK_OS_FLAVORS])
+
+    # If we found struct sockaddr and we're on any of the BSDs, we're
+    # happy.  I.e., this:
+    #if defined(__NetBSD__) || defined(__FreeBSD__) || \
+    #    defined(__OpenBSD__) || defined(__DragonFly__)
+    AC_MSG_CHECKING([struct sockaddr])
+    AS_IF([test "$pmix_found_sockaddr" = "yes"],
+          [AC_MSG_RESULT([yes (cached)])
+           AC_MSG_CHECKING([NetBSD, FreeBSD, OpenBSD, or DragonFly])
+           AS_IF([test "$pmix_found_netbsd" = "yes" || \
+                  test "$pmix_found_freebsd" = "yes" || \
+                  test "$pmix_found_openbsd" = "yes" || \
+                  test "$pmix_found_dragonfly" = "yes"],
+                 [AC_MSG_RESULT([yes])
+                  $1],
+                 [AC_MSG_RESULT([no])
+                  $2])],
+          [AC_MSG_RESULT([no (cached)])
+           $2])
+])

opal/mca/pmix/pmix3x/pmix/src/mca/pif/bsdx_ipv4/pif_bsdx.c

@@ -0,0 +1,193 @@
+/*
+ * Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2010      Oracle and/or its affiliates.  All rights reserved.
+ * Copyright (c) 2016      Intel, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "pmix_config.h"
+#include "pmix_common.h"
+
+#include <string.h>
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#include <errno.h>
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_SOCKET_H
+#include <sys/socket.h>
+#endif
+#ifdef HAVE_SYS_SOCKIO_H
+#include <sys/sockio.h>
+#endif
+#ifdef HAVE_SYS_IOCTL_H
+#include <sys/ioctl.h>
+#endif
+#ifdef HAVE_NETINET_IN_H
+#include <netinet/in.h>
+#endif
+#ifdef HAVE_ARPA_INET_H
+#include <arpa/inet.h>
+#endif
+#ifdef HAVE_NET_IF_H
+#if defined(__APPLE__) && defined(_LP64)
+/* Apple engineering suggested using options align=power as a
+   workaround for a bug in OS X 10.4 (Tiger) that prevented ioctl(...,
+   SIOCGIFCONF, ...) from working properly in 64 bit mode on Power PC.
+   It turns out that the underlying issue is the size of struct
+   ifconf, which the kernel expects to be 12 and natural 64 bit
+   alignment would make 16.  The same bug appears in 64 bit mode on
+   Intel macs, but align=power is a no-op there, so instead, use the
+   pack pragma to instruct the compiler to pack on 4 byte words, which
+   has the same effect as align=power for our needs and works on both
+   Intel and Power PC Macs. */
+#pragma pack(push,4)
+#endif
+#include <net/if.h>
+#if defined(__APPLE__) && defined(_LP64)
+#pragma pack(pop)
+#endif
+#endif
+#ifdef HAVE_NETDB_H
+#include <netdb.h>
+#endif
+#ifdef HAVE_IFADDRS_H
+#include <ifaddrs.h>
+#endif
+
+#include "src/util/output.h"
+#include "src/util/pif.h"
+#include "src/mca/pif/pif.h"
+#include "src/mca/pif/base/base.h"
+
+static int if_bsdx_open(void);
+
+/* Supports specific flavors of BSD:
+ * NetBSD
+ * FreeBSD
+ * OpenBSD
+ * DragonFly
+ */
+pmix_pif_base_component_t mca_pif_bsdx_ipv4_component = {
+    /* First, the mca_component_t struct containing meta information
+       about the component itself */
+    .base = {
+        PMIX_PIF_BASE_VERSION_2_0_0,
+
+        /* Component name and version */
+        "bsdx_ipv4",
+        PMIX_MAJOR_VERSION,
+        PMIX_MINOR_VERSION,
+        PMIX_RELEASE_VERSION,
+
+        /* Component open and close functions */
+        if_bsdx_open,
+        NULL
+    },
+    .data = {
+        /* This component is checkpointable */
+        PMIX_MCA_BASE_METADATA_PARAM_CHECKPOINT
+    },
+};
+
+/* convert a netmask (in network byte order) to CIDR notation */
+static int prefix (uint32_t netmask)
+{
+    uint32_t mask = ntohl(netmask);
+    int plen = 0;
+
+    if (0 == mask) {
+        plen = 32;
+    } else {
+        while ((mask % 2) == 0) {
+            plen += 1;
+            mask /= 2;
+        }
+    }
+
+    return (32 - plen);
+}
+
+/* configure using getifaddrs(3) */
+static int if_bsdx_open(void)
+{
+    struct ifaddrs **ifadd_list;
+    struct ifaddrs *cur_ifaddrs;
+    struct sockaddr_in* sin_addr;
+
+    /*
+     * the manpage claims that getifaddrs() allocates the memory,
+     * and freeifaddrs() is later used to release the allocated memory.
+     * however, without this malloc the call to getifaddrs() segfaults
+     */
+    ifadd_list = (struct ifaddrs **) malloc(sizeof(struct ifaddrs*));
+
+    /* create the linked list of ifaddrs structs */
+    if (getifaddrs(ifadd_list) < 0) {
+        pmix_output(0, "pmix_ifinit: getifaddrs() failed with error=%d\n",
+                    errno);
+        return PMIX_ERROR;
+    }
+
+    for (cur_ifaddrs = *ifadd_list; NULL != cur_ifaddrs;
+         cur_ifaddrs = cur_ifaddrs->ifa_next) {
+        pmix_pif_t *intf;
+        struct in_addr a4;
+
+        /* skip non- af_inet interface addresses */
+        if (AF_INET != cur_ifaddrs->ifa_addr->sa_family) {
+            continue;
+        }
+
+        /* skip interface if it is down (IFF_UP not set) */
+        if (0 == (cur_ifaddrs->ifa_flags & IFF_UP)) {
+            continue;
+        }
+
+        /* skip interface if it is a loopback device (IFF_LOOPBACK set) */
+        if (!pmix_if_retain_loopback && 0 != (cur_ifaddrs->ifa_flags & IFF_LOOPBACK)) {
+            continue;
+        }
+
+        /* or if it is a point-to-point interface */
+        /* TODO: do we really skip p2p? */
+        if (0 != (cur_ifaddrs->ifa_flags & IFF_POINTOPOINT)) {
+            continue;
+        }
+
+        sin_addr = (struct sockaddr_in *) cur_ifaddrs->ifa_addr;
+
+        intf = PMIX_NEW(pmix_pif_t);
+        if (NULL == intf) {
+            pmix_output(0, "pmix_ifinit: unable to allocate %d bytes\n",
+                        (int) sizeof(pmix_pif_t));
+            return PMIX_ERR_OUT_OF_RESOURCE;
+        }
+        intf->af_family = AF_INET;
+
+        /* fill values into the pmix_pif_t */
+        memcpy(&a4, &(sin_addr->sin_addr), sizeof(struct in_addr));
+
+        strncpy(intf->if_name, cur_ifaddrs->ifa_name, IF_NAMESIZE);
+        intf->if_index = pmix_list_get_size(&pmix_if_list) + 1;
+        ((struct sockaddr_in*) &intf->if_addr)->sin_addr = a4;
+        ((struct sockaddr_in*) &intf->if_addr)->sin_family = AF_INET;
+        ((struct sockaddr_in*) &intf->if_addr)->sin_len =  cur_ifaddrs->ifa_addr->sa_len;
+
+        intf->if_mask = prefix( sin_addr->sin_addr.s_addr);
+        intf->if_flags = cur_ifaddrs->ifa_flags;
+
+        intf->if_kernel_index =
+            (uint16_t) if_nametoindex(cur_ifaddrs->ifa_name);
+
+        pmix_list_append(&pmix_if_list, &(intf->super));
+    }   /*  of for loop over ifaddrs list */
+
+    return PMIX_SUCCESS;
+}

opal/mca/pmix/pmix3x/pmix/src/mca/pif/bsdx_ipv6/Makefile.am

@@ -0,0 +1,13 @@
+#
+# Copyright (c) 2010     Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2016      Intel, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+noinst_LTLIBRARIES = libmca_pif_bsdx_ipv6.la
+
+libmca_pif_bsdx_ipv6_la_SOURCES = pif_bsdx_ipv6.c

opal/mca/pmix/pmix3x/pmix/src/mca/pif/bsdx_ipv6/configure.m4

@@ -0,0 +1,48 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2015      Research Organization for Information Science
+#                         and Technology (RIST). All rights reserved.
+# Copyright (c) 2016      Intel, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+AC_DEFUN([MCA_pmix_pif_bsdx_ipv6_COMPILE_MODE], [
+    AC_MSG_CHECKING([for MCA component $1:$2 compile mode])
+    $3="static"
+    AC_MSG_RESULT([$$3])
+])
+
+# MCA_pif_config_CONFIG(action-if-can-compile,
+#                        [action-if-cant-compile])
+# ------------------------------------------------
+AC_DEFUN([MCA_pmix_pif_bsdx_ipv6_CONFIG], [
+    AC_CONFIG_FILES([src/mca/pif/bsdx_ipv6/Makefile])
+
+    AC_REQUIRE([PMIX_CHECK_OS_FLAVORS])
+
+    # If we found struct sockaddr and we're on any of the BSDs, we're
+    # happy.  I.e., this:
+    #if defined( __NetBSD__) || defined(__OpenBSD__) || defined(__FreeBSD__) || \
+    #             defined(__386BSD__) || defined(__bsdi__) || defined(__APPLE__)
+    AC_MSG_CHECKING([struct sockaddr])
+    AS_IF([test "$pmix_found_sockaddr" = "yes"],
+          [AC_MSG_RESULT([yes (cached)])
+           AC_MSG_CHECKING([some flavor of BSD])
+           AS_IF([test "$pmix_found_netbsd" = "yes" || \
+                  test "$pmix_found_freebsd" = "yes" || \
+                  test "$pmix_found_openbsd" = "yes" || \
+                  test "$pmix_found_386bsd" = "yes" || \
+                  test "$pmix_found_bsdi" = "yes" ||
+                  test "$pmix_found_apple" = "yes"],
+                 [AC_MSG_RESULT([yes])
+                  $1],
+                 [AC_MSG_RESULT([no])
+                  $2])],
+          [AC_MSG_RESULT([no (cached)])
+           $2])
+])dnl

opal/mca/pmix/pmix3x/pmix/src/mca/pif/bsdx_ipv6/pif_bsdx_ipv6.c

@@ -0,0 +1,229 @@
+/*
+ * Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2010      Oracle and/or its affiliates.  All rights reserved.
+ * Copyright (c) 2016      Intel, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "pmix_config.h"
+#include "pmix_common.h"
+
+#include <string.h>
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#include <errno.h>
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_SOCKET_H
+#include <sys/socket.h>
+#endif
+#ifdef HAVE_SYS_SOCKIO_H
+#include <sys/sockio.h>
+#endif
+#ifdef HAVE_SYS_IOCTL_H
+#include <sys/ioctl.h>
+#endif
+#ifdef HAVE_NETINET_IN_H
+#include <netinet/in.h>
+#endif
+#ifdef HAVE_ARPA_INET_H
+#include <arpa/inet.h>
+#endif
+#ifdef HAVE_NET_IF_H
+#if defined(__APPLE__) && defined(_LP64)
+/* Apple engineering suggested using options align=power as a
+   workaround for a bug in OS X 10.4 (Tiger) that prevented ioctl(...,
+   SIOCGIFCONF, ...) from working properly in 64 bit mode on Power PC.
+   It turns out that the underlying issue is the size of struct
+   ifconf, which the kernel expects to be 12 and natural 64 bit
+   alignment would make 16.  The same bug appears in 64 bit mode on
+   Intel macs, but align=power is a no-op there, so instead, use the
+   pack pragma to instruct the compiler to pack on 4 byte words, which
+   has the same effect as align=power for our needs and works on both
+   Intel and Power PC Macs. */
+#pragma pack(push,4)
+#endif
+#include <net/if.h>
+#if defined(__APPLE__) && defined(_LP64)
+#pragma pack(pop)
+#endif
+#endif
+#ifdef HAVE_NETDB_H
+#include <netdb.h>
+#endif
+#ifdef HAVE_IFADDRS_H
+#include <ifaddrs.h>
+#endif
+
+#include "src/util/output.h"
+#include "src/util/pif.h"
+#include "src/mca/pif/pif.h"
+#include "src/mca/pif/base/base.h"
+
+static int if_bsdx_ipv6_open(void);
+
+/* Discovers IPv6 interfaces for:
+ *
+ * NetBSD
+ * OpenBSD
+ * FreeBSD
+ * 386BSD
+ * bsdi
+ * Apple
+ */
+pmix_pif_base_component_t mca_pif_bsdx_ipv6_component = {
+    /* First, the mca_component_t struct containing meta information
+       about the component itself */
+    .base = {
+        PMIX_PIF_BASE_VERSION_2_0_0,
+
+        /* Component name and version */
+        "bsdx_ipv6",
+        PMIX_MAJOR_VERSION,
+        PMIX_MINOR_VERSION,
+        PMIX_RELEASE_VERSION,
+
+        /* Component open and close functions */
+        if_bsdx_ipv6_open,
+        NULL
+    },
+    .data = {
+        /* This component is checkpointable */
+        PMIX_MCA_BASE_METADATA_PARAM_CHECKPOINT
+    },
+};
+
+/* configure using getifaddrs(3) */
+static int if_bsdx_ipv6_open(void)
+{
+    struct ifaddrs **ifadd_list;
+    struct ifaddrs *cur_ifaddrs;
+    struct sockaddr_in6* sin_addr;
+
+    pmix_output_verbose(1, pmix_pif_base_framework.framework_output,
+                        "searching for IPv6 interfaces");
+
+    /*
+     * the manpage claims that getifaddrs() allocates the memory,
+     * and freeifaddrs() is later used to release the allocated memory.
+     * however, without this malloc the call to getifaddrs() segfaults
+     */
+    ifadd_list = (struct ifaddrs **) malloc(sizeof(struct ifaddrs*));
+
+    /* create the linked list of ifaddrs structs */
+    if (getifaddrs(ifadd_list) < 0) {
+        pmix_output(0, "pmix_ifinit: getifaddrs() failed with error=%d\n",
+                    errno);
+        free(ifadd_list);
+        return PMIX_ERROR;
+    }
+
+    for (cur_ifaddrs = *ifadd_list; NULL != cur_ifaddrs;
+         cur_ifaddrs = cur_ifaddrs->ifa_next) {
+        pmix_pif_t *intf;
+        struct in6_addr a6;
+
+        /* skip non-ipv6 interface addresses */
+        if (AF_INET6 != cur_ifaddrs->ifa_addr->sa_family) {
+            pmix_output_verbose(1, pmix_pif_base_framework.framework_output,
+                                "skipping non-ipv6 interface %s[%d].\n",
+                                cur_ifaddrs->ifa_name, (int)cur_ifaddrs->ifa_addr->sa_family);
+            continue;
+        }
+
+        /* skip interface if it is down (IFF_UP not set) */
+        if (0 == (cur_ifaddrs->ifa_flags & IFF_UP)) {
+            pmix_output_verbose(1, pmix_pif_base_framework.framework_output,
+                                "skipping non-up interface %s.\n", cur_ifaddrs->ifa_name);
+            continue;
+        }
+
+        /* skip interface if it is a loopback device (IFF_LOOPBACK set) */
+        if (!pmix_if_retain_loopback && 0 != (cur_ifaddrs->ifa_flags & IFF_LOOPBACK)) {
+            pmix_output_verbose(1, pmix_pif_base_framework.framework_output,
+                                "skipping loopback interface %s.\n", cur_ifaddrs->ifa_name);
+            continue;
+        }
+
+        /* or if it is a point-to-point interface */
+        /* TODO: do we really skip p2p? */
+        if (0!= (cur_ifaddrs->ifa_flags & IFF_POINTOPOINT)) {
+            pmix_output_verbose(1, pmix_pif_base_framework.framework_output,
+                                "skipping p2p interface %s.\n", cur_ifaddrs->ifa_name);
+            continue;
+        }
+
+        sin_addr = (struct sockaddr_in6 *) cur_ifaddrs->ifa_addr;
+
+        /*
+         * skip IPv6 address starting with fe80:, as this is supposed to be
+         * link-local scope. sockaddr_in6->sin6_scope_id doesn't always work
+         * TODO: test whether scope id is set to a sensible value on
+         * linux and/or bsd (including osx)
+         *
+         * MacOSX: fe80::... has a scope of 0, but ifconfig -a shows
+         * a scope of 4 on that particular machine,
+         * so the scope returned by getifaddrs() isn't working properly
+         */
+
+        if ((IN6_IS_ADDR_LINKLOCAL (&sin_addr->sin6_addr))) {
+            pmix_output_verbose(1, pmix_pif_base_framework.framework_output,
+                                "skipping link-local ipv6 address on interface "
+                                "%s with scope %d.\n",
+                                cur_ifaddrs->ifa_name, sin_addr->sin6_scope_id);
+            continue;
+        }
+
+        if (0 < pmix_output_get_verbosity(pmix_pif_base_framework.framework_output)) {
+            char *addr_name = (char *) malloc(48*sizeof(char));
+            inet_ntop(AF_INET6, &sin_addr->sin6_addr, addr_name, 48*sizeof(char));
+            pmix_output(0, "ipv6 capable interface %s discovered, address %s.\n",
+                        cur_ifaddrs->ifa_name, addr_name);
+            free(addr_name);
+        }
+
+        /* fill values into the pmix_pif_t */
+        memcpy(&a6, &(sin_addr->sin6_addr), sizeof(struct in6_addr));
+
+        intf = PMIX_NEW(pmix_pif_t);
+        if (NULL == intf) {
+            pmix_output(0, "pmix_ifinit: unable to allocate %lu bytes\n",
+                        sizeof(pmix_pif_t));
+            free(ifadd_list);
+            return PMIX_ERR_OUT_OF_RESOURCE;
+        }
+        intf->af_family = AF_INET6;
+        strncpy(intf->if_name, cur_ifaddrs->ifa_name, IF_NAMESIZE);
+        intf->if_index = pmix_list_get_size(&pmix_if_list) + 1;
+        ((struct sockaddr_in6*) &intf->if_addr)->sin6_addr = a6;
+        ((struct sockaddr_in6*) &intf->if_addr)->sin6_family = AF_INET6;
+
+        /* since every scope != 0 is ignored, we just set the scope to 0 */
+        ((struct sockaddr_in6*) &intf->if_addr)->sin6_scope_id = 0;
+
+        /*
+         * hardcoded netmask, adrian says that's ok
+         */
+        intf->if_mask = 64;
+        intf->if_flags = cur_ifaddrs->ifa_flags;
+
+        /*
+         * FIXME: figure out how to gain access to the kernel index
+         * (or create our own), getifaddrs() does not contain such
+         * data
+         */
+        intf->if_kernel_index =
+            (uint16_t) if_nametoindex(cur_ifaddrs->ifa_name);
+        pmix_list_append(&pmix_if_list, &(intf->super));
+    }   /*  of for loop over ifaddrs list */
+
+    free(ifadd_list);
+
+    return PMIX_SUCCESS;
+}

opal/mca/pmix/pmix3x/pmix/src/mca/pif/linux_ipv6/Makefile.am

@@ -0,0 +1,13 @@
+#
+# Copyright (c) 2010     Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2016      Intel, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+noinst_LTLIBRARIES = libmca_pif_linux_ipv6.la
+
+libmca_pif_linux_ipv6_la_SOURCES = pif_linux_ipv6.c

opal/mca/pmix/pmix3x/pmix/src/mca/pif/linux_ipv6/configure.m4

@@ -0,0 +1,36 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2015      Research Organization for Information Science
+#                         and Technology (RIST). All rights reserved.
+# Copyright (c) 2016      Intel, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+AC_DEFUN([MCA_pmix_pif_linux_ipv6_COMPILE_MODE], [
+    AC_MSG_CHECKING([for MCA component $1:$2 compile mode])
+    $3="static"
+    AC_MSG_RESULT([$$3])
+])
+
+# MCA_pif_config_CONFIG(action-if-can-compile,
+#                        [action-if-cant-compile])
+# ------------------------------------------------
+AC_DEFUN([MCA_pmix_pif_linux_ipv6_CONFIG], [
+    AC_CONFIG_FILES([src/mca/pif/linux_ipv6/Makefile])
+
+    AC_REQUIRE([PMIX_CHECK_OS_FLAVORS])
+
+    AC_MSG_CHECKING([if we are on Linux with TCP])
+    # If we have struct sockaddr and we're on Linux, then we're
+    # happy.
+    AS_IF([test "$pmix_found_sockaddr" = "yes" && test "$pmix_found_linux" = "yes"],
+          [AC_MSG_RESULT([yes])
+           $1],
+          [AC_MSG_RESULT([no])
+           $2])
+])dnl

opal/mca/pmix/pmix3x/pmix/src/mca/pif/linux_ipv6/pif_linux_ipv6.c

@@ -0,0 +1,173 @@
+/*
+ * Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2010      Oracle and/or its affiliates.  All rights reserved.
+ * Copyright (c) 2016      Intel, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "pmix_config.h"
+#include "pmix_common.h"
+
+#include <string.h>
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#include <errno.h>
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_SOCKET_H
+#include <sys/socket.h>
+#endif
+#ifdef HAVE_SYS_SOCKIO_H
+#include <sys/sockio.h>
+#endif
+#ifdef HAVE_SYS_IOCTL_H
+#include <sys/ioctl.h>
+#endif
+#ifdef HAVE_NETINET_IN_H
+#include <netinet/in.h>
+#endif
+#ifdef HAVE_ARPA_INET_H
+#include <arpa/inet.h>
+#endif
+#ifdef HAVE_NET_IF_H
+#if defined(__APPLE__) && defined(_LP64)
+/* Apple engineering suggested using options align=power as a
+   workaround for a bug in OS X 10.4 (Tiger) that prevented ioctl(...,
+   SIOCGIFCONF, ...) from working properly in 64 bit mode on Power PC.
+   It turns out that the underlying issue is the size of struct
+   ifconf, which the kernel expects to be 12 and natural 64 bit
+   alignment would make 16.  The same bug appears in 64 bit mode on
+   Intel macs, but align=power is a no-op there, so instead, use the
+   pack pragma to instruct the compiler to pack on 4 byte words, which
+   has the same effect as align=power for our needs and works on both
+   Intel and Power PC Macs. */
+#pragma pack(push,4)
+#endif
+#include <net/if.h>
+#if defined(__APPLE__) && defined(_LP64)
+#pragma pack(pop)
+#endif
+#endif
+#ifdef HAVE_NETDB_H
+#include <netdb.h>
+#endif
+#ifdef HAVE_IFADDRS_H
+#include <ifaddrs.h>
+#endif
+
+#include "src/util/output.h"
+#include "src/util/pif.h"
+#include "src/mca/pif/pif.h"
+#include "src/mca/pif/base/base.h"
+
+static int if_linux_ipv6_open(void);
+
+/* Discovers Linux IPv6 interfaces */
+pmix_pif_base_component_t mca_pif_linux_ipv6_component = {
+    /* First, the mca_component_t struct containing meta information
+       about the component itself */
+    .base = {
+        PMIX_PIF_BASE_VERSION_2_0_0,
+
+        /* Component name and version */
+        "linux_ipv6",
+        PMIX_MAJOR_VERSION,
+        PMIX_MINOR_VERSION,
+        PMIX_RELEASE_VERSION,
+
+        /* Component open and close functions */
+        if_linux_ipv6_open,
+        NULL
+    },
+    .data = {
+        /* This component is checkpointable */
+        PMIX_MCA_BASE_METADATA_PARAM_CHECKPOINT
+    },
+};
+
+/* configure using getifaddrs(3) */
+static int if_linux_ipv6_open(void)
+{
+    FILE *f;
+    if ((f = fopen("/proc/net/if_inet6", "r"))) {
+        char ifname[IF_NAMESIZE];
+        unsigned int idx, pfxlen, scope, dadstat;
+        struct in6_addr a6;
+        int iter;
+        uint32_t flag;
+        unsigned int addrbyte[16];
+
+        while (fscanf(f, "%2x%2x%2x%2x%2x%2x%2x%2x%2x%2x%2x%2x%2x%2x%2x%2x %x %x %x %x %20s\n",
+                      &addrbyte[0], &addrbyte[1], &addrbyte[2], &addrbyte[3],
+                      &addrbyte[4], &addrbyte[5], &addrbyte[6], &addrbyte[7],
+                      &addrbyte[8], &addrbyte[9], &addrbyte[10], &addrbyte[11],
+                      &addrbyte[12], &addrbyte[13], &addrbyte[14], &addrbyte[15],
+                      &idx, &pfxlen, &scope, &dadstat, ifname) != EOF) {
+            pmix_pif_t *intf;
+
+            pmix_output_verbose(1, pmix_pif_base_framework.framework_output,
+                                "found interface %2x%2x:%2x%2x:%2x%2x:%2x%2x:%2x%2x:%2x%2x:%2x%2x:%2x%2x scope %x\n",
+                                addrbyte[0], addrbyte[1], addrbyte[2], addrbyte[3],
+                                addrbyte[4], addrbyte[5], addrbyte[6], addrbyte[7],
+                                addrbyte[8], addrbyte[9], addrbyte[10], addrbyte[11],
+                                addrbyte[12], addrbyte[13], addrbyte[14], addrbyte[15], scope);
+
+            /* we don't want any other scope less than link-local */
+            if (scope < 0x20) {
+                pmix_output_verbose(1, pmix_pif_base_framework.framework_output,
+                                    "skipping interface %2x%2x:%2x%2x:%2x%2x:%2x%2x:%2x%2x:%2x%2x:%2x%2x:%2x%2x scope %x\n",
+                                    addrbyte[0], addrbyte[1], addrbyte[2], addrbyte[3],
+                                    addrbyte[4], addrbyte[5], addrbyte[6], addrbyte[7],
+                                    addrbyte[8], addrbyte[9], addrbyte[10], addrbyte[11],
+                                    addrbyte[12], addrbyte[13], addrbyte[14], addrbyte[15], scope);
+                continue;
+            }
+
+            intf = PMIX_NEW(pmix_pif_t);
+            if (NULL == intf) {
+                pmix_output(0, "pmix_ifinit: unable to allocate %lu bytes\n",
+                            (unsigned long)sizeof(pmix_pif_t));
+                fclose(f);
+                return PMIX_ERR_OUT_OF_RESOURCE;
+            }
+            intf->af_family = AF_INET6;
+
+            for (iter = 0; iter < 16; iter++) {
+                a6.s6_addr[iter] = addrbyte[iter];
+            }
+
+            /* now construct the pmix_pif_t */
+            strncpy(intf->if_name, ifname, IF_NAMESIZE);
+            intf->if_index = pmix_list_get_size(&pmix_if_list)+1;
+            intf->if_kernel_index = (uint16_t) idx;
+            ((struct sockaddr_in6*) &intf->if_addr)->sin6_addr = a6;
+            ((struct sockaddr_in6*) &intf->if_addr)->sin6_family = AF_INET6;
+            ((struct sockaddr_in6*) &intf->if_addr)->sin6_scope_id = scope;
+            intf->if_mask = pfxlen;
+            if (PMIX_SUCCESS == pmix_ifindextoflags(pmix_ifnametoindex (ifname), &flag)) {
+                intf->if_flags = flag;
+            } else {
+                intf->if_flags = IFF_UP;
+            }
+
+            /* copy new interface information to heap and append
+               to list */
+            pmix_list_append(&pmix_if_list, &(intf->super));
+            pmix_output_verbose(1, pmix_pif_base_framework.framework_output,
+                                "added interface %2x%2x:%2x%2x:%2x%2x:%2x%2x:%2x%2x:%2x%2x:%2x%2x:%2x%2x\n",
+                                addrbyte[0], addrbyte[1], addrbyte[2], addrbyte[3],
+                                addrbyte[4], addrbyte[5], addrbyte[6], addrbyte[7],
+                                addrbyte[8], addrbyte[9], addrbyte[10], addrbyte[11],
+                                addrbyte[12], addrbyte[13], addrbyte[14], addrbyte[15]);
+        } /* of while */
+        fclose(f);
+    }
+
+    return PMIX_SUCCESS;
+}

opal/mca/pmix/pmix3x/pmix/src/mca/pif/pif.h

@@ -0,0 +1,137 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2010-2013 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2016      Intel, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef PMIX_MCA_PIF_PIF_H
+#define PMIX_MCA_PIF_PIF_H
+
+#include "pmix_config.h"
+
+#include <string.h>
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#include <errno.h>
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_SOCKET_H
+#include <sys/socket.h>
+#endif
+#ifdef HAVE_SYS_SOCKIO_H
+#include <sys/sockio.h>
+#endif
+#ifdef HAVE_SYS_IOCTL_H
+#include <sys/ioctl.h>
+#endif
+#ifdef HAVE_NETINET_IN_H
+#include <netinet/in.h>
+#endif
+#ifdef HAVE_ARPA_INET_H
+#include <arpa/inet.h>
+#endif
+#ifdef HAVE_NET_IF_H
+#if defined(__APPLE__) && defined(_LP64)
+/* Apple engineering suggested using options align=power as a
+   workaround for a bug in OS X 10.4 (Tiger) that prevented ioctl(...,
+   SIOCGIFCONF, ...) from working properly in 64 bit mode on Power PC.
+   It turns out that the underlying issue is the size of struct
+   ifconf, which the kernel expects to be 12 and natural 64 bit
+   alignment would make 16.  The same bug appears in 64 bit mode on
+   Intel macs, but align=power is a no-op there, so instead, use the
+   pack pragma to instruct the compiler to pack on 4 byte words, which
+   has the same effect as align=power for our needs and works on both
+   Intel and Power PC Macs. */
+#pragma pack(push,4)
+#endif
+#include <net/if.h>
+#if defined(__APPLE__) && defined(_LP64)
+#pragma pack(pop)
+#endif
+#endif
+#ifdef HAVE_NETDB_H
+#include <netdb.h>
+#endif
+#ifdef HAVE_IFADDRS_H
+#include <ifaddrs.h>
+#endif
+
+#include "src/util/pif.h"
+#include "src/mca/mca.h"
+#include "src/mca/base/base.h"
+
+BEGIN_C_DECLS
+
+/*
+ * Define INADDR_NONE if we don't have it.  Solaris is the only system
+ * where I have found that it does not exist, and the man page for
+ * inet_addr() says that it returns -1 upon failure.  On Linux and
+ * other systems with INADDR_NONE, it's just a #define to -1 anyway.
+ * So just #define it to -1 here if it doesn't already exist.
+ */
+
+#if !defined(INADDR_NONE)
+#define INADDR_NONE -1
+#endif
+
+#define DEFAULT_NUMBER_INTERFACES 10
+#define MAX_PIFCONF_SIZE 10 * 1024 * 1024
+
+
+typedef struct pmix_pif_t {
+    pmix_list_item_t     super;
+    char                if_name[IF_NAMESIZE];
+    int                 if_index;
+    uint16_t            if_kernel_index;
+    uint16_t            af_family;
+    int                 if_flags;
+    int                 if_speed;
+    struct sockaddr_storage  if_addr;
+    uint32_t            if_mask;
+    uint32_t            if_bandwidth;
+    uint8_t             if_mac[6];
+    int                 ifmtu; /* Can't use if_mtu because of a
+                                  #define collision on some BSDs */
+} pmix_pif_t;
+PMIX_CLASS_DECLARATION(pmix_pif_t);
+
+
+/* "global" list of available interfaces */
+extern pmix_list_t pmix_if_list;
+
+/* global flags */
+extern bool pmix_if_do_not_resolve;
+extern bool pmix_if_retain_loopback;
+
+/**
+ * Structure for if components.
+ */
+struct pmix_pif_base_component_2_0_0_t {
+    /** MCA base component */
+    pmix_mca_base_component_t base;
+    /** MCA base data */
+    pmix_mca_base_component_data_t data;
+};
+/**
+ * Convenience typedef
+ */
+typedef struct pmix_pif_base_component_2_0_0_t pmix_pif_base_component_t;
+
+/*
+ * Macro for use in components that are of type pif
+ */
+#define PMIX_PIF_BASE_VERSION_2_0_0 \
+    PMIX_MCA_BASE_VERSION_1_0_0("pif", 2, 0, 0)
+
+END_C_DECLS
+
+#endif /* PMIX_MCA_PIF_PIF_H */

opal/mca/pmix/pmix3x/pmix/src/mca/pif/posix_ipv4/Makefile.am

@@ -0,0 +1,13 @@
+#
+# Copyright (c) 2010     Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2016      Intel, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+noinst_LTLIBRARIES = libmca_pif_posix_ipv4.la
+
+libmca_pif_posix_ipv4_la_SOURCES = pif_posix.c

opal/mca/pmix/pmix3x/pmix/src/mca/pif/posix_ipv4/configure.m4

@@ -0,0 +1,55 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2015      Research Organization for Information Science
+#                         and Technology (RIST). All rights reserved.
+# Copyright (c) 2016      Intel, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+AC_DEFUN([MCA_pmix_pif_posix_ipv4_COMPILE_MODE], [
+    AC_MSG_CHECKING([for MCA component $1:$2 compile mode])
+    $3="static"
+    AC_MSG_RESULT([$$3])
+])
+
+# MCA_pif_config_CONFIG(action-if-can-compile,
+#                        [action-if-cant-compile])
+# ------------------------------------------------
+AC_DEFUN([MCA_pmix_pif_posix_ipv4_CONFIG], [
+    AC_CONFIG_FILES([src/mca/pif/posix_ipv4/Makefile])
+
+    PMIX_VAR_SCOPE_PUSH([pmix_pif_posix_ipv4_happy])
+    pmix_pif_posix_ipv4_happy=no
+
+    AC_REQUIRE([PMIX_CHECK_OS_FLAVORS])
+
+    # If we found struct sockaddr and we're NOT on most of the BSDs,
+    # we're happy.  I.e., if posix but not:
+    #if defined(__NetBSD__) || defined(__FreeBSD__) || \
+    #    defined(__OpenBSD__) || defined(__DragonFly__)
+    AC_MSG_CHECKING([struct sockaddr])
+    AS_IF([test "$pmix_found_sockaddr" = "yes"],
+          [AC_MSG_RESULT([yes (cached)])
+           AC_MSG_CHECKING([not NetBSD, FreeBSD, OpenBSD, or DragonFly])
+           AS_IF([test "$pmix_found_netbsd" = "no" && test "$pmix_found_freebsd" = "no" && test "$pmix_found_openbsd" = "no" && test "$pmix_found_dragonfly" = "no"],
+                 [AC_MSG_RESULT([yes])
+                  pmix_pif_posix_ipv4_happy=yes],
+                 [AC_MSG_RESULT([no])]
+                )],
+          [AC_MSG_RESULT([no (cached)])]
+         )
+
+    AS_IF([test "$pmix_pif_posix_ipv4_happy" = "yes"],
+          [AC_CHECK_MEMBERS([struct ifreq.ifr_hwaddr], [], [],
+                           [[#include <net/if.h>]])
+           AC_CHECK_MEMBERS([struct ifreq.ifr_mtu], [], [],
+                           [[#include <net/if.h>]])
+          ])
+
+    AS_IF([test "$pmix_pif_posix_ipv4_happy" = "yes"], [$1], [$2]);
+])dnl

opal/mca/pmix/pmix3x/pmix/src/mca/pif/posix_ipv4/pif_posix.c

@@ -0,0 +1,343 @@
+/*
+ * Copyright (c) 2010-2013 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2010      Oracle and/or its affiliates.  All rights reserved.
+ * Copyright (c) 2013      The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2015-2016 Intel, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "pmix_config.h"
+#include "pmix_common.h"
+
+#include <string.h>
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#include <errno.h>
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_SOCKET_H
+#include <sys/socket.h>
+#endif
+#ifdef HAVE_SYS_SOCKIO_H
+#include <sys/sockio.h>
+#endif
+#ifdef HAVE_SYS_IOCTL_H
+#include <sys/ioctl.h>
+#endif
+#ifdef HAVE_NETINET_IN_H
+#include <netinet/in.h>
+#endif
+#ifdef HAVE_ARPA_INET_H
+#include <arpa/inet.h>
+#endif
+#ifdef HAVE_NET_IF_H
+#if defined(__APPLE__) && defined(_LP64)
+/* Apple engineering suggested using options align=power as a
+   workaround for a bug in OS X 10.4 (Tiger) that prevented ioctl(...,
+   SIOCGIFCONF, ...) from working properly in 64 bit mode on Power PC.
+   It turns out that the underlying issue is the size of struct
+   ifconf, which the kernel expects to be 12 and natural 64 bit
+   alignment would make 16.  The same bug appears in 64 bit mode on
+   Intel macs, but align=power is a no-op there, so instead, use the
+   pack pragma to instruct the compiler to pack on 4 byte words, which
+   has the same effect as align=power for our needs and works on both
+   Intel and Power PC Macs. */
+#pragma pack(push,4)
+#endif
+#include <net/if.h>
+#if defined(__APPLE__) && defined(_LP64)
+#pragma pack(pop)
+#endif
+#endif
+#ifdef HAVE_NETDB_H
+#include <netdb.h>
+#endif
+#ifdef HAVE_IFADDRS_H
+#include <ifaddrs.h>
+#endif
+
+#include "src/util/output.h"
+#include "src/util/pif.h"
+#include "src/mca/pif/pif.h"
+#include "src/mca/pif/base/base.h"
+
+static int if_posix_open(void);
+
+/* Supports all flavors of posix except those
+ * BSD-flavors supported elsewhere
+ */
+pmix_pif_base_component_t mca_pif_posix_ipv4_component = {
+    /* First, the mca_component_t struct containing meta information
+       about the component itself */
+    .base = {
+        PMIX_PIF_BASE_VERSION_2_0_0,
+
+        /* Component name and version */
+        "posix_ipv4",
+        PMIX_MAJOR_VERSION,
+        PMIX_MINOR_VERSION,
+        PMIX_RELEASE_VERSION,
+
+        /* Component open and close functions */
+        if_posix_open,
+        NULL
+    },
+    .data = {
+        /* This component is checkpointable */
+        PMIX_MCA_BASE_METADATA_PARAM_CHECKPOINT
+    },
+};
+
+/* convert a netmask (in network byte order) to CIDR notation */
+static int prefix (uint32_t netmask)
+{
+    uint32_t mask = ntohl(netmask);
+    int plen = 0;
+
+    if (0 == mask) {
+        plen = 32;
+    } else {
+        while ((mask % 2) == 0) {
+            plen += 1;
+            mask /= 2;
+        }
+    }
+
+    return (32 - plen);
+}
+
+/* configure using getifaddrs(3) */
+static int if_posix_open(void)
+{
+    int sd;
+    int lastlen, rem;
+    char *ptr;
+    struct ifconf ifconf;
+    int ifc_len;
+    bool successful_locate = false;
+
+    /* Create the internet socket to test with.  Must use AF_INET;
+       using AF_UNSPEC or AF_INET6 will cause everything to
+       fail. */
+    if ((sd = socket(AF_INET, SOCK_DGRAM, 0)) < 0) {
+        pmix_output(0, "pmix_ifinit: socket() failed with errno=%d\n",
+                    errno);
+        return PMIX_ERROR;
+    }
+
+    /*
+     * Get Network Interface configuration
+     *
+     * Some notes on the behavior of ioctl(..., SIOCGIFCONF,...)
+     * when not enough space is allocated for all the entries.
+     *
+     * - Solaris returns -1, errno EINVAL if there is not enough
+     *   space
+     * - OS X returns 0, sets .ifc_len to the space used by the
+     *   by the entries that did fit.
+     * - Linux returns 0, sets .ifc_len to the space required to
+     *   hold all the entries (although it only writes what will
+     *   fit in the buffer of .ifc_len passed to the function).
+     * - FreeBSD returns 0, sets .ifc_len to 0.
+     *
+     * Everyone else seems to do one of the four.
+     */
+    lastlen = 0;
+    ifc_len = sizeof(struct ifreq) * DEFAULT_NUMBER_INTERFACES;
+    do {
+        ifconf.ifc_len = ifc_len;
+        ifconf.ifc_req = malloc(ifc_len);
+        if (NULL == ifconf.ifc_req) {
+            close(sd);
+            return PMIX_ERROR;
+        }
+
+        /* initialize the memory so valgrind and purify won't
+         * complain.  Since this isn't performance critical, just
+         * always memset.
+         */
+        memset(ifconf.ifc_req, 0, ifconf.ifc_len);
+
+        if (ioctl(sd, SIOCGIFCONF, &ifconf) < 0) {
+            /* if we got an einval, we probably don't have enough
+               space.  so we'll fall down and try to expand our
+               space */
+            if (errno != EINVAL && lastlen != 0) {
+                pmix_output(0, "pmix_ifinit: ioctl(SIOCGIFCONF) \
+                            failed with errno=%d",
+                            errno);
+                free(ifconf.ifc_req);
+                close(sd);
+                return PMIX_ERROR;
+            }
+        } else {
+            /* if ifc_len is 0 or different than what we set it to
+               at call to ioctl, try again with a bigger buffer.
+               else stop */
+            if (ifconf.ifc_len == lastlen && ifconf.ifc_len > 0) {
+                /* we didn't expand.  we're done */
+                successful_locate = true;
+                break;
+            }
+            lastlen = ifconf.ifc_len;
+        }
+
+        /* Yes, we overflowed (or had an EINVAL on the ioctl).
+           Loop back around and try again with a bigger buffer */
+        free(ifconf.ifc_req);
+        ifc_len = (ifc_len == 0) ? 1 : ifc_len * 2;
+    } while (ifc_len < MAX_PIFCONF_SIZE);
+    if (!successful_locate) {
+        pmix_output(0, "pmix_ifinit: unable to find network interfaces.");
+        close(sd);
+        return PMIX_ERROR;
+    }
+
+    /*
+     * Setup indexes
+     */
+    ptr = (char*) ifconf.ifc_req;
+    rem = ifconf.ifc_len;
+
+    /* loop through all interfaces */
+    while (rem > 0) {
+        struct ifreq* ifr = (struct ifreq*) ptr;
+        pmix_pif_t *intf;
+        int length;
+
+        /* compute offset for entries */
+#ifdef HAVE_STRUCT_SOCKADDR_SA_LEN
+        length = sizeof(struct sockaddr);
+
+        if (ifr->ifr_addr.sa_len > length) {
+            length = ifr->ifr_addr.sa_len;
+        }
+
+        length += sizeof(ifr->ifr_name);
+#else
+        length = sizeof(struct ifreq);
+#endif
+
+        rem -= length;
+        ptr += length;
+
+        /* see if we like this entry */
+        if (AF_INET != ifr->ifr_addr.sa_family) {
+            continue;
+        }
+
+        if (ioctl(sd, SIOCGIFFLAGS, ifr) < 0) {
+            pmix_output(0, "pmix_ifinit: ioctl(SIOCGIFFLAGS) failed with errno=%d", errno);
+            continue;
+        }
+        if ((ifr->ifr_flags & IFF_UP) == 0) {
+            continue;
+        }
+#ifdef IFF_SLAVE
+        /* Is this a slave to a load balancer or bonded channel?
+           If so, don't use it -- pick up the master instead */
+        if ((ifr->ifr_flags & IFF_SLAVE) != 0) {
+            continue;
+        }
+#endif
+#if 0
+        if (!pmix_if_retain_loopback && (ifr->ifr_flags & IFF_LOOPBACK) != 0) {
+            continue;
+        }
+#endif
+
+        intf = PMIX_NEW(pmix_pif_t);
+        if (NULL == intf) {
+            pmix_output(0, "pmix_ifinit: unable to allocated %lu bytes\n", (unsigned long)sizeof(pmix_pif_t));
+            free(ifconf.ifc_req);
+            close(sd);
+            return PMIX_ERR_OUT_OF_RESOURCE;
+        }
+        intf->af_family = AF_INET;
+
+        /* copy entry over into our data structure */
+        memset(intf->if_name, 0, sizeof(intf->if_name));
+        strncpy(intf->if_name, ifr->ifr_name, sizeof(intf->if_name) - 1);
+        intf->if_flags = ifr->ifr_flags;
+
+        /* every new address gets its own internal if_index */
+        intf->if_index = pmix_list_get_size(&pmix_if_list)+1;
+
+        pmix_output_verbose(1, pmix_pif_base_framework.framework_output,
+                            "found interface %s", intf->if_name);
+
+        /* assign the kernel index to distinguish different NICs */
+#ifndef SIOCGIFINDEX
+        intf->if_kernel_index = intf->if_index;
+#else
+        if (ioctl(sd, SIOCGIFINDEX, ifr) < 0) {
+            pmix_output(0,"pmix_ifinit: ioctl(SIOCGIFINDEX) failed with errno=%d", errno);
+            PMIX_RELEASE(intf);
+            continue;
+        }
+#if defined(ifr_ifindex)
+        intf->if_kernel_index = ifr->ifr_ifindex;
+#elif defined(ifr_index)
+        intf->if_kernel_index = ifr->ifr_index;
+#else
+        intf->if_kernel_index = -1;
+#endif
+#endif /* SIOCGIFINDEX */
+
+        /* This call returns IPv4 addresses only. Use SIOCGLIFADDR
+           instead */
+        if (ioctl(sd, SIOCGIFADDR, ifr) < 0) {
+            pmix_output(0, "pmix_ifinit: ioctl(SIOCGIFADDR) failed with errno=%d", errno);
+            PMIX_RELEASE(intf);
+            break;
+        }
+        if (AF_INET != ifr->ifr_addr.sa_family) {
+            PMIX_RELEASE(intf);
+            continue;
+        }
+
+        /* based on above, we know this is an IPv4 address... */
+        memcpy(&intf->if_addr, &ifr->ifr_addr, sizeof(struct sockaddr_in));
+
+        if (ioctl(sd, SIOCGIFNETMASK, ifr) < 0) {
+            pmix_output(0, "pmix_ifinit: ioctl(SIOCGIFNETMASK) failed with errno=%d", errno);
+            PMIX_RELEASE(intf);
+            continue;
+        }
+
+        /* generate CIDR and assign to netmask */
+        intf->if_mask = prefix(((struct sockaddr_in*) &ifr->ifr_addr)->sin_addr.s_addr);
+
+#if defined(SIOCGIFHWADDR) && defined(HAVE_STRUCT_IFREQ_IFR_HWADDR)
+        /* get the MAC address */
+        if (ioctl(sd, SIOCGIFHWADDR, ifr) < 0) {
+            pmix_output(0, "pmix_ifinit: ioctl(SIOCGIFHWADDR) failed with errno=%d", errno);
+            break;
+        }
+        memcpy(intf->if_mac, ifr->ifr_hwaddr.sa_data, 6);
+#endif
+
+#if defined(SIOCGIFMTU) && defined(HAVE_STRUCT_IFREQ_IFR_MTU)
+        /* get the MTU */
+        if (ioctl(sd, SIOCGIFMTU, ifr) < 0) {
+            pmix_output(0, "pmix_ifinit: ioctl(SIOCGIFMTU) failed with errno=%d", errno);
+            break;
+        }
+        intf->ifmtu = ifr->ifr_mtu;
+#endif
+
+        pmix_list_append(&pmix_if_list, &(intf->super));
+    }
+    free(ifconf.ifc_req);
+    close(sd);
+
+    return PMIX_SUCCESS;
+}

opal/mca/pmix/pmix3x/pmix/src/mca/pif/solaris_ipv6/Makefile.am

@@ -0,0 +1,13 @@
+#
+# Copyright (c) 2010     Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2016      Intel, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+noinst_LTLIBRARIES = libmca_pif_solaris_ipv6.la
+
+libmca_pif_solaris_ipv6_la_SOURCES = pif_solaris_ipv6.c

opal/mca/pmix/pmix3x/pmix/src/mca/pif/solaris_ipv6/configure.m4

@@ -0,0 +1,32 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2016      Intel, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+AC_DEFUN([MCA_pmix_pif_solaris_ipv6_COMPILE_MODE], [
+    AC_MSG_CHECKING([for MCA component $1:$2 compile mode])
+    $3="static"
+    AC_MSG_RESULT([$$3])
+])
+
+# MCA_pif_config_CONFIG(action-if-can-compile,
+#                        [action-if-cant-compile])
+# ------------------------------------------------
+AC_DEFUN([MCA_pmix_pif_solaris_ipv6_CONFIG], [
+    AC_CONFIG_FILES([src/mca/pif/solaris_ipv6/Makefile])
+
+    AC_REQUIRE([PMIX_CHECK_OS_FLAVORS])
+
+    # check to see if we are on a solaris machine
+    AS_IF([test "$pmix_found_sun" = "yes"], [$1], [$2])
+])dnl
+
+#
+# ifdef __sun__
+#

opal/mca/pmix/pmix3x/pmix/src/mca/pif/solaris_ipv6/pif_solaris_ipv6.c

@@ -0,0 +1,212 @@
+/*
+ * Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2010      Oracle and/or its affiliates.  All rights reserved.
+ * Copyright (c) 2016      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016      Intel, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "pmix_config.h"
+#include "pmix_common.h"
+
+#include <string.h>
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#include <errno.h>
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_SOCKET_H
+#include <sys/socket.h>
+#endif
+#ifdef HAVE_SYS_SOCKIO_H
+#include <sys/sockio.h>
+#endif
+#ifdef HAVE_SYS_IOCTL_H
+#include <sys/ioctl.h>
+#endif
+#ifdef HAVE_NETINET_IN_H
+#include <netinet/in.h>
+#endif
+#ifdef HAVE_ARPA_INET_H
+#include <arpa/inet.h>
+#endif
+#ifdef HAVE_NET_IF_H
+#if defined(__APPLE__) && defined(_LP64)
+/* Apple engineering suggested using options align=power as a
+   workaround for a bug in OS X 10.4 (Tiger) that prevented ioctl(...,
+   SIOCGIFCONF, ...) from working properly in 64 bit mode on Power PC.
+   It turns out that the underlying issue is the size of struct
+   ifconf, which the kernel expects to be 12 and natural 64 bit
+   alignment would make 16.  The same bug appears in 64 bit mode on
+   Intel macs, but align=power is a no-op there, so instead, use the
+   pack pragma to instruct the compiler to pack on 4 byte words, which
+   has the same effect as align=power for our needs and works on both
+   Intel and Power PC Macs. */
+#pragma pack(push,4)
+#endif
+#include <net/if.h>
+#if defined(__APPLE__) && defined(_LP64)
+#pragma pack(pop)
+#endif
+#endif
+#ifdef HAVE_NETDB_H
+#include <netdb.h>
+#endif
+#ifdef HAVE_IFADDRS_H
+#include <ifaddrs.h>
+#endif
+
+#include "src/util/output.h"
+#include "src/util/pif.h"
+#include "src/mca/pif/pif.h"
+#include "src/mca/pif/base/base.h"
+
+static int if_solaris_ipv6_open(void);
+
+/* Discovers Solaris IPv6 interfaces */
+pmix_pif_base_component_t mca_pif_solaris_ipv6_component = {
+    /* First, the mca_component_t struct containing meta information
+       about the component itself */
+    .base = {
+        PMIX_PIF_BASE_VERSION_2_0_0,
+
+        /* Component name and version */
+        "solaris_ipv6",
+        PMIX_MAJOR_VERSION,
+        PMIX_MINOR_VERSION,
+        PMIX_RELEASE_VERSION,
+
+        /* Component open and close functions */
+        if_solaris_ipv6_open,
+        NULL
+    },
+    .data = {
+        /* This component is checkpointable */
+        PMIX_MCA_BASE_METADATA_PARAM_CHECKPOINT
+    },
+};
+
+/* configure using getifaddrs(3) */
+static int if_solaris_ipv6_open(void)
+{
+    int i;
+    int sd;
+    int error;
+    uint16_t kindex;
+    struct lifnum lifnum;
+    struct lifconf lifconf;
+    struct lifreq *lifreq, lifquery;
+
+    sd = socket (AF_INET6, SOCK_DGRAM, 0);
+    if (sd < 0) {
+        pmix_output (0, "pmix_ifinit: unable to open IPv6 socket\n");
+        return PMIX_ERROR;
+    }
+
+    /* we only ask for IPv6; IPv4 discovery has already been done */
+    lifnum.lifn_family = AF_INET6;
+    lifnum.lifn_flags = 0;
+    lifnum.lifn_count = 0;
+
+    /* get the number of interfaces in the system */
+    error = ioctl (sd, SIOCGLIFNUM, &lifnum);
+    if (error < 0) {
+        pmix_output (0,
+                     "pmix_ifinit: ioctl SIOCGLIFNUM failed with errno=%d\n", errno);
+        return PMIX_ERROR;
+    }
+
+    memset (&lifconf, 0, sizeof (struct lifconf));
+    memset (&lifquery, 0, sizeof (struct lifreq));
+    lifconf.lifc_family = AF_INET6;
+    lifconf.lifc_flags = 0;
+    lifconf.lifc_len = lifnum.lifn_count * sizeof (struct lifreq) * 2;
+    lifconf.lifc_buf = malloc (lifconf.lifc_len);
+    if (NULL == lifconf.lifc_buf) {
+        pmix_output (0, "pmix_ifinit: IPv6 discovery: malloc() failed\n");
+        return PMIX_ERR_OUT_OF_RESOURCE;
+    }
+
+    memset (lifconf.lifc_buf, 0, lifconf.lifc_len);
+
+    error = ioctl (sd, SIOCGLIFCONF, &lifconf);
+    if (error < 0) {
+        pmix_output (0,
+                     "pmix_ifinit: IPv6 SIOCGLIFCONF failed with errno=%d\n", errno);
+    }
+
+    for (i = 0; i + sizeof (struct lifreq) <= lifconf.lifc_len;
+         i += sizeof (*lifreq)) {
+
+        lifreq = (struct lifreq *)((caddr_t)lifconf.lifc_buf + i);
+        strncpy (lifquery.lifr_name, lifreq->lifr_name,
+                 sizeof (lifquery.lifr_name));
+
+        /* lookup kernel index */
+        error = ioctl (sd, SIOCGLIFINDEX, &lifquery);
+        if (error < 0) {
+            pmix_output (0,
+                         "pmix_ifinit: SIOCGLIFINDEX failed with errno=%d\n", errno);
+            return PMIX_ERROR;
+        }
+        kindex = lifquery.lifr_index;
+
+        /* lookup interface flags */
+        error = ioctl (sd, SIOCGLIFFLAGS, &lifquery);
+        if (error < 0) {
+            pmix_output (0,
+                         "pmix_ifinit: SIOCGLIFFLAGS failed with errno=%d\n", errno);
+            return PMIX_ERROR;
+        }
+
+        if (AF_INET6 == lifreq->lifr_addr.ss_family) {
+            struct sockaddr_in6* my_addr = (struct sockaddr_in6*) &lifreq->lifr_addr;
+            /* we surely want to check for sin6_scope_id, but Solaris
+               does not set it correctly, so we have to look for
+               global scope. For now, global is anything which is
+               neither loopback nor link local.
+
+               Bug, FIXME: site-local, multicast, ... missing
+               Check for 2000::/3?
+            */
+            if ( (!pmix_if_retain_loopback && !IN6_IS_ADDR_LOOPBACK (&my_addr->sin6_addr)) &&
+                 (! IN6_IS_ADDR_LINKLOCAL (&my_addr->sin6_addr))) {
+                /* create interface for newly found address */
+                pmix_pif_t *intf;
+
+                intf = PMIX_NEW(pmix_pif_t);
+                if (NULL == intf) {
+                    pmix_output (0,
+                                 "pmix_ifinit: unable to allocate %d bytes\n",
+                                 sizeof (pmix_pif_t));
+                    return PMIX_ERR_OUT_OF_RESOURCE;
+                }
+                intf->af_family = AF_INET6;
+
+                strncpy (intf->if_name, lifreq->lifr_name, IF_NAMESIZE);
+                intf->if_index = pmix_list_get_size(&pmix_if_list)+1;
+                memcpy(&intf->if_addr, my_addr, sizeof (*my_addr));
+                intf->if_mask = 64;
+                /* lifrq flags are uint64_t */
+                intf->if_flags =
+                    (uint32_t)(0x00000000ffffffff) & lifquery.lifr_flags;
+
+                /* append to list */
+                pmix_list_append (&pmix_if_list, &(intf->super));
+            }
+        }
+    } /* for */
+
+    if (NULL != lifconf.lifc_buf) {
+        free (lifconf.lifc_buf);
+    }
+
+    return PMIX_SUCCESS;
+}

opal/mca/pmix/pmix3x/pmix/src/mca/psec/base/psec_base_fns.c

@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2015-2016 Intel, Inc. All rights reserved.
+ * Copyright (c) 2015-2016 Intel, Inc.  All rights reserved.
  * Copyright (c) 2016      Mellanox Technologies, Inc.
  *                         All rights reserved.
  *

opal/mca/pmix/pmix3x/pmix/src/mca/psec/psec.h

@@ -1,7 +1,7 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2007-2008 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015-2016 Intel, Inc. All rights reserved.
+ * Copyright (c) 2015-2016 Intel, Inc.  All rights reserved.
  *
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.

opal/mca/pmix/pmix3x/pmix/src/runtime/pmix_finalize.c

@@ -67,9 +67,6 @@ void pmix_rte_finalize(void)
     if (!pmix_globals.external_evbase) {
         /* stop the progress thread */
         (void)pmix_progress_thread_finalize(NULL);
-        #ifdef HAVE_LIBEVENT_GLOBAL_SHUTDOWN
-            pmix_libevent_global_shutdown();
-        #endif
     }
 
     /* cleanup communications */

opal/mca/pmix/pmix3x/pmix/src/runtime/pmix_init.c

@@ -134,12 +134,6 @@ int pmix_rte_init(pmix_proc_type_t type,
         goto return_error;
     }
 
-    /* read any param files that were provided */
-    if (PMIX_SUCCESS != (ret = pmix_mca_base_var_cache_files(false))) {
-        error = "failed to cache files";
-        goto return_error;
-    }
-
     /* register params for pmix */
     if (PMIX_SUCCESS != (ret = pmix_register_params())) {
         error = "pmix_register_params";

opal/mca/pmix/pmix3x/pmix/src/runtime/pmix_params.c

@@ -43,9 +43,12 @@ bool pmix_timing_overhead = true;
 #endif
 
 static bool pmix_register_done = false;
+char *pmix_net_private_ipv4 = NULL;
 
 pmix_status_t pmix_register_params(void)
 {
+    int ret;
+
     if (pmix_register_done) {
         return PMIX_SUCCESS;
     }
@@ -80,6 +83,24 @@ pmix_status_t pmix_register_params(void)
                                   &pmix_timing_overhead);
 #endif
 
+    /* RFC1918 defines
+       - 10.0.0./8
+       - 172.16.0.0/12
+       - 192.168.0.0/16
+
+       RFC3330 also mentions
+       - 169.254.0.0/16 for DHCP onlink iff there's no DHCP server
+    */
+    pmix_net_private_ipv4 = "10.0.0.0/8;172.16.0.0/12;192.168.0.0/16;169.254.0.0/16";
+    ret = pmix_mca_base_var_register ("pmix", "pmix", "net", "private_ipv4",
+                                      "Semicolon-delimited list of CIDR notation entries specifying what networks are considered \"private\" (default value based on RFC1918 and RFC3330)",
+                                      PMIX_MCA_BASE_VAR_TYPE_STRING, NULL, 0, PMIX_MCA_BASE_VAR_FLAG_SETTABLE,
+                                      PMIX_INFO_LVL_3, PMIX_MCA_BASE_VAR_SCOPE_ALL_EQ,
+                                      &pmix_net_private_ipv4);
+    if (0 > ret) {
+        return ret;
+    }
+
     return PMIX_SUCCESS;
 }
 

opal/mca/pmix/pmix3x/pmix/src/runtime/pmix_rte.h

@@ -44,6 +44,7 @@ extern bool pmix_timing_overhead;
 #endif
 
 extern int pmix_initialized;
+extern char *pmix_net_private_ipv4;
 
 /** version string of pmix */
 extern const char pmix_version_string[];

opal/mca/pmix/pmix3x/pmix/src/usock/usock.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014-2016 Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2016 Intel, Inc.  All rights reserved.
  * Copyright (c) 2014      Artem Y. Polyakov <artpol84@gmail.com>.
  *                         All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science

opal/mca/pmix/pmix3x/pmix/src/util/Makefile.include

@@ -13,6 +13,8 @@
 # Copyright (c) 2007-2016 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2013      NVIDIA Corporation.  All rights reserved.
 # Copyright (c) 2013-2016 Intel, Inc. All rights reserved
+# Copyright (c) 2016      Research Organization for Information Science
+#                         and Technology (RIST). All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -42,7 +44,10 @@ headers += \
         util/path.h \
         util/getid.h \
         util/strnlen.h \
-        util/hash.h
+        util/hash.h \
+        util/net.h \
+        util/tsd.h \
+        util/pif.h
 
 sources += \
         util/argv.c \
@@ -60,7 +65,9 @@ sources += \
         util/show_help_lex.l \
         util/path.c \
         util/getid.c \
-        util/hash.c
+        util/hash.c \
+        util/net.c \
+        util/pif.c
 
 libpmix_la_LIBADD += \
         util/keyval/libpmixutilkeyval.la

opal/mca/pmix/pmix3x/pmix/src/util/error.c

@@ -143,6 +143,8 @@ const char* PMIx_Error_string(pmix_status_t errnum)
         return "PMIX_ERR_FATAL";
     case PMIX_ERR_VALUE_OUT_OF_BOUNDS:
         return "PMIX_ERR_VALUE_OUT_OF_BOUNDS";
+    case PMIX_ERR_NETWORK_NOT_PARSEABLE:
+        return "PMIX_ERR_NETWORK_NOT_PARSEABLE";
     case PMIX_ERR_PERM:
         return "PMIX_ERR_PERM";
     case PMIX_SUCCESS:

opal/mca/pmix/pmix3x/pmix/src/util/error.h

@@ -61,6 +61,7 @@
 #define PMIX_ERR_VALUE_OUT_OF_BOUNDS                    (PMIX_INTERNAL_ERR_BASE - 30)
 #define PMIX_ERR_PERM                                   (PMIX_INTERNAL_ERR_BASE - 31)
 #define PMIX_ERR_OPERATION_IN_PROGRESS                  (PMIX_INTERNAL_ERR_BASE - 32)
+#define PMIX_ERR_NETWORK_NOT_PARSEABLE                  (PMIX_INTERNAL_ERR_BASE - 33)
 
 #define PMIX_ERROR_LOG(r)                                           \
  do {                                                               \

opal/mca/pmix/pmix3x/pmix/src/util/net.c

@@ -0,0 +1,486 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2009-2015 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013-2016 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "pmix_config.h"
+#include "pmix_common.h"
+
+#include <stdio.h>
+#include <string.h>
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#include <errno.h>
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_SOCKET_H
+#include <sys/socket.h>
+#endif
+#ifdef HAVE_SYS_SOCKIO_H
+#include <sys/sockio.h>
+#endif
+#ifdef HAVE_SYS_IOCTL_H
+#include <sys/ioctl.h>
+#endif
+#ifdef HAVE_NETINET_IN_H
+#include <netinet/in.h>
+#endif
+#ifdef HAVE_ARPA_INET_H
+#include <arpa/inet.h>
+#endif
+#ifdef HAVE_NET_IF_H
+#if defined(__APPLE__) && defined(_LP64)
+/* Apple engineering suggested using options align=power as a
+   workaround for a bug in OS X 10.4 (Tiger) that prevented ioctl(...,
+   SIOCGIFCONF, ...) from working properly in 64 bit mode on Power PC.
+   It turns out that the underlying issue is the size of struct
+   ifconf, which the kernel expects to be 12 and natural 64 bit
+   alignment would make 16.  The same bug appears in 64 bit mode on
+   Intel macs, but align=power is a no-op there, so instead, use the
+   pack pragma to instruct the compiler to pack on 4 byte words, which
+   has the same effect as align=power for our needs and works on both
+   Intel and Power PC Macs. */
+#pragma pack(push,4)
+#endif
+#include <net/if.h>
+#if defined(__APPLE__) && defined(_LP64)
+#pragma pack(pop)
+#endif
+#endif
+#ifdef HAVE_NETDB_H
+#include <netdb.h>
+#endif
+#ifdef HAVE_IFADDRS_H
+#include <ifaddrs.h>
+#endif
+
+#include "src/util/net.h"
+#include "src/util/output.h"
+#include "src/util/argv.h"
+#include "src/util/show_help.h"
+#include "src/util/tsd.h"
+#include "src/runtime/pmix_rte.h"
+
+/* this function doesn't depend on sockaddr_h */
+bool pmix_net_isaddr(const char *name)
+{
+    struct addrinfo hint, *res = NULL;
+
+    /* initialize the hint */
+    memset(&hint, '\0', sizeof hint);
+
+    /* indicate that we don't know the family */
+    hint.ai_family = PF_UNSPEC;
+    hint.ai_flags = AI_NUMERICHOST;
+
+    if (0 != getaddrinfo(name, NULL, &hint, &res)) {
+        /* the input wasn't a recognizable address */
+        return false;
+    }
+    /* we don't care what family - all we care is that
+     * it is indeed an address
+     */
+    freeaddrinfo(res);
+    return true;
+}
+
+#ifdef HAVE_STRUCT_SOCKADDR_IN
+
+typedef struct private_ipv4_t {
+    in_addr_t addr;
+    uint32_t netmask_bits;
+} private_ipv4_t;
+
+static private_ipv4_t* private_ipv4 = NULL;
+
+static pmix_tsd_key_t hostname_tsd_key;
+
+
+static void
+hostname_cleanup(void *value)
+{
+    if (NULL != value) free(value);
+}
+
+
+static char*
+get_hostname_buffer(void)
+{
+    void *buffer;
+    int ret;
+
+    ret = pmix_tsd_getspecific(hostname_tsd_key, &buffer);
+    if (PMIX_SUCCESS != ret) return NULL;
+
+    if (NULL == buffer) {
+        buffer = (void*) malloc((NI_MAXHOST + 1) * sizeof(char));
+        ret = pmix_tsd_setspecific(hostname_tsd_key, buffer);
+    }
+
+    return (char*) buffer;
+}
+
+int
+pmix_net_init(void)
+{
+    char **args, *arg;
+    uint32_t a, b, c, d, bits, addr;
+    int i, count, found_bad = 0;
+
+    args = pmix_argv_split(pmix_net_private_ipv4, ';');
+    if( NULL != args ) {
+        count = pmix_argv_count(args);
+        private_ipv4 = (private_ipv4_t*)malloc( (count + 1) * sizeof(private_ipv4_t));
+        if( NULL == private_ipv4 ) {
+            pmix_output(0, "Unable to allocate memory for the private addresses array" );
+            pmix_argv_free(args);
+            goto do_local_init;
+        }
+        for( i = 0; i < count; i++ ) {
+            arg = args[i];
+
+            (void)sscanf( arg, "%u.%u.%u.%u/%u", &a, &b, &c, &d, &bits );
+
+            if( (a > 255) || (b > 255) || (c > 255) ||
+                (d > 255) || (bits > 32) ) {
+                if (0 == found_bad) {
+                    pmix_show_help("help-pmix-runtime.txt",
+                                   "malformed net_private_ipv4",
+                                   true, args[i]);
+                    found_bad = 1;
+                }
+                continue;
+            }
+            addr = (a << 24) | (b << 16) | (c << 8) | d;
+            private_ipv4[i].addr = htonl(addr);
+            private_ipv4[i].netmask_bits = bits;
+        }
+        private_ipv4[i].addr         = 0;
+        private_ipv4[i].netmask_bits = 0;
+        pmix_argv_free(args);
+    }
+
+ do_local_init:
+    return pmix_tsd_key_create(&hostname_tsd_key, hostname_cleanup);
+}
+
+
+int
+pmix_net_finalize()
+{
+    free(private_ipv4);
+    private_ipv4 = NULL;
+
+    return PMIX_SUCCESS;
+}
+
+
+/* convert a CIDR prefixlen to netmask (in network byte order) */
+uint32_t
+pmix_net_prefix2netmask(uint32_t prefixlen)
+{
+    return htonl (((1 << prefixlen) - 1) << (32 - prefixlen));
+}
+
+
+bool
+pmix_net_islocalhost(const struct sockaddr *addr)
+{
+    switch (addr->sa_family) {
+    case AF_INET:
+        {
+            const struct sockaddr_in *inaddr = (struct sockaddr_in*) addr;
+            /* if it's in the 127. domain, it shouldn't be routed
+               (0x7f == 127) */
+            if (0x7F000000 == (0x7F000000 & ntohl(inaddr->sin_addr.s_addr))) {
+                return true;
+            }
+            return false;
+        }
+        break;
+
+    case AF_INET6:
+        {
+            const struct sockaddr_in6 *inaddr = (struct sockaddr_in6*) addr;
+            if (IN6_IS_ADDR_LOOPBACK (&inaddr->sin6_addr)) {
+               return true; /* Bug, FIXME: check for 127.0.0.1/8 */
+            }
+            return false;
+        }
+        break;
+
+    default:
+        pmix_output(0, "unhandled sa_family %d passed to pmix_net_islocalhost",
+                    addr->sa_family);
+        return false;
+        break;
+    }
+}
+
+
+bool
+pmix_net_samenetwork(const struct sockaddr *addr1,
+                     const struct sockaddr *addr2,
+                     uint32_t plen)
+{
+    uint32_t prefixlen;
+
+    if(addr1->sa_family != addr2->sa_family) {
+        return false; /* address families must be equal */
+    }
+
+    switch (addr1->sa_family) {
+    case AF_INET:
+        {
+            if (0 == plen) {
+                prefixlen = 32;
+            } else {
+                prefixlen = plen;
+            }
+            struct sockaddr_in inaddr1, inaddr2;
+            /* Use temporary variables and memcpy's so that we don't
+               run into bus errors on Solaris/SPARC */
+            memcpy(&inaddr1, addr1, sizeof(inaddr1));
+            memcpy(&inaddr2, addr2, sizeof(inaddr2));
+            uint32_t netmask = pmix_net_prefix2netmask (prefixlen);
+
+            if((inaddr1.sin_addr.s_addr & netmask) ==
+               (inaddr2.sin_addr.s_addr & netmask)) {
+                return true;
+            }
+            return false;
+        }
+        break;
+
+    case AF_INET6:
+        {
+            struct sockaddr_in6 inaddr1, inaddr2;
+            /* Use temporary variables and memcpy's so that we don't
+               run into bus errors on Solaris/SPARC */
+            memcpy(&inaddr1, addr1, sizeof(inaddr1));
+            memcpy(&inaddr2, addr2, sizeof(inaddr2));
+            struct in6_addr *a6_1 = (struct in6_addr*) &inaddr1.sin6_addr;
+            struct in6_addr *a6_2 = (struct in6_addr*) &inaddr2.sin6_addr;
+
+            if (0 == plen) {
+                prefixlen = 64;
+            } else {
+                prefixlen = plen;
+            }
+            if (64 == prefixlen) {
+                /* prefixlen is always /64, any other case would be routing.
+                   Compare the first eight bytes (64 bits) and hope that
+                   endianess is not an issue on any system as long as
+                   addresses are always stored in network byte order.
+                */
+                if (((const uint32_t *) (a6_1))[0] ==
+                    ((const uint32_t *) (a6_2))[0] &&
+                    ((const uint32_t *) (a6_1))[1] ==
+                    ((const uint32_t *) (a6_2))[1]) {
+                    return true;
+                }
+            }
+            return false;
+        }
+        break;
+
+    default:
+        pmix_output(0, "unhandled sa_family %d passed to pmix_samenetwork",
+                    addr1->sa_family);
+    }
+
+    return false;
+}
+
+
+/**
+ * Returns true if the given address is a public IPv4 address.
+ */
+bool
+pmix_net_addr_isipv4public(const struct sockaddr *addr)
+{
+    switch (addr->sa_family) {
+        case AF_INET6:
+            return false;
+
+        case AF_INET:
+            {
+                const struct sockaddr_in *inaddr = (struct sockaddr_in*) addr;
+                int i;
+
+                if( NULL == private_ipv4 ) {
+                    return true;
+                }
+
+                for( i = 0; private_ipv4[i].addr != 0; i++ ) {
+                    if( private_ipv4[i].addr == (inaddr->sin_addr.s_addr &
+                                                 pmix_net_prefix2netmask(private_ipv4[i].netmask_bits)) )
+                        return false;
+                }
+
+            }
+            return true;
+        default:
+            pmix_output (0,
+                         "unhandled sa_family %d passed to pmix_net_addr_isipv4public\n",
+                         addr->sa_family);
+    }
+
+    return false;
+}
+
+
+char*
+pmix_net_get_hostname(const struct sockaddr *addr)
+{
+    char *name = get_hostname_buffer();
+    int error;
+    socklen_t addrlen;
+    char *p;
+
+    if (NULL == name) {
+        pmix_output(0, "pmix_sockaddr2str: malloc() failed\n");
+        return NULL;
+    }
+    memset(name, 0, sizeof(*name));
+
+    switch (addr->sa_family) {
+    case AF_INET:
+        addrlen = sizeof (struct sockaddr_in);
+        break;
+    case AF_INET6:
+#if defined( __NetBSD__)
+        /* hotfix for netbsd: on my netbsd machine, getnameinfo
+           returns an unkown error code. */
+        if(NULL == inet_ntop(AF_INET6, &((struct sockaddr_in6*) addr)->sin6_addr,
+                             name, NI_MAXHOST)) {
+            pmix_output(0, "pmix_sockaddr2str failed with error code %d", errno);
+            free(name);
+            return NULL;
+        }
+        return name;
+#else
+        addrlen = sizeof (struct sockaddr_in6);
+#endif
+        break;
+    default:
+        free(name);
+        return NULL;
+    }
+
+    error = getnameinfo(addr, addrlen,
+                        name, NI_MAXHOST, NULL, 0, NI_NUMERICHOST);
+
+    if (error) {
+       int err = errno;
+       pmix_output (0, "pmix_sockaddr2str failed:%s (return code %i)\n",
+                    gai_strerror(err), error);
+       free (name);
+       return NULL;
+    }
+    /* strip any trailing % data as it isn't pertinent */
+    if (NULL != (p = strrchr(name, '%'))) {
+        *p = '\0';
+    }
+    return name;
+}
+
+
+int
+pmix_net_get_port(const struct sockaddr *addr)
+{
+    switch (addr->sa_family) {
+    case AF_INET:
+        return ntohs(((struct sockaddr_in*) addr)->sin_port);
+        break;
+
+    case AF_INET6:
+        return ntohs(((struct sockaddr_in6*) addr)->sin6_port);
+        break;
+    }
+
+    return -1;
+}
+
+
+#else /* HAVE_STRUCT_SOCKADDR_IN */
+
+int
+pmix_net_init()
+{
+    return PMIX_SUCCESS;
+}
+
+
+int
+pmix_net_finalize()
+{
+    return PMIX_SUCCESS;
+}
+
+
+uint32_t
+pmix_net_prefix2netmask(uint32_t prefixlen)
+{
+    return 0;
+}
+
+
+bool
+pmix_net_islocalhost(const struct sockaddr *addr)
+{
+    return false;
+}
+
+
+bool
+pmix_net_samenetwork(const struct sockaddr *addr1,
+                     const struct sockaddr *addr2,
+                     uint32_t prefixlen)
+{
+    return false;
+}
+
+
+bool
+pmix_net_addr_isipv4public(const struct sockaddr *addr)
+{
+    return false;
+}
+
+
+char*
+pmix_net_get_hostname(const struct sockaddr *addr)
+{
+    return NULL;
+}
+
+
+int
+pmix_net_get_port(const struct sockaddr *addr)
+{
+    return -1;
+}
+
+#endif /* HAVE_STRUCT_SOCKADDR_IN */

opal/mca/pmix/pmix3x/pmix/src/util/net.h

@@ -0,0 +1,150 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2016      Intel, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+/* @file */
+
+#ifndef PMIX_UTIL_NET_H
+#define PMIX_UTIL_NET_H
+
+#include "pmix_config.h"
+
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_SOCKET_H
+#include <sys/socket.h>
+#endif
+#ifdef HAVE_NETINET_IN_H
+#include <netinet/in.h>
+#endif
+
+BEGIN_C_DECLS
+
+/**
+ * Intiailize the network helper subsystem
+ *
+ * Initialize the network helper subsystem.  Should be called exactly
+ * once for any process that will use any function in the network
+ * helper subsystem.
+ *
+ * @retval PMIX_SUCCESS   Success
+ * @retval PMIX_ERR_TEMP_OUT_OF_RESOURCE Not enough memory for static
+ *                        buffer creation
+ */
+PMIX_EXPORT int pmix_net_init(void);
+
+
+/**
+ * Finalize the network helper subsystem
+ *
+ * Finalize the network helper subsystem.  Should be called exactly
+ * once for any process that will use any function in the network
+ * helper subsystem.
+ *
+ * @retval PMIX_SUCCESS   Success
+ */
+PMIX_EXPORT int pmix_net_finalize(void);
+
+
+/**
+ * Calculate netmask in network byte order from CIDR notation
+ *
+ * @param prefixlen (IN)  CIDR prefixlen
+ * @return                netmask in network byte order
+ */
+PMIX_EXPORT uint32_t pmix_net_prefix2netmask(uint32_t prefixlen);
+
+
+/**
+ * Determine if given IP address is in the localhost range
+ *
+ * Determine if the given IP address is in the localhost range
+ * (127.0.0.0/8), meaning that it can't be used to connect to machines
+ * outside the current host.
+ *
+ * @param addr             struct sockaddr_in of IP address
+ * @return                 true if \c addr is a localhost address,
+ *                         false otherwise.
+ */
+PMIX_EXPORT bool pmix_net_islocalhost(const struct sockaddr *addr);
+
+
+/**
+ * Are we on the same network?
+ *
+ * For IPv6, we only need to check for /64, there are no other
+ * local netmasks.
+ *
+ * @param addr1             struct sockaddr of address
+ * @param addr2             struct sockaddr of address
+ * @param prefixlen         netmask (either CIDR or IPv6 prefixlen)
+ * @return                  true if \c addr1 and \c addr2 are on the
+ *                          same net, false otherwise.
+ */
+PMIX_EXPORT bool pmix_net_samenetwork(const struct sockaddr *addr1,
+                                        const struct sockaddr *addr2,
+                                        uint32_t prefixlen);
+
+
+/**
+ * Is the given address a public IPv4 address?  Returns false for IPv6
+ * address.
+ *
+ * @param addr      address as struct sockaddr
+ * @return          true, if \c addr is IPv4 public, false otherwise
+ */
+PMIX_EXPORT bool pmix_net_addr_isipv4public(const struct sockaddr *addr);
+
+
+/**
+ * Get string version of address
+ *
+ * Return the un-resolved address in a string format.  The string will
+ * be returned in a per-thread static buffer and should not be freed
+ * by the user.
+ *
+ * @param addr              struct sockaddr of address
+ * @return                  literal representation of \c addr
+ */
+PMIX_EXPORT char* pmix_net_get_hostname(const struct sockaddr *addr);
+
+
+/**
+ * Get port number from struct sockaddr
+ *
+ * Return the port number (as an integr) from either a struct
+ * sockaddr_in or a struct sockaddr_in6.
+ *
+ * @param addr             struct sockaddr containing address
+ * @return                 port number from \addr
+ */
+PMIX_EXPORT int pmix_net_get_port(const struct sockaddr *addr);
+
+/**
+ * Test if a string is actually an IP address
+ *
+ * Returns true if the string is of IPv4 or IPv6 address form
+ */
+PMIX_EXPORT bool pmix_net_isaddr(const char *name);
+
+END_C_DECLS
+
+#endif /* PMIX_UTIL_NET_H */

opal/mca/pmix/pmix3x/pmix/src/util/output.h

@@ -11,7 +11,7 @@
  *                         All rights reserved.
  * Copyright (c) 2007-2011 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2010      Oracle and/or its affiliates.  All rights reserved.
- * Copyright (c) 2015-2016 Intel, Inc. All rights reserved.
+ * Copyright (c) 2015-2016 Intel, Inc.  All rights reserved.
  * Copyright (c) 2016      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$

opal/mca/pmix/pmix3x/pmix/src/util/pif.c

@@ -0,0 +1,898 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2009 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2010-2015 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2014      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016      Intel, Inc. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "pmix_config.h"
+#include "pmix_common.h"
+
+#include <string.h>
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#include <errno.h>
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_SOCKET_H
+#include <sys/socket.h>
+#endif
+#ifdef HAVE_SYS_SOCKIO_H
+#include <sys/sockio.h>
+#endif
+#ifdef HAVE_SYS_IOCTL_H
+#include <sys/ioctl.h>
+#endif
+#ifdef HAVE_NETINET_IN_H
+#include <netinet/in.h>
+#endif
+#ifdef HAVE_ARPA_INET_H
+#include <arpa/inet.h>
+#endif
+#ifdef HAVE_NET_IF_H
+#if defined(__APPLE__) && defined(_LP64)
+/* Apple engineering suggested using options align=power as a
+   workaround for a bug in OS X 10.4 (Tiger) that prevented ioctl(...,
+   SIOCGIFCONF, ...) from working properly in 64 bit mode on Power PC.
+   It turns out that the underlying issue is the size of struct
+   ifconf, which the kernel expects to be 12 and natural 64 bit
+   alignment would make 16.  The same bug appears in 64 bit mode on
+   Intel macs, but align=power is a no-op there, so instead, use the
+   pack pragma to instruct the compiler to pack on 4 byte words, which
+   has the same effect as align=power for our needs and works on both
+   Intel and Power PC Macs. */
+#pragma pack(push,4)
+#endif
+#include <net/if.h>
+#if defined(__APPLE__) && defined(_LP64)
+#pragma pack(pop)
+#endif
+#endif
+#ifdef HAVE_NETDB_H
+#include <netdb.h>
+#endif
+#ifdef HAVE_IFADDRS_H
+#include <ifaddrs.h>
+#endif
+#include <ctype.h>
+
+#include "src/class/pmix_list.h"
+#include "src/util/error.h"
+#include "src/util/pif.h"
+#include "src/util/net.h"
+#include "src/util/output.h"
+#include "src/util/argv.h"
+#include "src/util/show_help.h"
+
+#include "src/mca/pif/base/base.h"
+
+#ifdef HAVE_STRUCT_SOCKADDR_IN
+
+#ifndef MIN
+#  define MIN(a,b)                ((a) < (b) ? (a) : (b))
+#endif
+
+/*
+ *  Look for interface by name and returns its address
+ *  as a dotted decimal formatted string.
+ */
+
+int pmix_ifnametoaddr(const char* if_name, struct sockaddr* addr, int length)
+{
+    pmix_pif_t* intf;
+
+    if (PMIX_SUCCESS != pmix_mca_base_framework_open(&pmix_pif_base_framework, 0)) {
+        return PMIX_ERROR;
+    }
+
+    for (intf =  (pmix_pif_t*)pmix_list_get_first(&pmix_if_list);
+        intf != (pmix_pif_t*)pmix_list_get_end(&pmix_if_list);
+        intf =  (pmix_pif_t*)pmix_list_get_next(intf)) {
+        if (strcmp(intf->if_name, if_name) == 0) {
+            memcpy(addr, &intf->if_addr, length);
+            return PMIX_SUCCESS;
+        }
+    }
+    return PMIX_ERROR;
+}
+
+
+/*
+ *  Look for interface by name and returns its
+ *  corresponding pmix_list index.
+ */
+
+int pmix_ifnametoindex(const char* if_name)
+{
+    pmix_pif_t* intf;
+
+    if (PMIX_SUCCESS != pmix_mca_base_framework_open(&pmix_pif_base_framework, 0)) {
+        return -1;
+    }
+
+    for (intf =  (pmix_pif_t*)pmix_list_get_first(&pmix_if_list);
+        intf != (pmix_pif_t*)pmix_list_get_end(&pmix_if_list);
+        intf =  (pmix_pif_t*)pmix_list_get_next(intf)) {
+        if (strcmp(intf->if_name, if_name) == 0) {
+            return intf->if_index;
+        }
+    }
+    return -1;
+}
+
+
+/*
+ *  Look for interface by name and returns its
+ *  corresponding kernel index.
+ */
+
+int16_t pmix_ifnametokindex(const char* if_name)
+{
+    pmix_pif_t* intf;
+
+    if (PMIX_SUCCESS != pmix_mca_base_framework_open(&pmix_pif_base_framework, 0)) {
+        return -1;
+    }
+
+    for (intf =  (pmix_pif_t*)pmix_list_get_first(&pmix_if_list);
+        intf != (pmix_pif_t*)pmix_list_get_end(&pmix_if_list);
+        intf =  (pmix_pif_t*)pmix_list_get_next(intf)) {
+        if (strcmp(intf->if_name, if_name) == 0) {
+            return intf->if_kernel_index;
+        }
+    }
+    return -1;
+}
+
+
+/*
+ *  Look for interface by pmix_list index and returns its
+ *  corresponding kernel index.
+ */
+
+int pmix_ifindextokindex(int if_index)
+{
+    pmix_pif_t* intf;
+
+    if (PMIX_SUCCESS != pmix_mca_base_framework_open(&pmix_pif_base_framework, 0)) {
+        return -1;
+    }
+
+    for (intf =  (pmix_pif_t*)pmix_list_get_first(&pmix_if_list);
+        intf != (pmix_pif_t*)pmix_list_get_end(&pmix_if_list);
+        intf =  (pmix_pif_t*)pmix_list_get_next(intf)) {
+        if (if_index == intf->if_index) {
+            return intf->if_kernel_index;
+        }
+    }
+    return -1;
+}
+
+
+/*
+ *  Attempt to resolve the adddress (given as either IPv4/IPv6 string
+ *  or hostname) and lookup corresponding interface.
+ */
+
+int pmix_ifaddrtoname(const char* if_addr, char* if_name, int length)
+{
+    pmix_pif_t* intf;
+    int error;
+    struct addrinfo hints, *res = NULL, *r;
+
+    /* if the user asked us not to resolve interfaces, then just return */
+    if (pmix_if_do_not_resolve) {
+        /* return not found so ifislocal will declare
+         * the node to be non-local
+         */
+        return PMIX_ERR_NOT_FOUND;
+    }
+
+    if (PMIX_SUCCESS != pmix_mca_base_framework_open(&pmix_pif_base_framework, 0)) {
+        return PMIX_ERROR;
+    }
+
+    memset(&hints, 0, sizeof(hints));
+    hints.ai_family = PF_UNSPEC;
+    hints.ai_socktype = SOCK_STREAM;
+    error = getaddrinfo(if_addr, NULL, &hints, &res);
+
+    if (error) {
+        if (NULL != res) {
+            freeaddrinfo (res);
+        }
+        return PMIX_ERR_NOT_FOUND;
+    }
+
+    for (r = res; r != NULL; r = r->ai_next) {
+        for (intf =  (pmix_pif_t*)pmix_list_get_first(&pmix_if_list);
+            intf != (pmix_pif_t*)pmix_list_get_end(&pmix_if_list);
+            intf =  (pmix_pif_t*)pmix_list_get_next(intf)) {
+
+            if (AF_INET == r->ai_family) {
+                struct sockaddr_in ipv4;
+                struct sockaddr_in *inaddr;
+
+                inaddr = (struct sockaddr_in*) &intf->if_addr;
+                memcpy (&ipv4, r->ai_addr, r->ai_addrlen);
+
+                if (inaddr->sin_addr.s_addr == ipv4.sin_addr.s_addr) {
+                    strncpy(if_name, intf->if_name, length);
+                    freeaddrinfo (res);
+                    return PMIX_SUCCESS;
+                }
+            }
+            else {
+                if (IN6_ARE_ADDR_EQUAL(&((struct sockaddr_in6*) &intf->if_addr)->sin6_addr,
+                    &((struct sockaddr_in6*) r->ai_addr)->sin6_addr)) {
+                    strncpy(if_name, intf->if_name, length);
+                    freeaddrinfo (res);
+                    return PMIX_SUCCESS;
+                }
+            }
+        }
+    }
+    if (NULL != res) {
+        freeaddrinfo (res);
+    }
+
+    /* if we get here, it wasn't found */
+    return PMIX_ERR_NOT_FOUND;
+}
+
+/*
+ *  Attempt to resolve the address (given as either IPv4/IPv6 string
+ *  or hostname) and return the kernel index of the interface
+ *  on the same network as the specified address
+ */
+int16_t pmix_ifaddrtokindex(const char* if_addr)
+{
+    pmix_pif_t* intf;
+    int error;
+    struct addrinfo hints, *res = NULL, *r;
+    int if_kernel_index;
+    size_t len;
+
+    if (PMIX_SUCCESS != pmix_mca_base_framework_open(&pmix_pif_base_framework, 0)) {
+        return PMIX_ERROR;
+    }
+
+    memset(&hints, 0, sizeof(hints));
+    hints.ai_family = PF_UNSPEC;
+    hints.ai_socktype = SOCK_STREAM;
+    error = getaddrinfo(if_addr, NULL, &hints, &res);
+
+    if (error) {
+        if (NULL != res) {
+            freeaddrinfo (res);
+        }
+        return PMIX_ERR_NOT_FOUND;
+    }
+
+    for (r = res; r != NULL; r = r->ai_next) {
+        PMIX_LIST_FOREACH(intf, &pmix_if_list, pmix_pif_t) {
+            if (AF_INET == r->ai_family && AF_INET == intf->af_family) {
+                struct sockaddr_in ipv4;
+                len = (r->ai_addrlen < sizeof(struct sockaddr_in)) ? r->ai_addrlen : sizeof(struct sockaddr_in);
+                memcpy(&ipv4, r->ai_addr, len);
+                if (pmix_net_samenetwork((struct sockaddr*)&ipv4, (struct sockaddr*)&intf->if_addr, intf->if_mask)) {
+                    if_kernel_index = intf->if_kernel_index;
+                    freeaddrinfo (res);
+                    return if_kernel_index;
+                }
+            } else if (AF_INET6 == r->ai_family && AF_INET6 == intf->af_family) {
+                struct sockaddr_in6 ipv6;
+                len = (r->ai_addrlen < sizeof(struct sockaddr_in6)) ? r->ai_addrlen : sizeof(struct sockaddr_in6);
+                memcpy(&ipv6, r->ai_addr, len);
+                if (pmix_net_samenetwork((struct sockaddr*)((struct sockaddr_in6*)&intf->if_addr),
+                                         (struct sockaddr*)&ipv6, intf->if_mask)) {
+                    if_kernel_index = intf->if_kernel_index;
+                    freeaddrinfo (res);
+                    return if_kernel_index;
+                }
+            }
+        }
+    }
+    if (NULL != res) {
+        freeaddrinfo (res);
+    }
+    return PMIX_ERR_NOT_FOUND;
+}
+
+/*
+ *  Return the number of discovered interface.
+ */
+
+int pmix_ifcount(void)
+{
+    if (PMIX_SUCCESS != pmix_mca_base_framework_open(&pmix_pif_base_framework, 0)) {
+        return 0;
+    }
+
+    return pmix_list_get_size(&pmix_if_list);
+}
+
+
+/*
+ *  Return the pmix_list interface index for the first
+ *  interface in our list.
+ */
+
+int pmix_ifbegin(void)
+{
+    pmix_pif_t *intf;
+
+    if (PMIX_SUCCESS != pmix_mca_base_framework_open(&pmix_pif_base_framework, 0)) {
+        return -1;
+    }
+
+    intf = (pmix_pif_t*)pmix_list_get_first(&pmix_if_list);
+    if (NULL != intf)
+        return intf->if_index;
+    return (-1);
+}
+
+
+/*
+ *  Located the current position in the list by if_index and
+ *  return the interface index of the next element in our list
+ *  (if it exists).
+ */
+
+int pmix_ifnext(int if_index)
+{
+    pmix_pif_t *intf;
+
+    if (PMIX_SUCCESS != pmix_mca_base_framework_open(&pmix_pif_base_framework, 0)) {
+        return -1;
+    }
+
+    for (intf =  (pmix_pif_t*)pmix_list_get_first(&pmix_if_list);
+        intf != (pmix_pif_t*)pmix_list_get_end(&pmix_if_list);
+        intf =  (pmix_pif_t*)pmix_list_get_next(intf)) {
+        if (intf->if_index == if_index) {
+            do {
+                pmix_pif_t* if_next = (pmix_pif_t*)pmix_list_get_next(intf);
+                pmix_pif_t* if_end =  (pmix_pif_t*)pmix_list_get_end(&pmix_if_list);
+                if (if_next == if_end) {
+                    return -1;
+                }
+                intf = if_next;
+            } while(intf->if_index == if_index);
+            return intf->if_index;
+        }
+    }
+    return (-1);
+}
+
+
+/*
+ *  Lookup the interface by pmix_list index and return the
+ *  primary address assigned to the interface.
+ */
+
+int pmix_ifindextoaddr(int if_index, struct sockaddr* if_addr, unsigned int length)
+{
+    pmix_pif_t* intf;
+
+    if (PMIX_SUCCESS != pmix_mca_base_framework_open(&pmix_pif_base_framework, 0)) {
+        return PMIX_ERROR;
+    }
+
+    for (intf =  (pmix_pif_t*)pmix_list_get_first(&pmix_if_list);
+         intf != (pmix_pif_t*)pmix_list_get_end(&pmix_if_list);
+         intf =  (pmix_pif_t*)pmix_list_get_next(intf)) {
+        if (intf->if_index == if_index) {
+            memcpy(if_addr, &intf->if_addr, MIN(length, sizeof (intf->if_addr)));
+            return PMIX_SUCCESS;
+        }
+    }
+    return PMIX_ERROR;
+}
+
+
+/*
+ *  Lookup the interface by pmix_list kindex and return the
+ *  primary address assigned to the interface.
+ */
+int pmix_ifkindextoaddr(int if_kindex, struct sockaddr* if_addr, unsigned int length)
+{
+    pmix_pif_t* intf;
+
+    if (PMIX_SUCCESS != pmix_mca_base_framework_open(&pmix_pif_base_framework, 0)) {
+        return PMIX_ERROR;
+    }
+
+    for (intf =  (pmix_pif_t*)pmix_list_get_first(&pmix_if_list);
+         intf != (pmix_pif_t*)pmix_list_get_end(&pmix_if_list);
+         intf =  (pmix_pif_t*)pmix_list_get_next(intf)) {
+        if (intf->if_kernel_index == if_kindex) {
+            memcpy(if_addr, &intf->if_addr, MIN(length, sizeof (intf->if_addr)));
+            return PMIX_SUCCESS;
+        }
+    }
+    return PMIX_ERROR;
+}
+
+
+/*
+ *  Lookup the interface by pmix_list index and return the
+ *  network mask assigned to the interface.
+ */
+
+int pmix_ifindextomask(int if_index, uint32_t* if_mask, int length)
+{
+    pmix_pif_t* intf;
+
+    if (PMIX_SUCCESS != pmix_mca_base_framework_open(&pmix_pif_base_framework, 0)) {
+        return PMIX_ERROR;
+    }
+
+    for (intf =  (pmix_pif_t*)pmix_list_get_first(&pmix_if_list);
+        intf != (pmix_pif_t*)pmix_list_get_end(&pmix_if_list);
+        intf =  (pmix_pif_t*)pmix_list_get_next(intf)) {
+        if (intf->if_index == if_index) {
+            memcpy(if_mask, &intf->if_mask, length);
+            return PMIX_SUCCESS;
+        }
+    }
+    return PMIX_ERROR;
+}
+
+/*
+ *  Lookup the interface by pmix_list index and return the
+ *  MAC assigned to the interface.
+ */
+
+int pmix_ifindextomac(int if_index, uint8_t mac[6])
+{
+    pmix_pif_t* intf;
+
+    for (intf = (pmix_pif_t*)pmix_list_get_first(&pmix_if_list);
+        intf != (pmix_pif_t*)pmix_list_get_end(&pmix_if_list);
+        intf = (pmix_pif_t*)pmix_list_get_next(intf)) {
+        if (intf->if_index == if_index) {
+            memcpy(mac, &intf->if_mac, 6);
+            return PMIX_SUCCESS;
+        }
+    }
+    return PMIX_ERROR;
+}
+
+/*
+ *  Lookup the interface by pmix_list index and return the
+ *  MTU assigned to the interface.
+ */
+
+int pmix_ifindextomtu(int if_index, int *mtu)
+{
+    pmix_pif_t* intf;
+
+    for (intf = (pmix_pif_t*)pmix_list_get_first(&pmix_if_list);
+        intf != (pmix_pif_t*)pmix_list_get_end(&pmix_if_list);
+        intf = (pmix_pif_t*)pmix_list_get_next(intf)) {
+        if (intf->if_index == if_index) {
+            *mtu = intf->ifmtu;
+            return PMIX_SUCCESS;
+        }
+    }
+    return PMIX_ERROR;
+}
+
+/*
+ *  Lookup the interface by pmix_list index and return the
+ *  flags assigned to the interface.
+ */
+
+int pmix_ifindextoflags(int if_index, uint32_t* if_flags)
+{
+    pmix_pif_t* intf;
+
+    if (PMIX_SUCCESS != pmix_mca_base_framework_open(&pmix_pif_base_framework, 0)) {
+        return PMIX_ERROR;
+    }
+
+    for (intf =  (pmix_pif_t*)pmix_list_get_first(&pmix_if_list);
+        intf != (pmix_pif_t*)pmix_list_get_end(&pmix_if_list);
+        intf =  (pmix_pif_t*)pmix_list_get_next(intf)) {
+        if (intf->if_index == if_index) {
+            memcpy(if_flags, &intf->if_flags, sizeof(uint32_t));
+            return PMIX_SUCCESS;
+        }
+    }
+    return PMIX_ERROR;
+}
+
+
+
+/*
+ *  Lookup the interface by pmix_list index and return
+ *  the associated name.
+ */
+
+int pmix_ifindextoname(int if_index, char* if_name, int length)
+{
+    pmix_pif_t *intf;
+
+    if (PMIX_SUCCESS != pmix_mca_base_framework_open(&pmix_pif_base_framework, 0)) {
+        return PMIX_ERROR;
+    }
+
+    for (intf =  (pmix_pif_t*)pmix_list_get_first(&pmix_if_list);
+        intf != (pmix_pif_t*)pmix_list_get_end(&pmix_if_list);
+        intf =  (pmix_pif_t*)pmix_list_get_next(intf)) {
+        if (intf->if_index == if_index) {
+            strncpy(if_name, intf->if_name, length);
+            return PMIX_SUCCESS;
+        }
+    }
+    return PMIX_ERROR;
+}
+
+
+/*
+ *  Lookup the interface by kernel index and return
+ *  the associated name.
+ */
+
+int pmix_ifkindextoname(int if_kindex, char* if_name, int length)
+{
+    pmix_pif_t *intf;
+
+    if (PMIX_SUCCESS != pmix_mca_base_framework_open(&pmix_pif_base_framework, 0)) {
+        return PMIX_ERROR;
+    }
+
+    for (intf =  (pmix_pif_t*)pmix_list_get_first(&pmix_if_list);
+        intf != (pmix_pif_t*)pmix_list_get_end(&pmix_if_list);
+        intf =  (pmix_pif_t*)pmix_list_get_next(intf)) {
+        if (intf->if_kernel_index == if_kindex) {
+            strncpy(if_name, intf->if_name, length);
+            return PMIX_SUCCESS;
+        }
+    }
+    return PMIX_ERROR;
+}
+
+
+#define ADDRLEN 100
+bool
+pmix_ifislocal(const char *hostname)
+{
+    char addrname[NI_MAXHOST]; /* should be larger than ADDRLEN, but I think
+                                  they really mean IFNAMESIZE */
+
+    if (PMIX_SUCCESS == pmix_ifaddrtoname(hostname, addrname, ADDRLEN)) {
+        return true;
+    }
+
+    return false;
+}
+
+static int parse_ipv4_dots(const char *addr, uint32_t* net, int* dots)
+{
+    const char *start = addr, *end;
+    uint32_t n[]={0,0,0,0};
+    int i;
+
+    /* now assemble the address */
+    for( i = 0; i < 4; i++ ) {
+        n[i] = strtoul(start, (char**)&end, 10);
+        if( end == start ) {
+            /* this is not an error, but indicates that
+             * we were given a partial address - e.g.,
+             * 192.168 - usually indicating an IP range
+             * in CIDR notation. So just return what we have
+             */
+            break;
+        }
+        /* did we read something sensible? */
+        if( n[i] > 255 ) {
+            return PMIX_ERR_NETWORK_NOT_PARSEABLE;
+        }
+        /* skip all the . */
+        for( start = end; '\0' != *start; start++ )
+            if( '.' != *start ) break;
+    }
+    *dots = i;
+    *net = PMIX_PIF_ASSEMBLE_NETWORK(n[0], n[1], n[2], n[3]);
+    return PMIX_SUCCESS;
+}
+
+int
+pmix_iftupletoaddr(const char *inaddr, uint32_t *net, uint32_t *mask)
+{
+    int pval, dots, rc = PMIX_SUCCESS;
+    const char *ptr;
+
+    /* if a mask was desired... */
+    if (NULL != mask) {
+        /* set default */
+        *mask = 0xFFFFFFFF;
+
+        /* if entry includes mask, split that off */
+        if (NULL != (ptr = strchr(inaddr, '/'))) {
+            ptr = ptr + 1;  /* skip the / */
+            /* is the mask a tuple? */
+            if (NULL != strchr(ptr, '.')) {
+                /* yes - extract mask from it */
+                rc = parse_ipv4_dots(ptr, mask, &dots);
+            } else {
+                /* no - must be an int telling us how much of the addr to use: e.g., /16
+                 * For more information please read http://en.wikipedia.org/wiki/Subnetwork.
+                 */
+                pval = strtol(ptr, NULL, 10);
+                if ((pval > 31) || (pval < 1)) {
+                    pmix_output(0, "pmix_iftupletoaddr: unknown mask");
+                    return PMIX_ERR_NETWORK_NOT_PARSEABLE;
+                }
+                *mask = 0xFFFFFFFF << (32 - pval);
+            }
+        } else {
+            /* use the number of dots to determine it */
+            for (ptr = inaddr, pval = 0; '\0'!= *ptr; ptr++) {
+                if ('.' == *ptr) {
+                    pval++;
+                }
+            }
+            /* if we have three dots, then we have four
+             * fields since it is a full address, so the
+             * default netmask is fine
+             */
+            if (3 == pval) {
+                *mask = 0xFFFFFFFF;
+            } else if (2 == pval) {         /* 2 dots */
+                *mask = 0xFFFFFF00;
+            } else if (1 == pval) {  /* 1 dot */
+                *mask = 0xFFFF0000;
+            } else if (0 == pval) {  /* no dots */
+                *mask = 0xFF000000;
+            } else {
+                pmix_output(0, "pmix_iftupletoaddr: unknown mask");
+                return PMIX_ERR_NETWORK_NOT_PARSEABLE;
+            }
+        }
+    }
+
+    /* if network addr is desired... */
+    if (NULL != net) {
+        /* now assemble the address */
+        rc = parse_ipv4_dots(inaddr, net, &dots);
+    }
+
+    return rc;
+}
+
+/*
+ *  Determine if the specified interface is loopback
+ */
+
+bool pmix_ifisloopback(int if_index)
+{
+    pmix_pif_t* intf;
+
+    if (PMIX_SUCCESS != pmix_mca_base_framework_open(&pmix_pif_base_framework, 0)) {
+        return PMIX_ERROR;
+    }
+
+    for (intf =  (pmix_pif_t*)pmix_list_get_first(&pmix_if_list);
+        intf != (pmix_pif_t*)pmix_list_get_end(&pmix_if_list);
+        intf =  (pmix_pif_t*)pmix_list_get_next(intf)) {
+        if (intf->if_index == if_index) {
+            if ((intf->if_flags & IFF_LOOPBACK) != 0) {
+                return true;
+            }
+        }
+    }
+    return false;
+}
+
+/* Determine if an interface matches any entry in the given list, taking
+ * into account that the list entries could be given as named interfaces,
+ * IP addrs, or subnet+mask
+ */
+int pmix_ifmatches(int kidx, char **nets)
+{
+    bool named_if;
+    int i, rc;
+    size_t j;
+    int kindex;
+    struct sockaddr_in inaddr;
+    uint32_t addr, netaddr, netmask;
+
+    /* get the address info for the given network in case we need it */
+    if (PMIX_SUCCESS != (rc = pmix_ifkindextoaddr(kidx, (struct sockaddr*)&inaddr, sizeof(inaddr)))) {
+        return rc;
+    }
+    addr = ntohl(inaddr.sin_addr.s_addr);
+
+    for (i=0; NULL != nets[i]; i++) {
+        /* if the specified interface contains letters in it, then it
+         * was given as an interface name and not an IP tuple
+         */
+        named_if = false;
+        for (j=0; j < strlen(nets[i]); j++) {
+            if (isalpha(nets[i][j]) && '.' != nets[i][j]) {
+                named_if = true;
+                break;
+            }
+        }
+        if (named_if) {
+            if (0 > (kindex = pmix_ifnametokindex(nets[i]))) {
+                continue;
+            }
+            if (kindex == kidx) {
+                return PMIX_SUCCESS;
+            }
+        } else {
+            if (PMIX_SUCCESS != (rc = pmix_iftupletoaddr(nets[i], &netaddr, &netmask))) {
+                pmix_show_help("help-pmix-util.txt", "invalid-net-mask", true, nets[i]);
+                return rc;
+            }
+            if (netaddr == (addr & netmask)) {
+                return PMIX_SUCCESS;
+            }
+        }
+    }
+    /* get here if not found */
+    return PMIX_ERR_NOT_FOUND;
+}
+
+void pmix_ifgetaliases(char ***aliases)
+{
+    pmix_pif_t* intf;
+    char ipv4[INET_ADDRSTRLEN];
+    struct sockaddr_in *addr;
+    char ipv6[INET6_ADDRSTRLEN];
+    struct sockaddr_in6 *addr6;
+
+    /* set default answer */
+    *aliases = NULL;
+
+    if (PMIX_SUCCESS != pmix_mca_base_framework_open(&pmix_pif_base_framework, 0)) {
+        return;
+    }
+
+    for (intf =  (pmix_pif_t*)pmix_list_get_first(&pmix_if_list);
+        intf != (pmix_pif_t*)pmix_list_get_end(&pmix_if_list);
+        intf =  (pmix_pif_t*)pmix_list_get_next(intf)) {
+        addr = (struct sockaddr_in*) &intf->if_addr;
+        /* ignore purely loopback interfaces */
+        if ((intf->if_flags & IFF_LOOPBACK) != 0) {
+            continue;
+        }
+        if (addr->sin_family == AF_INET) {
+            inet_ntop(AF_INET, &(addr->sin_addr.s_addr), ipv4, INET_ADDRSTRLEN);
+            pmix_argv_append_nosize(aliases, ipv4);
+        } else {
+            addr6 = (struct sockaddr_in6*) &intf->if_addr;
+            inet_ntop(AF_INET6, &(addr6->sin6_addr), ipv6, INET6_ADDRSTRLEN);
+            pmix_argv_append_nosize(aliases, ipv6);
+        }
+    }
+}
+
+#else /* HAVE_STRUCT_SOCKADDR_IN */
+
+/* if we don't have struct sockaddr_in, we don't have traditional
+   ethernet devices.  Just make everything a no-op error call */
+
+int
+pmix_ifnametoaddr(const char* if_name,
+                  struct sockaddr* if_addr, int size)
+{
+    return PMIX_ERR_NOT_SUPPORTED;
+}
+
+int
+pmix_ifaddrtoname(const char* if_addr,
+                  char* if_name, int size)
+{
+    return PMIX_ERR_NOT_SUPPORTED;
+}
+
+int
+pmix_ifnametoindex(const char* if_name)
+{
+    return PMIX_ERR_NOT_SUPPORTED;
+}
+
+int16_t
+pmix_ifnametokindex(const char* if_name)
+{
+    return PMIX_ERR_NOT_SUPPORTED;
+}
+
+int
+pmix_ifindextokindex(int if_index)
+{
+    return PMIX_ERR_NOT_SUPPORTED;
+}
+
+int
+pmix_ifcount(void)
+{
+    return PMIX_ERR_NOT_SUPPORTED;
+}
+
+int
+pmix_ifbegin(void)
+{
+    return PMIX_ERR_NOT_SUPPORTED;
+}
+
+int
+pmix_ifnext(int if_index)
+{
+    return PMIX_ERR_NOT_SUPPORTED;
+}
+
+int
+pmix_ifindextoname(int if_index, char* if_name, int length)
+{
+    return PMIX_ERR_NOT_SUPPORTED;
+}
+
+int
+pmix_ifkindextoname(int kif_index, char* if_name, int length)
+{
+    return PMIX_ERR_NOT_SUPPORTED;
+}
+
+int
+pmix_ifindextoaddr(int if_index, struct sockaddr* if_addr, unsigned int length)
+{
+    return PMIX_ERR_NOT_SUPPORTED;
+}
+
+int
+pmix_ifindextomask(int if_index, uint32_t* if_addr, int length)
+{
+    return PMIX_ERR_NOT_SUPPORTED;
+}
+
+bool
+pmix_ifislocal(const char *hostname)
+{
+    return false;
+}
+
+int
+pmix_iftupletoaddr(const char *inaddr, uint32_t *net, uint32_t *mask)
+{
+    return 0;
+}
+
+int pmix_ifmatches(int idx, char **nets)
+{
+    return PMIX_ERR_NOT_SUPPORTED;
+}
+
+void pmix_ifgetaliases(char ***aliases)
+{
+    /* set default answer */
+    *aliases = NULL;
+}
+
+#endif /* HAVE_STRUCT_SOCKADDR_IN */

opal/mca/pmix/pmix3x/pmix/src/util/pif.h

@@ -0,0 +1,231 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2016      Intel, Inc. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+/* @file */
+
+#ifndef PMIX_PIF_UTIL_
+#define PMIX_PIF_UTIL_
+
+#include "pmix_config.h"
+
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_SOCKET_H
+#include <sys/socket.h>
+#endif
+#ifdef HAVE_NETINET_IN_H
+#include <netinet/in.h>
+#endif
+
+#ifndef IF_NAMESIZE
+#define IF_NAMESIZE 32
+#endif
+
+BEGIN_C_DECLS
+
+#define PMIX_PIF_FORMAT_ADDR(n)                              \
+    (((n) >> 24) & 0x000000FF), (((n) >> 16) & 0x000000FF), \
+    (((n) >> 8) & 0x000000FF), ((n) & 0x000000FF)
+
+#define PMIX_PIF_ASSEMBLE_NETWORK(n1, n2, n3, n4)    \
+    (((n1) << 24) & 0xFF000000) |                   \
+    (((n2) << 16) & 0x00FF0000) |                   \
+    (((n3) <<  8) & 0x0000FF00) |                   \
+    ( (n4)        & 0x000000FF)
+
+/**
+ *  Lookup an interface by name and return its primary address.
+ *
+ *  @param if_name (IN)   Interface name
+ *  @param if_addr (OUT)  Interface address buffer
+ *  @param size    (IN)   Interface address buffer size
+ */
+PMIX_EXPORT int pmix_ifnametoaddr(const char* if_name,
+                                  struct sockaddr* if_addr,
+                                  int size);
+
+/**
+ *  Lookup an interface by address and return its name.
+ *
+ *  @param if_addr (IN)   Interface address (hostname or dotted-quad)
+ *  @param if_name (OUT)  Interface name buffer
+ *  @param size    (IN)   Interface name buffer size
+ */
+PMIX_EXPORT int pmix_ifaddrtoname(const char* if_addr,
+                                  char* if_name, int size);
+
+/**
+ *  Lookup an interface by name and return its pmix_list index.
+ *
+ *  @param if_name (IN)  Interface name
+ *  @return              Interface pmix_list index
+ */
+PMIX_EXPORT int pmix_ifnametoindex(const char* if_name);
+
+/**
+ *  Lookup an interface by name and return its kernel index.
+ *
+ *  @param if_name (IN)  Interface name
+ *  @return              Interface kernel index
+ */
+PMIX_EXPORT int16_t pmix_ifnametokindex(const char* if_name);
+
+/*
+ *  Attempt to resolve an address (given as either IPv4/IPv6 string
+ *  or hostname) and return the kernel index of the interface
+ *  that is on the same network as the specified address
+ */
+PMIX_EXPORT int16_t pmix_ifaddrtokindex(const char* if_addr);
+
+/**
+ *  Lookup an interface by pmix_list index and return its kernel index.
+ *
+ *  @param if_name (IN)  Interface pmix_list index
+ *  @return              Interface kernel index
+ */
+PMIX_EXPORT int pmix_ifindextokindex(int if_index);
+
+/**
+ *  Returns the number of available interfaces.
+ */
+PMIX_EXPORT int pmix_ifcount(void);
+
+/**
+ *  Returns the index of the first available interface.
+ */
+PMIX_EXPORT int pmix_ifbegin(void);
+
+/**
+ *  Lookup the current position in the interface list by
+ *  index and return the next available index (if it exists).
+ *
+ *  @param if_index   Returns the next available index from the
+ *                    current position.
+ */
+PMIX_EXPORT int pmix_ifnext(int if_index);
+
+/**
+ *  Lookup an interface by index and return its name.
+ *
+ *  @param if_index (IN)  Interface index
+ *  @param if_name (OUT)  Interface name buffer
+ *  @param size (IN)      Interface name buffer size
+ */
+PMIX_EXPORT int pmix_ifindextoname(int if_index, char* if_name, int);
+
+/**
+ *  Lookup an interface by kernel index and return its name.
+ *
+ *  @param if_index (IN)  Interface kernel index
+ *  @param if_name (OUT)  Interface name buffer
+ *  @param size (IN)      Interface name buffer size
+ */
+PMIX_EXPORT int pmix_ifkindextoname(int if_kindex, char* if_name, int);
+
+/**
+ *  Lookup an interface by index and return its primary address.
+ *
+ *  @param if_index (IN)  Interface index
+ *  @param if_name (OUT)  Interface address buffer
+ *  @param size (IN)      Interface address buffer size
+ */
+PMIX_EXPORT int pmix_ifindextoaddr(int if_index, struct sockaddr*,
+                                   unsigned int);
+PMIX_EXPORT int pmix_ifkindextoaddr(int if_kindex,
+                                    struct sockaddr* if_addr,
+                                    unsigned int length);
+
+/**
+ *  Lookup an interface by index and return its network mask (in CIDR
+ *  notation -- NOT the actual netmask itself!).
+ *
+ *  @param if_index (IN)  Interface index
+ *  @param if_name (OUT)  Interface address buffer
+ *  @param size (IN)      Interface address buffer size
+ */
+PMIX_EXPORT int pmix_ifindextomask(int if_index, uint32_t*, int);
+
+/**
+ *  Lookup an interface by index and return its MAC address.
+ *
+ *  @param if_index (IN)  Interface index
+ *  @param if_mac (OUT)   Interface's MAC address
+ */
+PMIX_EXPORT int pmix_ifindextomac(int if_index, uint8_t if_mac[6]);
+
+/**
+ *  Lookup an interface by index and return its MTU.
+ *
+ *  @param if_index (IN)  Interface index
+ *  @param if_mtu (OUT)   Interface's MTU
+ */
+PMIX_EXPORT int pmix_ifindextomtu(int if_index, int *mtu);
+
+/**
+ *  Lookup an interface by index and return its flags.
+ *
+ *  @param if_index (IN)  Interface index
+ *  @param if_flags (OUT) Interface flags
+ */
+PMIX_EXPORT int pmix_ifindextoflags(int if_index, uint32_t*);
+
+/**
+ * Determine if given hostname / IP address is a local address
+ *
+ * @param hostname (IN)    Hostname (or stringified IP address)
+ * @return                 true if \c hostname is local, false otherwise
+ */
+PMIX_EXPORT bool pmix_ifislocal(const char *hostname);
+
+/**
+ * Convert a dot-delimited network tuple to an IP address
+ *
+ * @param addr (IN) character string tuple
+ * @param net (IN) Pointer to returned network address
+ * @param mask (IN) Pointer to returned netmask
+ * @return PMIX_SUCCESS if no problems encountered
+ * @return PMIX_ERROR if data could not be released
+ */
+PMIX_EXPORT int pmix_iftupletoaddr(const char *addr, uint32_t *net, uint32_t *mask);
+
+/**
+ * Determine if given interface is loopback
+ *
+ *  @param if_index (IN)  Interface index
+ */
+PMIX_EXPORT bool pmix_ifisloopback(int if_index);
+
+/*
+ * Determine if a specified interface is included in a NULL-terminated argv array
+ */
+PMIX_EXPORT int pmix_ifmatches(int kidx, char **nets);
+
+/*
+ * Provide a list of strings that contain all known aliases for this node
+ */
+PMIX_EXPORT void pmix_ifgetaliases(char ***aliases);
+
+END_C_DECLS
+
+#endif

opal/mca/pmix/pmix3x/pmix/src/util/tsd.h

@@ -0,0 +1,172 @@
+/*
+ * Copyright (c) 2007-2013 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2008      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016      Intel, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+
+#ifndef PMIX_THREADS_TSD_H
+#define PMIX_THREADS_TSD_H
+
+#include "pmix_config.h"
+#include "pmix_common.h"
+
+#include <pthread.h>
+
+
+BEGIN_C_DECLS
+
+/**
+ * @file
+ *
+ * Thread Specific Datastore Interface
+ *
+ * Functions for providing thread-specific datastore capabilities.
+ */
+
+
+/**
+ * Prototype for callback when tsd data is being destroyed
+ */
+typedef void (*pmix_tsd_destructor_t)(void *value);
+
+#if defined(DOXYGEN)
+
+/**
+ * Typedef for thread-specific data key
+ */
+typedef void* pmix_tsd_key_t;
+
+
+/**
+ * Create thread-specific data key
+ *
+ * Create a thread-specific data key visible to all threads in the
+ * current process.  The returned key is valid in all threads,
+ * although the values bound to the key by pmix_tsd_setspecific() are
+ * allocated on a per-thread basis and persist for the life of the
+ * calling thread.
+ *
+ * Upon key creation, the value NULL is associated with the new key in
+ * all active threads.  When a new thread is created, the value NULL
+ * is associated with all defined keys in the new thread.
+ *
+ * The destructor parameter may be NULL.  At thread exit, if
+ * destructor is non-NULL AND the thread has a non-NULL value
+ * associated with the key, the function is called with the current
+ * value as its argument.
+ *
+ * @param key[out]       The key for accessing thread-specific data
+ * @param destructor[in] Cleanup function to call when a thread exits
+ *
+ * @retval PMIX_SUCCESS  Success
+ * @retval EAGAIN        The system lacked the necessary resource to
+ *                       create another thread specific data key
+ * @retval ENOMEM        Insufficient memory exists to create the key
+ */
+PMIX_EXPORT int pmix_tsd_key_create(pmix_tsd_key_t *key,
+                                    pmix_tsd_destructor_t destructor);
+
+
+/**
+ * Delete a thread-specific data key
+ *
+ * Delete a thread-specific data key previously returned by
+ * pmix_tsd_key_create().  The destructor associated with the key is
+ * not fired in any thread and memory cleanup is the responsibility of
+ * the caller.
+ *
+ * @note Unlike pthread_key_delete, this function should not be called
+ * from within a destructor.  It can not be universally supported at
+ * this time.
+ *
+ * @param key[in]       The key for accessing thread-specific data
+ *
+ * @retval PMIX_SUCCESS Success
+ * @retval EINVAL       Invalid key
+ */
+PMIX_EXPORT int pmix_tsd_key_delete(pmix_tsd_key_t key);
+
+
+/**
+ * Set a thread-specific data value
+ *
+ * Associates value with key in the current thread.  The value for the
+ * key in other threads is not changed.  Different threads may assign
+ * different values to the same key.
+ *
+ * @note This function should not be called within
+ * pmix_tsd_key_delete().
+ *
+ * @param key[in]       Thread specific data key to modify
+ * @param value[in]     Value to associate with key
+ *
+ * @retval PMIX_SUCCESS Success
+ * @retval ENOMEM       Insufficient memory exists to associate the
+ *                      value with the key
+ * @retval EINVAL       Invalid key
+ */
+PMIX_EXPORT int pmix_tsd_setspecific(pmix_tsd_key_t key, void *value);
+
+
+/**
+ * Get a thread-specific data value
+ *
+ * Get the data associated with the given key, as set by
+ * pmix_tsd_setspecific().  If pmix_tsd_setspecific() hasn't been
+ * called in the current thread with the given key, NULL is returned
+ * in valuep.
+ *
+ * @param key[in]       Thread specific data key to modify
+ * @param value[out]     Value to associate with key
+ *
+ * @retval PMIX_SUCCESS Success
+ * @retval ENOMEM       Insufficient memory exists to associate the
+ *                      value with the key
+ * @retval EINVAL       Invalid key
+ */
+PMIX_EXPORT int pmix_tsd_getspecific(pmix_tsd_key_t key, void **valuep);
+
+#else
+
+typedef pthread_key_t pmix_tsd_key_t;
+
+static inline int
+pmix_tsd_key_create(pmix_tsd_key_t *key,
+                    pmix_tsd_destructor_t destructor)
+{
+    return pthread_key_create(key, destructor);
+}
+
+static inline int
+pmix_tsd_key_delete(pmix_tsd_key_t key)
+{
+    return pthread_key_delete(key);
+}
+
+static inline int
+pmix_tsd_setspecific(pmix_tsd_key_t key, void *value)
+{
+    return pthread_setspecific(key, value);
+}
+
+static inline int
+pmix_tsd_getspecific(pmix_tsd_key_t key, void **valuep)
+{
+    *valuep = pthread_getspecific(key);
+    return PMIX_SUCCESS;
+}
+
+#endif
+
+END_C_DECLS
+
+#endif /* PMIX_MTHREADS_TSD_H */