Update PMIx support to latest PMIx master

opal/mca/pmix/pmix3x/pmix/NEWS

@@ -23,6 +23,16 @@ current release as well as the "stable" bug fix release branch.
 Master (not on release branches yet)
 ------------------------------------
 
+1.1.5
+-----
+- Add pmix_version.h to support direct detection of PMIx library version
+- Fix support for Solaris 10 by using abstract version of strnlen
+- Fix native security module for Solaris by using getpeerucred in
+  that environment
+- Ensure man pages don't get installed in embedded builds
+- Pass temporary directory locations in info keys instead of
+  the environment
+
 1.1.4
 -----
 - Properly increment the reference count for PMIx_Init

opal/mca/pmix/pmix3x/pmix/VERSION

@@ -30,7 +30,7 @@ greek=
 # command, or with the date (if "git describe" fails) in the form of
 # "date<date>".
 
-repo_rev=git9cbb1bf
+repo_rev=git6ea0747
 
 # If tarball_version is not empty, it is used as the version string in
 # the tarball filename, regardless of all other versions listed in
@@ -44,7 +44,7 @@ tarball_version=
 
 # The date when this release was created
 
-date="Aug 19, 2016"
+date="Oct 11, 2016"
 
 # The shared library version of each of PMIx's public libraries.
 # These versions are maintained in accordance with the "Library

opal/mca/pmix/pmix3x/pmix/config/pmix.m4

@@ -17,7 +17,7 @@ dnl Copyright (c) 2009      Los Alamos National Security, LLC.  All rights
 dnl                         reserved.
 dnl Copyright (c) 2009-2011 Oak Ridge National Labs.  All rights reserved.
 dnl Copyright (c) 2011-2013 NVIDIA Corporation.  All rights reserved.
-dnl Copyright (c) 2013-2015 Intel, Inc. All rights reserved
+dnl Copyright (c) 2013-2016 Intel, Inc. All rights reserved
 dnl Copyright (c) 2015-2016 Research Organization for Information Science
 dnl                         and Technology (RIST). All rights reserved.
 dnl Copyright (c) 2016      Mellanox Technologies, Inc.
@@ -104,6 +104,12 @@ AC_DEFUN([PMIX_SETUP_CORE],[
     AC_DEFINE_UNQUOTED([PMIX_MINOR_VERSION], [$PMIX_MINOR_VERSION],
                        [The library minor version is always available, contrary to VERSION])
 
+    pmixmajor=${PMIX_MAJOR_VERSION}L
+    pmixminor=${PMIX_MINOR_VERSION}L
+    AC_SUBST(pmixmajor)
+    AC_SUBST(pmixminor)
+    AC_CONFIG_FILES(pmix_config_prefix[include/pmix_version.h])
+
     PMIX_RELEASE_VERSION="`$PMIX_top_srcdir/config/pmix_get_version.sh $PMIX_top_srcdir/VERSION --release`"
     if test "$?" != "0"; then
         AC_MSG_ERROR([Cannot continue])
@@ -136,6 +142,23 @@ AC_DEFUN([PMIX_SETUP_CORE],[
     # replaced, not the entire file.
     AC_CONFIG_HEADERS(pmix_config_prefix[src/include/pmix_config.h])
 
+    # Rename symbols?
+    AC_ARG_WITH([pmix-symbol-rename],
+                AC_HELP_STRING([--with-pmix-symbol-rename=PREFIX],
+                               [Provide a prefix to rename PMIx symbols]))
+    AC_MSG_CHECKING([for symbol rename])
+    AS_IF([test ! -z "$with_pmix_symbol_rename" && test "$with_pmix_symbol_rename" != "yes"],
+          [AC_MSG_RESULT([$with_pmix_symbol_rename])
+           pmix_symbol_rename="$with_pmix_symbol_rename"
+           PMIX_RENAME=$with_pmix_symbol_rename],
+          [AC_MSG_RESULT([no])
+           pmix_symbol_rename=""
+           PMIX_RENAME=])
+    AC_DEFINE_UNQUOTED(PMIX_SYMBOL_RENAME, [$pmix_symbol_rename],
+                       [The pmix symbol rename include directive])
+    AC_SUBST(PMIX_RENAME)
+    AC_CONFIG_FILES(pmix_config_prefix[include/pmix_rename.h])
+
     # GCC specifics.
     if test "x$GCC" = "xyes"; then
         PMIX_GCC_CFLAGS="-Wall -Wmissing-prototypes -Wundef"
@@ -592,20 +615,6 @@ AC_DEFUN([PMIX_SETUP_CORE],[
 
     PMIX_HWLOC_CONFIG
 
-    ##################################
-    # SASL
-    ##################################
-    pmix_show_title "SASL"
-
-    PMIX_SASL_CONFIG
-
-    ##################################
-    # Munge
-    ##################################
-    pmix_show_title "Munge"
-
-    PMIX_MUNGE_CONFIG
-
     ##################################
     # MCA
     ##################################
@@ -698,16 +707,6 @@ AC_DEFUN([PMIX_DEFINE_ARGS],[
           [pmix_mode=standalone
            AC_MSG_RESULT([no])])
 
-    # Rename symbols?
-    AC_ARG_WITH([pmix-symbol-rename],
-                AC_HELP_STRING([--with-pmix-symbol-rename=FILE],
-                               [Provide an include file that contains directives to rename PMIx symbols]))
-    AS_IF([test ! -z "$with_pmix_symbol_rename" && test "$with_pmix_symbol_rename" != "yes"],
-          [pmix_symbol_rename="$with_pmix_symbol_rename"],
-          [pmix_symbol_rename=\"src/include/rename.h\"])
-    AC_DEFINE_UNQUOTED(PMIX_SYMBOL_RENAME, [$pmix_symbol_rename],
-                       [The pmix symbol rename include directive])
-
     # Install tests and examples?
     AC_MSG_CHECKING([if tests and examples are to be installed])
     AC_ARG_WITH([tests-examples],
@@ -840,16 +839,17 @@ AC_MSG_CHECKING([if want shared memory datastore])
 AC_ARG_ENABLE([dstore],
               [AC_HELP_STRING([--disable-dstore],
                               [Using shared memory datastore (default: enabled)])])
-if test "$enable_dstore" != "no" ; then
+if test "$enable_dstore" == "no" ; then
-    AC_MSG_RESULT([yes])
-    WANT_DSTORE=1
-else
     AC_MSG_RESULT([no])
     WANT_DSTORE=0
+else
+    AC_MSG_RESULT([yes])
+    WANT_DSTORE=1
 fi
 AC_DEFINE_UNQUOTED([PMIX_ENABLE_DSTORE],
-                   [$WANT_DSTORE],
+                 [$WANT_DSTORE],
-                   [if want shared memory dstore feature])
+                 [if want shared memory dstore feature])
+
 #
 
 # Ident string

opal/mca/pmix/pmix3x/pmix/config/pmix_check_munge.m4

@@ -1,83 +0,0 @@
-# -*- shell-script -*-
-#
-# Copyright (c) 2015-2016 Intel, Inc. All rights reserved
-# Copyright (c) 2015      Cisco Systems, Inc.  All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-# MCA_munge_CONFIG([action-if-found], [action-if-not-found])
-# --------------------------------------------------------------------
-AC_DEFUN([PMIX_MUNGE_CONFIG],[
-
-    PMIX_VAR_SCOPE_PUSH([pmix_munge_dir pmix_munge_libdir])
-
-    AC_ARG_WITH([munge],
-                [AC_HELP_STRING([--with-munge=DIR],
-                                [Search for munge headers and libraries in DIR ])])
-
-    AC_ARG_WITH([munge-libdir],
-                [AC_HELP_STRING([--with-munge-libdir=DIR],
-                                [Search for munge libraries in DIR ])])
-
-    pmix_munge_support=0
-    if test ! -z "$with_munge" && test "$with_munge" != "no"; then
-        AC_MSG_CHECKING([for munge in])
-        if test "$with_munge" != "yes"; then
-            if test -d $with_munge/include/munge; then
-                pmix_munge_dir=$with_munge/include/munge
-            else
-                pmix_munge_dir=$with_munge
-            fi
-            if test -d $with_munge/lib; then
-                pmix_munge_libdir=$with_munge/lib
-            elif test -d $with_munge/lib64; then
-                pmix_munge_libdir=$with_munge/lib64
-            else
-                AC_MSG_RESULT([Could not find $with_munge/lib or $with_munge/lib64])
-                AC_MSG_ERROR([Can not continue])
-            fi
-            AC_MSG_RESULT([$pmix_munge_dir and $pmix_munge_libdir])
-        else
-            AC_MSG_RESULT([(default search paths)])
-            pmix_munge_dir=
-        fi
-        AS_IF([test ! -z "$with_munge_libdir" && test "$with_munge_libdir" != "yes"],
-              [pmix_munge_libdir="$with_munge_libdir"])
-
-        PMIX_CHECK_PACKAGE([pmix_munge],
-                           [munge.h],
-                           [munge],
-                           [munge_encode],
-                           [-lmunge],
-                           [$pmix_munge_dir],
-                           [$pmix_munge_libdir],
-                           [pmix_munge_support=1],
-                           [pmix_munge_support=0])
-        if test $pmix_munge_support == "1"; then
-            CPPFLAGS="$pmix_munge_CPPFLAGS $CPPFLAGS"
-            LIBS="$LIBS -lmunge"
-            LDFLAGS="$pmix_munge_LDFLAGS $LDFLAGS"
-        fi
-    fi
-
-    if test ! -z "$with_munge" && test "$with_munge" != "no" && test "$pmix_munge_support" != "1"; then
-        AC_MSG_WARN([MUNGE SUPPORT REQUESTED AND NOT FOUND.])
-        AC_MSG_ERROR([CANNOT CONTINUE])
-    fi
-
-    AC_MSG_CHECKING([will munge support be built])
-    if test "$pmix_munge_support" != "1"; then
-        AC_MSG_RESULT([no])
-    else
-        AC_MSG_RESULT([yes])
-    fi
-
-    AC_DEFINE_UNQUOTED([PMIX_WANT_MUNGE], [$pmix_munge_support],
-                       [Whether we want munge support or not])
-
-    PMIX_VAR_SCOPE_POP
-])dnl

opal/mca/pmix/pmix3x/pmix/config/pmix_check_sasl.m4

@@ -1,82 +0,0 @@
-# -*- shell-script -*-
-#
-# Copyright (c) 2015      Intel, Inc. All rights reserved
-# Copyright (c) 2015      Cisco Systems, Inc.  All rights reserved.
-# Copyright (c) 2015      Research Organization for Information Science
-#                         and Technology (RIST). All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-# MCA_sasl_CONFIG([action-if-found], [action-if-not-found])
-# --------------------------------------------------------------------
-AC_DEFUN([PMIX_SASL_CONFIG],[
-
-    PMIX_VAR_SCOPE_PUSH([pmix_sasl_dir pmix_sasl_libdir])
-
-    AC_ARG_WITH([sasl],
-                [AC_HELP_STRING([--with-sasl=DIR],
-                                [Search for sasl headers and libraries in DIR ])],
-               [], [with_sasl=no])
-
-    AC_ARG_WITH([sasl-libdir],
-                [AC_HELP_STRING([--with-sasl-libdir=DIR],
-                                [Search for sasl libraries in DIR ])])
-
-    pmix_sasl_support=0
-    if test ! -z "$with_sasl" && test "$with_sasl" != "no"; then
-        AC_MSG_CHECKING([for sasl in])
-        if test "$with_sasl" != "yes"; then
-            pmix_sasl_dir=$with_sasl/include/sasl
-            if test -d $with_sasl/lib; then
-                pmix_sasl_libdir=$with_sasl/lib
-            elif test -d $with_sasl/lib64; then
-                pmix_sasl_libdir=$with_sasl/lib64
-            else
-                AC_MSG_RESULT([Could not find $with_sasl/lib or $with_sasl/lib64])
-                AC_MSG_ERROR([Can not continue])
-            fi
-            AC_MSG_RESULT([$pmix_sasl_dir and $pmix_sasl_libdir])
-        else
-            AC_MSG_RESULT([(default search paths)])
-            pmix_sasl_dir=
-        fi
-        AS_IF([test ! -z "$with_sasl_libdir" && test "$with_sasl_libdir" != "yes"],
-              [pmix_sasl_libdir="$with_sasl_libdir"])
-
-        PMIX_CHECK_PACKAGE([pmix_sasl],
-                           [sasl/sasl.h],
-                           [sasl2],
-                           [sasl_server_init],
-                           [-lsasl2],
-                           [$pmix_sasl_dir],
-                           [$pmix_sasl_libdir],
-                           [pmix_sasl_support=1],
-                           [pmix_sasl_support=0])
-        if test $pmix_sasl_support == "1"; then
-            CPPFLAGS="$pmix_sasl_CPPFLAGS $CPPFLAGS"
-            LIBS="$LIBS -lsasl2"
-            LDFLAGS="$pmix_sasl_LDFLAGS $LDFLAGS"
-        fi
-    fi
-
-    if test ! -z "$with_sasl" && test "$with_sasl" != "no" && test "$pmix_sasl_support" != "1"; then
-        AC_MSG_WARN([SASL SUPPORT REQUESTED AND NOT FOUND.])
-        AC_MSG_ERROR([CANNOT CONTINUE])
-    fi
-
-    AC_MSG_CHECKING([will sasl support be built])
-    if test "$pmix_sasl_support" != "1"; then
-        AC_MSG_RESULT([no])
-    else
-        AC_MSG_RESULT([yes])
-    fi
-
-    AC_DEFINE_UNQUOTED(PMIX_HAVE_SASL, [$pmix_sasl_support],
-                       [Whether we have sasl support or not])
-
-    PMIX_VAR_SCOPE_POP
-])dnl

opal/mca/pmix/pmix3x/pmix/config/pmix_setup_libevent.m4

@@ -124,7 +124,7 @@ AC_DEFUN([_PMIX_LIBEVENT_EXTERNAL],[
     # Set output variables
     PMIX_EVENT_HEADER="<event.h>"
     PMIX_EVENT2_THREAD_HEADER="<event2/thread.h>"
-    PMIX_EVENT_LIB=-levent
+    PMIX_EVENT_LIB="-levent -levent_pthreads"
     AS_IF([test "$pmix_event_dir" != ""],
         [PMIX_EVENT_CPPFLAGS="-I$pmix_event_dir/include"])
     AS_IF([test "$pmix_event_libdir" != ""],

opal/mca/pmix/pmix3x/pmix/include/Makefile.am

@@ -19,4 +19,8 @@ include_HEADERS = \
         pmi2.h \
         pmix_tool.h
 
+nodist_include_HEADERS = \
+    pmix_version.h \
+    pmix_rename.h
+
 endif ! PMIX_EMBEDDED_MODE

opal/mca/pmix/pmix3x/pmix/include/pmix.h

@@ -48,7 +48,7 @@
 
 /* Structure and constant definitions */
 #include <pmix_common.h>
-
+#include <pmix_rename.h>
 
 #if defined(c_plusplus) || defined(__cplusplus)
 extern "C" {

opal/mca/pmix/pmix3x/pmix/include/pmix_common.h

@@ -59,6 +59,7 @@
 #include <sys/time.h> /* for struct timeval */
 #include <unistd.h> /* for uid_t and gid_t */
 #include <sys/types.h> /* for uid_t and gid_t */
+#include <pmix_version.h>
 
 #if defined(c_plusplus) || defined(__cplusplus)
 extern "C" {
@@ -136,6 +137,7 @@ typedef uint32_t pmix_rank_t;
 #define PMIX_TMPDIR                         "pmix.tmpdir"           // (char*) top-level tmp dir assigned to session
 #define PMIX_NSDIR                          "pmix.nsdir"            // (char*) sub-tmpdir assigned to namespace
 #define PMIX_PROCDIR                        "pmix.pdir"             // (char*) sub-nsdir assigned to proc
+#define PMIX_TDIR_RMCLEAN                   "pmix.tdir.rmclean"     // (bool)  Resource Manager will clean session directories
 
 /* information about relative ranks as assigned by the RM */
 #define PMIX_JOBID                          "pmix.jobid"            // (char*) jobid assigned by scheduler
@@ -168,6 +170,7 @@ typedef uint32_t pmix_rank_t;
 #define PMIX_LOCAL_SIZE                     "pmix.local.size"       // (uint32_t) #procs in this job on this node
 #define PMIX_NODE_SIZE                      "pmix.node.size"        // (uint32_t) #procs across all jobs on this node
 #define PMIX_MAX_PROCS                      "pmix.max.size"         // (uint32_t) max #procs for this job
+#define PMIX_NUM_NODES                      "pmix.num.nodes"        // (uint32_t) #nodes in this nspace
 
 /* topology info */
 #define PMIX_NET_TOPO                       "pmix.ntopo"            // (char*) xml-representation of network topology
@@ -256,8 +259,7 @@ typedef uint32_t pmix_rank_t;
 #define PMIX_QUERY_LOCAL_PROC_TABLE         "pmix.qry.lptable"       // (char*) input nspace of job whose info is being requested
                                                                      //     returns (pmix_data_array_t) an array of pmix_proc_info_t for
                                                                      //     procs in job on same node
-#define PMIX_QUERY_AUTHORIZATIONS           "pmix.qry.auths"         // return operations tool is authorized to perform
+#define PMIX_QUERY_AUTHORIZATIONS           "pmix.qry.auths"         // return operations tool is authorized to perform"
-#define PMIX_QUERY_MEMPROFILE               "pmix.qry.mempfle"       // (pmix_usage_t) return an array of memory usage stats
 
 /* log attributes */
 #define PMIX_LOG_STDERR                     "pmix.log.stderr"        // (bool) log data to stderr
@@ -559,29 +561,6 @@ typedef struct pmix_proc_info {
     } while (0)
 
 
-/****    PMIX USAGE STRUCT    ****/
-typedef struct pmix_usage {
-    pmix_proc_t proc;
-    char key[PMIX_MAX_KEYLEN+1];
-    float usage;
-} pmix_usage_t;
-#define PMIX_USAGE_CREATE(m, n)                                     \
-    do {                                                            \
-        (m) = (pmix_usage_t*)calloc((n) , sizeof(pmix_usage_t));    \
-    } while (0)
-
-#define PMIX_USAGE_RELEASE(m)  PMIX_USAGE_FREE((m))
-
-#define PMIX_USAGE_CONSTRUCT(m)                 \
-    do {                                        \
-        memset((m), 0, sizeof(pmix_usage_t));   \
-    } while (0)
-
-#define PMIX_USAGE_DESTRUCT(m)
-
-#define PMIX_USAGE_FREE(m, n)  free((m))
-
-
 /****    PMIX VALUE STRUCT    ****/
 typedef struct pmix_data_array {
     pmix_data_type_t type;

opal/mca/pmix/pmix3x/pmix/include/pmix_rename.h.in

@@ -0,0 +1,71 @@
+/*
+ * Copyright (c) 2016      Intel, Inc. All rights reserved
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef PMIX_RENAME_H
+#define PMIX_RENAME_H
+
+#define @PMIX_RENAME@PMIx_Init                      PMIx_Init
+#define @PMIX_RENAME@PMIx_Finalize                  PMIx_Finalize
+#define @PMIX_RENAME@PMIx_Abort                     PMIx_Abort
+#define @PMIX_RENAME@PMIx_Put                       PMIx_Put
+#define @PMIX_RENAME@PMIx_Commit                    PMIx_Commit
+#define @PMIX_RENAME@PMIx_Fence                     PMIx_Fence
+#define @PMIX_RENAME@PMIx_Fence_nb                  PMIx_Fence_nb
+#define @PMIX_RENAME@PMIx_Get                       PMIx_Get
+#define @PMIX_RENAME@PMIx_Get_nb                    PMIx_Get_nb
+#define @PMIX_RENAME@PMIx_Publish                   PMIx_Publish
+#define @PMIX_RENAME@PMIx_Publish_nb                PMIx_Publish_nb
+#define @PMIX_RENAME@PMIx_Lookup                    PMIx_Lookup
+#define @PMIX_RENAME@PMIx_Lookup_nb                 PMIx_Lookup_nb
+#define @PMIX_RENAME@PMIx_Unpublish                 PMIx_Unpublish
+#define @PMIX_RENAME@PMIx_Unpublish_nb              PMIx_Unpublish_nb
+#define @PMIX_RENAME@PMIx_Spawn                     PMIx_Spawn
+#define @PMIX_RENAME@PMIx_Spawn_nb                  PMIx_Spawn_nb
+#define @PMIX_RENAME@PMIx_Connect                   PMIx_Connect
+#define @PMIX_RENAME@PMIx_Connect_nb                PMIx_Connect_nb
+#define @PMIX_RENAME@PMIx_Disconnect                PMIx_Disconnect
+#define @PMIX_RENAME@PMIx_Disconnect_nb             PMIx_Disconnect_nb
+#define @PMIX_RENAME@PMIx_Resolve_peers             PMIx_Resolve_peers
+#define @PMIX_RENAME@PMIx_Resolve_nodes             PMIx_Resolve_nodes
+#define @PMIX_RENAME@PMIx_Query_info_nb             PMIx_Query_info_nb
+#define @PMIX_RENAME@PMIx_Log_nb                    PMIx_Log_nb
+
+#define @PMIX_RENAME@PMIx_server_init               PMIx_server_init
+#define @PMIX_RENAME@PMIx_server_finalize           PMIx_server_finalize
+#define @PMIX_RENAME@PMIx_generate_regex            PMIx_generate_regex
+#define @PMIX_RENAME@PMIx_generate_ppn              PMIx_generate_ppn
+#define @PMIX_RENAME@PMIx_server_register_nspace    PMIx_server_register_nspace
+#define @PMIX_RENAME@PMIx_server_deregister_nspace  PMIx_server_deregister_nspace
+#define @PMIX_RENAME@PMIx_server_register_client    PMIx_server_register_client
+#define @PMIX_RENAME@PMIx_server_deregister_client  PMIx_server_deregister_client
+#define @PMIX_RENAME@PMIx_server_setup_fork         PMIx_server_setup_fork
+#define @PMIX_RENAME@PMIx_server_dmodex_request     PMIx_server_dmodex_request
+
+#define @PMIX_RENAME@PMIx_tool_init                 PMIx_tool_init
+#define @PMIX_RENAME@PMIx_tool_finalize             PMIx_tool_finalize
+
+#define @PMIX_RENAME@PMIx_Register_event_handler    PMIx_Register_event_handler
+#define @PMIX_RENAME@PMIx_Deregister_event_handler  PMIx_Deregister_event_handler
+#define @PMIX_RENAME@PMIx_Notify_event              PMIx_Notify_event
+#define @PMIX_RENAME@PMIx_Error_string              PMIx_Error_string
+#define @PMIX_RENAME@PMIx_Proc_state_string         PMIx_Proc_state_string
+#define @PMIX_RENAME@PMIx_Persistence_string        PMIx_Persistence_string
+#define @PMIX_RENAME@PMIx_Data_range_string         PMIx_Data_range_string
+#define @PMIX_RENAME@PMIx_Info_directives_string    PMIx_Info_directives_string
+#define @PMIX_RENAME@PMIx_Data_type_string          PMIx_Data_type_string
+#define @PMIX_RENAME@PMIx_Get_version               PMIx_Get_version
+#define @PMIX_RENAME@PMIx_Store_internal            PMIx_Store_internal
+
+#define @PMIX_RENAME@pmix_value_load                pmix_value_load
+#define @PMIX_RENAME@pmix_value_xfer                pmix_value_xfer
+#define @PMIX_RENAME@pmix_globals                   pmix_globals
+#define @PMIX_RENAME@pmix_output                    pmix_output
+#define @PMIX_RENAME@pmix_output_verbose            pmix_output_verbose
+
+#endif

opal/mca/pmix/pmix3x/pmix/include/pmix_server.h

@@ -385,7 +385,7 @@ pmix_status_t PMIx_generate_regex(const char *input, char **regex);
  * of ranges. Thus, an input of:
  *     "1-4;2-5;8,10,11,12;6,7,9"
  * would generate a regex of
- *     "[pmix:3x(3);8,10-12;6-7,9]"
+ *     "[pmix:2x(3);8,10-12;6-7,9]"
  *
  * Note that the "pmix" at the beginning of each regex indicates
  * that the PMIx native parser is to be used by the client for

opal/mca/pmix/pmix3x/pmix/include/pmix_version.h.in

@@ -0,0 +1,20 @@
+/*
+ * Copyright (c) 2016      Mellanox Technologies, Inc.
+ *                         All rights reserved.
+ * Copyright (c) 2016      Intel, Inc. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+
+#ifndef PMIx_VERSION_H
+#define PMIx_VERSION_H
+
+/* define PMIx version */
+#define PMIX_VERSION_MAJOR @pmixmajor@
+#define PMIX_VERSION_MINOR @pmixminor@
+
+#endif

opal/mca/pmix/pmix3x/pmix/src/Makefile.am

@@ -73,7 +73,6 @@ include tool/Makefile.include
 include common/Makefile.include
 include buffer_ops/Makefile.am
 include usock/Makefile.am
-include sec/Makefile.am
 if WANT_DSTORE
 include sm/Makefile.include
 include dstore/Makefile.include
@@ -82,3 +81,4 @@ endif
 MAINTAINERCLEANFILES = Makefile.in config.h config.h.in
 DISTCLEANFILES = Makefile
 CLEANFILES = core.* *~
+AM_CFLAGS = -Wall

opal/mca/pmix/pmix3x/pmix/src/buffer_ops/copy.c

@@ -270,7 +270,7 @@ bool pmix_value_cmp(pmix_value_t *p, pmix_value_t *p1)
 /* COPY FUNCTIONS FOR GENERIC PMIX TYPES - we
  * are not allocating memory and so we cannot
  * use the regular copy functions */
-pmix_status_t pmix_value_xfer(pmix_value_t *p, pmix_value_t *src)
+PMIX_EXPORT pmix_status_t pmix_value_xfer(pmix_value_t *p, pmix_value_t *src)
 {
     size_t n, m;
     pmix_status_t rc;

opal/mca/pmix/pmix3x/pmix/src/buffer_ops/print.c

@@ -877,28 +877,31 @@ pmix_status_t pmix_bfrop_print_status(char **output, char *prefix,
                       prefx, (long)src->data.bo.size);
         break;
         case PMIX_PERSIST:
-        rc = asprintf(output, "%sPMIX_VALUE: Data type: PMIX_PERSIST\tValue: %d",
+        rc = asprintf(output, "%sPMIX_VALUE: Data type: PMIX_PERSIST\tValue: %s",
-                      prefx, (int)src->data.persist);
+                      prefx, PMIx_Persistence_string(src->data.persist));
         break;
         case PMIX_SCOPE:
-        rc = asprintf(output, "%sPMIX_VALUE: Data type: PMIX_SCOPE\tValue: %d",
+        rc = asprintf(output, "%sPMIX_VALUE: Data type: PMIX_SCOPE\tValue: %s",
-                      prefx, (int)src->data.scope);
+                      prefx, PMIx_Scope_string(src->data.scope));
         break;
         case PMIX_DATA_RANGE:
-        rc = asprintf(output, "%sPMIX_VALUE: Data type: PMIX_DATA_RANGE\tValue: %d",
+        rc = asprintf(output, "%sPMIX_VALUE: Data type: PMIX_DATA_RANGE\tValue: %s",
-                      prefx, (int)src->data.range);
+                      prefx, PMIx_Data_range_string(src->data.range));
         break;
         case PMIX_PROC_STATE:
-        rc = asprintf(output, "%sPMIX_VALUE: Data type: PMIX_STATE\tValue: %d",
+        rc = asprintf(output, "%sPMIX_VALUE: Data type: PMIX_STATE\tValue: %s",
-                      prefx, (int)src->data.state);
+                      prefx, PMIx_Proc_state_string(src->data.state));
         break;
         case PMIX_PROC_INFO:
-        rc = asprintf(output, "%sPMIX_VALUE: Data type: PMIX_PROC_INFO\tValue: %s:%lu",
+        rc = asprintf(output, "%sPMIX_VALUE: Data type: PMIX_PROC_INFO\tProc: %s:%lu\n%s\tHost: %s\tExecutable: %s\tPid: %lu",
-                      prefx, src->data.proc->nspace, (unsigned long)src->data.proc->rank);
+                      prefx, src->data.pinfo->proc.nspace, (unsigned long)src->data.pinfo->proc.rank,
+                      prefx, src->data.pinfo->hostname, src->data.pinfo->executable_name,
+                      (unsigned long)src->data.pinfo->pid);
         break;
         case PMIX_DATA_ARRAY:
         rc = asprintf(output, "%sPMIX_VALUE: Data type: DATA_ARRAY\tARRAY SIZE: %ld",
                       prefx, (long)src->data.darray->size);
+        break;
         /**** DEPRECATED ****/
         case PMIX_INFO_ARRAY:
         rc = asprintf(output, "%sPMIX_VALUE: Data type: INFO_ARRAY\tARRAY SIZE: %ld",

opal/mca/pmix/pmix3x/pmix/src/client/pmi2.c

@@ -291,7 +291,8 @@ PMIX_EXPORT int PMI2_Job_Connect(const char jobid[], PMI2_Connect_comm_t *conn)
         return PMI2_ERR_INVALID_ARGS;
     }
 
-    (void)strncpy(proc.nspace, (jobid ? jobid : myproc.nspace), sizeof(myproc.nspace));
+    memset(proc.nspace, 0, sizeof(proc.nspace));
+    (void)strncpy(proc.nspace, (jobid ? jobid : proc.nspace), sizeof(proc.nspace)-1);
     proc.rank = PMIX_RANK_WILDCARD;
     rc = PMIx_Connect(&proc, 1, NULL, 0);
     return convert_err(rc);
@@ -304,7 +305,8 @@ PMIX_EXPORT int PMI2_Job_Disconnect(const char jobid[])
 
     PMI2_CHECK();
 
-    (void)strncpy(proc.nspace, (jobid ? jobid : myproc.nspace), sizeof(myproc.nspace));
+    memset(proc.nspace, 0, sizeof(proc.nspace));
+    (void)strncpy(proc.nspace, (jobid ? jobid : proc.nspace), sizeof(proc.nspace)-1);
     proc.rank = PMIX_RANK_WILDCARD;
     rc = PMIx_Disconnect(&proc, 1, NULL, 0);
     return convert_err(rc);

opal/mca/pmix/pmix3x/pmix/src/client/pmix_client.c

@@ -22,6 +22,7 @@
 #include <src/include/pmix_socket_errno.h>
 
 #include <pmix.h>
+#include <pmix_rename.h>
 
 #include "src/include/pmix_globals.h"
 
@@ -63,8 +64,8 @@
 #include "src/util/hash.h"
 #include "src/util/output.h"
 #include "src/runtime/pmix_progress_threads.h"
+#include "src/runtime/pmix_rte.h"
 #include "src/usock/usock.h"
-#include "src/sec/pmix_sec.h"
 #include "src/include/pmix_globals.h"
 #if defined(PMIX_ENABLE_DSTORE) && (PMIX_ENABLE_DSTORE == 1)
 #include "src/dstore/pmix_dstore.h"
@@ -245,14 +246,13 @@ PMIX_EXPORT const char* PMIx_Get_version(void)
 }
 
 PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc,
-                          pmix_info_t info[], size_t ninfo)
+                                    pmix_info_t info[], size_t ninfo)
 {
     char **uri, *evar;
-    int rc, debug_level;
+    int rc;
     struct sockaddr_un address;
     pmix_nspace_t *nsptr;
     pmix_cb_t cb;
-    size_t n;
 
     if (NULL == proc) {
         return PMIX_ERR_BAD_PARAM;
@@ -274,35 +274,17 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc,
         return PMIX_ERR_INVALID_NAMESPACE;
     }
 
-    /* scan incoming info for directives */
+    /* setup the runtime - this init's the globals,
-    if (NULL != info) {
+     * opens and initializes the required frameworks */
-        for (n=0; n < ninfo; n++) {
+    if (PMIX_SUCCESS != (rc = pmix_rte_init(PMIX_PROC_CLIENT, info, ninfo,
-            if (0 == strcmp(PMIX_EVENT_BASE, info[n].key)) {
+                                            pmix_client_notify_recv))) {
-                pmix_globals.evbase = (pmix_event_base_t*)info[n].value.data.ptr;
+        PMIX_ERROR_LOG(rc);
-                pmix_globals.external_evbase = true;
+        return rc;
-            }
-        }
-    }
-    /* setup the globals */
-    pmix_globals_init();
-    PMIX_CONSTRUCT(&pmix_client_globals.pending_requests, pmix_list_t);
-    PMIX_CONSTRUCT(&pmix_client_globals.myserver, pmix_peer_t);
-    /* mark that we are a client */
-    pmix_globals.server = false;
-    /* get our effective id's */
-    pmix_globals.uid = geteuid();
-    pmix_globals.gid = getegid();
-    /* initialize the output system */
-    if (!pmix_output_init()) {
-        return PMIX_ERROR;
     }
 
-    /* see if debug is requested */
+    /* setup the globals */
-    if (NULL != (evar = getenv("PMIX_DEBUG"))) {
+    PMIX_CONSTRUCT(&pmix_client_globals.pending_requests, pmix_list_t);
-        debug_level = strtol(evar, NULL, 10);
+    PMIX_CONSTRUCT(&pmix_client_globals.myserver, pmix_peer_t);
-        pmix_globals.debug_output = pmix_output_open(NULL);
-        pmix_output_set_verbosity(pmix_globals.debug_output, debug_level);
-    }
 
     pmix_output_verbose(2, pmix_globals.debug_output,
                         "pmix: init called");
@@ -310,9 +292,6 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc,
     /* we require the nspace */
     if (NULL == (evar = getenv("PMIX_NAMESPACE"))) {
         /* let the caller know that the server isn't available yet */
-        pmix_output_close(pmix_globals.debug_output);
-        pmix_output_finalize();
-        pmix_class_finalize();
         return PMIX_ERR_INVALID_NAMESPACE;
     }
     if (NULL != proc) {
@@ -327,17 +306,11 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc,
      * then we need to return an error */
     if (NULL == (evar = getenv("PMIX_SERVER_URI"))) {
         /* let the caller know that the server isn't available */
-        pmix_output_close(pmix_globals.debug_output);
-        pmix_output_finalize();
-        pmix_class_finalize();
         return PMIX_ERR_SERVER_NOT_AVAIL;
     }
     uri = pmix_argv_split(evar, ':');
     if (3 != pmix_argv_count(uri)) {
         pmix_argv_free(uri);
-        pmix_output_close(pmix_globals.debug_output);
-        pmix_output_finalize();
-        pmix_class_finalize();
         return PMIX_ERROR;
     }
 
@@ -356,9 +329,6 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc,
     /* if the rendezvous file doesn't exist, that's an error */
     if (0 != access(uri[2], R_OK)) {
         pmix_argv_free(uri);
-        pmix_output_close(pmix_globals.debug_output);
-        pmix_output_finalize();
-        pmix_class_finalize();
         return PMIX_ERR_NOT_FOUND;
     }
     pmix_argv_free(uri);
@@ -366,9 +336,6 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc,
     /* we also require our rank */
     if (NULL == (evar = getenv("PMIX_RANK"))) {
         /* let the caller know that the server isn't available yet */
-        pmix_output_close(pmix_globals.debug_output);
-        pmix_output_finalize();
-        pmix_class_finalize();
         return PMIX_ERR_DATA_VALUE_NOT_FOUND;
     }
     pmix_globals.myid.rank = strtol(evar, NULL, 10);
@@ -380,15 +347,9 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc,
     /* setup the support */
 #if defined(PMIX_ENABLE_DSTORE) && (PMIX_ENABLE_DSTORE == 1)
     if (PMIX_SUCCESS != (rc = pmix_dstore_init(NULL, 0))) {
-        pmix_output_close(pmix_globals.debug_output);
-        pmix_output_finalize();
-        pmix_class_finalize();
         return PMIX_ERR_DATA_VALUE_NOT_FOUND;
     }
 #endif /* PMIX_ENABLE_DSTORE */
-    pmix_bfrop_open();
-    pmix_usock_init(pmix_client_notify_recv);
-    pmix_sec_init();
 
     if (!pmix_globals.external_evbase) {
         /* tell the event library we need thread support */
@@ -396,12 +357,6 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc,
 
         /* create an event base and progress thread for us */
         if (NULL == (pmix_globals.evbase = pmix_progress_thread_init(NULL))) {
-            pmix_sec_finalize();
-            pmix_usock_finalize();
-            pmix_bfrop_close();
-            pmix_output_close(pmix_globals.debug_output);
-            pmix_output_finalize();
-            pmix_class_finalize();
             return -1;
 
         }
@@ -413,13 +368,6 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc,
     /* connect to the server - returns job info if successful */
     if (PMIX_SUCCESS != (rc = connect_to_server(&address, &cb))){
         PMIX_DESTRUCT(&cb);
-        pmix_progress_thread_finalize(NULL);
-        pmix_sec_finalize();
-        pmix_usock_finalize();
-        pmix_bfrop_close();
-        pmix_output_close(pmix_globals.debug_output);
-        pmix_output_finalize();
-        pmix_class_finalize();
         return rc;
     }
     PMIX_WAIT_FOR_COMPLETION(cb.active);
@@ -492,31 +440,25 @@ PMIX_EXPORT pmix_status_t PMIx_Finalize(const pmix_info_t info[], size_t ninfo)
         /* create a callback object as we need to pass it to the
          * recv routine so we know which callback to use when
          * the return message is recvd */
-         cb = PMIX_NEW(pmix_cb_t);
+        cb = PMIX_NEW(pmix_cb_t);
-         cb->active = true;
+        cb->active = true;
 
-         pmix_output_verbose(2, pmix_globals.debug_output,
+        pmix_output_verbose(2, pmix_globals.debug_output,
                              "pmix:client sending finalize sync to server");
 
         /* push the message into our event base to send to the server */
-         PMIX_ACTIVATE_SEND_RECV(&pmix_client_globals.myserver, msg, wait_cbfunc, cb);
+        PMIX_ACTIVATE_SEND_RECV(&pmix_client_globals.myserver, msg, wait_cbfunc, cb);
 
         /* wait for the ack to return */
-         PMIX_WAIT_FOR_COMPLETION(cb->active);
+        PMIX_WAIT_FOR_COMPLETION(cb->active);
-         PMIX_RELEASE(cb);
+        PMIX_RELEASE(cb);
-         pmix_output_verbose(2, pmix_globals.debug_output,
+        pmix_output_verbose(2, pmix_globals.debug_output,
                              "pmix:client finalize sync received");
-     }
+    }
+
+    pmix_rte_finalize();
 
      PMIX_DESTRUCT(&pmix_client_globals.myserver);
-     if (!pmix_globals.external_evbase) {
-        #ifdef HAVE_LIBEVENT_GLOBAL_SHUTDOWN
-            libevent_global_shutdown();
-        #endif
-         pmix_progress_thread_finalize(NULL);
-     }
-
-     pmix_usock_finalize();
      PMIX_LIST_DESTRUCT(&pmix_client_globals.pending_requests);
 
      if (0 <= pmix_client_globals.myserver.sd) {
@@ -524,16 +466,6 @@ PMIX_EXPORT pmix_status_t PMIx_Finalize(const pmix_info_t info[], size_t ninfo)
     }
 
     pmix_bfrop_close();
-    pmix_sec_finalize();
-#if defined(PMIX_ENABLE_DSTORE) && (PMIX_ENABLE_DSTORE == 1)
-    pmix_dstore_finalize();
-#endif /* PMIX_ENABLE_DSTORE */
-
-    pmix_globals_finalize();
-
-    pmix_output_close(pmix_globals.debug_output);
-    pmix_output_finalize();
-    pmix_class_finalize();
 
     return PMIX_SUCCESS;
 }
@@ -776,7 +708,7 @@ static void _commitfn(int sd, short args, void *cbdata)
     }
 
     /* if we are a server, or we aren't connected, don't attempt to send */
-    if (pmix_globals.server) {
+    if (PMIX_PROC_SERVER == pmix_globals.proc_type) {
         return PMIX_SUCCESS;  // not an error
     }
     if (!pmix_globals.connected) {
@@ -950,6 +882,8 @@ static pmix_status_t send_connect_ack(int sd)
     pmix_usock_hdr_t hdr;
     size_t sdsize=0, csize=0;
     char *cred = NULL;
+    char *bfrop;
+    char *sec;
 
     pmix_output_verbose(2, pmix_globals.debug_output,
                         "pmix: SEND CONNECT ACK");
@@ -962,17 +896,26 @@ static pmix_status_t send_connect_ack(int sd)
     /* reserve space for the nspace and rank info */
     sdsize = strlen(pmix_globals.myid.nspace) + 1 + sizeof(int);
 
-    /* get a credential, if the security system provides one. Not
+    /* a security module was assigned to us during rte_init based
-     * every SPC will do so, thus we must first check */
+     * on a list of available security modules provided by our
-    if (NULL != pmix_sec.create_cred) {
+     * local PMIx server. Now use that module to
-        if (NULL == (cred = pmix_sec.create_cred())) {
+     * get a credential, if the security system provides one. Not
+     * every psec module will do so, thus we must first check */
+    if (NULL != pmix_globals.mypeer->compat.psec->create_cred) {
+        if (NULL == (cred = pmix_globals.mypeer->compat.psec->create_cred())) {
             /* an error occurred - we cannot continue */
             return PMIX_ERR_INVALID_CRED;
         }
         csize = strlen(cred) + 1;  // must NULL terminate the string!
     }
+
+    /* add our active bfrops and sec module info, and what type
+     * of buffers we are using */
+   // bfrop = pmix_globals.mypeer->compat.bfrops->name;
+    sec = pmix_globals.mypeer->compat.psec->name;
+
     /* set the number of bytes to be read beyond the header */
-    hdr.nbytes = sdsize + strlen(PMIX_VERSION) + 1 + csize;  // must NULL terminate the VERSION string!
+    hdr.nbytes = sdsize + strlen(PMIX_VERSION) + 1 + strlen(sec) + 1 + csize;  // must NULL terminate the VERSION string!
 
     /* create a space for our message */
     sdsize = (sizeof(hdr) + hdr.nbytes);
@@ -994,6 +937,12 @@ static pmix_status_t send_connect_ack(int sd)
     csize += sizeof(int);
     memcpy(msg+csize, PMIX_VERSION, strlen(PMIX_VERSION));
     csize += strlen(PMIX_VERSION)+1;
+  //  memcpy(msg+csize, bfrop, strlen(bfrop));
+  //  csize += strlen(bfrop)+1;
+    memcpy(msg+csize, sec, strlen(sec));
+    csize += strlen(sec)+1;
+  //  memcpy(msg+csize, &pmix_globals.mypeer->comm.type, sizeof(pmix_bfrop_buffer_type_t));
+  //  csize += sizeof(pmix_bfrop_buffer_type_t);
     if (NULL != cred) {
         memcpy(msg+csize, cred, strlen(cred));  // leaves last position in msg set to NULL
     }
@@ -1035,14 +984,14 @@ static pmix_status_t send_connect_ack(int sd)
        }
    } else {
         /* set a timeout on the blocking recv so we don't hang */
-    tv.tv_sec  = 2;
+        tv.tv_sec  = 2;
-    tv.tv_usec = 0;
+        tv.tv_usec = 0;
-    if (0 != setsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv))) {
+        if (0 != setsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv))) {
-        pmix_output_verbose(2, pmix_globals.debug_output,
+            pmix_output_verbose(2, pmix_globals.debug_output,
-                            "pmix: recv_connect_ack could not setsockopt SO_RCVTIMEO");
+                                "pmix: recv_connect_ack could not setsockopt SO_RCVTIMEO");
-        return PMIX_ERR_UNREACH;
+            return PMIX_ERR_UNREACH;
+        }
     }
-}
 
     /* receive the status reply */
     rc = pmix_usock_recv_blocking(sd, (char*)&reply, sizeof(int));
@@ -1053,10 +1002,10 @@ static pmix_status_t send_connect_ack(int sd)
 
     /* see if they want us to do the handshake */
     if (PMIX_ERR_READY_FOR_HANDSHAKE == reply) {
-        if (NULL == pmix_sec.client_handshake) {
+        if (NULL == pmix_globals.mypeer->compat.psec->client_handshake) {
             return PMIX_ERR_HANDSHAKE_FAILED;
         }
-        if (PMIX_SUCCESS != (rc = pmix_sec.client_handshake(sd))) {
+        if (PMIX_SUCCESS != (rc = pmix_globals.mypeer->compat.psec->client_handshake(sd))) {
             return rc;
         }
     } else if (PMIX_SUCCESS != reply) {

opal/mca/pmix/pmix3x/pmix/src/client/pmix_client_connect.c

@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2014-2015 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014-2016 Intel, Inc.  All rights reserved.
  * Copyright (c) 2014-2015 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2014      Artem Y. Polyakov <artpol84@gmail.com>.
@@ -21,6 +21,7 @@
 #include <src/include/pmix_stdint.h>
 
 #include <pmix.h>
+#include <pmix_rename.h>
 
 #include "src/include/pmix_globals.h"
 
@@ -51,7 +52,6 @@
 #include "src/util/error.h"
 #include "src/util/output.h"
 #include "src/usock/usock.h"
-#include "src/sec/pmix_sec.h"
 
 #include "pmix_client_ops.h"
 

opal/mca/pmix/pmix3x/pmix/src/client/pmix_client_fence.c

@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2014-2015 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014-2016 Intel, Inc.  All rights reserved.
  * Copyright (c) 2014-2015 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2014      Artem Y. Polyakov <artpol84@gmail.com>.
@@ -21,6 +21,7 @@
 #include <src/include/pmix_stdint.h>
 
 #include <pmix.h>
+#include <pmix_rename.h>
 
 #include "src/include/pmix_globals.h"
 
@@ -52,7 +53,6 @@
 #include "src/util/hash.h"
 #include "src/util/output.h"
 #include "src/usock/usock.h"
-#include "src/sec/pmix_sec.h"
 
 #include "pmix_client_ops.h"
 

opal/mca/pmix/pmix3x/pmix/src/client/pmix_client_get.c

@@ -21,6 +21,7 @@
 #include <src/include/pmix_stdint.h>
 
 #include <pmix.h>
+#include <pmix_rename.h>
 
 #include "src/include/pmix_globals.h"
 
@@ -52,7 +53,6 @@
 #include "src/util/hash.h"
 #include "src/util/output.h"
 #include "src/usock/usock.h"
-#include "src/sec/pmix_sec.h"
 #if defined(PMIX_ENABLE_DSTORE) && (PMIX_ENABLE_DSTORE == 1)
 #include "src/dstore/pmix_dstore.h"
 #endif /* PMIX_ENABLE_DSTORE */
@@ -613,7 +613,8 @@ static void _getnbfn(int fd, short flags, void *cbdata)
     /* if we got here, then we don't have the data for this proc. If we
      * are a server, or we are a client and not connected, then there is
      * nothing more we can do */
-    if (pmix_globals.server || (!pmix_globals.server && !pmix_globals.connected)) {
+    if (PMIX_PROC_SERVER == pmix_globals.proc_type ||
+        (PMIX_PROC_SERVER != pmix_globals.proc_type && !pmix_globals.connected)) {
         cb->value_cbfunc(PMIX_ERR_NOT_FOUND, NULL, cb->cbdata);
         PMIX_RELEASE(cb);
         return;

opal/mca/pmix/pmix3x/pmix/src/client/pmix_client_pub.c

@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2014-2015 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014-2016 Intel, Inc.  All rights reserved.
  * Copyright (c) 2014-2015 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2014      Artem Y. Polyakov <artpol84@gmail.com>.
@@ -21,6 +21,7 @@
 #include <src/include/pmix_stdint.h>
 
 #include <pmix.h>
+#include <pmix_rename.h>
 
 #include "src/include/pmix_globals.h"
 
@@ -51,7 +52,6 @@
 #include "src/util/error.h"
 #include "src/util/output.h"
 #include "src/usock/usock.h"
-#include "src/sec/pmix_sec.h"
 
 #include "pmix_client_ops.h"
 

opal/mca/pmix/pmix3x/pmix/src/client/pmix_client_spawn.c

@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2014      Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014-2016 Intel, Inc.  All rights reserved.
  * Copyright (c) 2014-2015 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2014      Artem Y. Polyakov <artpol84@gmail.com>.
@@ -21,6 +21,7 @@
 #include <src/include/pmix_stdint.h>
 
 #include <pmix.h>
+#include <pmix_rename.h>
 
 #include "src/include/pmix_globals.h"
 
@@ -51,7 +52,6 @@
 #include "src/util/error.h"
 #include "src/util/output.h"
 #include "src/usock/usock.h"
-#include "src/sec/pmix_sec.h"
 
 #include "pmix_client_ops.h"
 

opal/mca/pmix/pmix3x/pmix/src/common/pmix_log.c

@@ -19,6 +19,7 @@
 #include <pmix.h>
 #include <pmix_common.h>
 #include <pmix_server.h>
+#include <pmix_rename.h>
 
 #include "src/util/argv.h"
 #include "src/util/error.h"
@@ -73,7 +74,7 @@ PMIX_EXPORT pmix_status_t PMIx_Log_nb(const pmix_info_t data[], size_t ndata,
 
     /* if we are the server, then we just log and
      * return the response */
-    if (pmix_globals.server) {
+    if (PMIX_PROC_SERVER == pmix_globals.proc_type) {
             if (NULL == pmix_host_server.log) {
                 /* nothing we can do */
                 return PMIX_ERR_NOT_SUPPORTED;

opal/mca/pmix/pmix3x/pmix/src/common/pmix_query.c

@@ -19,6 +19,7 @@
 #include <pmix.h>
 #include <pmix_common.h>
 #include <pmix_server.h>
+#include <pmix_rename.h>
 
 #include "src/util/argv.h"
 #include "src/util/error.h"
@@ -113,7 +114,7 @@ PMIX_EXPORT pmix_status_t PMIx_Query_info_nb(pmix_query_t queries[], size_t nque
 
     /* if we are the server, then we just issue the query and
      * return the response */
-    if (pmix_globals.server) {
+    if (PMIX_PROC_SERVER == pmix_globals.proc_type) {
             if (NULL == pmix_host_server.query) {
                 /* nothing we can do */
                 return PMIX_ERR_NOT_SUPPORTED;

opal/mca/pmix/pmix3x/pmix/src/common/pmix_strings.c

@@ -32,6 +32,8 @@
 #endif
 
 #include <pmix_common.h>
+#include <pmix_rename.h>
+
 #include "src/buffer_ops/internal.h"
 #include "src/include/pmix_globals.h"
 

opal/mca/pmix/pmix3x/pmix/src/dstore/pmix_dstore.c

@@ -1,6 +1,7 @@
 /*
  * Copyright (c) 2016      Mellanox Technologies, Inc.
  *                         All rights reserved.
+ * Copyright (c) 2016      Intel, Inc. All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -75,18 +76,26 @@ int pmix_dstore_fetch(const char *nspace, pmix_rank_t rank,
     return pmix_dstore.fetch(nspace, rank, key, kvs);
 }
 
-int pmix_dstore_patch_env(char ***env)
+int pmix_dstore_patch_env(const char *nspace, char ***env)
 {
     if (!pmix_dstore.patch_env) {
         return PMIX_ERR_NOT_SUPPORTED;
     }
-    return pmix_dstore.patch_env(env);
+    return pmix_dstore.patch_env(nspace, env);
 }
 
-int pmix_dstore_nspace_add(const char *nspace)
+int pmix_dstore_nspace_add(const char *nspace, pmix_info_t info[], size_t ninfo)
 {
-    if (!pmix_dstore.nspace) {
+    if (!pmix_dstore.nspace_add) {
         return PMIX_ERR_NOT_SUPPORTED;
     }
-    return pmix_dstore.nspace(nspace);
+    return pmix_dstore.nspace_add(nspace, info, ninfo);
+}
+
+int pmix_dstore_nspace_del(const char *nspace)
+{
+    if (!pmix_dstore.nspace_del) {
+        return PMIX_ERR_NOT_SUPPORTED;
+    }
+    return pmix_dstore.nspace_del(nspace);
 }

opal/mca/pmix/pmix3x/pmix/src/dstore/pmix_dstore.h

@@ -27,8 +27,9 @@ void pmix_dstore_finalize(void);
 int pmix_dstore_store(const char *nspace, pmix_rank_t rank, pmix_kval_t *kv);
 int pmix_dstore_fetch(const char *nspace, pmix_rank_t rank,
                       const char *key, pmix_value_t **kvs);
-int pmix_dstore_patch_env(char ***env);
+int pmix_dstore_patch_env(const char *nspace, char ***env);
-int pmix_dstore_nspace_add(const char *nspace);
+int pmix_dstore_nspace_add(const char *nspace, pmix_info_t info[], size_t ninfo);
+int pmix_dstore_nspace_del(const char *nspace);
 
 /**
  * Initialize the module. Returns an error if the module cannot
@@ -82,7 +83,7 @@ typedef int (*pmix_dstore_base_module_fetch_fn_t)(const char *nspace,
 *
 * @return PMIX_SUCCESS on success.
 */
-typedef int (*pmix_dstore_base_module_proc_patch_env_fn_t)(char ***env);
+typedef int (*pmix_dstore_base_module_proc_patch_env_fn_t)(const char *nspace, char ***env);
 
 /**
 * get base dstore path.
@@ -93,7 +94,18 @@ typedef int (*pmix_dstore_base_module_proc_patch_env_fn_t)(char ***env);
 *
 * @return PMIX_SUCCESS on success.
 */
-typedef int (*pmix_dstore_base_module_add_nspace_fn_t)(const char *nspace);
+typedef int (*pmix_dstore_base_module_add_nspace_fn_t)(const char *nspace,
+                                                        pmix_info_t info[],
+                                                        size_t ninfo);
+
+/**
+* finalize nspace.
+*
+* @param nspace   namespace string
+*
+* @return PMIX_SUCCESS on success.
+*/
+typedef int (*pmix_dstore_base_module_del_nspace_fn_t)(const char *nspace);
 
 /**
 * structure for dstore modules
@@ -105,7 +117,9 @@ typedef struct {
     pmix_dstore_base_module_store_fn_t       store;
     pmix_dstore_base_module_fetch_fn_t       fetch;
     pmix_dstore_base_module_proc_patch_env_fn_t   patch_env;
-    pmix_dstore_base_module_add_nspace_fn_t  nspace;
+    pmix_dstore_base_module_add_nspace_fn_t  nspace_add;
+    pmix_dstore_base_module_del_nspace_fn_t  nspace_del;
+
 } pmix_dstore_base_module_t;
 
 END_C_DECLS

opal/mca/pmix/pmix3x/pmix/src/dstore/pmix_esh.h

@@ -1,6 +1,8 @@
 /*
  * Copyright (c) 2015-2016 Mellanox Technologies, Inc.
  *                         All rights reserved.
+ * Copyright (c) 2016      Intel, Inc. All rights reserved.
+ *
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -12,6 +14,7 @@
 #define PMIX_DSTORE_ESH_H
 
 #include <src/include/pmix_config.h>
+#include "src/class/pmix_value_array.h"
 
 
 #include "pmix_dstore.h"
@@ -25,9 +28,11 @@ BEGIN_C_DECLS
 
 #define PMIX_DSTORE_ESH_BASE_PATH "PMIX_DSTORE_ESH_BASE_PATH"
 
-#define _ESH_LOCK_EX F_WRLCK
+/* this structs are used to store information about
-#define _ESH_LOCK_SH F_RDLCK
+ * shared segments addresses locally at each process,
-#define _ESH_LOCK_UN F_UNLCK
+ * so they are common for different types of segments
+ * and don't have a specific content (namespace's info,
+ * rank's meta info, ranks's data). */
 
 typedef enum {
     INITIAL_SEGMENT,
@@ -35,14 +40,47 @@ typedef enum {
     NS_DATA_SEGMENT
 } segment_type;
 
+typedef struct seg_desc_t seg_desc_t;
+struct seg_desc_t {
+    segment_type type;
+    pmix_sm_seg_t seg_info;
+    uint32_t id;
+    seg_desc_t *next;
+};
+
+typedef struct ns_map_data_s ns_map_data_t;
+typedef struct session_s session_t;
+typedef struct ns_map_s ns_map_t;
+
+struct session_s {
+    int in_use;
+    uid_t jobuid;
+    char *nspace_path;
+    char *lockfile;
+    int lockfd;
+    seg_desc_t *sm_seg_first;
+    seg_desc_t *sm_seg_last;
+};
+
+struct ns_map_data_s {
+    char name[PMIX_MAX_NSLEN+1];
+    size_t tbl_idx;
+    int track_idx;
+};
+
+struct ns_map_s {
+    int in_use;
+    ns_map_data_t data;
+};
+
 /* initial segment format:
  * size_t num_elems;
- * int full; //indicate to client that it needs to attach to the next segment
+ * size_t full; //indicate to client that it needs to attach to the next segment
  * ns_seg_info_t ns_seg_info[max_ns_num];
  */
 
 typedef struct {
-    char ns_name[PMIX_MAX_NSLEN+1];
+    ns_map_data_t ns_map;
     size_t num_meta_seg;/* read by clients to attach to this number of segments. */
     size_t num_data_seg;
 } ns_seg_info_t;
@@ -58,29 +96,14 @@ typedef struct {
     size_t count;
 } rank_meta_info;
 
-/* this structs are used to store information about
- * shared segments addresses locally at each process,
- * so they are common for different types of segments
- * and don't have a specific content (namespace's info,
- * rank's meta info, ranks's data). */
-
-typedef struct seg_desc_t seg_desc_t;
-struct seg_desc_t {
-    segment_type type;
-    pmix_sm_seg_t seg_info;
-    uint32_t id;
-    seg_desc_t *next;
-};
-
 typedef struct {
-    pmix_list_item_t super;
+    pmix_value_array_t super;
-    char ns_name[PMIX_MAX_NSLEN+1];
+    ns_map_data_t ns_map;
     size_t num_meta_seg;
     size_t num_data_seg;
     seg_desc_t *meta_seg;
     seg_desc_t *data_seg;
 } ns_track_elem_t;
-PMIX_CLASS_DECLARATION(ns_track_elem_t);
 
 extern pmix_dstore_base_module_t pmix_dstore_esh_module;
 

opal/mca/pmix/pmix3x/pmix/src/event/pmix_event.h

@@ -22,7 +22,6 @@
 #define PMIX_EVENT_H
 
 #include <src/include/pmix_config.h>
-#include <src/include/rename.h>
 
 #include <pmix_common.h>
 #include "src/class/pmix_list.h"
@@ -116,7 +115,7 @@ PMIX_CLASS_DECLARATION(pmix_event_chain_t);
 /* invoke the error handler that is registered against the given
  * status, passing it the provided info on the procs that were
  * affected, plus any additional info provided by the server */
- void pmix_invoke_local_event_hdlr(pmix_event_chain_t *chain);
+void pmix_invoke_local_event_hdlr(pmix_event_chain_t *chain);
 
 #define PMIX_REPORT_EVENT(e)                        \
     do {                                            \

opal/mca/pmix/pmix3x/pmix/src/event/pmix_event_notification.c

@@ -8,11 +8,11 @@
  * $HEADER$
  */
 #include <src/include/pmix_config.h>
-#include <src/include/rename.h>
 
-#include "include/pmix.h"
+#include <pmix.h>
-#include "include/pmix_common.h"
+#include <pmix_common.h>
-#include "include/pmix_server.h"
+#include <pmix_server.h>
+#include <pmix_rename.h>
 
 #include "src/util/error.h"
 #include "src/util/output.h"
@@ -44,7 +44,7 @@ PMIX_EXPORT pmix_status_t PMIx_Notify_event(pmix_status_t status,
 {
     int rc;
 
-    if (pmix_globals.server) {
+    if (PMIX_PROC_SERVER == pmix_globals.proc_type) {
         rc = notify_client_of_event(status, source, range,
                                     info, ninfo,
                                     cbfunc, cbdata);

opal/mca/pmix/pmix3x/pmix/src/event/pmix_event_registration.c

@@ -8,11 +8,11 @@
  * $HEADER$
  */
 #include <src/include/pmix_config.h>
-#include <src/include/rename.h>
 
 #include <pmix.h>
 #include <pmix_common.h>
 #include <pmix_server.h>
+#include <pmix_rename.h>
 
 #include "src/util/error.h"
 #include "src/util/output.h"
@@ -234,7 +234,7 @@ static pmix_status_t _add_hdlr(pmix_list_t *list, pmix_list_item_t *item,
     /* if we are a client, and we haven't already registered a handler of this
      * type with our server, or if we have directives, then we need to notify
      * the server */
-    if (!pmix_globals.server &&
+    if (PMIX_PROC_SERVER != pmix_globals.proc_type &&
        (need_register || 0 < pmix_list_get_size(xfer))) {
         pmix_output_verbose(2, pmix_globals.debug_output,
                             "pmix: _add_hdlr sending to server");
@@ -254,7 +254,7 @@ static pmix_status_t _add_hdlr(pmix_list_t *list, pmix_list_item_t *item,
     /* if we are a server and are registering for events, then we only contact
      * our host if we want environmental events */
 
-    if (pmix_globals.server && cd->enviro &&
+    if (PMIX_PROC_SERVER == pmix_globals.proc_type && cd->enviro &&
         NULL != pmix_host_server.register_events) {
             pmix_output_verbose(2, pmix_globals.debug_output,
                                 "pmix: _add_hdlr registering with server");
@@ -450,7 +450,7 @@ static void dereg_event_hdlr(int sd, short args, void *cbdata)
 
     /* if I am not the server, then I need to notify the server
      * to remove my registration */
-    if (!pmix_globals.server) {
+    if (PMIX_PROC_SERVER != pmix_globals.proc_type) {
         msg = PMIX_NEW(pmix_buffer_t);
         if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, &cmd, 1, PMIX_CMD))) {
             PMIX_RELEASE(msg);

opal/mca/pmix/pmix3x/pmix/src/include/Makefile.include

@@ -37,8 +37,7 @@ headers += \
         include/prefetch.h \
         include/types.h \
         include/pmix_config_top.h \
-        include/pmix_config_bottom.h \
+        include/pmix_config_bottom.h
-        include/rename.h
 
 endif ! PMIX_EMBEDDED_MODE
 

opal/mca/pmix/pmix3x/pmix/src/include/frameworks.h

@@ -1,21 +0,0 @@
-/*
- * This file is autogenerated by autogen.pl. Do not edit this file by hand.
- */
-#ifndef PMIX_FRAMEWORKS_H
-#define PMIX_FRAMEWORKS_H
-
-#include <pmix/mca/base/mca_base_framework.h>
-
-extern pmix_mca_base_framework_t pmix_src_base_framework;
-extern pmix_mca_base_framework_t pmix_pdl_base_framework;
-extern pmix_mca_base_framework_t pmix_pinstalldirs_base_framework;
-
-static pmix_mca_base_framework_t *pmix_frameworks[] = {
-    &pmix_src_base_framework,
-    &pmix_pdl_base_framework,
-    &pmix_pinstalldirs_base_framework,
-    NULL
-};
-
-#endif /* PMIX_FRAMEWORKS_H */
-

opal/mca/pmix/pmix3x/pmix/src/include/pmix_globals.c

@@ -41,18 +41,6 @@
 #include "src/class/pmix_list.h"
 
 
-pmix_globals_t pmix_globals = {
-    .init_cntr = 0,
-    .pindex = 0,
-    .evbase = NULL,
-    .debug_output = -1,
-    .server = false,
-    .connected = false,
-    .cache_local = NULL,
-    .cache_remote = NULL
-};
-
-
 void pmix_globals_init(void)
 {
     memset(&pmix_globals.myid, 0, sizeof(pmix_proc_t));

opal/mca/pmix/pmix3x/pmix/src/include/pmix_globals.h

@@ -37,6 +37,8 @@
 #include "src/class/pmix_list.h"
 #include "src/event/pmix_event.h"
 
+#include "src/mca/psec/psec.h"
+
 BEGIN_C_DECLS
 
 /* some limits */
@@ -179,6 +181,12 @@ typedef struct pmix_rank_info_t {
 } pmix_rank_info_t;
 PMIX_CLASS_DECLARATION(pmix_rank_info_t);
 
+/* define a structure for holding personality pointers
+ * to plugins for cross-version support */
+typedef struct pmix_personality_t {
+    pmix_psec_module_t *psec;
+} pmix_personality_t;
+
 /* object for tracking peers - each peer can have multiple
  * connections. This can occur if the initial app executes
  * a fork/exec, and the child initiates its own connection
@@ -198,6 +206,7 @@ typedef struct pmix_peer_t {
     pmix_list_t send_queue;      /**< list of messages to send */
     pmix_usock_send_t *send_msg; /**< current send in progress */
     pmix_usock_recv_t *recv_msg; /**< current recv in progress */
+    pmix_personality_t compat;
 } pmix_peer_t;
 PMIX_CLASS_DECLARATION(pmix_peer_t);
 
@@ -329,6 +338,8 @@ PMIX_CLASS_DECLARATION(pmix_info_caddy_t);
 typedef struct {
     int init_cntr;                       // #times someone called Init - #times called Finalize
     pmix_proc_t myid;
+    pmix_peer_t *mypeer;                 // my own peer object
+    pmix_proc_type_t proc_type;
     uid_t uid;                           // my effective uid
     gid_t gid;                           // my effective gid
     int pindex;
@@ -336,7 +347,6 @@ typedef struct {
     bool external_evbase;
     int debug_output;
     pmix_events_t events;                // my event handler registrations.
-    bool server;
     bool connected;
     pmix_list_t nspaces;                 // list of pmix_nspace_t for the nspaces we know about
     pmix_buffer_t *cache_local;          // data PUT by me to local scope

opal/mca/pmix/pmix3x/pmix/src/include/rename.h

@@ -1,14 +0,0 @@
-/*
- * Copyright (c) 2016      Intel, Inc. All rights reserved
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#ifndef PMIX_RENAME_H
-#define PMIX_RENAME_H
-
-#endif
-

opal/mca/pmix/pmix3x/pmix/src/mca/base/base.h

@@ -23,8 +23,8 @@
  * $HEADER$
  */
 
-#ifndef MCA_BASE_H
+#ifndef PMIX_MCA_BASE_H
-#define MCA_BASE_H
+#define PMIX_MCA_BASE_H
 
 #include <src/include/pmix_config.h>
 

opal/mca/pmix/pmix3x/pmix/src/mca/base/pmix_mca_base_component_repository.h

@@ -34,8 +34,8 @@
  * (i.e., have a separate header file just for retain()).
  */
 
-#ifndef MCA_BASE_COMPONENT_REPOSITORY_H
+#ifndef PMIX_MCA_BASE_COMPONENT_REPOSITORY_H
-#define MCA_BASE_COMPONENT_REPOSITORY_H
+#define PMIX_MCA_BASE_COMPONENT_REPOSITORY_H
 
 #include <src/include/pmix_config.h>
 

opal/mca/pmix/pmix3x/pmix/src/mca/pdl/pdlopen/configure.m4

@@ -1,6 +1,9 @@
 # -*- shell-script -*-
 #
 # Copyright (c) 2009-2015 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2016      Research Organization for Information Science
+#                         and Technology (RIST). All rights reserved.
+# Copyright (c) 2016      Intel, Inc. All rights reserved.
 #
 # $COPYRIGHT$
 #

opal/mca/pmix/pmix3x/pmix/src/mca/pinstalldirs/config/configure.m4

@@ -4,6 +4,8 @@
 #                         reserved.
 # Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2016      Intel, Inc. All rights reserved
+# Copyright (c) 2016      Research Organization for Information Science
+#                         and Technology (RIST). All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow

opal/mca/pmix/pmix3x/pmix/src/mca/pinstalldirs/env/configure.m4

@@ -4,6 +4,8 @@
 #                         reserved.
 # Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2016      Intel, Inc. All rights reserved
+# Copyright (c) 2016      Research Organization for Information Science
+#                         and Technology (RIST). All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow

opal/mca/pmix/pmix3x/pmix/src/mca/psec/Makefile.am

@@ -0,0 +1,44 @@
+# -*- makefile -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2012      Los Alamos National Security, Inc.  All rights reserved.
+# Copyright (c) 2013-2016 Intel, Inc. All rights reserved
+# Copyright (c) 2016 Cisco Systems, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+AM_CPPFLAGS = $(LTDLINCL)
+
+# main library setup
+noinst_LTLIBRARIES = libmca_psec.la
+libmca_psec_la_SOURCES =
+
+# local files
+headers = psec.h
+sources =
+
+# Conditionally install the header files
+if WANT_INSTALL_HEADERS
+pmixdir = $(pmixincludedir)/$(subdir)
+nobase_pmix_HEADERS = $(headers)
+endif
+
+include base/Makefile.include
+
+libmca_psec_la_SOURCES += $(headers) $(sources)
+
+distclean-local:
+	rm -f base/static-components.h

opal/mca/pmix/pmix3x/pmix/src/mca/psec/base/Makefile.include

@@ -0,0 +1,32 @@
+# -*- makefile -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2012      Los Alamos National Security, Inc.  All rights reserved.
+# Copyright (c) 2013-2016 Intel, Inc. All rights reserved
+# Copyright (c) 2016      Cisco Systems, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# This makefile.am does not stand on its own - it is included from
+# src/Makefile.am
+
+headers += \
+         base/base.h
+
+sources += \
+        base/psec_base_frame.c \
+        base/psec_base_select.c \
+        base/psec_base_fns.c

opal/mca/pmix/pmix3x/pmix/src/mca/psec/base/base.h

@@ -0,0 +1,85 @@
+/* -*- C -*-
+ *
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2006 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2012      Los Alamos National Security, Inc.  All rights reserved.
+ * Copyright (c) 2014-2016 Intel, Inc. All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+#ifndef PMIX_BFROP_BASE_H_
+#define PMIX_BFROP_BASE_H_
+
+#include <src/include/pmix_config.h>
+
+
+#ifdef HAVE_SYS_TIME_H
+#include <sys/time.h> /* for struct timeval */
+#endif
+#ifdef HAVE_STRING_H
+#include <string.h>
+#endif
+#if PMIX_HAVE_HWLOC
+#include PMIX_HWLOC_HEADER
+#endif
+
+#include "src/class/pmix_pointer_array.h"
+#include "src/mca/mca.h"
+#include "src/mca/base/pmix_mca_base_framework.h"
+
+#include "src/mca/psec/psec.h"
+
+
+BEGIN_C_DECLS
+
+/*
+ * MCA Framework
+ */
+extern pmix_mca_base_framework_t pmix_psec_base_framework;
+/**
+ * BFROP select function
+ *
+ * Cycle across available components and construct the list
+ * of active modules
+ */
+pmix_status_t pmix_psec_base_select(void);
+
+/**
+ * Track an active component / module
+ */
+struct pmix_psec_base_active_module_t {
+    pmix_list_item_t super;
+    int pri;
+    pmix_psec_module_t *module;
+    pmix_psec_base_component_t *component;
+};
+typedef struct pmix_psec_base_active_module_t pmix_psec_base_active_module_t;
+PMIX_CLASS_DECLARATION(pmix_psec_base_active_module_t);
+
+
+/* framework globals */
+struct pmix_psec_globals_t {
+  pmix_list_t actives;
+  bool initialized;
+};
+typedef struct pmix_psec_globals_t pmix_psec_globals_t;
+
+extern pmix_psec_globals_t pmix_psec_globals;
+
+END_C_DECLS
+
+#endif

opal/mca/pmix/pmix3x/pmix/src/mca/psec/base/psec_base_fns.c

@@ -0,0 +1,82 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2015-2016 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2016      Mellanox Technologies, Inc.
+ *                         All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include <src/include/pmix_config.h>
+
+#include <pmix_common.h>
+#include "src/include/pmix_globals.h"
+
+#include "src/class/pmix_list.h"
+#include "src/util/argv.h"
+#include "src/util/output.h"
+
+
+#include "src/mca/psec/base/base.h"
+
+
+char* pmix_psec_base_get_available_modules(void)
+{
+    pmix_psec_base_active_module_t *active;
+    char **tmp=NULL, *reply=NULL;
+
+    if (!pmix_psec_globals.initialized) {
+        return NULL;
+    }
+
+    PMIX_LIST_FOREACH(active, &pmix_psec_globals.actives, pmix_psec_base_active_module_t) {
+        pmix_argv_append_nosize(&tmp, active->component->base.pmix_mca_component_name);
+    }
+    if (NULL != tmp) {
+        reply = pmix_argv_join(tmp, ',');
+        pmix_argv_free(tmp);
+    }
+    return reply;
+}
+
+pmix_psec_module_t* pmix_psec_base_assign_module(const char *options)
+{
+    pmix_psec_base_active_module_t *active;
+    char **tmp=NULL;
+    int i;
+    pmix_psec_module_t *mod;
+
+    if (!pmix_psec_globals.initialized) {
+        return NULL;
+    }
+    if (NULL != options) {
+        tmp = pmix_argv_split(options, ',');
+    }
+
+    PMIX_LIST_FOREACH(active, &pmix_psec_globals.actives, pmix_psec_base_active_module_t) {
+        if (NULL == tmp) {
+            if (NULL != (mod = active->component->assign_module())) {
+                return mod;
+            }
+        } else {
+            for (i=0; NULL != tmp[i]; i++) {
+                if (0 == strcmp(tmp[i], active->component->base.pmix_mca_component_name)) {
+                    if (NULL != (mod = active->component->assign_module())) {
+                        pmix_argv_free(tmp);
+                        return mod;
+                    }
+                }
+            }
+        }
+    }
+
+    /* we only get here if nothing was found */
+    if (NULL != tmp) {
+        pmix_argv_free(tmp);
+    }
+    return NULL;
+}

opal/mca/pmix/pmix3x/pmix/src/mca/psec/base/psec_base_frame.c

@@ -0,0 +1,81 @@
+/* -*- Mode: C; c-basic-offset:4 ; -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2009 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2012-2013 Los Alamos National Security, Inc.  All rights reserved.
+ * Copyright (c) 2014-2016 Intel, Inc. All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+/** @file:
+ *
+ */
+#include <src/include/pmix_config.h>
+
+#include <pmix_common.h>
+
+#ifdef HAVE_STRING_H
+#include <string.h>
+#endif
+
+#include "src/class/pmix_list.h"
+#include "src/mca/psec/base/base.h"
+
+/*
+ * The following file was created by configure.  It contains extern
+ * statements and the definition of an array of pointers to each
+ * component's public mca_base_component_t struct.
+ */
+
+#include "src/mca/psec/base/static-components.h"
+
+/* Instantiate the global vars */
+pmix_psec_globals_t pmix_psec_globals = {0};
+
+static pmix_status_t pmix_psec_close(void)
+{
+  pmix_psec_base_active_module_t *active;
+
+    if (!pmix_psec_globals.initialized) {
+        return PMIX_SUCCESS;
+    }
+    pmix_psec_globals.initialized = false;
+
+    PMIX_LIST_FOREACH(active, &pmix_psec_globals.actives, pmix_psec_base_active_module_t) {
+      if (NULL != active->component->finalize) {
+        active->component->finalize();
+      }
+    }
+    PMIX_DESTRUCT(&pmix_psec_globals.actives);
+
+    return PMIX_SUCCESS;
+}
+
+static pmix_status_t pmix_psec_open(pmix_mca_base_open_flag_t flags)
+{
+  /* initialize globals */
+  pmix_psec_globals.initialized = true;
+  PMIX_CONSTRUCT(&pmix_psec_globals.actives, pmix_list_t);
+  return PMIX_SUCCESS;
+}
+
+PMIX_MCA_BASE_FRAMEWORK_DECLARE(pmix, psec, "PMIx Security Operations",
+                                NULL, pmix_psec_open, pmix_psec_close,
+                                mca_psec_base_static_components, 0);
+
+PMIX_CLASS_INSTANCE(pmix_psec_base_active_module_t,
+                    pmix_list_item_t,
+                    NULL, NULL);

opal/mca/pmix/pmix3x/pmix/src/mca/psec/base/psec_base_select.c

@@ -0,0 +1,112 @@
+/*
+ * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2016      Intel, Inc. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include <src/include/pmix_config.h>
+#include <pmix_common.h>
+
+#include <string.h>
+
+#include "src/mca/mca.h"
+#include "src/mca/base/base.h"
+
+#include "src/mca/psec/base/base.h"
+
+static bool selected = false;
+
+/* Function for selecting a prioritized list of components
+ * from all those that are available. */
+int pmix_psec_base_select(void)
+{
+    pmix_mca_base_component_list_item_t *cli = NULL;
+    pmix_mca_base_component_t *component = NULL;
+    pmix_mca_base_module_t *module = NULL;
+    pmix_psec_module_t *nmodule;
+    pmix_psec_base_active_module_t *newmodule, *mod;
+    int rc, priority;
+    bool inserted;
+
+    if (selected) {
+        /* ensure we don't do this twice */
+        return PMIX_SUCCESS;
+    }
+    selected = true;
+
+    /* Query all available components and ask if they have a module */
+    PMIX_LIST_FOREACH(cli, &pmix_psec_base_framework.framework_components, pmix_mca_base_component_list_item_t) {
+        component = (pmix_mca_base_component_t *) cli->cli_component;
+
+        pmix_output_verbose(5, pmix_psec_base_framework.framework_output,
+                            "mca:psec:select: checking available component %s", component->pmix_mca_component_name);
+
+        /* If there's no query function, skip it */
+        if (NULL == component->pmix_mca_query_component) {
+            pmix_output_verbose(5, pmix_psec_base_framework.framework_output,
+                                "mca:psec:select: Skipping component [%s]. It does not implement a query function",
+                                component->pmix_mca_component_name );
+            continue;
+        }
+
+        /* Query the component */
+        pmix_output_verbose(5, pmix_psec_base_framework.framework_output,
+                            "mca:psec:select: Querying component [%s]",
+                            component->pmix_mca_component_name);
+        rc = component->pmix_mca_query_component(&module, &priority);
+
+        /* If no module was returned, then skip component */
+        if (PMIX_SUCCESS != rc || NULL == module) {
+            pmix_output_verbose(5, pmix_psec_base_framework.framework_output,
+                                "mca:psec:select: Skipping component [%s]. Query failed to return a module",
+                                component->pmix_mca_component_name );
+            continue;
+        }
+
+        /* If we got a module, keep it */
+        nmodule = (pmix_psec_module_t*) module;
+        /* add to the list of selected modules */
+        newmodule = PMIX_NEW(pmix_psec_base_active_module_t);
+        newmodule->pri = priority;
+        newmodule->module = nmodule;
+        newmodule->component = (pmix_psec_base_component_t*)cli->cli_component;
+
+        /* maintain priority order */
+        inserted = false;
+        PMIX_LIST_FOREACH(mod, &pmix_psec_globals.actives, pmix_psec_base_active_module_t) {
+            if (priority > mod->pri) {
+                pmix_list_insert_pos(&pmix_psec_globals.actives,
+                                     (pmix_list_item_t*)mod, &newmodule->super);
+                inserted = true;
+                break;
+            }
+        }
+        if (!inserted) {
+            /* must be lowest priority - add to end */
+            pmix_list_append(&pmix_psec_globals.actives, &newmodule->super);
+        }
+    }
+
+    if (4 < pmix_output_get_verbosity(pmix_psec_base_framework.framework_output)) {
+        pmix_output(0, "Final psec priorities");
+        /* show the prioritized list */
+        PMIX_LIST_FOREACH(mod, &pmix_psec_globals.actives, pmix_psec_base_active_module_t) {
+            pmix_output(0, "\tpsec: %s Priority: %d", mod->component->base.pmix_mca_component_name, mod->pri);
+        }
+    }
+
+    return PMIX_SUCCESS;;
+}

opal/mca/pmix/pmix3x/pmix/src/mca/psec/munge/Makefile.am

@@ -0,0 +1,50 @@
+# -*- makefile -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2012      Los Alamos National Security, Inc.  All rights reserved.
+# Copyright (c) 2013-2016 Intel, Inc. All rights reserved
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+headers = psec_munge.h
+sources = \
+        psec_munge_component.c \
+        psec_munge.c
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if MCA_BUILD_pmix_psec_munge_DSO
+lib =
+lib_sources =
+component = mca_psec_munge.la
+component_sources = $(headers) $(sources)
+else
+lib = libmca_psec_munge.la
+lib_sources = $(headers) $(sources)
+component =
+component_sources =
+endif
+
+mcacomponentdir = $(pmixlibdir)
+mcacomponent_LTLIBRARIES = $(component)
+mca_psec_munge_la_SOURCES = $(component_sources)
+mca_psec_munge_la_LDFLAGS = -module -avoid-version
+
+noinst_LTLIBRARIES = $(lib)
+libmca_psec_munge_la_SOURCES = $(lib_sources)
+libmca_psec_munge_la_LDFLAGS = -module -avoid-version

opal/mca/pmix/pmix3x/pmix/src/mca/psec/munge/configure.m4

@@ -0,0 +1,88 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2015-2016 Intel, Inc. All rights reserved
+# Copyright (c) 2015      Research Organization for Information Science
+#                         and Technology (RIST). All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# MCA_psec_munge_CONFIG([action-if-found], [action-if-not-found])
+# --------------------------------------------------------------------
+AC_DEFUN([MCA_pmix_psec_munge_CONFIG],[
+    AC_CONFIG_FILES([src/mca/psec/munge/Makefile])
+
+    PMIX_VAR_SCOPE_PUSH([psec_munge_support psec_munge_dir psec_munge_libdir save_cpp save_ld])
+
+    AC_ARG_WITH([munge],
+                [AC_HELP_STRING([--with-munge=DIR],
+                                [Search for munge headers and libraries in DIR ])])
+
+    AC_ARG_WITH([munge-libdir],
+                [AC_HELP_STRING([--with-munge-libdir=DIR],
+                                [Search for munge libraries in DIR ])])
+
+    psec_munge_support=0
+    if test ! -z "$with_munge" && test "$with_munge" != "no"; then
+        AC_MSG_CHECKING([for munge in])
+        if test -n "$with_munge" && test "$with_munge" != "yes"; then
+            if test -d $with_munge/include/munge; then
+                psec_munge_dir=$with_munge/include/munge
+            else
+                psec_munge_dir=$with_munge
+            fi
+            if test -d $with_munge/lib; then
+                psec_munge_libdir=$with_munge/lib
+            elif -d $with_munge/lib64; then
+                psec_munge_libdir=$with_munge/lib64
+            else
+                AC_MSG_RESULT([Could not find $with_munge/lib or $with_munge/lib64])
+                AC_MSG_ERROR([Can not continue])
+            fi
+            AC_MSG_RESULT([$psec_munge_dir and $psec_munge_libdir])
+        else
+            AC_MSG_RESULT([(default search paths)])
+            psec_munge_dir=
+        fi
+        AS_IF([test -n "$with_munge_libdir" && test "$with_munge_libdir" != "yes"],
+              [psec_munge_libdir="$with_munge_libdir"])
+
+        save_cpp=$CPPFLAGS
+        save_ld=$LDFLAGS
+
+        PMIX_CHECK_PACKAGE([psec_munge],
+                           [munge.h],
+                           [munge],
+                           [munge_encode],
+                           [-lmunge],
+                           [$psec_munge_dir],
+                           [$psec_munge_libdir],
+                           [psec_munge_support=1],
+                           [psec_munge_support=0])
+
+        CPPFLAGS=$save_cpp
+        LDFLAGS=$save_ld
+    fi
+
+    if test -n "$with_munge" && test "$with_munge" != "no" && test "$psec_munge_support" != "1"; then
+        AC_MSG_WARN([MUNGE SUPPORT REQUESTED AND NOT FOUND.])
+        AC_MSG_ERROR([CANNOT CONTINUE])
+    fi
+
+    AC_MSG_CHECKING([will munge support be built])
+    AS_IF([test "$psec_munge_support" != "1"],
+          [AC_MSG_RESULT([no])
+           $2],
+          [AC_MSG_RESULT([yes])
+           $1])
+
+    # set build flags to use in makefile
+    AC_SUBST([psec_munge_CPPFLAGS])
+    AC_SUBST([psec_munge_LDFLAGS])
+    AC_SUBST([psec_munge_LIBS])
+
+    PMIX_VAR_SCOPE_POP
+])dnl

opal/mca/pmix/pmix3x/pmix/src/mca/psec/munge/psec_munge.c

@@ -1,7 +1,5 @@
 /*
- * Copyright (c) 2015      Intel, Inc.  All rights reserved.
+ * Copyright (c) 2015-2016 Intel, Inc.  All rights reserved.
- * Copyright (c) 2016      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
  *
  * NOTE: THE MUNGE CLIENT LIBRARY (libmunge) IS LICENSED AS LGPL
  *
@@ -18,9 +16,8 @@
 
 #include "src/include/pmix_globals.h"
 #include "src/util/argv.h"
-#include "src/util/output.h"
 #include "src/util/error.h"
-#include "src/usock/usock.h"
+#include "src/util/output.h"
 
 #include <unistd.h>
 #ifdef HAVE_SYS_TYPES_H
@@ -28,15 +25,15 @@
 #endif
 #include <munge.h>
 
-#include "pmix_sec.h"
+#include "src/mca/psec/psec.h"
-#include "pmix_munge.h"
+#include "psec_munge.h"
 
 static int munge_init(void);
 static void munge_finalize(void);
 static char* create_cred(void);
 static int validate_cred(pmix_peer_t *peer, char *cred);
 
-pmix_sec_base_module_t pmix_munge_module = {
+pmix_psec_module_t pmix_munge_module = {
     "munge",
     munge_init,
     munge_finalize,
@@ -55,7 +52,7 @@ static int munge_init(void)
     int rc;
 
     pmix_output_verbose(2, pmix_globals.debug_output,
-                        "sec: munge init");
+                        "psec: munge init");
 
     /* attempt to get a credential as a way of checking that
      * the munge server is available - cache the credential
@@ -63,7 +60,7 @@ static int munge_init(void)
 
     if (EMUNGE_SUCCESS != (rc = munge_encode(&mycred, NULL, NULL, 0))) {
         pmix_output_verbose(2, pmix_globals.debug_output,
-                            "sec: munge failed to create credential: %s",
+                            "psec: munge failed to create credential: %s",
                             munge_strerror(rc));
         return PMIX_ERR_SERVER_NOT_AVAIL;
     }
@@ -75,7 +72,7 @@ static int munge_init(void)
 static void munge_finalize(void)
 {
     pmix_output_verbose(2, pmix_globals.debug_output,
-                        "sec: munge finalize");
+                        "psec: munge finalize");
     if (initialized) {
         if (NULL != mycred) {
             free(mycred);
@@ -90,7 +87,7 @@ static char* create_cred(void)
     char *resp=NULL;
 
     pmix_output_verbose(2, pmix_globals.debug_output,
-                        "sec: munge create_cred");
+                        "psec: munge create_cred");
 
     if (initialized) {
         if (!refresh) {
@@ -104,7 +101,7 @@ static char* create_cred(void)
             }
             if (EMUNGE_SUCCESS != (rc = munge_encode(&mycred, NULL, NULL, 0))) {
                 pmix_output_verbose(2, pmix_globals.debug_output,
-                                    "sec: munge failed to create credential: %s",
+                                    "psec: munge failed to create credential: %s",
                                     munge_strerror(rc));
                 return NULL;
             }
@@ -121,12 +118,12 @@ static int validate_cred(pmix_peer_t *peer, char *cred)
     munge_err_t rc;
 
     pmix_output_verbose(2, pmix_globals.debug_output,
-                        "sec: munge validate_cred %s", cred ? cred : "NULL");
+                        "psec: munge validate_cred %s", cred ? cred : "NULL");
 
     /* parse the inbound string */
     if (EMUNGE_SUCCESS != (rc = munge_decode(cred, NULL, NULL, NULL, &uid, &gid))) {
         pmix_output_verbose(2, pmix_globals.debug_output,
-                            "sec: munge failed to decode credential: %s",
+                            "psec: munge failed to decode credential: %s",
                             munge_strerror(rc));
         return PMIX_ERR_INVALID_CRED;
     }
@@ -142,7 +139,6 @@ static int validate_cred(pmix_peer_t *peer, char *cred)
     }
 
     pmix_output_verbose(2, pmix_globals.debug_output,
-                        "sec: munge credential valid");
+                        "psec: munge credential valid");
     return PMIX_SUCCESS;
 }
-

opal/mca/pmix/pmix3x/pmix/src/mca/psec/munge/psec_munge.h

@@ -0,0 +1,25 @@
+/*
+ * Copyright (c) 2015-2016 Intel, Inc.  All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef PMIX_MUNGE_H
+#define PMIX_MUNGE_H
+
+BEGIN_C_DECLS
+
+#include <src/include/pmix_config.h>
+
+
+/* the component must be visible data for the linker to find it */
+PMIX_EXPORT extern pmix_psec_base_component_t mca_psec_munge_component;
+extern pmix_psec_module_t pmix_munge_module;
+
+END_C_DECLS
+
+#endif

opal/mca/pmix/pmix3x/pmix/src/mca/psec/munge/psec_munge_component.c

@@ -0,0 +1,91 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2016      Intel, Inc. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ * These symbols are in a file by themselves to provide nice linker
+ * semantics.  Since linkers generally pull in symbols by object
+ * files, keeping these symbols as the only symbols in this file
+ * prevents utility programs such as "ompi_info" from having to import
+ * entire components just to query their version and parameters.
+ */
+
+#include <src/include/pmix_config.h>
+#include "pmix_common.h"
+
+
+#include "src/mca/psec/psec.h"
+#include "psec_munge.h"
+
+static pmix_status_t component_open(void);
+static pmix_status_t component_close(void);
+static pmix_status_t component_query(pmix_mca_base_module_t **module, int *priority);
+static pmix_psec_module_t* assign_module(void);
+
+/*
+ * Instantiate the public struct with all of our public information
+ * and pointers to our public functions in it
+ */
+pmix_psec_base_component_t mca_psec_munge_component = {
+    .base = {
+        PMIX_PSEC_BASE_VERSION_1_0_0,
+
+        /* Component name and version */
+        .pmix_mca_component_name = "munge",
+        PMIX_MCA_BASE_MAKE_VERSION(component,
+                                   PMIX_MAJOR_VERSION,
+                                   PMIX_MINOR_VERSION,
+                                   PMIX_RELEASE_VERSION),
+
+        /* Component open and close functions */
+        .pmix_mca_open_component = component_open,
+        .pmix_mca_close_component = component_close,
+        .pmix_mca_query_component = component_query,
+    },
+    .data = {
+        /* The component is checkpoint ready */
+        PMIX_MCA_BASE_METADATA_PARAM_CHECKPOINT
+    },
+    .assign_module = assign_module
+};
+
+
+static int component_open(void)
+{
+    return PMIX_SUCCESS;
+}
+
+
+static int component_query(pmix_mca_base_module_t **module, int *priority)
+{
+    *priority = 80;
+    *module = (pmix_mca_base_module_t *)&pmix_munge_module;
+    return PMIX_SUCCESS;
+}
+
+
+static int component_close(void)
+{
+    return PMIX_SUCCESS;
+}
+
+static pmix_psec_module_t* assign_module(void)
+{
+    return &pmix_munge_module;
+}

opal/mca/pmix/pmix3x/pmix/src/mca/psec/native/Makefile.am

@@ -0,0 +1,50 @@
+# -*- makefile -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2012      Los Alamos National Security, Inc.  All rights reserved.
+# Copyright (c) 2013-2016 Intel, Inc. All rights reserved
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+headers = psec_native.h
+sources = \
+        psec_native_component.c \
+        psec_native.c
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if MCA_BUILD_pmix_psec_native_DSO
+lib =
+lib_sources =
+component = mca_psec_native.la
+component_sources = $(headers) $(sources)
+else
+lib = libmca_psec_native.la
+lib_sources = $(headers) $(sources)
+component =
+component_sources =
+endif
+
+mcacomponentdir = $(pmixlibdir)
+mcacomponent_LTLIBRARIES = $(component)
+mca_psec_native_la_SOURCES = $(component_sources)
+mca_psec_native_la_LDFLAGS = -module -avoid-version
+
+noinst_LTLIBRARIES = $(lib)
+libmca_psec_native_la_SOURCES = $(lib_sources)
+libmca_psec_native_la_LDFLAGS = -module -avoid-version

opal/mca/pmix/pmix3x/pmix/src/mca/psec/native/psec_native.c

@@ -1,8 +1,6 @@
 /*
  * Copyright (c) 2015-2016 Intel, Inc.  All rights reserved.
  * Copyright (c) 2016      IBM Corporation.  All rights reserved.
- * Copyright (c) 2016      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
  *
  * $COPYRIGHT$
  *
@@ -20,44 +18,37 @@
 #include "src/util/argv.h"
 #include "src/util/error.h"
 #include "src/util/output.h"
-#include "src/usock/usock.h"
 
 #include <unistd.h>
 #ifdef HAVE_SYS_TYPES_H
 #include <sys/types.h>
 #endif
-#ifdef HAVE_UCRED_H
-#include <ucred.h>
-#endif
 
-#include "pmix_sec.h"
+#include "src/mca/psec/psec.h"
-#include "pmix_native.h"
+#include "psec_native.h"
 
 static int native_init(void);
 static void native_finalize(void);
 static pmix_status_t validate_cred(pmix_peer_t *peer, char *cred);
 
-pmix_sec_base_module_t pmix_native_module = {
+pmix_psec_module_t pmix_native_module = {
-    "native",
+    .name = "native",
-    native_init,
+    .init = native_init,
-    native_finalize,
+    .finalize = native_finalize,
-    NULL,
+    .validate_cred = validate_cred
-    NULL,
-    validate_cred,
-    NULL
 };
 
 static int native_init(void)
 {
     pmix_output_verbose(2, pmix_globals.debug_output,
-                        "sec: native init");
+                        "psec: native init");
     return PMIX_SUCCESS;
 }
 
 static void native_finalize(void)
 {
     pmix_output_verbose(2, pmix_globals.debug_output,
-                        "sec: native finalize");
+                        "psec: native finalize");
 }
 
 static pmix_status_t validate_cred(pmix_peer_t *peer, char *cred)
@@ -70,24 +61,22 @@ static pmix_status_t validate_cred(pmix_peer_t *peer, char *cred)
     struct ucred ucred;
 #endif
     socklen_t crlen = sizeof (ucred);
-#endif
-#ifdef HAVE_GETPEERUCRED
-    ucred_t *ucred = NULL;
 #endif
     uid_t euid;
     gid_t gid;
 
     pmix_output_verbose(2, pmix_globals.debug_output,
-                        "sec: native validate_cred %s", cred ? cred : "NULL");
+                        "psec: native validate_cred %s", cred ? cred : "NULL");
 
 #if defined(SO_PEERCRED) && (defined(HAVE_STRUCT_UCRED_UID) || defined(HAVE_STRUCT_UCRED_CR_UID))
     /* Ignore received 'cred' and validate ucred for socket instead. */
     pmix_output_verbose(2, pmix_globals.debug_output,
-                        "sec:native checking getsockopt for peer credentials");
+                        "psec:native checking getsockopt for peer credentials");
     if (getsockopt (peer->sd, SOL_SOCKET, SO_PEERCRED, &ucred, &crlen) < 0) {
         pmix_output_verbose(2, pmix_globals.debug_output,
-                            "sec: getsockopt SO_PEERCRED failed: %s",
+                            "psec: getsockopt SO_PEERCRED failed: %s",
                             strerror (pmix_socket_errno));
+        pmix_output(0, "ONE");
         return PMIX_ERR_INVALID_CRED;
     }
 #if defined(HAVE_STRUCT_UCRED_UID)
@@ -100,51 +89,37 @@ static pmix_status_t validate_cred(pmix_peer_t *peer, char *cred)
 
 #elif defined(HAVE_GETPEEREID)
     pmix_output_verbose(2, pmix_globals.debug_output,
-                        "sec:native checking getpeereid for peer credentials");
+                        "psec:native checking getpeereid for peer credentials");
     if (0 != getpeereid(peer->sd, &euid, &gid)) {
         pmix_output_verbose(2, pmix_globals.debug_output,
-                            "sec: getsockopt getpeereid failed: %s",
+                            "psec: getsockopt getpeereid failed: %s",
                             strerror (pmix_socket_errno));
+        pmix_output(0, "TWO");
         return PMIX_ERR_INVALID_CRED;
     }
-#elif defined(HAVE_GETPEERUCRED)
-    pmix_output_verbose(2, pmix_globals.debug_output,
-                        "sec:native checking getpeerucred for peer credentials");
-    if (0 != getpeerucred(peer->sd, &ucred)) {
-        pmix_output_verbose(2, pmix_globals.debug_output,
-                            "sec: getsockopt getpeerucred failed: %s",
-                            strerror (pmix_socket_errno));
-        pmix_output_verbose(2, pmix_globals.debug_output,
-                            "sec: getsockopt getpeerucred failed: %s",
-                            strerror (errno));
-        return PMIX_ERR_INVALID_CRED;
-    }
-    euid = ucred_geteuid(ucred);
-    gid = ucred_getrgid(ucred);
-    ucred_free(ucred);
 #else
-    pmix_output_verbose(2, pmix_globals.debug_output,
+    pmix_output(0, "FIVE");
-                        "sec: native cannot validate_cred on this system");
     return PMIX_ERR_NOT_SUPPORTED;
 #endif
 
     /* check uid */
     if (euid != peer->info->uid) {
         pmix_output_verbose(2, pmix_globals.debug_output,
-                            "sec: socket cred contains invalid uid %u", euid);
+                            "psec: socket cred contains invalid uid %u", euid);
+        pmix_output(0, "THREE");
         return PMIX_ERR_INVALID_CRED;
     }
 
     /* check gid */
     if (gid != peer->info->gid) {
         pmix_output_verbose(2, pmix_globals.debug_output,
-                            "sec: socket cred contains invalid gid %u", gid);
+                            "psec: socket cred contains invalid gid %u", gid);
+        pmix_output(0, "FOUR");
         return PMIX_ERR_INVALID_CRED;
     }
 
     pmix_output_verbose(2, pmix_globals.debug_output,
-                        "sec: native credential %u:%u valid",
+                        "psec: native credential %u:%u valid",
                         euid, gid);
     return PMIX_SUCCESS;
 }
-

opal/mca/pmix/pmix3x/pmix/src/mca/psec/native/psec_native.h

@@ -0,0 +1,27 @@
+/*
+ * Copyright (c) 2015-2016 Intel, Inc.  All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef PMIX_NATIVE_H
+#define PMIX_NATIVE_H
+
+#include <src/include/pmix_config.h>
+
+
+#include "src/mca/psec/psec.h"
+
+BEGIN_C_DECLS
+
+/* the component must be visible data for the linker to find it */
+PMIX_EXPORT extern pmix_psec_base_component_t mca_psec_native_component;
+extern pmix_psec_module_t pmix_native_module;
+
+END_C_DECLS
+
+#endif

opal/mca/pmix/pmix3x/pmix/src/mca/psec/native/psec_native_component.c

@@ -0,0 +1,91 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2016      Intel, Inc. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ * These symbols are in a file by themselves to provide nice linker
+ * semantics.  Since linkers generally pull in symbols by object
+ * files, keeping these symbols as the only symbols in this file
+ * prevents utility programs such as "ompi_info" from having to import
+ * entire components just to query their version and parameters.
+ */
+
+#include <src/include/pmix_config.h>
+#include "pmix_common.h"
+
+
+#include "src/mca/psec/psec.h"
+#include "psec_native.h"
+
+static pmix_status_t component_open(void);
+static pmix_status_t component_close(void);
+static pmix_status_t component_query(pmix_mca_base_module_t **module, int *priority);
+static pmix_psec_module_t* assign_module(void);
+
+/*
+ * Instantiate the public struct with all of our public information
+ * and pointers to our public functions in it
+ */
+pmix_psec_base_component_t mca_psec_native_component = {
+    .base = {
+        PMIX_PSEC_BASE_VERSION_1_0_0,
+
+        /* Component name and version */
+        .pmix_mca_component_name = "native",
+        PMIX_MCA_BASE_MAKE_VERSION(component,
+                                   PMIX_MAJOR_VERSION,
+                                   PMIX_MINOR_VERSION,
+                                   PMIX_RELEASE_VERSION),
+
+        /* Component open and close functions */
+        .pmix_mca_open_component = component_open,
+        .pmix_mca_close_component = component_close,
+        .pmix_mca_query_component = component_query,
+    },
+    .data = {
+        /* The component is checkpoint ready */
+        PMIX_MCA_BASE_METADATA_PARAM_CHECKPOINT
+    },
+    .assign_module = assign_module
+};
+
+
+static int component_open(void)
+{
+    return PMIX_SUCCESS;
+}
+
+
+static int component_query(pmix_mca_base_module_t **module, int *priority)
+{
+    *priority = 10;
+    *module = (pmix_mca_base_module_t *)&pmix_native_module;
+    return PMIX_SUCCESS;
+}
+
+
+static int component_close(void)
+{
+    return PMIX_SUCCESS;
+}
+
+static pmix_psec_module_t* assign_module(void)
+{
+    return &pmix_native_module;
+}

opal/mca/pmix/pmix3x/pmix/src/mca/psec/psec.h

@@ -0,0 +1,141 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2007-2008 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2016 Intel, Inc.  All rights reserved.
+ *
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+/**
+ * @file
+ *
+ * This interface is for psecurity support. PMIx doesn't need much in
+ * this regard, but we do need a mechanism for authenticating connections.
+ *
+ * Only *one* plugin will be active in a client, but multiple plugins may
+ * be active in a server. Thus, this is a multi-select framework.
+ *
+ * Available plugins may be defined at runtime via the typical MCA parameter
+ * syntax.
+ */
+
+#ifndef PMIX_PSEC_H
+#define PMIX_PSEC_H
+
+#include <src/include/pmix_config.h>
+
+#include "src/mca/mca.h"
+#include "src/mca/base/pmix_mca_base_var.h"
+#include "src/mca/base/pmix_mca_base_framework.h"
+
+BEGIN_C_DECLS
+
+/*** forward declaration ***/
+struct pmix_peer_t;
+
+/******    MODULE DEFINITION    ******/
+
+/**
+ * Initialize the module. Returns an error if the module cannot
+ * run, success if it can and wants to be used.
+ */
+typedef int (*pmix_psec_base_module_init_fn_t)(void);
+
+/**
+ * Finalize the module. Tear down any allocated storage, disconnect
+ * from any system support (e.g., LDAP server)
+ */
+typedef void (*pmix_psec_base_module_fini_fn_t)(void);
+
+/****    CLIENT-SIDE FUNCTIONS    ****/
+/**
+ * Create and return a string representation of a credential for this
+ * client
+ */
+typedef char* (*pmix_psec_base_module_create_cred_fn_t)(void);
+
+/**
+ * Perform the client-side handshake. Note that it is not required
+ * (and indeed, would be rare) for a protocol to use both the
+ * credential and handshake interfaces. It is acceptable, therefore,
+ * for one of them to be NULL */
+typedef pmix_status_t (*pmix_psec_base_module_client_hndshk_fn_t)(int sd);
+
+
+/****    SERVER-SIDE FUNCTIONS    ****/
+/**
+ * Validate a client's credential
+ */
+typedef pmix_status_t (*pmix_psec_base_module_validate_cred_fn_t)(struct pmix_peer_t *peer, char *cred);
+
+/**
+ * Perform the server-side handshake. Note that it is not required
+ * (and indeed, would be rare) for a protocol to use both the
+ * credential and handshake interfaces. It is acceptable, therefore,
+ * for one of them to be NULL */
+typedef pmix_status_t (*pmix_psec_base_module_server_hndshk_fn_t)(struct pmix_peer_t *peer);
+
+/**
+ * Base structure for a PSEC module
+ */
+typedef struct {
+    char *name;
+    /* init/finalize */
+    pmix_psec_base_module_init_fn_t           init;
+    pmix_psec_base_module_fini_fn_t           finalize;
+    /** Client-side */
+    pmix_psec_base_module_create_cred_fn_t    create_cred;
+    pmix_psec_base_module_client_hndshk_fn_t  client_handshake;
+    /** Server-side */
+    pmix_psec_base_module_validate_cred_fn_t  validate_cred;
+    pmix_psec_base_module_server_hndshk_fn_t  server_handshake;
+} pmix_psec_module_t;
+
+
+/****    COMPONENT STRUCTURE DEFINITION    ****/
+
+/* define a component-level API for initializing the component */
+typedef pmix_status_t (*pmix_psec_base_component_init_fn_t)(void);
+
+/* define a component-level API for finalizing the component */
+typedef void (*pmix_psec_base_component_finalize_fn_t)(void);
+
+/* define a component-level API for getting a module */
+typedef pmix_psec_module_t* (*pmix_psec_base_component_assign_module_fn_t)(void);
+
+/*
+ * the standard component data structure
+ */
+struct pmix_psec_base_component_t {
+    pmix_mca_base_component_t                        base;
+    pmix_mca_base_component_data_t                   data;
+    int                                              priority;
+    pmix_psec_base_component_init_fn_t               init;
+    pmix_psec_base_component_finalize_fn_t           finalize;
+    pmix_psec_base_component_assign_module_fn_t      assign_module;
+};
+typedef struct pmix_psec_base_component_t pmix_psec_base_component_t;
+
+/* Select a psec module for a given peer */
+pmix_psec_module_t* pmix_psec_base_assign_module(const char *options);
+
+/* get a list of available options - caller must free results
+ * when done */
+char* pmix_psec_base_get_available_modules(void);
+
+
+/*
+ * Macro for use in components that are of type psec
+ */
+#define PMIX_PSEC_BASE_VERSION_1_0_0 \
+    PMIX_MCA_BASE_VERSION_1_0_0("psec", 1, 0, 0)
+
+END_C_DECLS
+
+#endif

opal/mca/pmix/pmix3x/pmix/src/runtime/pmix_finalize.c

@@ -24,8 +24,6 @@
 
 #include <src/include/pmix_config.h>
 
-#if 0
-
 #include "src/class/pmix_object.h"
 #include "src/client/pmix_client_ops.h"
 #include "src/usock/usock.h"
@@ -35,8 +33,8 @@
 #include "src/mca/base/base.h"
 #include "src/mca/base/pmix_mca_base_var.h"
 #include "src/mca/pinstalldirs/base/base.h"
-#include "src/mca/bfrops/base/base.h"
 #include "src/mca/psec/base/base.h"
+#include "src/dstore/pmix_dstore.h"
 #include PMIX_EVENT_HEADER
 
 #include "src/runtime/pmix_rte.h"
@@ -68,7 +66,7 @@ void pmix_rte_finalize(void)
 
     /* shutdown communications */
     pmix_usock_finalize();
-    if (PMIX_PROC_CLIENT == pmix_globals.proc_type &&
+    if (PMIX_PROC_SERVER != pmix_globals.proc_type &&
         0 <= pmix_client_globals.myserver.sd) {
         CLOSE_THE_SOCKET(pmix_client_globals.myserver.sd);
     }
@@ -97,8 +95,10 @@ void pmix_rte_finalize(void)
        much */
     pmix_output_finalize();
 
+#if 0
     /* close the bfrops */
     (void)pmix_mca_base_framework_close(&pmix_bfrops_base_framework);
+#endif
 
     if (!pmix_globals.external_evbase) {
         /* stop the progress thread */
@@ -123,4 +123,3 @@ void pmix_rte_finalize(void)
         pmix_cleanup();
     #endif
 }
-#endif

opal/mca/pmix/pmix3x/pmix/src/runtime/pmix_init.c

@@ -29,18 +29,19 @@
 
 #include <src/include/pmix_config.h>
 
-#if 0
-
 #ifdef HAVE_UNISTD_H
 #include <unistd.h>
 #endif
+#include PMIX_EVENT_HEADER
+#include "event2/thread.h"
+
+#include <pmix_rename.h>
 
 #include "src/util/output.h"
 #include "src/util/show_help.h"
 #include "src/mca/base/base.h"
 #include "src/mca/base/pmix_mca_base_var.h"
 #include "src/mca/pinstalldirs/base/base.h"
-#include "src/mca/bfrops/base/base.h"
 #include "src/mca/psec/base/base.h"
 
 #include "src/event/pmix_event.h"
@@ -61,14 +62,17 @@ const char pmix_version_string[] = PMIX_IDENT_STRING;
 
 int pmix_initialized = 0;
 bool pmix_init_called = false;
-pmix_globals_t pmix_globals = {
+/* we have to export the pmix_globals object so
+ * all plugins can access it. However, it is included
+ * in the pmix_rename.h file for external protection */
+PMIX_EXPORT pmix_globals_t pmix_globals = {
     .init_cntr = 0,
     .mypeer = NULL,
+    .proc_type = PMIX_PROC_UNDEF,
     .pindex = 0,
     .evbase = NULL,
     .external_evbase = false,
     .debug_output = -1,
-    .proc_type = PMIX_PROC_UNDEF,
     .connected = false,
     .cache_local = NULL,
     .cache_remote = NULL
@@ -76,7 +80,8 @@ pmix_globals_t pmix_globals = {
 
 
 int pmix_rte_init(pmix_proc_type_t type,
-                  pmix_info_t info[], size_t ninfo)
+                  pmix_info_t info[], size_t ninfo,
+                  pmix_usock_cbfunc_t notifycbfunc)
 {
     int ret, debug_level;
     char *error = NULL, *evar;
@@ -163,6 +168,10 @@ int pmix_rte_init(pmix_proc_type_t type,
     }
     /* create our peer object */
     pmix_globals.mypeer = PMIX_NEW(pmix_peer_t);
+    if (NULL == pmix_globals.mypeer) {
+        ret = PMIX_ERR_NOMEM;
+        goto return_error;
+    }
 
     /* scan incoming info for directives */
     if (NULL != info) {
@@ -173,7 +182,9 @@ int pmix_rte_init(pmix_proc_type_t type,
             }
         }
     }
+    pmix_bfrop_open();
 
+#if 0
     /* open the bfrops - we will select the active plugin later */
     if( PMIX_SUCCESS != (ret = pmix_mca_base_framework_open(&pmix_bfrops_base_framework, 0)) ) {
         error = "pmix_bfrops_base_open";
@@ -183,6 +194,7 @@ int pmix_rte_init(pmix_proc_type_t type,
         error = "pmix_bfrops_base_select";
         goto return_error;
     }
+#endif
 
     /* open the psec and select the default module for this environment */
     if (PMIX_SUCCESS != (ret = pmix_mca_base_framework_open(&pmix_psec_base_framework, 0))) {
@@ -194,7 +206,12 @@ int pmix_rte_init(pmix_proc_type_t type,
         goto return_error;
     }
     param = getenv("PMIX_SEC_MODULE");  // if directive was given, use it
-    pmix_globals.mypeer->comm.sec = pmix_psec_base_assign_module(param);
+    pmix_globals.mypeer->compat.psec = pmix_psec_base_assign_module(param);
+    if (NULL == pmix_globals.mypeer->compat.psec) {
+        ret = PMIX_ERR_NOT_SUPPORTED;
+        error = "no psec module";
+        goto return_error;
+    }
 
     /* tell libevent that we need thread support */
     pmix_event_use_threads();
@@ -207,6 +224,7 @@ int pmix_rte_init(pmix_proc_type_t type,
         }
     }
 
+#if 0
     /* open the PMIx Transport Layer (ptl) - we will select the
      * active plugin later */
     if( PMIX_SUCCESS != (ret = pmix_mca_base_framework_open(&pmix_ptl_base_framework, 0)) ) {
@@ -228,13 +246,10 @@ int pmix_rte_init(pmix_proc_type_t type,
         error = "pmix_pdstor_base_select";
         goto return_error;
     }
+#endif
 
     /* setup the usock service */
-    if (PMIX_PROC_SERVER == type) {
+    pmix_usock_init(notifycbfunc);
-        pmix_usock_init(NULL);
-    } else {
-        pmix_usock_init(pmix_client_notify_recv);
-    }
 
     return PMIX_SUCCESS;
 
@@ -244,5 +259,3 @@ int pmix_rte_init(pmix_proc_type_t type,
                     error, ret );
     return ret;
 }
-
-#endif

opal/mca/pmix/pmix3x/pmix/src/runtime/pmix_rte.h

@@ -56,7 +56,8 @@ extern const char pmix_version_string[];
  *
  */
 pmix_status_t pmix_rte_init(pmix_proc_type_t type,
-                            pmix_info_t info[], size_t ninfo);
+                            pmix_info_t info[], size_t ninfo,
+                            pmix_usock_cbfunc_t cbfunc);
 
 /**
  * Finalize the PMIX layer, including the MCA system.
@@ -70,9 +71,6 @@ void pmix_rte_finalize(void);
 pmix_status_t pmix_register_params(void);
 pmix_status_t pmix_deregister_params(void);
 
-void pmix_client_notify_recv(struct pmix_peer_t *peer, pmix_usock_hdr_t *hdr,
-                             pmix_buffer_t *buf, void *cbdata);
-
 END_C_DECLS
 
 #endif /* PMIX_RTE_H */

opal/mca/pmix/pmix3x/pmix/src/sec/Makefile.am

@@ -1,32 +0,0 @@
-#
-# Copyright (c) 2015      Intel, Inc.  All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-headers += \
-        sec/pmix_sec.h \
-        sec/pmix_native.h
-
-sources += \
-        sec/pmix_sec.c \
-        sec/pmix_native.c
-
-if PMIX_WANT_MUNGE
-headers += \
-        sec/pmix_munge.h
-
-sources += \
-        sec/pmix_munge.c
-endif
-
-if PMIX_WANT_SASL
-headers += \
-        sec/pmix_sasl.h
-
-sources += \
-        sec/pmix_sasl.c
-endif

opal/mca/pmix/pmix3x/pmix/src/sec/pmix_munge.h

@@ -1,23 +0,0 @@
-/*
- * Copyright (c) 2015      Intel, Inc.  All rights reserved.
- *
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#ifndef PMIX_MUNGE_H
-#define PMIX_MUNGE_H
-
-BEGIN_C_DECLS
-
-#include <src/include/pmix_config.h>
-
-
-extern pmix_sec_base_module_t pmix_munge_module;
-
-END_C_DECLS
-
-#endif

opal/mca/pmix/pmix3x/pmix/src/sec/pmix_native.h

@@ -1,25 +0,0 @@
-/*
- * Copyright (c) 2015      Intel, Inc.  All rights reserved.
- *
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#ifndef PMIX_NATIVE_H
-#define PMIX_NATIVE_H
-
-#include <src/include/pmix_config.h>
-
-
-#include "src/sec/pmix_sec.h"
-
-BEGIN_C_DECLS
-
-extern pmix_sec_base_module_t pmix_native_module;
-
-END_C_DECLS
-
-#endif

opal/mca/pmix/pmix3x/pmix/src/sec/pmix_sasl.c

@@ -1,97 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2014-2015 Intel, Inc.  All rights reserved.
- * Copyright (c) 2014      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2014-2015 Artem Y. Polyakov <artpol84@gmail.com>.
- *                         All rights reserved.
- * Copyright (c) 2016      Mellanox Technologies, Inc.
- *                         All rights reserved.
- * Copyright (c) 2016      IBM Corporation.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include <src/include/pmix_config.h>
-
-#include <src/include/types.h>
-#include <src/include/pmix_stdint.h>
-#include <src/include/pmix_socket_errno.h>
-
-#include "src/include/pmix_globals.h"
-
-#ifdef HAVE_STRING_H
-#include <string.h>
-#endif
-#include <fcntl.h>
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-#ifdef HAVE_SYS_SOCKET_H
-#include <sys/socket.h>
-#endif
-#ifdef HAVE_SYS_UN_H
-#include <sys/un.h>
-#endif
-#ifdef HAVE_SYS_UIO_H
-#include <sys/uio.h>
-#endif
-#ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
-#endif
-#include PMIX_EVENT_HEADER
-#include <sasl/sasl.h>
-
-#include "src/class/pmix_list.h"
-#include "src/buffer_ops/buffer_ops.h"
-#include "src/util/argv.h"
-#include "src/util/error.h"
-#include "src/util/output.h"
-#include "src/util/pmix_environ.h"
-#include "src/usock/usock.h"
-
-#include "pmix_sasl.h"
-
-static int sasl_init(void);
-static void sasl_finalize(void);
-static int client_handshake(int sd);
-static int server_handshake(pmix_peer_t *peer);
-
-pmix_sec_base_module_t pmix_sasl_module = {
-    "sasl",
-    sasl_init,
-    sasl_finalize,
-    NULL,
-    client_handshake,
-    NULL,
-    server_handshake
-};
-
-
-static int sasl_init(void)
-{
-    pmix_output_verbose(2, pmix_globals.debug_output,
-                        "sec: sasl init");
-
-    return PMIX_ERR_NOT_SUPPORTED;
-}
-
-static void sasl_finalize(void)
-{
-    pmix_output_verbose(2, pmix_globals.debug_output,
-                        "sec: sasl finalize");
-}
-
-static int client_handshake(int sd)
-{
-    return PMIX_ERR_NOT_SUPPORTED;
-}
-
-
-static int server_handshake(pmix_peer_t *peer)
-{
-    return PMIX_ERR_NOT_IMPLEMENTED;
-}

opal/mca/pmix/pmix3x/pmix/src/sec/pmix_sasl.h

@@ -1,25 +0,0 @@
-/*
- * Copyright (c) 2015      Intel, Inc.  All rights reserved.
- *
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#ifndef PMIX_SASL_H
-#define PMIX_SASL_H
-
-#include <src/include/pmix_config.h>
-
-
-#include "src/sec/pmix_sec.h"
-
-BEGIN_C_DECLS
-
-extern pmix_sec_base_module_t pmix_sasl_module;
-
-END_C_DECLS
-
-#endif

opal/mca/pmix/pmix3x/pmix/src/sec/pmix_sec.c

@@ -1,170 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2015      Intel, Inc.  All rights reserved.
- * Copyright (c) 2016      Mellanox Technologies, Inc.
- *                         All rights reserved.
- * Copyright (c) 2016      IBM Corporation.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include <src/include/pmix_config.h>
-
-#include <src/include/types.h>
-#include <src/include/pmix_stdint.h>
-
-#include <pmix_common.h>
-#include "src/include/pmix_globals.h"
-
-#ifdef HAVE_STRING_H
-#include <string.h>
-#endif
-#include <fcntl.h>
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-#ifdef HAVE_SYS_SOCKET_H
-#include <sys/socket.h>
-#endif
-#ifdef HAVE_SYS_UN_H
-#include <sys/un.h>
-#endif
-#ifdef HAVE_SYS_UIO_H
-#include <sys/uio.h>
-#endif
-#ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
-#endif
-#include PMIX_EVENT_HEADER
-
-#include "src/util/argv.h"
-#include "src/util/output.h"
-#include "pmix_sec.h"
-
-#include "src/sec/pmix_native.h"
-#if PMIX_WANT_MUNGE
-#include "src/sec/pmix_munge.h"
-#endif
-#
-
-/*
- * Array of all possible SPCs
- */
-
-/****  ENSURE THE FOLLOWING VALUE IS AT LEAST AS
- ****  LARGE AS THE TOTAL NUMBER OF SUPPORTED SPCs
- ****  IN THE ARRAY BELOW
- */
-#define PMIX_SEC_NAVAIL  3
-
-static pmix_sec_base_module_t *all[] = {
-#if PMIX_WANT_MUNGE
-    &pmix_munge_module,
-#endif
-    &pmix_native_module,
-
-    /* Always end the array with a NULL */
-    NULL
-};
-
-pmix_sec_base_module_t pmix_sec = {0};
-
-int pmix_sec_init(void)
-{
-    pmix_sec_base_module_t *available[PMIX_SEC_NAVAIL];
-    int i, j, navail=0;
-    char *evar, **options;
-    bool exclude;
-
-    /* see if the PMIX_SEC_MODE envar has been provided */
-    if (NULL != (evar = getenv("PMIX_SECURITY_MODE"))) {
-        /* if the leading character is '^', then we are excluding */
-        if ('^' == evar[0]) {
-            options = pmix_argv_split(&evar[1], ',');
-            /* cycle thru the built modules and xfer them
-             * to the available array if they aren't on the list */
-            for (i = 0; NULL != all[i]; ++i) {
-                exclude = false;
-                for (j = 0; NULL != options[j]; ++j) {
-                    if (0 == strcmp(options[j], all[i]->name)) {
-                        pmix_output_verbose(30, pmix_globals.debug_output,
-                                           "Security mode %s excluded", all[i]->name);
-                        exclude = true;
-                        break;
-                    }
-                }
-                if (!exclude) {
-                    available[navail] = all[i];
-                    ++navail;
-                }
-            }
-        } else {
-            options = pmix_argv_split(evar, ',');
-            /* cycle thru the built modules and xfer them
-             * to the available array in the specified order */
-            for (j = 0; NULL != options[j]; ++j) {
-                for (i = 0; NULL != all[i]; ++i) {
-                    if (0 == strcmp(options[j], all[i]->name)) {
-                        pmix_output_verbose(30, pmix_globals.debug_output,
-                                           "Security SPC include: %s", all[i]->name);
-                        available[navail] = all[i];
-                        ++navail;
-                        break;
-                    }
-                }
-                if (NULL == all[i]) {
-                    /* we didn't find one they specified */
-                    pmix_output(0, "Security mode %s is not available", options[j]);
-                    pmix_argv_free(options);
-                    return PMIX_ERR_NOT_FOUND;
-                }
-            }
-        }
-        pmix_argv_free(options);
-    } else {
-        /* everything is available */
-        for (i=0; NULL != all[i]; i++) {
-            available[i] = all[i];
-            navail++;
-        }
-    }
-
-    /* if nothing is available, then that is an error - we need
-     * at least our native SPC */
-    if (0 == navail) {
-        pmix_output(0, "No Security modes are available");
-        return PMIX_ERR_NOT_FOUND;
-    }
-
-    /* now go thru the available modules in order until someone
-     * indicates they are able/willing to run */
-    for (i=0; i < navail; i++) {
-        if (NULL == available[i]->init) {
-            /* we assume this is acceptable */
-            pmix_sec = *available[i];
-            pmix_output_verbose(2, pmix_globals.debug_output,
-                                "sec: SPC %s active", pmix_sec.name);
-            return PMIX_SUCCESS;
-        }
-        if (PMIX_SUCCESS == available[i]->init()) {
-            pmix_sec = *available[i];
-            pmix_output_verbose(2, pmix_globals.debug_output,
-                                "sec: SPC %s active", pmix_sec.name);
-            return PMIX_SUCCESS;
-        }
-    }
-    /* if we get here, then none of the available modules
-     * returned success */
-    return PMIX_ERR_NOT_FOUND;
-}
-
-void pmix_sec_finalize(void)
-{
-    /* tell the active module to finalize */
-    if (NULL != pmix_sec.finalize) {
-        pmix_sec.finalize();
-    }
-}

opal/mca/pmix/pmix3x/pmix/src/sec/pmix_sec.h

@@ -1,143 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2007-2008 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Intel, Inc.  All rights reserved.
- *
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-/**
- * @file
- *
- * This interface is designed to serve as a low-weight plugin mechansim
- * for security support. PMIx doesn't need much in this regard, but we do
- * need a mechanism for authenticating connections. Dlopen and
- * friends are not used, but all the functionality is accessed through
- * struct's of function pointers, so you can swap between multiple
- * different implementations at run time, just like typical plugins.
- * Hence, these entities are referred to as "Security
- * Pseudo-Components" (SPCs).
- *
- * The SPCs are referenced by their names (e.g., "sasl", "munge").
- *
- * Only *one* SPC will be active in a given system. The SPC to be
- * used, however, may be selected at runtime by setting the "PMIX_SEC_MODE"
- * environmental parameter. This param can consist of either:
- *
- * a single SPC name - if this SPC is not available, an error will
- * be returned
- *
- * a comma-separated list of SPC names - the available SPCs will be
- * queried in the specified order. The first one to return "success"
- * will be selected and used. An error will be returned if none of
- * the available SPCs return success.
- *
- * either of the above, with a '^' as the first character - this
- * indicates that the specified SPC(s) are to be excluded from
- * consideration. The remaining SPCs will be queried until one returns
- * "success" - if none return success, then an error will be returned.
- *
- * Module interface:
- *
- * module_init(): The PMIx client and server init functions
- * call pmix_sec_init(), which will invoke this init function on
- * each SPC to see if it wants to run.  SPCs can gracefully
- * remove themselves from consideration in this process by returning
- * PMIX_ERR_NOT_SUPPORTED.
- *
- * initiate_connection(): Executes the client side of the
- * authentication procedure. Returns 0 if successful, or
- * a non-zero error.
- *
- * accept_connection(): Executes the server side of the
- * authentication procedure. Returns 0 if successful, or
- * a non-zero error.
- *
- * module_finalize(): The PMIx client and server finalize functions
- * call pmix_sec_finalize(), which, in turn, calls the
- * module_finalize() function on all available SPCs.
- */
-
-#ifndef PMIX_SEC_H
-#define PMIX_SEC_H
-
-#include <src/include/pmix_config.h>
-
-
-#include "src/usock/usock.h"
-
-BEGIN_C_DECLS
-
-/******    MODULE DEFINITION    ******/
-
-/**
- * Initialize the module. Returns an error if the module cannot
- * run, success if it can and wants to be used.
- */
-typedef int (*pmix_sec_base_module_init_fn_t)(void);
-
-/**
- * Finalize the module. Tear down any allocated storage, disconnect
- * from any system support (e.g., LDAP server)
- */
-typedef void (*pmix_sec_base_module_fini_fn_t)(void);
-
-/****    CLIENT-SIDE FUNCTIONS    ****/
-/**
- * Create and return a string representation of a credential for this
- * client
- */
-typedef char* (*pmix_sec_base_module_create_cred_fn_t)(void);
-
-/**
- * Perform the client-side handshake. Note that it is not required
- * (and indeed, would be rare) for a protocol to use both the
- * credential and handshake interfaces. It is acceptable, therefore,
- * for one of them to be NULL */
-typedef pmix_status_t (*pmix_sec_base_module_client_hndshk_fn_t)(int sd);
-
-
-/****    SERVER-SIDE FUNCTIONS    ****/
-/**
- * Validate a client's credential
- */
-typedef pmix_status_t (*pmix_sec_base_module_validate_cred_fn_t)(pmix_peer_t *peer, char *cred);
-
-/**
- * Perform the server-side handshake. Note that it is not required
- * (and indeed, would be rare) for a protocol to use both the
- * credential and handshake interfaces. It is acceptable, therefore,
- * for one of them to be NULL */
-typedef pmix_status_t (*pmix_sec_base_module_server_hndshk_fn_t)(pmix_peer_t *peer);
-
-/**
- * Struct for holding CPC module function pointers
- */
-typedef struct {
-    char *name;
-    /* init/finalize */
-    pmix_sec_base_module_init_fn_t           init;
-    pmix_sec_base_module_fini_fn_t           finalize;
-    /** Client-side */
-    pmix_sec_base_module_create_cred_fn_t    create_cred;
-    pmix_sec_base_module_client_hndshk_fn_t  client_handshake;
-    /** Server-side */
-    pmix_sec_base_module_validate_cred_fn_t  validate_cred;
-    pmix_sec_base_module_server_hndshk_fn_t  server_handshake;
-} pmix_sec_base_module_t;
-
-extern pmix_sec_base_module_t pmix_sec;
-
-/* initialize and finalize the security system */
-int pmix_sec_init(void);
-void pmix_sec_finalize(void);
-
-END_C_DECLS
-
-#endif

opal/mca/pmix/pmix3x/pmix/src/server/help-pmix-server.txt

@@ -18,7 +18,7 @@ system's restriction for Unix's socket's path-length.
 Please try to set TMPDIR to something short (like /tmp) or change
 your computer's name to something shorter (see uname -n).
 [listener-failed-start]
-The PMIx server was unable to start its listening thread. This is
+The PMIx server was unable to setup a rendezvous point. This is
 usually due to a conflicting stale named pipe from a prior failed
 job, thus preventing the server from binding to its assigned socket.
 
@@ -33,3 +33,6 @@ The PMIx server was unable to store the specified key-value:
 The precise reason for the failure was provided in the above
 "error-log" message. This is probably something that should
 be referred to the PMIx developers.
+[listener-thread-start]
+The PMIx server's listener thread failed to start. We cannot
+continue.

opal/mca/pmix/pmix3x/pmix/src/server/pmix_server.c

@@ -23,6 +23,8 @@
 
 #include <pmix_server.h>
 #include <pmix_common.h>
+#include <pmix_rename.h>
+
 #include "src/include/pmix_globals.h"
 
 #ifdef HAVE_STRING_H
@@ -58,8 +60,8 @@
 #include "src/mca/base/pmix_mca_base_var.h"
 #include "src/mca/pinstalldirs/base/base.h"
 #include "src/runtime/pmix_progress_threads.h"
+#include "src/runtime/pmix_rte.h"
 #include "src/usock/usock.h"
-#include "src/sec/pmix_sec.h"
 #if defined(PMIX_ENABLE_DSTORE) && (PMIX_ENABLE_DSTORE == 1)
 #include "src/dstore/pmix_dstore.h"
 #endif /* PMIX_ENABLE_DSTORE */
@@ -122,35 +124,9 @@ void pmix_server_queue_message(int fd, short args, void *cbdata)
 
 static pmix_status_t initialize_server_base(pmix_server_module_t *module)
 {
-    int debug_level;
     char *tdir, *evar;
     char * pmix_pid;
     pmix_listener_t *listener;
-    pmix_status_t ret;
-
-    /* initialize the output system */
-    if (!pmix_output_init()) {
-        fprintf(stderr, "PMIx server was unable to initialize its output system\n");
-        return PMIX_ERR_INIT;
-    }
-    /* initialize install dirs code */
-    if (PMIX_SUCCESS != (ret = pmix_mca_base_framework_open(&pmix_pinstalldirs_base_framework, 0))) {
-        fprintf(stderr, "pmix_pinstalldirs_base_open() failed -- process will likely abort (%s:%d, returned %d instead of PMIX_SUCCESS)\n",
-                __FILE__, __LINE__, ret);
-        return ret;
-    }
-
-    if (PMIX_SUCCESS != pmix_show_help_init()) {
-        fprintf(stderr, "PMIx server was unable to initialize its show_help system\n");
-        return PMIX_ERR_INIT;
-    }
-
-    /* setup the globals */
-    pmix_globals_init();
-    memset(&pmix_server_globals, 0, sizeof(pmix_server_globals));
-
-    /* mark that I am a server */
-    pmix_globals.server = true;
 
     /* look for our namespace, if one was given */
     if (NULL == (evar = getenv("PMIX_SERVER_NAMESPACE"))) {
@@ -168,9 +144,6 @@ static pmix_status_t initialize_server_base(pmix_server_module_t *module)
         pmix_globals.myid.rank = strtol(evar, NULL, 10);
     }
 
-    /* initialize the datatype support */
-    pmix_bfrop_open();
-
     /* setup the server-specific globals */
     PMIX_CONSTRUCT(&pmix_server_globals.clients, pmix_pointer_array_t);
     pmix_pointer_array_init(&pmix_server_globals.clients, 1, INT_MAX, 1);
@@ -183,13 +156,6 @@ static pmix_status_t initialize_server_base(pmix_server_module_t *module)
     PMIX_CONSTRUCT(&pmix_server_globals.listeners, pmix_list_t);
     pmix_ring_buffer_init(&pmix_server_globals.notifications, 256);
 
-    /* see if debug is requested */
-    if (NULL != (evar = getenv("PMIX_DEBUG"))) {
-        debug_level = strtol(evar, NULL, 10);
-        pmix_globals.debug_output = pmix_output_open(NULL);
-        pmix_output_set_verbosity(pmix_globals.debug_output, debug_level);
-    }
-
     pmix_output_verbose(2, pmix_globals.debug_output,
                         "pmix:server init called");
 
@@ -197,10 +163,6 @@ static pmix_status_t initialize_server_base(pmix_server_module_t *module)
     memset(&pmix_host_server, 0, sizeof(pmix_server_module_t));
     pmix_host_server = *module;
 
-    /* init security */
-    pmix_sec_init();
-    security_mode = strdup(pmix_sec.name);
-
     /* find the temp dir */
     if (NULL != mytmpdir) {
         tdir = mytmpdir;
@@ -268,6 +230,13 @@ PMIX_EXPORT pmix_status_t PMIx_server_init(pmix_server_module_t *module,
     pmix_output_verbose(2, pmix_globals.debug_output,
                         "pmix:server init called");
 
+    /* setup the runtime - this init's the globals,
+     * opens and initializes the required frameworks */
+    if (PMIX_SUCCESS != (rc = pmix_rte_init(PMIX_PROC_SERVER, info, ninfo, NULL))) {
+        PMIX_ERROR_LOG(rc);
+        return rc;
+    }
+
     /* Check for the info keys that are not independent from
      * initialize_server_base() and even may be needed there */
     if (NULL != info) {
@@ -298,17 +267,6 @@ PMIX_EXPORT pmix_status_t PMIx_server_init(pmix_server_module_t *module,
     }
 #endif /* PMIX_ENABLE_DSTORE */
 
-    /* and the usock system */
-    pmix_usock_init(NULL);
-
-    /* tell the event library we need thread support */
-    pmix_event_use_threads();
-
-    /* create an event base and progress thread for us */
-    if (NULL == (pmix_globals.evbase = pmix_progress_thread_init(NULL))) {
-        return PMIX_ERR_INIT;
-    }
-
     /* check the info keys for a directive about the uid/gid
      * to be set for the rendezvous file */
     if (NULL != info) {
@@ -441,13 +399,14 @@ PMIX_EXPORT pmix_status_t PMIx_server_init(pmix_server_module_t *module,
     need_listener = false;
     PMIX_LIST_FOREACH(lt, &pmix_server_globals.listeners, pmix_listener_t) {
         if (PMIX_SUCCESS != pmix_prepare_listening(lt, &need_listener)) {
+            pmix_show_help("help-pmix-server.txt", "listener-failed-start", true, lt->address.sun_path);
             PMIx_server_finalize();
             return PMIX_ERR_INIT;
         }
     }
     if (need_listener) {
         if (PMIX_SUCCESS != pmix_start_listening()) {
-            pmix_show_help("help-pmix-server.txt", "listener-failed-start", true, tl->address.sun_path);
+            pmix_show_help("help-pmix-server.txt", "listener-thread-start", true);
             PMIx_server_finalize();
             return PMIX_ERR_INIT;
         }
@@ -513,10 +472,6 @@ static void cleanup_server_state(void)
     PMIX_DESTRUCT(&pmix_server_globals.gdata);
     PMIX_LIST_DESTRUCT(&pmix_server_globals.listeners);
 
-    if (NULL != security_mode) {
-        free(security_mode);
-    }
-
     if (NULL != mytmpdir) {
         free(mytmpdir);
     }
@@ -526,14 +481,7 @@ static void cleanup_server_state(void)
     }
 
     pmix_bfrop_close();
-    pmix_sec_finalize();
+    pmix_rte_finalize();
-    pmix_globals_finalize();
-    pmix_output_verbose(2, pmix_globals.debug_output,
-                        "pmix:server finalize complete");
-
-    pmix_output_close(pmix_globals.debug_output);
-    pmix_output_finalize();
-    pmix_class_finalize();
 }
 
 PMIX_EXPORT pmix_status_t PMIx_server_finalize(void)
@@ -551,17 +499,6 @@ PMIX_EXPORT pmix_status_t PMIx_server_finalize(void)
         pmix_stop_listening();
     }
 
-    pmix_progress_thread_finalize(NULL);
-#ifdef HAVE_LIBEVENT_GLOBAL_SHUTDOWN
-    libevent_global_shutdown();
-#endif
-
-    pmix_usock_finalize();
-
-#if defined(PMIX_ENABLE_DSTORE) && (PMIX_ENABLE_DSTORE == 1)
-    pmix_dstore_finalize();
-#endif /* PMIX_ENABLE_DSTORE */
-
     cleanup_server_state();
     pmix_output_verbose(2, pmix_globals.debug_output,
                         "pmix:server finalize complete");
@@ -721,7 +658,7 @@ static void _register_nspace(int sd, short args, void *cbdata)
     /* do not destruct the kv object - no memory leak will result */
 
 #if defined(PMIX_ENABLE_DSTORE) && (PMIX_ENABLE_DSTORE == 1)
-    if (0 > pmix_dstore_nspace_add(cd->proc.nspace)) {
+    if (PMIX_SUCCESS != (rc = pmix_dstore_nspace_add(cd->proc.nspace, cd->info, cd->ninfo))) {
         PMIX_ERROR_LOG(rc);
         goto release;
     }
@@ -768,6 +705,7 @@ static void _deregister_nspace(int sd, short args, void *cbdata)
 {
     pmix_setup_caddy_t *cd = (pmix_setup_caddy_t*)cbdata;
     pmix_nspace_t *tmp;
+    pmix_status_t rc = PMIX_SUCCESS;
 
     pmix_output_verbose(2, pmix_globals.debug_output,
                         "pmix:server _deregister_nspace %s",
@@ -782,8 +720,14 @@ static void _deregister_nspace(int sd, short args, void *cbdata)
         }
     }
 
+#if defined(PMIX_ENABLE_DSTORE) && (PMIX_ENABLE_DSTORE == 1)
+    if (0 > (rc = pmix_dstore_nspace_del(cd->proc.nspace))) {
+        PMIX_ERROR_LOG(rc);
+    }
+#endif
+
     if (NULL != cd->opcbfunc) {
-        cd->opcbfunc(PMIX_SUCCESS, cd->cbdata);
+        cd->opcbfunc(rc, cd->cbdata);
     }
     PMIX_RELEASE(cd);
 }
@@ -1094,6 +1038,7 @@ PMIX_EXPORT pmix_status_t PMIx_server_setup_fork(const pmix_proc_t *proc, char *
 {
     char rankstr[128];
     pmix_listener_t *lt;
+    pmix_status_t rc;
 
     pmix_output_verbose(2, pmix_globals.debug_output,
                         "pmix:server setup_fork for nspace %s rank %d",
@@ -1115,7 +1060,10 @@ PMIX_EXPORT pmix_status_t PMIx_server_setup_fork(const pmix_proc_t *proc, char *
 
 #if defined(PMIX_ENABLE_DSTORE) && (PMIX_ENABLE_DSTORE == 1)
     /* pass dstore path to files */
-    pmix_dstore_patch_env(env);
+    if (PMIX_SUCCESS != (rc = pmix_dstore_patch_env(proc->nspace, env))) {
+        PMIX_ERROR_LOG(rc);
+        return rc;
+    }
 #endif
 
     return PMIX_SUCCESS;
@@ -1290,7 +1238,7 @@ PMIX_EXPORT pmix_status_t PMIx_Store_internal(const pmix_proc_t *proc,
         return rc;
     }
 
-    if (pmix_globals.server) {
+    if (PMIX_PROC_SERVER == pmix_globals.proc_type) {
         PMIX_THREADSHIFT(cd, _store_internal);
         PMIX_WAIT_FOR_COMPLETION(cd->active);
     } else {

opal/mca/pmix/pmix3x/pmix/src/server/pmix_server_get.c

@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2014-2015 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014-2016 Intel, Inc.  All rights reserved.
  * Copyright (c) 2014-2015 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2014-2015 Artem Y. Polyakov <artpol84@gmail.com>.
@@ -22,6 +22,7 @@
 #include <src/include/pmix_socket_errno.h>
 
 #include <pmix_server.h>
+#include <pmix_rename.h>
 #include "src/include/pmix_globals.h"
 
 #ifdef HAVE_STRING_H
@@ -52,7 +53,6 @@
 #include "src/util/output.h"
 #include "src/util/pmix_environ.h"
 #include "src/usock/usock.h"
-#include "src/sec/pmix_sec.h"
 #if defined(PMIX_ENABLE_DSTORE) && (PMIX_ENABLE_DSTORE == 1)
 #include "src/dstore/pmix_dstore.h"
 #endif /* PMIX_ENABLE_DSTORE */

opal/mca/pmix/pmix3x/pmix/src/server/pmix_server_listener.c

@@ -58,7 +58,7 @@
 #include "src/util/pmix_environ.h"
 #include "src/util/strnlen.h"
 #include "src/usock/usock.h"
-#include "src/sec/pmix_sec.h"
+#include "src/mca/psec/psec.h"
 
 #include "pmix_server_ops.h"
 
@@ -413,9 +413,6 @@ static void process_cbfunc(int sd, short args, void *cbdata)
         return;
     }
 
-    /* set the socket non-blocking for all further operations */
-    pmix_usock_set_nonblocking(pnd->sd);
-
     /* add this nspace to our pool */
     nptr = PMIX_NEW(pmix_nspace_t);
     (void)strncpy(nptr->nspace, cd->proc.nspace, PMIX_MAX_NSLEN);
@@ -426,6 +423,9 @@ static void process_cbfunc(int sd, short args, void *cbdata)
     PMIX_RETAIN(nptr);
     info->nptr = nptr;
     info->rank = 0;
+    /* need to include the uid/gid for validation */
+    info->uid = pnd->uid;
+    info->gid = pnd->gid;
     pmix_list_append(&nptr->server->ranks, &info->super);
 
     /* setup a peer object for this tool */
@@ -434,6 +434,71 @@ static void process_cbfunc(int sd, short args, void *cbdata)
     peer->info = info;
     peer->proc_cnt = 1;
     peer->sd = pnd -> sd;
+
+    /* get the appropriate compatibility modules */
+   // peer->comm.type = pnd->buffer_type;
+   // peer->comm.bfrops = pmix_bfrops_base_assign_module(pnd->bfrop);
+    peer->compat.psec = pmix_psec_base_assign_module(pnd->psec);
+
+    /* if a credential is available, then check it */
+    if (NULL != peer->compat.psec->validate_cred) {
+        if (PMIX_SUCCESS != (rc = peer->compat.psec->validate_cred(peer, pnd->cred))) {
+            pmix_output_verbose(2, pmix_globals.debug_output,
+                                "validation of tool credential failed: %s",
+                                PMIx_Error_string(rc));
+            PMIX_RELEASE(peer);
+            pmix_list_remove_item(&pmix_globals.nspaces, &nptr->super);
+            PMIX_RELEASE(nptr);  // will release the info object
+            CLOSE_THE_SOCKET(pnd->sd);
+            goto done;
+        }
+        pmix_output_verbose(2, pmix_globals.debug_output,
+                            "tool credential validated");
+        /* send them success */
+        rc = PMIX_SUCCESS;
+        if (PMIX_SUCCESS != (rc = pmix_usock_send_blocking(pnd->sd, (char*)&rc, sizeof(int)))) {
+            PMIX_ERROR_LOG(rc);
+            PMIX_RELEASE(peer);
+            pmix_list_remove_item(&pmix_globals.nspaces, &nptr->super);
+            PMIX_RELEASE(nptr);  // will release the info object
+            CLOSE_THE_SOCKET(pnd->sd);
+            goto done;
+        }
+    } else if (NULL != peer->compat.psec->server_handshake) {
+        /* execute the handshake if the security mode calls for it */
+        pmix_output_verbose(2, pmix_globals.debug_output,
+                            "connect-ack executing handshake");
+        rc = PMIX_ERR_READY_FOR_HANDSHAKE;
+        if (PMIX_SUCCESS != (rc = pmix_usock_send_blocking(sd, (char*)&rc, sizeof(int)))) {
+            PMIX_ERROR_LOG(rc);
+            PMIX_RELEASE(peer);
+            pmix_list_remove_item(&pmix_globals.nspaces, &nptr->super);
+            PMIX_RELEASE(nptr);  // will release the info object
+            CLOSE_THE_SOCKET(pnd->sd);
+            goto done;
+        }
+        if (PMIX_SUCCESS != (rc = peer->compat.psec->server_handshake(peer))) {
+            PMIX_ERROR_LOG(rc);
+            PMIX_RELEASE(peer);
+            pmix_list_remove_item(&pmix_globals.nspaces, &nptr->super);
+            PMIX_RELEASE(nptr);  // will release the info object
+            CLOSE_THE_SOCKET(pnd->sd);
+            goto done;
+        }
+        pmix_output_verbose(2, pmix_globals.debug_output,
+                            "tool connect-ack handshake complete");
+    } else {
+        /* this is not allowed */
+        PMIX_RELEASE(peer);
+        pmix_list_remove_item(&pmix_globals.nspaces, &nptr->super);
+        PMIX_RELEASE(nptr);  // will release the info object
+        CLOSE_THE_SOCKET(pnd->sd);
+        goto done;
+    }
+
+    /* set the socket non-blocking for all further operations */
+    pmix_usock_set_nonblocking(pnd->sd);
+
     if (0 > (peer->index = pmix_pointer_array_add(&pmix_server_globals.clients, peer))) {
         PMIX_RELEASE(pnd);
         PMIX_RELEASE(cd);
@@ -454,6 +519,8 @@ static void process_cbfunc(int sd, short args, void *cbdata)
     pmix_output_verbose(2, pmix_globals.debug_output,
                         "pmix:server tool %s:%d has connected on socket %d",
                         peer->info->nptr->nspace, peer->info->rank, peer->sd);
+
+  done:
     PMIX_RELEASE(pnd);
     PMIX_RELEASE(cd);
 }
@@ -474,13 +541,16 @@ static void cnct_cbfunc(pmix_status_t status,
 }
 
 /* Parse init-ack message:
- *    NSPACE<0><rank>VERSION<0>[CRED<0>]
+ *    NSPACE<0><rank>VERSION<0>BFROP<0>SEC<0>BUFTYPE<0>[CRED<0>]
  */
 static pmix_status_t parse_connect_ack (char *msg,
                                         pmix_listener_protocol_t protocol,
                                         int len,
                                         char **nspace, pmix_rank_t *rank,
-                                        char **version, char **cred)
+                                        char **version,
+                                        char **bfrop, char **sec,
+                                        pmix_bfrop_buffer_type_t *type,
+                                        char **cred)
 {
     int msglen;
 
@@ -491,15 +561,16 @@ static pmix_status_t parse_connect_ack (char *msg,
             msg += strlen(*nspace) + 1;
             len -= strlen(*nspace) + 1;
         } else {
+            PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
             return PMIX_ERR_BAD_PARAM;
         }
 
-        PMIX_STRNLEN(msglen, msg, len);
+        if ((int)sizeof(pmix_rank_t) <= len) {
-        if (msglen <= len) {
             memcpy(rank, msg, sizeof(pmix_rank_t));
             msg += sizeof(pmix_rank_t);
             len -= sizeof(pmix_rank_t);
         } else {
+            PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
             return PMIX_ERR_BAD_PARAM;
         }
     }
@@ -510,9 +581,43 @@ static pmix_status_t parse_connect_ack (char *msg,
         msg += strlen(*version) + 1;
         len -= strlen(*version) + 1;
     } else {
+        PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
         return PMIX_ERR_BAD_PARAM;
     }
 
+#if 0
+    PMIX_STRNLEN(msglen, msg, len);
+    if (msglen < len) {
+        *bfrop = msg;
+        msg += strlen(*bfrop) + 1;
+        len -= strlen(*bfrop) + 1;
+    } else {
+        PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
+        return PMIX_ERR_BAD_PARAM;
+    }
+#endif
+
+    PMIX_STRNLEN(msglen, msg, len);
+    if (msglen < len) {
+        *sec = msg;
+        msg += strlen(*sec) + 1;
+        len -= strlen(*sec) + 1;
+    } else {
+        PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+#if 0
+    if ((int)sizeof(pmix_bfrop_buffer_type_t) <= len) {
+        memcpy(type, msg, sizeof(pmix_bfrop_buffer_type_t));
+        msg += sizeof(pmix_bfrop_buffer_type_t);
+        len -= sizeof(pmix_bfrop_buffer_type_t);
+    } else {
+        PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
+        return PMIX_ERR_BAD_PARAM;
+    }
+#endif
+
     PMIX_STRNLEN(msglen, msg, len);
     if (msglen < len)
         *cred = msg;
@@ -530,7 +635,7 @@ static pmix_status_t pmix_server_authenticate(pmix_pending_connection_t *pnd,
                                               pmix_rank_t *out_rank,
                                               pmix_peer_t **peer)
 {
-    char *msg, *nspace, *version, *cred;
+    char *msg, *nspace=NULL, *version=NULL, *bfrop=NULL, *sec=NULL, *cred=NULL;
     pmix_status_t rc;
     pmix_rank_t rank=PMIX_RANK_UNDEF;
     pmix_usock_hdr_t hdr;
@@ -541,6 +646,7 @@ static pmix_status_t pmix_server_authenticate(pmix_pending_connection_t *pnd,
     pmix_proc_t proc;
     uid_t uid;
     gid_t gid;
+    pmix_bfrop_buffer_type_t buffer_type = 0;
 
     pmix_output_verbose(2, pmix_globals.debug_output,
                         "RECV CONNECT ACK FROM PEER ON SOCKET %d",
@@ -574,154 +680,23 @@ static pmix_status_t pmix_server_authenticate(pmix_pending_connection_t *pnd,
         free(msg);
         return PMIX_ERR_UNREACH;
     }
-    if (PMIX_SUCCESS != (rc = parse_connect_ack(msg, pnd->protocol, hdr.nbytes, &nspace,
+    if (PMIX_SUCCESS != (rc = parse_connect_ack(msg, pnd->protocol,
-                                                &rank, &version, &cred))) {
+                                                hdr.nbytes, &nspace,
+                                                &rank, &version,
+                                                &bfrop, &sec,
+                                                &buffer_type, &cred))) {
         pmix_output_verbose(2, pmix_globals.debug_output,
                             "error parsing connect-ack from client ON SOCKET %d", pnd->sd);
         free(msg);
         return rc;
     }
 
-    /* if the attaching process is not a tool, then set it up as
+    /* if the attaching process is a tool, then kickoff that procedure */
-     * a known peer */
+    if (PMIX_PROTOCOL_TOOL == pnd->protocol) {
-    if (PMIX_PROTOCOL_TOOL != pnd->protocol) {
-        pmix_output_verbose(2, pmix_globals.debug_output,
-                            "connect-ack recvd from peer %s:%d:%s",
-                            nspace, rank, version);
-
-        /* do not check the version - we only retain it at this
-         * time in case we need to check it at some future date.
-         * For now, our intent is to retain backward compatibility
-         * and so we will assume that all versions are compatible. */
-
-        /* see if we know this nspace */
-        nptr = NULL;
-        PMIX_LIST_FOREACH(tmp, &pmix_globals.nspaces, pmix_nspace_t) {
-            if (0 == strcmp(tmp->nspace, nspace)) {
-                nptr = tmp;
-                break;
-            }
-        }
-        if (NULL == nptr) {
-            /* we don't know this namespace, reject it */
-            free(msg);
-            /* send an error reply to the client */
-            rc = PMIX_ERR_NOT_FOUND;
-            goto error;
-        }
-
-        /* see if we have this peer in our list */
-        info = NULL;
-        found = false;
-        PMIX_LIST_FOREACH(info, &nptr->server->ranks, pmix_rank_info_t) {
-            if (info->rank == rank) {
-                found = true;
-                break;
-            }
-        }
-        if (!found) {
-            /* rank unknown, reject it */
-            free(msg);
-            /* send an error reply to the client */
-            rc = PMIX_ERR_NOT_FOUND;
-            goto error;
-        }
-        *out_rank = rank;
-        /* a peer can connect on multiple sockets since it can fork/exec
-         * a child that also calls PMIx_Init, so add it here if necessary.
-         * Create the tracker for this peer */
-        psave = PMIX_NEW(pmix_peer_t);
-        PMIX_RETAIN(info);
-        psave->info = info;
-        info->proc_cnt++; /* increase number of processes on this rank */
-        psave->sd = pnd->sd;
-        if (0 > (psave->index = pmix_pointer_array_add(&pmix_server_globals.clients, psave))) {
-            free(msg);
-            PMIX_RELEASE(psave);
-            /* probably cannot send an error reply if we are out of memory */
-            return PMIX_ERR_OUT_OF_RESOURCE;
-        }
-        /* see if there is a credential */
-        if (NULL != pmix_sec.validate_cred) {
-            if (PMIX_SUCCESS != (rc = pmix_sec.validate_cred(psave, cred))) {
-                pmix_output_verbose(2, pmix_globals.debug_output,
-                                    "validation of client credential failed");
-                free(msg);
-                pmix_pointer_array_set_item(&pmix_server_globals.clients, psave->index, NULL);
-                PMIX_RELEASE(psave);
-                /* send an error reply to the client */
-                goto error;
-            }
-            pmix_output_verbose(2, pmix_globals.debug_output,
-                                "client credential validated");
-        }
-    }
-    free(msg);
-
-    /* execute the handshake if the security mode calls for it */
-    if (NULL != pmix_sec.server_handshake) {
-        pmix_output_verbose(2, pmix_globals.debug_output,
-                            "connect-ack executing handshake");
-        rc = PMIX_ERR_READY_FOR_HANDSHAKE;
-        if (PMIX_SUCCESS != (rc = pmix_usock_send_blocking(pnd->sd, (char*)&rc, sizeof(int)))) {
-            PMIX_ERROR_LOG(rc);
-            if (NULL != psave) {
-                pmix_pointer_array_set_item(&pmix_server_globals.clients, psave->index, NULL);
-                PMIX_RELEASE(psave);
-            }
-            return rc;
-        }
-        if (PMIX_SUCCESS != (rc = pmix_sec.server_handshake(psave))) {
-            PMIX_ERROR_LOG(rc);
-            if (NULL != psave) {
-                pmix_pointer_array_set_item(&pmix_server_globals.clients, psave->index, NULL);
-                PMIX_RELEASE(psave);
-            }
-            return rc;
-        }
-        pmix_output_verbose(2, pmix_globals.debug_output,
-                            "connect-ack handshake complete");
-    } else {
-        /* send them success */
-        rc = PMIX_SUCCESS;
-        if (PMIX_SUCCESS != (rc = pmix_usock_send_blocking(pnd->sd, (char*)&rc, sizeof(int)))) {
-            PMIX_ERROR_LOG(rc);
-            if (NULL != psave) {
-                pmix_pointer_array_set_item(&pmix_server_globals.clients, psave->index, NULL);
-                PMIX_RELEASE(psave);
-            }
-            return rc;
-        }
-    }
-
-    /* if the attaching process is not a tool, then send its index */
-    if (PMIX_PROTOCOL_TOOL != pnd->protocol) {
-        /* send the client's array index */
-        if (PMIX_SUCCESS != (rc = pmix_usock_send_blocking(pnd->sd, (char*)&psave->index, sizeof(int)))) {
-            PMIX_ERROR_LOG(rc);
-            pmix_pointer_array_set_item(&pmix_server_globals.clients, psave->index, NULL);
-            PMIX_RELEASE(psave);
-            return rc;
-        }
-
-        pmix_output_verbose(2, pmix_globals.debug_output,
-                            "connect-ack from client completed");
-
-        *peer = psave;
-        /* let the host server know that this client has connected */
-        if (NULL != pmix_host_server.client_connected) {
-            (void)strncpy(proc.nspace, psave->info->nptr->nspace, PMIX_MAX_NSLEN);
-            proc.rank = psave->info->rank;
-            rc = pmix_host_server.client_connected(&proc, psave->info->server_object,
-                                                   NULL, NULL);
-            if (PMIX_SUCCESS != rc) {
-                PMIX_ERROR_LOG(rc);
-            }
-        }
-    } else {
         /* get the tool socket's uid and gid so we can pass them to
          * the host RM for validation */
         if (PMIX_SUCCESS != (rc = pmix_util_getid(pnd->sd, &uid, &gid))) {
+            free(msg);
             return rc;
         }
         /* we pass this info in an array of pmix_info_t structs,
@@ -734,11 +709,189 @@ static pmix_status_t pmix_server_authenticate(pmix_pending_connection_t *pnd,
         (void)strncpy(pnd->info[1].key, PMIX_GRPID, PMIX_MAX_KEYLEN);
         pnd->info[0].value.type = PMIX_UINT32;
         pnd->info[0].value.data.uint32 = gid;
+        /* pass along the bfrop, buffer_type, and sec fields so
+         * we can assign them once we create a peer object */
+        pnd->uid = uid;
+        pnd->gid = gid;
+        if (NULL != bfrop) {
+            pnd->bfrop = strdup(bfrop);
+        }
+        if (NULL != sec) {
+            pnd->psec = strdup(sec);
+        }
+        pnd->buffer_type = buffer_type;
+        /* pass along the credential for later */
+        if (NULL != cred) {
+            pnd->cred = strdup(cred);
+        }
+        /* release the msg */
+        free(msg);
         /* request an nspace for this requestor - it will
          * automatically be assigned rank=0 */
         pmix_host_server.tool_connected(pnd->info, pnd->ninfo, cnct_cbfunc, pnd);
         return PMIX_ERR_OPERATION_IN_PROGRESS;
     }
+
+    /* if the attaching process is not a tool, then set it up as
+     * a known peer */
+    pmix_output_verbose(2, pmix_globals.debug_output,
+                        "connect-ack recvd from peer %s:%d:%s",
+                        nspace, rank, version);
+
+    /* do not check the version - we only retain it at this
+     * time in case we need to check it at some future date.
+     * For now, our intent is to retain backward compatibility
+     * and so we will assume that all versions are compatible. */
+
+    /* see if we know this nspace */
+    nptr = NULL;
+    PMIX_LIST_FOREACH(tmp, &pmix_globals.nspaces, pmix_nspace_t) {
+        if (0 == strcmp(tmp->nspace, nspace)) {
+            nptr = tmp;
+            break;
+        }
+    }
+    if (NULL == nptr) {
+        /* we don't know this namespace, reject it */
+        free(msg);
+        /* send an error reply to the client */
+        rc = PMIX_ERR_NOT_FOUND;
+        goto error;
+    }
+
+    /* see if we have this peer in our list */
+    info = NULL;
+    found = false;
+    PMIX_LIST_FOREACH(info, &nptr->server->ranks, pmix_rank_info_t) {
+        if (info->rank == rank) {
+            found = true;
+            break;
+        }
+    }
+    if (!found) {
+        /* rank unknown, reject it */
+        free(msg);
+        /* send an error reply to the client */
+        rc = PMIX_ERR_NOT_FOUND;
+        goto error;
+    }
+    *out_rank = rank;
+
+    /* a peer can connect on multiple sockets since it can fork/exec
+     * a child that also calls PMIx_Init, so add it here if necessary.
+     * Create the tracker for this peer */
+    psave = PMIX_NEW(pmix_peer_t);
+    if (NULL == psave) {
+        rc = PMIX_ERR_NOMEM;
+        goto error;
+    }
+    PMIX_RETAIN(info);
+    psave->info = info;
+    info->proc_cnt++; /* increase number of processes on this rank */
+    psave->sd = pnd->sd;
+    if (0 > (psave->index = pmix_pointer_array_add(&pmix_server_globals.clients, psave))) {
+        free(msg);
+        PMIX_RELEASE(psave);
+        /* probably cannot send an error reply if we are out of memory */
+        return PMIX_ERR_OUT_OF_RESOURCE;
+    }
+    if (PMIX_PROTOCOL_V1 == pnd->protocol) {
+        /* we ignore the version string from 1.1 and before */
+        /* assign the 1.1 bfrops module to this peer */
+      //  psave->compat.type = buffer_type;
+      //  psave->compat.bfrops = pmix_bfrops_base_assign_module("1.1");
+        /* take the default sec module as that is what we
+         * told them to use */
+        psave->compat.psec = pmix_psec_base_assign_module(NULL);
+    } else if (PMIX_PROTOCOL_V2 == pnd->protocol) {
+        /* the newer protocol contains two fields:
+         * (a) a string containing the supported bfrops modules
+         * (b) a string containing the supported sec modules
+         */
+        /* set the bfrops module to match this peer */
+      //  psave->compat.type = buffer_type;
+      //  psave->compat.bfrops = pmix_bfrops_base_assign_module(bfrop);
+        /* set the sec module to match this peer */
+        psave->compat.psec = pmix_psec_base_assign_module(sec);
+    } else {
+        /* unrecognized */
+        free(msg);
+        pmix_pointer_array_set_item(&pmix_server_globals.clients, psave->index, NULL);
+        PMIX_RELEASE(psave);
+        PMIX_RELEASE(info);
+        return PMIX_ERR_BAD_PARAM;
+    }
+    free(msg);
+
+    /* see if there is a credential */
+    if (NULL != psave->compat.psec->validate_cred) {
+        if (PMIX_SUCCESS != (rc = psave->compat.psec->validate_cred(psave, cred))) {
+            pmix_output_verbose(2, pmix_globals.debug_output,
+                                "validation of client credential failed");
+            pmix_pointer_array_set_item(&pmix_server_globals.clients, psave->index, NULL);
+            PMIX_RELEASE(psave);
+            /* send an error reply to the client */
+            goto error;
+        }
+        pmix_output_verbose(2, pmix_globals.debug_output,
+                            "client credential validated");
+        /* send them success */
+        rc = PMIX_SUCCESS;
+        if (PMIX_SUCCESS != (rc = pmix_usock_send_blocking(pnd->sd, (char*)&rc, sizeof(int)))) {
+            PMIX_ERROR_LOG(rc);
+            pmix_pointer_array_set_item(&pmix_server_globals.clients, psave->index, NULL);
+            PMIX_RELEASE(psave);
+            return rc;
+        }
+    } else if (NULL != pmix_globals.mypeer->compat.psec->server_handshake) {
+        pmix_output_verbose(2, pmix_globals.debug_output,
+                            "connect-ack executing handshake");
+        rc = PMIX_ERR_READY_FOR_HANDSHAKE;
+        if (PMIX_SUCCESS != (rc = pmix_usock_send_blocking(pnd->sd, (char*)&rc, sizeof(int)))) {
+            PMIX_ERROR_LOG(rc);
+            pmix_pointer_array_set_item(&pmix_server_globals.clients, psave->index, NULL);
+            PMIX_RELEASE(psave);
+            return rc;
+        }
+        if (PMIX_SUCCESS != (rc = pmix_globals.mypeer->compat.psec->server_handshake(psave))) {
+            PMIX_ERROR_LOG(rc);
+            pmix_pointer_array_set_item(&pmix_server_globals.clients, psave->index, NULL);
+            PMIX_RELEASE(psave);
+            return rc;
+        }
+        pmix_output_verbose(2, pmix_globals.debug_output,
+                            "connect-ack handshake complete");
+    } else {
+        /* this is not allowed */
+        pmix_pointer_array_set_item(&pmix_server_globals.clients, psave->index, NULL);
+        PMIX_RELEASE(psave);
+        /* send an error reply to the client */
+        rc = PMIX_ERR_INVALID_CRED;
+        goto error;
+    }
+
+    /* send the client's array index */
+    if (PMIX_SUCCESS != (rc = pmix_usock_send_blocking(pnd->sd, (char*)&psave->index, sizeof(int)))) {
+        PMIX_ERROR_LOG(rc);
+        pmix_pointer_array_set_item(&pmix_server_globals.clients, psave->index, NULL);
+        PMIX_RELEASE(psave);
+        return rc;
+    }
+
+    pmix_output_verbose(2, pmix_globals.debug_output,
+                        "connect-ack from client completed");
+
+    *peer = psave;
+    /* let the host server know that this client has connected */
+    if (NULL != pmix_host_server.client_connected) {
+        (void)strncpy(proc.nspace, psave->info->nptr->nspace, PMIX_MAX_NSLEN);
+        proc.rank = psave->info->rank;
+        rc = pmix_host_server.client_connected(&proc, psave->info->server_object,
+                                               NULL, NULL);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+        }
+    }
     return rc;
 
   error:

opal/mca/pmix/pmix3x/pmix/src/server/pmix_server_ops.c

@@ -55,7 +55,6 @@
 #include "src/util/output.h"
 #include "src/util/pmix_environ.h"
 #include "src/usock/usock.h"
-#include "src/sec/pmix_sec.h"
 
 #include "pmix_server_ops.h"
 
@@ -1583,12 +1582,24 @@ static void pccon(pmix_pending_connection_t *p)
     memset(p->nspace, 0, PMIX_MAX_NSLEN+1);
     p->info = NULL;
     p->ninfo = 0;
+    p->bfrop = NULL;
+    p->psec = NULL;
+    p->cred = NULL;
 }
 static void pcdes(pmix_pending_connection_t *p)
 {
     if (NULL != p->info) {
         PMIX_INFO_FREE(p->info, p->ninfo);
     }
+    if (NULL != p->bfrop) {
+        free(p->bfrop);
+    }
+    if (NULL != p->psec) {
+        free(p->psec);
+    }
+    if (NULL != p->cred) {
+        free(p->cred);
+    }
 }
 PMIX_CLASS_INSTANCE(pmix_pending_connection_t,
                     pmix_object_t,

opal/mca/pmix/pmix3x/pmix/src/server/pmix_server_ops.h

@@ -90,7 +90,7 @@ PMIX_CLASS_DECLARATION(pmix_dmdx_local_t);
 typedef uint16_t pmix_listener_protocol_t;
 #define PMIX_PROTOCOL_V1        0
 #define PMIX_PROTOCOL_TOOL      1
-#define PMIX_PROTOCOL V2        2
+#define PMIX_PROTOCOL_V2        2
 
 /* connection support */
 typedef struct {
@@ -103,6 +103,12 @@ typedef struct {
     size_t ninfo;
     pmix_status_t status;
     struct sockaddr_storage addr;
+    char *bfrop;
+    char *psec;
+    pmix_bfrop_buffer_type_t buffer_type;
+    char *cred;
+    uid_t uid;
+    gid_t gid;
 } pmix_pending_connection_t;
 PMIX_CLASS_DECLARATION(pmix_pending_connection_t);
 

opal/mca/pmix/pmix3x/pmix/src/server/pmix_server_regex.c

@@ -19,6 +19,7 @@
 
 #include <src/include/types.h>
 #include <src/include/pmix_stdint.h>
+#include <pmix_rename.h>
 
 #include "src/include/pmix_globals.h"
 

opal/mca/pmix/pmix3x/pmix/src/tool/pmix_tool.c

@@ -22,6 +22,7 @@
 
 #include "src/client/pmix_client_ops.h"
 #include <pmix_tool.h>
+#include <pmix_rename.h>
 
 #include "src/include/pmix_globals.h"
 
@@ -66,8 +67,9 @@ extern pmix_client_globals_t pmix_client_globals;
 #include "src/util/hash.h"
 #include "src/util/output.h"
 #include "src/runtime/pmix_progress_threads.h"
+#include "src/runtime/pmix_rte.h"
 #include "src/usock/usock.h"
-#include "src/sec/pmix_sec.h"
+#include "src/mca/psec/psec.h"
 #include "src/include/pmix_globals.h"
 #if defined(PMIX_ENABLE_DSTORE) && (PMIX_ENABLE_DSTORE == 1)
 #include "src/dstore/pmix_dstore.h"
@@ -187,7 +189,6 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc,
                                pmix_info_t info[], size_t ninfo)
 {
     char *evar, *tdir, *tmp;
-    int debug_level;
     struct sockaddr_un address;
     size_t n;
     pmix_kval_t *kptr;
@@ -222,10 +223,7 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc,
     /* scan incoming info for directives */
     if (NULL != info) {
         for (n=0; n < ninfo; n++) {
-            if (0 == strcmp(PMIX_EVENT_BASE, info[n].key)) {
+            if (strcmp(info[n].key, PMIX_SERVER_PIDINFO) == 0) {
-                pmix_globals.evbase = (pmix_event_base_t*)info[n].value.data.ptr;
-                pmix_globals.external_evbase = true;
-            } else if (strcmp(info[n].key, PMIX_SERVER_PIDINFO) == 0) {
                 server_pid = info[n].value.data.pid;
                 server_pid_given = true;
             } else if (strcmp(info[n].key, PMIX_CONNECT_TO_SYSTEM) == 0) {
@@ -244,26 +242,16 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc,
         }
     }
 
-    /* setup the globals */
+    /* setup the runtime - this init's the globals,
-    pmix_globals_init();
+     * opens and initializes the required frameworks */
-    PMIX_CONSTRUCT(&pmix_client_globals.pending_requests, pmix_list_t);
+    if (PMIX_SUCCESS != (rc = pmix_rte_init(PMIX_PROC_TOOL, info, ninfo,
-    PMIX_CONSTRUCT(&pmix_client_globals.myserver, pmix_peer_t);
+                                            pmix_tool_notify_recv))) {
-    /* mark that we are a client */
+        PMIX_ERROR_LOG(rc);
-    pmix_globals.server = false;
+        return rc;
-    /* get our effective id's */
-    pmix_globals.uid = geteuid();
-    pmix_globals.gid = getegid();
-    /* initialize the output system */
-    if (!pmix_output_init()) {
-        return PMIX_ERROR;
     }
 
-    /* see if debug is requested */
+    PMIX_CONSTRUCT(&pmix_client_globals.pending_requests, pmix_list_t);
-    if (NULL != (evar = getenv("PMIX_DEBUG"))) {
+    PMIX_CONSTRUCT(&pmix_client_globals.myserver, pmix_peer_t);
-        debug_level = strtol(evar, NULL, 10);
-        pmix_globals.debug_output = pmix_output_open(NULL);
-        pmix_output_set_verbosity(pmix_globals.debug_output, debug_level);
-    }
 
     pmix_output_verbose(2, pmix_globals.debug_output,
                         "pmix: init called");
@@ -369,24 +357,9 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc,
         }
     }
 
-    pmix_bfrop_open();
-    pmix_usock_init(pmix_tool_notify_recv);
-    pmix_sec_init();
-
-
-    /* tell the event library we need thread support */
-    pmix_event_use_threads();
-
-    if (!pmix_globals.external_evbase) {
-        /* create an event base and progress thread for us */
-        if (NULL == (pmix_globals.evbase = pmix_progress_thread_init(NULL))) {
-            return -1;
-
-        }
-    }
-
     /* connect to the server */
     if (PMIX_SUCCESS != (rc = connect_to_server(&address))) {
+        PMIX_ERROR_LOG(rc);
         return rc;
     }
     /* increment our init reference counter */
@@ -653,24 +626,12 @@ PMIX_EXPORT pmix_status_t PMIx_tool_finalize(void)
     pmix_output_verbose(2, pmix_globals.debug_output,
                         "pmix:tool finalize called");
 
-    if (!pmix_globals.external_evbase) {
+    /* shutdown services */
-        pmix_progress_thread_finalize(NULL);
+    pmix_rte_finalize();
-    }
 
-    pmix_usock_finalize();
     PMIX_DESTRUCT(&pmix_client_globals.myserver);
     PMIX_LIST_DESTRUCT(&pmix_client_globals.pending_requests);
 
-    if (0 <= pmix_client_globals.myserver.sd) {
-        CLOSE_THE_SOCKET(pmix_client_globals.myserver.sd);
-    }
-    event_base_free(pmix_globals.evbase);
-#ifdef HAVE_LIBEVENT_GLOBAL_SHUTDOWN
-    libevent_global_shutdown();
-#endif
-    pmix_bfrop_close();
-    pmix_sec_finalize();
-
     if (NULL != mytmpdir) {
         free(mytmpdir);
     }
@@ -678,9 +639,6 @@ PMIX_EXPORT pmix_status_t PMIx_tool_finalize(void)
         free(systmpdir);
     }
     pmix_globals_finalize();
-
-    pmix_output_close(pmix_globals.debug_output);
-    pmix_output_finalize();
     pmix_class_finalize();
 
     return PMIX_SUCCESS;
@@ -699,6 +657,7 @@ static pmix_status_t send_connect_ack(int sd)
     pmix_usock_hdr_t hdr;
     size_t sdsize=0, csize=0;
     char *cred = NULL;
+    char *sec;
 
     pmix_output_verbose(2, pmix_globals.debug_output,
                         "pmix: TOOL SEND CONNECT ACK");
@@ -710,15 +669,21 @@ static pmix_status_t send_connect_ack(int sd)
 
     /* get a credential, if the security system provides one. Not
      * every SPC will do so, thus we must first check */
-    if (NULL != pmix_sec.create_cred) {
+    if (NULL != pmix_globals.mypeer->compat.psec->create_cred) {
-        if (NULL == (cred = pmix_sec.create_cred())) {
+        if (NULL == (cred = pmix_globals.mypeer->compat.psec->create_cred())) {
             /* an error occurred - we cannot continue */
             return PMIX_ERR_INVALID_CRED;
         }
         csize = strlen(cred) + 1;  // must NULL terminate the string!
     }
+
+    /* add our active bfrops and sec module info, and what type
+     * of buffers we are using */
+   // bfrop = pmix_globals.mypeer->compat.bfrops->name;
+    sec = pmix_globals.mypeer->compat.psec->name;
+
     /* set the number of bytes to be read beyond the header */
-    hdr.nbytes = strlen(PMIX_VERSION) + 1 + csize;  // must NULL terminate the VERSION string!
+    hdr.nbytes = strlen(PMIX_VERSION) + 1 + strlen(sec) + 1 + csize;  // must NULL terminate the VERSION string!
 
     /* create a space for our message */
     sdsize = (sizeof(hdr) + hdr.nbytes);
@@ -737,6 +702,12 @@ static pmix_status_t send_connect_ack(int sd)
     /* load the message */
     memcpy(msg+csize, PMIX_VERSION, strlen(PMIX_VERSION));
     csize += strlen(PMIX_VERSION)+1;
+  //  memcpy(msg+csize, bfrop, strlen(bfrop));
+  //  csize += strlen(bfrop)+1;
+    memcpy(msg+csize, sec, strlen(sec));
+    csize += strlen(sec)+1;
+  //  memcpy(msg+csize, &pmix_globals.mypeer->compat.type, sizeof(pmix_bfrop_buffer_type_t));
+  //  csize += sizeof(pmix_bfrop_buffer_type_t);
     if (NULL != cred) {
         memcpy(msg+csize, cred, strlen(cred));  // leaves last position in msg set to NULL
     }
@@ -791,12 +762,6 @@ static pmix_status_t recv_connect_ack(int sd)
         }
     }
 
-    /* get the returned status from the security handshake */
-    pmix_usock_recv_blocking(sd, (char*)&reply, sizeof(pmix_status_t));
-    if (PMIX_SUCCESS != reply) {
-        return reply;
-    }
-
     /* get the returned status from the request for namespace */
     pmix_usock_recv_blocking(sd, (char*)&reply, sizeof(pmix_status_t));
     if (PMIX_SUCCESS != reply) {
@@ -825,6 +790,23 @@ static pmix_status_t recv_connect_ack(int sd)
                         pmix_client_globals.myserver.info->nptr->nspace,
                         pmix_client_globals.myserver.info->rank);
 
+    /* get the returned status from the security handshake */
+    pmix_usock_recv_blocking(sd, (char*)&reply, sizeof(pmix_status_t));
+    if (PMIX_SUCCESS != reply) {
+        /* see if they want us to do the handshake */
+        if (PMIX_ERR_READY_FOR_HANDSHAKE == reply) {
+            if (NULL == pmix_globals.mypeer->compat.psec->client_handshake) {
+                return PMIX_ERR_HANDSHAKE_FAILED;
+            }
+            if (PMIX_SUCCESS != (reply = pmix_globals.mypeer->compat.psec->client_handshake(sd))) {
+                return reply;
+            }
+            /* if the handshake succeeded, then fall thru to the next step */
+        } else {
+            return reply;
+        }
+    }
+
     if (sockopt) {
         if (0 != setsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, &save, sz)) {
             return PMIX_ERR_UNREACH;
@@ -899,12 +881,14 @@ static pmix_status_t usock_connect(struct sockaddr_un *addr, int *fd)
 
     /* send any authentication credentials to the server */
     if (PMIX_SUCCESS != (rc = send_connect_ack(sd))) {
+        PMIX_ERROR_LOG(rc);
         CLOSE_THE_SOCKET(sd);
         return rc;
     }
 
     /* do whatever handshake is required */
     if (PMIX_SUCCESS != (rc = recv_connect_ack(sd))) {
+        PMIX_ERROR_LOG(rc);
         CLOSE_THE_SOCKET(sd);
         return rc;
     }

opal/mca/pmix/pmix3x/pmix/src/usock/usock.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014-2015 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014-2016 Intel, Inc.  All rights reserved.
  * Copyright (c) 2014      Artem Y. Polyakov <artpol84@gmail.com>.
  *                         All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
@@ -303,12 +303,16 @@ PMIX_CLASS_INSTANCE(pmix_usock_sr_t,
 static void pcon(pmix_peer_t *p)
 {
     p->info = NULL;
+    p->proc_cnt = 0;
+    p->server_object = NULL;
+    p->index = 0;
     p->sd = -1;
     p->send_ev_active = false;
     p->recv_ev_active = false;
     PMIX_CONSTRUCT(&p->send_queue, pmix_list_t);
     p->send_msg = NULL;
     p->recv_msg = NULL;
+    memset(&p->compat, 0, sizeof(p->compat));
 }
 static void pdes(pmix_peer_t *p)
 {

opal/mca/pmix/pmix3x/pmix/src/usock/usock_sendrecv.c

@@ -69,7 +69,7 @@ static void lost_connection(pmix_peer_t *peer, pmix_status_t err)
     }
     CLOSE_THE_SOCKET(peer->sd);
 
-    if (pmix_globals.server) {
+    if (PMIX_PROC_SERVER == pmix_globals.proc_type) {
         /* if I am a server, then we need to ensure that
          * we properly account for the loss of this client
          * from any local collectives in which it was

opal/mca/pmix/pmix3x/pmix/src/util/output.c

@@ -317,7 +317,7 @@ void pmix_output_close(int output_id)
 /*
  * Main function to send output to a stream
  */
-void pmix_output(int output_id, const char *format, ...)
+PMIX_EXPORT void pmix_output(int output_id, const char *format, ...)
 {
     if (output_id >= 0 && output_id < PMIX_OUTPUT_MAX_STREAMS) {
         va_list arglist;
@@ -331,7 +331,7 @@ void pmix_output(int output_id, const char *format, ...)
 /*
  * Send a message to a stream if the verbose level is high enough
  */
-void pmix_output_verbose(int level, int output_id, const char *format, ...)
+ PMIX_EXPORT void pmix_output_verbose(int level, int output_id, const char *format, ...)
 {
     if (output_id >= 0 && output_id < PMIX_OUTPUT_MAX_STREAMS &&
         info[output_id].ldi_verbose_level >= level) {
@@ -456,7 +456,7 @@ void pmix_output_hexdump(int verbose_level, int output_id,
             out_pos += ret;
             for (j = 0; j < 16; j++) {
                 if (i + j < buflen)
-                ret = sprintf(out_buf + out_pos, "%03x ",
+                ret = sprintf(out_buf + out_pos, "%02x ",
                         buf[i + j]);
                 else
                 ret = sprintf(out_buf + out_pos, "   ");

opal/mca/pmix/pmix3x/pmix/test/pmix_test.c

@@ -35,6 +35,9 @@
 
 #include "server_callbacks.h"
 #include "utils.h"
+#include "src/include/pmix_globals.h"
+
+bool spawn_wait = false;
 
 int main(int argc, char **argv)
 {
@@ -175,6 +178,10 @@ int main(int argc, char **argv)
         test_fail = 1;
     }
 
+    if (0 != params.test_spawn) {
+        PMIX_WAIT_FOR_COMPLETION(spawn_wait);
+    }
+
     pmix_argv_free(client_argv);
     pmix_argv_free(client_env);
 

opal/mca/pmix/pmix3x/pmix/test/server_callbacks.c

@@ -13,9 +13,12 @@
  *
  */
 
+#include <pthread.h>
 #include "server_callbacks.h"
 #include "src/util/argv.h"
 
+extern int spawn_wait;
+
 pmix_server_module_t mymodule = {
     .client_connected = connected,
     .client_finalized = finalized,
@@ -263,13 +266,27 @@ typedef struct {
     void *cbdata;
 } release_cbdata;
 
-static void release_cb(pmix_status_t status, void *cbdata)
+
+static void * _release_cb(void *arg)
 {
-    release_cbdata *cb = (release_cbdata*)cbdata;
+    release_cbdata *cb = (release_cbdata*)arg;
     if (NULL != cb->cbfunc) {
         cb->cbfunc(cb->status, "foobar", cb->cbdata);
     }
     free(cb);
+    spawn_wait = false;
+    pthread_exit(NULL);
+}
+
+static void release_cb(pmix_status_t status, void *cbdata)
+{
+    pthread_t thread;
+
+    if (0 > pthread_create(&thread, NULL, _release_cb, cbdata)) {
+        spawn_wait = false;
+        return;
+    }
+    pthread_detach(thread);
 }
 
 pmix_status_t spawn_fn(const pmix_proc_t *proc,
@@ -278,9 +295,12 @@ pmix_status_t spawn_fn(const pmix_proc_t *proc,
              pmix_spawn_cbfunc_t cbfunc, void *cbdata)
 {
     release_cbdata *cb = malloc(sizeof(release_cbdata));
+
     cb->status = PMIX_SUCCESS;
     cb->cbfunc = cbfunc;
     cb->cbdata = cbdata;
+
+    spawn_wait = true;
     PMIx_server_register_nspace("foobar", napps, NULL, 0, release_cb, (void*)cb);
     return PMIX_SUCCESS;
 }

opal/mca/pmix/pmix3x/pmix/test/simple/simptest.c

@@ -269,6 +269,7 @@ int main(int argc, char **argv)
         free(tmp);
     }
     tmp = pmix_argv_join(atmp, ',');
+    pmix_argv_free(atmp);
     x = PMIX_NEW(myxfer_t);
     set_namespace(nprocs, tmp, "foobar", opcbfunc, x);
 
@@ -322,6 +323,9 @@ int main(int argc, char **argv)
             exit(0);
         }
     }
+    free(executable);
+    pmix_argv_free(client_argv);
+    pmix_argv_free(client_env);
 
     /* hang around until the client(s) finalize */
     while (0 < wakeup) {
@@ -330,8 +334,6 @@ int main(int argc, char **argv)
         ts.tv_nsec = 100000;
         nanosleep(&ts, NULL);
     }
-    pmix_argv_free(client_argv);
-    pmix_argv_free(client_env);
 
     /* deregister the errhandler */
     PMIx_Deregister_event_handler(0, NULL, NULL);

opal/mca/pmix/pmix3x/pmix/test/test_fence.c

@@ -313,7 +313,7 @@ int test_fence(test_params params, char *my_nspace, pmix_rank_t my_rank)
                     PMIX_LIST_DESTRUCT(&test_fences);
                     return rc;
                 }
-                GET(uint32_t, fence_num+p->proc.rank, p->proc.nspace, p->proc.rank, fence_num, put_ind++, params.use_same_keys, 0, 0);
+                GET(int, (int)(fence_num+p->proc.rank), p->proc.nspace, p->proc.rank, fence_num, put_ind++, params.use_same_keys, 0, 0);
                 if (PMIX_SUCCESS != rc) {
                     TEST_ERROR(("%s:%d: PMIx_Get failed (%d) from %s:%d", my_nspace, my_rank, rc, p->proc.nspace, p->proc.rank));
                     PMIX_PROC_FREE(pcs, npcs);
@@ -364,7 +364,7 @@ static int get_local_peers(char *my_nspace, int my_rank, pmix_rank_t **_peers, p
     pmix_proc_t proc;
 
     (void)strncpy(proc.nspace, my_nspace, PMIX_MAX_NSLEN);
-    proc.rank = my_rank;
+    proc.rank = PMIX_RANK_WILDCARD;
     /* get number of neighbours on this node */
     if (PMIX_SUCCESS != (rc = PMIx_Get(&proc, PMIX_LOCAL_SIZE, NULL, 0, &val))) {
         TEST_ERROR(("%s:%d: PMIx_Get local peer # failed: %d", my_nspace, my_rank, rc));