Merge pull request #2609 from rhc54/topic/psrv

Bring across some more patches from the debugger work

opal/mca/pmix/ext2x/pmix2x.c

@@ -733,6 +733,9 @@ pmix_persistence_t pmix2x_convert_opalpersist(opal_pmix_persistence_t persist)
 void pmix2x_value_load(pmix_value_t *v,
                        opal_value_t *kv)
 {
+    opal_pmix2x_jobid_trkr_t *job;
+    bool found;
+
     switch(kv->type) {
         case OPAL_UNDEF:
             v->type = PMIX_UNDEF;
@@ -829,7 +832,18 @@ void pmix2x_value_load(pmix_value_t *v,
             v->type = PMIX_PROC;
             /* have to stringify the jobid */
             PMIX_PROC_CREATE(v->data.proc, 1);
-            (void)opal_snprintf_jobid(v->data.proc->nspace, PMIX_MAX_NSLEN, kv->data.name.vpid);
+            /* see if this job is in our list of known nspaces */
+            found = false;
+            OPAL_LIST_FOREACH(job, &mca_pmix_ext2x_component.jobids, opal_pmix2x_jobid_trkr_t) {
+                if (job->jobid == kv->data.name.jobid) {
+                    (void)strncpy(v->data.proc->nspace, job->nspace, PMIX_MAX_NSLEN);
+                    found = true;
+                    break;
+                }
+            }
+            if (!found) {
+                (void)opal_snprintf_jobid(v->data.proc->nspace, PMIX_MAX_NSLEN, kv->data.name.vpid);
+            }
             v->data.proc->rank = pmix2x_convert_opalrank(kv->data.name.vpid);
             break;
         case OPAL_BYTE_OBJECT:
@@ -875,7 +889,8 @@ int pmix2x_value_unload(opal_value_t *kv,
                        const pmix_value_t *v)
 {
     int rc=OPAL_SUCCESS;
-
+    bool found;
+    opal_pmix2x_jobid_trkr_t *job;
 
     switch(v->type) {
     case PMIX_UNDEF:
@@ -969,8 +984,19 @@ int pmix2x_value_unload(opal_value_t *kv,
         break;
     case PMIX_PROC:
         kv->type = OPAL_NAME;
-        if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&kv->data.name.jobid, v->data.proc->nspace))) {
-            return pmix2x_convert_opalrc(rc);
+        /* see if this job is in our list of known nspaces */
+        found = false;
+        OPAL_LIST_FOREACH(job, &mca_pmix_ext2x_component.jobids, opal_pmix2x_jobid_trkr_t) {
+            if (0 == strncmp(job->nspace, v->data.proc->nspace, PMIX_MAX_NSLEN)) {
+                kv->data.name.jobid = job->jobid;
+                found = true;
+                break;
+            }
+        }
+        if (!found) {
+            if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&kv->data.name.jobid, v->data.proc->nspace))) {
+                return pmix2x_convert_opalrc(rc);
+            }
         }
         kv->data.name.vpid = pmix2x_convert_rank(v->data.proc->rank);
         break;
@@ -1330,11 +1356,8 @@ static void pmix2x_log(opal_list_t *info,
     pmix2x_opcaddy_t *cd;
     pmix_status_t prc;
 
-    /* setup the operation */
+    /* create the caddy */
     cd = OBJ_NEW(pmix2x_opcaddy_t);
-    cd->opcbfunc = cbfunc;
-    cd->cbdata = cbdata;
-    cd->ninfo = ninfo;
 
     /* bozo check */
     if (NULL == info || 0 == (ninfo = opal_list_get_size(info))) {
@@ -1342,6 +1365,11 @@ static void pmix2x_log(opal_list_t *info,
         goto CLEANUP;
     }
 
+    /* setup the operation */
+    cd->opcbfunc = cbfunc;
+    cd->cbdata = cbdata;
+    cd->ninfo = ninfo;
+
     /* convert the list to an array of info objects */
     PMIX_INFO_CREATE(cd->info, cd->ninfo);
     n=0;
@@ -1358,6 +1386,7 @@ static void pmix2x_log(opal_list_t *info,
         rc = pmix2x_convert_rc(prc);
         goto CLEANUP;
     }
+
     return;
 
   CLEANUP:

opal/mca/pmix/ext2x/pmix2x_client.c

@@ -892,7 +892,6 @@ int pmix2x_spawn(opal_list_t *job_info, opal_list_t *apps, opal_jobid_t *jobid)
     n=0;
     OPAL_LIST_FOREACH(app, apps, opal_pmix_app_t) {
         papps[n].cmd = strdup(app->cmd);
-        papps[n].argc = app->argc;
         papps[n].argv = opal_argv_copy(app->argv);
         papps[n].env = opal_argv_copy(app->env);
         papps[n].maxprocs = app->maxprocs;
@@ -993,7 +992,6 @@ int pmix2x_spawnnb(opal_list_t *job_info, opal_list_t *apps,
     n=0;
     OPAL_LIST_FOREACH(app, apps, opal_pmix_app_t) {
         op->apps[n].cmd = strdup(app->cmd);
-        op->apps[n].argc = app->argc;
         op->apps[n].argv = opal_argv_copy(app->argv);
         op->apps[n].env = opal_argv_copy(app->env);
         op->apps[n].maxprocs = app->maxprocs;

opal/mca/pmix/ext2x/pmix2x_component.c

@@ -27,8 +27,8 @@
 /*
  * Public string showing the pmix external component version number
  */
-const char *opal_pmix_pmix2x_component_version_string =
-    "OPAL pmix2x MCA component version " OPAL_VERSION;
+const char *opal_pmix_ext2x_component_version_string =
+    "OPAL ext2x MCA component version " OPAL_VERSION;
 
 /*
  * Local function

opal/mca/pmix/ext2x/pmix2x_server_north.c

@@ -588,7 +588,6 @@ static pmix_status_t server_spawn_fn(const pmix_proc_t *p,
         if (NULL != apps[n].cmd) {
             app->cmd = strdup(apps[n].cmd);
         }
-        app->argc = apps[n].argc;
         if (NULL != apps[n].argv) {
             app->argv = opal_argv_copy(apps[n].argv);
         }
@@ -785,7 +784,6 @@ static pmix_status_t server_notify_event(pmix_status_t code,
 
     /* convert the source */
     if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&src.jobid, source->nspace))) {
-        opal_output(0, "FILE: %s LINE %d", __FILE__, __LINE__);
         OBJ_RELEASE(opalcaddy);
         return pmix2x_convert_opalrc(rc);
     }
@@ -882,7 +880,6 @@ static pmix_status_t server_query(pmix_proc_t *proct,
 
     /* convert the requestor */
     if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&requestor.jobid, proct->nspace))) {
-        opal_output(0, "FILE: %s LINE %d", __FILE__, __LINE__);
         OBJ_RELEASE(opalcaddy);
         return pmix2x_convert_opalrc(rc);
     }
@@ -923,13 +920,22 @@ static void toolcbfunc(int status,
     pmix2x_opalcaddy_t *opalcaddy = (pmix2x_opalcaddy_t*)cbdata;
     pmix_status_t rc;
     pmix_proc_t p;
+    opal_pmix2x_jobid_trkr_t *job;
 
     /* convert the status */
     rc = pmix2x_convert_opalrc(status);
 
-    /* convert the process name */
-    (void)opal_snprintf_jobid(p.nspace, PMIX_MAX_NSLEN, proc.jobid);
-    p.rank = pmix2x_convert_opalrank(proc.vpid);
+    memset(&p, 0, sizeof(pmix_proc_t));
+    if (OPAL_SUCCESS == status) {
+        /* convert the process name */
+        (void)opal_snprintf_jobid(p.nspace, PMIX_MAX_NSLEN, proc.jobid);
+        p.rank = pmix2x_convert_opalrank(proc.vpid);
+        /* store this job in our list of known nspaces */
+        job = OBJ_NEW(opal_pmix2x_jobid_trkr_t);
+        (void)strncpy(job->nspace, p.nspace, PMIX_MAX_NSLEN);
+        job->jobid = proc.jobid;
+        opal_list_append(&mca_pmix_ext2x_component.jobids, &job->super);
+    }
 
     /* pass it down */
     if (NULL != opalcaddy->toolcbfunc) {
@@ -997,7 +1003,6 @@ static void server_log(const pmix_proc_t *proct,
 
     /* convert the requestor */
     if (OPAL_SUCCESS != (rc = opal_convert_string_to_jobid(&requestor.jobid, proct->nspace))) {
-    opal_output(0, "FILE: %s LINE %d", __FILE__, __LINE__);
     OBJ_RELEASE(opalcaddy);
     ret = pmix2x_convert_opalrc(rc);
         if (NULL != cbfunc) {

opal/mca/pmix/ext2x/pmix2x_server_south.c

@@ -71,6 +71,28 @@ static void errreg_cbfunc (pmix_status_t status,
     *active = false;
 }
 
+static void opcbfunc(pmix_status_t status, void *cbdata)
+{
+    pmix2x_opcaddy_t *op = (pmix2x_opcaddy_t*)cbdata;
+
+    if (NULL != op->opcbfunc) {
+        op->opcbfunc(pmix2x_convert_rc(status), op->cbdata);
+    }
+    if (op->active) {
+        op->status = status;
+        op->active = false;
+    } else {
+        OBJ_RELEASE(op);
+    }
+}
+
+static void op2cbfunc(pmix_status_t status, void *cbdata)
+{
+    volatile bool *active = (volatile bool*)cbdata;
+
+    *active = false;
+}
+
 int pmix2x_server_init(opal_pmix_server_module_t *module,
                       opal_list_t *info)
 {
@@ -123,6 +145,12 @@ int pmix2x_server_init(opal_pmix_server_module_t *module,
     PMIx_Register_event_handler(NULL, 0, NULL, 0, pmix2x_event_hdlr, errreg_cbfunc, (void*)&active);
     PMIX_WAIT_FOR_COMPLETION(active);
 
+    /* as we might want to use some client-side functions, be sure
+     * to register our own nspace */
+    active = true;
+    PMIx_server_register_nspace(job->nspace, 1, NULL, 0, op2cbfunc, (void*)&active);
+    PMIX_WAIT_FOR_COMPLETION(active);
+
     return OPAL_SUCCESS;
 }
 
@@ -163,21 +191,6 @@ int pmix2x_server_gen_ppn(const char *input, char **ppn)
     return pmix2x_convert_rc(rc);
 }
 
-static void opcbfunc(pmix_status_t status, void *cbdata)
-{
-    pmix2x_opcaddy_t *op = (pmix2x_opcaddy_t*)cbdata;
-
-    if (NULL != op->opcbfunc) {
-        op->opcbfunc(pmix2x_convert_rc(status), op->cbdata);
-    }
-    if (op->active) {
-        op->status = status;
-        op->active = false;
-    } else {
-        OBJ_RELEASE(op);
-    }
-}
-
 static void _reg_nspace(int sd, short args, void *cbdata)
 {
     pmix2x_threadshift_t *cd = (pmix2x_threadshift_t*)cbdata;
@@ -486,6 +499,7 @@ int pmix2x_server_notify_event(int status,
         OPAL_LIST_FOREACH(kv, info, opal_value_t) {
             (void)strncpy(pinfo[n].key, kv->key, PMIX_MAX_KEYLEN);
             pmix2x_value_load(&pinfo[n].value, kv);
+            ++n;
         }
     } else {
         sz = 0;

opal/mca/pmix/pmix2x/pmix/Makefile.am

@@ -23,7 +23,7 @@
 # via AC_CONFIG_MACRO_DIR in configure.ac.
 ACLOCAL_AMFLAGS = -I ./config
 
-SUBDIRS = config include src
+SUBDIRS = config contrib include src
 
 
 headers =

opal/mca/pmix/pmix2x/pmix/VERSION

@@ -30,7 +30,7 @@ greek=
 # command, or with the date (if "git describe" fails) in the form of
 # "date<date>".
 
-repo_rev=git9089b99
+repo_rev=gitb9778a7
 
 # If tarball_version is not empty, it is used as the version string in
 # the tarball filename, regardless of all other versions listed in
@@ -44,7 +44,7 @@ tarball_version=
 
 # The date when this release was created
 
-date="Dec 13, 2016"
+date="Dec 19, 2016"
 
 # The shared library version of each of PMIx's public libraries.
 # These versions are maintained in accordance with the "Library

opal/mca/pmix/pmix2x/pmix/configure.ac

@@ -218,7 +218,8 @@ AC_SUBST([CONFIGURE_DEPENDENCIES], ['$(top_srcdir)/VERSION'])
 . $srcdir/VERSION
 AC_SUBST([libpmix_so_version])
 
-AC_CONFIG_FILES(pmix_config_prefix[examples/Makefile]
+AC_CONFIG_FILES(pmix_config_prefix[contrib/Makefile]
+                pmix_config_prefix[examples/Makefile]
                 pmix_config_prefix[man/Makefile]
                 pmix_config_prefix[test/Makefile]
                 pmix_config_prefix[test/simple/Makefile])

opal/mca/pmix/pmix2x/pmix/contrib/Makefile.am

@@ -0,0 +1,39 @@
+#
+# Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2009 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2010      IBM Corporation.  All rights reserved.
+# Copyright (c) 2010-2011 Oak Ridge National Labs.  All rights reserved.
+# Copyright (c) 2013-2016 Los Alamos National Security, Inc.  All rights
+#                         reserved.
+# Copyright (c) 2013-2016 Intel, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+EXTRA_DIST = \
+	make_dist_tarball \
+	buildrpm.sh \
+	cron-run-all-md2nroff.pl \
+        md2nroff.pl \
+	platform/optimized \
+        pmix_jenkins.sh \
+        pmix-release.sh \
+        pmix.spec \
+        update-my-copyright.pl \
+        whitespace-purge.sh
+
+include perf_tools/Makefile.include
+
+dist_pmixdata_DATA = pmix-valgrind.supp

opal/mca/pmix/pmix2x/pmix/contrib/buildrpm.sh

@@ -0,0 +1,288 @@
+#!/bin/sh -f
+#
+# Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2015      Intel, Inc. All rights reserved.
+#
+
+#
+# General config vars
+# The following vars can be set from outside and will affect script behave:
+# prefix,rpmbuild_options,configure_options,build_srpm,build_single,build_multiple,rpmtopdir
+#
+
+
+specfile="pmix.spec"
+prefix=${prefix:-"/opt/pmix"}
+rpmbuild_options=${rpmbuild_options:-"--define 'mflags -j4' --define '_source_filedigest_algorithm md5'  --define '_binary_filedigest_algorithm md5'"}
+configure_options=${configure_options:-""}
+
+# Helpful when debugging
+#rpmbuild_options="--define 'mflags -j4' --define 'install_in_opt 1' --define 'cflags -g' --define 'install_modulefile 1' --define 'modules_rpm_name dhcp'"
+#configure_options="--disable-mpi-f77 --without-io-romio --disable-mpi-f90"
+
+# Some distro's will attempt to force using bizarre, custom compiler
+# names (e.g., i386-redhat-linux-gnu-gcc).  So hardwire them to use
+# "normal" names.
+#export CC=gcc
+#export CXX=g++
+#export F77=f77
+#export FC=
+
+# Note that this script can build one or all of the following RPMs:
+# SRPM, all-in-one, multiple.
+
+# If you want to build the SRPM, put "yes" here
+build_srpm=${build_srpm:-"yes"}
+# If you want to build the "all in one RPM", put "yes" here
+build_single=${build_single:-"no"}
+# If you want to build the "multiple" RPMs, put "yes" here
+build_multiple=${build_multiple:-"no"}
+
+#########################################################################
+# You should not need to change anything below this line
+#########################################################################
+
+#
+# get the tarball name
+#
+
+tarball="$1"
+if test "$tarball" = ""; then
+    echo "Usage: buildrpm.sh <tarball>"
+    exit 1
+fi
+if test ! -f $tarball; then
+    echo "Can't find $tarball"
+    exit 1
+fi
+echo "--> Found tarball: $tarball"
+
+#
+# get the extension from the tarball (gz or bz2)
+#
+
+extension=`echo $tarball | egrep '\.bz2'`
+if test -n "$extension"; then
+    extension=bz2
+else
+    extension=gz
+fi
+
+#
+# Get the version number
+#
+
+first="`basename $tarball | cut -d- -f2`"
+version="`echo $first | sed -e 's/\.tar\.'$extension'//'`"
+unset first
+echo "--> Found PMIx version: $version"
+
+#
+# do we have the spec files?
+#
+
+if test ! -f $specfile; then
+    echo "can't find $specfile"
+    exit 1
+fi
+echo "--> Found specfile: $specfile"
+
+#
+# Find where the top RPM-building directory is
+#
+
+rpmtopdir=${rpmtopdir:-"`grep %_topdir $HOME/.rpmmacros | awk '{ print $2 }'`"}
+if test "$rpmtopdir" != ""; then
+        rpmbuild_options="$rpmbuild_options --define '_topdir $rpmtopdir'"
+    if test ! -d "$rpmtopdir"; then
+        mkdir -p "$rpmtopdir"
+        mkdir -p "$rpmtopdir/BUILD"
+        mkdir -p "$rpmtopdir/RPMS"
+        mkdir -p "$rpmtopdir/RPMS/i386"
+        mkdir -p "$rpmtopdir/RPMS/i586"
+        mkdir -p "$rpmtopdir/RPMS/i686"
+        mkdir -p "$rpmtopdir/RPMS/noarch"
+        mkdir -p "$rpmtopdir/RPMS/athlon"
+        mkdir -p "$rpmtopdir/SOURCES"
+        mkdir -p "$rpmtopdir/SPECS"
+        mkdir -p "$rpmtopdir/SRPMS"
+    fi
+    need_root=0
+elif test -d /usr/src/RPM; then
+    need_root=1
+    rpmtopdir="/usr/src/RPM"
+elif test -d /usr/src/packages; then
+    need_root=1
+    rpmtopdir="/usr/src/packages"
+else
+    need_root=1
+    rpmtopdir="/usr/src/redhat"
+fi
+echo "--> Found RPM top dir: $rpmtopdir"
+
+#
+# If we're not root, try to sudo
+#
+
+if test "$need_root" = "1" -a "`whoami`" != "root"; then
+    echo "--> Trying to sudo: \"$0 $*\""
+    echo "------------------------------------------------------------"
+    sudo -u root sh -c "$0 $tarball"
+    echo "------------------------------------------------------------"
+    echo "--> sudo finished"
+    exit 0
+fi
+
+#
+# make sure we have write access to the directories we need
+#
+
+if test ! -w $rpmtopdir/SOURCES ; then
+    echo "Problem creating rpms: You do not have a $rpmtopdir directory"
+    echo "tree or you do not have write access to the $rpmtopdir directory"
+    echo "tree.  Please remedy and try again."
+    exit 1
+fi
+echo "--> Have write access to $rpmtopdir/SOURCES"
+
+#
+# move the tarball file to the rpm directory
+#
+
+cp $tarball $rpmtopdir/SOURCES
+
+#
+# Print out the compilers
+#
+
+cat <<EOF
+--> Hard-wired for compilers:
+    CC = $CC
+    CXX = $CXX
+    F77 = $F77
+    FC = $FC
+EOF
+
+#
+# what command should we use?
+# RH 8.0 changed from using "rpm -ba" to "rpmbuild -ba".  ARRGGH!!!
+#
+
+which rpmbuild 2>&1 >/dev/null
+if test "$?" = "0"; then
+    rpm_cmd="rpmbuild"
+else
+    rpm_cmd="rpm"
+fi
+
+
+#
+# from the specfile
+#
+
+specdest="$rpmtopdir/SPECS/pmix-$version.spec"
+sed -e 's/\$VERSION/'$version'/g' \
+    -e 's/\$EXTENSION/'$extension'/g' \
+    $specfile > "$specdest"
+echo "--> Created destination specfile: $specdest"
+release=`egrep -i release: $specdest | cut -d\  -f2`
+
+#
+# Setup compiler string
+#
+
+if test "$CC" != ""; then
+    configure_options="$configure_options CC=$CC"
+fi
+if test "$CXX" != ""; then
+    configure_options="$configure_options CXX=$CXX"
+fi
+if test "$F77" != ""; then
+    configure_options="$configure_options F77=$F77"
+fi
+if test "$FC" != ""; then
+    configure_options="$configure_options FC=$FC"
+fi
+
+#
+# Make the SRPM
+#
+
+if test "$build_srpm" = "yes"; then
+    echo "--> Building the PMIx SRPM"
+    rpmbuild_options="$rpmbuild_options --define 'dist %{nil}'"
+    cmd="$rpm_cmd $rpmbuild_options -bs $specdest"
+    echo "--> $cmd"
+    eval $cmd
+
+    if test $? != 0; then
+        echo "*** FAILURE BUILDING SRPM!"
+        echo "Aborting"
+        exit 1
+    fi
+    echo "--> Done building the SRPM"
+fi
+
+#
+# Make the single RPM
+#
+
+if test "$build_single" = "yes"; then
+    echo "--> Building the single PMIx RPM"
+    cmd="$rpm_cmd -bb $rpmbuild_options --define 'build_all_in_one_rpm 1'"
+    if test "$configure_options" != ""; then
+        cmd="$cmd --define 'configure_options $configure_options'"
+    fi
+    cmd="$cmd $specdest"
+    echo "--> $cmd"
+    eval $cmd
+
+    if test $? != 0; then
+        echo "*** FAILURE BUILDING SINGLE RPM!"
+        echo "Aborting"
+        exit 1
+    fi
+    echo "--> Done building the single RPM"
+fi
+
+#
+# Make the multi RPM
+#
+
+if test "$build_multiple" = "yes"; then
+    echo "--> Building the multiple PMIx RPM"
+    cmd="$rpm_cmd -bb $rpmbuild_options --define 'build_all_in_one_rpm 0'"
+    if test "$configure_options" != ""; then
+        cmd="$cmd --define 'configure_options $configure_options'"
+    fi
+    cmd="$cmd $specdest"
+    echo "--> $cmd"
+    eval $cmd
+
+    if test $? != 0; then
+        echo "*** FAILURE BUILDING MULTIPLE RPM!"
+        echo "Aborting"
+        exit 1
+    fi
+    echo "--> Done building the multiple RPM"
+fi
+
+#
+# Done
+#
+
+cat <<EOF
+
+------------------------------------------------------------------------------
+====                FINISHED BUILDING PMIx RPM                        ====
+------------------------------------------------------------------------------
+A copy of the tarball is located in: $rpmtopdir/SOURCES/
+The completed rpms are located in:   $rpmtopdir/RPMS/i<something>86/
+The sources rpms are located in:     $rpmtopdir/SRPMS/
+The spec files are located in:       $rpmtopdir/SPECS/
+------------------------------------------------------------------------------
+
+EOF

opal/mca/pmix/pmix2x/pmix/contrib/cron-run-all-md2nroff.pl

@@ -0,0 +1,187 @@
+#!/usr/bin/env perl
+
+# Script to pull down the latest markdown man pages from the PMIx
+# git repo.  Iterate over them, converting each to an nroff man page
+# and also copying+committing them to the gh-pages branch.  Finally,
+# git push them back upstream (so that Github will render + serve them
+# up as web pages).
+
+use strict;
+use warnings;
+
+use POSIX;
+use File::Basename;
+use Getopt::Long;
+use File::Temp;
+
+my $repo_arg;
+my $source_branch_arg = "master";
+my $pages_branch_arg = "gh-pages";
+my $logfile_dir_arg = "/tmp";
+my $verbose_arg;
+my $help_arg;
+
+my $ok = Getopt::Long::GetOptions("repo=s" => \$repo_arg,
+                                  "source-branch=s" => \$source_branch_arg,
+                                  "pages-branch=s" => \$pages_branch_arg,
+                                  "logfile-dir=s" => \$logfile_dir_arg,
+                                  "help|h" => \$help_arg,
+                                  "verbose" => \$verbose_arg,
+                                  );
+
+if (!$ok || $help_arg) {
+    print "Invalid command line argument.\n\n"
+        if (!$ok);
+    print "Options:
+  --help | -h      Print this message
+  --repo           Git repo to be updated
+  --source-branch  Branch containing source files (default: master)
+  --pages-branch   Branch where man pages are to be output (default: gh-pages)
+  --logfile-dir    Directory where execution log is to be written (default: /tmp)
+  --verbose        Print debug info during execution\n";
+    exit($ok ? 0 : 1);
+}
+
+# Sanity checks
+die "Must specify a git repo"
+    if (!defined($repo_arg));
+
+#####################################################################
+
+my $logfile_dir = $logfile_dir_arg;
+my $logfile_counter = 1;
+
+sub doit {
+    my $allowed_to_fail = shift;
+    my $cmd = shift;
+    my $stdout_file = shift;
+
+    # Put a prefix on the logfiles so that we know that they belong to
+    # this script, and put a counter so that we know the sequence of
+    # logfiles
+    $stdout_file = "runall-md2nroff-$logfile_counter-$stdout_file";
+    ++$logfile_counter;
+
+    # Redirect stdout if requested
+    if (defined $stdout_file) {
+        $stdout_file = "$logfile_dir/$stdout_file.log";
+        unlink($stdout_file);
+        $cmd .= " >$stdout_file";
+    } elsif (!$verbose_arg && $cmd !~ />/) {
+        $cmd .= " >/dev/null";
+    }
+    $cmd .= " 2>&1";
+
+    my $rc = system($cmd);
+    if (0 != $rc && !$allowed_to_fail) {
+        # If we die/fail, ensure to change out of the temp tree so
+        # that it can be removed upon exit.
+        chdir("/");
+        die "Command $cmd failed: exit status $rc";
+    }
+
+    system("cat $stdout_file")
+        if ($verbose_arg && defined($stdout_file) && -f $stdout_file);
+}
+
+sub verbose {
+    print @_
+        if ($verbose_arg);
+}
+
+#####################################################################
+
+# Setup a logfile dir just for this run
+my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) =
+    localtime(time);
+$logfile_dir =
+    sprintf("%s/cron-run-all-md2nroff-logs-%04d-%02d-%02d-%02d%02d",
+            $logfile_dir_arg, $year + 1900, $mon + 1, $mday,
+            $hour, $min);
+my $rc = system("mkdir $logfile_dir");
+if ($rc != 0 || ! -d $logfile_dir || ! -w $logfile_dir) {
+    chdir("/");
+    die "mkdir of $logfile_dir failed, or can't write to it";
+}
+
+# First, git clone the source branch of the repo
+verbose("*** Cloning repo: $repo_arg / $source_branch_arg...\n");
+my $tmpdir = File::Temp->newdir();
+
+chdir($tmpdir);
+doit(0, "git clone --single-branch --branch $source_branch_arg $repo_arg source", "git-clone");
+
+# Next, git clone the pages branch of repo
+if (defined($pages_branch_arg)) {
+    verbose("*** Cloning repo: $repo_arg / $pages_branch_arg...\n");
+    doit(0, "git clone --single-branch --branch $pages_branch_arg $repo_arg pages", "git-clone2");
+}
+
+#####################################################################
+
+# Find all the *.\d.md files in the source repo
+verbose("*** Finding markdown man pages...\n");
+opendir(DIR, "source/man");
+my @markdown_files = grep { /\.\d\.md$/ && -f "source/man/$_" } readdir(DIR);
+closedir(DIR);
+verbose("Found: @markdown_files\n");
+
+#####################################################################
+
+# Copy each of the markdown files to the pages branch checkout
+if (defined($pages_branch_arg)) {
+    chdir("pages/master");
+    foreach my $file (@markdown_files) {
+        doit(0, "cp ../../source/man/$file man/$file", "loop-cp");
+
+        # Is there a new man page?  If so, we need to "git add" it.
+        my $out = `git status --porcelain man/$file`;
+        doit(0, "git add man/$file", "loop-git-add")
+            if ($out =~ /^\?\?/);
+    }
+
+    # Git commit those files in the pages repo and push them to the
+    # upstream repo so that they go live.  If nothing changed, the commit
+    # and push will be no-ops.
+    chdir("..");
+    doit(1, "git commit --no-verify -a -m \"Updated Markdown man pages from $source_branch_arg\"",
+         "git-commit-first");
+    doit(1, "git push", "git-push-first");
+}
+
+#####################################################################
+
+# Now process each of the Markdown files in the source repo and
+# generate new nroff man pages.
+chdir("$tmpdir/source/man");
+foreach my $file (@markdown_files) {
+    doit(0, "../contrib/md2nroff.pl --source $file", "loop2-md2nroff");
+
+    # Did we generate a new man page?  If so, we need to "git add" it.
+    my $man_file = basename($file);
+
+    $man_file =~ m/\.(\d)\.md$/;
+    my $section = $1;
+
+    $man_file =~ s/\.md$//;
+
+    my $full_filename = "man$section/$man_file";
+
+    my $out = `git status --porcelain $full_filename`;
+    doit(0, "git add $full_filename", "loop2-git-add")
+        if ($out =~ /^\?\?/);
+}
+
+# Similar to above: commit the newly-generated nroff pages and push
+# them back upstream.  If nothing changed, these will be no-ops.
+doit(1, "git commit --no-verify -a -m \"Updated nroff-generated man pages\"", "git-commit-final");
+doit(1, "git push", "git-push-final");
+
+# chdir out of the tmpdir so that it can be removed
+chdir("/");
+
+# If we get here, we finished successfully, so there's no need to keep
+# the logfile dir around
+system("rm -rf $logfile_dir");
+
+exit(0);

opal/mca/pmix/pmix2x/pmix/contrib/make_dist_tarball

@@ -0,0 +1,428 @@
+#!/bin/sh
+#
+# Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2008-2013 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2015-2016 Intel, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+#
+# Version of auto tools that we want
+#
+
+M4_TARGET_VERSION=1.4.17
+AM_TARGET_VERSION=1.15
+AC_TARGET_VERSION=2.69
+LT_TARGET_VERSION=2.4.6
+
+#
+# When running "make distcheck", use these parallelization flags.  Can
+# significantly decrease the time required for "make distcheck" because
+# that target includes multiple builds of the entire code base.
+#
+
+DISTCHECK_MAKE_FLAGS=-j4
+
+#########################################################################
+
+#
+# Check command line flags
+#
+
+# Default to requiring *exact* versions if we're making distribution
+# tarballs; but higher-than-expected versions are ok for
+# non-distribution tarballs.
+dirty_ok=0
+gnu_version_ignore=0
+mac=0
+dist_target=distcheck
+distcheck_flags="AM_MAKEFLAGS=$DISTCHECK_MAKE_FLAGS"
+if test "`basename $0`" = "make_tarball"; then
+    dist_target=dist
+    distcheck_flags=
+    highok=1
+    dirty_ok=1
+else
+    highok=0
+fi
+
+greekonly=0
+autogen_args=
+config_args=
+distdir=".."
+for i in "$@"; do
+    case $i in
+        -greekonly) greekonly=1 ;;
+        --greekonly) greekonly=1 ;;
+        -highok) highok=1 ;;
+        --highok) highok=1 ;;
+        -autogen-args=*) autogen_args="${i#*=}"; shift ;;
+        --autogen-args=*) autogen_args="${i#*=}"; shift ;;
+        -config-args=*) config_args="${i#*=}"; shift ;;
+        --config-args=*) config_args="${i#*=}"; shift ;;
+        -distdir=*) distdir="${i#*=}"; shift ;;
+        --distdir=*) distdir="${i#*=}"; shift ;;
+        --dirtyok) dirty_ok=1; shift ;;
+        --verok) gnu_version_ignore=1 ;;
+        -verok) gnu_version_ignore=1 ;;
+        --macosx) mac=1 ;;
+        -macosx) mac=1 ;;
+        *)
+            cat <<EOF
+Unrecognized argument: $1
+
+Valid arguments:
+  --greekonly     Only build the greek tarball (vs. both tarballs)
+  --highok        Ok if Autotools versions are too high
+  --autogen-args  Arguments to pass to autogen
+  --config-args   Arguments to pass to configure (e.g., --with-libevent=<foo>)
+  --distdir       Move the tarball(s) to this directory when done
+  --dirtyok       Ok if the source tree is dirty
+  --verok         Ignore result of autotools version checking
+  --macosx        Executing on Mac OSX, so do not check libtool version
+EOF
+            exit 1
+            ;;
+    esac
+    shift
+done
+
+# pmix needs libevent to build.  If $LIBEVENT is set in the
+# environment, automatically add it to the config_args.
+if test "$LIBEVENT" != ""; then
+    echo "*** Found \$LIBEVENT: $LIBEVENT"
+    echo "*** Adding --with-libevent=$LIBEVENT to config_args"
+    config_args="--with-libevent=$LIBEVENT $config_args"
+fi
+
+# if config_args isn't empty, then add that to the distcheck_flags
+# (because we'll assumedly need those to run configure under "make
+# distcheck").
+if test "$config_args" != ""; then
+    echo "*** Adding to distcheck_flags: $config_args"
+    distcheck_flags="$distcheck_flags AM_DISTCHECK_CONFIGURE_FLAGS=\"$config_args\""
+fi
+
+export DISTCHECK_CONFIGURE_FLAGS=$config_args
+
+#
+# First things first -- check that the auto versions that we have are
+# the ones that we want.
+#
+
+check_gnu_version() {
+    prog="$1"
+    target="$2"
+
+    ver="`$prog --version | head -n 1 | sed -e's/([^)]*)//g' -e's/[^0-9 .][^ ]* / /g' -e's/ //g'`"
+    echo $prog version is $ver
+
+    ver_major=`echo $ver | cut -d. -f1`
+    ver_minor=`echo $ver | cut -d. -f2`
+    ver_release=`echo $ver | cut -d. -f3`
+    if test "$ver_release" = ""; then
+        ver_release=0
+    fi
+
+    target_major=`echo $target | cut -d. -f1`
+    target_minor=`echo $target | cut -d. -f2`
+    target_release=`echo $target | cut -d. -f3`
+    if test "$target_release" = ""; then
+        target_release=0
+    fi
+
+    # Gah -- Libtool released version 2.2.6b, the "b" of which totally
+    # screws up the -lt and -gt comparisons, below.  So strip out any
+    # trailing letters in the target_release and ver_release variables
+    # -- if they don't match, we'll just get a "they don't match
+    # somehow" kind of message (because I'm not going to code up a
+    # complex/clever alphanumeric lower/higher comparison thingy).
+    # Sigh.
+    ver_release=`echo $ver_release | sed 's/[A-Za-z]//g'`
+    target_release=`echo $target_release | sed 's/[A-Za-z]//g'`
+
+    result=same
+    if test "$ver" != "$target"; then
+        if test "$ver_major" -lt "$target_major"; then
+            result=low
+        elif test "$ver_major" = "$target_major" -a "$ver_minor" -lt "$target_minor"; then
+            result=low
+        elif test "$ver_major" = "$target_major" -a "$ver_minor" = "$target_minor" -a "$ver_release" -lt "$target_release"; then
+            result=low
+        elif test "$ver_major" -gt "$target_major"; then
+            result=high
+        elif test "$ver_major" = "$target_major" -a "$ver_minor" -gt "$target_minor"; then
+            result=high
+        elif test "$ver_major" = "$target_major" -a "$ver_minor" = "$target_minor" -a "$ver_release" = "$target_release"; then
+            result=same
+        elif test "$ver_major" = "$target_major" -a "$ver_minor" = "$target_minor" -a "$ver_release" -gt "$target_release"; then
+            result=high
+        elif test "$ver_major" = "$target_major" -a "$ver_minor" = "$target_minor" -a "$ver_release" -lt "$target_release"; then
+            result=low
+        else
+            result=unknown
+        fi
+    fi
+
+    if test "$result" = "low"; then
+        cat <<EOF
+----------------------------------------------------------------------
+ERROR: Program "$prog" does not have a high enough version:
+       Found: $ver
+       Expected: $target
+
+Expected versions:
+m4: $M4_TARGET_VERSION
+Automake: $AM_TARGET_VERSION
+Autoconf: $AC_TARGET_VERSION
+Libtool:  $LT_TARGET_VERSION
+
+Either change this script to match the found version, or install
+the correct version of the tools.
+----------------------------------------------------------------------
+EOF
+        if test "$gnu_version_ignore" = "0"; then
+            exit 1
+        fi
+
+    elif test "$result" = "high"; then
+        if test "$highok" = "0"; then
+            cat <<EOF
+----------------------------------------------------------------------
+ERROR: Program "$prog" has a higher version than expected:
+       Found: $ver
+       Expected: $target
+
+Expected versions:
+m4: $M4_TARGET_VERSION
+Automake: $AM_TARGET_VERSION
+Autoconf: $AC_TARGET_VERSION
+Libtool:  $LT_TARGET_VERSION
+
+Either change this script to match the found version, or install
+the correct version of the tools.
+----------------------------------------------------------------------
+EOF
+        if test "$gnu_version_ignore" = "0"; then
+            exit 1
+        fi
+        else
+            cat <<EOF
+----------------------------------------------------------------------
+WARNING: Program "$prog" has a higher version than expected:
+         Found: $ver
+         Expected: $target
+
+Expected versions:
+m4: $M4_TARGET_VERSION
+Automake: $AM_TARGET_VERSION
+Autoconf: $AC_TARGET_VERSION
+Libtool:  $LT_TARGET_VERSION
+
+----------------------------------------------------------------------
+EOF
+        fi
+
+    elif test "$result" = "unknown"; then
+        cat <<EOF
+----------------------------------------------------------------------
+ERROR: Program "$prog" does not have the correct version:
+       Found: $ver
+       Expected: $target
+
+Expected versions:
+m4: $M4_TARGET_VERSION
+Automake: $AM_TARGET_VERSION
+Autoconf: $AC_TARGET_VERSION
+Libtool:  $LT_TARGET_VERSION
+
+Either change this script to match the found version, or install
+the correct version of the tools.
+----------------------------------------------------------------------
+EOF
+        if test "$gnu_version_ignore" = "0"; then
+            exit 1
+        fi
+    fi
+
+}
+
+#
+# Subroutine to actually make a tarball
+#
+
+make_tarball() {
+    #
+    # Autogen
+    #
+    echo "*** Running autogen $autogen_args..."
+    rm -f success
+    (./autogen.sh $autogen_args 2>&1 && touch success) | tee auto.out
+    if test ! -f success; then
+        echo "Autogen failed.  Aborting"
+        exit 1
+    fi
+
+    #
+    # Configure
+    #
+    echo "*** Running configure..."
+    rm -f success
+    (./configure $config_args 2>&1 && touch success) | tee config.out
+    if test ! -f success; then
+        echo "Configure failed.  Aborting"
+        exit 1
+    fi
+
+    #
+    #
+    # make tarball
+    #
+    echo "*** Running make $dist_target..."
+    save_LD=$LD_LIBRARY_PATH
+    LD_LIBRARY_PATH=
+    rm -f success
+    (eval make $distcheck_flags $dist_target 2>&1 && touch success) | tee dist.out
+    if test ! -f success; then
+        echo "Make $dist_target failed.  Aborting"
+        exit 1
+    fi
+    rm -f success
+    LD_LIBRARY_PATH=$save_LD
+
+    #
+    # move
+    #
+    echo "*** Moving tarballs..."
+    mv pmix-* $distdir
+
+    echo "*** All done"
+}
+
+#########################################################################
+# main
+#########################################################################
+
+start=`date`
+echo "*** Start time: $start"
+
+echo "*** Checking tools versions..."
+check_gnu_version m4 $M4_TARGET_VERSION
+check_gnu_version automake $AM_TARGET_VERSION
+check_gnu_version autoconf $AC_TARGET_VERSION
+if test $mac -eq 0; then
+    check_gnu_version libtool $LT_TARGET_VERSION
+fi
+#
+# Verify that we're in a top PMIx dir
+#
+echo "*** Checking to ensure in top-level PMIx directory..."
+if test -f VERSION -a -f configure.ac -a -f config/pmix.m4 ; then
+    happy=1
+else
+    echo "Do not appear to be in an PMIx top directory.  Abort!"
+    exit 1
+fi
+
+echo "*** Removing old VERSION file..."
+rm -f VERSION
+echo "*** Restoring pristine VERSION file..."
+git checkout VERSION
+echo "*** Getting git version number..."
+repo_rev=git`git log -1 --oneline | head -1 | cut -d' ' -f1`
+echo "    Repo rev number: $repo_rev"
+
+# Sanity checks
+if test "$repo_rev" = ""; then
+    echo "Somehow the repo rev number is empty.  Abort!"
+    exit 1
+elif test "`echo $repo_rev | grep ' '`" != ""; then
+    echo "Somehow the repo rev number has a space in it -- bad!"
+    exit 1
+fi
+
+# Ensure we have a clean repo
+if test $dirty_ok -eq 0; then
+    echo "*** Checking if source tree is dirty..."
+    dirty=0
+    if test "`git status | grep 'Changes not staged for commit'`" != ""; then
+        dirty=1
+    fi
+
+    if test $dirty -eq 1; then
+        echo "    Source tree is dirty.  Cannot continue."
+        exit 1
+    fi
+fi
+
+#
+# Ditch repo rev from all version numbers
+#
+echo "*** Removing git version numbers from VERSION..."
+version_files=VERSION
+release_date=`date '+%b %d, %Y'`
+echo "    Release date: $release_date"
+for file in $version_files; do
+    echo " - $file"
+    sed -e 's/^want_repo_rev=.*/want_repo_rev=0/' \
+        -e 's/^repo_rev=.*/'repo_rev=$repo_rev/ \
+        -e "s/^date=.*/date=\"$release_date\"/" \
+        $file > $file.new
+    cp -f $file.new $file
+    rm $file.new
+done
+
+#
+# Make 2 tarballs:
+#
+# - one with the greek
+# - one without the greek
+#
+# unless the user specifically said --greekonly, then only make the
+# greek tarball.  Making both tarballs at once allows us to guarantee
+# to have two tarballs -- one greek and one not -- that have exactly
+# the same GIT r number (as opposed to, for example, running this
+# script to make a greek tarball, then running it again to make a
+# non-greek tarball -- there is a race condition that someone could
+# commit in the meantime and change the GIT r number in the 2nd
+# tarball)
+#
+
+# First, make greek tarball
+
+echo "*** Making greek tarball"
+make_tarball
+
+# Now if ! --greekonly, make the non-greek tarball
+
+if test "$greekonly" = "0"; then
+    echo "*** REMOVING ALL GREEK FROM VERSION NUMBERS!!"
+    for file in $version_files; do
+        echo " - $file"
+        sed -e 's/^greek=.*/greek=/' $file > $file.new
+        cp -f $file.new $file
+        rm $file.new
+    done
+    echo "Making non-greek tarball"
+    make_tarball
+fi
+
+# Put the VERSION file back the way it was
+git checkout VERSION
+
+echo " "
+echo "*** Start time:  $start"
+echo "*** Finish time: `date`"

opal/mca/pmix/pmix2x/pmix/contrib/md2nroff.pl

@@ -0,0 +1,164 @@
+#!/usr/bin/env perl
+
+# Script to convert markdown to nroff man pages.
+#
+# The main conversion work is done via pandoc.  But pandoc doesn't do
+# everything exactly the way we want it, so use some perl regular
+# expressions to fix up what pandoc doesn't get right.
+#
+# Do a "smart" write of the resulting output man page -- only write to
+# the output file if the contents have actually changed compared to
+# what was already there.
+
+use strict;
+use warnings;
+
+use POSIX;
+use File::Basename;
+use Getopt::Long;
+use File::Temp qw/tempfile/;
+
+my $source_arg;
+my $target_arg;
+my $help_arg;
+
+my $ok = Getopt::Long::GetOptions("source=s" => \$source_arg,
+                                  "target=s" => \$target_arg,
+                                  "help|h" => \$help_arg,
+                                  );
+
+if ($help_arg) {
+    print "$0 --source input_MD_file --target output_nroff_file\n";
+    exit(0);
+}
+
+# Sanity checks
+die "Must specify a source file"
+    if (!defined($source_arg));
+die "Source file does not exist ($source_arg)"
+    if (! -r $source_arg);
+
+my $pandoc = `which pandoc`;
+die "Cannot find pandoc executable"
+    if ($pandoc eq "");
+
+#####################################################################
+
+my $file = $source_arg;
+$file =~ m/(\d+).md/;
+my $section = $1;
+die "Could not figure out the man page section: $source_arg"
+    if (!defined($section));
+my $shortfile = basename($file);
+$shortfile =~ s/\.$section\.md$//;
+
+# If the target file was not specified, derive it from the source file
+my $target;
+if (!defined($target_arg)) {
+    $target_arg = $source_arg;
+
+    $target_arg =~ m/\.(\d)\.md$/;
+    my $section = $1;
+
+    my $dirname = dirname($target_arg);
+    my $basename = basename($target_arg);
+    $basename =~ s/\.md$//;
+
+    $target = "$dirname/man$section/$basename";
+} else {
+    $target = $target_arg;
+}
+
+print "*** Processing: $file -> $target\n";
+
+# Read in the file
+my $pandoc_input;
+open(IN, $file)
+    || die "Can't open $file";
+$pandoc_input .= $_
+    while (<IN>);
+close(IN);
+
+# Remove the Jekyll header
+$pandoc_input =~ s/.*---\n.+?---\n//s;
+
+# Remove the {% include ... %} directives
+$pandoc_input =~ s/\n{0,1}\s*{%\s+include .+?\s+%}\s*\n/\n/g;
+
+# Change {% highlight c %} to ```c
+$pandoc_input =~ s/^\s*{%\s+highlight\s+c\s+%}\s*$/\n```c/gmi;
+
+# Change {% endhighlight %} to ```
+$pandoc_input =~ s/^\s*\{\%\s+endhighlight\s+\%\}\s*$/```\n/gmi;
+
+# Pandoc does not handle markdown links in output nroff properly,
+# so just remove all links.
+while ($pandoc_input =~ m/\[(.+?)\]\(.+?\)/) {
+    my $text = $1;
+    $pandoc_input =~ s/\[(.+?)\]\(.+?\)/$text/;
+}
+
+# Add the pandoc header
+$pandoc_input = "% $shortfile($section) PMIx Programmer's Manual | \@VERSION\@
+% PMIx
+% \@DATE\@\n\n$pandoc_input";
+
+# Generate the nroff output
+my ($fh, $temp_filename) = tempfile();
+print $fh $pandoc_input;
+close($fh);
+
+open(IN, "pandoc -s --from=markdown --to=man $temp_filename|")
+    || die "Can't run pandoc";
+my $pandoc_nroff;
+$pandoc_nroff .= $_
+    while (<IN>);
+close(IN);
+unlink($temp_filename);
+
+# Now that we have the nroff string result, is it different than the
+# target file?
+my $write_nroff = 1;
+if (-r $target) {
+    # If the target file exists, read it in
+    open(IN, $target)
+        || die "Can't open $target";
+    my $target_nroff;
+    $target_nroff .= $_
+        while (<IN>);
+    close(IN);
+
+    # Remove the date from the target nroff string so that we can
+    # compare and ignore if the date has changed.  Note that some
+    # versions of pandoc render dates as xxxx\-xx\-xx, and others
+    # render it as xxxx-xx-xx.  Handle both.
+    $target_nroff =~ s/\"\d\d\d\d\\\-\d\d\\\-\d\d\"/\"\\\@DATE\\\@\"/;
+    $target_nroff =~ s/\"\d\d\d\d\-\d\d\-\d\d\"/\"\\\@DATE\\\@\"/;
+
+    $write_nroff = 0
+        if ($pandoc_nroff eq $target_nroff);
+}
+
+# Do we need to write a new target nroff?
+if ($write_nroff) {
+
+    # What's the date right now?
+    my $now_string = strftime "%Y\\-%m\\-%d", localtime;
+    $pandoc_nroff =~ s/\\\@DATE\\\@/$now_string/g;
+
+    # Make sure the target directory exists
+    my $dirname = dirname($target);
+    mkdir($dirname)
+        if (! -d $dirname);
+
+    open(OUT, ">$target")
+        || die "Can't write to $target";
+    print OUT $pandoc_nroff;
+    close(OUT);
+
+    print "--> Wrote new $target\n";
+} else {
+    print "--> $target unchanged; not written\n";
+}
+
+exit(0);

opal/mca/pmix/pmix2x/pmix/contrib/perf_tools/Makefile

@@ -0,0 +1,21 @@
+PMIX_BASE = <pmix-path>
+PMIX_INC= -I$(PMIX_BASE)/include/
+PMIX_LIB= -L$(PMIX_BASE)/lib/ -lpmix
+
+PMI2_BASE = /usr/
+#PMI2_INC= -I$(PMI2_BASE)/include/
+#PMI2_LIB= -L$(PMI2_BASE)/lib/ -lpmi2
+PMI2_LIB= -lpmi2
+
+CFLAGS = -O2 -g
+
+all: pmix pmi2
+
+pmix: pmi_intra_perf.c pmi.h pmix.c
+	gcc $(PMIX_INC) $(CFLAGS) -o pmix_intra_perf pmi_intra_perf.c pmix.c $(PMIX_LIB) -lrt
+
+pmi2: pmi_intra_perf.c pmi.h pmi2.c pmi2_pmap_parser.c pmi2_pmap_parser.h pmi2_utils.c pmi2_utils.h
+	gcc $(PMI2_INC) $(CFLAGS) -o pmi2_intra_perf pmi_intra_perf.c pmi2.c pmi2_utils.c pmi2_pmap_parser.c -lrt $(PMI2_LIB)
+
+clean:
+	rm -f pmix_intra_perf pmi2_intra_perf

opal/mca/pmix/pmix2x/pmix/contrib/perf_tools/Makefile.include

@@ -0,0 +1,36 @@
+# -*- makefile -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2007 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2013-2016 Intel, Inc. All rights reserved
+# Copyright (c) 2016      Cisco Systems, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# This makefile.am does not stand on its own - it is included from
+# Makefile.am
+
+EXTRA_DIST += \
+        perf_tools/Makefile \
+        perf_tools/pmi_intra_perf.c \
+        perf_tools/pmi.h \
+        perf_tools/pmi2_pmap_parser.c \
+        perf_tools/pmi2_pmap_parser.h \
+        perf_tools/pmi2_utils.c \
+        perf_tools/pmi2_utils.h \
+        perf_tools/pmi2.c \
+        perf_tools/pmix.c \
+        perf_tools/README \
+        perf_tools/run.sh

opal/mca/pmix/pmix2x/pmix/contrib/perf_tools/README

@@ -0,0 +1,27 @@
+Building instructions:
+
+* Open the Makefile and fix the following variables:
+  * PMIX_BASE should point to the directory where PMIx is installed.
+    If you don't want to test PMIx - remove pmix from the list of "all" targets.
+  * PMI2_BASE should point to the directory where SLURM is installed.
+    If you don't want to test PMI2 - remove pmi2 from the list of "all" targets.
+* run `make`.
+
+The follwoing files (or one of them) will be built as the result:
+* pmix_intra_perf - that is for testing pmix performance
+- pmi2_intra_perf - for testing pmi2 performance
+
+Running instructions:
+
+The following options are supported by both of the binaries:
+  -s, --key-size=<size>     size of the key's submitted (default is 100 * sizeof(int) bytes)
+  -c, --key-count=<size>    number of keys submitted to local and remote parts (default is 10)
+  -d, --direct-modex        use direct modex if available
+  --debug                   force all processes to print out the timings themself
+
+You can run it directly with your favorite launcher (mpirun/srun), just
+make sure that both MPI and PMIx libraries are visible for the loader.
+
+For PMIx testing "convenience" there is a `run.sh` script that can be used to
+ensure that environment is set properly (not a production grade so may not work
+for all environments)

opal/mca/pmix/pmix2x/pmix/contrib/perf_tools/pmi.h

@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2016      Mellanox Technologies, Inc.
+ *                         All rights reserved.
+ * Copyright (c) 2016      Intel, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef PMI_INTERFACE_H
+#define PMI_INTERFACE_H
+
+#include <stdio.h>
+
+void pmi_init(int *rank, int *size);
+void pmi_get_local_ranks(int **local_ranks, int *local_cnt);
+void pmi_put_key_loc(char *key, int *key_val, int key_size);
+void pmi_put_key_rem(char *key, int *key_val, int key_size);
+void pmi_put_double(char *key, double val);
+void pmi_commit();
+void pmi_fence(int collect);
+void pmi_fini();
+void pmi_get_key_loc(int rank, char *key_name, int **key_val, int *key_size);
+void pmi_get_key_rem(int rank, char *key_name, int **key_val, int *key_size);
+double pmi_get_double(int rank, char *key);
+
+#endif

opal/mca/pmix/pmix2x/pmix/contrib/perf_tools/pmi2.c

@@ -0,0 +1,187 @@
+/*
+ * Copyright (c) 2016      Mellanox Technologies, Inc.
+ *                         All rights reserved.
+ * Copyright (c) 2016      Intel, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include <stdio.h>
+#include <slurm/pmi2.h>
+#include "pmi2_pmap_parser.h"
+#include "pmi2_utils.h"
+
+static int my_rank, my_node;
+static char * kvs_name;
+
+void pmi_init(int *rank, int *size)
+{
+    int spawned, appnum;
+    int rc;
+
+    *size = -1;
+    *rank = -1;
+    appnum = -1;
+
+    if (PMI2_SUCCESS != (rc = PMI2_Init(&spawned, size, rank, &appnum))) {
+        fprintf(stderr, "pmi2: PMI2_Init: error rc = %d\n", rc);
+        abort();
+    }
+    if( *size < 0 || *rank < 0 ){
+        fprintf(stderr, "pmi2: PMI2_Init: bad size=%d or rank=%d values\n",
+               *size, *rank);
+        abort();
+    }
+    my_rank = *rank;
+
+    kvs_name = (char*)malloc(PMI2_MAX_VALLEN);
+    if( kvs_name == NULL ){
+        fprintf(stderr, "pmi_init (pmi2): cannot malloc kvs name buffer\n");
+        abort();
+    }
+    rc = PMI2_Job_GetId(kvs_name, PMI2_MAX_VALLEN);
+    if( PMI2_SUCCESS != rc ){
+        fprintf(stderr, "pmi_init (pmi2): cannot get kvs name: rc = %d\n", rc);
+        abort();
+    }
+}
+
+void pmi_get_local_ranks(int **local_ranks, int *local_cnt)
+{
+    int rc, found;
+
+    char *pmapping = (char*)malloc(PMI2_MAX_VALLEN);
+    if( pmapping == NULL ){
+        fprintf(stderr,"pmi_get_local_ranks (pmi2): cannot mallic for pmappig\n");
+        abort();
+    }
+
+
+    rc = PMI2_Info_GetJobAttr("PMI_process_mapping", pmapping, PMI2_MAX_VALLEN, &found);
+    if( !found || PMI2_SUCCESS != rc ) {
+        fprintf(stderr,"pmi_get_local_ranks (pmi2): PMI2_Info_GetJobAttr rc = %d\n", rc);
+        abort();
+    }
+
+    *local_ranks = NULL;
+    *local_ranks = mca_common_pmi2_parse_pmap(pmapping, my_rank, &my_node, local_cnt);
+    if (NULL == *local_ranks) {
+        fprintf(stderr,"mca_common_pmi2_parse_pmap: error\n");
+        abort();
+    }
+
+    free(pmapping);
+}
+
+void pmi_put_key_loc(char *key, int *key_val, int key_size)
+{
+    char *encoded = pmi_encode(key_val, key_size * sizeof(int));
+    if( NULL == encoded ){
+        fprintf(stderr, "pmi_encode: error on key: %s\n", key);
+        abort();
+    }
+    PMI2_Info_PutNodeAttr(key, encoded);
+}
+
+void pmi_put_key_rem(char *key, int *key_val, int key_size)
+{
+    char *encoded = pmi_encode(key_val, key_size * sizeof(int));
+    if( NULL == encoded ){
+        fprintf(stderr, "pmi_encode: error on key: %s\n", key);
+        abort();
+    }
+
+    PMI2_KVS_Put(key, encoded);
+}
+
+void pmi_put_double(char *key, double val)
+{
+    char buf[128];
+    sprintf(buf, "%lf", val);
+    PMI2_KVS_Put(key, buf);
+}
+
+
+void pmi_commit()
+{
+
+}
+
+void pmi_fence(int collect)
+{
+    int rc;
+    /* now call fence */
+    if (PMI2_SUCCESS != (rc = PMI2_KVS_Fence()) ) {
+        fprintf(stderr, "PMI2_KVS_Fence: error %d\n", rc);
+        abort();
+    }
+}
+
+void pmi_fini()
+{
+    PMI2_Finalize();
+}
+
+void pmi_get_key_loc(int rank, char *key_name, int **key_val, int *key_size)
+{
+    int found, rc;
+    size_t tmp_size;
+    char *tmp = calloc(PMI2_MAX_VALLEN, sizeof(char));
+    if( (rc = PMI2_Info_GetNodeAttr(key_name, tmp, PMI2_MAX_VALLEN, &found, 1) ) ){
+        fprintf(stderr,"PMI2_Info_GetNodeAttr: error rc = %d\n", rc);
+        abort();
+    }
+    if( !found ){
+        fprintf(stderr,"pmi_get_key_loc: key %s not found\n", key_name);
+        abort();
+    }
+
+    *key_val = (int*)pmi_decode(tmp, &tmp_size);
+    *key_size = tmp_size / sizeof(int);
+
+    if( NULL == *key_val ){
+        fprintf(stderr,"pmi_decode: cannot decode key %s\n", key_name);
+        abort();
+    }
+    free(tmp);
+}
+
+void pmi_get_key_rem(int rank, char *key_name, int **key_val, int *key_size)
+{
+    int len, rc;
+    size_t tmp_size;
+
+    char *tmp = calloc(PMI2_MAX_VALLEN, sizeof(char));
+    if( (rc = PMI2_KVS_Get(kvs_name, PMI2_ID_NULL, key_name, tmp, PMI2_MAX_VALLEN, &len) ) ){
+        fprintf(stderr,"PMI2_Info_GetNodeAttr: error rc = %d\n", rc);
+        abort();
+    }
+
+    *key_val = (int*)pmi_decode(tmp, &tmp_size);
+    *key_size = tmp_size / sizeof(int);
+
+    if( NULL == *key_val ){
+        fprintf(stderr,"pmi_decode: cannot decode key %s\n", key_name);
+        abort();
+    }
+    free(tmp);
+}
+
+float pmi_get_double(int rank, char *key)
+{
+    int len, rc;
+    size_t tmp_size;
+    char *tmp = calloc(PMI2_MAX_VALLEN, sizeof(char));
+    double v;
+
+    if( (rc = PMI2_KVS_Get(kvs_name, PMI2_ID_NULL, key, tmp, PMI2_MAX_VALLEN, &len) ) ){
+        fprintf(stderr,"PMI2_Info_GetNodeAttr: error rc = %d\n", rc);
+        abort();
+    }
+    sscanf(tmp, "%lf", &v);
+    free(tmp);
+    return v;
+}

opal/mca/pmix/pmix2x/pmix/contrib/perf_tools/pmi2_pmap_parser.c

@@ -0,0 +1,244 @@
+/* -*- Mode: C; c-basic-offset:4 ; -*- */
+/*
+ *
+ * Copyright (c) 2013      Mellanox Technologies, Inc.
+ *                         All rights reserved.
+ * Copyright (c) 2014-2016 Intel, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+
+/* This code was taken from Open MPI project, file
+   opal/mca/pmix/s2/pmi2_pmap_parser.c
+*/
+
+#include "pmi2_pmap_parser.h"
+
+/**
+ pmi2 process mapping is returned as a
+ comma separated list of tuples:
+ex: (vector,(0,4,4),(0,4,1))
+slurm cyclic distro of 4 ranks over 2 nodes:
+(vector,(0,2,1),(0,2,1))
+slurm block distro of 4 ranks over 2 nodes:
+(vector,(0,2,2))
+
+
+ Format of each tuple is (base, H, L), where
+ H is number of nodes spawned by tuple,
+ L is number of ranks per node,
+ base is offset from node 0.
+
+ Tuple can be visualized as a rectangle on two
+ dimensional (Hosts, Local Ranks) plane:
+
+           ------------------------------------ Hosts ->
+           |              H
+           |           +--------+
+           |<- base -->|        |
+           |           |        | L
+           |           +--------+
+        Local Ranks
+           V
+
+Note that ranks increase by column. Tuple (0,2,3) looks like:
+0 3
+1 4
+2 5
+
+*/
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+static int find_my_node(char *map, int me)
+{
+    int abs_rank;
+    int base, H, L;
+    char *p;
+
+    p = map;
+    abs_rank = 0;
+    while (NULL != (p = strstr(p+1, ",("))) {
+        if (3 != sscanf(p, ",(%d,%d,%d)", &base, &H, &L)) {
+            return -1;
+        }
+        if (me >= abs_rank && me < abs_rank + H*L) {
+            /* found my rectangle, compute node */
+            return base + (me - abs_rank)/L;
+        }
+        abs_rank += H*L;
+    }
+    return -1;
+}
+
+static int *find_lrs(char *map, int my_node, int *nlrs)
+{
+    int abs_rank;
+    int base, H, L;
+    char *p;
+    int *lrs;
+    int max_lr;
+    int i;
+
+    p = map;
+    abs_rank = 0;
+    *nlrs = 0;
+    max_lr = 16;
+    lrs = malloc(max_lr * sizeof(int));
+    while (NULL != (p = strstr(p+1, ",("))) {
+        if (3 != sscanf(p, ",(%d,%d,%d)", &base, &H, &L)) {
+            free(lrs);
+            return NULL;
+        }
+        if (base <= my_node && my_node < base + H) {
+            if (*nlrs + L >= max_lr) {
+                lrs = realloc(lrs, (max_lr + L) * sizeof(int));
+                if (NULL == lrs) {
+                    *nlrs = 0;
+                    free(lrs);
+                    return NULL;
+                }
+                max_lr += L;
+            }
+            /* skip (my_node - base) columns of L elems,
+             * numbers in my column are local to me
+             */
+            for (i = 0; i < L; i++) {
+                lrs[*nlrs] = (my_node - base) * L + i + abs_rank;
+                (*nlrs) ++;
+            }
+        }
+        abs_rank += H*L;
+    }
+
+    if (0 == *nlrs) {
+        free(lrs);
+        lrs = 0;
+    }
+    return lrs;
+}
+
+/**
+ * @param pmap process map as returned by PMI_process_mapping
+ *             attribute
+ * @param my_rank
+ * @param node set to my node id
+ * @param nlrs set to the number of local ranks returned
+ *
+ * @return array that contains ranks local to my_rank or NULL
+ * on failure. Array must be freed by the caller.
+ */
+int *mca_common_pmi2_parse_pmap(char *pmap, int my_rank,
+                                  int *node, int *nlrs)
+{
+    char *p;
+
+    p = strstr(pmap, "(vector");
+    if (NULL == p) {
+        return NULL;
+    }
+
+    *node = find_my_node(p, my_rank);
+    if (0 > *node) {
+       return NULL;
+    }
+
+    return find_lrs(p, *node, nlrs);
+}
+
+
+#ifdef STANDALONE_TEST
+#include <assert.h>
+static void dump_lrs(int *lrs, int me, int node, int n)
+{
+    int i;
+
+    printf("Total %d ranks/node, node %d me %d\n", n, node, me);
+    for (i = 0; i < n; i++) {
+        printf("%d ", lrs[i]);
+    }
+    printf("\n");
+    free(lrs);
+}
+
+int main(int argc, char **argv)
+{
+    int me, n, node;
+    int *lrs;
+    char *pmap;
+    int a1[] = {0, 1};
+    int a2[] = {2, 3};
+    int a3[] = {0, 2};
+    int a4[] = {1, 3};
+    int a5[] = {0,1,3,2,16,17};
+    int a6[] = {8,9,10,11,19};
+
+
+    if (argc == 3) {
+        me = atoi(argv[1]);
+        lrs = orte_grpcomm_pmi2_parse_pmap(argv[2], me, &node, &n);
+        if (NULL == lrs) {
+            printf("can not parse pmap\n");
+            exit(1);
+        }
+        dump_lrs(lrs, me, node, n);
+        exit(0);
+    }
+    /* built in cases */
+
+    pmap = "(vector,(0,2,2))";
+    me = 1;
+    lrs = orte_grpcomm_pmi2_parse_pmap(pmap, me, &node, &n);
+    assert(lrs);
+    assert(n == 2);
+    assert(memcmp(lrs, a1, 2) == 0);
+    free(lrs);
+
+
+    pmap = "(vector,(0,2,2))";
+    me = 2;
+    lrs = orte_grpcomm_pmi2_parse_pmap(pmap, me, &node, &n);
+    assert(lrs);
+    assert(n == 2);
+    assert(memcmp(lrs, a2, 2) == 0);
+    free(lrs);
+
+
+    /* cyclic distro which skips node 0 */
+    pmap = "(vector,(1,2,1),(1,2,1))";
+    me = 0;
+    lrs = orte_grpcomm_pmi2_parse_pmap(pmap, me, &node, &n);
+    assert(lrs);
+    assert(n == 2);
+    assert(memcmp(lrs, a3, n) == 0);
+    free(lrs);
+
+    pmap = "(vector,(1,2,1),(1,2,1))";
+    me = 3;
+    lrs = orte_grpcomm_pmi2_parse_pmap(pmap, me, &node, &n);
+    assert(lrs);
+    assert(n == 2);
+    assert(memcmp(lrs, a4, n) == 0);
+    free(lrs);
+
+    pmap = "(vector,(0,4,4),(0,1,2),(1,3,1))";
+    me = 3;
+    lrs = orte_grpcomm_pmi2_parse_pmap(pmap, me, &node, &n);
+    assert(lrs);
+    assert(n == 6);
+    assert(memcmp(lrs, a5, n) == 0);
+    free(lrs);
+
+    pmap = "(vector,(0,4,4),(0,1,2),(1,3,1))";
+    me = 10;
+    lrs = orte_grpcomm_pmi2_parse_pmap(pmap, me, &node, &n);
+    assert(lrs);
+    assert(n == 5);
+    assert(memcmp(lrs, a6, n) == 0);
+    free(lrs);
+    return 0;
+}
+#endif

opal/mca/pmix/pmix2x/pmix/contrib/perf_tools/pmi2_pmap_parser.h

@@ -0,0 +1,24 @@
+/* -*- Mode: C; c-basic-offset:4 ; -*- */
+/*
+ *
+ * Copyright (c) 2013      Mellanox Technologies, Inc.
+ *                         All rights reserved.
+ * Copyright (c) 2014-2016 Intel, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+
+/* This code was taken from Open MPI project, file
+   opal/mca/pmix/s2/pmi2_pmap_parser.h
+*/
+
+
+#ifndef PMI2_PMAP_PARSER_H
+#define PMI2_PMAP_PARSER_H
+
+int *mca_common_pmi2_parse_pmap(char *pmap, int my_rank,
+                                  int *node, int *nlrs);
+#endif

opal/mca/pmix/pmix2x/pmix/contrib/perf_tools/pmi2_utils.c

@@ -0,0 +1,132 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2012-2015 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2014-2016 Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2016 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016      Mellanox Technologies, Inc.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+
+/* This code was taken from Open MPI project, file
+   opal/mca/pmix/base/pmix_base_fns.c
+*/
+
+#include <assert.h>
+#include <string.h>
+#include "pmi2_utils.h"
+
+/* base64 encoding with illegal (to Cray PMI) characters removed ('=' is replaced by ' ') */
+static inline unsigned char pmi_base64_encsym (unsigned char value) {
+    assert (value < 64);
+
+    if (value < 26) {
+    return 'A' + value;
+    } else if (value < 52) {
+    return 'a' + (value - 26);
+    } else if (value < 62) {
+    return '0' + (value - 52);
+    }
+
+    return (62 == value) ? '+' : '/';
+}
+
+static inline unsigned char pmi_base64_decsym (unsigned char value) {
+    if ('+' == value) {
+    return 62;
+    } else if ('/' == value) {
+    return 63;
+    } else if (' ' == value) {
+    return 64;
+    } else if (value <= '9') {
+    return (value - '0') + 52;
+    } else if (value <= 'Z') {
+    return (value - 'A');
+    } else if (value <= 'z') {
+    return (value - 'a') + 26;
+    }
+
+    return 64;
+}
+
+static inline void pmi_base64_encode_block (const unsigned char in[3], char out[4], int len) {
+    out[0] = pmi_base64_encsym (in[0] >> 2);
+    out[1] = pmi_base64_encsym (((in[0] & 0x03) << 4) | ((in[1] & 0xf0) >> 4));
+    /* Cray PMI doesn't allow = in PMI attributes so pad with spaces */
+    out[2] = 1 < len ? pmi_base64_encsym(((in[1] & 0x0f) << 2) | ((in[2] & 0xc0) >> 6)) : ' ';
+    out[3] = 2 < len ? pmi_base64_encsym(in[2] & 0x3f) : ' ';
+}
+
+static inline int pmi_base64_decode_block (const char in[4], unsigned char out[3]) {
+    char in_dec[4];
+
+    in_dec[0] = pmi_base64_decsym (in[0]);
+    in_dec[1] = pmi_base64_decsym (in[1]);
+    in_dec[2] = pmi_base64_decsym (in[2]);
+    in_dec[3] = pmi_base64_decsym (in[3]);
+
+    out[0] = in_dec[0] << 2 | in_dec[1] >> 4;
+    if (64 == in_dec[2]) {
+    return 1;
+    }
+
+    out[1] = in_dec[1] << 4 | in_dec[2] >> 2;
+    if (64 == in_dec[3]) {
+    return 2;
+    }
+
+    out[2] = ((in_dec[2] << 6) & 0xc0) | in_dec[3];
+    return 3;
+}
+
+
+/* PMI only supports strings. For now, do a simple base64. */
+char *pmi_encode(const void *val, size_t vallen)
+{
+    char *outdata, *tmp;
+    size_t i;
+
+    outdata = calloc (((2 + vallen) * 4) / 3 + 2, 1);
+    if (NULL == outdata) {
+    return NULL;
+    }
+
+    for (i = 0, tmp = outdata ; i < vallen ; i += 3, tmp += 4) {
+        pmi_base64_encode_block((unsigned char *) val + i, tmp, vallen - i);
+    }
+
+    tmp[0] = (unsigned char)'\0';
+
+    return outdata;
+}
+
+uint8_t *pmi_decode (const char *data, size_t *retlen)
+{
+    size_t input_len = strlen(data) / 4;
+    unsigned char *ret;
+    int out_len;
+    size_t i;
+
+    /* default */
+    *retlen = 0;
+
+    ret = calloc (1, 3 * input_len + 1);
+    if (NULL == ret) {
+        return ret;
+    }
+
+    for (i = 0, out_len = 0 ; i < input_len ; i++, data += 4) {
+    out_len += pmi_base64_decode_block(data, ret + 3 * i);
+    }
+
+    ret[out_len] = '\0';
+    *retlen = out_len;
+    return ret;
+}

opal/mca/pmix/pmix2x/pmix/contrib/perf_tools/pmi2_utils.h

@@ -0,0 +1,11 @@
+#ifndef PMI2_UTILS_H
+#define PMI2_UTILS_H
+
+#include <stdlib.h>
+#include <stdint.h>
+#include <inttypes.h>
+
+char *pmi_encode(const void *val, size_t vallen);
+uint8_t *pmi_decode (const char *data, size_t *retlen);
+
+#endif // PMI2_UTILS_H

opal/mca/pmix/pmix2x/pmix/contrib/perf_tools/pmi_intra_perf.c

@@ -0,0 +1,517 @@
+/*
+ * Copyright (c) 2015      Mellanox Technologies, Inc.  All rights reserved.
+ * Copyright (c) 2016      Intel, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <time.h>
+#include <getopt.h>
+#include <limits.h>
+#include <string.h>
+
+#include "pmi.h"
+
+#include <time.h>
+#define GET_TS ({ \
+    struct timespec ts;                     \
+    double ret;                             \
+    clock_gettime(CLOCK_MONOTONIC, &ts);    \
+    ret = ts.tv_sec + 1E-9 * ts.tv_nsec;    \
+    ret;                                    \
+})
+
+
+int key_size = 100, key_count = 10, rank_shift;
+int direct_modex = 0, debug_on = 0;
+
+static void usage(const char *argv0)
+{
+    printf("Usage:\n");
+    printf("  %s [options]              start the benchmark\n", argv0);
+    printf("\n");
+    printf("  -s, --key-size=<size>     size of the key's submitted\n");
+    printf("  -c, --key-count=<size>    number of keys submitted to local and remote parts\n");
+    printf("  -d, --direct-modex        use direct modex if available\n");
+    printf("  --debug                   force all processes to print out the timings themself\n");
+}
+
+
+void parse_options(int argc, char **argv)
+{
+    extern char *optarg;
+    extern int optind;
+    struct option long_options[] = {
+        { "help",           0, NULL, 'h' },
+        /* IB options */
+        { "key-size",       1, NULL, 's' },
+        { "key-count",      1, NULL, 'c' },
+        { "direct-modex",   0, NULL, 'd' },
+        { "debug",          0, NULL, '0' },
+        { 0 }
+    };
+
+    while (1) {
+        int c;
+        c = getopt_long(argc, argv, "hs:c:d0", long_options, NULL);
+
+        if (c == -1)
+            break;
+        switch (c) {
+        case 's':
+            key_size = atoi(optarg);
+            break;
+        case 'c':
+            key_count = atoi(optarg);
+            break;
+        case 'd':
+            direct_modex = 1;
+            break;
+        case '0':
+            debug_on = 1;
+            break;
+        case 'h':
+        default:
+            usage(argv[0]);
+            exit(0);
+        }
+    }
+
+    rank_shift = 10;
+    while( rank_shift <= key_count ){
+        rank_shift *= 10;
+    }
+}
+
+void fill_remote_ranks(int *local_ranks, int local_cnt, int *remote_ranks, int size)
+{
+    int i, k;
+    for(i = 0, k = 0; i < size && k < (size - local_cnt); i++ ){
+        int j, flag = 1;
+        for(j=0; j < local_cnt; j++){
+            if( i == local_ranks[j] ){
+                flag = 0;
+                break;
+            }
+        }
+        if( flag ){
+            remote_ranks[k] = i;
+            k++;
+        }
+    }
+}
+
+int store_double(char *name, double val)
+{
+    char buf[128];
+    sprintf(buf,"%lf",val);
+
+}
+
+int get_mem_usage(double *_pss, double *_rss) {
+    char data[PATH_MAX];
+    FILE *smaps;
+    double pss = 0.0, rss = 0.0;
+    char *line = NULL;
+    size_t size = 0;
+    pid_t pid = getpid();
+
+    *_pss = 0.0;
+    *_rss = 0.0;
+
+    memset(data, 0, sizeof(data));
+    snprintf(data, sizeof(data), "/proc/%d/smaps", pid);
+
+    if (NULL == (smaps = fopen(data, "r"))) {
+        return -1;
+    }
+
+    while ((size = getline(&line, &size, smaps)) != -1) {
+        if (0 == strncmp(line, "Pss", strlen("Pss"))) {
+            sscanf(line, "Pss: %lf", &pss);
+            *_pss += pss;
+        }
+        if (0 == strncmp(line, "Rss", strlen("Pss"))) {
+            sscanf(line, "Rss: %lf", &rss);
+            *_rss += pss;
+        }
+    }
+    free(line);
+    fclose(smaps);
+
+    return 0;
+}
+
+int main(int argc, char **argv)
+{
+    int rc;
+    char *key_name;
+    int *key_val;
+    int rank, nproc;
+    int cnt;
+    int *local_ranks, local_cnt;
+    int *remote_ranks, remote_cnt;
+    double start, total_start, get_loc_time = 0, get_rem_time = 0, put_loc_time = 0,
+            put_rem_time = 0, commit_time = 0, fence_time = 0, init_time = 0, total_time = 0;
+    int get_loc_cnt = 0, get_rem_cnt = 0, put_loc_cnt = 0, put_rem_cnt = 0;
+    double mem_pss = 0.0, mem_rss = 0.0;
+
+    parse_options(argc, argv);
+
+    total_start = GET_TS;
+    start = GET_TS;
+    pmi_init(&rank, &nproc);
+    init_time += GET_TS - start;
+
+    pmi_get_local_ranks(&local_ranks, &local_cnt);
+    remote_cnt = nproc - local_cnt;
+    if( remote_cnt ){
+        remote_ranks = calloc(remote_cnt, sizeof(int));
+        fill_remote_ranks(local_ranks, local_cnt, remote_ranks, nproc);
+    }
+
+    if( 0 == rank && debug_on ){
+        int i;
+        fprintf(stderr,"%d: local ranks: ", rank);
+        for(i = 0; i < local_cnt; i++){
+            fprintf(stderr,"%d ", local_ranks[i]);
+        }
+        fprintf(stderr,"\n");
+        fflush(stderr);
+    }
+
+    key_val = calloc(key_size, sizeof(int));
+    for (cnt=0; cnt < key_count; cnt++) {
+        int i;
+        if( local_cnt > 0 ){
+            (void)asprintf(&key_name, "KEY-%d-local-%d", rank, cnt);
+            for(i=0; i < key_size; i++){
+                key_val[i] = rank * rank_shift + cnt;
+            }
+            put_loc_cnt++;
+            start = GET_TS;
+            pmi_put_key_loc(key_name, key_val, key_size);
+            put_loc_time += GET_TS - start;
+            free(key_name);
+        }
+        if( remote_cnt > 0 ){
+            (void)asprintf(&key_name, "KEY-%d-remote-%d", rank, cnt);
+            for(i=0; i < key_size; i++){
+                key_val[i] = rank * rank_shift + cnt;
+            }
+            put_rem_cnt++;
+            start = GET_TS;
+            pmi_put_key_rem(key_name, key_val, key_size);
+            put_rem_time += GET_TS - start;
+            free(key_name);
+        }
+    }
+    free(key_val);
+
+    start = GET_TS;
+    pmi_commit();
+    commit_time += GET_TS - start;
+
+    start = GET_TS;
+    pmi_fence( !direct_modex );
+    fence_time += GET_TS - start;
+
+
+
+    for (cnt=0; cnt < key_count; cnt++) {
+        int i;
+
+        for(i = 0; i < remote_cnt; i++){
+            int rank = remote_ranks[i], j;
+            int *key_val, key_size_new;
+            double start;
+            (void)asprintf(&key_name, "KEY-%d-remote-%d", rank, cnt);
+
+            start = GET_TS;
+            pmi_get_key_rem(rank, key_name, &key_val, &key_size_new);
+            get_rem_time += GET_TS - start;
+            get_rem_cnt++;
+
+            if( key_size != key_size_new ){
+                fprintf(stderr,"%d: error in key %s sizes: %d vs %d\n",
+                        rank, key_name, key_size, key_size_new);
+                abort();
+            }
+
+            for(j=0; j < key_size; j++){
+                if( key_val[j] != rank * rank_shift + cnt ){
+                    fprintf(stderr, "%d: error in key %s value (byte %d)\n",
+                            rank, key_name, j);
+                    abort();
+                }
+            }
+            free(key_name);
+            free(key_val);
+        }
+
+         // check the returned data
+        for(i = 0; i < local_cnt; i++){
+            int rank = local_ranks[i], j;
+            int *key_val, key_size_new;
+            double start;
+            (void)asprintf(&key_name, "KEY-%d-local-%d", rank, cnt);
+
+            start = GET_TS;
+            pmi_get_key_loc(rank, key_name, &key_val, &key_size_new);
+            get_loc_time += GET_TS - start;
+            get_loc_cnt++;
+
+            if( key_size != key_size_new ){
+                fprintf(stderr,"%d: error in key %s sizes: %d vs %d\n",
+                        rank, key_name, key_size, key_size_new);
+                abort();
+            }
+
+            for(j=0; j < key_size; j++){
+                if( key_val[j] != rank * rank_shift + cnt ){
+                    fprintf(stderr, "%d: error in key %s value (byte %d)",
+                            rank, key_name, j);
+                    abort();
+                }
+            }
+            free(key_name);
+            free(key_val);
+        }
+    }
+
+    total_time = GET_TS - total_start;
+
+    if (0 != get_mem_usage(&mem_pss, &mem_rss)) {
+        fprintf(stderr, "Rank %d: error get memory usage", rank);
+        abort();
+    }
+
+    if( debug_on ){
+        fprintf(stderr,"%d: get: total %lf avg loc %lf rem %lf all %lf ; put: %lf %lf commit: %lf fence %lf\n",
+                rank, (get_loc_time + get_rem_time),
+                get_loc_time/get_loc_cnt, get_rem_time/get_rem_cnt,
+                (get_loc_time + get_rem_time)/(get_loc_cnt + get_rem_cnt),
+                put_loc_time/put_loc_cnt, put_rem_time/put_rem_cnt,
+                commit_time, fence_time);
+    }
+
+    /* Out of the perf path - send our results to rank 0 using same PMI */
+    char key[128];
+    sprintf(key, "PMIX_PERF_get_total_time.%d", rank);
+    pmi_put_double(key, get_rem_time + get_loc_time);
+
+    sprintf(key, "PMIX_PERF_get_loc_time.%d", rank);
+    pmi_put_double(key, get_loc_cnt ? get_loc_time/get_loc_cnt : 0 );
+
+    sprintf(key, "PMIX_PERF_get_rem_time.%d", rank);
+    pmi_put_double(key, get_rem_cnt ? get_rem_time/get_rem_cnt : 0);
+
+    sprintf(key, "PMIX_PERF_get_time.%d", rank);
+    pmi_put_double(key, (get_loc_time + get_rem_time)/(get_loc_cnt + get_rem_cnt) );
+
+    sprintf(key, "PMIX_PERF_put_loc_time.%d", rank);
+    pmi_put_double(key, put_loc_cnt ? put_loc_time / put_loc_cnt : 0);
+
+    sprintf(key, "PMIX_PERF_put_rem_time.%d", rank);
+    pmi_put_double(key, put_rem_cnt ? put_rem_time / put_rem_cnt : 0);
+
+    sprintf(key, "PMIX_PERF_commit_time.%d", rank);
+    pmi_put_double(key, commit_time);
+
+    sprintf(key, "PMIX_PERF_fence_time.%d", rank);
+    pmi_put_double(key, fence_time);
+
+    sprintf(key, "PMIX_PERF_init_time.%d", rank);
+    pmi_put_double(key, init_time);
+
+    sprintf(key, "PMIX_PERF_total_time.%d", rank);
+    pmi_put_double(key, total_time);
+
+    sprintf(key, "PMIX_PERF_mem_pss.%d", rank);
+    pmi_put_double(key, mem_pss);
+
+    sprintf(key, "PMIX_PERF_mem_rss.%d", rank);
+    pmi_put_double(key, mem_rss);
+
+    pmi_commit();
+    pmi_fence( 1 );
+
+    if( rank == 0 ){
+        double  cum_get_total_time = 0,
+                cum_get_loc_time = 0,
+                cum_get_rem_time = 0,
+                cum_get_time = 0,
+                cum_put_total_time = 0,
+                cum_put_loc_time = 0,
+                cum_put_rem_time = 0,
+                cum_commit_time = 0,
+                cum_fence_time = 0,
+                cum_init_time = 0,
+                cum_total_time = 0,
+                cum_mem_pss = 0.0;
+
+        double  min_get_loc_time = get_loc_time / get_loc_cnt,
+                max_get_loc_time = get_loc_time / get_loc_cnt,
+                min_get_rem_time = get_rem_time / get_rem_cnt,
+                max_get_rem_time = get_rem_time / get_rem_cnt,
+                min_init_time = init_time,
+                max_init_time = init_time,
+                min_total_time = total_time,
+                max_total_time = total_time,
+                min_mem_pss = mem_pss,
+                max_mem_pss = 0.0;
+
+        int min_get_loc_idx = 0, max_get_loc_idx = 0;
+        int min_get_rem_idx = 0, max_get_rem_idx = 0;
+
+        char c_get_ltime[128], c_get_rtime[128], c_get_ttime[128];
+        char c_put_ltime[128], c_put_rtime[128];
+        int i;
+        for(i = 0; i < nproc; i++){
+            double val;
+            sprintf(key, "PMIX_PERF_get_total_time.%d", i);
+            cum_get_total_time += pmi_get_double(i, key);
+
+            sprintf(key, "PMIX_PERF_get_loc_time.%d", i);
+            val = pmi_get_double(i, key);
+            cum_get_loc_time += val;
+            if( min_get_loc_time > val ){
+                min_get_loc_time = val;
+                min_get_loc_idx = i;
+            }
+            if( max_get_loc_time < val ){
+                max_get_loc_time = val;
+                max_get_loc_idx = i;
+            }
+
+            sprintf(key, "PMIX_PERF_get_rem_time.%d", i);
+            val = pmi_get_double(i, key);
+            cum_get_rem_time += val;
+            if( min_get_rem_time > val ){
+                min_get_rem_time = val;
+                min_get_rem_idx = i;
+            }
+            if( max_get_rem_time < val ){
+                max_get_rem_time = val;
+                max_get_rem_idx = i;
+            }
+
+            sprintf(key, "PMIX_PERF_get_time.%d", i);
+            cum_get_time += pmi_get_double(i, key);
+
+            sprintf(key, "PMIX_PERF_put_loc_time.%d", i);
+            cum_put_loc_time += pmi_get_double(i, key);
+
+            sprintf(key, "PMIX_PERF_put_rem_time.%d", i);
+            cum_put_rem_time += pmi_get_double(i, key);
+
+            sprintf(key, "PMIX_PERF_commit_time.%d", i);
+            cum_commit_time += pmi_get_double(i, key);
+
+            sprintf(key, "PMIX_PERF_fence_time.%d", i);
+            cum_fence_time += pmi_get_double(i, key);
+
+            sprintf(key, "PMIX_PERF_init_time.%d", i);
+            val = pmi_get_double(i, key);
+            cum_init_time += val;
+            if (min_init_time > val) {
+                min_init_time = val;
+            }
+            if (max_init_time < val) {
+                max_init_time = val;
+            }
+
+            sprintf(key, "PMIX_PERF_total_time.%d", i);
+            val = pmi_get_double(i, key);
+            cum_total_time += val;
+            if (min_total_time > val) {
+                min_total_time = val;
+            }
+            if (max_total_time < val) {
+                max_total_time = val;
+            }
+
+            sprintf(key, "PMIX_PERF_mem_pss.%d", i);
+            val = pmi_get_double(i, key);
+            cum_mem_pss += val;
+            if (min_mem_pss > val) {
+                min_mem_pss = val;
+            }
+            if (max_mem_pss < val) {
+                max_mem_pss = val;
+            }
+        }
+
+        if( get_loc_cnt ){
+            sprintf(c_get_ltime,"%lf", cum_get_loc_time / nproc);
+        } else {
+            sprintf(c_get_ltime,"--------");
+        }
+        if( get_rem_cnt ){
+            sprintf(c_get_rtime,"%lf", cum_get_rem_time / nproc);
+        } else {
+            sprintf(c_get_rtime,"--------");
+        }
+
+        if( get_loc_cnt + get_rem_cnt ){
+            sprintf(c_get_ttime,"%lf", cum_get_time / nproc);
+        } else {
+            sprintf(c_get_ttime,"--------");
+        }
+
+        if( put_loc_cnt ){
+            sprintf(c_put_ltime,"%lf", cum_put_loc_time / nproc);
+            cum_put_total_time += cum_put_loc_time;
+        } else {
+            sprintf(c_put_ltime,"--------");
+        }
+        if( put_rem_cnt ){
+            sprintf(c_put_rtime,"%lf", cum_put_rem_time / nproc);
+            cum_put_total_time += cum_put_rem_time;
+        } else {
+            sprintf(c_put_rtime,"--------");
+        }
+
+        fprintf(stderr,"init: %lf; put: %lf; commit: %lf; fence: %lf; get: %lf; total: %lf\n",
+                cum_init_time / nproc,
+                cum_put_total_time / nproc,
+                cum_commit_time / nproc, cum_fence_time / nproc,
+                cum_get_total_time / nproc,
+                cum_total_time / nproc);
+        fprintf(stderr,"init:          max %lf min %lf\n",  max_init_time, min_init_time);
+        fprintf(stderr,"put:           loc %s rem %s\n", c_put_ltime, c_put_rtime);
+        fprintf(stderr,"get:           loc %s rem %s all %s\n", c_get_ltime, c_get_rtime, c_get_ttime);
+        fprintf(stderr,"get:           min loc %lf rem %lf (loc: %d, rem: %d)\n",
+                min_get_loc_time, min_get_rem_time, min_get_loc_idx, min_get_rem_idx);
+        fprintf(stderr,"get:           max loc %lf rem %lf (loc: %d, rem: %d)\n",
+                max_get_loc_time, max_get_rem_time, max_get_loc_idx, max_get_rem_idx);
+        fprintf(stderr,"total:         max %lf min %lf\n", max_total_time, min_total_time);
+        fprintf(stderr,"mem:           loc %0.2lf rem %0.2lf min %0.2lf max %0.2lf total %0.2lf Kb\n",
+                mem_pss, cum_mem_pss / nproc, min_mem_pss, max_mem_pss, cum_mem_pss);
+
+
+        /* debug printout */
+/*
+        for(i = 0; i < nproc; i++){
+            double val;
+            printf("%d: ", i);
+            sprintf(key, "PMIX_PERF_get_loc_time.%d", i);
+            printf("local = %lf ", pmi_get_double(i, key));
+
+            sprintf(key, "PMIX_PERF_get_rem_time.%d", i);
+            printf("remote = %lf\n", pmi_get_double(i, key));
+        }
+*/
+    }
+
+    pmi_fini();
+
+    return 0;
+}

opal/mca/pmix/pmix2x/pmix/contrib/perf_tools/pmix.c

@@ -0,0 +1,227 @@
+/*
+ * Copyright (c) 2015      Mellanox Technologies, Inc.  All rights reserved.
+ * Copyright (c) 2016      Intel, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+
+#include <stdio.h>
+#include <pmix.h>
+
+pmix_proc_t this_proc;
+
+void pmi_init(int *rank, int *size)
+{
+    pmix_value_t value, *val = &value;
+    int rc;
+
+    /* init us */
+#if (PMIX_VERSION_MAJOR == 1 )
+    if (PMIX_SUCCESS != (rc = PMIx_Init(&this_proc)))
+#else
+    if (PMIX_SUCCESS != (rc = PMIx_Init(&this_proc, NULL, 0)))
+#endif
+    {
+        fprintf(stderr, "Client ns %s rank %d: PMIx_Init failed: %d", this_proc.nspace, this_proc.rank, rc);
+        abort();
+    }
+
+    /* get our job size */
+    if (PMIX_SUCCESS != (rc = PMIx_Get(&this_proc, PMIX_JOB_SIZE, NULL, 0, &val))) {
+        fprintf(stderr, "Client ns %s rank %d: PMIx_Get job size failed: %d", this_proc.nspace, this_proc.rank, rc);
+        abort();
+    }
+    *size = val->data.uint32;
+    *rank = this_proc.rank;
+    PMIX_VALUE_RELEASE(val);
+}
+
+void pmi_get_local_ranks(int **local_ranks, int *local_cnt)
+{
+    pmix_value_t value, *val = &value;
+    char *ptr;
+    int i, rc;
+
+    /* get our job size */
+    if (PMIX_SUCCESS != (rc = PMIx_Get(&this_proc, PMIX_LOCAL_SIZE, NULL, 0, &val))) {
+        fprintf(stderr, "Client ns %s rank %d: PMIx_Get PMIX_LOCAL_SIZE failed: %d", this_proc.nspace, this_proc.rank, rc);
+        abort();
+    }
+    *local_cnt = val->data.uint32;
+    PMIX_VALUE_RELEASE(val);
+
+    *local_ranks = calloc(*local_cnt, sizeof(int));
+    /* get our job size */
+    if (PMIX_SUCCESS != (rc = PMIx_Get(&this_proc, PMIX_LOCAL_PEERS, NULL, 0, &val))) {
+        fprintf(stderr, "Client ns %s rank %d: PMIx_Get PMIX_LOCAL_PEERS failed: %d", this_proc.nspace, this_proc.rank, rc);
+        abort();
+    }
+    ptr = val->data.string;
+    for(i=0; NULL != ptr && i < *local_cnt; i++ ){
+         char *loc_rank = strsep(&ptr, ",");
+         (*local_ranks)[i] = atoi(loc_rank);
+    }
+    if( i != *local_cnt || NULL != ptr ){
+        fprintf(stderr, "Client ns %s rank %d: number of local peers doesn't match",
+                this_proc.nspace, this_proc.rank);
+        abort();
+    }
+}
+
+static void _put_key(char *key, int *key_val, int key_size, pmix_scope_t scope)
+{
+    pmix_value_t value;
+    int rc;
+
+    value.type = PMIX_BYTE_OBJECT;
+    value.data.bo.size = key_size * sizeof(int);
+    value.data.bo.bytes = (char*)key_val;
+    if (PMIX_SUCCESS != (rc = PMIx_Put(scope, key, &value))) {
+        fprintf(stderr, "Client ns %s rank %d: PMIx_Put internal failed: %d", this_proc.nspace, this_proc.rank, rc);
+        abort();
+    }
+}
+
+void pmi_put_key_loc(char *key, int *key_val, int key_size)
+{
+    _put_key(key, key_val, key_size, PMIX_LOCAL);
+}
+
+void pmi_put_key_rem(char *key, int *key_val, int key_size)
+{
+    _put_key(key, key_val, key_size, PMIX_REMOTE);
+}
+
+void pmi_put_double(char *key, double v)
+{
+    pmix_value_t value;
+    int rc;
+
+    value.type = PMIX_DOUBLE;
+    value.data.dval = v;
+    if (PMIX_SUCCESS != (rc = PMIx_Put(PMIX_GLOBAL, key, &value))) {
+        fprintf(stderr, "Client ns %s rank %d: PMIx_Put internal failed: %d", this_proc.nspace, this_proc.rank, rc);
+        abort();
+    }
+}
+
+void pmi_commit()
+{
+    int rc;
+
+    if (PMIX_SUCCESS != (rc = PMIx_Commit())) {
+        fprintf(stderr, "Client ns %s rank %d: PMIx_Commit failed: %d",
+                this_proc.nspace, this_proc.rank, rc);
+        abort();
+    }
+}
+
+
+void pmi_fence(int collect)
+{
+    pmix_info_t *info = NULL;
+    pmix_proc_t proc;
+    bool value = 1;
+    int ninfo = 0;
+    int rc;
+
+    if( collect ){
+        PMIX_INFO_CREATE(info, 1);
+        (void)strncpy(info->key, PMIX_COLLECT_DATA, PMIX_MAX_KEYLEN);
+        pmix_value_load(&info->value, &value, PMIX_BOOL);
+        ninfo = 1;
+    }
+
+    /* call fence to ensure the data is received */
+    PMIX_PROC_CONSTRUCT(&proc);
+    (void)strncpy(proc.nspace, this_proc.nspace, PMIX_MAX_NSLEN);
+    proc.rank = PMIX_RANK_WILDCARD;
+
+    if (PMIX_SUCCESS != (rc = PMIx_Fence(&proc, 1, info, ninfo))) {
+        fprintf(stderr,  "Client ns %s rank %d: PMIx_Fence failed: %d",
+                this_proc.nspace, this_proc.rank, rc);
+        abort();
+    }
+
+    if( collect ){
+        PMIX_INFO_FREE(info, ninfo);
+    }
+}
+
+void _get_key(int rank, char *key_name, int **key_val, int *key_size)
+{
+    pmix_proc_t proc;
+    pmix_value_t value, *val = &value;
+    int rc;
+
+    (void)strncpy(proc.nspace, this_proc.nspace, PMIX_MAX_NSLEN);
+    proc.rank = rank;
+    if (PMIX_SUCCESS != (rc = PMIx_Get(&proc, key_name, NULL, 0, &val))) {
+        fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s failed: %d",
+                this_proc.nspace, this_proc.rank, key_name, rc);
+        abort();
+    }
+    if (PMIX_BYTE_OBJECT != val->type) {
+        fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s returned wrong type: %d",
+                this_proc.nspace, this_proc.rank, key_name, val->type);
+        PMIX_VALUE_RELEASE(val);
+        abort();
+    }
+    *key_val = (int*)val->data.bo.bytes;
+    *key_size = val->data.bo.size / sizeof(int);
+    val->data.bo.bytes = NULL;
+    PMIX_VALUE_RELEASE(val);
+}
+
+void pmi_get_key_loc(int rank, char *key_name, int **key_val, int *key_size)
+{
+    _get_key(rank, key_name, key_val, key_size);
+}
+
+void pmi_get_key_rem(int rank, char *key_name, int **key_val, int *key_size)
+{
+    _get_key(rank, key_name, key_val, key_size);
+}
+
+double pmi_get_double(int rank, char *key_name)
+{
+    pmix_proc_t proc;
+    pmix_value_t value, *val = &value;
+    int rc;
+    double v;
+
+    (void)strncpy(proc.nspace, this_proc.nspace, PMIX_MAX_NSLEN);
+    proc.rank = rank;
+    if (PMIX_SUCCESS != (rc = PMIx_Get(&proc, key_name, NULL, 0, &val))) {
+        fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s failed: %d",
+                this_proc.nspace, this_proc.rank, key_name, rc);
+        abort();
+    }
+    if (PMIX_DOUBLE != val->type) {
+        fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s returned wrong type: %d",
+                this_proc.nspace, this_proc.rank, key_name, val->type);
+        PMIX_VALUE_RELEASE(val);
+        abort();
+    }
+    v = val->data.dval;
+    PMIX_VALUE_RELEASE(val);
+    return v;
+}
+
+pmi_fini()
+{
+    int rc;
+#if (PMIX_VERSION_MAJOR == 1 )
+    if (PMIX_SUCCESS != (rc = PMIx_Finalize()))
+#else
+    if (PMIX_SUCCESS != (rc = PMIx_Finalize(NULL, 0)))
+#endif
+    {
+        fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize failed: %d\n", this_proc.nspace, this_proc.rank, rc);
+        abort();
+    }
+}

opal/mca/pmix/pmix2x/pmix/contrib/platform/optimized

@@ -0,0 +1,3 @@
+enable_mem_debug=no
+enable_mem_profile=no
+enable_debug=no

opal/mca/pmix/pmix2x/pmix/contrib/pmix-release.sh

@@ -0,0 +1,52 @@
+#!/bin/sh -x
+
+# The tarballs to make
+if [ $# -eq 0 ] ; then
+    branches="v1.0"
+else
+    branches=$1
+    shift
+fi
+
+# Build root - scratch space
+build_root=/home/mpiteam/pmix/release
+
+# Script to execute
+script=contrib/make_dist_tarball
+
+export PATH=$HOME/local/bin:$PATH
+export LD_LIBRARY_PATH=$HOME/local/lib:$LD_LIBRARY_PATH
+
+#####
+#
+# Actually do stuff
+#
+#####
+
+# load the modules configuration
+. /etc/profile.d/modules.sh
+module use ~/modules
+
+# move to the directory
+# Loop making them
+for branch in $branches; do
+    cd $build_root/$branch
+
+    module load "autotools/pmix-$branch"
+    module load libevent/pmix-$branch
+
+    ./$script $@ >dist.out 2>&1
+        if test "$?" != "0"; then
+                cat <<EOF
+=============================================================================
+== Dist failure
+== Last few lines of output (full results in dist.out file):
+=============================================================================
+EOF
+                tail -n 20 dist.out
+                exit 1
+        fi
+
+    module unload libevent
+    module unload autotools
+done

opal/mca/pmix/pmix2x/pmix/contrib/pmix.spec

@@ -0,0 +1,685 @@
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2006-2014 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2013      Mellanox Technologies, Inc.
+#                         All rights reserved.
+# Copyright (c) 2015-2016 Intel, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+############################################################################
+#
+# Copyright (c) 2003, The Regents of the University of California, through
+# Lawrence Berkeley National Laboratory (subject to receipt of any
+# required approvals from the U.S. Dept. of Energy).  All rights reserved.
+#
+# Initially written by:
+#       Greg Kurtzer, <gmkurtzer@lbl.gov>
+#
+############################################################################
+
+
+#############################################################################
+#
+# Configuration Options
+#
+# Options that can be passed in via rpmbuild's --define option.  Note
+# that --define takes *1* argument: a multi-token string where the first
+# token is the name of the variable to define, and all remaining tokens
+# are the value.  For example:
+#
+# shell$ rpmbuild ... --define 'install_in_opt 1' ...
+#
+# Or (a multi-token example):
+#
+# shell$ rpmbuild ... \
+#    --define 'configure_options CFLAGS=-g --with-openib=/usr/local/ofed' ...
+#
+#############################################################################
+
+# Define this if you want to make this SRPM build in
+# /opt/NAME/VERSION-RELEASE instead of the default /usr/.
+# type: bool (0/1)
+%{!?install_in_opt: %define install_in_opt 0}
+
+# Define this if you want this RPM to install environment setup
+# shell scripts.
+# type: bool (0/1)
+%{!?install_shell_scripts: %define install_shell_scripts 0}
+# type: string (root path to install shell scripts)
+%{!?shell_scripts_path: %define shell_scripts_path %{_bindir}}
+# type: string (base name of the shell scripts)
+%{!?shell_scripts_basename: %define shell_scripts_basename mpivars}
+
+# Define this to 1 if you want this RPM to install a modulefile.
+# type: bool (0/1)
+%{!?install_modulefile: %define install_modulefile 0}
+# type: string (root path to install modulefiles)
+%{!?modulefile_path: %define modulefile_path /usr/share/Modules/modulefiles}
+# type: string (subdir to install modulefile)
+%{!?modulefile_subdir: %define modulefile_subdir %{name}}
+# type: string (name of modulefile)
+%{!?modulefile_name: %define modulefile_name %{version}}
+
+# The name of the modules RPM.  Can vary from system to system.
+# RHEL6 calls it "environment-modules".
+# type: string (name of modules RPM)
+%{!?modules_rpm_name: %define modules_rpm_name environment-modules}
+
+# Should we use the mpi-selector functionality?
+# type: bool (0/1)
+%{!?use_mpi_selector: %define use_mpi_selector 0}
+# The name of the mpi-selector RPM.  Can vary from system to system.
+# type: string (name of mpi-selector RPM)
+%{!?mpi_selector_rpm_name: %define mpi_selector_rpm_name mpi-selector}
+# The location of the mpi-selector executable (can be a relative path
+# name if "mpi-selector" can be found in the path)
+# type: string (path to mpi-selector exectuable)
+%{!?mpi_selector: %define mpi_selector mpi-selector}
+
+# Should we build a debuginfo RPM or not?
+# type: bool (0/1)
+%{!?build_debuginfo_rpm: %define build_debuginfo_rpm 0}
+
+# Should we build an all-in-one RPM, or several sub-package RPMs?
+# type: bool (0/1)
+%{!?build_all_in_one_rpm: %define build_all_in_one_rpm 1}
+
+# Should we use the default "check_files" RPM step (i.e., check for
+# unpackaged files)?  It is discouraged to disable this, but some
+# installers need it (e.g., older versions of OFED, because they
+# installed lots of other stuff in the BUILD_ROOT before PMIx/SHMEM).
+# type: bool (0/1)
+%{!?use_check_files: %define use_check_files 1}
+
+# By default, RPM supplies a bunch of optimization flags, some of
+# which may not work with non-gcc compilers.  We attempt to weed some
+# of these out (below), but sometimes it's better to just ignore them
+# altogether (e.g., PGI 6.2 will warn about unknown compiler flags,
+# but PGI 7.0 will error -- and RPM_OPT_FLAGS contains a lot of flags
+# that PGI 7.0 does not understand).  The default is to use the flags,
+# but you can set this variable to 0, indicating that RPM_OPT_FLAGS
+# should be erased (in which case you probabl want to supply your own
+# optimization flags!).
+# type: bool (0/1)
+%{!?use_default_rpm_opt_flags: %define use_default_rpm_opt_flags 1}
+
+# Some compilers can be installed via tarball or RPM (e.g., Intel,
+# PGI).  If they're installed via RPM, then rpmbuild's auto-dependency
+# generation stuff will work fine.  But if they're installed via
+# tarball, then rpmbuild's auto-dependency generation stuff will
+# break; complaining that it can't find a bunch of compiler .so files.
+# So provide an option to turn this stuff off.
+# type: bool (0/1)
+%{!?disable_auto_requires: %define disable_auto_requires 0}
+
+# On some platforms, PMIx/SHMEM just flat-out doesn't work with
+# -D_FORTIFY_SOURCE (e.g., some users have reported that there are
+# problems on ioa64 platforms).  In this case, just turn it off
+# (meaning: this specfile will strip out that flag from the
+# OS-provided compiler flags).  We already strip out _FORTIFY_SOURCE
+# for non-GCC compilers; setting this option to 0 will *always* strip
+# it out, even if you're using GCC.
+# type: bool (0/1)
+%{!?allow_fortify_source: %define allow_fortify_source 1}
+
+# Select md5 packing algorithm, that src.rpm created on one distro can be read on another.
+%global _binary_filedigest_algorithm 1
+%global _source_filedigest_algorithm 1
+
+#############################################################################
+#
+# Configuration Logic
+#
+#############################################################################
+
+%if %{install_in_opt}
+%define _prefix /opt/%{name}/%{version}
+%define _sysconfdir /opt/%{name}/%{version}/etc
+%define _libdir /opt/%{name}/%{version}/lib
+%define _includedir /opt/%{name}/%{version}/include
+%define _mandir /opt/%{name}/%{version}/man
+# Note that the name "pmix" is hard-coded in
+# opal/mca/installdirs/config for pkgdatadir; there is currently no
+# easy way to have PMIx change this directory name internally.  So we
+# just hard-code that name here as well (regardless of the value of
+# %{name} or %{_name}).
+%define _pkgdatadir /opt/%{name}/%{version}/share/pmix
+# Per advice from Doug Ledford at Red Hat, docdir is supposed to be in
+# a fixed location.  But if you're installing a package in /opt, all
+# bets are off.  So feel free to install it anywhere in your tree.  He
+# suggests $prefix/doc.
+%define _defaultdocdir /opt/%{name}/%{version}/doc
+%endif
+
+%if !%{build_debuginfo_rpm}
+%define debug_package %{nil}
+%endif
+
+%if %(test "%{_prefix}" = "/usr" && echo 1 || echo 0)
+%global _sysconfdir /etc
+%else
+%global _sysconfdir %{_prefix}/etc
+%endif
+
+# Is the sysconfdir under the prefix directory?  This affects
+# whether we list the sysconfdir separately in the files sections,
+# below.
+%define sysconfdir_in_prefix %(test "`echo %{_sysconfdir} | grep %{_prefix}`" = "" && echo 0 || echo 1)
+
+%{!?_pkgdatadir: %define _pkgdatadir %{_datadir}/pmix}
+
+%if !%{use_check_files}
+%define __check_files %{nil}
+%endif
+
+%{!?configure_options: %define configure_options %{nil}}
+
+%if !%{use_default_rpm_opt_flags}
+%define optflags ""
+%endif
+
+%if %{use_mpi_selector}
+%define install_shell_scripts 1
+%endif
+
+#############################################################################
+#
+# Preamble Section
+#
+#############################################################################
+
+Summary: An extended/exascale implementation of PMI
+Name: %{?_name:%{_name}}%{!?_name:pmix}
+Version: $VERSION
+Release: 1%{?dist}
+License: BSD
+Group: Development/Libraries
+Source: pmix-%{version}.tar.$EXTENSION
+Packager: %{?_packager:%{_packager}}%{!?_packager:%{_vendor}}
+Vendor: %{?_vendorinfo:%{_vendorinfo}}%{!?_vendorinfo:%{_vendor}}
+Distribution: %{?_distribution:%{_distribution}}%{!?_distribution:%{_vendor}}
+Prefix: %{_prefix}
+Provides: mpi
+Provides: pmix = %{version}
+BuildRoot: /var/tmp/%{name}-%{version}-%{release}-root
+%if %{disable_auto_requires}
+AutoReq: no
+%endif
+%if %{install_modulefile}
+Requires: %{modules_rpm_name}
+%endif
+%if %{use_mpi_selector}
+Requires: %{mpi_selector_rpm_name}
+%endif
+
+%description
+The Process Management Interface (PMI) has been used for quite some time as a
+means of exchanging wireup information needed for interprocess communication. Two
+versions (PMI-1 and PMI-2) have been released as part of the MPICH effort. While
+PMI-2 demonstrates better scaling properties than its PMI-1 predecessor, attaining
+rapid launch and wireup of the roughly 1M processes executing across 100k nodes
+expected for exascale operations remains challenging.
+
+PMI Exascale (PMIx) represents an attempt to resolve these questions by providing
+an extended version of the PMI standard specifically designed to support clusters
+up to and including exascale sizes. The overall objective of the project is not to
+branch the existing pseudo-standard definitions - in fact, PMIx fully supports both
+of the existing PMI-1 and PMI-2 APIs - but rather to (a) augment and extend those
+APIs to eliminate some current restrictions that impact scalability, and (b) provide
+a reference implementation of the PMI-server that demonstrates the desired level of
+scalability.
+
+This RPM contains all the tools necessary to compile and link against PMIx.
+
+#############################################################################
+#
+# Prepatory Section
+#
+#############################################################################
+%prep
+# Unbelievably, some versions of RPM do not first delete the previous
+# installation root (e.g., it may have been left over from a prior
+# failed build).  This can lead to Badness later if there's files in
+# there that are not meant to be packaged.
+rm -rf $RPM_BUILD_ROOT
+
+%setup -q -n pmix-%{version}
+
+#############################################################################
+#
+# Build Section
+#
+#############################################################################
+
+%build
+
+# rpmbuild processes seem to be geared towards the GNU compilers --
+# they pass in some flags that will only work with gcc.  So if we're
+# trying to build with some other compiler, the process will choke.
+# This is *not* something the user can override with a well-placed
+# --define on the rpmbuild command line, unless they find and override
+# all "global" CFLAGS kinds of RPM macros (every distro names them
+# differently).  For example, non-gcc compilers cannot use
+# FORTIFY_SOURCE (at least, not as of 6 Oct 2006).  We can really only
+# examine the basename of the compiler, so search for it in a few
+# places.
+
+%if %{allow_fortify_source}
+using_gcc=1
+if test "$CC" != ""; then
+    # Do horrible things to get the basename of just the compiler,
+    # particularly in the case of multword values for $CC
+    eval "set $CC"
+    if test "`basename $1`" != "gcc"; then
+        using_gcc=0
+    fi
+fi
+
+if test "$using_gcc" = "1"; then
+    # Do wretched things to find a CC=* token
+    eval "set -- %{configure_options}"
+    compiler=
+    while test "$1" != "" -a "$compiler" = ""; do
+         case "$1" in
+         CC=*)
+                 compiler=`echo $1 | cut -d= -f2-`
+                 ;;
+         esac
+         shift
+    done
+
+    # Now that we *might* have the compiler name, do a best-faith
+    # effort to see if it's gcc.  Blah!
+    if test "$compiler" != ""; then
+        if test "`basename $compiler`" != "gcc"; then
+            using_gcc=0
+        fi
+    fi
+fi
+%else
+# If we're not allowing _FORTIFY_SOURCE, then just set using_gcc to 0 and
+# the logic below will strip _FORTIFY_SOURCE out if it's present.
+using_gcc=0
+%endif
+
+# If we're not using the default RPM_OPT_FLAGS, then wipe them clean
+# (the "optflags" macro has already been wiped clean, above).
+
+%if !%{use_default_rpm_opt_flags}
+RPM_OPT_FLAGS=
+export RPM_OPT_FLAGS
+%endif
+
+# If we're not GCC, strip out any GCC-specific arguments in the
+# RPM_OPT_FLAGS before potentially propagating them everywhere.
+
+if test "$using_gcc" = 0; then
+
+    # Non-gcc compilers cannot handle FORTIFY_SOURCE (at least, not as
+    # of Oct 2006)
+    RPM_OPT_FLAGS="`echo $RPM_OPT_FLAGS | sed -e 's@-D_FORTIFY_SOURCE[=0-9]*@@'`"
+
+    # Non-gcc compilers will generate warnings for several flags
+    # placed in RPM_OPT_FLAGS by RHEL5, but -mtune=generic will cause
+    # an error for icc 9.1.
+    RPM_OPT_FLAGS="`echo $RPM_OPT_FLAGS | sed -e 's@-mtune=generic@@'`"
+fi
+
+CFLAGS="%{?cflags:%{cflags}}%{!?cflags:$RPM_OPT_FLAGS}"
+CXXFLAGS="%{?cxxflags:%{cxxflags}}%{!?cxxflags:$RPM_OPT_FLAGS}"
+FFLAGS="%{?f77flags:%{f77flags}}%{!?f7flags:$RPM_OPT_FLAGS}"
+FCFLAGS="%{?fcflags:%{fcflags}}%{!?fcflags:$RPM_OPT_FLAGS}"
+export CFLAGS CXXFLAGS F77FLAGS FCFLAGS
+
+%configure %{configure_options}
+%{__make} %{?mflags}
+
+
+#############################################################################
+#
+# Install Section
+#
+#############################################################################
+
+%install
+%{__make} install DESTDIR=$RPM_BUILD_ROOT %{?mflags_install}
+
+# We've had cases of config.log being left in the installation tree.
+# We don't need that in an RPM.
+find $RPM_BUILD_ROOT -name config.log -exec rm -f {} \;
+
+# First, the [optional] modulefile
+
+%if %{install_modulefile}
+%{__mkdir_p} $RPM_BUILD_ROOT/%{modulefile_path}/%{modulefile_subdir}/
+cat <<EOF >$RPM_BUILD_ROOT/%{modulefile_path}/%{modulefile_subdir}/%{modulefile_name}
+#%Module
+
+# NOTE: This is an automatically-generated file!  (generated by the
+# PMIx/SHMEM RPM).  Any changes made here will be lost a) if the RPM is
+# uninstalled, or b) if the RPM is upgraded or uninstalled.
+
+proc ModulesHelp { } {
+   puts stderr "This module adds PMIx/SHMEM v%{version} to various paths"
+}
+
+module-whatis   "Sets up PMIx/SHMEM v%{version} in your enviornment"
+
+prepend-path PATH "%{_prefix}/bin/"
+prepend-path LD_LIBRARY_PATH %{_libdir}
+prepend-path MANPATH %{_mandir}
+EOF
+%endif
+# End of modulefile if
+
+# Next, the [optional] shell scripts
+
+%if %{install_shell_scripts}
+%{__mkdir_p} $RPM_BUILD_ROOT/%{shell_scripts_path}
+cat <<EOF > $RPM_BUILD_ROOT/%{shell_scripts_path}/%{shell_scripts_basename}.sh
+# NOTE: This is an automatically-generated file!  (generated by the
+# PMIx/SHMEM RPM).  Any changes made here will be lost if the RPM is
+# uninstalled or upgraded.
+
+# PATH
+if test -z "\`echo \$PATH | grep %{_bindir}\`"; then
+    PATH=%{_bindir}:\${PATH}
+    export PATH
+fi
+
+# LD_LIBRARY_PATH
+if test -z "\`echo \$LD_LIBRARY_PATH | grep %{_libdir}\`"; then
+    LD_LIBRARY_PATH=%{_libdir}\${LD_LIBRARY_PATH:+:}\${LD_LIBRARY_PATH}
+    export LD_LIBRARY_PATH
+fi
+
+# MANPATH
+if test -z "\`echo \$MANPATH | grep %{_mandir}\`"; then
+    MANPATH=%{_mandir}:\${MANPATH}
+    export MANPATH
+fi
+
+# MPI_ROOT
+MPI_ROOT=%{_prefix}
+export MPI_ROOT
+EOF
+cat <<EOF > $RPM_BUILD_ROOT/%{shell_scripts_path}/%{shell_scripts_basename}.csh
+# NOTE: This is an automatically-generated file!  (generated by the
+# PMIx/SHMEM RPM).  Any changes made here will be lost if the RPM is
+# uninstalled or upgraded.
+
+# path
+if ("" == "\`echo \$path | grep %{_bindir}\`") then
+    set path=(%{_bindir} \$path)
+endif
+
+# LD_LIBRARY_PATH
+if ("1" == "\$?LD_LIBRARY_PATH") then
+    if ("\$LD_LIBRARY_PATH" !~ *%{_libdir}*) then
+        setenv LD_LIBRARY_PATH %{_libdir}:\${LD_LIBRARY_PATH}
+    endif
+else
+    setenv LD_LIBRARY_PATH %{_libdir}
+endif
+
+# MANPATH
+if ("1" == "\$?MANPATH") then
+    if ("\$MANPATH" !~ *%{_mandir}*) then
+        setenv MANPATH %{_mandir}:\${MANPATH}
+    endif
+else
+    setenv MANPATH %{_mandir}:
+endif
+
+# MPI_ROOT
+setenv MPI_ROOT %{_prefix}
+EOF
+%endif
+# End of shell_scripts if
+
+#############################################################################
+#
+# Clean Section
+#
+#############################################################################
+%clean
+# We may be in the directory that we're about to remove, so cd out of
+# there before we remove it
+cd /tmp
+
+# Remove installed driver after rpm build finished
+rm -rf $RPM_BUILD_DIR/%{name}-%{version}
+
+test "x$RPM_BUILD_ROOT" != "x" && rm -rf $RPM_BUILD_ROOT
+
+#############################################################################
+#
+# Post Install Section
+#
+#############################################################################
+%if %{use_mpi_selector}
+%post
+%{mpi_selector} \
+        --register %{name}-%{version} \
+        --source-dir %{shell_scripts_path} \
+        --yes
+%endif
+
+#############################################################################
+#
+# Pre Uninstall Section
+#
+#############################################################################
+%if %{use_mpi_selector}
+%preun
+%{mpi_selector} --unregister %{name}-%{version} --yes || \
+      /bin/true > /dev/null 2> /dev/null
+%endif
+
+#############################################################################
+#
+# Files Section
+#
+#############################################################################
+
+#
+# All in one RPM
+#
+# Easy; just list the prefix and then specifically call out the doc
+# files.
+#
+
+%files
+%defattr(-, root, root, -)
+%{_prefix}
+# If the sysconfdir is not under the prefix, then list it explicitly.
+%if !%{sysconfdir_in_prefix}
+%{_sysconfdir}
+%endif
+# If %{install_in_opt}, then we're installing PMIx to
+# /opt/pmix/<version>.  But be sure to also explicitly mention
+# /opt/pmix so that it can be removed by RPM when everything under
+# there is also removed.
+%if %{install_in_opt}
+%dir /opt/%{name}
+%endif
+# If we're installing the modulefile, get that, too
+%if %{install_modulefile}
+%{modulefile_path}
+%endif
+# If we're installing the shell scripts, get those, too
+%if %{install_shell_scripts}
+%{shell_scripts_path}/%{shell_scripts_basename}.sh
+%{shell_scripts_path}/%{shell_scripts_basename}.csh
+%endif
+%doc README INSTALL LICENSE
+
+#############################################################################
+#
+# Changelog
+#
+#############################################################################
+%changelog
+* Fri Jun 19 2015 Ralph H. Castain <rhc@open-mpi.org>
+- Port to PMIx
+
+* Tue Jan 20 2015 Bert Wesarg <bert.wesarg@tu-dresden.de>
+- Remove VampirTrace wrapper from package.
+
+* Mon Jul 07 2014 Jeff Squyres <jsquyres@cisco.com>
+- Several minor fixes from Oliver Lahaye: fix dates in changelog,
+  added %{?dist} tag to the Release field, and added some Provides
+  fields in case %{name} is overridden.
+
+* Mon Jun 24 2013 Igor Ivanov <Igor.Ivanov@itseez.com>
+- Add Open SHMEM parallel programming library as part of Open MPI
+
+* Tue Dec 11 2012 Jeff Squyres <jsquyres@cisco.com>
+- Re-release 1.6.0-1.6.3 SRPMs (with new SRPM Release numbers) with
+  patch for VampirTrace's configure script to make it install the
+  private "libtool" script in the right location (the script is used
+  to build user VT applications).
+- Update the regexps/methodology used to generate the lists of files
+  in the multi-RPM sub-packages; it's been broken for a little while.
+- No longer explicitly list the bin dir executables in the multi-RPM
+  sub-packages
+- Per https://svn.open-mpi.org/trac/ompi/ticket/3382, remove all files
+  named "config.log" from the install tree so that we can use this
+  spec file to re-release all OMPI v1.6.x SRPMs.
+
+* Wed Jun 27 2012 Jeff Squyres <jsquyres@cisco.com>
+- Remove the "ofed" and "munge_build_into_install" options, because
+  OFED no longer distributes MPI implementations.  Yay!
+
+* Mon Jun 04 2012 Jeff Squyres <jsquyres@cisco.com>
+- Didn't change the specfile, but changed the script that generates
+  the SRPM to force the use of MD5 checksums (vs. SHA1 checksums) so
+  that the SRPM is friendly to older versions of RPM, such as that on
+  RHEL 5.x.
+
+* Fri Feb 17 2012 Jeff Squyres <jsquyres@cisco.com>
+- Removed OSCAR defines.
+- If use_mpi_selector==1, then also set install_shell_scripts to 1.
+- Change modules default RPM name and modulefiles path to the defaults
+  on RHEL6.
+
+* Mon Dec 14 2009 Jeff Squyres <jsquyres@cisco.com>
+- Add missing executables to specfile (ompi-server, etc.)
+- Fix: pull in VT files when building multiple RPMs (reported by Jim
+  Kusznir).
+- Add allow_fortify_source option to let users selectively disable
+  _FORTIFY_SOURCE processing on platforms where it just doesn't work
+  (even with gcc; also reported by Jim Kusznir).
+
+* Tue Sep  8 2009 Jeff Squyres <jsquyres@cisco.com>
+- Change shell_scripts_basename to not include version number to
+  accomodate what mpi-selector expects.
+
+* Mon Feb  4 2008 Jeff Squyres <jsquyres@cisco.com>
+- OFED 1.3 has a much better installer; remove all the
+  leave_build_root kludge nastyness.  W00t!
+
+* Fri Jan 18 2008 Jeff Squyres <jsquyres@cisco.com>
+- Remove the hard-coded "pmix" name from two Requires statements
+  and use %{name} instead (FWIW, %{_name} caused rpmbuild to barf).
+
+* Wed Jan  2 2008 Jeff Squyres <jsquyres@cisco.com>
+- Remove duplicate %{_sysconfdir} in the % files sections when
+  building the sub-packages.
+- When building the sub-packages, ensure that devel.files also picks
+  up the F90 module.
+- Hard-code the directory name "openmpi" into _pkglibdir (vs. using
+  %{name}) because the OMPI code base has it hard-coded as well.
+  Thanks to Jim Kusznir for noticing the problem.
+
+* Tue Dec  4 2007 Jeff Squyres <jsquyres@cisco.com>
+- Added define option for disabling the use of rpmbuild's
+  auto-dependency generation stuff.  This is necessary for some
+  compilers that allow themselves to be installed via tarball (not
+  RPM), such as the Portland Group compiler.
+
+* Thu Jul 12 2007 Jeff Squyres <jsquyres@cisco.com>
+- Change default doc location when using install_in_opt.  Thanks to
+  Alex Tumanov for pointing this out and to Doug Ledford for
+  suggestions where to put docdir in this case.
+
+* Thu May  3 2007 Jeff Squyres <jsquyres@cisco.com>
+- Ensure to move out of $RPM_BUILD_ROOT before deleting it in % clean.
+- Remove a debugging "echo" that somehow got left in there
+
+* Thu Apr 12 2007 Jeff Squyres <jsquyres@cisco.com>
+- Ensure that _pkglibdir is always defined, suggested by Greg Kurtzer.
+
+* Wed Apr  4 2007 Jeff Squyres <jsquyres@cisco.com>
+- Fix several mistakes in the generated profile.d scripts
+- Fix several bugs with identifying non-GNU compilers, stripping of
+  FORTIFY_SOURCE, -mtune, etc.
+
+* Fri Feb  9 2007 Jeff Squyres <jsquyres@cisco.com>
+- Revamp to make profile.d scripts more general: default to making the
+  shell script files be %{_bindir}/mpivars.{sh|csh}
+- Add %{munge_build_into_install} option for OFED 1.2 installer on SLES
+- Change shell script files and modulefile to *pre*pend all the OMPI paths
+- Make shell script and modulefile installation indepdendent of
+  %{install_in_opt} (they're really separate issues)
+- Add more "ofed" shortcut qualifiers
+- Slightly better test for basename CC in the fortify source section
+- Fix some problems in the csh shell script
+
+* Fri Oct  6 2006 Jeff Squyres <jsquyres@cisco.com>
+- Remove LANL section; they don't want it
+- Add some help for OFED building
+- Remove some outdated "rm -f" lines for executables that we no longer ship
+
+* Wed Apr 26 2006 Jeff Squyres <jsquyres@cisco.com>
+- Revamp files listings to ensure that rpm -e will remove directories
+  if rpm -i created them.
+- Simplify options for making modulefiles and profile.d scripts.
+- Add oscar define.
+- Ensure to remove the previous installation root during prep.
+- Cleanup the modulefile specification and installation; also ensure
+  that the profile.d scripts get installed if selected.
+- Ensure to list sysconfdir in the files list if it's outside of the
+  prefix.
+
+* Thu Mar 30 2006 Jeff Squyres <jsquyres@cisco.com>
+- Lots of bit rot updates
+- Reorganize and rename the subpackages
+- Add / formalize a variety of rpmbuild --define options
+- Comment out the docs subpackage for the moment (until we have some
+  documentation -- coming in v1.1!)
+
+* Tue May 03 2005 Jeff Squyres <jsquyres@open-mpi.org>
+- Added some defines for LANL defaults
+- Added more defines for granulatirty of installation location for
+  modulefile
+- Differentiate between installing in /opt and whether we want to
+  install environment script files
+- Filled in files for man and mca-general subpackages
+
+* Thu Apr 07 2005 Greg Kurtzer <GMKurtzer@lbl.gov>
+- Added opt building
+- Added profile.d/modulefile logic and creation
+- Minor cleanups
+
+* Fri Apr 01 2005 Greg Kurtzer <GMKurtzer@lbl.gov>
+- Added comments
+- Split package into subpackages
+- Cleaned things up a bit
+- Sold the code to Microsoft, and now I am retiring. Thanks guys!
+
+* Wed Mar 23 2005 Mezzanine <mezzanine@kainx.org>
+- Specfile auto-generated by Mezzanine

opal/mca/pmix/pmix2x/pmix/contrib/pmix_jenkins.sh

@@ -0,0 +1,397 @@
+#!/bin/bash -xeE
+export PATH=/hpc/local/bin::/usr/local/bin:/bin:/usr/bin:/usr/sbin:${PATH}
+
+rel_path=$(dirname $0)
+abs_path=$(readlink -f $rel_path)
+
+jenkins_test_build=${jenkins_test_build:="yes"}
+jenkins_test_check=${jenkins_test_check:="yes"}
+jenkins_test_src_rpm=${jenkins_test_src_rpm:="yes"}
+jenkins_test_cov=${jenkins_test_cov:="yes"}
+jenkins_test_comments=${jenkins_test_comments:="no"}
+jenkins_test_vg=${jenkins_test_vg:="no"}
+
+timeout_exe=${timout_exe:="timeout -s SIGKILL 1m"}
+
+# prepare to run from command line w/o jenkins
+if [ -z "$WORKSPACE" ]; then
+    WORKSPACE=$PWD
+    JOB_URL=$WORKSPACE
+    BUILD_NUMBER=1
+    JENKINS_RUN_TESTS=yes
+    NOJENKINS=${NOJENKINS:="yes"}
+fi
+
+prefix=jenkins
+rm -rf ${WORKSPACE}/${prefix}
+mkdir -p ${WORKSPACE}/${prefix}
+pmix_dir=${WORKSPACE}/${prefix}/install
+build_dir=${WORKSPACE}/${prefix}/build
+rpm_dir=${WORKSPACE}/${prefix}/rpms
+cov_dir=${WORKSPACE}/${prefix}/cov
+tarball_dir=${WORKSPACE}/${prefix}/tarball
+
+
+make_opt="-j$(nproc)"
+
+# extract jenkins commands from function args
+function check_commands
+{
+    local cmd=$1
+    local pat=""
+    local test_list="threads src_rpm oshmem check help_txt known_issues cov all"
+    for pat in $(echo $test_list); do
+        echo -n "checking $pat "
+        if [[ $cmd =~ jenkins\:.*no${pat}.* ]]; then
+            echo disabling
+            eval "jenkins_test_${pat}=no"
+        elif [[ $cmd =~ jenkins\:.*${pat}.* ]]; then
+            echo enabling
+            eval "jenkins_test_${pat}=yes"
+        else
+            echo no directive for ${pat}
+        fi
+    done
+
+    if [ "$jenkins_test_all" = "yes" ]; then
+        echo Enabling all tests
+        for pat in $(echo $test_list); do
+            eval "jenkins_test_${pat}=yes"
+        done
+    fi
+}
+
+function test_cov
+{
+    local cov_root_dir=$1
+    local cov_proj=$2
+    local cov_make_cmd=$3
+    local cov_directive=$4
+
+    local nerrors=0;
+
+    module load tools/cov
+
+    local cov_build_dir=$cov_dir/$cov_proj
+
+    rm -rf $cov_build_dir
+    cov-build   --dir $cov_build_dir $cov_make_cmd
+
+    for excl in $cov_exclude_file_list; do
+        cov-manage-emit --dir $cov_build_dir --tu-pattern "file('$excl')" delete
+    done
+
+    cov-analyze --dir $cov_build_dir
+    nerrors=$(cov-format-errors --dir $cov_build_dir | awk '/Processing [0-9]+ errors?/ { print $2 }')
+
+    index_html=$(cd $cov_build_dir && find . -name index.html | cut -c 3-)
+
+    if [ -n "$nerrors" ]; then
+        if [ "$nerrors" = "0" ]; then
+            echo ok - coverity found no issues for $cov_proj >> $cov_stat_tap
+        else
+            echo "not ok - coverity detected $nerrors failures in $cov_proj # $cov_directive" >> $cov_stat_tap
+            local cov_proj_disp="$(echo $cov_proj|cut -f1 -d_)"
+            echo "" >> $gh_cov_msg
+            echo "* Coverity found $nerrors errors for ${cov_proj_disp}" >> $gh_cov_msg
+            echo "<li><a href=${cov_proj}/output/errors/index.html>Report for ${cov_proj}</a>" >> $cov_dir/index.html
+        fi
+    else
+        echo "not ok - coverity failed to run for $cov_proj # SKIP failed to init coverity" >> $cov_stat_tap
+    fi
+
+    module unload tools/cov
+
+    return $nerrors
+}
+
+# check for jenkins commands in PR title
+if [ -n "$ghprbPullTitle" ]; then
+    check_commands "$ghprbPullTitle"
+fi
+
+# check for jenkins command in PR last comment
+if [ -n "$ghprbPullLink" ]; then
+    set +xeE
+    pr_url=$(echo $ghprbPullLink | sed -e s,github.com,api.github.com/repos,g -e s,pull,issues,g)
+    pr_url="${pr_url}/comments"
+    pr_file="$WORKSPACE/github_pr_${ghprbPullId}.json"
+    curl -s $pr_url > $pr_file
+    echo Fetching PR comments from URL: $pr_url
+
+    # extracting last comment
+    pr_comments="$(cat $pr_file | jq -M -a '.[length-1] | .body')"
+
+    echo Last comment: $pr_comments
+    if [ -n "$pr_comments" ]; then
+        check_commands "$pr_comments"
+    fi
+    set -xeE
+fi
+
+echo Running following tests:
+set|grep jenkins_test_
+
+function on_start()
+{
+    echo Starting on host: $(hostname)
+
+    export distro_name=$(python -c 'import platform ; print platform.dist()[0]' | tr '[:upper:]' '[:lower:]')
+    export distro_ver=$(python  -c 'import platform ; print platform.dist()[1]' | tr '[:upper:]' '[:lower:]')
+    if [ "$distro_name" == "suse" ]; then
+        patch_level=$(egrep PATCHLEVEL /etc/SuSE-release|cut -f2 -d=|sed -e "s/ //g")
+        if [ -n "$patch_level" ]; then
+            export distro_ver="${distro_ver}.${patch_level}"
+        fi
+    fi
+    echo $distro_name -- $distro_ver
+
+    # save current environment to support debugging
+#    set +x
+#    env| sed -ne "s/\(\w*\)=\(.*\)\$/export \1='\2'/p" > $WORKSPACE/test_env.sh
+#    chmod 755 $WORKSPACE/test_env.sh
+#    set -x
+}
+
+function on_exit
+{
+    set +x
+    rc=$((rc + $?))
+    echo exit code=$rc
+    if [ $rc -ne 0 ]; then
+        # FIX: when rpmbuild fails, it leaves folders w/o any permissions even for owner
+        # jenkins fails to remove such and fails
+        find $rpm_dir -type d -exec chmod +x {} \;
+    fi
+}
+
+# $1 - test name
+# $2 - test command
+function check_result()
+{
+    set +e
+    eval $timeout_exe $2
+    ret=$?
+    set -e
+    if [ $ret -gt 0 ]; then
+        echo "not ok $test_id $1" >> $run_tap
+    else
+        echo "ok $test_id $1" >> $run_tap
+    fi
+    test_id=$((test_id+1))
+}
+
+trap "on_exit" INT TERM ILL KILL FPE SEGV ALRM
+
+on_start
+
+
+cd $WORKSPACE
+if [ "$jenkins_test_build" = "yes" ]; then
+    echo "Checking for build ..."
+
+    cd ${WORKSPACE}/${prefix}
+    wget http://downloads.sourceforge.net/levent/libevent-2.0.22-stable.tar.gz
+    tar zxf libevent-2.0.22-stable.tar.gz
+    cd libevent-2.0.22-stable
+    libevent_dir=$PWD/install
+    ./autogen.sh && ./configure --prefix=$libevent_dir && make && make install
+
+    cd $WORKSPACE
+    if [ -x "autogen.sh" ]; then
+        autogen_script=./autogen.sh
+    else
+        autogen_script=./autogen.pl
+    fi
+
+    configure_args="--with-libevent=$libevent_dir"
+
+    # build pmix
+    $autogen_script
+    echo ./configure --prefix=$pmix_dir $configure_args | bash -xeE
+    make $make_opt install
+    jenkins_build_passed=1
+
+    # make check
+    if [ "$jenkins_test_check" = "yes" ]; then
+        make $make_opt check || exit 12
+    fi
+fi
+
+cd $WORKSPACE
+if [ -n "jenkins_build_passed" -a "$jenkins_test_cov" = "yes" ]; then
+    echo "Checking for coverity ..."
+
+    vpath_dir=$WORKSPACE
+    cov_proj="all"
+    gh_cov_msg=$WORKSPACE/cov_gh_msg.txt
+    cov_stat=$vpath_dir/cov_stat.txt
+    cov_stat_tap=$vpath_dir/cov_stat.tap
+    cov_build_dir=$vpath_dir/${prefix}/cov_build
+    cov_url_webroot=${JOB_URL}/${BUILD_ID}/Coverity_Report
+
+    rm -f $cov_stat $cov_stat_tap
+
+    if [ -d "$vpath_dir" ]; then
+        mkdir -p $cov_build_dir
+        pushd $vpath_dir
+        for dir in $cov_proj; do
+            if [ "$dir" = "all" ]; then
+                make_cov_opt=""
+                cov_directive="SKIP"
+            else
+                if [ ! -d "$dir" ]; then
+                    continue
+                fi
+                cov_directive="TODO"
+                make_cov_opt="-C $dir"
+            fi
+            echo Working on $dir
+
+            cov_proj="cov_$(basename $dir)"
+            set +eE
+            make $make_cov_opt $make_opt clean 2>&1 > /dev/null
+            test_cov $cov_build_dir $cov_proj "make $make_cov_opt $make_opt all" $cov_directive
+            set -eE
+        done
+        if [ -n "$ghprbPullId" -a -f "$gh_cov_msg" ]; then
+            echo "* Coverity report at $cov_url_webroot" >> $gh_cov_msg
+            if [ "$jenkins_test_comments" = "yes" ]; then
+                gh pr $ghprbPullId --comment "$(cat $gh_cov_msg)"
+            fi
+        fi
+        popd
+    fi
+fi
+
+cd $WORKSPACE
+if [ "$jenkins_test_src_rpm" = "yes" ]; then
+    echo "Checking for rpm ..."
+
+    # check distclean
+    make $make_opt distclean
+    $autogen_script
+    echo ./configure --prefix=$pmix_dir $configure_args | bash -xeE || exit 11
+
+    if [ -x /usr/bin/dpkg-buildpackage ]; then
+        echo "Do not support PMIX on debian"
+    else
+        echo "Building PMIX src.rpm"
+        rm -rf $tarball_dir
+        mkdir -p $tarball_dir
+
+        make_dist_args="--highok --distdir=$tarball_dir --greekonly"
+
+        for arg in no-git-update dirtyok verok; do
+            if grep $arg contrib/make_tarball 2>&1 > /dev/null; then
+                make_dist_args="$make_dist_args --${arg}"
+            fi
+        done
+
+        # ugly hack, make_tarball has hardcoded "-j32" and sometimes it fails on some race
+        sed -i -e s,-j32,-j8,g contrib/make_tarball
+
+        export LIBEVENT=$libevent_dir
+        chmod +x ./contrib/make* ./contrib/buildrpm.sh
+        echo contrib/make_tarball $make_dist_args | bash -xeE || exit 11
+
+        # build src.rpm
+        # svn_r=$(git rev-parse --short=7 HEAD| tr -d '\n') ./contrib/make_tarball --distdir=$tarball_dir
+        tarball_src=$(ls -1 $tarball_dir/pmix-*.tar.bz2|sort -r|head -1)
+
+        echo "Building PMIX bin.rpm"
+        rpm_flags="--define 'mflags -j8' --define '_source_filedigest_algorithm md5' --define '_binary_filedigest_algorithm md5'"
+        (cd ./contrib/ && env rpmbuild_options="$rpm_flags" rpmtopdir=$rpm_dir ./buildrpm.sh $tarball_src)
+    fi
+fi
+
+#
+# JENKINS_RUN_TESTS should be set in jenkins slave node to indicate that node can run tests
+#
+cd $WORKSPACE
+if [ -n "$JENKINS_RUN_TESTS" -a "$JENKINS_RUN_TESTS" -ne "0" ]; then
+    echo "Checking for tests ..."
+
+    run_tap=$WORKSPACE/run_test.tap
+    rm -rf $run_tap
+
+    # build pmix
+    $autogen_script
+    echo ./configure --prefix=$pmix_dir $configure_args --disable-visibility | bash -xeE
+    make $make_opt install
+
+    cd $WORKSPACE/test
+
+    echo "1..11" > $run_tap
+
+    test_id=1
+    # 1 blocking fence with data exchange among all processes from two namespaces:
+    test_exec='./pmix_test -n 4 --ns-dist 3:1 --fence "[db | 0:0-2;1:3]"'
+    check_result "blocking fence w/ data all" "$test_exec"
+    test_exec='./pmix_test -n 4 --ns-dist 3:1 --fence "[db | 0:;1:3]"'
+    check_result "blocking fence w/ data all" "$test_exec"
+    test_exec='./pmix_test -n 4 --ns-dist 3:1 --fence "[db | 0:;1:]"'
+    check_result "blocking fence w/ data all" "$test_exec"
+
+    # 1 non-blocking fence without data exchange among processes from the 1st namespace
+    test_exec='./pmix_test -n 4 --ns-dist 3:1 --fence "[0:]"'
+    check_result "non-blocking fence w/o data" "$test_exec"
+
+    # blocking fence without data exchange among processes from the 1st namespace
+    test_exec='./pmix_test -n 4 --ns-dist 3:1 --fence "[b | 0:]"'
+    check_result "blocking fence w/ data" "$test_exec"
+
+    # non-blocking fence with data exchange among processes from the 1st namespace. Ranks 0, 1 from ns 0 are sleeping for 2 sec before doing fence test.
+    test_exec='./pmix_test -n 4 --ns-dist 3:1 --fence "[d | 0:]" --noise "[0:0,1]"'
+    check_result "non-blocking fence w/ data" "$test_exec"
+
+    # blocking fence with data exchange across processes from the same namespace.
+    test_exec='./pmix_test -n 4 --job-fence -c'
+    check_result "blocking fence w/ data on the same nspace" "$test_exec"
+
+    # 3 fences: 1 - non-blocking without data exchange across processes from ns 0,
+    # 2 - non-blocking across processes 0 and 1 from ns 0 and process 3 from ns 1,
+    # 3 - blocking with data exchange across processes from their own namespace.
+#    Disabled as incorrect at the moment
+#    test_exec='./pmix_test -n 4 --job-fence -c --fence "[0:][d|0:0-1;1:]" --use-same-keys --ns-dist "3:1"'
+#    check_result "mix fence" $test_exec
+
+    # test publish/lookup/unpublish functionality.
+    test_exec='./pmix_test -n 2 --test-publish'
+    check_result "publish" "$test_exec"
+
+    # test spawn functionality.
+    test_exec='./pmix_test -n 2 --test-spawn'
+    check_result "spawn" "$test_exec"
+
+    # test connect/disconnect between processes from the same namespace.
+    test_exec='./pmix_test -n 2 --test-connect'
+    check_result "connect" "$test_exec"
+
+    # resolve peers from different namespaces.
+    test_exec='./pmix_test -n 5 --test-resolve-peers --ns-dist "1:2:2"'
+    check_result "resolve peers" "$test_exec"
+
+    # run valgrind
+    if [ "$jenkins_test_vg" = "yes" ]; then
+        set +e
+        module load tools/valgrind
+
+        vg_opt="--tool=memcheck --leak-check=full --error-exitcode=0 --trace-children=yes  --trace-children-skip=*/sed,*/collect2,*/gcc,*/cat,*/rm,*/ls --track-origins=yes --xml=yes --xml-file=valgrind%p.xml --fair-sched=try --gen-suppressions=all"
+
+        valgrind $vg_opt  ./pmix_test -n 4 --timeout 60 --ns-dist 3:1 --fence "[db | 0:;1:3]"
+
+        valgrind $vg_opt  ./pmix_test -n 4 --timeout 60 --job-fence -c
+
+        valgrind $vg_opt  ./pmix_test -n 2 --timeout 60 --test-publish
+
+        valgrind $vg_opt  ./pmix_test -n 2 --timeout 60 --test-spawn
+
+        valgrind $vg_opt  ./pmix_test -n 2 --timeout 60 --test-connect
+
+        valgrind $vg_opt  ./pmix_test -n 5 --timeout 60 --test-resolve-peers --ns-dist "1:2:2"
+
+        module unload tools/valgrind
+        set -e
+    fi
+fi

opal/mca/pmix/pmix2x/pmix/contrib/update-my-copyright.pl

@@ -0,0 +1,347 @@
+#!/usr/bin/env perl
+#
+# Copyright (c) 2010-2014 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2016      Intel, Inc. All rights reserved.
+# $COPYRIGHT$
+#
+
+# Short version:
+#
+# This script automates the tedious task of updating copyright notices
+# in the tops of PMIX source files before committing back to
+# the respository.  Set the environment variable
+# PMIX_COPYRIGHT_SEARCH_NAME to a short (case-insensitive) name that
+# indicates your copyright line (e.g., "cisco"), and set the env
+# variable PMIX_COPYRIGHT_FORMAL_NAME with your organization's formal
+# name and copyright statement (e.g., "Cisco Systems, Inc.  All rights
+# reserved.") before running the script.
+
+# More details:
+#
+# This is a simple script to traverse the tree looking for added and
+# changed files (via "svn st ." or "hg st .", depending on what meta
+# directory is found in this tree).  Note that the search starts in
+# the current directory -- not the top-level directory.
+#
+# All added and changed files are examined.  If the special
+# "$COPYRIGHT$" token is found, then lines above that token are
+# examined to find the "search" copyright name.
+#
+# - If the search name is found, that line is examined to see if the
+#   current year is in the copyright year range.  If it is not, the line
+#   is modified to include the current year.
+# - If the search name is not found, a new line is created in the
+#   copyright block of the file using the formal name and the current
+#   year.
+#
+# NOTE: this script currently doesn't handle multi-line copyright
+# statements, such as:
+#
+# Copyright (c) 2010 University of Blabbityblah and the Trustees of
+#                    Schblitbittyboo.  All rights reserved.
+#
+# Someone could certainly extend this script to do so, if they cared
+# (my organizations' copyright fits on a single line, so I wasn't
+# motivated to handle the multi-line case :-) ).
+#
+
+use strict;
+use Cwd;
+use Getopt::Long;
+
+# Set to true if the script should merely check for up-to-date copyrights.
+# Will exit with status 111 if there are out of date copyrights which this
+# script can correct.
+my $CHECK_ONLY = 0;
+# used by $CHECK_ONLY logic for bookeeping
+my $would_replace = 0;
+
+# Set to true to suppress most informational messages.  Only out of date files
+# will be printed.
+my $QUIET = 0;
+
+# Set to true if we just want to see the help message
+my $HELP = 0;
+
+# Defaults
+my $my_search_name = "Intel";
+my $my_formal_name = "Intel, Inc.  All rights reserved.";
+
+# Override the defaults if some values are set in the environment
+$my_search_name = $ENV{PMIX_COPYRIGHT_SEARCH_NAME}
+    if (defined($ENV{PMIX_COPYRIGHT_SEARCH_NAME}));
+$my_formal_name = $ENV{PMIX_COPYRIGHT_FORMAL_NAME}
+    if (defined($ENV{PMIX_COPYRIGHT_FORMAL_NAME}));
+
+GetOptions(
+    "help" => \$HELP,
+    "quiet" => \$QUIET,
+    "check-only" => \$CHECK_ONLY,
+    "search-name=s" => \$my_search_name,
+    "formal-name=s" => \$my_formal_name,
+) or die "unable to parse options, stopped";
+
+if ($HELP) {
+    print <<EOT;
+$0 [options]
+
+--help | -h          This help message
+--quiet | -q         Only output critical messages to stdout
+--check-only         exit(111) if there are files with copyrights to edit
+--search-name=NAME   Set search name to NAME
+--formal-same=NAME   Set formal name to NAME
+EOT
+    exit(0);
+}
+
+#-------------------------------------------------------------------------------
+# predeclare sub for print-like syntax
+sub quiet_print {
+    unless ($QUIET) {
+        print @_;
+    }
+}
+
+#-------------------------------------------------------------------------------
+
+quiet_print "==> Copyright search name: $my_search_name\n";
+quiet_print "==> Copyright formal name: $my_formal_name\n";
+
+# Get the year
+my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime;
+$year += 1900;
+quiet_print "==> This year: $year\n";
+
+# Find the top-level PMIx source tree dir
+my $start = cwd();
+my $top = $start;
+while (! -f "$top/VERSION") {
+    chdir("..");
+    $top = cwd();
+    die "Can't find top-level PMIx directory"
+        if ($top eq "/");
+}
+chdir($start);
+
+quiet_print "==> Top-level PMIx dir: $top\n";
+quiet_print "==> Current directory: $start\n";
+
+# Select VCS used to obtain modification info.  Choose in increasing priority
+# order (last hit wins).
+my $vcs;
+$vcs = "git"
+    if (-d "$top/.git");
+$vcs = "hg"
+    if (-d "$top/.hg");
+$vcs = "svn"
+    if (-d "$top/.svn");
+
+my @files = find_modified_files($vcs);
+
+if ($#files < 0) {
+    quiet_print "No added / changed files -- nothing to do\n";
+    exit(0);
+}
+
+# Examine each of the files and see if they need an updated copyright
+foreach my $f (@files) {
+    quiet_print "Processing added/changed file: $f\n";
+    open(FILE, $f) || die "Can't open file: $f";
+
+    # Read in the file, and look for the "$COPYRIGHT$" token; that's
+    # the end of the copyright block that we're allowed to edit.  Do
+    # not edit any copyright notices that may appear below that.
+
+    my $i = 0;
+    my $found_copyright = 0;
+    my $found_me = 0;
+    my @lines;
+    my $my_line_index;
+    my $token_line_index;
+    while (<FILE>) {
+        push(@lines, $_);
+        if (!$found_copyright && $_ =~ /\$COPYRIGHT\$/) {
+            $token_line_index = $i;
+            $found_copyright = 1;
+        }
+        if (!$found_me &&
+            !defined($token_line_index) && $_ =~ /$my_search_name/i) {
+            $my_line_index = $i;
+            $found_me = 1;
+        }
+        ++$i;
+    }
+    close(FILE);
+
+    # If there was not copyright token, don't do anything
+    if (!defined($token_line_index)) {
+        quiet_print "==> WARNING: Did not find the \$COPYRIGHT\$ token!\n";
+        quiet_print "    File left unchanged\n";
+        next;
+    }
+
+    # Figure out the line prefix
+    $lines[$token_line_index] =~ m/^(.+)\$COPYRIGHT\$/;
+    my $prefix = $1;
+
+    # Now act on it
+    if (!defined($my_line_index)) {
+        quiet_print "--- My copyright line not found; adding:\n";
+        my $str = "${prefix}Copyright (c) $year      $my_formal_name\n";
+        quiet_print "    $str";
+        $lines[$token_line_index] = $str . $lines[$token_line_index];
+    } else {
+        quiet_print "--- Found existing copyright line:\n";
+        quiet_print "    $lines[$my_line_index]";
+        $lines[$my_line_index] =~ m/([\d+\-]+)/;
+        my $years = $1;
+        die "Could not find years in copyright line!"
+            if (!defined($years));
+
+        # If it's a range, separate them out
+        my $first_year;
+        my $last_year;
+        if ($years =~ /\-/) {
+            $years =~ m/(\d+)\s*-\s*(\d+)/;
+            $first_year = $1;
+            $last_year = $2;
+        } else {
+            $first_year = $last_year = $years;
+        }
+
+        # Sanity check
+        die "Copyright looks like it extends before 1990...?"
+            if ($first_year < 1990);
+        die "Copyright in the future...?"
+            if ($last_year > $year);
+
+        # Do we need to do anything?
+        if ($year > $last_year) {
+            $lines[$my_line_index] = "${prefix}Copyright (c) $first_year-$year $my_formal_name\n";
+            quiet_print "    Updated to:\n";
+            quiet_print "    $lines[$my_line_index]";
+        } else {
+            quiet_print "    This year already included in copyright; not changing file\n";
+            next;
+        }
+    }
+
+    # If we got this far, we want to write out a new file
+    my $newf = "$f.new-copyright";
+    unlink($newf);
+    open(FILE, ">$newf") || die "Can't open file: $newf";
+    print FILE join('', @lines);
+    close(FILE);
+
+    if ($CHECK_ONLY) {
+        # intentional "loud" print to be more useful in a pre-commit hook
+        print "==> '$f' has a stale/missing copyright\n";
+        unlink($newf);
+        ++$would_replace;
+    }
+    else {
+        # Now replace the old one
+        unlink($f);
+        rename($newf, $f);
+    }
+}
+
+if ($CHECK_ONLY and $would_replace) {
+    exit(111);
+}
+
+#-------------------------------------------------------------------------------
+
+# Takes two arguments, the top level directory and the VCS method.  Returns a
+# list of file names (relative to pwd) which the VCS considers to be modified.
+sub find_modified_files {
+    my $vcs = shift;
+    my @files = ();
+
+    if ($vcs eq "git") {
+        # Number of path entries to remove from ${top}-relative paths.
+        # (--show-cdup either returns the empty string or sequence of "../"
+        # entries, always ending in a "/")
+        my $n_strip = scalar(split(m!/!, scalar(`git rev-parse --show-cdup`))) - 1;
+
+        # "." restricts scope, but does not get us relative path names
+        my $cmd = "git status -z --porcelain --untracked-files=no .";
+        quiet_print "==> Running: \"$cmd\"\n";
+        my $lines = `$cmd`;
+
+        # From git-status(1):
+        # X          Y     Meaning
+        # -------------------------------------------------
+        #           [MD]   not updated
+        # M        [ MD]   updated in index
+        # A        [ MD]   added to index
+        # D         [ M]   deleted from index
+        # R        [ MD]   renamed in index
+        # C        [ MD]   copied in index
+        # [MARC]           index and work tree matches
+        # [ MARC]     M    work tree changed since index
+        # [ MARC]     D    deleted in work tree
+        # -------------------------------------------------
+        # D           D    unmerged, both deleted
+        # A           U    unmerged, added by us
+        # U           D    unmerged, deleted by them
+        # U           A    unmerged, added by them
+        # D           U    unmerged, deleted by us
+        # A           A    unmerged, both added
+        # U           U    unmerged, both modified
+        # -------------------------------------------------
+        # ?           ?    untracked
+        # -------------------------------------------------
+        foreach my $line (split /\x{00}/, $lines) {
+            my $keep = 0;
+            my ($s1, $s2, $fullname) = $line =~ m/^(.)(.) (.*)$/;
+
+            # ignore all merge cases
+            next if ($s1 eq "D" and $s2 eq "D");
+            next if ($s1 eq "A" and $s2 eq "A");
+            next if ($s1 eq "U" or $s2 eq "U");
+
+            # only update for actually added/modified cases, no copies,
+            # renames, etc.
+            $keep = 1 if ($s1 eq "M" or $s2 eq "M");
+            $keep = 1 if ($s1 eq "A");
+
+            if ($keep) {
+                my $relname = $fullname;
+                $relname =~ s!^([^/]*/){$n_strip}!!g;
+
+                push @files, $relname
+                    if (-f $relname);
+            }
+        }
+    }
+    elsif ($vcs eq "hg" or $vcs eq "svn") {
+        my $cmd = "$vcs st .";
+
+        # Run the command, parsing the output.  Make a list of files that are
+        # added or modified.
+        quiet_print "==> Running: \"$cmd\"\n";
+        open(CMD, "$cmd|") || die "Can't run command";
+        while (<CMD>) {
+            chomp;
+            if ($_ =~ /^M/ || $_ =~ /^A/) {
+                my @tokens = split(/\s+/, $_);
+                # Handle output of both forms:
+                # M       filenameA
+                # A  +    filenameB
+                my $filename = $tokens[1];
+                $filename = $tokens[2]
+                if ($tokens[1] =~ /\+/);
+                # Don't bother saving directory names
+                push(@files, $filename)
+                    if (-f $filename);
+            }
+        }
+        close(CMD);
+    }
+    else {
+        die "unknown VCS '$vcs', stopped";
+    }
+
+    return @files;
+}

opal/mca/pmix/pmix2x/pmix/contrib/whitespace-purge.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+#
+# Copyright (c) 2015-2016 Intel, Inc.  All rights reserved.
+# Copyright (c) 2015      Los Alamos National Security, LLC. All rights
+#                         reserved
+# Copyright (c) 2015 Cisco Systems, Inc.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+for file in $(git ls-files) ; do
+    # check for the mime-type and do not follow symbolic links. this
+    # will cause file to print application/x-symlink for the mime-type
+    # allowing us to only have to check if the type is application to
+    # skip sym links, pdfs, etc. If any other file types should be
+    # skipped add the check here.
+    type=$(file -b --mime-type -h $file)
+    if test ${type::4} == "text" ; then
+        # Eliminate whitespace at the end of lines
+        perl -pi -e 's/\s*$/\n/' $file
+    fi
+done

opal/mca/pmix/pmix2x/pmix/examples/Makefile.am

@@ -22,11 +22,26 @@
 AM_CPPFLAGS = -I$(top_builddir)/src -I$(top_builddir)/src/include -I$(top_builddir)/src/api
 
 noinst_PROGRAMS = client dmodex dynamic fault pub tool
+if !WANT_HIDDEN
+# these examples use internal symbols
+# use --disable-visibility
+noinst_PROGRAMS += debugger debuggerd
+endif
 
 client_SOURCES = client.c
 client_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS)
 client_LDADD =  $(top_builddir)/src/libpmix.la
 
+if !WANT_HIDDEN
+debugger_SOURCES = debugger.c
+debugger_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS)
+debugger_LDADD =  $(top_builddir)/src/libpmix.la
+
+debuggerd_SOURCES = debuggerd.c
+debuggerd_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS)
+debuggerd_LDADD =  $(top_builddir)/src/libpmix.la
+endif
+
 dmodex_SOURCES = dmodex.c
 dmodex_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS)
 dmodex_LDADD =  $(top_builddir)/src/libpmix.la
@@ -48,4 +63,4 @@ tool_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS)
 tool_LDADD = $(top_builddir)/src/libpmix.la
 
 distclean-local:
-	rm -f *.o client dmodex dynamic fault pub server
+	rm -f *.o client debugger debuggerd dmodex dynamic fault pub server

opal/mca/pmix/pmix2x/pmix/examples/client.c

@@ -32,9 +32,68 @@
 
 #include <pmix.h>
 
+static volatile bool waiting_for_debugger = true;
+static pmix_proc_t myproc;
+
+/* this is the event notification function we pass down below
+ * when registering for general events - i.e.,, the default
+ * handler. We don't technically need to register one, but it
+ * is usually good practice to catch any events that occur */
+static void notification_fn(size_t evhdlr_registration_id,
+                            pmix_status_t status,
+                            const pmix_proc_t *source,
+                            pmix_info_t info[], size_t ninfo,
+                            pmix_info_t results[], size_t nresults,
+                            pmix_event_notification_cbfunc_fn_t cbfunc,
+                            void *cbdata)
+{
+    if (NULL != cbfunc) {
+        cbfunc(PMIX_EVENT_ACTION_COMPLETE, NULL, 0, NULL, NULL, cbdata);
+    }
+}
+
+/* this is an event notification function that we explicitly request
+ * be called when the PMIX_ERR_DEBUGGER_RELEASE notification is issued.
+ * We could catch it in the general event notification function and test
+ * the status to see if it was "debugger release", but it often is simpler
+ * to declare a use-specific notification callback point. In this case,
+ * we are asking to know when we are told the debugger released us */
+static void release_fn(size_t evhdlr_registration_id,
+                       pmix_status_t status,
+                       const pmix_proc_t *source,
+                       pmix_info_t info[], size_t ninfo,
+                       pmix_info_t results[], size_t nresults,
+                       pmix_event_notification_cbfunc_fn_t cbfunc,
+                       void *cbdata)
+{
+    if (NULL != cbfunc) {
+        cbfunc(PMIX_EVENT_ACTION_COMPLETE, NULL, 0, NULL, NULL, cbdata);
+    }
+    waiting_for_debugger = false;
+}
+
+/* event handler registration is done asynchronously because it
+ * may involve the PMIx server registering with the host RM for
+ * external events. So we provide a callback function that returns
+ * the status of the request (success or an error), plus a numerical index
+ * to the registered event. The index is used later on to deregister
+ * an event handler - if we don't explicitly deregister it, then the
+ * PMIx server will do so when it see us exit */
+static void evhandler_reg_callbk(pmix_status_t status,
+                                 size_t evhandler_ref,
+                                 void *cbdata)
+{
+    volatile int *active = (volatile int*)cbdata;
+
+    if (PMIX_SUCCESS != status) {
+        fprintf(stderr, "Client %s:%d EVENT HANDLER REGISTRATION FAILED WITH STATUS %d, ref=%lu\n",
+                   myproc.nspace, myproc.rank, status, (unsigned long)evhandler_ref);
+    }
+    *active = status;
+}
+
 int main(int argc, char **argv)
 {
-    pmix_proc_t myproc;
     int rc;
     pmix_value_t value;
     pmix_value_t *val = &value;
@@ -43,18 +102,66 @@ int main(int argc, char **argv)
     uint32_t nprocs, n;
     pmix_info_t *info;
     bool flag;
+    volatile int active;
+    pmix_status_t dbg = PMIX_ERR_DEBUGGER_RELEASE;
 
-    /* init us */
+    /* init us - note that the call to "init" includes the return of
+     * any job-related info provided by the RM. This includes any
+     * debugger flag instructing us to stop-in-init. If such a directive
+     * is included, then the process will be stopped in this call until
+     * the "debugger release" notification arrives */
     if (PMIX_SUCCESS != (rc = PMIx_Init(&myproc, NULL, 0))) {
         fprintf(stderr, "Client ns %s rank %d: PMIx_Init failed: %d\n", myproc.nspace, myproc.rank, rc);
         exit(0);
     }
     fprintf(stderr, "Client ns %s rank %d: Running\n", myproc.nspace, myproc.rank);
 
+
+    /* register our default event handler - again, this isn't strictly
+     * required, but is generally good practice */
+    active = -1;
+    PMIx_Register_event_handler(NULL, 0, NULL, 0,
+                                notification_fn, evhandler_reg_callbk, (void*)&active);
+    while (-1 == active) {
+        sleep(1);
+    }
+    if (0 != active) {
+        fprintf(stderr, "[%s:%d] Default handler registration failed\n", myproc.nspace, myproc.rank);
+        exit(active);
+    }
+
+    /* job-related info is found in our nspace, assigned to the
+     * wildcard rank as it doesn't relate to a specific rank. Setup
+     * a name to retrieve such values */
     PMIX_PROC_CONSTRUCT(&proc);
     (void)strncpy(proc.nspace, myproc.nspace, PMIX_MAX_NSLEN);
     proc.rank = PMIX_RANK_WILDCARD;
 
+    /* check to see if we have been instructed to wait for a debugger
+     * to attach to us. We won't get both a stop-in-init AND a
+     * wait-for-notify directive, so we should never stop twice. This
+     * directive is provided so that something like an MPI implementation
+     * can do some initial setup in MPI_Init prior to pausing for the
+     * debugger */
+    if (PMIX_SUCCESS == (rc = PMIx_Get(&proc, PMIX_DEBUG_WAIT_FOR_NOTIFY, NULL, 0, &val))) {
+        /* register for debugger release */
+        active = -1;
+        PMIx_Register_event_handler(&dbg, 1, NULL, 0,
+                                    release_fn, evhandler_reg_callbk, (void*)&active);
+        /* wait for registration to complete */
+        while (-1 == active) {
+            sleep(1);
+        }
+        if (0 != active) {
+            fprintf(stderr, "[%s:%d] Debug handler registration failed\n", myproc.nspace, myproc.rank);
+            exit(active);
+        }
+        /* wait for debugger release */
+        while (waiting_for_debugger) {
+            sleep(1);
+        }
+    }
+
     /* get our universe size */
     if (PMIX_SUCCESS != (rc = PMIx_Get(&proc, PMIX_UNIV_SIZE, NULL, 0, &val))) {
         fprintf(stderr, "Client ns %s rank %d: PMIx_Get universe size failed: %d\n", myproc.nspace, myproc.rank, rc);
@@ -98,12 +205,15 @@ int main(int argc, char **argv)
     }
     free(tmp);
 
+    /* push the data to our PMIx server */
     if (PMIX_SUCCESS != (rc = PMIx_Commit())) {
         fprintf(stderr, "Client ns %s rank %d: PMIx_Commit failed: %d\n", myproc.nspace, myproc.rank, rc);
         goto done;
     }
 
-    /* call fence to ensure the data is received */
+    /* call fence to synchronize with our peers - instruct
+     * the fence operation to collect and return all "put"
+     * data from our peers */
     PMIX_INFO_CREATE(info, 1);
     flag = true;
     PMIX_INFO_LOAD(info, PMIX_COLLECT_DATA, &flag, PMIX_BOOL);

opal/mca/pmix/pmix2x/pmix/examples/debugger.c

@@ -0,0 +1,544 @@
+/*
+ * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2011 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2013 Los Alamos National Security, LLC.
+ *                         All rights reserved.
+ * Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2011      Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2013-2016 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2015      Mellanox Technologies, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <time.h>
+
+#include <pmix_tool.h>
+
+
+/* define a structure for collecting returned
+ * info from a query */
+typedef struct {
+    volatile bool active;
+    pmix_info_t *info;
+    size_t ninfo;
+} myquery_data_t;
+
+static int attach_to_running_job(char *nspace);
+static bool waiting_for_debugger = true;
+static pmix_proc_t myproc;
+
+/* this is a callback function for the PMIx_Query
+ * API. The query will callback with a status indicating
+ * if the request could be fully satisfied, partially
+ * satisfied, or completely failed. The info parameter
+ * contains an array of the returned data, with the
+ * info->key field being the key that was provided in
+ * the query call. Thus, you can correlate the returned
+ * data in the info->value field to the requested key.
+ *
+ * Once we have dealt with the returned data, we must
+ * call the release_fn so that the PMIx library can
+ * cleanup */
+static void cbfunc(pmix_status_t status,
+                   pmix_info_t *info, size_t ninfo,
+                   void *cbdata,
+                   pmix_release_cbfunc_t release_fn,
+                   void *release_cbdata)
+{
+    myquery_data_t *mq = (myquery_data_t*)cbdata;
+    size_t n;
+
+    /* save the returned info - the PMIx library "owns" it
+     * and will release it and perform other cleanup actions
+     * when release_fn is called */
+    if (0 < ninfo) {
+        PMIX_INFO_CREATE(mq->info, ninfo);
+        mq->ninfo = ninfo;
+        for (n=0; n < ninfo; n++) {
+            fprintf(stderr, "Transferring %s\n", info[n].key);
+            PMIX_INFO_XFER(&mq->info[n], &info[n]);
+        }
+    }
+
+    /* let the library release the data and cleanup from
+     * the operation */
+    if (NULL != release_fn) {
+        release_fn(release_cbdata);
+    }
+
+    /* release the block */
+    mq->active = false;
+}
+
+/* this is the event notification function we pass down below
+ * when registering for general events - i.e.,, the default
+ * handler. We don't technically need to register one, but it
+ * is usually good practice to catch any events that occur */
+static void notification_fn(size_t evhdlr_registration_id,
+                            pmix_status_t status,
+                            const pmix_proc_t *source,
+                            pmix_info_t info[], size_t ninfo,
+                            pmix_info_t results[], size_t nresults,
+                            pmix_event_notification_cbfunc_fn_t cbfunc,
+                            void *cbdata)
+{
+    /* this example doesn't do anything with default events */
+    if (NULL != cbfunc) {
+        cbfunc(PMIX_EVENT_ACTION_COMPLETE, NULL, 0, NULL, NULL, cbdata);
+    }
+}
+
+/* this is an event notification function that we explicitly request
+ * be called when the PMIX_ERR_JOB_TERMINATED notification is issued.
+ * We could catch it in the general event notification function and test
+ * the status to see if it was "job terminated", but it often is simpler
+ * to declare a use-specific notification callback point. In this case,
+ * we are asking to know whenever a job terminates, and we will then
+ * know we can exit */
+static void release_fn(size_t evhdlr_registration_id,
+                       pmix_status_t status,
+                       const pmix_proc_t *source,
+                       pmix_info_t info[], size_t ninfo,
+                       pmix_info_t results[], size_t nresults,
+                       pmix_event_notification_cbfunc_fn_t cbfunc,
+                       void *cbdata)
+{
+    /* tell the event handler state machine that we are the last step */
+    if (NULL != cbfunc) {
+        cbfunc(PMIX_EVENT_ACTION_COMPLETE, NULL, 0, NULL, NULL, cbdata);
+    }
+    /* flag that the debugger is complete so we can exit */
+    waiting_for_debugger = false;
+}
+
+/* event handler registration is done asynchronously because it
+ * may involve the PMIx server registering with the host RM for
+ * external events. So we provide a callback function that returns
+ * the status of the request (success or an error), plus a numerical index
+ * to the registered event. The index is used later on to deregister
+ * an event handler - if we don't explicitly deregister it, then the
+ * PMIx server will do so when it see us exit */
+static void evhandler_reg_callbk(pmix_status_t status,
+                                 size_t evhandler_ref,
+                                 void *cbdata)
+{
+    volatile int *active = (volatile int*)cbdata;
+
+    if (PMIX_SUCCESS != status) {
+        fprintf(stderr, "Client %s:%d EVENT HANDLER REGISTRATION FAILED WITH STATUS %d, ref=%lu\n",
+                   myproc.nspace, myproc.rank, status, (unsigned long)evhandler_ref);
+    }
+    *active = status;
+}
+
+int main(int argc, char **argv)
+{
+    pmix_status_t rc;
+    pmix_info_t *info, *dinfo;
+    pmix_app_t *app, *debugger;
+    size_t ninfo, napps, dninfo;
+    char *tdir, *nspace = NULL;
+    char appspace[PMIX_MAX_NSLEN+1], dspace[PMIX_MAX_NSLEN+1];
+    int i;
+    pmix_query_t *query;
+    size_t nq, n;
+    myquery_data_t myquery_data;
+    bool cospawn = false, stop_on_exec = false;
+    char cwd[1024];
+    volatile int active;
+    pmix_status_t code = PMIX_ERR_JOB_TERMINATED;
+
+    /* Process any arguments we were given */
+    for (i=1; i < argc; i++) {
+        if (0 == strcmp(argv[i], "-h") ||
+            0 == strcmp(argv[i], "--help")) {
+            /* print the usage message and exit */
+
+        }
+        if (0 == strcmp(argv[i], "-a") ||
+            0 == strcmp(argv[i], "--attach")) {
+            if (NULL != nspace) {
+                /* can only support one */
+                fprintf(stderr, "Cannot attach to more than one nspace\n");
+                exit(1);
+            }
+            /* the next argument must be the nspace */
+            ++i;
+            if (argc == i) {
+                /* they goofed */
+                fprintf(stderr, "The %s option requires an <nspace> argument\n", argv[i]);
+                exit(1);
+            }
+            nspace = strdup(argv[i]);
+        } else {
+            fprintf(stderr, "Unknown option: %s\n", argv[i]);
+            exit(1);
+        }
+    }
+
+    /* we need to provide some info to the PMIx tool library so
+     * it can find the server's contact info. The simplest way
+     * of doing this here is to look for an environmental variable
+     * that tells us where to look. The PMIx reference server only
+     * allows one instantiation of the server per user, so setting
+     * this up is something a user could do in their login script.
+     * The reference server is based on OpenMPI, and so the contact
+     * info will always be found at:
+     *
+     * $TMPDIR/ompi.<nodename>.<numerical-userid>/dvm
+     *
+     * NOTE: we will eliminate this requirement in a future version
+     */
+
+    if (NULL == (tdir = getenv("PMIX_SERVER_TMPDIR"))) {
+        fprintf(stderr, "Tool usage requires that the PMIX_SERVER_TMPDIR envar\n");
+        fprintf(stderr, "be set to point at the directory where the PMIx Reference\n");
+        fprintf(stderr, "Server leaves its contact info file.\n");
+        exit(1);
+    }
+
+    /* init us - pass along the location of the contact file */
+    ninfo = 1;
+    PMIX_INFO_CREATE(info, ninfo);
+    PMIX_INFO_LOAD(&info[0], PMIX_SERVER_TMPDIR, tdir, PMIX_STRING);
+
+    if (PMIX_SUCCESS != (rc = PMIx_tool_init(&myproc, info, ninfo))) {
+        fprintf(stderr, "PMIx_tool_init failed: %d\n", rc);
+        exit(rc);
+    }
+    PMIX_INFO_FREE(info, ninfo);
+
+    fprintf(stderr, "Tool ns %s rank %d: Running\n", myproc.nspace, myproc.rank);
+
+    /* register a default event handler */
+    active = -1;
+    PMIx_Register_event_handler(NULL, 0, NULL, 0,
+                                notification_fn, evhandler_reg_callbk, (void*)&active);
+    while (-1 == active) {
+        usleep(10);
+    }
+    if (0 != active) {
+        exit(active);
+    }
+
+    /* register another handler specifically for when the debugger
+     * job completes */
+    active = -1;
+    PMIx_Register_event_handler(&code, 1, NULL, 0,
+                                release_fn, evhandler_reg_callbk, (void*)&active);
+    while (-1 == active) {
+        usleep(10);
+    }
+    if (0 != active) {
+        exit(active);
+    }
+
+    /* if we are attaching to a running job, then attach to it */
+    if (NULL != nspace) {
+        if (PMIX_SUCCESS != (rc = attach_to_running_job(nspace))) {
+            fprintf(stderr, "Failed to attach to nspace %s: error code %d\n",
+                    nspace, rc);
+            goto done;
+        }
+    } else {
+        /* this is an initial launch - we need to launch the application
+         * plus the debugger daemons, letting the RM know we are debugging
+         * so that it will "pause" the app procs until we are ready. First
+         * we need to know if this RM supports co-spawning of daemons with
+         * the application, or if we need to launch the daemons as a separate
+         * spawn command. The former is faster and more scalable, but not
+         * every RM may support it. We also need to ask for debug support
+         * so we know if the RM can stop-on-exec, or only supports stop-in-init */
+        nq = 1;
+        PMIX_QUERY_CREATE(query, nq);
+        PMIX_ARGV_APPEND(query[0].keys, PMIX_QUERY_SPAWN_SUPPORT);
+        PMIX_ARGV_APPEND(query[0].keys, PMIX_QUERY_DEBUG_SUPPORT);
+        /* setup the caddy to retrieve the data */
+        myquery_data.info = NULL;
+        myquery_data.ninfo = 0;
+        myquery_data.active = true;
+        /* execute the query */
+        fprintf(stderr, "Debugger: querying capabilities\n");
+        if (PMIX_SUCCESS != (rc = PMIx_Query_info_nb(query, nq, cbfunc, (void*)&myquery_data))) {
+            fprintf(stderr, "PMIx_Query_info failed: %d\n", rc);
+            goto done;
+        }
+        while (myquery_data.active) {
+            usleep(10);
+        }
+
+        /* we should have received back two info structs, one containing
+         * a comma-delimited list of PMIx spawn attributes the RM supports,
+         * and the other containing a comma-delimited list of PMIx debugger
+         * attributes it supports */
+        if (2 != myquery_data.ninfo) {
+            /* this is an error */
+            fprintf(stderr, "PMIx Query returned an incorrect number of results: %lu\n", myquery_data.ninfo);
+            PMIX_INFO_FREE(myquery_data.info, myquery_data.ninfo);
+            goto done;
+        }
+
+        /* we would like to co-spawn the debugger daemons with the app, but
+         * let's first check to see if this RM supports that operation by
+         * looking for the PMIX_COSPAWN_APP attribute in the spawn support
+         *
+         * We will also check to see if "stop_on_exec" is supported. Few RMs
+         * do so, which is why we have to check. The reference server sadly is
+         * not one of them, so we shouldn't find it here
+         *
+         * Note that the PMIx reference server always returns the query results
+         * in the same order as the query keys. However, this is not guaranteed,
+         * so we should search the returned info structures to find the desired key
+         */
+        for (n=0; n < myquery_data.ninfo; n++) {
+            if (0 == strcmp(myquery_data.info[n].key, PMIX_QUERY_SPAWN_SUPPORT)) {
+                /* see if the cospawn attribute is included */
+                if (NULL != strstr(myquery_data.info[n].value.data.string, PMIX_COSPAWN_APP)) {
+                    cospawn = true;
+                } else {
+                    cospawn = false;
+                }
+            } else if (0 == strcmp(myquery_data.info[n].key, PMIX_QUERY_DEBUG_SUPPORT)) {
+                if (NULL != strstr(myquery_data.info[n].value.data.string, PMIX_DEBUG_STOP_ON_EXEC)) {
+                    stop_on_exec = true;
+                } else {
+                    stop_on_exec = false;
+                }
+            }
+        }
+
+        /* if cospawn is true, then we can launch both the app and the debugger
+         * daemons at the same time */
+        if (cospawn) {
+
+        } else {
+            /* we must do these as separate launches, so do the app first */
+            napps = 1;
+            PMIX_APP_CREATE(app, napps);
+            /* setup the executable */
+            app[0].cmd = strdup("client");
+            PMIX_ARGV_APPEND(app[0].argv, "./client");
+            getcwd(cwd, 1024);  // point us to our current directory
+            app[0].cwd = strdup(cwd);
+            app[0].maxprocs = 2;
+            /* provide job-level directives so the apps do what the user requested */
+            ninfo = 2;
+            PMIX_INFO_CREATE(info, ninfo);
+            PMIX_INFO_LOAD(&info[0], PMIX_MAPBY, "slot", PMIX_STRING);  // map by slot
+            if (stop_on_exec) {
+                PMIX_INFO_LOAD(&info[1], PMIX_DEBUG_STOP_ON_EXEC, NULL, PMIX_BOOL);  // procs are to stop on first instruction
+            } else {
+                PMIX_INFO_LOAD(&info[1], PMIX_DEBUG_STOP_IN_INIT, NULL, PMIX_BOOL);  // procs are to pause in PMIx_Init for debugger attach
+            }
+            /* spawn the job - the function will return when the app
+             * has been launched */
+            fprintf(stderr, "Debugger: spawning %s\n", app[0].cmd);
+            if (PMIX_SUCCESS != (rc = PMIx_Spawn(info, ninfo, app, napps, appspace))) {
+                fprintf(stderr, "Application failed to launch with error: %s\n", PMIx_Error_string(rc));
+                goto done;
+            }
+            PMIX_INFO_FREE(info, ninfo);
+            PMIX_APP_FREE(app, napps);
+
+            /* setup the debugger */
+            PMIX_APP_CREATE(debugger, 1);
+            debugger[0].cmd = strdup("./debuggerd");
+            PMIX_ARGV_APPEND(debugger[0].argv, "./debuggerd");
+            debugger[0].cwd = strdup(cwd);
+            /* provide directives so the daemons go where we want, and
+             * let the RM know these are debugger daemons */
+            dninfo = 5;
+            PMIX_INFO_CREATE(dinfo, dninfo);
+            PMIX_INFO_LOAD(&dinfo[0], PMIX_MAPBY, "ppr:1:node", PMIX_STRING);  // instruct the RM to launch one copy of the executable on each node
+            PMIX_INFO_LOAD(&dinfo[1], PMIX_DEBUGGER_DAEMONS, NULL, PMIX_BOOL); // these are debugger daemons
+            PMIX_INFO_LOAD(&dinfo[2], PMIX_DEBUG_JOB, appspace, PMIX_STRING); // the nspace being debugged
+            PMIX_INFO_LOAD(&dinfo[3], PMIX_NOTIFY_COMPLETION, NULL, PMIX_BOOL); // notify us when the debugger job completes
+            PMIX_INFO_LOAD(&dinfo[4], PMIX_DEBUG_WAITING_FOR_NOTIFY, NULL, PMIX_BOOL);  // tell the daemon that the proc is waiting to be released
+            /* spawn the daemons */
+            fprintf(stderr, "Debugger: spawning %s\n", debugger[0].cmd);
+            if (PMIX_SUCCESS != (rc = PMIx_Spawn(dinfo, dninfo, debugger, 1, dspace))) {
+                fprintf(stderr, "Debugger daemons failed to launch with error: %s\n", PMIx_Error_string(rc));
+                goto done;
+            }
+
+            /* cleanup */
+            PMIX_INFO_FREE(dinfo, dninfo);
+            PMIX_APP_FREE(debugger, 1);
+        }
+
+
+        /* this is where a debugger tool would wait until the debug operation is complete */
+        while (waiting_for_debugger) {
+            sleep(1);
+        }
+    }
+
+  done:
+    PMIx_tool_finalize();
+
+    return(rc);
+}
+
+typedef struct {
+    volatile bool active;
+    pmix_status_t status;
+    pmix_info_t *info;
+    size_t ninfo;
+} mydbug_query_t;
+
+
+static void infocbfunc(pmix_status_t status,
+                       pmix_info_t *info, size_t ninfo,
+                       void *cbdata,
+                       pmix_release_cbfunc_t release_fn,
+                       void *release_cbdata)
+{
+    mydbug_query_t *q = (mydbug_query_t*)cbdata;
+    size_t n;
+
+    q->status = status;
+    q->info = NULL;
+    q->ninfo = ninfo;
+    if (0 < ninfo) {
+        PMIX_INFO_CREATE(q->info, q->ninfo);
+        for (n=0; n < ninfo; n++) {
+            PMIX_INFO_XFER(&q->info[n], &info[n]);
+        }
+    }
+    if (NULL != release_fn) {
+        release_fn(release_cbdata);
+    }
+    q->active = false;
+}
+
+static int attach_to_running_job(char *nspace)
+{
+    pmix_status_t rc;
+    pmix_proc_t myproc;
+    pmix_query_t *query;
+    size_t nq;
+    mydbug_query_t *q;
+
+    /* query the active nspaces so we can verify that the
+     * specified one exists */
+    nq = 1;
+    PMIX_QUERY_CREATE(query, nq);
+    query[0].keys = (char**)malloc(2 * sizeof(char*));
+    query[0].keys[0] = strdup(PMIX_QUERY_NAMESPACES);
+    query[0].keys[1] = NULL;
+
+    q = (mydbug_query_t*)malloc(sizeof(mydbug_query_t));
+    q->active = true;
+
+    if (PMIX_SUCCESS != (rc = PMIx_Query_info_nb(query, nq, infocbfunc, (void*)q))) {
+        fprintf(stderr, "Client ns %s rank %d: PMIx_Query_info failed: %d\n", myproc.nspace, myproc.rank, rc);
+        return -1;
+    }
+    /* wait for a response */
+    while (q->active) {
+        sleep(1);
+    }
+
+    if (NULL == q->info) {
+        fprintf(stderr, "Query returned no info\n");
+        return -1;
+    }
+    /* the query should have returned a comma-delimited list of nspaces */
+    if (PMIX_STRING != q->info[0].value.type) {
+        fprintf(stderr, "Query returned incorrect data type: %d\n", q->info[0].value.type);
+        return -1;
+    }
+    if (NULL == q->info[0].value.data.string) {
+        fprintf(stderr, "Query returned no active nspaces\n");
+        return -1;
+    }
+
+    fprintf(stderr, "Query returned %s\n", q->info[0].value.data.string);
+    return 0;
+
+#if 0
+    /* split the returned string and look for the given nspace */
+
+    /* if not found, then we have an error */
+    PMIX_INFO_FREE(info, ninfo);
+
+    /* get the proctable for this nspace */
+    ninfo = 1;
+    PMIX_INFO_CREATE(info, ninfo);
+    (void)strncpy(info[0].key, PMIX_QUERY_PROC_TABLE, PMIX_MAX_KEYLEN);
+    (void)strncpy(info[0].qualifier, nspace, PMIX_MAX_KEYLEN);
+    if (PMIX_SUCCESS != (rc = PMIx_Query_info_nb(info, ninfo, infocbfunc, (void*)&active))) {
+        fprintf(stderr, "Client ns %s rank %d: PMIx_Query_info_nb failed: %d\n", myproc.nspace, myproc.rank, rc);
+        return -1;
+    }
+    /* wait to get a response */
+
+    /* the query should have returned a data_array */
+    if (PMIX_DATA_ARRAY != info[0].type) {
+        fprintf(stderr, "Query returned incorrect data type: %d\n", info[0].type);
+        return -1;
+    }
+    if (NULL == info[0].data.darray.array) {
+        fprintf(stderr, "Query returned no proctable info\n");
+        return -1;
+    }
+    /* the data array consists of a struct:
+     *     size_t size;
+     *     void* array;
+     *
+     * In this case, the array is composed of pmix_proc_info_t structs:
+     *     pmix_proc_t proc;   // contains the nspace,rank of this proc
+     *     char* hostname;
+     *     char* executable_name;
+     *     pid_t pid;
+     *     int exit_code;
+     *     pmix_proc_state_t state;
+     */
+
+    /* this is where a debugger tool would process the proctable to
+     * create whatever blob it needs to provide to its daemons */
+    PMIX_INFO_FREE(info, ninfo);
+
+    /* setup the debugger daemon spawn request */
+    napps = 1;
+    PMIX_APP_CREATE(app, napps);
+    /* setup the name of the daemon executable to launch */
+    app[0].cmd = strdup("debuggerdaemon");
+    app[0].argc = 1;
+    app[0].argv = (char**)malloc(2*sizeof(char*));
+    app[0].argv[0] = strdup("debuggerdaemon");
+    app[0].argv[1] = NULL;
+    /* provide directives so the daemons go where we want, and
+     * let the RM know these are debugger daemons */
+    ninfo = 3;
+    PMIX_INFO_CREATE(app[0].info, ninfo);
+    PMIX_INFO_LOAD(&app[0].info[0], PMIX_MAPBY, "ppr:1:node", PMIX_STRING);  // instruct the RM to launch one copy of the executable on each node
+    PMIX_INFO_LOAD(&app[0].info[1], PMIX_DEBUGGER_DAEMONS, true, PMIX_BOOL); // these are debugger daemons
+    PMIX_INFO_LOAD(&app[0].info[2], PMIX_DEBUG_TARGET, nspace, PMIX_STRING); // the "jobid" of the application to be debugged
+
+    /* spawn the daemons */
+    PMIx_Spawn(NULL, 0, app, napps, dspace);
+    /* cleanup */
+    PMIX_APP_FREE(app, napps);
+
+    /* this is where a debugger tool would wait until the debug operation is complete */
+
+    return 0;
+#endif
+}

opal/mca/pmix/pmix2x/pmix/examples/debuggerd.c

@@ -0,0 +1,227 @@
+/*
+ * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2011 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2013 Los Alamos National Security, LLC.
+ *                         All rights reserved.
+ * Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2011      Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2013-2016 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2015      Mellanox Technologies, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+
+#define _GNU_SOURCE
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <time.h>
+
+#include <pmix_tool.h>
+
+/* define a structure for collecting returned
+ * info from a query */
+typedef struct {
+    volatile bool active;
+    pmix_info_t *info;
+    size_t ninfo;
+} myquery_data_t;
+
+
+static pmix_proc_t myproc;
+
+/* this is a callback function for the PMIx_Query
+ * API. The query will callback with a status indicating
+ * if the request could be fully satisfied, partially
+ * satisfied, or completely failed. The info parameter
+ * contains an array of the returned data, with the
+ * info->key field being the key that was provided in
+ * the query call. Thus, you can correlate the returned
+ * data in the info->value field to the requested key.
+ *
+ * Once we have dealt with the returned data, we must
+ * call the release_fn so that the PMIx library can
+ * cleanup */
+static void cbfunc(pmix_status_t status,
+                   pmix_info_t *info, size_t ninfo,
+                   void *cbdata,
+                   pmix_release_cbfunc_t release_fn,
+                   void *release_cbdata)
+{
+    myquery_data_t *mq = (myquery_data_t*)cbdata;
+    size_t n;
+
+    /* save the returned info - it will be
+     * released in the release_fn */
+    if (0 < ninfo) {
+        PMIX_INFO_CREATE(mq->info, ninfo);
+        mq->ninfo = ninfo;
+        for (n=0; n < ninfo; n++) {
+            fprintf(stderr, "Transferring %s\n", info[n].key);
+            PMIX_INFO_XFER(&mq->info[n], &info[n]);
+        }
+    }
+
+    /* let the library release the data */
+    if (NULL != release_fn) {
+        release_fn(release_cbdata);
+    }
+
+    /* release the block */
+    mq->active = false;
+}
+
+/* this is the event notification function we pass down below
+ * when registering for general events - i.e.,, the default
+ * handler. We don't technically need to register one, but it
+ * is usually good practice to catch any events that occur */
+static void notification_fn(size_t evhdlr_registration_id,
+                            pmix_status_t status,
+                            const pmix_proc_t *source,
+                            pmix_info_t info[], size_t ninfo,
+                            pmix_info_t results[], size_t nresults,
+                            pmix_event_notification_cbfunc_fn_t cbfunc,
+                            void *cbdata)
+{
+    if (NULL != cbfunc) {
+        cbfunc(PMIX_EVENT_ACTION_COMPLETE, NULL, 0, NULL, NULL, cbdata);
+    }
+}
+
+/* event handler registration is done asynchronously because it
+ * may involve the PMIx server registering with the host RM for
+ * external events. So we provide a callback function that returns
+ * the status of the request (success or an error), plus a numerical index
+ * to the registered event. The index is used later on to deregister
+ * an event handler - if we don't explicitly deregister it, then the
+ * PMIx server will do so when it see us exit */
+static void evhandler_reg_callbk(pmix_status_t status,
+                                 size_t evhandler_ref,
+                                 void *cbdata)
+{
+    volatile int *active = (volatile int*)cbdata;
+
+    if (PMIX_SUCCESS != status) {
+        fprintf(stderr, "Client %s:%d EVENT HANDLER REGISTRATION FAILED WITH STATUS %d, ref=%lu\n",
+                   myproc.nspace, myproc.rank, status, (unsigned long)evhandler_ref);
+    }
+    *active = status;
+}
+
+int main(int argc, char **argv)
+{
+    pmix_status_t rc;
+    pmix_value_t *val;
+    pmix_proc_t proc;
+    pmix_info_t *info;
+    size_t ninfo;
+    volatile int active;
+    pmix_query_t *query;
+    size_t nq, n;
+    myquery_data_t myquery_data;
+
+    /* init us - since we were launched by the RM, our connection info
+     * will have been provided at startup. */
+    if (PMIX_SUCCESS != (rc = PMIx_tool_init(&myproc, NULL, 0))) {
+        fprintf(stderr, "Debugger daemon ns %s rank %d: PMIx_tool_init failed: %d\n", myproc.nspace, myproc.rank, rc);
+        exit(0);
+    }
+    fprintf(stderr, "Debugger daemon ns %s rank %d: Running\n", myproc.nspace, myproc.rank);
+
+
+    /* register our default event handler */
+    active = -1;
+    PMIx_Register_event_handler(NULL, 0, NULL, 0,
+                                notification_fn, evhandler_reg_callbk, (void*)&active);
+    while (-1 == active) {
+        usleep(10);
+    }
+    if (0 != active) {
+        exit(active);
+    }
+
+    /* get the nspace of the job we are to debug */
+    (void)strncpy(proc.nspace, myproc.nspace, PMIX_MAX_NSLEN);
+    proc.rank = PMIX_RANK_WILDCARD;
+    if (PMIX_SUCCESS != (rc = PMIx_Get(&proc, PMIX_DEBUG_JOB, NULL, 0, &val))) {
+        fprintf(stderr, "[%s:%d] Failed to get job being debugged - error %d\n", myproc.nspace, myproc.rank, rc);
+        goto done;
+    }
+    if (NULL == val) {
+        fprintf(stderr, "Got NULL return\n");
+        goto done;
+    }
+    fprintf(stderr, "[%s:%d] Debugging %s\n", myproc.nspace, myproc.rank, val->data.string);
+
+    /* get our local proctable - for scalability reasons, we don't want to
+     * have our "root" debugger process get the proctable for everybody and
+     * send it out to us. So ask the local PMIx server for the pid's of
+     * our local target processes */
+    nq = 1;
+    PMIX_QUERY_CREATE(query, nq);
+    PMIX_ARGV_APPEND(query[0].keys, PMIX_QUERY_LOCAL_PROC_TABLE);
+    query[0].nqual = 1;
+    PMIX_INFO_CREATE(query[0].qualifiers, 1);
+    PMIX_INFO_LOAD(&query[0].qualifiers[0], PMIX_NSPACE, val->data.string, PMIX_STRING);  // the nspace we are enquiring about
+    /* setup the caddy to retrieve the data */
+    myquery_data.info = NULL;
+    myquery_data.ninfo = 0;
+    myquery_data.active = true;
+    /* execute the query */
+    if (PMIX_SUCCESS != (rc = PMIx_Query_info_nb(query, nq, cbfunc, (void*)&myquery_data))) {
+        fprintf(stderr, "PMIx_Query_info failed: %d\n", rc);
+        goto done;
+    }
+    while (myquery_data.active) {
+        usleep(10);
+    }
+    fprintf(stderr, "[%s:%d] Local proctable received\n", myproc.nspace, myproc.rank);
+
+
+    /* now that we have the proctable for our local processes, we can do our
+     * magic debugger stuff and attach to them. We then send a "release" event
+     * to them - i.e., it's the equivalent to setting the MPIR breakpoint. We
+     * do this with the event notification system */
+    (void)strncpy(proc.nspace, val->data.string, PMIX_MAX_NSLEN);
+    proc.rank = PMIX_RANK_WILDCARD;
+    /* we send the notification to just the local procs of the job being debugged */
+    ninfo = 1;
+    PMIX_INFO_CREATE(info, ninfo);
+    PMIX_INFO_LOAD(&info[0], PMIX_EVENT_CUSTOM_RANGE, &proc, PMIX_PROC);  // deliver to the target nspace
+    fprintf(stderr, "[%s:%u] Sending release\n", myproc.nspace, myproc.rank);
+    PMIx_Notify_event(PMIX_ERR_DEBUGGER_RELEASE,
+                      NULL, PMIX_RANGE_LOCAL,
+                      info, ninfo, NULL, NULL);
+
+    /* do some debugger magic */
+    n = 0;
+    fprintf(stderr, "[%s:%u] Hanging around awhile, doing debugger magic\n", myproc.nspace, myproc.rank);
+    while (n < 5) {
+        usleep(10);
+        ++n;
+    }
+
+  done:
+    /* finalize us */
+    fprintf(stderr, "Debugger daemon ns %s rank %d: Finalizing\n", myproc.nspace, myproc.rank);
+    if (PMIX_SUCCESS != (rc = PMIx_Finalize(NULL, 0))) {
+        fprintf(stderr, "Debugger daemon ns %s rank %d:PMIx_Finalize failed: %d\n", myproc.nspace, myproc.rank, rc);
+    } else {
+        fprintf(stderr, "Debugger daemon ns %s rank %d:PMIx_Finalize successfully completed\n", myproc.nspace, myproc.rank);
+    }
+    fflush(stderr);
+    return(0);
+}

opal/mca/pmix/pmix2x/pmix/include/pmix_common.h

@@ -264,6 +264,8 @@ typedef uint32_t pmix_rank_t;
 #define PMIX_FWD_STDOUT                     "pmix.fwd.stdout"        // (bool) forward stdout from spawned procs to me
 #define PMIX_FWD_STDERR                     "pmix.fwd.stderr"        // (bool) forward stderr from spawned procs to me
 #define PMIX_DEBUGGER_DAEMONS               "pmix.debugger"          // (bool) spawned app consists of debugger daemons
+#define PMIX_COSPAWN_APP                    "pmix.cospawn"           // (bool) designated app is to be spawned as a disconnected
+                                                                     //     job - i.e., not part of the "comm_world" of the job
 
 /* query attributes */
 #define PMIX_QUERY_NAMESPACES               "pmix.qry.ns"            // (char*) request a comma-delimited list of active nspaces
@@ -275,7 +277,7 @@ typedef uint32_t pmix_rank_t;
 #define PMIX_QUERY_LOCAL_PROC_TABLE         "pmix.qry.lptable"       // (char*) input nspace of job whose info is being requested
                                                                      //     returns (pmix_data_array_t) an array of pmix_proc_info_t for
                                                                      //     procs in job on same node
-#define PMIX_QUERY_AUTHORIZATIONS           "pmix.qry.auths"         // return operations tool is authorized to perform"
+#define PMIX_QUERY_AUTHORIZATIONS           "pmix.qry.auths"         // return operations tool is authorized to perform
 #define PMIX_QUERY_SPAWN_SUPPORT            "pmix.qry.spawn"         // return a comma-delimited list of supported spawn attributes
 #define PMIX_QUERY_DEBUG_SUPPORT            "pmix.qry.debug"         // return a comma-delimited list of supported debug attributes
 

opal/mca/pmix/pmix2x/pmix/src/dstore/pmix_esh.c

@@ -1250,7 +1250,6 @@ static int _esh_nspace_del(const char *nspace)
 
     if (NULL == (ns_map_data = _esh_session_map_search(nspace))) {
         rc = PMIX_ERR_NOT_AVAILABLE;
-        PMIX_ERROR_LOG(rc);
         return rc;
     }
 

opal/mca/pmix/pmix2x/pmix/src/event/pmix_event.h

@@ -117,6 +117,13 @@ PMIX_CLASS_DECLARATION(pmix_event_chain_t);
  * affected, plus any additional info provided by the server */
 void pmix_invoke_local_event_hdlr(pmix_event_chain_t *chain);
 
+/* invoke the server event notification handler */
+pmix_status_t pmix_server_notify_client_of_event(pmix_status_t status,
+                                                 const pmix_proc_t *source,
+                                                 pmix_data_range_t range,
+                                                 pmix_info_t info[], size_t ninfo,
+                                                 pmix_op_cbfunc_t cbfunc, void *cbdata);
+
 #define PMIX_REPORT_EVENT(e, f)                     \
     do {                                            \
         pmix_event_chain_t *_ch;                    \

opal/mca/pmix/pmix2x/pmix/src/event/pmix_event_notification.c

@@ -27,12 +27,6 @@ static pmix_status_t notify_server_of_event(pmix_status_t status,
                                             pmix_info_t info[], size_t ninfo,
                                             pmix_op_cbfunc_t cbfunc, void *cbdata);
 
-static pmix_status_t notify_client_of_event(pmix_status_t status,
-                                            const pmix_proc_t *source,
-                                            pmix_data_range_t range,
-                                            pmix_info_t info[], size_t ninfo,
-                                            pmix_op_cbfunc_t cbfunc, void *cbdata);
-
 /* if we are a client, we call this function to notify the server of
  * an event. If we are a server, our host RM will call this function
  * to notify us of an event */
@@ -45,9 +39,9 @@ PMIX_EXPORT pmix_status_t PMIx_Notify_event(pmix_status_t status,
     int rc;
 
     if (PMIX_PROC_SERVER == pmix_globals.proc_type) {
-        rc = notify_client_of_event(status, source, range,
-                                    info, ninfo,
-                                    cbfunc, cbdata);
+        rc = pmix_server_notify_client_of_event(status, source, range,
+                                                info, ninfo,
+                                                cbfunc, cbdata);
         pmix_output_verbose(2, pmix_globals.debug_output,
                             "pmix_server_notify_event source = %s:%d event_status = %d, rc= %d",
                             (NULL == source) ? "UNKNOWN" : source->nspace,
@@ -501,9 +495,8 @@ static void _notify_client_event(int sd, short args, void *cbdata)
                     }
                 }
                 pmix_output_verbose(2, pmix_globals.debug_output,
-                                    "pmix_server: notifying client %s:%d of status %s",
-                                    pr->peer->info->nptr->nspace, pr->peer->info->rank,
-                                    PMIx_Error_string(cd->status));
+                                    "pmix_server: notifying client %s:%d",
+                                    pr->peer->info->nptr->nspace, pr->peer->info->rank);
                 PMIX_RETAIN(cd->buf);
                 PMIX_SERVER_QUEUE_REPLY(pr->peer, 0, cd->buf);
             }
@@ -525,11 +518,11 @@ static void _notify_client_event(int sd, short args, void *cbdata)
  * (b) callback any of our own functions that have registered
  *     for this event
  */
-static pmix_status_t notify_client_of_event(pmix_status_t status,
-                                            const pmix_proc_t *source,
-                                            pmix_data_range_t range,
-                                            pmix_info_t info[], size_t ninfo,
-                                            pmix_op_cbfunc_t cbfunc, void *cbdata)
+pmix_status_t pmix_server_notify_client_of_event(pmix_status_t status,
+                                                 const pmix_proc_t *source,
+                                                 pmix_data_range_t range,
+                                                 pmix_info_t info[], size_t ninfo,
+                                                 pmix_op_cbfunc_t cbfunc, void *cbdata)
 {
     pmix_notify_caddy_t *cd;
     pmix_cmd_t cmd = PMIX_NOTIFY_CMD;
@@ -558,7 +551,7 @@ static pmix_status_t notify_client_of_event(pmix_status_t status,
                 cd->nondefault = true;
             } else if (0 == strncmp(info[n].key, PMIX_EVENT_CUSTOM_RANGE, PMIX_MAX_KEYLEN)) {
                 /* provides an array of pmix_proc_t identifying the procs
-                 * that are to receive this notification */
+                 * that are to receive this notification, or a single pmix_proc_t  */
                 if (PMIX_DATA_ARRAY == info[n].value.type &&
                     NULL != info[n].value.data.darray &&
                     NULL != info[n].value.data.darray->array) {

opal/mca/pmix/pmix2x/pmix/src/mca/ptl/tcp/ptl_tcp_component.c

@@ -185,11 +185,13 @@ static pmix_status_t component_open(void)
 
     /* check for environ-based directives
      * on system tmpdir to use */
-    if (NULL == (tdir = getenv("PMIX_SYSTEM_TMPDIR"))) {
-        if (NULL == (tdir = getenv("TMPDIR"))) {
-            if (NULL == (tdir = getenv("TEMP"))) {
-                if (NULL == (tdir = getenv("TMP"))) {
-                    tdir = "/tmp";
+    if (NULL == (tdir = getenv("PMIX_SERVER_TMPDIR"))) {
+        if (NULL == (tdir = getenv("PMIX_SYSTEM_TMPDIR"))) {
+            if (NULL == (tdir = getenv("TMPDIR"))) {
+                if (NULL == (tdir = getenv("TEMP"))) {
+                    if (NULL == (tdir = getenv("TMP"))) {
+                        tdir = "/tmp";
+                    }
                 }
             }
         }

opal/mca/pmix/pmix2x/pmix/src/server/pmix_server.c

@@ -517,9 +517,7 @@ static void _deregister_nspace(int sd, short args, void *cbdata)
     }
 
 #if defined(PMIX_ENABLE_DSTORE) && (PMIX_ENABLE_DSTORE == 1)
-    if (0 > (rc = pmix_dstore_nspace_del(cd->proc.nspace))) {
-        PMIX_ERROR_LOG(rc);
-    }
+    rc = pmix_dstore_nspace_del(cd->proc.nspace);
 #endif
 
     if (NULL != cd->opcbfunc) {

opal/mca/pmix/pmix2x/pmix/src/server/pmix_server_ops.c

@@ -1339,6 +1339,20 @@ pmix_status_t pmix_server_event_recvd_from_client(pmix_peer_t *peer,
         }
     }
 
+    /* check the range directive - if it is LOCAL, then we just
+     * process it ourselves. Otherwise, it needs to go up to our
+     * host for dissemination */
+    if (PMIX_RANGE_LOCAL == cd->range) {
+        if (PMIX_SUCCESS != (rc = pmix_server_notify_client_of_event(cd->status,
+                                                                     &cd->source,
+                                                                     cd->range,
+                                                                     cd->info, cd->ninfo,
+                                                                     local_cbfunc, cd))) {
+            goto exit;
+        }
+        return PMIX_SUCCESS;
+    }
+
     /* when we receive an event from a client, we just pass it to
      * our host RM for distribution - if any targeted recipients
      * are local to us, the host RM will let us know */

opal/mca/pmix/pmix2x/pmix/src/tool/pmix_tool.c

@@ -209,7 +209,6 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc,
         return rc;
     }
 
-pmix_output(0, "TOOL: SERVER CONNECTION COMPLETE");
     /* increment our init reference counter */
     pmix_globals.init_cntr++;
 
@@ -229,7 +228,6 @@ pmix_output(0, "TOOL: SERVER CONNECTION COMPLETE");
         }
     }
     if (NULL == nsptr) {
-pmix_output(0, "TOOL: NSPACE NOT FOUND");
         return PMIX_ERR_NOT_FOUND;
     }
 
@@ -464,8 +462,26 @@ pmix_output(0, "TOOL: NSPACE NOT FOUND");
     return rc;
 }
 
+/* callback for wait completion */
+static void wait_cbfunc(struct pmix_peer_t *pr,
+                        pmix_ptl_hdr_t *hdr,
+                        pmix_buffer_t *buf, void *cbdata)
+{
+    volatile bool *active = (volatile bool*)cbdata;
+
+    pmix_output_verbose(2, pmix_globals.debug_output,
+                        "pmix:tool wait_cbfunc received");
+
+    *active = false;
+}
+
 PMIX_EXPORT pmix_status_t PMIx_tool_finalize(void)
 {
+    pmix_buffer_t *msg;
+    pmix_cmd_t cmd = PMIX_FINALIZE_CMD;
+    pmix_status_t rc;
+    volatile bool active;
+
     if (1 != pmix_globals.init_cntr) {
         --pmix_globals.init_cntr;
         return PMIX_SUCCESS;
@@ -475,6 +491,32 @@ PMIX_EXPORT pmix_status_t PMIx_tool_finalize(void)
     pmix_output_verbose(2, pmix_globals.debug_output,
                         "pmix:tool finalize called");
 
+    /* setup a cmd message to notify the PMIx
+     * server that we are normally terminating */
+    msg = PMIX_NEW(pmix_buffer_t);
+    /* pack the cmd */
+    if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, &cmd, 1, PMIX_CMD))) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_RELEASE(msg);
+        return rc;
+    }
+
+
+    pmix_output_verbose(2, pmix_globals.debug_output,
+                         "pmix:tool sending finalize sync to server");
+
+    /* send to the server */
+    active = true;;
+    if (PMIX_SUCCESS != (rc = pmix_ptl.send_recv(&pmix_client_globals.myserver, msg,
+                                                 wait_cbfunc, (void*)&active))){
+        return rc;
+    }
+
+    /* wait for the ack to return */
+    PMIX_WAIT_FOR_COMPLETION(active);
+    pmix_output_verbose(2, pmix_globals.debug_output,
+                         "pmix:tool finalize sync received");
+
     /* shutdown services */
     pmix_rte_finalize();
 

opal/mca/pmix/pmix2x/pmix/src/util/error.c

@@ -153,10 +153,10 @@ const char* PMIx_Error_string(pmix_status_t errnum)
         return "PMIX_ERR_PERM";
     case PMIX_ERR_JOB_TERMINATED:
         return "PMIX_ERR_JOB_TERMINATED";
-    case PMIX_SUCCESS:
-        return "SUCCESS";
     case PMIX_MAX_ERR_CONSTANT:
         return "PMIX_ERR_WILDCARD";
+    case PMIX_SUCCESS:
+        return "SUCCESS";
     default:
         return "ERROR STRING NOT FOUND";
     }

opal/mca/pmix/pmix2x/pmix/test/Makefile.am

@@ -20,7 +20,7 @@
 #
 
 if !WANT_HIDDEN
-# these tests use interanl symbols
+# these tests use internal symbols
 # use --disable-visibility
 SUBDIRS = simple
 endif

opal/mca/pmix/pmix2x/pmix/test/pmi2_client.c

@@ -10,10 +10,12 @@
  *
  */
 
+#define _GNU_SOURCE
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <time.h>
+#include <stdarg.h>
 
 #include "pmi2.h"
 
@@ -22,24 +24,53 @@ static int _legacy = 0;
 /* Verbose level 0-silent, 1-fatal, 2-error, 3+ debug*/
 static int _verbose = 1;
 
-#define log_fatal(fmt, ...) \
-    do {                                                     \
-        if (_verbose > 0)                               \
-            fprintf(stderr, "FATAL " fmt, ##__VA_ARGS__);    \
-            exit(rc);    \
-    } while (0)
+static void log_fatal(const char *format, ...)
+{
+    va_list arglist;
+    char **output = NULL;
 
-#define log_error(fmt, ...) \
-    do {                                                     \
-        if (_verbose > 1)                          \
-            fprintf(stderr, "ERROR " fmt, ##__VA_ARGS__);    \
-    } while (0)
+    va_start(arglist, format);
+    if (_verbose > 0) {
+        if (0 > vasprintf(output, format, arglist)) {
+            return;
+        }
+        fprintf(stderr, "FATAL: %s", *output);
+        va_end(arglist);
+        free(*output);
+    }
+}
 
-#define log_info(fmt, ...) \
-    do {                                                     \
-        if (_verbose > 2)                          \
-            fprintf(stderr, "INFO  " fmt, ##__VA_ARGS__);    \
-    } while (0)
+static void log_error(const char *format, ...)
+{
+    va_list arglist;
+    char **output = NULL;
+
+    va_start(arglist, format);
+    if (_verbose > 0) {
+        if (0 > vasprintf(output, format, arglist)) {
+            return;
+        }
+        fprintf(stderr, "ERROR: %s", *output);
+        va_end(arglist);
+        free(*output);
+    }
+}
+
+static void log_info(const char *format, ...)
+{
+    va_list arglist;
+    char **output = NULL;
+
+    va_start(arglist, format);
+    if (_verbose > 0) {
+        if (0 > vasprintf(output, format, arglist)) {
+            return;
+        }
+        fprintf(stderr, "INFO: %s", *output);
+        va_end(arglist);
+        free(*output);
+    }
+}
 
 #define log_assert(e, msg) \
     do {                                                                \

opal/mca/pmix/pmix2x/pmix/test/pmi_client.c

@@ -10,10 +10,12 @@
  *
  */
 
+#define _GNU_SOURCE
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <time.h>
+#include <stdarg.h>
 
 #include "pmi.h"
 
@@ -22,24 +24,53 @@ static int _legacy = 0;
 /* Verbose level 0-silent, 1-fatal, 2-error, 3+ debug*/
 static int _verbose = 1;
 
-#define log_fatal(fmt, ...) \
-    do {                                                     \
-        if (_verbose > 0)                               \
-            fprintf(stderr, "FATAL " fmt, ##__VA_ARGS__);    \
-            exit(rc);    \
-    } while (0)
+static void log_fatal(const char *format, ...)
+{
+    va_list arglist;
+    char **output = NULL;
 
-#define log_error(fmt, ...) \
-    do {                                                     \
-        if (_verbose > 1)                          \
-            fprintf(stderr, "ERROR " fmt, ##__VA_ARGS__);    \
-    } while (0)
+    va_start(arglist, format);
+    if (_verbose > 0) {
+        if (0 > vasprintf(output, format, arglist)) {
+            return;
+        }
+        fprintf(stderr, "FATAL: %s", *output);
+        va_end(arglist);
+        free(*output);
+    }
+}
 
-#define log_info(fmt, ...) \
-    do {                                                     \
-        if (_verbose > 2)                          \
-            fprintf(stderr, "INFO  " fmt, ##__VA_ARGS__);    \
-    } while (0)
+static void log_error(const char *format, ...)
+{
+    va_list arglist;
+    char **output = NULL;
+
+    va_start(arglist, format);
+    if (_verbose > 0) {
+        if (0 > vasprintf(output, format, arglist)) {
+            return;
+        }
+        fprintf(stderr, "ERROR: %s", *output);
+        va_end(arglist);
+        free(*output);
+    }
+}
+
+static void log_info(const char *format, ...)
+{
+    va_list arglist;
+    char **output = NULL;
+
+    va_start(arglist, format);
+    if (_verbose > 0) {
+        if (0 > vasprintf(output, format, arglist)) {
+            return;
+        }
+        fprintf(stderr, "INFO: %s", *output);
+        va_end(arglist);
+        free(*output);
+    }
+}
 
 #define log_assert(e, msg) \
     do {                                                                \

opal/mca/pmix/pmix2x/pmix/test/test_common.c

@@ -205,6 +205,9 @@ void parse_cmd(int argc, char **argv, test_params *params)
     }
 
     // Fix rank if running under SLURM
+#if 0
+    /* the following "if" statement can never be true as rank is
+     * an unsigned 32-bit int */
     if( 0 > params->rank ){
         char *ranklist = getenv("SLURM_GTIDS");
         char *rankno = getenv("SLURM_LOCALID");
@@ -222,6 +225,7 @@ void parse_cmd(int argc, char **argv, test_params *params)
             pmix_argv_free(argv);
         }
     }
+#endif
 
     // Fix namespace if running under SLURM
     if( NULL == params->nspace ){

orte/mca/grpcomm/base/grpcomm_base_stubs.c

@@ -317,8 +317,9 @@ static int create_dmns(orte_grpcomm_signature_t *sig,
                          ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
                          (NULL == sig->signature) ? "NULL" : "NON-NULL"));
 
-    /* if NULL == procs, then all daemons are participating */
-    if (NULL == sig->signature) {
+    /* if NULL == procs, or the target jobid is our own,
+     * then all daemons are participating */
+    if (NULL == sig->signature || ORTE_PROC_MY_NAME->jobid == sig->signature[0].jobid) {
         *ndmns = orte_process_info.num_procs;
         *dmns = NULL;
         return ORTE_SUCCESS;
@@ -348,6 +349,7 @@ static int create_dmns(orte_grpcomm_signature_t *sig,
                 *dmns = dns;
                 return ORTE_SUCCESS;
             }
+            ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
             ORTE_FORCED_TERMINATE(ORTE_ERR_NOT_FOUND);
             *ndmns = 0;
             *dmns = NULL;

orte/mca/state/base/state_base_fns.c

@@ -852,9 +852,9 @@ void orte_state_base_check_all_complete(int fd, short args, void *cbdata)
                 continue;
             }
             OPAL_OUTPUT_VERBOSE((2, orte_state_base_framework.framework_output,
-                                 "%s releasing procs from node %s",
+                                 "%s releasing procs for job %s from node %s",
                                  ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
-                                 node->name));
+                                 ORTE_JOBID_PRINT(jdata->jobid), node->name));
             for (i = 0; i < node->procs->size; i++) {
                 if (NULL == (proc = (orte_proc_t*)opal_pointer_array_get_item(node->procs, i))) {
                     continue;

orte/mca/state/dvm/state_dvm.c

@@ -406,7 +406,7 @@ static void check_complete(int fd, short args, void *cbdata)
                 continue;
             }
             OPAL_OUTPUT_VERBOSE((2, orte_state_base_framework.framework_output,
-                                 "%s releasing procs from node %s",
+                                 "%s state:dvm releasing procs from node %s",
                                  ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
                                  node->name));
             for (i = 0; i < node->procs->size; i++) {
@@ -420,7 +420,7 @@ static void check_complete(int fd, short args, void *cbdata)
                 node->slots_inuse--;
                 node->num_procs--;
                 OPAL_OUTPUT_VERBOSE((2, orte_state_base_framework.framework_output,
-                                     "%s releasing proc %s from node %s",
+                                     "%s state:dvm releasing proc %s from node %s",
                                      ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
                                      ORTE_NAME_PRINT(&proc->name), node->name));
                 /* set the entry in the node array to NULL */

orte/orted/pmix/pmix_server_gen.c

@@ -59,17 +59,17 @@ static void _client_conn(int sd, short args, void *cbdata)
     } else {
         /* find the named process */
         p = NULL;
-        if (NULL == (jdata = orte_get_job_data_object(cd->proc->jobid))) {
+        if (NULL == (jdata = orte_get_job_data_object(cd->proc.jobid))) {
             return;
         }
         for (i=0; i < jdata->procs->size; i++) {
             if (NULL == (ptr = (orte_proc_t*)opal_pointer_array_get_item(jdata->procs, i))) {
                 continue;
             }
-            if (cd->proc->jobid != ptr->name.jobid) {
+            if (cd->proc.jobid != ptr->name.jobid) {
                 continue;
             }
-            if (cd->proc->vpid == ptr->name.vpid) {
+            if (cd->proc.vpid == ptr->name.vpid) {
                 p = ptr;
                 break;
             }
@@ -108,21 +108,29 @@ static void _client_finalized(int sd, short args, void *cbdata)
     } else {
         /* find the named process */
         p = NULL;
-        if (NULL == (jdata = orte_get_job_data_object(cd->proc->jobid))) {
+        if (NULL == (jdata = orte_get_job_data_object(cd->proc.jobid))) {
             return;
         }
         for (i=0; i < jdata->procs->size; i++) {
             if (NULL == (ptr = (orte_proc_t*)opal_pointer_array_get_item(jdata->procs, i))) {
                 continue;
             }
-            if (cd->proc->jobid != ptr->name.jobid) {
+            if (cd->proc.jobid != ptr->name.jobid) {
                 continue;
             }
-            if (cd->proc->vpid == ptr->name.vpid) {
+            if (cd->proc.vpid == ptr->name.vpid) {
                 p = ptr;
                 break;
             }
         }
+        /* if we came thru this code path, then this client must be an
+         * independent tool that connected to us - i.e., it wasn't
+         * something we spawned. For accounting purposes, we have to
+         * ensure the job complete procedure is run - otherwise, slots
+         * and other resources won't correctly be released */
+        ORTE_FLAG_SET(p, ORTE_PROC_FLAG_IOF_COMPLETE);
+        ORTE_FLAG_SET(p, ORTE_PROC_FLAG_WAITPID);
+        ORTE_ACTIVATE_PROC_STATE(&cd->proc, ORTE_PROC_STATE_TERMINATED);
     }
     if (NULL != p) {
         ORTE_FLAG_SET(p, ORTE_PROC_FLAG_HAS_DEREG);
@@ -157,17 +165,17 @@ static void _client_abort(int sd, short args, void *cbdata)
     } else {
         /* find the named process */
         p = NULL;
-        if (NULL == (jdata = orte_get_job_data_object(cd->proc->jobid))) {
+        if (NULL == (jdata = orte_get_job_data_object(cd->proc.jobid))) {
             return;
         }
         for (i=0; i < jdata->procs->size; i++) {
             if (NULL == (ptr = (orte_proc_t*)opal_pointer_array_get_item(jdata->procs, i))) {
                 continue;
             }
-            if (cd->proc->jobid != ptr->name.jobid) {
+            if (cd->proc.jobid != ptr->name.jobid) {
                 continue;
             }
-            if (cd->proc->vpid == ptr->name.vpid) {
+            if (cd->proc.vpid == ptr->name.vpid) {
                 p = ptr;
                 break;
             }
@@ -540,7 +548,7 @@ int pmix_server_query_fn(opal_process_name_t *requestor,
 
     /* need to threadshift this request */
     cd = OBJ_NEW(orte_pmix_server_op_caddy_t);
-    cd->proc = requestor;
+    cd->proc = *requestor;
     cd->info = queries;
     cd->infocbfunc = cbfunc;
     cd->cbdata = cbdata;

orte/orted/pmix/pmix_server_internal.h

@@ -75,7 +75,7 @@ typedef struct {
     opal_object_t super;
     opal_event_t ev;
     int status;
-    opal_process_name_t *proc;
+    opal_process_name_t proc;
     const char *msg;
     void *server_object;
     opal_list_t *procs;
@@ -140,7 +140,8 @@ OBJ_CLASS_DECLARATION(orte_pmix_mdx_caddy_t);
     do {                                                        \
         orte_pmix_server_op_caddy_t *_cd;                       \
         _cd = OBJ_NEW(orte_pmix_server_op_caddy_t);             \
-        _cd->proc = (p);                                        \
+        _cd->proc.jobid = (p)->jobid;                           \
+        _cd->proc.vpid = (p)->vpid;                             \
         _cd->server_object = (s);                               \
         _cd->status = (st);                                     \
         _cd->msg = (m);                                         \
@@ -237,4 +238,3 @@ extern pmix_server_globals_t orte_pmix_server_globals;
 END_C_DECLS
 
 #endif /* PMIX_SERVER_INTERNAL_H_ */
-