Add an "affinity" Open MPI extension (also describe the

--enable-mpi-ext configure switch in the top-level README file).

See Josh's excellent wiki page about OMPI extensions:

    https://svn.open-mpi.org/trac/ompi/wiki/MPIExtensions

This extension exposes a new API to MPI applications: 

{{{
int OMPI_Affinity_str(char ompi_bound[OMPI_AFFINITY_STRING_MAX],
                      char current_binding[OMPI_AFFINITY_STRING_MAX],
                      char exists[OMPI_AFFINITY_STRING_MAX]);
}}}

It returns 3 things.  Each are a prettyprint string describing sets of
processors in terms of sockets and cores:

 1. What Open MPI bound this process to.  If Open MPI didn't bind this
    process, the prettyprint string says so.
 1. What this process is currently bound to.  If the process is
    unbound, the prettyprint string says so.  This string is a
    separate OUT parameter to detect the case where some other entity
    bound the process (potentially after Open MPI bound it).
 1. What processors are availabile in the system, mainly for reference.

This commit was SVN r23018.

README

@@ -790,6 +790,12 @@ for a full list); a summary of the more commonly used ones follows:
   generates these bindings at compile time with a maximum number of
   dimensions as specified by this parameter.  The default value is 4.
 
+--enable-mpi-ext(=<list>)
+  Enable Open MPI's non-portable API extensions.  If no <list> is
+  specified, all of the extensions are enabled.
+
+  See "Open MPI API Extensions", below, for more details.
+
 --enable-mpirun-prefix-by-default
   This option forces the "mpirun" command to always behave as if
   "--prefix $prefix" was present on the command line (where $prefix is
@@ -1160,6 +1166,67 @@ Modular Component Architecture (MCA)" section, below.
 
 ===========================================================================
 
+Open MPI API Extensions
+-----------------------
+
+Open MPI contains a framework for extending the MPI API that is
+available to applications.  Each extension is usually a standalone set of
+functionality that is distinct from other extensions (similar to how
+Open MPI's plugins are usually unrelated to wach other).  These
+extensions provide new functions and/or constants that are available
+to MPI applications.
+
+WARNING: These extensions are neither standard nor portable to other
+MPI implementations!
+
+Compiling the extensions
+------------------------
+
+Open MPI extensions are not enabled by default; they must be enabled
+by Open MPI's configure script.  The --enable-mpi-ext command line
+switch accepts a comma-delimited list of extensions to enable, or, if
+it is specified without a list, all extensions are enabled.
+
+Since extensions are meant to be used by advanced users only, this
+file does not document which extensions are available or what they
+do.  Look in the ompi/mpiext/ directory to see the extensions; each
+subdirectory off that directory contains an extension.  Each has a
+README file that describes what it does.
+
+Using the extensions
+--------------------
+
+To reinforce the fact that thse extensions are non-standard, you must
+include a separate header file after <mpi.h> to obtain the function
+prototypes, constant declarations, etc.  For example:
+
+#include <mpi.h>
+#include <mpi-ext.h>
+
+int main() {
+    MPI_Init(NULL, NULL);
+#if defined(OPEN_MPI) && OPEN_MPI
+    {
+        char ompi_bound[OMPI_AFFINITY_STRING_MAX];
+        char current_binding[OMPI_AFFINITY_STRING_MAX];
+        char exists[OMPI_AFFINITY_STRING_MAX];
+        OMPI_Affinity_str(ompi_bound, current_bindings, exists);
+    }
+#endif
+    MPI_Finalize();
+    return 0;
+}
+
+Notice that the Open MPI-specific code is surrounded by the #if
+statement to ensure that it is only ever compiled by Open MPI.  
+
+The Open MPI wrapper compilers (mpicc and friends) should
+automatically insert all relevant compiler and linker flags necessary
+to use the extensions.  No special flags or steps should be necessary
+compared to "normal" MPI applications.
+
+===========================================================================
+
 Compiling Open MPI Applications
 -------------------------------
 

ompi/mpiext/affinity/Makefile.am

@@ -0,0 +1,34 @@
+#
+# Copyright (c) 2004-2009 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+
+include $(top_srcdir)/Makefile.man-page-rules
+
+headers = mpiext_affinity_c.h
+
+sources = c/affinity_str.c
+
+man_pages = OMPI_Affinity_str.3
+
+lib = libext_mpiext_affinity.la
+lib_sources = $(sources)
+
+extcomponentdir = $(pkglibdir)
+
+noinst_LTLIBRARIES = $(lib)
+libext_mpiext_affinity_la_SOURCES = $(lib_sources)
+libext_mpiext_affinity_la_LDFLAGS = -module -avoid-version
+
+ompidir = $(includedir)/openmpi/ompi/mpiext/affinity
+ompi_HEADERS = $(headers)
+
+nodist_man_MANS = $(man_pages)
+EXTRA_DIST = $(man_pages:.3=.3in)

ompi/mpiext/affinity/OMPI_Affinity_str.3in

@@ -0,0 +1,88 @@
+.\"Copyright 2007-2008 Sun Microsystems, Inc.
+.\" Copyright (c) 1996 Thinking Machines Corporation
+.\" Copyright (c) 2010 Cisco Systems, Inc.  All rights reserved.
+.TH OMPI_Affinity_str 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
+.SH NAME
+\fBOMPI_Affinity_str\fP \- Obtain prettyprint strings of processor affinity information for this process
+
+.SH SYNTAX
+.ft R
+.SH C Syntax
+.nf
+#include <mpi.h>
+#include <mpi-ext.h>
+
+int OMPI_Affinity_str(char \fIompi_bound\fP[OMPI_AFFINITY_STRING_MAX],
+                      char \fIcurrent_binding\fP[OMPI_AFFINITY_STRING_MAX],
+                      char \fIexists\fP[OMPI_AFFINITY_STRING_MAX])
+.
+.SH Fortran Syntax
+.nf
+There is no Fortran binding for this function.
+.
+.SH C++ Syntax
+.nf
+There is no C++ binding for this function.
+.
+.SH OUTPUT PARAMETERS
+.ft R
+.TP 1i
+ompi_bound
+A prettyprint string describing what processor(s) Open MPI bound this
+process to, or a string indicating that Open MPI did not bind this
+process.
+.
+.TP 1i
+current_binding
+A prettyprint string describing what processor(s) this process is
+currently bound to, or a string indicating that the process is bound
+to all available processors (and is therefore considered "unbound").
+.
+.TP 1i
+exists
+A prettyprint string describing the available sockets and sockets on
+this host.
+
+.SH DESCRIPTION
+.ft R
+Open MPI may bind a process to specific sockets and/or cores at
+process launch time.  This non-standard Open MPI function call returns
+prettyprint information about three things:
+.
+.TP
+Where Open MPI bound this process.
+The string returned in 
+.B
+ompi_bound
+will either indicate that Open MPI did not bind this process to
+anything, or it will contain a prettyprint description of the
+processor(s) to which Open MPI bound this process.
+.
+.TP
+Where this process is currently bound.
+Regardless of whether Open MPI bound this process or not, another
+entity may have bound it.  The string returned in
+.B current_binding
+will indicate what the
+.I
+current
+binding is of this process, regardless of what Open MPI may have done
+earlier.  The string returned will either indicate that the process is
+unbound (meaning that it is bound to all available processors) or it
+will contain a prettyprint description of the sockets and cores to
+which the process is currently bound.
+.
+.TP
+What processors exist.
+As a convenience to the user, the
+.B
+exists
+string will contain a prettyprint description of the sockets and cores
+that this process can see (which is
+.I usually
+all processors in the system).
+
+.SH SEE ALSO
+.ft R
+.nf
+mpirun(1)

ompi/mpiext/affinity/README.txt

@@ -0,0 +1,17 @@
+Copyright (c) 2010 Cisco Systems, Inc.  All rights reserved.
+
+Jeff Squyres
+19 April 2010
+
+This extension provides a single new function, OMPI_Affinity_str(),
+that provides 3 prettyprint strings as output:
+
+ompi_bound: describes what sockets/cores Open MPI bound this process
+to (or indicates that Open MPI did not bind this process).
+
+currently_bound: describes what sockets/cores this process is
+currently bound to (or indicates that it is unbound).
+
+exists: describes what processors are available in the current host.
+
+See OMPI_Affinity_str(3) for more details.

ompi/mpiext/affinity/c/affinity_str.c

@@ -0,0 +1,138 @@
+/*
+ * Copyright (c) 2004-2009 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ *
+ * Simple routine to expose three things to the MPI process:
+ *
+ * 1. What processor(s) Open MPI bound this process to
+ * 2. What processor(s) this process is bound to
+ * 3. What processor(s) exist on this host
+ * 
+ * Note that 1 and 2 may be different!
+ */
+
+#include "ompi_config.h"
+
+#include <stdio.h>
+#include <string.h>
+
+#include "opal/mca/paffinity/base/base.h"
+
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/mpiext/affinity/mpiext_affinity_c.h"
+
+static const char FUNC_NAME[] = "OMPI_Affinity";
+
+static int fill_ompi_bound(char str[OMPI_AFFINITY_STRING_MAX]);
+static int fill_current_binding(char str[OMPI_AFFINITY_STRING_MAX]);
+static int fill_exists(char str[OMPI_AFFINITY_STRING_MAX]);
+
+
+int OMPI_Affinity_str(char ompi_bound[OMPI_AFFINITY_STRING_MAX],
+                      char current_binding[OMPI_AFFINITY_STRING_MAX],
+                      char exists[OMPI_AFFINITY_STRING_MAX])
+{
+    int ret;
+
+    memset(ompi_bound, 0, sizeof(ompi_bound));
+    memset(current_binding, 0, sizeof(current_binding));
+
+    if (OPAL_SUCCESS != (ret = fill_ompi_bound(ompi_bound)) ||
+        OPAL_SUCCESS != (ret = fill_current_binding(current_binding)) ||
+        OPAL_SUCCESS != (ret = fill_exists(exists))) {
+        return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, ret, FUNC_NAME);
+    }
+
+    return MPI_SUCCESS;
+}
+
+static int fill_ompi_bound(char str[OMPI_AFFINITY_STRING_MAX])
+{
+    int ret;
+    opal_paffinity_base_cpu_set_t cset;
+
+    /* If OMPI did not bind, indicate that */
+    if (!opal_paffinity_base_bound) {
+        const char tmp[] = "Open MPI did not bind this process";
+        strncpy(str, tmp, OMPI_AFFINITY_STRING_MAX - 1);
+        return OPAL_SUCCESS;
+    }
+
+    /* Find out what OMPI bound us to and prettyprint it */
+    ret = 
+        opal_paffinity_base_parse_binding(opal_paffinity_base_applied_binding,
+                                          &cset);
+    if (OPAL_SUCCESS != ret) {
+        return ret;
+    }
+
+    return opal_paffinity_base_cset2str(str, OMPI_AFFINITY_STRING_MAX, &cset);
+}
+
+static int fill_current_binding(char str[OMPI_AFFINITY_STRING_MAX])
+{
+    int ret, flag;
+    opal_paffinity_base_cpu_set_t cset;
+
+    /* Are we bound anywhere? */
+    OPAL_PAFFINITY_PROCESS_IS_BOUND(cset, &flag);
+    if (!flag) {
+        const char tmp[] = "Not bound (or bound to all available processors)";
+        strncat(str, tmp, OMPI_AFFINITY_STRING_MAX - 1);
+        return OPAL_SUCCESS;
+    }
+
+    /* Get our binding */
+    ret = opal_paffinity_base_get(&cset);
+    if (OPAL_SUCCESS != ret) {
+        return ret;
+    }
+    return opal_paffinity_base_cset2str(str, OMPI_AFFINITY_STRING_MAX, &cset);
+}
+
+
+/* Prettyprint a list of all available processors */
+static int fill_exists(char str[OMPI_AFFINITY_STRING_MAX])
+{
+    int ret, i, num_sockets, num_cores;
+    char tmp[BUFSIZ];
+    const int stmp = sizeof(tmp) - 1;
+
+    tmp[stmp] = '\0';
+
+    /* Loop over the number of sockets in this machine */
+    ret = opal_paffinity_base_get_socket_info(&num_sockets);
+    if (OPAL_SUCCESS != ret) {
+        return ret;
+    }
+    for (i = 0; i < num_sockets; ++i) {
+        if (i > 0) {
+            strncat(str, ", ", OMPI_AFFINITY_STRING_MAX - strlen(str));
+        }
+        snprintf(tmp, stmp, "socket %d has ", i);
+        strncat(str, tmp, OMPI_AFFINITY_STRING_MAX - strlen(str));
+
+        /* Loop over the number of cores in this socket */
+        ret = opal_paffinity_base_get_core_info(i, &num_cores);
+        if (OPAL_SUCCESS != ret) {
+            return ret;
+        }
+        if (1 == num_cores) {
+            strncat(str, "1 core", OMPI_AFFINITY_STRING_MAX - strlen(str));
+        } else {
+            snprintf(tmp, stmp, "%d cores", num_cores);
+            strncat(str, tmp, OMPI_AFFINITY_STRING_MAX - strlen(str));
+        }
+    }
+
+    return OPAL_SUCCESS;
+}

ompi/mpiext/affinity/configure.params

@@ -0,0 +1,12 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2004-2009 The Trustees of Indiana University.
+#                         All rights reserved.
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+
+PARAM_CONFIG_FILES="Makefile"

ompi/mpiext/affinity/mpiext_affinity_c.h

@@ -0,0 +1,18 @@
+/*
+ * Copyright (c) 2004-2009 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ *
+ */
+
+#define OMPI_AFFINITY_STRING_MAX 1024
+
+OMPI_DECLSPEC int OMPI_Affinity_str(char ompi_bound[OMPI_AFFINITY_STRING_MAX],
+                                    char current_binding[OMPI_AFFINITY_STRING_MAX],
+                                    char exists[OMPI_AFFINITY_STRING_MAX]);
+