Bump up to hwloc 1.1.1

This commit was SVN r24312.

opal/mca/paffinity/hwloc/hwloc/NEWS

@@ -1,7 +1,7 @@
 Copyright © 2009 CNRS
-Copyright © 2009-2010 INRIA
+Copyright © 2009-2011 INRIA
-Copyright © 2009-2010 Université Bordeaux 1
+Copyright © 2009-2011 Université Bordeaux 1
-Copyright © 2009-2010 Cisco Systems, Inc.  All rights reserved.
+Copyright © 2009-2011 Cisco Systems, Inc.  All rights reserved.
 
 $COPYRIGHT$
 
@@ -16,6 +16,33 @@ bug fixes (and other actions) for each version of hwloc since version
 0.9 (as initially released as "libtopology", then re-branded to "hwloc"
 in v0.9.1).
 
+
+Version 1.1.1
+-------------
+* Add hwloc_get_api_version() which returns the version of hwloc used
+  at runtime. Thanks to Guy Streeter for the suggestion.
+* Fix the number of hugepages reported for NUMA nodes on Linux.
+* Fix hwloc_bitmap_to_ulong() right after allocating the bitmap.
+  Thanks to Bernd Kallies for reporting the problem.
+* Fix hwloc_bitmap_from_ith_ulong() to properly zero the first ulong.
+  Thanks to Guy Streeter for reporting the problem.
+* Fix hwloc_get_membind_nodeset() on Linux.
+  Thanks to Bernd Kallies for reporting the problem and providing a patch.
+* Fix some file descriptor leaks in the Linux discovery.
+* Fix the minimum width of NUMA nodes, caches and the legend in the graphical
+  lstopo output. Thanks to Jirka Hladky for reporting the problem.
+* Various fixes to bitmap conversion from/to taskset-strings.
+* Fix and document snprintf functions behavior when the buffer size is too
+  small or zero. Thanks to Guy Streeter for reporting the problem.
+* Fix configure to avoid spurious enabling of the cpuid backend.
+  Thanks to Tim Anderson for reporting the problem.
+* Cleanup error management in hwloc-gather-topology.sh.
+  Thanks to Jirka Hladky for reporting the problem and providing a patch.
+* Add a manpage and usage for hwloc-gather-topology.sh on Linux.
+  Thanks to Jirka Hladky for providing a patch.
+* Memory binding documentation enhancements.
+
+
 Version 1.1.0
 -------------
 

opal/mca/paffinity/hwloc/hwloc/README

@@ -23,7 +23,7 @@ from PLPA to hwloc.
 hwloc supports the following operating systems:
 
   * Linux (including old kernels not having sysfs topology information, with
- knowledge of cpusets, offline cpus, ScaleMP vSMP, and Kerrighed support)
+ knowledge of cpusets, offline CPUs, ScaleMP vSMP, and Kerrighed support)
   * Solaris
   * AIX
   * Darwin / OS X
@@ -345,8 +345,8 @@ that it contains. These bitmaps may be used for CPU binding and Memory binding.
 hwloc offers an extensive bitmap manipulation interface in hwloc/bitmap.h.
 
 Moreover, hwloc also comes with additional helpers for interoperability with
-several commonly used environments. See the Interoperability with other
+several commonly used environments. See the Interoperability With Other
-software section for details.
+Software section for details.
 
 The complete API documentation is available in a full set of HTML pages, man
 pages, and self-contained PDF files (formatted for both both US letter and A4
@@ -671,12 +671,12 @@ Further Reading
 The documentation chapters include
 
   * Terms and Definitions
-  * Command-line tools
+  * Command-Line Tools
-  * Environment variables
+  * Environment Variables
-  * CPU Binding and Memory Binding
+  * CPU and Memory Binding Overview
-  * Interoperability with other software
+  * Interoperability With Other Software
-  * Thread safety
+  * Thread Safety
-  * Embedding hwloc in other software
+  * Embedding hwloc in Other Software
   * Switching from PLPA to hwloc
   * Frequently Asked Questions
 
@@ -684,5 +684,5 @@ Make sure to have had a look at those too!
 
 -------------------------------------------------------------------------------
 
-Generated on Thu Dec 16 2010 16:48:29 for Hardware Locality (hwloc) by  doxygen
+Generated on Mon Jan 24 2011 17:12:06 for Hardware Locality (hwloc) by  doxygen
-1.7.1
+1.7.2

opal/mca/paffinity/hwloc/hwloc/VERSION

@@ -7,7 +7,7 @@
 
 major=1
 minor=1
-release=0
+release=1
 
 # greek is used for alpha or beta release tags.  If it is non-empty,
 # it will be appended to the version number.  It does not have to be
@@ -33,11 +33,11 @@ want_svn=0
 # distribution tarball is being made from an SVN checkout, the value
 # of svn_r in this file is replaced with the output of "svnversion".
 
-svn_r=r2935
+svn_r=r3129
 
 # The date when this release was created
 
-date="Dec 16, 2010"
+date="Jan 24, 2011"
 
 # The shared library version of hwloc's public library.  This version
 # is maintained in accordance with the "Library Interface Versions"
@@ -50,4 +50,4 @@ date="Dec 16, 2010"
 # 2. Version numbers are described in the Libtool current:revision:age
 # format.
 
-libhwloc_so_version=1:0:1
+libhwloc_so_version=2:0:2

opal/mca/paffinity/hwloc/hwloc/config/hwloc.m4

@@ -1,7 +1,7 @@
 dnl -*- Autoconf -*-
 dnl
-dnl Copyright (c) 2009 INRIA
+dnl Copyright (c) 2009-2010 INRIA
-dnl Copyright (c) 2009 Université Bordeaux 1
+dnl Copyright (c) 2009-2011 Université Bordeaux 1
 dnl Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
 dnl                         University Research and Technology
 dnl                         Corporation.  All rights reserved.
@@ -9,8 +9,6 @@ dnl Copyright (c) 2004-2005 The Regents of the University of California.
 dnl                         All rights reserved.
 dnl Copyright (c) 2004-2008 High Performance Computing Center Stuttgart, 
 dnl                         University of Stuttgart.  All rights reserved.
-dnl Copyright © 2010  INRIA
-dnl Copyright © 2010  Université of Bordeaux
 dnl Copyright © 2006-2010  Cisco Systems, Inc.  All rights reserved.
 
 # Main hwloc m4 macro, to be invoked by the user
@@ -238,7 +236,7 @@ AC_DEFUN([HWLOC_SETUP_CORE_AFTER_C99],[
         AC_MSG_WARN([***********************************************************])
         AC_MSG_WARN([*** hwloc does not support this system.])
         AC_MSG_WARN([*** hwloc will *attempt* to build (but it may not work).])
-        AC_MSG_WARN([*** hwloc's run-time results may be reduced to showing just one processor.])
+        AC_MSG_WARN([*** hwloc run-time results may be reduced to showing just one processor.])
         AC_MSG_WARN([*** You have been warned.])
         AC_MSG_WARN([*** Pausing to give you time to read this message...])
         AC_MSG_WARN([***********************************************************])
@@ -283,7 +281,7 @@ AC_DEFUN([HWLOC_SETUP_CORE_AFTER_C99],[
     _HWLOC_CHECK_VISIBILITY
     HWLOC_CFLAGS="$HWLOC_FLAGS $HWLOC_VISIBILITY_CFLAGS"
     AS_IF([test "$HWLOC_VISIBILITY_CFLAGS" != ""],
-          [AC_MSG_WARN(["$HWLOC_VISIBILITY_CFLAGS" has been added to hwloc's CFLAGS])])
+          [AC_MSG_WARN(["$HWLOC_VISIBILITY_CFLAGS" has been added to the hwloc CFLAGS])])
 
     #
     # Check for inline compatibility support
@@ -584,7 +582,7 @@ AC_DEFUN([HWLOC_SETUP_CORE_AFTER_C99],[
     AC_MSG_CHECKING([for cpuid])
     old_CPPFLAGS="$CPPFLAGS"
     CFLAGS="$CFLAGS -I$HWLOC_top_srcdir/include"
-    AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
+    AC_LINK_IFELSE([AC_LANG_PROGRAM([[
         #include <stdio.h>
         #include <private/cpuid.h>
       ]], [[

opal/mca/paffinity/hwloc/hwloc/config/hwloc_internal.m4

@@ -105,6 +105,22 @@ EOF
                  [hwloc_generate_doxs=yes], [hwloc_generate_doxs=no])
     AC_MSG_RESULT([$hwloc_generate_doxs])
     
+    # Linux and OS X take different sed arguments.
+    AC_PROG_SED
+    AC_MSG_CHECKING([if the sed -i option requires an argument])
+    rm -f conftest
+    cat > conftest <<EOF
+hello
+EOF
+    $SED -i -e s/hello/goodbye/ conftest 2> /dev/null
+    AS_IF([test -f conftest-e],
+          [SED_I="$SED -i ''"
+           AC_MSG_RESULT([yes])],
+          [SED_I="$SED -i"
+           AC_MSG_RESULT([no])])
+    rm -f conftest conftest-e
+    AC_SUBST([SED_I])
+
     # Making the top-level README requires w3m or lynx.
     AC_ARG_VAR([W3M], [Location of the w3m program (required to building the top-level hwloc README file)])
     AC_PATH_TOOL([W3M], [w3m])

opal/mca/paffinity/hwloc/hwloc/include/hwloc.h

@@ -1,8 +1,8 @@
 /*
  * Copyright © 2009 CNRS
  * Copyright © 2009-2010 INRIA
- * Copyright © 2009-2010 Université Bordeaux 1
+ * Copyright © 2009-2011 Université Bordeaux 1
- * Copyright © 2009-2010 Cisco Systems, Inc.  All rights reserved.
+ * Copyright © 2009-2011 Cisco Systems, Inc.  All rights reserved.
  * See COPYING in top-level directory.
  */
 
@@ -50,6 +50,9 @@ extern "C" {
 /** \brief Indicate at build time which hwloc API version is being used. */
 #define HWLOC_API_VERSION 0x00010100
 
+/** \brief Indicate at runtime which hwloc API version was used at build time. */
+HWLOC_DECLSPEC unsigned hwloc_get_api_version(void);
+
 /** @} */
 
 
@@ -69,21 +72,41 @@ typedef struct hwloc_topology * hwloc_topology_t;
 
 
 
-/** \defgroup hwlocality_sets Object sets
+/** \defgroup hwlocality_sets Object sets (hwloc_cpuset_t and hwloc_nodeset_t)
+ *
+ * Hwloc uses bitmaps to represent two distinct kinds of object sets:
+ * CPU sets (::hwloc_cpuset_t) and NUMA node sets (::hwloc_nodeset_t).
+ * These types are both typedefs to a common back end type
+ * (::hwloc_bitmap_t), and therefore all the hwloc bitmap functions
+ * are applicable to both ::hwloc_cpuset_t and ::hwloc_nodeset_t (see
+ * \ref hwlocality_bitmap).
+ *
+ * The rationale for having two different types is that even though
+ * the actions one wants to perform on these types are the same (e.g.,
+ * enable and disable individual items in the set/mask), they're used
+ * in very different contexts: one for specifying which processors to
+ * use and one for specifying which NUMA nodes to use.  Hence, the
+ * name difference is really just to reflect the intent of where the
+ * type is used.
+ *
  * @{
  */
 
-/** \brief A CPU set is a bitmap whose bits are set according to CPU physical OS indexes.
+/** \brief A CPU set is a bitmap whose bits are set according to CPU
+ * physical OS indexes.
  *
- * It may be consulted and modified with the bitmap API as any ::hwloc_bitmap_t (see hwloc/bitmap.h).
+ * It may be consulted and modified with the bitmap API as any
+ * ::hwloc_bitmap_t (see hwloc/bitmap.h).
  */
 typedef hwloc_bitmap_t hwloc_cpuset_t;
 /** \brief A non-modifiable ::hwloc_cpuset_t. */
 typedef hwloc_const_bitmap_t hwloc_const_cpuset_t;
 
-/** \brief A node set is a bitmap whose bits are set according to NUMA memory node physical OS indexes.
+/** \brief A node set is a bitmap whose bits are set according to NUMA
+ * memory node physical OS indexes.
  *
- * It may be consulted and modified with the bitmap API as any ::hwloc_bitmap_t (see hwloc/bitmap.h).
+ * It may be consulted and modified with the bitmap API as any
+ * ::hwloc_bitmap_t (see hwloc/bitmap.h).
  *
  * When binding memory on a system without any NUMA node
  * (when the whole memory is considered as a single memory bank),
@@ -149,7 +172,7 @@ typedef enum {
   HWLOC_OBJ_GROUP,	/**< \brief Group objects.
 			  * Objects which do not fit in the above but are
 			  * detected by hwloc and are useful to take into
-			  * account for affinity. For instance, some OSes
+			  * account for affinity. For instance, some operating systems
 			  * expose their arbitrary processors aggregation this
 			  * way.  And hwloc may insert such objects to group
 			  * NUMA nodes according to their distances.
@@ -216,7 +239,7 @@ struct hwloc_obj_memory_s {
 
 /** \brief Structure of a topology object
  *
- * Applications mustn't modify any field except ::userdata .
+ * Applications must not modify any field except hwloc_obj.userdata.
  */
 struct hwloc_obj {
   /* physical information */
@@ -800,8 +823,10 @@ HWLOC_DECLSPEC hwloc_obj_type_t hwloc_obj_type_of_string (const char * string) _
  * It differs from hwloc_obj_type_string() because it prints type attributes such
  * as cache depth.
  *
- * \return how many characters were actually written (not including the ending
+ * If \p size is 0, \p string may safely be \c NULL.
- * \\0), or -1 on error.
+ *
+ * \return the number of character that were actually written if not truncating,
+ * or that would have been written (not including the ending \\0).
  */
 HWLOC_DECLSPEC int hwloc_obj_type_snprintf(char * __hwloc_restrict string, size_t size, hwloc_obj_t obj,
 				   int verbose);
@@ -812,8 +837,10 @@ HWLOC_DECLSPEC int hwloc_obj_type_snprintf(char * __hwloc_restrict string, size_
  *
  * Only the major attributes are printed in non-verbose mode.
  *
- * \return how many characters were actually written (not including the ending
+ * If \p size is 0, \p string may safely be \c NULL.
- * \\0), or -1 on error.
+ *
+ * \return the number of character that were actually written if not truncating,
+ * or that would have been written (not including the ending \\0).
  */
 HWLOC_DECLSPEC int hwloc_obj_attr_snprintf(char * __hwloc_restrict string, size_t size, hwloc_obj_t obj, const char * __hwloc_restrict separator,
 				   int verbose);
@@ -833,8 +860,10 @@ HWLOC_DECLSPEC int hwloc_obj_attr_snprintf(char * __hwloc_restrict string, size_
  * \p indexprefix is used to prefix the \p os_index attribute number of
  * the object in the description. If \c NULL, the \c # character is used.
  *
- * \return how many characters were actually written (not including the ending
+ * If \p size is 0, \p string may safely be \c NULL.
- * \\0), or -1 on error.
+ *
+ * \return the number of character that were actually written if not truncating,
+ * or that would have been written (not including the ending \\0).
  */
 HWLOC_DECLSPEC int hwloc_obj_snprintf(char * __hwloc_restrict string, size_t size,
 			     hwloc_topology_t topology, hwloc_obj_t obj,
@@ -842,7 +871,11 @@ HWLOC_DECLSPEC int hwloc_obj_snprintf(char * __hwloc_restrict string, size_t siz
 
 /** \brief Stringify the cpuset containing a set of objects.
  *
- * \return how many characters were actually written (not including the ending \\0). */
+ * If \p size is 0, \p string may safely be \c NULL.
+ *
+ * \return the number of character that were actually written if not truncating,
+ * or that would have been written (not including the ending \\0).
+ */
 HWLOC_DECLSPEC int hwloc_obj_cpuset_snprintf(char * __hwloc_restrict str, size_t size, size_t nobj, const hwloc_obj_t * __hwloc_restrict objs);
 
 /** \brief Search the given key name in object infos and return the corresponding value.
@@ -867,14 +900,15 @@ hwloc_obj_get_info_by_name(hwloc_obj_t obj, const char *name)
  *
  * It is often useful to call hwloc_bitmap_singlify() first so that a single CPU
  * remains in the set. This way, the process will not even migrate between
- * different CPUs. Some OSes also only support that kind of binding.
+ * different CPUs. Some operating systems also only support that kind of binding.
  *
- * \note Some OSes do not provide all ways to bind processes, threads, etc and
+ * \note Some operating systems do not provide all hwloc-supported
- * the corresponding binding functions may fail. -1 is returned and errno is set
+ * mechanisms to bind processes, threads, etc. and the corresponding
- * to ENOSYS when it is not possible to bind the requested kind of object
+ * binding functions may fail. -1 is returned and errno is set to
- * processes/threads. errno is set to EXDEV when the requested cpuset can not be
+ * ENOSYS when it is not possible to bind the requested kind of object
- * enforced (e.g. some systems only allow one CPU, and some other systems only
+ * processes/threads. errno is set to EXDEV when the requested cpuset
- * allow one NUMA node).
+ * can not be enforced (e.g. some systems only allow one CPU, and some
+ * other systems only allow one NUMA node).
  *
  * The most portable version that should be preferred over the others, whenever
  * possible, is
@@ -883,7 +917,7 @@ hwloc_obj_get_info_by_name(hwloc_obj_t obj, const char *name)
  * hwloc_set_cpubind(topology, set, 0),
  * \endcode
  *
- * as it just binds the current program, assuming it is monothread, or
+ * as it just binds the current program, assuming it is single-threaded, or
  *
  * \code
  * hwloc_set_cpubind(topology, set, HWLOC_CPUBIND_THREAD),
@@ -895,20 +929,22 @@ hwloc_obj_get_info_by_name(hwloc_obj_t obj, const char *name)
  * \note To unbind, just call the binding function with either a full cpuset or
  * a cpuset equal to the system cpuset.
  *
- * \note On some OSes, CPU binding may have effects on memory binding, see
+ * \note On some operating systems, CPU binding may have effects on memory binding, see
  * ::HWLOC_CPUBIND_NOMEMBIND
  * @{
  */
 
 /** \brief Process/Thread binding flags.
  *
- * These flags can be used to refine the binding policy.
+ * These bit flags can be used to refine the binding policy.
  *
- * The default (0) is to bind the current process, assumed to be mono-thread,
+ * The default (0) is to bind the current process, assumed to be
- * in a non-strict way.  This is the most portable way to bind as all OSes
+ * single-threaded, in a non-strict way.  This is the most portable
- * usually provide it.
+ * way to bind as all operating systems usually provide it.
  *
- * \note Not all systems support all kinds of binding.
+ * \note Not all systems support all kinds of binding.  See the
+ * "Detailed Description" section of \ref hwlocality_cpubinding for a
+ * description of errors that can occur.
  */
 typedef enum {
   HWLOC_CPUBIND_PROCESS = (1<<0), /**< \brief Bind all threads of the current
@@ -920,7 +956,7 @@ typedef enum {
                                    * \hideinitializer
                                    *
                                    * By default, when the designated CPUs are
-                                   * all busy while other CPUs are idle, OSes
+                                   * all busy while other CPUs are idle, operating systems
                                    * may execute the thread/process on those
                                    * other CPUs instead of the designated CPUs,
                                    * to let them progress anyway.  Strict
@@ -929,10 +965,10 @@ typedef enum {
                                    * designated CPUs, even when those are busy
                                    * with other tasks and other CPUs are idle.
                                    *
-                                   * \note Depending on OSes and
+                                   * \note Depending on the operating system,
-                                   * implementations, strict binding may not be
+                                   * strict binding may not be
-                                   * possible (implementation reason) or not
+                                   * possible (e.g., the OS does not implement it) or not
-                                   * allowed (administrative reasons), and the
+                                   * allowed (e.g., for an administrative reasons), and the
                                    * function will fail in that case.
 				   *
 				   * When retrieving the binding of a process,
@@ -947,7 +983,7 @@ typedef enum {
   HWLOC_CPUBIND_NOMEMBIND = (1<<3)/**< \brief Avoid any effect on memory binding
                                    * \hideinitializer
                                    *
-                                   * On some OSes, some CPU binding function
+                                   * On some operating systems, some CPU binding function
                                    * would also bind the memory on the
                                    * corresponding NUMA node.  It is often not
                                    * a problem for the application, but if it
@@ -1016,165 +1052,475 @@ HWLOC_DECLSPEC int hwloc_get_thread_cpubind(hwloc_topology_t topology, hwloc_thr
 
 /** \defgroup hwlocality_membinding Memory binding
  *
- * \note Not all OSes support all ways to bind existing allocated memory
+ * \note Not all operating systems support all ways to bind existing
- * (migration), future memory allocation, explicit memory allocation, etc. and
+ * allocated memory (e.g., migration), future memory allocation,
- * the corresponding binding functions may fail. -1 is returned and errno is
+ * explicit memory allocation, etc.  Using a binding flag or policy
- * set to ENOSYS when it is not possible to bind the requested kind of object
+ * that is not supported by the underlying OS will cause hwloc's
- * processes/threads). errno is set to EXDEV when the requested cpuset can not
+ * binding functions to fail and return -1.  errno will be set to
- * be enforced (e.g. some systems only allow one NUMA node).
+ * ENOSYS when the system does support the specified action or policy
+ * (e.g., some systems only allow binding memory on a per-thread
+ * basis, whereas other systems only allow binding memory for all
+ * threads in a process).  errno will be set to EXDEV when the
+ * requested cpuset can not be enforced (e.g., some systems only allow
+ * binding memory to a single NUMA node).
  *
- * The most portable version that should be preferred over the others, whenever
+ * The most portable form that should be preferred over the others
- * possible, is
+ * whenever possible is as follows:
  *
  * \code
- * hwloc_alloc_membind_policy(topology, size, set, HWLOC_MEMBIND_DEFAULT, 0),
+ * hwloc_alloc_membind_policy(topology, size, set, 
+ *                            HWLOC_MEMBIND_DEFAULT, 0);
  * \endcode
  *
- * which will try to allocate new data bound to the given set, possibly by
+ * This will allocate some memory hopefully bound to the specified set.
- * changing the current memory binding policy, or at worse allocate memory
+ * To do so, hwloc will possibly have to change the current memory
- * without binding it at all.  Since HWLOC_MEMBIND_STRICT is not given, this
+ * binding policy in order to actually get the memory bound, if the OS
- * will even not fail unless a mere malloc() itself would fail, i.e. ENOMEM.
+ * does not provide any other way to simply allocate bound memory
+ * without changing the policy for all allocations. That is the
+ * difference with hwloc_alloc_membind(), which will never change the
+ * current memory binding policy. Note that since HWLOC_MEMBIND_STRICT
+ * was not specified, failures to bind will not be reported --
+ * generally, only memory allocation failures will be reported (e.g.,
+ * even a plain malloc() would have failed with ENOMEM).
  *
- * Each binding is available with a CPU set argument or a NUMA memory node set
+ * Each hwloc memory binding function is available in two forms: one
- * argument. The name of the latter ends with _nodeset. It is also possible to
+ * that takes a CPU set argument and another that takes a NUMA memory
- * convert between CPU set and node set using ::hwloc_cpuset_to_nodeset or
+ * node set argument (see \ref hwlocality_sets and \ref
- * ::hwloc_cpuset_from_nodeset.
+ * hwlocality_bitmap for a discussion of CPU sets and NUMA memory node
+ * sets).  The names of the latter form end with _nodeset.  It is also
+ * possible to convert between CPU set and node set using
+ * hwloc_cpuset_to_nodeset() or hwloc_cpuset_from_nodeset().
  *
- * \note On some OSes, memory binding may have effects on CPU binding, see
+ * \note On some operating systems, memory binding affects the CPU
- * ::HWLOC_MEMBIND_NOCPUBIND
+ * binding; see ::HWLOC_MEMBIND_NOCPUBIND 
  * @{
  */
 
 /** \brief Memory binding policy.
  *
- * These can be used to choose the binding policy.
+ * These constants can be used to choose the binding policy.  Only one policy can
+ * be used at a time (i.e., the values cannot be OR'ed together).
  *
- * Note that not all systems support all kinds of binding.
+ * \note Not all systems support all kinds of binding.  See the
+ * "Detailed Description" section of \ref hwlocality_membinding for a
+ * description of errors that can occur.
  */
 typedef enum {
   HWLOC_MEMBIND_DEFAULT =	0,	/**< \brief Reset the memory allocation policy to the system default.
 					 * \hideinitializer */
-  HWLOC_MEMBIND_FIRSTTOUCH =	1,	/**< \brief Allocate memory on the given nodes, but preferably on the
+  HWLOC_MEMBIND_FIRSTTOUCH =	1,	/**< \brief Allocate memory
-					  node where the first accessor is running.
+                                         * but do not immediately bind
+                                         * it to a specific locality.
+                                         * Instead, each page in the
+                                         * allocation is bound only
+                                         * when it is first touched.
+                                         * Pages are individually
+                                         * bound to the local NUMA
+                                         * node of the first thread
+                                         * that touches it.
                                          * \hideinitializer */
-  HWLOC_MEMBIND_BIND =		2,	/**< \brief Allocate memory on the given nodes.
+  HWLOC_MEMBIND_BIND =		2,	/**< \brief Allocate memory on the specified nodes.
 					 * \hideinitializer */
-  HWLOC_MEMBIND_INTERLEAVE =	3,	/**< \brief Allocate memory on the given nodes in a round-robin manner.
+  HWLOC_MEMBIND_INTERLEAVE =	3,	/**< \brief Allocate memory on
+                                         * the given nodes in an
+                                         * interleaved / round-robin
+                                         * manner.  The precise layout
+                                         * of the memory across
+                                         * multiple NUMA nodes is
+                                         * OS/system specific.
+                                         * Interleaving can be useful
+                                         * when threads distributed across
+                                         * the specified NUMA nodes
+                                         * will all be accessing the whole
+                                         * memory range concurrently, since
+                                         * the interleave will then balance
+                                         * the memory references.
                                          * \hideinitializer */
-  HWLOC_MEMBIND_REPLICATE =	4,	/**< \brief Replicate memory on the given nodes.
+  HWLOC_MEMBIND_REPLICATE =	4,	/**< \brief Replicate memory
+					 * on the given nodes; reads
+					 * from this memory will
+					 * attempt to be serviced from
+					 * the NUMA node local to the
+					 * reading thread.
+					 * Replicating can be useful
+					 * when multiple threads from
+					 * the specified NUMA nodes
+					 * will be sharing the same
+					 * read-only data.
+                                         *
+                                         * This policy can only be
+					 * used with existing memory
+					 * allocations (i.e., the
+					 * hwloc_set_*membind*()
+					 * functions); it cannot be
+					 * used with functions that
+					 * allocate new memory (i.e.,
+					 * the hwloc_alloc*()
+					 * functions).
 					 * \hideinitializer */
-  HWLOC_MEMBIND_NEXTTOUCH =	5	/**< \brief On next touch of existing allocated memory, migrate it to the node
+  HWLOC_MEMBIND_NEXTTOUCH =	5,	/**< \brief For each page bound
-					 * where the memory reference happened.
+                                         * with this policy, by next time
+                                         * it is touched (and next time only),
+                                         * it is moved from
+                                         * its current location to the
+                                         * local NUMA node of the
+                                         * thread where the memory
+                                         * reference occurred (if it
+                                         * needs to be moved at all).
+					 * \hideinitializer */
+  HWLOC_MEMBIND_MIXED = -1              /**< \brief Returned by hwloc_get_membind*()
+                                         * functions when multiple threads or
+                                         * parts of a memory area have
+                                         * differing memory binding policies.
                                          * \hideinitializer */
 } hwloc_membind_policy_t;
 
 /** \brief Memory binding flags.
  *
- * These flags can be used to refine the binding policy.
+ * These flags can be used to refine the binding policy.  All flags
+ * can be logically OR'ed together with the exception of
+ * HWLOC_MEMBIND_PROCESS and HWLOC_MEMBIND_THREAD; these two flags are
+ * mutually exclusive.
  *
- * \note Not all systems support all kinds of binding.
+ * \note Not all systems support all kinds of binding.  See the
+ * "Detailed Description" section of \ref hwlocality_membinding for a
+ * description of errors that can occur.
  */
 typedef enum {
-  HWLOC_MEMBIND_PROCESS =       (1<<0), /**< \brief Set policy for all threads of the
+  HWLOC_MEMBIND_PROCESS =       (1<<0), /**< \brief Set policy for all
-                                         * current (possibly multithreaded) process.
+                                         * threads of the specified
+                                         * (possibly multithreaded)
+                                         * process.  This flag is
+                                         * mutually exclusive with
+                                         * HWLOC_MEMBIND_THREAD.
                                          * \hideinitializer */
-  HWLOC_MEMBIND_THREAD =        (1<<1), /**< \brief Set policy for the current thread of
+  HWLOC_MEMBIND_THREAD =        (1<<1), /**< \brief Set policy for a
-                                         * the current process.
+                                         * specific thread of the
+                                         * current process.  This flag
+                                         * is mutually exclusive with
+                                         * HWLOC_MEMBIND_PROCESS.
                                          * \hideinitializer */
-  HWLOC_MEMBIND_STRICT =        (1<<2), /**< Request strict binding from the OS.
+  HWLOC_MEMBIND_STRICT =        (1<<2), /**< Request strict binding
-                                         * The function will fail if the
+                                         * from the OS.  The function
-                                         * binding can not be completely
+                                         * will fail if the binding
-                                         * enforced.
+                                         * can not be guaranteed /
-                                         * \hideinitializer  */
+                                         * completely enforced.
-  HWLOC_MEMBIND_MIGRATE =       (1<<3), /**< \brief Migrate existing allocated memory.
-                                         * If memory can not be migrated and the STRICT
-                                         * flag is passed, an error will be returned.
-                                         * \hideinitializer  */
-  HWLOC_MEMBIND_NOCPUBIND =     (1<<4)  /**< \brief Avoid any effect on CPU binding
-                                         * \hideinitializer
                                          *
-                                         * On some OSes, some memory binding function
+                                         * This flag has slightly
-                                         * would also bind the application on
+                                         * different meanings
-                                         * the corresponding CPUs. It is often
+                                         * depending on which function
-                                         * not a problem for the application, but if it
+                                         * it is used with.  
-                                         * is, setting this flag will make hwloc
+                                         * \hideinitializer  */
-                                         * avoid using OS functions that would also
+  HWLOC_MEMBIND_MIGRATE =       (1<<3), /**< \brief Migrate existing
-                                         * bind on CPUs.  This will however reduce the
+                                         * allocated memory.  If the
-                                         * support of memory bindings, i.e. potentially
+                                         * memory cannot be migrated
-                                         * return ENOSYS in some cases.
+                                         * and the
+                                         * HWLOC_MEMBIND_STRICT flag
+                                         * is passed, an error will be
+                                         * returned.
+                                         * \hideinitializer  */
+  HWLOC_MEMBIND_NOCPUBIND =     (1<<4)  /**< \brief Avoid any effect
+                                         * on CPU binding.
+                                         *
+                                         * On some operating systems,
+                                         * some underlying memory
+                                         * binding functions also bind
+                                         * the application to the
+                                         * corresponding CPU(s).
+                                         * Using this flag will cause
+                                         * hwloc to avoid using OS
+                                         * functions that could
+                                         * potentially affect CPU
+                                         * bindings.  Note, however,
+                                         * that using NOCPUBIND may
+                                         * reduce hwloc's overall
+                                         * memory binding support.
+                                         * Specifically: some of
+                                         * hwloc's memory binding
+                                         * functions may fail with
+                                         * errno set to ENOSYS when
+                                         * used with NOCPUBIND.
+                                         * \hideinitializer
                                          */
 } hwloc_membind_flags_t;
 
-/** \brief Bind current process memory on the given nodeset \p nodeset
+/** \brief Set the default memory binding policy of the current
+ * process or thread to prefer the NUMA node(s) specified by \p nodeset
+ *
+ * If neither HWLOC_MEMBIND_PROCESS nor HWLOC_MEMBIND_THREAD is
+ * specified, the current process is assumed to be single-threaded.
+ * This is the most portable form as it permits hwloc to use either
+ * process-based OS functions or thread-based OS functions, depending
+ * on which are available.
  *
  * \return -1 with errno set to ENOSYS if the action is not supported
  * \return -1 with errno set to EXDEV if the binding cannot be enforced
  */
 HWLOC_DECLSPEC int hwloc_set_membind_nodeset(hwloc_topology_t topology, hwloc_const_nodeset_t nodeset, hwloc_membind_policy_t policy, int flags);
 
-/** \brief Bind current process memory on memory nodes near the given cpuset \p cpuset
+/** \brief Set the default memory binding policy of the current
+ * process or thread to prefer the NUMA node(s) near the specified \p
+ * cpuset
+ *
+ * If neither HWLOC_MEMBIND_PROCESS nor HWLOC_MEMBIND_THREAD is
+ * specified, the current process is assumed to be single-threaded.
+ * This is the most portable form as it permits hwloc to use either
+ * process-based OS functions or thread-based OS functions, depending
+ * on which are available.
  *
  * \return -1 with errno set to ENOSYS if the action is not supported
  * \return -1 with errno set to EXDEV if the binding cannot be enforced
  */
 HWLOC_DECLSPEC int hwloc_set_membind(hwloc_topology_t topology, hwloc_const_cpuset_t cpuset, hwloc_membind_policy_t policy, int flags);
 
-/** \brief Get current process memory binding in nodeset \p nodeset
+/** \brief Query the default memory binding policy and locality of the
+ * current process or thread.
+ *
+ * This function has two output parameters: \p nodeset and \p policy.
+ * The values returned in these parameters depend on both the \p flags
+ * passed in and the current memory binding policies and nodesets in
+ * the queried target.
+ *
+ * Passing the HWLOC_MEMBIND_PROCESS flag specifies that the query
+ * target is the current policies and nodesets for all the threads in
+ * the current process.  Passing HWLOC_MEMBIND_THREAD specifies that
+ * the query target is the current policy and nodeset for only the
+ * thread invoking this function.
+ *
+ * If neither of these flags are passed (which is the most portable
+ * method), the process is assumed to be single threaded.  This allows
+ * hwloc to use either process-based OS functions or thread-based OS
+ * functions, depending on which are available.
+ *
+ * HWLOC_MEMBIND_STRICT is only meaningful when HWLOC_MEMBIND_PROCESS
+ * is also specified.  In this case, hwloc will check the default
+ * memory policies and nodesets for all threads in the process.  If
+ * they are not identical, -1 is returned and errno is set to EXDEV.
+ * If they are identical, the values are returned in \p nodeset and \p
+ * policy.
+ *
+ * Otherwise, if HWLOC_MEMBIND_PROCESS is specified (and
+ * HWLOC_MEMBIND_STRICT is \em not specified), \p nodeset is set to
+ * the logical OR of all threads' default nodeset.  If all threads'
+ * default policies are the same, \p policy is set to that policy.  If
+ * they are different, \p policy is set to HWLOC_MEMBIND_MIXED.
+ *
+ * In the HWLOC_MEMBIND_THREAD case (or when neither
+ * HWLOC_MEMBIND_PROCESS or HWLOC_MEMBIND_THREAD is specified), there
+ * is only one nodeset and policy; they are returned in \p nodeset and
+ * \p policy, respectively.
+ *
+ * If any other flags are specified, -1 is returned and errno is set
+ * to EINVAL.
  */
 HWLOC_DECLSPEC int hwloc_get_membind_nodeset(hwloc_topology_t topology, hwloc_nodeset_t nodeset, hwloc_membind_policy_t * policy, int flags);
 
-/** \brief Get current process memory binding in cpuset \p cpuset
+/** \brief Query the default memory binding policy and locality of the
+ * current process or thread (the locality is returned in \p cpuset as
+ * CPUs near the locality's actual NUMA node(s)).
+ *
+ * This function has two output parameters: \p cpuset and \p policy.
+ * The values returned in these parameters depend on both the \p flags
+ * passed in and the current memory binding policies and nodesets in
+ * the queried target.
+ *
+ * Passing the HWLOC_MEMBIND_PROCESS flag specifies that the query
+ * target is the current policies and nodesets for all the threads in
+ * the current process.  Passing HWLOC_MEMBIND_THREAD specifies that
+ * the query target is the current policy and nodeset for only the
+ * thread invoking this function.
+ *
+ * If neither of these flags are passed (which is the most portable
+ * method), the process is assumed to be single threaded.  This allows
+ * hwloc to use either process-based OS functions or thread-based OS
+ * functions, depending on which are available.
+ *
+ * HWLOC_MEMBIND_STRICT is only meaningful when HWLOC_MEMBIND_PROCESS
+ * is also specified.  In this case, hwloc will check the default
+ * memory policies and nodesets for all threads in the process.  If
+ * they are not identical, -1 is returned and errno is set to EXDEV.
+ * If they are identical, the policy is returned in \p policy.  \p
+ * cpuset is set to the union of CPUs near the NUMA node(s) in the
+ * nodeset.
+ *
+ * Otherwise, if HWLOC_MEMBIND_PROCESS is specified (and
+ * HWLOC_MEMBIND_STRICT is \em not specified), the default nodeset
+ * from each thread is logically OR'ed together.  \p cpuset is set to
+ * the union of CPUs near the NUMA node(s) in the resulting nodeset.
+ * If all threads' default policies are the same, \p policy is set to
+ * that policy.  If they are different, \p policy is set to
+ * HWLOC_MEMBIND_MIXED.
+ *
+ * In the HWLOC_MEMBIND_THREAD case (or when neither
+ * HWLOC_MEMBIND_PROCESS or HWLOC_MEMBIND_THREAD is specified), there
+ * is only one nodeset and policy.  The policy is returned in \p
+ * policy; \p cpuset is set to the union of CPUs near the NUMA node(s)
+ * in the \p nodeset.
+ *
+ * If any other flags are specified, -1 is returned and errno is set
+ * to EINVAL.
  */
 HWLOC_DECLSPEC int hwloc_get_membind(hwloc_topology_t topology, hwloc_cpuset_t cpuset, hwloc_membind_policy_t * policy, int flags);
 
-/** \brief Bind given process memory on the given nodeset \p nodeset
+/** \brief Set the default memory binding policy of the specified
+ * process to prefer the NUMA node(s) specified by \p nodeset
  *
  * \return -1 with errno set to ENOSYS if the action is not supported
  * \return -1 with errno set to EXDEV if the binding cannot be enforced
  */
 HWLOC_DECLSPEC int hwloc_set_proc_membind_nodeset(hwloc_topology_t topology, hwloc_pid_t pid, hwloc_const_nodeset_t nodeset, hwloc_membind_policy_t policy, int flags);
 
-/** \brief Bind given process memory on memory nodes near the given cpuset \p cpuset
+/** \brief Set the default memory binding policy of the specified
+ * process to prefer the NUMA node(s) near the specified \p cpuset
  *
  * \return -1 with errno set to ENOSYS if the action is not supported
  * \return -1 with errno set to EXDEV if the binding cannot be enforced
  */
 HWLOC_DECLSPEC int hwloc_set_proc_membind(hwloc_topology_t topology, hwloc_pid_t pid, hwloc_const_cpuset_t cpuset, hwloc_membind_policy_t policy, int flags);
 
-/** \brief Get current process memory binding in nodeset \p nodeset
+/** \brief Query the default memory binding policy and locality of the
+ * specified process.
+ *
+ * This function has two output parameters: \p nodeset and \p policy.
+ * The values returned in these parameters depend on both the \p flags
+ * passed in and the current memory binding policies and nodesets in
+ * the queried target.
+ *
+ * Passing the HWLOC_MEMBIND_PROCESS flag specifies that the query
+ * target is the current policies and nodesets for all the threads in
+ * the specified process.  If HWLOC_MEMBIND_PROCESS is not specified
+ * (which is the most portable method), the process is assumed to be
+ * single threaded.  This allows hwloc to use either process-based OS
+ * functions or thread-based OS functions, depending on which are
+ * available.
+ *
+ * Note that it does not make sense to pass HWLOC_MEMBIND_THREAD to
+ * this function.
+ *
+ * If HWLOC_MEMBIND_STRICT is specified, hwloc will check the default
+ * memory policies and nodesets for all threads in the specified
+ * process.  If they are not identical, -1 is returned and errno is
+ * set to EXDEV.  If they are identical, the values are returned in \p
+ * nodeset and \p policy.
+ *
+ * Otherwise, \p nodeset is set to the logical OR of all threads'
+ * default nodeset.  If all threads' default policies are the same, \p
+ * policy is set to that policy.  If they are different, \p policy is
+ * set to HWLOC_MEMBIND_MIXED.
+ *
+ * If any other flags are specified, -1 is returned and errno is set
+ * to EINVAL.
  */
 HWLOC_DECLSPEC int hwloc_get_proc_membind_nodeset(hwloc_topology_t topology, hwloc_pid_t pid, hwloc_nodeset_t nodeset, hwloc_membind_policy_t * policy, int flags);
 
-/** \brief Get current process memory binding in cpuset \p cpuset
+/** \brief Query the default memory binding policy and locality of the
+ * specified process (the locality is returned in \p cpuset as CPUs
+ * near the locality's actual NUMA node(s)).
+ *
+ * This function has two output parameters: \p cpuset and \p policy.
+ * The values returned in these parameters depend on both the \p flags
+ * passed in and the current memory binding policies and nodesets in
+ * the queried target.
+ *
+ * Passing the HWLOC_MEMBIND_PROCESS flag specifies that the query
+ * target is the current policies and nodesets for all the threads in
+ * the specified process.  If HWLOC_MEMBIND_PROCESS is not specified
+ * (which is the most portable method), the process is assumed to be
+ * single threaded.  This allows hwloc to use either process-based OS
+ * functions or thread-based OS functions, depending on which are
+ * available.
+ *
+ * Note that it does not make sense to pass HWLOC_MEMBIND_THREAD to
+ * this function.
+ *
+ * If HWLOC_MEMBIND_STRICT is specified, hwloc will check the default
+ * memory policies and nodesets for all threads in the specified
+ * process.  If they are not identical, -1 is returned and errno is
+ * set to EXDEV.  If they are identical, the policy is returned in \p
+ * policy.  \p cpuset is set to the union of CPUs near the NUMA
+ * node(s) in the nodeset.
+ *
+ * Otherwise, the default nodeset from each thread is logically OR'ed
+ * together.  \p cpuset is set to the union of CPUs near the NUMA
+ * node(s) in the resulting nodeset.  If all threads' default policies
+ * are the same, \p policy is set to that policy.  If they are
+ * different, \p policy is set to HWLOC_MEMBIND_MIXED.
+ *
+ * If any other flags are specified, -1 is returned and errno is set
+ * to EINVAL.
  */
 HWLOC_DECLSPEC int hwloc_get_proc_membind(hwloc_topology_t topology, hwloc_pid_t pid, hwloc_cpuset_t cpuset, hwloc_membind_policy_t * policy, int flags);
 
-/** \brief Bind some memory range on the given nodeset \p nodeset
+/** \brief Bind the already-allocated memory identified by (addr, len)
+ * to the NUMA node(s) in \p nodeset.
  *
  * \return -1 with errno set to ENOSYS if the action is not supported
  * \return -1 with errno set to EXDEV if the binding cannot be enforced
  */
 HWLOC_DECLSPEC int hwloc_set_area_membind_nodeset(hwloc_topology_t topology, const void *addr, size_t len, hwloc_const_nodeset_t nodeset, hwloc_membind_policy_t policy, int flags);
 
-/** \brief Bind some memory range on memory nodes near the given cpuset \p cpuset
+/** \brief Bind the already-allocated memory identified by (addr, len)
+ * to the NUMA node(s) near \p cpuset.
  *
  * \return -1 with errno set to ENOSYS if the action is not supported
  * \return -1 with errno set to EXDEV if the binding cannot be enforced
  */
 HWLOC_DECLSPEC int hwloc_set_area_membind(hwloc_topology_t topology, const void *addr, size_t len, hwloc_const_cpuset_t cpuset, hwloc_membind_policy_t policy, int flags);
 
-/** \brief Get some memory range memory binding in nodeset \p nodeset
+/** \brief Query the NUMA node(s) and binding policy of the memory
+ * identified by (\p addr, \p len ).
+ *
+ * This function has two output parameters: \p nodeset and \p policy.
+ * The values returned in these parameters depend on both the \p flags
+ * passed in and the memory binding policies and nodesets of the pages
+ * in the address range.
+ *
+ * If HWLOC_MEMBIND_STRICT is specified, the target pages are first
+ * checked to see if they all have the same memory binding policy and
+ * nodeset.  If they do not, -1 is returned and errno is set to EXDEV.
+ * If they are identical across all pages, the nodeset and policy are
+ * returned in \p nodeset and \p policy, respectively.
+ *
+ * If HWLOC_MEMBIND_STRICT is not specified, \p nodeset is set to the
+ * union of all NUMA node(s) containing pages in the address range.
+ * If all pages in the target have the same policy, it is returned in
+ * \p policy.  Otherwise, \p policy is set to HWLOC_MEMBIND_MIXED.
+ *
+ * If any other flags are specified, -1 is returned and errno is set
+ * to EINVAL.
  */
 HWLOC_DECLSPEC int hwloc_get_area_membind_nodeset(hwloc_topology_t topology, const void *addr, size_t len, hwloc_nodeset_t nodeset, hwloc_membind_policy_t * policy, int flags);
 
-/** \brief Get some memory range memory binding in cpuset \p cpuset
+/** \brief Query the CPUs near the NUMA node(s) and binding policy of
+ * the memory identified by (\p addr, \p len ).
+ *
+ * This function has two output parameters: \p cpuset and \p policy.
+ * The values returned in these parameters depend on both the \p flags
+ * passed in and the memory binding policies and nodesets of the pages
+ * in the address range.
+ *
+ * If HWLOC_MEMBIND_STRICT is specified, the target pages are first
+ * checked to see if they all have the same memory binding policy and
+ * nodeset.  If they do not, -1 is returned and errno is set to EXDEV.
+ * If they are identical across all pages, the policy is returned in
+ * \p policy.  \p cpuset is set to the union of CPUs near the NUMA
+ * node(s) in the nodeset.
+ *
+ * If HWLOC_MEMBIND_STRICT is not specified, the union of all NUMA
+ * node(s) containing pages in the address range is calculated.  \p
+ * cpuset is then set to the CPUs near the NUMA node(s) in this union.
+ * If all pages in the target have the same policy, it is returned in
+ * \p policy.  Otherwise, \p policy is set to HWLOC_MEMBIND_MIXED.
+ *
+ * If any other flags are specified, -1 is returned and errno is set
+ * to EINVAL.
  */
 HWLOC_DECLSPEC int hwloc_get_area_membind(hwloc_topology_t topology, const void *addr, size_t len, hwloc_cpuset_t cpuset, hwloc_membind_policy_t * policy, int flags);
 
 /** \brief Allocate some memory
  *
- * This is equivalent to malloc(), except it tries to allocated page-aligned
+ * This is equivalent to malloc(), except that it tries to allocate
- * memory from the OS.
+ * page-aligned memory from the OS.
  *
  * \note The allocated memory should be freed with hwloc_free().
  */
@@ -1202,7 +1548,8 @@ HWLOC_DECLSPEC void *hwloc_alloc_membind_nodeset(hwloc_topology_t topology, size
  */
 HWLOC_DECLSPEC void *hwloc_alloc_membind(hwloc_topology_t topology, size_t len, hwloc_const_cpuset_t cpuset, hwloc_membind_policy_t policy, int flags) __hwloc_attribute_malloc;
 
-/** \brief Free some memory allocated by hwloc_alloc() or hwloc_alloc_membind().
+/** \brief Free memory that was previously allocated by hwloc_alloc()
+ * or hwloc_alloc_membind().
  */
 HWLOC_DECLSPEC int hwloc_free(hwloc_topology_t topology, void *addr, size_t len);
 

opal/mca/paffinity/hwloc/hwloc/include/hwloc/bitmap.h

@@ -24,11 +24,17 @@ extern "C" {
 
 /** \defgroup hwlocality_bitmap The bitmap API
  *
- * For use in hwloc itself, a hwloc_bitmap_t usually represents a set of
+ * The ::hwloc_bitmap_t type represents a set of objects, typically OS
- * objects, typically logical processors or memory nodes, indexed by OS
+ * processors -- which may actually be hardware threads (represented
- * physical number.
+ * by ::hwloc_cpuset_t, which is a typedef for ::hwloc_bitmap_t) -- or
+ * memory nodes (represented by ::hwloc_nodeset_t, which is also a
+ * typedef for ::hwloc_bitmap_t).  
  *
- * A bitmap may be infinite.
+ * <em>Both CPU and node sets are always indexed by OS physical number.</em>
+ *
+ * \note CPU sets and nodesets are described in \ref hwlocality_sets.
+ *
+ * A bitmap may be of infinite size.
  * @{
  */
 
@@ -37,6 +43,7 @@ extern "C" {
  * Set of bits represented as an opaque pointer to an internal bitmap.
  */
 typedef struct hwloc_bitmap_s * hwloc_bitmap_t;
+/** \brief a non-modifiable ::hwloc_bitmap_t */
 typedef const struct hwloc_bitmap_s * hwloc_const_bitmap_t;
 
 
@@ -73,21 +80,18 @@ HWLOC_DECLSPEC void hwloc_bitmap_copy(hwloc_bitmap_t dst, hwloc_const_bitmap_t s
  *
  * Up to \p buflen characters may be written in buffer \p buf.
  *
+ * If \p buflen is 0, \p buf may safely be \c NULL.
+ *
  * \return the number of character that were actually written if not truncating,
  * or that would have been written (not including the ending \\0).
  */
 HWLOC_DECLSPEC int hwloc_bitmap_snprintf(char * __hwloc_restrict buf, size_t buflen, hwloc_const_bitmap_t bitmap);
 
 /** \brief Stringify a bitmap into a newly allocated string.
- *
- * \return the number of character that were actually written
- * (not including the ending \\0).
  */
 HWLOC_DECLSPEC int hwloc_bitmap_asprintf(char ** strp, hwloc_const_bitmap_t bitmap);
 
 /** \brief Parse a bitmap string and stores it in bitmap \p bitmap.
- *
- * Must start and end with a digit.
  */
 HWLOC_DECLSPEC int hwloc_bitmap_sscanf(hwloc_bitmap_t bitmap, const char * __hwloc_restrict string);
 
@@ -95,6 +99,13 @@ HWLOC_DECLSPEC int hwloc_bitmap_sscanf(hwloc_bitmap_t bitmap, const char * __hwl
  *
  * The taskset command manipulates bitmap strings that contain a single
  * (possible very long) hexadecimal number starting with 0x.
+ *
+ * Up to \p buflen characters may be written in buffer \p buf.
+ *
+ * If \p buflen is 0, \p buf may safely be \c NULL.
+ *
+ * \return the number of character that were actually written if not truncating,
+ * or that would have been written (not including the ending \\0).
  */
 HWLOC_DECLSPEC int hwloc_bitmap_taskset_snprintf(char * __hwloc_restrict buf, size_t buflen, hwloc_const_bitmap_t bitmap);
 

opal/mca/paffinity/hwloc/hwloc/include/hwloc/helper.h

@@ -582,7 +582,7 @@ hwloc_get_obj_below_array_by_type (hwloc_topology_t topology, int nr, hwloc_obj_
  *
  * Array \p cpuset will be filled with \p n cpusets recursively distributed
  * linearly over the topology under \p root, down to depth \p until (which can
- * be MAX_INT to distribute down to the finest level).
+ * be INT_MAX to distribute down to the finest level).
  *
  * This is typically useful when an application wants to distribute \p n
  * threads over a machine, giving each of them as much private cache as
@@ -680,7 +680,7 @@ hwloc_alloc_membind_policy(hwloc_topology_t topology, size_t len, hwloc_const_cp
 /** \defgroup hwlocality_helper_cpuset Cpuset Helpers
  * @{
  */
-/* \brief Get complete CPU set
+/** \brief Get complete CPU set
  *
  * \return the complete CPU set of logical processors of the system. If the
  * topology is the result of a combination of several systems, NULL is
@@ -695,7 +695,7 @@ hwloc_topology_get_complete_cpuset(hwloc_topology_t topology)
   return hwloc_get_root_obj(topology)->complete_cpuset;
 }
 
-/* \brief Get topology CPU set
+/** \brief Get topology CPU set
  *
  * \return the CPU set of logical processors of the system for which hwloc
  * provides topology information. This is equivalent to the cpuset of the
@@ -748,7 +748,7 @@ hwloc_topology_get_allowed_cpuset(hwloc_topology_t topology)
 /** \defgroup hwlocality_helper_nodeset Nodeset Helpers
  * @{
  */
-/* \brief Get complete node set
+/** \brief Get complete node set
  *
  * \return the complete node set of memory of the system. If the
  * topology is the result of a combination of several systems, NULL is
@@ -763,7 +763,7 @@ hwloc_topology_get_complete_nodeset(hwloc_topology_t topology)
   return hwloc_get_root_obj(topology)->complete_nodeset;
 }
 
-/* \brief Get topology node set
+/** \brief Get topology node set
  *
  * \return the node set of memory of the system for which hwloc
  * provides topology information. This is equivalent to the nodeset of the

opal/mca/paffinity/hwloc/hwloc/include/hwloc/rename.h

@@ -36,6 +36,8 @@ extern "C" {
 
 /* Names from hwloc.h */
 
+#define hwloc_get_api_version HWLOC_NAME(get_api_version)
+
 #define hwloc_topology HWLOC_NAME(topology)
 #define hwloc_topology_t HWLOC_NAME(topology_t)
 
@@ -181,6 +183,9 @@ extern "C" {
 #define hwloc_bitmap_snprintf HWLOC_NAME(bitmap_snprintf)
 #define hwloc_bitmap_asprintf HWLOC_NAME(bitmap_asprintf)
 #define hwloc_bitmap_sscanf HWLOC_NAME(bitmap_sscanf)
+#define hwloc_bitmap_taskset_snprintf HWLOC_NAME(bitmap_taskset_snprintf)
+#define hwloc_bitmap_taskset_asprintf HWLOC_NAME(bitmap_taskset_asprintf)
+#define hwloc_bitmap_taskset_sscanf HWLOC_NAME(bitmap_taskset_sscanf)
 #define hwloc_bitmap_zero HWLOC_NAME(bitmap_zero)
 #define hwloc_bitmap_fill HWLOC_NAME(bitmap_fill)
 #define hwloc_bitmap_from_ulong HWLOC_NAME(bitmap_from_ulong)
@@ -214,6 +219,52 @@ extern "C" {
 #define hwloc_bitmap_compare HWLOC_NAME(bitmap_compare)
 #define hwloc_bitmap_weight HWLOC_NAME(bitmap_weight)
 
+/* hwloc/cpuset.h -- deprecated but still available */
+
+#define hwloc_cpuset HWLOC_NAME(cpuset)
+#define hwloc_cpuset_s HWLOC_NAME(cpuset_s)
+#define hwloc_cpuset_t HWLOC_NAME(cpuset_t)
+#define hwloc_const_cpuset_t HWLOC_NAME(const_cpuset_t)
+
+#define hwloc_cpuset_alloc HWLOC_NAME(cpuset_alloc)
+#define hwloc_cpuset_free HWLOC_NAME(cpuset_free)
+#define hwloc_cpuset_dup HWLOC_NAME(cpuset_dup)
+#define hwloc_cpuset_copy HWLOC_NAME(cpuset_copy)
+#define hwloc_cpuset_snprintf HWLOC_NAME(cpuset_snprintf)
+#define hwloc_cpuset_asprintf HWLOC_NAME(cpuset_asprintf)
+#define hwloc_cpuset_from_string HWLOC_NAME(cpuset_from_string)
+#define hwloc_cpuset_zero HWLOC_NAME(cpuset_zero)
+#define hwloc_cpuset_fill HWLOC_NAME(cpuset_fill)
+#define hwloc_cpuset_from_ulong HWLOC_NAME(cpuset_from_ulong)
+
+#define hwloc_cpuset_from_ith_ulong HWLOC_NAME(cpuset_from_ith_ulong)
+#define hwloc_cpuset_to_ulong HWLOC_NAME(cpuset_to_ulong)
+#define hwloc_cpuset_to_ith_ulong HWLOC_NAME(cpuset_to_ith_ulong)
+#define hwloc_cpuset_cpu HWLOC_NAME(cpuset_cpu)
+#define hwloc_cpuset_all_but_cpu HWLOC_NAME(cpuset_all_but_cpu)
+#define hwloc_cpuset_set HWLOC_NAME(cpuset_set)
+#define hwloc_cpuset_set_range HWLOC_NAME(cpuset_set_range)
+#define hwloc_cpuset_clr HWLOC_NAME(cpuset_clr)
+#define hwloc_cpuset_clr_range HWLOC_NAME(cpuset_clr_range)
+#define hwloc_cpuset_isset HWLOC_NAME(cpuset_isset)
+#define hwloc_cpuset_iszero HWLOC_NAME(cpuset_iszero)
+#define hwloc_cpuset_isfull HWLOC_NAME(cpuset_isfull)
+#define hwloc_cpuset_isequal HWLOC_NAME(cpuset_isequal)
+#define hwloc_cpuset_intersects HWLOC_NAME(cpuset_intersects)
+#define hwloc_cpuset_isincluded HWLOC_NAME(cpuset_isincluded)
+#define hwloc_cpuset_or HWLOC_NAME(cpuset_or)
+#define hwloc_cpuset_and HWLOC_NAME(cpuset_and)
+#define hwloc_cpuset_andnot HWLOC_NAME(cpuset_andnot)
+#define hwloc_cpuset_xor HWLOC_NAME(cpuset_xor)
+#define hwloc_cpuset_not HWLOC_NAME(cpuset_not)
+#define hwloc_cpuset_first HWLOC_NAME(cpuset_first)
+#define hwloc_cpuset_last HWLOC_NAME(cpuset_last)
+#define hwloc_cpuset_next HWLOC_NAME(cpuset_next)
+#define hwloc_cpuset_singlify HWLOC_NAME(cpuset_singlify)
+#define hwloc_cpuset_compare_first HWLOC_NAME(cpuset_compare_first)
+#define hwloc_cpuset_compare HWLOC_NAME(cpuset_compare)
+#define hwloc_cpuset_weight HWLOC_NAME(cpuset_weight)
+
 /* hwloc/helper.h */
 
 #define hwloc_get_type_or_below_depth HWLOC_NAME(get_type_or_below_depth)

opal/mca/paffinity/hwloc/hwloc/src/cpuset.c

@@ -36,8 +36,8 @@
 
 /* actual opaque type internals */
 struct hwloc_bitmap_s {
-  unsigned ulongs_count; /* how many ulong bitmasks are valid */
+  unsigned ulongs_count; /* how many ulong bitmasks are valid, >= 1 */
-  unsigned ulongs_allocated; /* how many ulong bitmasks are allocated */
+  unsigned ulongs_allocated; /* how many ulong bitmasks are allocated, >= ulongs_count */
   unsigned long *ulongs;
   int infinite; /* set to 1 if all bits beyond ulongs are set */
 #ifdef HWLOC_DEBUG
@@ -47,7 +47,11 @@ struct hwloc_bitmap_s {
 
 /* overzealous check in debug-mode, not as powerful as valgrind but still useful */
 #ifdef HWLOC_DEBUG
-#define HWLOC__BITMAP_CHECK(set) assert((set)->magic == HWLOC_BITMAP_MAGIC)
+#define HWLOC__BITMAP_CHECK(set) do {				\
+  assert((set)->magic == HWLOC_BITMAP_MAGIC);			\
+  assert((set)->ulongs_count >= 1);				\
+  assert((set)->ulongs_allocated >= (set)->ulongs_count);	\
+} while (0)
 #else
 #define HWLOC__BITMAP_CHECK(set)
 #endif
@@ -77,7 +81,7 @@ struct hwloc_bitmap_s * hwloc_bitmap_alloc(void)
   if (!set)
     return NULL;
 
-  set->ulongs_count = 0;
+  set->ulongs_count = 1;
   set->ulongs_allocated = 64/sizeof(unsigned long);
   set->ulongs = malloc(64);
   if (!set->ulongs) {
@@ -85,6 +89,7 @@ struct hwloc_bitmap_s * hwloc_bitmap_alloc(void)
     return NULL;
   }
 
+  set->ulongs[0] = HWLOC_SUBBITMAP_ZERO;
   set->infinite = 0;
 #ifdef HWLOC_DEBUG
   set->magic = HWLOC_BITMAP_MAGIC;
@@ -95,8 +100,10 @@ struct hwloc_bitmap_s * hwloc_bitmap_alloc(void)
 struct hwloc_bitmap_s * hwloc_bitmap_alloc_full(void)
 {
   struct hwloc_bitmap_s * set = hwloc_bitmap_alloc();
-  if (set)
+  if (set) {
     set->infinite = 1;
+    set->ulongs[0] = HWLOC_SUBBITMAP_FULL;
+  }
   return set;
 }
 
@@ -240,13 +247,9 @@ int hwloc_bitmap_snprintf(char * __hwloc_restrict buf, size_t buflen, const stru
       res = size>0 ? size - 1 : 0;
     tmp += res;
     size -= res;
-  }
+    /* optimize a common case: full bitmap should appear as 0xf...f instead of 0xf...f,0xffffffff */
-
+    if (set->ulongs_count == 1 && set->ulongs[0] == HWLOC_SUBBITMAP_FULL)
-  if (!set->infinite && !set->ulongs_count) {
+      return ret;
-    res = hwloc_snprintf(tmp, size, "0x0");
-    if (res < 0)
-      return -1;
-    return res;
   }
 
   i=set->ulongs_count-1;
@@ -318,9 +321,15 @@ int hwloc_bitmap_sscanf(struct hwloc_bitmap_s *set, const char * __hwloc_restric
     count++;
 
   current = string;
-  if (!strncmp("0xf...f,", current, 8)) {
+  if (!strncmp("0xf...f", current, 7)) {
+    current += 7;
+    if (*current != ',') {
+      /* special case for infinite/full cpuset */
+      hwloc_bitmap_fill(set);
+      return 0;
+    }
+    current++;
     infinite = 1;
-    current += 8;
     count--;
   }
 
@@ -384,6 +393,9 @@ int hwloc_bitmap_taskset_snprintf(char * __hwloc_restrict buf, size_t buflen, co
       res = size>0 ? size - 1 : 0;
     tmp += res;
     size -= res;
+    /* optimize a common case: full bitmap should appear as 0xf...f instead of 0xf...fffffffff */
+    if (set->ulongs_count == 1 && set->ulongs[0] == HWLOC_SUBBITMAP_FULL)
+      return ret;
   }
 
   i=set->ulongs_count-1;
@@ -391,8 +403,12 @@ int hwloc_bitmap_taskset_snprintf(char * __hwloc_restrict buf, size_t buflen, co
     unsigned long val = set->ulongs[i--];
     if (started) {
       /* print the whole subset */
+#if HWLOC_BITS_PER_LONG == 64
+      res = hwloc_snprintf(tmp, size, "%016lx", val);
+#else
       res = hwloc_snprintf(tmp, size, "%08lx", val);
-    } else if (val) {
+#endif
+    } else if (val || i == -1) {
       res = hwloc_snprintf(tmp, size, "0x%lx", val);
       started = 1;
     } else {
@@ -432,11 +448,25 @@ int hwloc_bitmap_taskset_sscanf(struct hwloc_bitmap_s *set, const char * __hwloc
 
   current = string;
   if (!strncmp("0xf...f", current, 7)) {
+    /* infinite bitmap */
     infinite = 1;
     current += 7;
-  } else if (!strncmp("0x", current, 2)) {
+    if (*current == '\0') {
-    current += 2;
+      /* special case for infinite/full bitmap */
+      hwloc_bitmap_fill(set);
+      return 0;
     }
+  } else {
+    /* finite bitmap */
+    if (!strncmp("0x", current, 2))
+      current += 2;
+    if (*current == '\0') {
+      /* special case for empty bitmap */
+      hwloc_bitmap_zero(set);
+      return 0;
+    }
+  }
+  /* we know there are other characters now */
 
   chars = strlen(current);
   count = (chars * 4 + HWLOC_BITS_PER_LONG - 1) / HWLOC_BITS_PER_LONG;
@@ -446,13 +476,13 @@ int hwloc_bitmap_taskset_sscanf(struct hwloc_bitmap_s *set, const char * __hwloc
 
   while (*current != '\0') {
     int tmpchars;
-    char ustr[9];
+    char ustr[17];
     unsigned long val;
     char *next;
 
-    tmpchars = chars % 8;
+    tmpchars = chars % (HWLOC_BITS_PER_LONG/4);
     if (!tmpchars)
-      tmpchars = 8;
+      tmpchars = (HWLOC_BITS_PER_LONG/4);
 
     memcpy(ustr, current, tmpchars);
     ustr[tmpchars] = '\0';
@@ -489,7 +519,7 @@ void hwloc_bitmap_zero(struct hwloc_bitmap_s * set)
 {
 	HWLOC__BITMAP_CHECK(set);
 
-	hwloc_bitmap_reset_by_ulongs(set, 0);
+	hwloc_bitmap_reset_by_ulongs(set, 1);
 	hwloc_bitmap__zero(set);
 }
 
@@ -505,7 +535,7 @@ void hwloc_bitmap_fill(struct hwloc_bitmap_s * set)
 {
 	HWLOC__BITMAP_CHECK(set);
 
-	hwloc_bitmap_reset_by_ulongs(set, 0);
+	hwloc_bitmap_reset_by_ulongs(set, 1);
 	hwloc_bitmap__fill(set);
 }
 
@@ -526,7 +556,7 @@ void hwloc_bitmap_from_ith_ulong(struct hwloc_bitmap_s *set, unsigned i, unsigne
 
 	hwloc_bitmap_reset_by_ulongs(set, i+1);
 	set->ulongs[i] = mask;
-	for(j=1; j<(unsigned) i; j++)
+	for(j=0; j<(unsigned) i; j++)
 		set->ulongs[j] = HWLOC_SUBBITMAP_ZERO;
 	set->infinite = 0;
 }

opal/mca/paffinity/hwloc/hwloc/src/topology-linux.c

@@ -886,18 +886,18 @@ hwloc_linux_membind_mask_from_nodeset(hwloc_topology_t topology __hwloc_attribut
 static void
 hwloc_linux_membind_mask_to_nodeset(hwloc_topology_t topology __hwloc_attribute_unused,
 				    hwloc_nodeset_t nodeset,
-				    unsigned _max_os_index, const unsigned long *linuxmask)
+				    unsigned max_os_index, const unsigned long *linuxmask)
 {
-  unsigned max_os_index;
   unsigned i;
 
-  /* round up to the nearest multiple of BITS_PER_LONG */
+#ifdef HWLOC_DEBUG
-  max_os_index = (_max_os_index + HWLOC_BITS_PER_LONG) & ~(HWLOC_BITS_PER_LONG - 1);
+  /* max_os_index comes from hwloc_linux_find_kernel_max_numnodes() so it's a multiple of HWLOC_BITS_PER_LONG */
+  assert(!(max_os_index%HWLOC_BITS_PER_LONG));
+#endif
 
   hwloc_bitmap_zero(nodeset);
   for(i=0; i<max_os_index/HWLOC_BITS_PER_LONG; i++)
     hwloc_bitmap_set_ith_ulong(nodeset, i, linuxmask[i]);
-  /* if we don't trust the kernel, we could clear bits from _max_os_index+1 to max_os_index-1 */
 }
 #endif /* HWLOC_HAVE_SET_MEMPOLICY || HWLOC_HAVE_MBIND */
 
@@ -1161,6 +1161,7 @@ hwloc_parse_sysfs_unsigned(const char *mappath, unsigned *value, int fsroot_fd)
 
   if (!fgets(string, 11, fd)) {
     *value = -1;
+    fclose(fd);
     return -1;
   }
   *value = strtoul(string, NULL, 10);
@@ -1725,7 +1726,7 @@ hwloc_sysfs_node_meminfo_info(struct hwloc_topology *topology,
   hwloc_parse_meminfo_info(topology, meminfopath,
 			   hwloc_snprintf(NULL, 0, "Node %d ", node),
 			   &memory->local_memory,
-			   &meminfo_hugepages_count, &meminfo_hugepages_size,
+			   &meminfo_hugepages_count, NULL /* no hugepage size in node-specific meminfo */,
 			   memory->page_types == NULL);
 
   if (memory->page_types) {
@@ -1734,12 +1735,14 @@ hwloc_sysfs_node_meminfo_info(struct hwloc_topology *topology,
       /* read from node%d/hugepages/hugepages-%skB/nr_hugepages */
       hwloc_parse_hugepages_info(topology, path, memory, &remaining_local_memory);
     } else {
+      /* get hugepage size from machine-specific meminfo since there is no size in node-specific meminfo,
+       * hwloc_get_procfs_meminfo_info must have been called earlier */
+      meminfo_hugepages_size = topology->levels[0][0]->memory.page_types[1].size;
       /* use what we found in meminfo */
-      /* hwloc_get_procfs_meminfo_info must have been called earlier */
       if (meminfo_hugepages_size) {
         memory->page_types[1].count = meminfo_hugepages_count;
-        memory->page_types[1].size = topology->levels[0][0]->memory.page_types[1].size;
+        memory->page_types[1].size = meminfo_hugepages_size;
-        remaining_local_memory -= meminfo_hugepages_count * memory->page_types[1].size;
+        remaining_local_memory -= meminfo_hugepages_count * meminfo_hugepages_size;
       } else {
         memory->page_types_len = 1;
       }
@@ -1761,8 +1764,10 @@ hwloc_parse_node_distance(const char *distancepath, unsigned nbnodes, unsigned *
   if (!fd)
     return;
 
-  if (!fgets(string, sizeof(string), fd))
+  if (!fgets(string, sizeof(string), fd)) {
+    fclose(fd);
     return;
+  }
 
   tmp = string;
   while (tmp) {

opal/mca/paffinity/hwloc/hwloc/src/topology-synthetic.c

@@ -85,7 +85,6 @@ hwloc_backend_synthetic_init(struct hwloc_topology *topology, const char *descri
 
     topology->backend_params.synthetic.arity[count-1] = (unsigned)item;
     topology->backend_params.synthetic.type[count] = type;
-    topology->backend_params.synthetic.id[count] = 0;
     count++;
   }
 
@@ -162,7 +161,6 @@ hwloc_backend_synthetic_init(struct hwloc_topology *topology, const char *descri
     topology->backend_params.synthetic.type[0] = HWLOC_OBJ_MACHINE;
     nb_machine_levels++;
   }
-  topology->backend_params.synthetic.id[0] = 0;
 
   if (cache_depth == 1)
     /* if there is a single cache level, make it L2 */
@@ -301,6 +299,12 @@ hwloc_look_synthetic(struct hwloc_topology *topology)
 
   topology->support.discovery->pu = 1;
 
+  /* start with id=0 for each level */
+  for (i = 0; topology->backend_params.synthetic.arity[i] > 0; i++)
+    topology->backend_params.synthetic.id[i] = 0;
+  /* ... including the last one */
+  topology->backend_params.synthetic.id[i] = 0;
+
   /* update first level type according to the synthetic type array */
   topology->levels[0][0]->type = topology->backend_params.synthetic.type[0];
 

opal/mca/paffinity/hwloc/hwloc/src/topology-windows.c

@@ -309,7 +309,7 @@ static BOOL WINAPI (*QueryWorkingSetExProc)(HANDLE hProcess, PVOID pv, DWORD cb)
 
 static int hwloc_win_get_VirtualAllocExNumaProc(void) {
   if (VirtualAllocExNumaProc == NULL) {
-    FARPROC alloc_fun, free_fun;
+    FARPROC alloc_fun = NULL, free_fun = NULL;
     HMODULE kernel32;
 
     kernel32 = LoadLibrary("kernel32.dll");
@@ -318,7 +318,7 @@ static int hwloc_win_get_VirtualAllocExNumaProc(void) {
       free_fun = GetProcAddress(kernel32, "VirtualFreeEx");
     }
 
-    if (!kernel32 || !alloc_fun || !free_fun) {
+    if (!alloc_fun || !free_fun) {
       VirtualAllocExNumaProc = (FARPROC) -1;
       errno = ENOSYS;
       return -1;
@@ -378,7 +378,7 @@ hwloc_win_free_membind(hwloc_topology_t topology __hwloc_attribute_unused, void
 
 static int hwloc_win_get_QueryWorkingSetExProc(void) {
   if (QueryWorkingSetExProc == NULL) {
-    FARPROC fun;
+    FARPROC fun = NULL;
     HMODULE kernel32, psapi;
 
     kernel32 = LoadLibrary("kernel32.dll");

opal/mca/paffinity/hwloc/hwloc/src/topology.c

@@ -43,6 +43,12 @@
 #include <windows.h>
 #endif
 
+unsigned hwloc_get_api_version(void)
+{
+  return HWLOC_API_VERSION;
+}
+
+
 static void
 hwloc_topology_clear (struct hwloc_topology *topology);
 

opal/mca/paffinity/hwloc/hwloc/src/traversal.c

@@ -193,6 +193,7 @@ hwloc_obj_type_snprintf(char * __hwloc_restrict string, size_t size, hwloc_obj_t
 	  /* TODO: more pretty presentation? */
     return hwloc_snprintf(string, size, "%s%u", hwloc_obj_type_string(type), obj->attr->group.depth);
   default:
+    if (size > 0)
       *string = '\0';
     return 0;
   }