- Add a first cut of the memory affinity (maffinity) framework. It's

API is still a bit unstable and may change.
- Add a primitive "first use" component that simply has each process
  "touch" the pages that they want to use, thereby [hopefully] locking
  them locally to a specific processor
- Add hooks in ompi_mpi_init to enable memory affinity when processor
  affinity is used.
- Added hooks in ompi_mpi_finalize to shut down memory affinity when
  it was initialized during ompi_mpi_init.
- Added right hooks in ompi_info to display maffinity components.

This commit was SVN r7044.

ompi/runtime/mpiruntime.h

@@ -49,6 +49,10 @@ extern "C" {
     /** Identifier of the main thread */
     OMPI_DECLSPEC extern struct opal_thread_t *ompi_mpi_main_thread;
 
+    /** Did we setup maffinity in MPI_INIT (and therefore need to shut
+        it down during MPI_FINALIZE)? */
+    OMPI_DECLSPEC extern bool ompi_mpi_maffinity_setup;
+
 
   /**
    * Initialize the Open MPI MPI environment

ompi/runtime/ompi_mpi_finalize.c

@@ -16,45 +16,46 @@
 
 #include "orte_config.h"
 
-#include "ompi/include/constants.h"
-#include "include/orte_constants.h"
-#include "mca/schema/schema.h"
+#include "opal/event/event.h"
+#include "opal/util/sys_info.h"
+#include "opal/runtime/opal_progress.h"
+#include "opal/mca/maffinity/base/base.h"
+#include "opal/mca/base/base.h"
+
+#include "orte/util/proc_info.h"
+#include "orte/include/orte_constants.h"
+#include "orte/mca/schema/schema.h"
+#include "orte/mca/oob/base/base.h"
+#include "orte/mca/ns/ns.h"
+#include "orte/mca/gpr/gpr.h"
+#include "orte/mca/rml/rml.h"
+#include "orte/mca/soh/soh.h"
+#include "orte/mca/soh/base/base.h"
+#include "orte/mca/errmgr/errmgr.h"
+#include "orte/runtime/runtime.h"
 
 #include "mpi.h"
-#include "opal/event/event.h"
-#include "group/group.h"
-#include "errhandler/errcode.h"
-#include "errhandler/errclass.h"
-#include "communicator/communicator.h"
-#include "datatype/datatype.h"
-#include "op/op.h"
-#include "file/file.h"
-#include "info/info.h"
-#include "util/proc_info.h"
-#include "util/sys_info.h"
+#include "ompi/include/constants.h"
+#include "ompi/group/group.h"
+#include "ompi/errhandler/errcode.h"
+#include "ompi/errhandler/errclass.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/datatype/datatype.h"
+#include "ompi/op/op.h"
+#include "ompi/file/file.h"
+#include "ompi/info/info.h"
 #include "ompi/runtime/mpiruntime.h"
-#include "orte/runtime/runtime.h"
-#include "opal/runtime/opal_progress.h"
-#include "attribute/attribute.h"
-
-#include "mca/base/base.h"
-#include "mca/pml/base/pml_base_module_exchange.h"
-#include "mca/pml/pml.h"
-#include "mca/pml/base/base.h"
-#include "mca/coll/coll.h"
-#include "mca/coll/base/base.h"
-#include "mca/topo/topo.h"
-#include "mca/topo/base/base.h"
-#include "mca/io/io.h"
-#include "mca/io/base/base.h"
-#include "mca/oob/base/base.h"
-#include "mca/ns/ns.h"
-#include "mca/gpr/gpr.h"
-#include "mca/rml/rml.h"
-#include "mca/soh/soh.h"
-#include "mca/soh/base/base.h"
-#include "mca/errmgr/errmgr.h"
-#include "mca/mpool/base/base.h"
+#include "ompi/attribute/attribute.h"
+#include "ompi/mca/pml/base/pml_base_module_exchange.h"
+#include "ompi/mca/pml/pml.h"
+#include "ompi/mca/pml/base/base.h"
+#include "ompi/mca/coll/coll.h"
+#include "ompi/mca/coll/base/base.h"
+#include "ompi/mca/topo/topo.h"
+#include "ompi/mca/topo/base/base.h"
+#include "ompi/mca/io/io.h"
+#include "ompi/mca/io/base/base.h"
+#include "ompi/mca/mpool/base/base.h"
 
 
 int ompi_mpi_finalize(void)
@@ -65,6 +66,10 @@ int ompi_mpi_finalize(void)
 #if OMPI_ENABLE_PROGRESS_THREADS == 0
     opal_progress_events(OPAL_EVLOOP_NONBLOCK);
 #endif
+    /* If maffinity was setup, tear it down */
+    if (ompi_mpi_maffinity_setup) {
+        opal_maffinity_base_close();
+    }
 
     /* Change progress function priority back to RTE level stuff */
     opal_progress_mpi_disable();

ompi/runtime/ompi_mpi_init.c

@@ -19,6 +19,7 @@
 #include "mpi.h"
 #include "opal/mca/base/base.h"
 #include "opal/mca/paffinity/base/base.h"
+#include "opal/mca/maffinity/base/base.h"
 #include "opal/runtime/opal_progress.h"
 #include "opal/util/sys_info.h"
 #include "opal/threads/threads.h"
@@ -78,6 +79,9 @@ int ompi_mpi_thread_provided = MPI_THREAD_SINGLE;
 
 opal_thread_t *ompi_mpi_main_thread = NULL;
 
+bool ompi_mpi_maffinity_setup = false;
+
+
 int ompi_mpi_init(int argc, char **argv, int requested, int *provided)
 {
     int ret;
@@ -158,6 +162,16 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided)
                                true, vpid);
                 free(vpid);
             }
+
+            /* If we were able to set processor affinity, try setting
+               up memory affinity */
+
+            else {
+                if (OPAL_SUCCESS == opal_maffinity_base_open() &&
+                    OPAL_SUCCESS == opal_maffinity_base_select()) {
+                    ompi_mpi_maffinity_setup = true;
+                }
+            }
         }
     }
 

ompi/tools/ompi_info/components.cc

@@ -31,12 +31,14 @@
 #include "opal/mca/paffinity/paffinity.h"
 #include "opal/mca/paffinity/base/base.h"
 #include "opal/mca/paffinity/base/internal.h"
+#include "opal/mca/maffinity/maffinity.h"
+#include "opal/mca/maffinity/base/base.h"
+#include "opal/mca/maffinity/base/internal.h"
 #include "opal/mca/memory/memory.h"
 #include "opal/mca/memory/base/base.h"
 #include "opal/mca/timer/timer.h"
 #include "opal/mca/timer/base/base.h"
 
-
 #include "ompi/mca/allocator/allocator.h"
 #include "ompi/mca/allocator/base/base.h"
 #include "ompi/mca/coll/coll.h"
@@ -155,10 +157,12 @@ void ompi_info::open_components()
   opal_paffinity_base_open();
   component_map["paffinity"] = &opal_paffinity_base_components_opened;
 
+  opal_maffinity_base_open();
+  component_map["maffinity"] = &opal_maffinity_base_components_opened;
+
   opal_timer_base_open();
   component_map["timer"] = &opal_timer_base_components_opened;
 
-
   // ORTE frameworks
 
   mca_oob_base_open();
@@ -268,6 +272,7 @@ void ompi_info::close_components()
     
         opal_memory_base_close();
         opal_paffinity_base_close();
+        opal_maffinity_base_close();
         opal_timer_base_close();
 
         component_map.clear();

ompi/tools/ompi_info/ompi_info.cc

@@ -167,6 +167,7 @@ int main(int argc, char *argv[])
 
   ompi_info::mca_types.push_back("memory");
   ompi_info::mca_types.push_back("paffinity");
+  ompi_info::mca_types.push_back("maffinity");
   ompi_info::mca_types.push_back("timer");
 
   ompi_info::mca_types.push_back("allocator");

opal/dynamic-mca/maffinity/Makefile.am

@@ -0,0 +1,26 @@
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University.
+#                         All rights reserved.
+# Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+#                         All rights reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+
+include $(top_srcdir)/config/Makefile.options
+
+SUBDIRS			= $(MCA_maffinity_DSO_SUBDIRS)
+DISTCLEANFILES		= $(SUBDIRS)
+
+# Every directory under here is a sym link to something in the main
+# src/mca tree.  Hence, we don't want to distribute anything under
+# here.
+
+DIST_SUBDIRS		=

opal/mca/maffinity/Makefile.am

@@ -0,0 +1,33 @@
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University.
+#                         All rights reserved.
+# Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+#                         All rights reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+
+include $(top_srcdir)/config/Makefile.options
+
+SUBDIRS = base $(MCA_maffinity_STATIC_SUBDIRS)
+DIST_SUBDIRS = base $(MCA_maffinity_ALL_SUBDIRS)
+
+# Source code files
+
+headers = maffinity.h maffinity_types.h
+
+# Conditionally install the header files
+
+if WANT_INSTALL_HEADERS
+ompidir = $(includedir)/openmpi/ompi/mca/maffinity
+ompi_HEADERS = $(headers)
+else
+ompidir = $(includedir)
+endif

opal/mca/maffinity/base/Makefile.am

@@ -0,0 +1,42 @@
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University.
+#                         All rights reserved.
+# Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+#                         All rights reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+
+include $(top_srcdir)/config/Makefile.options
+
+noinst_LTLIBRARIES = libmca_maffinity_base.la
+
+headers = \
+        base.h \
+        internal.h
+
+libmca_maffinity_base_la_SOURCES = \
+        $(headers) \
+        maffinity_base_close.c \
+        maffinity_base_select.c \
+        maffinity_base_open.c \
+        maffinity_base_wrappers.c
+
+# Conditionally install the header files
+
+if WANT_INSTALL_HEADERS
+ompidir = $(includedir)/openmpi/ompi/mca/maffinity/base
+ompi_HEADERS = $(headers)
+else
+ompidir = $(includedir)
+endif
+
+distclean-local:
+	rm -f static-components.h

opal/mca/maffinity/base/base.h

@@ -0,0 +1,124 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ *
+ */
+
+#ifndef OPAL_MAFFINITY_BASE_H
+#define OPAL_MAFFINITY_BASE_H
+
+#include "ompi_config.h"
+
+#include "mca/maffinity/maffinity.h"
+
+
+/*
+ * Global functions for MCA overall maffinity open and close
+ */
+
+#if defined(c_plusplus) || defined(__cplusplus)
+extern "C" {
+#endif
+
+    /**
+     * Initialize the maffinity MCA framework
+     *
+     * @retval OPAL_SUCCESS Upon success
+     * @retval OPAL_ERROR Upon failure
+     *
+     * This must be the first function invoked in the maffinity MCA
+     * framework.  It initializes the maffinity MCA framework, finds
+     * and opens maffinity components, etc.
+     *
+     * This function fills in the internal global variable
+     * opal_maffinity_base_components_opened, which is a list of all
+     * maffinity components that were successfully opened.  This
+     * variable should \em only be used by other maffinity base
+     * functions -- it is not considered a public interface member --
+     * and is only mentioned here for completeness.
+     */
+    OMPI_DECLSPEC int opal_maffinity_base_open(void);
+    
+    /**
+     * Select an available component.
+     *
+     * @return OPAL_SUCCESS Upon success.
+     * @return OPAL_NOT_FOUND If no component can be selected.
+     * @return OPAL_ERROR Upon other failure.
+     *
+     * This function invokes the selection process for maffinity
+     * components, which works as follows:
+     *
+     * - If the \em maffinity MCA parameter is not specified, the
+     *   selection set is all available maffinity components.
+     * - If the \em maffinity MCA parameter is specified, the
+     *   selection set is just that component.
+     * - All components in the selection set are queried to see if
+     *   they want to run.  All components that want to run are ranked
+     *   by their priority and the highest priority component is
+     *   selected.  All non-selected components have their "close"
+     *   function invoked to let them know that they were not selected.
+     * - The selected component will have its "init" function invoked to
+     *   let it know that it was selected.
+     *
+     * If we fall through this entire process and no component is
+     * selected, then return OPAL_NOT_FOUND (this is not a fatal
+     * error).
+     *
+     * At the end of this process, we'll either have a single
+     * component that is selected and initialized, or no component was
+     * selected.  If no component was selected, subsequent invocation
+     * of the maffinity wrapper functions will return an error.
+     */
+    OMPI_DECLSPEC int opal_maffinity_base_select(void);
+
+    /**
+     * Set memory affinity.
+     *
+     * @param segments Array describing segments and what process they
+     * belong to
+     * @param num_segments Length of the segments array
+     * @param am_allocator True if this process created the shared
+     * memory block
+     *
+     * @retval OPAL_SUCCESS upon success
+     * @retval OPAL_NOT_FOUND if no maffinity components are available.
+     * @retval OPAL_ERROR upon other error.
+     *
+     * Set the affinity of the memory segments described in the \em
+     * segments array.
+     *
+     * If no maffinity components were available, or if the
+     * opal_maffinity_base_select() was never invoked, OPAL_NOT_FOUND
+     * is returned.
+     */
+    OMPI_DECLSPEC int opal_maffinity_base_set(opal_maffinity_base_segment_t *segments, size_t num_segments, bool am_allocator);
+
+    /**
+     * Shut down the maffinity MCA framework.
+     *
+     * @retval OPAL_SUCCESS Always
+     *
+     * This function shuts down everything in the maffinity MCA
+     * framework.
+     *
+     * It must be the last function invoked on the maffinity MCA
+     * framework.
+     */
+    OMPI_DECLSPEC int opal_maffinity_base_close(void);
+    
+#if defined(c_plusplus) || defined(__cplusplus)
+}
+#endif
+#endif /* OPAL_BASE_MAFFINITY_H */

opal/mca/maffinity/base/internal.h

@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ *
+ */
+
+#ifndef OPAL_MAFFINITY_BASE_INTERNAL_H
+#define OPAL_MAFFINITY_BASE_INTERNAL_H
+
+#include "ompi_config.h"
+
+#include "opal/class/opal_list.h"
+#include "mca/maffinity/maffinity.h"
+
+
+/** 
+ * @file 
+ *
+ * Global functions for MCA overall maffinity open and close
+ */
+
+#if defined(c_plusplus) || defined(__cplusplus)
+extern "C" {
+#endif
+
+    /**
+     * Indication of whether a component was successfully selected or
+     * not
+     */
+    OMPI_DECLSPEC extern bool opal_maffinity_base_selected;
+
+    /**
+     * Global component struct for the selected component
+     */
+    OMPI_DECLSPEC extern const opal_maffinity_base_component_1_0_0_t 
+        *opal_maffinity_base_component;
+    /**
+     * Global module struct for the selected module
+     */
+    OMPI_DECLSPEC extern const opal_maffinity_base_module_1_0_0_t 
+        *opal_maffinity_base_module;
+
+    /**
+     * Indicator as to whether the list of opened maffinity components
+     * is valid or not.
+     */
+    OMPI_DECLSPEC extern bool opal_maffinity_base_components_opened_valid;
+    /**
+     * List of all opened components; created when the maffinity
+     * framework is initialized and destroyed when we reduce the list
+     * to all available maffinity components.
+     */
+    OMPI_DECLSPEC extern opal_list_t opal_maffinity_base_components_opened;
+
+    /**
+     * Debugging output stream
+     */
+    OMPI_DECLSPEC extern int opal_maffinity_base_output;
+
+#if defined(c_plusplus) || defined(__cplusplus)
+}
+#endif
+#endif /* OPAL_MAFFINITY_BASE_INTERNAL_H */

opal/mca/maffinity/base/maffinity_base_close.c

@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "opal/include/constants.h"
+#include "opal/mca/mca.h"
+#include "opal/mca/base/base.h"
+#include "opal/mca/maffinity/maffinity.h"
+#include "opal/mca/maffinity/base/base.h"
+#include "opal/mca/maffinity/base/internal.h"
+
+
+int opal_maffinity_base_close(void)
+{
+    /* Close all components that are still open (this should only
+       happen during ompi_info). */
+
+    if (opal_maffinity_base_components_opened_valid) {
+        mca_base_components_close(opal_maffinity_base_output,
+                                  &opal_maffinity_base_components_opened, NULL);
+        OBJ_DESTRUCT(&opal_maffinity_base_components_opened);
+        opal_maffinity_base_components_opened_valid = false;
+    }
+
+    /* All done */
+
+    return OPAL_SUCCESS;
+}

opal/mca/maffinity/base/maffinity_base_open.c

@@ -0,0 +1,81 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+
+#include "ompi_config.h"
+
+#include "opal/include/constants.h"
+#include "opal/util/output.h"
+#include "opal/mca/mca.h"
+#include "opal/mca/base/base.h"
+#include "opal/mca/base/mca_base_param.h"
+#include "opal/mca/maffinity/maffinity.h"
+#include "opal/mca/maffinity/base/base.h"
+
+
+/*
+ * The following file was created by configure.  It contains extern
+ * statements and the definition of an array of pointers to each
+ * component's public mca_base_component_t struct.
+ */
+#include "opal/mca/maffinity/base/static-components.h"
+
+
+/*
+ * Globals
+ */
+int opal_maffinity_base_output = -1;
+bool opal_maffinity_base_components_opened_valid = false;
+opal_list_t opal_maffinity_base_components_opened;
+
+
+/*
+ * Function for finding and opening either all MCA components, or the one
+ * that was specifically requested via a MCA parameter.
+ */
+int opal_maffinity_base_open(void)
+{
+    int value;
+
+    /* Debugging / verbose output */
+
+    mca_base_param_reg_int_name("maffinity_base", "verbose", 
+                                "Verbosity level of the maffinity framework",
+                                false, false,
+                                0, &value);
+    if (0 != value) {
+        opal_maffinity_base_output = opal_output_open(NULL);
+    } else {
+        opal_maffinity_base_output = -1;
+    }
+
+    opal_maffinity_base_components_opened_valid = false;
+
+    /* Open up all available components */
+
+    if (OMPI_SUCCESS !=
+        mca_base_components_open("maffinity", opal_maffinity_base_output,
+                                 mca_maffinity_base_static_components,
+                                 &opal_maffinity_base_components_opened, 
+                                 true)) {
+        return OPAL_ERROR;
+    }
+    opal_maffinity_base_components_opened_valid = true;
+
+    /* All done */
+
+    return OPAL_SUCCESS;
+}

opal/mca/maffinity/base/maffinity_base_select.c

@@ -0,0 +1,146 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+
+#include "ompi_config.h"
+
+#include "opal/include/constants.h"
+#include "opal/util/output.h"
+#include "opal/mca/mca.h"
+#include "opal/mca/base/base.h"
+#include "opal/mca/base/mca_base_param.h"
+#include "opal/mca/maffinity/maffinity.h"
+#include "opal/mca/maffinity/base/base.h"
+#include "opal/mca/maffinity/base/internal.h"
+
+
+/*
+ * Globals
+ */
+bool opal_maffinity_base_selected = false;
+const opal_maffinity_base_component_1_0_0_t *opal_maffinity_base_component = NULL;
+const opal_maffinity_base_module_1_0_0_t *opal_maffinity_base_module = NULL;
+
+
+int opal_maffinity_base_select(void)
+{
+    int priority = 0, best_priority = 0;
+    opal_list_item_t *item = NULL;
+    mca_base_component_list_item_t *cli = NULL;
+    opal_maffinity_base_component_1_0_0_t *component = NULL,
+        *best_component = NULL;
+    const opal_maffinity_base_module_1_0_0_t *module = NULL, 
+        *best_module = NULL;
+    char *value;
+
+    /* Register the framework MCA param and look it up */
+
+    mca_base_param_reg_string_name("maffinity", NULL,
+                                   "Which maffinity component to use (empty = auto-select)",
+                                   false, false,
+                                   NULL, &value);
+    if (NULL == value || 0 == strlen(value)) {
+        opal_output_verbose(10, opal_maffinity_base_output,
+                            "maffinity:select: auto-selecting");
+    } else {
+        opal_output_verbose(10, opal_maffinity_base_output,
+                            "maffinity:select: looking for %s component", 
+                            value);
+    }
+
+    /* Traverse the list of available components; call their init
+       functions. */
+
+    best_priority = -1;
+    best_component = NULL;
+    module = NULL;
+    for (item = opal_list_get_first(&opal_maffinity_base_components_opened);
+         opal_list_get_end(&opal_maffinity_base_components_opened) != item;
+         item = opal_list_get_next(item) ) {
+        cli = (mca_base_component_list_item_t *) item;
+        component = (opal_maffinity_base_component_1_0_0_t *) cli->cli_component;
+
+        /* if there is an include list - item must be in the list to
+           be included */
+
+        if (NULL != value && strlen(value) > 0 &&
+            0 != strcmp(component->maffinityc_version.mca_component_name, 
+                        value)) {
+            opal_output_verbose(10, opal_maffinity_base_output,
+                                 "maffinity:select: skipping %s component",
+                                 component->maffinityc_version.mca_component_name);
+            continue;
+        }
+
+        if (NULL == component->maffinityc_query) {
+            opal_output_verbose(10, opal_maffinity_base_output,
+                                "pafinity:select: no init function; ignoring component %s",
+                                component->maffinityc_version.mca_component_name );
+            continue;
+        }
+        opal_output_verbose(10, opal_maffinity_base_output, 
+                            "maffinity:select: initializing component %s",
+                            component->maffinityc_version.mca_component_name);
+        module = component->maffinityc_query(&priority);
+        if (NULL == module) {
+            opal_output_verbose(10, opal_maffinity_base_output,
+                                "maffinity:select: init returned failure for component %s",
+                                component->maffinityc_version.mca_component_name );
+            continue;
+        }
+        opal_output_verbose(10, opal_maffinity_base_output,
+                            "maffinity:select: init returned priority %d", 
+                            priority );
+        if (priority > best_priority) {
+            best_priority = priority;
+            best_component = component;
+            best_module = module;
+        }
+    }
+
+    /* Finished querying all components.  Check for the bozo case. */
+
+    if (NULL == best_component ) {
+        return OPAL_ERR_NOT_FOUND;
+    } 
+
+    /* Now go through the opened list and close all the non-selected
+       components */
+
+    mca_base_components_close(opal_maffinity_base_output, 
+                              &opal_maffinity_base_components_opened, 
+                              (mca_base_component_t *) best_component);
+    
+    /* Save the winner */
+
+    opal_maffinity_base_component = best_component;
+    opal_maffinity_base_module = best_module;
+    opal_output_verbose(10, opal_maffinity_base_output, 
+                        "maffinity:select: component %s selected",
+                        best_component->maffinityc_version.mca_component_name);
+    opal_maffinity_base_selected = true;
+
+    /* Initialize the winner */
+
+    if (NULL != opal_maffinity_base_module) {
+        if (OPAL_SUCCESS != opal_maffinity_base_module->maff_module_init()) {
+            return OPAL_ERROR;
+        }
+    }
+
+    return OPAL_SUCCESS;
+}
+

opal/mca/maffinity/base/maffinity_base_wrappers.c

@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+
+#include "ompi_config.h"
+
+#include "opal/include/constants.h"
+#include "opal/mca/maffinity/maffinity.h"
+#include "opal/mca/maffinity/base/base.h"
+#include "opal/mca/maffinity/base/internal.h"
+
+
+int opal_maffinity_base_set(opal_maffinity_base_segment_t *segments,
+                            size_t num_segments, bool am_allocator)
+{
+    if (!opal_maffinity_base_selected) {
+        return OPAL_ERR_NOT_FOUND;
+    }
+    return opal_maffinity_base_module->maff_module_set(segments, num_segments,
+                                                       am_allocator);
+}

opal/mca/maffinity/first_use/Makefile.am

@@ -0,0 +1,45 @@
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University.
+#                         All rights reserved.
+# Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+#                         All rights reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+
+# Use the top-level Makefile.options
+
+include $(top_ompi_srcdir)/config/Makefile.options
+
+sources = \
+        maffinity_first_use.h \
+        maffinity_first_use_component.c \
+        maffinity_first_use_module.c
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if OMPI_BUILD_maffinity_first_use_DSO
+component_noinst =
+component_install = mca_maffinity_first_use.la
+else
+component_noinst = libmca_maffinity_first_use.la
+component_install =
+endif
+
+mcacomponentdir = $(libdir)/openmpi
+mcacomponent_LTLIBRARIES = $(component_install)
+mca_maffinity_first_use_la_SOURCES = $(sources)
+mca_maffinity_first_use_la_LDFLAGS = -module -avoid-version
+
+noinst_LTLIBRARIES = $(component_noinst)
+libmca_maffinity_first_use_la_SOURCES =$(sources)
+libmca_maffinity_first_use_la_LDFLAGS = -module -avoid-version

opal/mca/maffinity/first_use/configure.params

@@ -0,0 +1,19 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University.
+#                         All rights reserved.
+# Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+#                         All rights reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+
+PARAM_INIT_FILE=maffinity_first_use_component.c
+PARAM_CONFIG_FILES="Makefile"

opal/mca/maffinity/first_use/maffinity_first_use.h

@@ -0,0 +1,107 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+/**
+ * @file
+ *
+ * Processor affinity for First_Use.
+ *
+ * First_Use sucks.  There are at least 3 different ways that
+ * sched_setaffinity is implemented (only one of which -- the most
+ * outdated -- is documented in the sched_setaffinity(2) man page):
+ *
+ *-----------------------------------------------------------------
+ * 1. int sched_setaffinity(pid_t pid, unsigned int len, unsigned
+ * long *mask);
+ *
+ * This originated in 2.5 kernels (which we won't worry about) and
+ * some distros back-ported it to their 2.4 kernels.  It's unknown if
+ * this appears in any 2.6 kernels.
+ *
+ * 2. int sched_setaffinity (pid_t __pid, size_t __cpusetsize,
+ * const cpu_set_t *__cpuset);
+ *
+ * This appears to be in recent 2.6 kernels (confirmed in Gentoo
+ * 2.6.11).  I don't know when #1 changed into #2.  However, this
+ * prototype is nice -- the cpu_set_t type is accompanied by
+ * fdset-like CPU_ZERO(), CPU_SET(), CPU_ISSET(), etc. macros.
+ *
+ * 3. int sched_setaffinity (pid_t __pid, const cpu_set_t *__mask);
+ *
+ * (note the missing len parameter) This is in at least some First_Use
+ * distros (e.g., MDK 10.0 with a 2.6.3 kernel, and SGI Altix, even
+ * though the Altix uses a 2.4-based kernel and therefore likely
+ * back-ported the 2.5 work but modified it for their needs).  Similar
+ * to #2, the cpu_set_t type is accompanied by fdset-like CPU_ZERO(),
+ * CPU_SET(), CPU_ISSET(), etc. macros.
+ *-----------------------------------------------------------------
+ *
+ * This component has to figure out which one to use.  :-\
+ *
+ * Also note that at least some distros of First_Use have a broken
+ * CPU_ZERO macro (a pair of typos in /usr/include/bits/sched.h).
+ * MDK 9.2 is the screaming example, but it's pretty old and
+ * probably only matters because one of the developers uses that as
+ * a compilation machine :-) (it also appears to have been fixed in
+ * MDK 10.0, but they also changed from #2 to #3 -- arrgh!).
+ * However, there's no way of knowing where these typos came from
+ * and if they exist elsewhere.  So it seems safest to extend this
+ * configure script to check for a bad CPU_ZERO macro.  #$%#@%$@!!!
+ */
+
+
+#ifndef MCA_MAFFINITY_FIRST_USE_EXPORT_H
+#define MCA_MAFFINITY_FIRST_USE_EXPORT_H
+
+#include "ompi_config.h"
+
+#include "opal/mca/mca.h"
+#include "opal/mca/maffinity/maffinity.h"
+
+
+/**
+ * Determine whether we have a working CPU_ZERO() macro or not.  If
+ * not, use memset().
+ */
+#ifdef HAVE_CPU_ZERO
+#define OMPI_CPU_ZERO(foo) CPU_ZERO(foo)
+#else
+#include <string.h>
+#define OMPI_CPU_ZERO(foo) memset(foo, 0, sizeof(*foo))
+#endif
+
+
+#if defined(c_plusplus) || defined(__cplusplus)
+extern "C" {
+#endif
+
+    /**
+     * Globally exported variable
+     */
+    OMPI_COMP_EXPORT extern const opal_maffinity_base_component_1_0_0_t
+        mca_maffinity_first_use_component;
+
+
+    /**
+     * maffinity query API function
+     */
+    const opal_maffinity_base_module_1_0_0_t *
+        opal_maffinity_first_use_component_query(int *query);
+
+#if defined(c_plusplus) || defined(__cplusplus)
+}
+#endif
+#endif /* MCA_MAFFINITY_FIRST_USE_EXPORT_H */

opal/mca/maffinity/first_use/maffinity_first_use_component.c

@@ -0,0 +1,91 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ *
+ * These symbols are in a file by themselves to provide nice linker
+ * semantics.  Since linkers generally pull in symbols by object
+ * files, keeping these symbols as the only symbols in this file
+ * prevents utility programs such as "ompi_info" from having to import
+ * entire components just to query their version and parameters.
+ */
+
+#include "ompi_config.h"
+
+#include "opal/include/constants.h"
+#include "opal/mca/maffinity/maffinity.h"
+#include "maffinity_first_use.h"
+
+/*
+ * Public string showing the maffinity ompi_first_use component version number
+ */
+const char *opal_maffinity_first_use_component_version_string =
+    "OPAL first_use maffinity MCA component version " OMPI_VERSION;
+
+/*
+ * Local function
+ */
+static int first_use_open(void);
+
+/*
+ * Instantiate the public struct with all of our public information
+ * and pointers to our public functions in it
+ */
+
+const opal_maffinity_base_component_1_0_0_t mca_maffinity_first_use_component = {
+
+    /* First, the mca_component_t struct containing meta information
+       about the component itself */
+
+    {
+        /* Indicate that we are a maffinity v1.0.0 component (which also
+           implies a specific MCA version) */
+        
+        OPAL_MAFFINITY_BASE_VERSION_1_0_0,
+
+        /* Component name and version */
+
+        "first_use",
+        OMPI_MAJOR_VERSION,
+        OMPI_MINOR_VERSION,
+        OMPI_RELEASE_VERSION,
+
+        /* Component open and close functions */
+
+        first_use_open,
+        NULL
+    },
+
+    /* Next the MCA v1.0.0 component meta data */
+
+    {
+        /* Whether the component is checkpointable or not */
+        
+        true
+    },
+
+    /* Query function */
+
+    opal_maffinity_first_use_component_query
+};
+
+
+static int first_use_open(void)
+{
+    mca_base_param_reg_int(&mca_maffinity_first_use_component.maffinityc_version,
+                           "priority",
+                           "Priority of the first_use maffinity component",
+                           false, false, 10, NULL);
+
+    return OPAL_SUCCESS;
+}

opal/mca/maffinity/first_use/maffinity_first_use_module.c

@@ -0,0 +1,90 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include <sys/types.h>
+#include <unistd.h>
+#include <string.h>
+
+#include "opal/include/constants.h"
+#include "opal/mca/maffinity/maffinity.h"
+#include "opal/mca/maffinity/base/base.h"
+#include "maffinity_first_use.h"
+
+
+/*
+ * Local functions
+ */
+static int first_use_module_init(void);
+static int first_use_module_set(opal_maffinity_base_segment_t *segments,
+                                size_t num_segments, bool am_allocator);
+
+/*
+ * First_Use maffinity module
+ */
+static const opal_maffinity_base_module_1_0_0_t module = {
+
+    /* Initialization function */
+
+    first_use_module_init,
+
+    /* Module function pointers */
+
+    first_use_module_set
+};
+
+
+const opal_maffinity_base_module_1_0_0_t *
+opal_maffinity_first_use_component_query(int *query)
+{
+    int param;
+
+    param = mca_base_param_find("maffinity", "first_use", "priority");
+    mca_base_param_lookup_int(param, query);
+
+    return &module;
+}
+
+
+static int first_use_module_init(void)
+{
+    /* Nothing to do */
+
+    return OPAL_SUCCESS;
+}
+
+
+static int first_use_module_set(opal_maffinity_base_segment_t *segments,
+                                size_t num_segments, bool am_allocator)
+{
+    size_t i;
+    pid_t mypid = getpid();
+
+    /* Crude: zero out all the segments that belong to me.  We could
+       probably get away with touching a byte in each page (which
+       would potentially be much faster), but this is during setup so
+       it's not a huge deal.  Consider this a target for future
+       optimization... */
+
+    for (i = 0; i < num_segments; ++i) {
+        if (segments[i].mbs_owner_pid == mypid) {
+            memset(segments[i].mbs_start_addr, 0, segments[i].mbs_len);
+        }
+    }
+
+    return OPAL_SUCCESS;
+}

opal/mca/maffinity/maffinity.h

@@ -0,0 +1,149 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+/**
+ * @file
+ *
+ * maffinity (memory affinity) framework component interface
+ * definitions.
+ *
+ * Intent
+ *
+ * Simple component to set memory affinity for pages.  Note that this
+ * framework assumes that processor affinity is being used (it doesn't
+ * make much sense to use memory affinity unless processes are bound
+ * to specific processors, otherwise the processes may move around and
+ * the memory may end up being remote).
+ *
+ * maffinity components are typically used with shared memory
+ * operations, but can be used elsewhere as well.  The idea is to get
+ * memory physically located with the process that is going to use it.
+ * For memory allocated to a processor-bound process, functions such
+ * as malloc() do this naturally.  However, when working with memory
+ * shared by multiple processes on a NUMA machine, it can be extremely
+ * advantageous to ensure that pages containing the data structures
+ * for a given process are physically local to the processor where
+ * that process is bound.
+ *
+ * One process will allocate a large shared memory block and one or
+ * more processors will need to participate to make pages local to
+ * specific processors.  
+ *
+ * Some systems have an API (e.g., SGI Altix) where a single function
+ * call by the allocating process can assign the page locality of the
+ * pages throughout the entire shared block.  Other systems follow a
+ * "first use" rule, where pages in the shared block are assigned to
+ * memory local to the processor of the first process that uses it.
+ *
+ * This API can handle both models.  There is one main module function
+ * (opal_maffinity_base_module_set_fn_t) that takes an array of
+ * segment descriptions within the block.  This is enough information
+ * to describe each section in the shared block and what process owns
+ * it.  Components can then do whatever is necessary to make pages
+ * local to their respective processes (i.e., the processors where the
+ * processes are running).
+ */
+
+#ifndef OPAL_MAFFINITY_H
+#define OPAL_MAFFINITY_H
+
+#include "ompi_config.h"
+
+#include "opal/mca/mca.h"
+#include "opal/mca/base/base.h"
+#include "opal/mca/maffinity/maffinity_types.h"
+
+
+/**
+ * Query function for maffinity components.  Simply returns a priority
+ * to rank it against other available maffinity components (assumedly,
+ * only one component will be available per platform, but it's
+ * possible that there could be more than one available).
+ */
+typedef const struct opal_maffinity_base_module_1_0_0_t *
+    (*opal_maffinity_base_component_query_1_0_0_fn_t)
+    (int *priority);
+
+
+/**
+ * Module initialization function.  Should return OPAL_SUCCESS.
+ */
+typedef int (*opal_maffinity_base_module_init_1_0_0_fn_t)(void);
+
+
+/**
+ * Module function to set memory affinity.  Take an array of
+ * maffinity_base_segment_t instances to describe which memory should
+ * physically reside with which process.
+ *
+ * This function is intended to be invoked by each process immediately
+ * after they mmap / attach the shared memory.  In some cases, it will
+ * be a no-op for some processes (i.e., machines where a single
+ * function call in the creating process sets the memory affinity for
+ * the entire region), but in other cases all processes will need to
+ * participate (e.g., the first_use component, each each process will
+ * "touch" the pages that are supposed to be local to them).
+ */
+typedef int (*opal_maffinity_base_module_set_fn_t)
+    (opal_maffinity_base_segment_t *segments, size_t num_segments,
+     bool am_allocator);
+
+
+/**
+ * Structure for maffinity v1.0.0 components.
+ * Chained to MCA v1.0.0
+ */
+struct opal_maffinity_base_component_1_0_0_t {
+    /** MCA base component */
+    mca_base_component_t maffinityc_version;
+    /** MCA base data */
+    mca_base_component_data_1_0_0_t maffinityc_data;
+
+    /** Component query function */
+    opal_maffinity_base_component_query_1_0_0_fn_t maffinityc_query;
+};
+/**
+ * Convenience typedef
+ */
+typedef struct opal_maffinity_base_component_1_0_0_t opal_maffinity_base_component_1_0_0_t;
+
+
+/**
+ * Structure for maffinity v1.0.0 modules
+ */
+struct opal_maffinity_base_module_1_0_0_t {
+
+    /** Module initialization function */
+    opal_maffinity_base_module_init_1_0_0_fn_t maff_module_init;
+
+    /** Set memory affinity */
+    opal_maffinity_base_module_set_fn_t maff_module_set;
+};
+/**
+ * Convenience typedef
+ */
+typedef struct opal_maffinity_base_module_1_0_0_t opal_maffinity_base_module_1_0_0_t;
+
+
+/*
+ * Macro for use in components that are of type maffinity v1.0.0
+ */
+#define OPAL_MAFFINITY_BASE_VERSION_1_0_0 \
+    /* maffinity v1.0 is chained to MCA v1.0 */ \
+    MCA_BASE_VERSION_1_0_0, \
+    /* maffinity v1.0 */ \
+    "maffinity", 1, 0, 0
+
+#endif /* OPAL_MAFFINITY_H */

opal/mca/maffinity/maffinity_types.h

@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+/**
+ * @file
+ *
+ * Common types used in the maffinity framework
+ */
+
+#ifndef OPAL_MAFFINITY_TYPES_H
+#define OPAL_MAFFINITY_TYPES_H
+
+#include "ompi_config.h"
+
+#include <sys/types.h>
+
+#if defined(c_plusplus) || defined(__cplusplus)
+extern "C" {
+#endif
+
+    /**
+     * Struct used with opal_maffinity_base_module_set_fn_t.  It
+     * describes a section of memory (starting address and length) and
+     * what process owns it (by PID).  PID is unique enough because
+     * all processes are guaranteed to be on the same machine.
+     */
+    struct opal_maffinity_base_segment_t {
+        /** Owning process */
+        pid_t mbs_owner_pid;
+        /** Starting address of segment */
+        void *mbs_start_addr;
+        /** Length of segment */
+        size_t mbs_len;
+    };
+    /**
+     * Convenience typedef
+     */
+    typedef struct opal_maffinity_base_segment_t opal_maffinity_base_segment_t;
+
+#if defined(c_plusplus) || defined(__cplusplus)
+}
+#endif
+
+#endif /* OPAL_MAFFINITY_TYPES_H */