- Add MCA param mpi_paffinity_alone, which, when nonzero, will assume

that this ORTE job is the only one on the nodes involved, and if
  told what processors to assign the processes to, will bind MPI
  processes to specific processors.
- Convert #include's to new style
- Convert some <tab>'s to spaces

This commit was SVN r6904.

ompi/runtime/ompi_mpi_init.c

@@ -16,52 +16,53 @@
 
 #include "ompi_config.h"
 
+#include "mpi.h"
+#include "opal/mca/base/base.h"
+#include "opal/mca/paffinity/base/base.h"
+#include "opal/runtime/opal_progress.h"
+#include "opal/util/sys_info.h"
+#include "opal/threads/threads.h"
+#include "opal/util/show_help.h"
+#include "opal/util/stacktrace.h"
+#include "opal/runtime/opal.h"
+#include "opal/event/event.h"
+
+#include "orte/util/proc_info.h"
+#include "orte/util/session_dir.h"
+#include "orte/runtime/runtime.h"
+#include "orte/mca/oob/oob.h"
+#include "orte/mca/oob/base/base.h"
+#include "orte/mca/ns/ns.h"
+#include "orte/mca/gpr/gpr.h"
+#include "orte/mca/rml/rml.h"
+#include "orte/mca/schema/schema.h"
+#include "orte/mca/soh/soh.h"
+#include "orte/mca/soh/base/base.h"
+#include "orte/mca/errmgr/errmgr.h"
+
 #include "ompi/include/constants.h"
 #include "ompi/runtime/mpiruntime.h"
 #include "ompi/runtime/params.h"
-#include "orte/runtime/runtime.h"
+#include "ompi/communicator/communicator.h"
-#include "opal/runtime/opal_progress.h"
+#include "ompi/group/group.h"
-#include "util/sys_info.h"
+#include "ompi/info/info.h"
-#include "util/proc_info.h"
+#include "ompi/errhandler/errcode.h"
-#include "util/session_dir.h"
+#include "ompi/errhandler/errclass.h"
-#include "mpi.h"
+#include "ompi/request/request.h"
-#include "communicator/communicator.h"
+#include "ompi/op/op.h"
-#include "group/group.h"
+#include "ompi/file/file.h"
-#include "info/info.h"
+#include "ompi/attribute/attribute.h"
-#include "opal/util/show_help.h"
+#include "ompi/mca/allocator/base/base.h"
-#include "opal/util/stacktrace.h"
+#include "ompi/mca/allocator/allocator.h"
-#include "errhandler/errcode.h"
+#include "ompi/mca/mpool/base/base.h"
-#include "errhandler/errclass.h"
+#include "ompi/mca/mpool/mpool.h"
-#include "request/request.h"
+#include "ompi/mca/pml/pml.h"
-#include "op/op.h"
+#include "ompi/mca/pml/base/pml_base_module_exchange.h"
-#include "file/file.h"
+#include "ompi/mca/pml/base/base.h"
-#include "attribute/attribute.h"
+#include "ompi/mca/coll/coll.h"
-#include "opal/threads/threads.h"
+#include "ompi/mca/coll/base/base.h"
-
+#include "ompi/mca/io/io.h"
-#include "mca/base/base.h"
+#include "ompi/mca/io/base/base.h"
-#include "mca/allocator/base/base.h"
-#include "mca/allocator/allocator.h"
-#include "mca/mpool/base/base.h"
-#include "mca/mpool/mpool.h"
-#include "mca/pml/pml.h"
-#include "mca/pml/base/pml_base_module_exchange.h"
-#include "mca/pml/base/base.h"
-#include "mca/coll/coll.h"
-#include "mca/coll/base/base.h"
-#include "mca/io/io.h"
-#include "mca/io/base/base.h"
-#include "mca/oob/oob.h"
-#include "mca/oob/base/base.h"
-#include "mca/ns/ns.h"
-#include "mca/gpr/gpr.h"
-#include "mca/rml/rml.h"
-#include "mca/schema/schema.h"
-#include "mca/soh/soh.h"
-#include "mca/soh/base/base.h"
-#include "mca/errmgr/errmgr.h"
-
-#include "opal/runtime/opal.h"
-#include "opal/event/event.h"
 
 /*
  * Global variables and symbols for the MPI layer
@@ -91,10 +92,12 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided)
         error = "ompi_mpi_init: opal_init failed";
         goto error;
     }
+
+    /* Setup ORTE stage 1 */
     
     if (ORTE_SUCCESS != (ret = orte_init_stage1())) {
         error = "ompi_mpi_init: orte_init_stage1 failed";
-	    goto error;
+        goto error;
     }
 
     /* If we are not the seed nor a singleton, AND we have not set the
@@ -132,15 +135,30 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided)
         goto error;
     }
 
+    /* Setup process affinity */
+
+    if (ompi_mpi_paffinity_alone) {
+        int param, value;
+        param = mca_base_param_find("mpi", NULL, "paffinity_processor");
+        if (param >= 0) {
+            mca_base_param_lookup_int(param, &value);
+            if (value >= 0) {
+                printf("Setting affinity! proc %d\n", value);
+                opal_paffinity_base_set(value);
+            }
+        }
+    }
+
 #ifndef WIN32
     if (OMPI_SUCCESS != (ret = opal_util_register_stackhandlers ())) {
-	error = "util_register_stackhandlers() failed";
+        error = "util_register_stackhandlers() failed";
-	goto error;
+        goto error;
     }
 #endif
 
-    /* initialize datatypes. This step should be done early as it will create the local convertor
+    /* initialize datatypes. This step should be done early as it will
-     * and local arch used in the proc init.
+     * create the local convertor and local arch used in the proc
+     * init.
      */
     if (OMPI_SUCCESS != (ret = ompi_ddt_init())) {
         error = "ompi_ddt_init() failed";
@@ -309,16 +327,16 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided)
     if (compound_cmd) {
         if (OMPI_SUCCESS != (ret = orte_gpr.exec_compound_cmd())) {
             ORTE_ERROR_LOG(ret);
-    	    error = "ompi_rte_init: orte_gpr.exec_compound_cmd failed";
+            error = "ompi_rte_init: orte_gpr.exec_compound_cmd failed";
-    	    goto error;
+            goto error;
         }
     }
 
      /* FIRST BARRIER - WAIT FOR MSG FROM RMGR_PROC_STAGE_GATE_MGR TO ARRIVE */
     if (ORTE_SUCCESS != (ret = orte_rml.xcast(NULL, NULL, 0, NULL, NULL))) {
         ORTE_ERROR_LOG(ret);
-	    error = "ompi_mpi_init: failed to see all procs register\n";
+        error = "ompi_mpi_init: failed to see all procs register\n";
-	    goto error;
+        goto error;
     }
 
     /* start PTL's */
@@ -389,8 +407,8 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided)
     /* setup I/O forwarding */
     if (orte_process_info.seed == false) {
         if (OMPI_SUCCESS != (ret = ompi_mpi_init_io())) {
-	        error = "ompi_rte_init_io failed";
+            error = "ompi_rte_init_io failed";
-	        goto error;
+            goto error;
         }
     }
 #endif
@@ -455,8 +473,8 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided)
     ompi_mpi_initialized = true;
 
     if (orte_debug_flag) {
-	opal_output(0, "[%lu,%lu,%lu] ompi_mpi_init completed",
+        opal_output(0, "[%lu,%lu,%lu] ompi_mpi_init completed",
-		    ORTE_NAME_ARGS(orte_process_info.my_name));
+                    ORTE_NAME_ARGS(orte_process_info.my_name));
     }
 
     return MPI_SUCCESS;

ompi/runtime/ompi_mpi_params.c

@@ -38,6 +38,8 @@ bool ompi_debug_show_handle_leaks = false;
 bool ompi_debug_no_free_handles = false;
 bool ompi_mpi_show_mca_params = false;
 char *ompi_mpi_show_mca_params_file = NULL;
+bool ompi_mpi_paffinity_alone = false;
+
 
 int ompi_mpi_register_params(void)
 {
@@ -125,6 +127,18 @@ int ompi_mpi_register_params(void)
                                    false, false,
                                    "", &ompi_mpi_show_mca_params_file);
     
+    /* User-level process pinning controls */
+    mca_base_param_reg_int_name("mpi", "paffinity_alone",
+                                "If nonzero, assume that this job is the only (set of) process(es) running on each node and bind processes to processors, starting with processor ID 0",
+                                false, false, 
+                                (int) ompi_mpi_paffinity_alone, &value);
+    ompi_mpi_paffinity_alone = (bool) value;
+
+    mca_base_param_reg_int_name("mpi", "paffinity_processor",
+                                "If set, pin this process to the processor number indicated by the value",
+                                true, false, 
+                                -1, NULL);
+
     /* All done */
 
     return OMPI_SUCCESS;

ompi/runtime/params.h

@@ -79,6 +79,13 @@ OMPI_DECLSPEC extern bool ompi_mpi_show_mca_params;
  */
 OMPI_DECLSPEC extern char * ompi_mpi_show_mca_params_file;
 
+/**
+ * If this value is true, assume that this ORTE job is the only job
+ * running on the nodes that have been allocated to it, and bind
+ * processes to processors (starting with processor 0).
+ */
+OMPI_DECLSPEC extern bool ompi_mpi_paffinity_alone;
+
    /**
      * Register MCA parameters used by the MPI layer.
      *