The mpi_leave_pinned and mpi_leave_pinned_pipeline MCA parameters were

needlessly registered in multiple different places, and none of them
had a good help string.  There was also an inconsistent check for
setting both mpi_leave_pinned and mpi_leave_pinned_pipeline (i.e., it
was only in ob1).  This commit moves the registration of these params
to one central place (ompi/runtime/ompi_mpi_params.c, with all other
mpi_* MCA params) and uses globals to propagate the values as
relevant.  The error check was also moved to the central location to
ensure that we can consistency everywhere.

This commit was SVN r13226.

ompi/mca/mpool/base/mpool_base_open.c

@@ -9,6 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
+ * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
  * 
  * Additional copyrights may follow
@@ -22,6 +23,7 @@
 
 #include "opal/mca/mca.h"
 #include "opal/mca/base/base.h"
+#include "ompi/runtime/params.h"
 #include "ompi/mca/mpool/mpool.h"
 #include "ompi/mca/mpool/base/base.h"
 #include "ompi/constants.h"
@@ -128,20 +130,10 @@ int mca_mpool_base_open(void)
 #else 
     if(0 == mca_mpool_base_use_mem_hooks ) {
 #endif
-         int param;
+        if (ompi_mpi_leave_pinned || ompi_mpi_leave_pinned_pipeline) {
-         mca_base_param_register_int("mpi", NULL, "leave_pinned", "leave_pinned", 0);
+            mca_mpool_base_use_mem_hooks = 1;
-         param = mca_base_param_find("mpi", NULL, "leave_pinned");
+        }
-         mca_base_param_lookup_int(param, &mca_mpool_base_use_mem_hooks);
+    }
-         
-         if(0 == mca_mpool_base_use_mem_hooks) { 
-             /* and now check leave_pinned_pipeline if necessary */
-             mca_base_param_register_int("mpi", NULL, 
-                                         "leave_pinned_pipeline", 
-                                         "leave_pinned_pipeline", 0); 
-             param = mca_base_param_find("mpi", NULL, "leave_pinned_pipeline");
-             mca_base_param_lookup_int(param, &mca_mpool_base_use_mem_hooks);
-         }
-    }  
     
     /* get the page size for this architecture*/ 
     mca_mpool_base_page_size = sysconf(_SC_PAGESIZE); 

ompi/mca/mpool/rdma/mpool_rdma_component.c

@@ -10,6 +10,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2006      Voltaire. All rights reserved.
+ * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
  *
  * $COPYRIGHT$
  *
@@ -25,6 +26,7 @@
 #include "mpool_rdma.h"
 #include "orte/util/proc_info.h"
 #include "orte/util/sys_info.h"
+#include "ompi/runtime/params.h"
 #include <unistd.h>
 #ifdef HAVE_MALLOC_H
 #include <malloc.h>
@@ -72,7 +74,7 @@ mca_mpool_rdma_component_t mca_mpool_rdma_component = {
   */
 static int mca_mpool_rdma_open(void)
 {
-    int param, val;
+    int val;
 
     mca_base_param_reg_string(&mca_mpool_rdma_component.super.mpool_version,
             "rcache_name",
@@ -93,17 +95,8 @@ static int mca_mpool_rdma_open(void)
 
     mca_mpool_rdma_component.print_stats = val?true:false;
 
-    mca_base_param_register_int("mpi", NULL, "leave_pinned", "leave_pinned", 0);
+    mca_mpool_rdma_component.leave_pinned = (int) 
-    param = mca_base_param_find("mpi", NULL, "leave_pinned");
+        (ompi_mpi_leave_pinned || ompi_mpi_leave_pinned_pipeline);
-    mca_base_param_lookup_int(param, (int*)&mca_mpool_rdma_component.leave_pinned);
-
-    if(0 == mca_mpool_rdma_component.leave_pinned) {
-        /* and now check leave_pinned_pipeline if necessary */
-        mca_base_param_register_int("mpi", NULL, "leave_pinned_pipeline",
-                "leave_pinned_pipeline", 0);
-        param = mca_base_param_find("mpi", NULL, "leave_pinned_pipeline");
-        mca_base_param_lookup_int(param, (int*)&mca_mpool_rdma_component.leave_pinned);
-    }
 
     return OMPI_SUCCESS;
 }

ompi/mca/pml/ob1/pml_ob1_component.c

@@ -9,6 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
+ * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
  * 
  * Additional copyrights may follow
@@ -20,6 +21,7 @@
 #include "opal/sys/cache.h"
 #include "opal/event/event.h"
 #include "mpi.h"
+#include "ompi/runtime/params.h"
 #include "ompi/mca/pml/pml.h"
 #include "ompi/mca/btl/btl.h"
 #include "ompi/mca/btl/base/base.h"
@@ -86,8 +88,6 @@ static inline int mca_pml_ob1_param_register_int(
 
 int mca_pml_ob1_component_open(void)
 {
-    int param, value; 
-
     mca_pml_ob1.free_list_num =
         mca_pml_ob1_param_register_int("free_list_num", 4);
     mca_pml_ob1.free_list_max =
@@ -166,21 +166,10 @@ int mca_pml_ob1_component_open(void)
     OBJ_CONSTRUCT(&mca_pml_ob1.recv_pending, opal_list_t);
     OBJ_CONSTRUCT(&mca_pml_ob1.pckt_pending, opal_list_t);
     OBJ_CONSTRUCT(&mca_pml_ob1.rdma_pending, opal_list_t);
-    
-    mca_base_param_register_int("mpi", NULL, "leave_pinned", "leave_pinned", 0); 
-    param = mca_base_param_find("mpi", NULL, "leave_pinned"); 
-    mca_base_param_lookup_int(param, &value); 
-    mca_pml_ob1.leave_pinned = OPAL_INT_TO_BOOL(value); 
-    
-    mca_base_param_register_int("mpi", NULL, "leave_pinned_pipeline", "leave_pinned_pipeline", 0); 
-    param = mca_base_param_find("mpi", NULL, "leave_pinned_pipeline"); 
-    mca_base_param_lookup_int(param, &value); 
-    mca_pml_ob1.leave_pinned_pipeline = value; 
 
-    if(mca_pml_ob1.leave_pinned_pipeline && mca_pml_ob1.leave_pinned) { 
+    mca_pml_ob1.leave_pinned = ompi_mpi_leave_pinned;
-        mca_pml_ob1.leave_pinned_pipeline = 0;
+    mca_pml_ob1.leave_pinned_pipeline = (int) ompi_mpi_leave_pinned_pipeline;
-        opal_output(0, "WARNING: Cannot set both mpi_leave_pinned and mpi_leave_pinned_pipeline, defaulting to mpi_leave_pinned ONLY\n");
+
-    }
     mca_pml_ob1.enabled = false; 
     return mca_bml_base_open(); 
     

ompi/runtime/help-mpi-runtime.txt

@@ -10,6 +10,7 @@
 #                         University of Stuttgart.  All rights reserved.
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
+# Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
 # $COPYRIGHT$
 # 
 # Additional copyrights may follow
@@ -32,6 +33,10 @@ developer):
 WARNING: The MCA parameter mpi_param_check has been set to true, but
 parameter checking has been compiled out of Open MPI.  The
 mpi_param_check value has therefore been ignored.
+[mpi-params:leave-pinned-and-pipeline-selected]
+WARNING: Cannot set both the MCA parameters mpi_leave_pinned and
+mpi_leave_pinned_pipeline to "true".  Defaulting to mpi_leave_pinned
+ONLY.
 [mpi_init:startup:paffinity-unavailable]
 The MCA parameter "mpi_paffinity_alone" was set to a nonzero value,
 but Open MPI was unable to bind MPI_COMM_WORLD rank %s to a processor.

ompi/runtime/ompi_mpi_params.c

@@ -50,6 +50,8 @@ bool ompi_mpi_abort_print_stack = false;
 int ompi_mpi_abort_delay = 0;
 bool ompi_mpi_keep_peer_hostnames = true;
 bool ompi_mpi_preconnect_all = false;
+bool ompi_mpi_leave_pinned = false;
+bool ompi_mpi_leave_pinned_pipeline = false;
 
 
 int ompi_mpi_register_params(void)
@@ -187,7 +189,28 @@ int ompi_mpi_register_params(void)
                                 (int) ompi_mpi_preconnect_all, &value);
     
     ompi_mpi_preconnect_all = OPAL_INT_TO_BOOL(value); 
+
+    /* Leave pinned parameter */
+
+    mca_base_param_reg_int_name("mpi", "leave_pinned",
+                                "Whether to use the \"leave pinned\" protocol or not.  Enabling this setting can help bandwidth performance when repeatedly sending and receiving large messages with the same buffers over RDMA-based networks.",
+                                false, false,
+                                (int) ompi_mpi_leave_pinned, &value);
+    ompi_mpi_leave_pinned = OPAL_INT_TO_BOOL(value);
+
+    mca_base_param_reg_int_name("mpi", "leave_pinned_pipeline",
+                                "Whether to use the \"leave pinned pipeline\" protocol or not.",
+                                false, false,
+                                (int) ompi_mpi_leave_pinned_pipeline, &value);
+    ompi_mpi_leave_pinned_pipeline = OPAL_INT_TO_BOOL(value);
     
+    if (ompi_mpi_leave_pinned && ompi_mpi_leave_pinned_pipeline) {
+        ompi_mpi_leave_pinned_pipeline = 0;
+        opal_show_help("help-mpi-runtime.txt", 
+                       "mpi-params:leave-pinned-and-pipeline-selected",
+                       true);
+    }
+
     /* The ddt engine has a few parameters */
 
     return ompi_ddt_register_params();

ompi/runtime/params.h

@@ -9,7 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2006-2007 Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
  * 
  * Additional copyrights may follow
@@ -118,6 +118,16 @@ OMPI_DECLSPEC extern bool ompi_mpi_paffinity_alone;
      */
     OMPI_DECLSPEC extern int ompi_mpi_abort_delay;
 
+    /**
+     * Whether to use the "leave pinned" protocol or not.
+     */
+    OMPI_DECLSPEC extern bool ompi_mpi_leave_pinned;
+
+    /**
+     * Whether to use the "leave pinned pipeline" protocol or not.
+     */
+    OMPI_DECLSPEC extern bool ompi_mpi_leave_pinned_pipeline;
+
    /**
      * Register MCA parameters used by the MPI layer.
      *