Fixes trac:1379:

* Use synonym/deprecated MCA param API for some mca base params
 * In openib BTL, if we have appropriate memory hooks support, and if
   mpi_leave_pinned and mpi_leave_pinned_pipeline were not set by the
   user, set mpi_leave_pinned to 1.
 * Defer checking mpi_leave_pinned_* until as late as possible (i.e.,
   until after the btl's have had a chance to set mpi_leave_pinned to
   1):
   * in ob1 pml
   * in rdma mpool

This commit was SVN r19022.

The following Trac tickets were found above:
  Ticket 1379 --> https://svn.open-mpi.org/trac/ompi/ticket/1379

ompi/mca/btl/openib/btl_openib_component.c

@@ -31,16 +31,17 @@
 #include <unistd.h>
 #include <sys/types.h>
 #include <sys/stat.h>
+#include <malloc.h>
 
 #include "ompi/constants.h"
 #include "opal/event/event.h"
 #include "opal/include/opal/align.h"
 #include "opal/util/if.h"
 #include "opal/util/argv.h"
-#include "orte/util/show_help.h"
 #include "opal/sys/timer.h"
 #include "opal/sys/atomic.h"
 #include "opal/util/argv.h"
+#include "opal/memoryhooks/memory.h"
 #include "opal/mca/base/mca_base_param.h"
 #include "opal/mca/carto/carto.h"
 #include "opal/mca/carto/base/base.h"
@@ -48,6 +49,7 @@
 #include "opal/mca/installdirs/installdirs.h"
 #include "opal_stdint.h"
 
+#include "orte/util/show_help.h"
 #include "orte/mca/errmgr/errmgr.h"
 #include "orte/util/proc_info.h"
 #include "orte/runtime/orte_globals.h"
@@ -1926,6 +1928,8 @@ btl_openib_component_init(int *num_btl_modules,
     mca_btl_openib_frag_init_data_t *init_data;
     struct dev_distance *dev_sorted;
     int distance;
+    int index, value;
+    mca_base_param_source_t source;
 
     /* initialization */
     *num_btl_modules = 0;
@@ -1961,6 +1965,44 @@ btl_openib_component_init(int *num_btl_modules,
         goto no_btls;
     }
 
+    /* If we have a memory manager available, unless the user
+       explicitly set mpi_leave_pinned==0 or
+       mpi_leave_pinned_pipeline==0, then set mpi_leave_pinned to 1.
+
+       We have a memory manager if:
+       - we have both FREE and MUNMAP support
+       - we have MUNMAP support and the linux mallopt */
+    value = opal_mem_hooks_support_level();
+    if (((OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT) == 
+         ((OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT) & value)) ||
+        (0 != (OPAL_MEMORY_MUNMAP_SUPPORT & value) &&
+         OMPI_MPOOL_BASE_HAVE_LINUX_MALLOPT)) {
+        ret = 0;
+        index = mca_base_param_find("mpi", NULL, "leave_pinned");
+        if (index >= 0) {
+            if (OPAL_SUCCESS == mca_base_param_lookup_int(index, &value) &&
+                OPAL_SUCCESS == mca_base_param_lookup_source(index, &source)) {
+                if (0 == value && MCA_BASE_PARAM_SOURCE_DEFAULT == source) {
+                    ++ret;
+                }
+            }
+        }
+        index = mca_base_param_find("mpi", NULL, "leave_pinned_pipeline");
+        if (index >= 0) {
+            if (OPAL_SUCCESS == mca_base_param_lookup_int(index, &value) &&
+                OPAL_SUCCESS == mca_base_param_lookup_source(index, &source)) {
+                if (0 == value && MCA_BASE_PARAM_SOURCE_DEFAULT == source) {
+                    ++ret;
+                }
+            }
+        }
+        /* If we were good on both parameters, then set leave_pinned=1 */
+        if (2 == ret) {
+            ompi_mpi_leave_pinned = 1;
+            ompi_mpi_leave_pinned_pipeline = 0;
+        }
+    }
+
     OBJ_CONSTRUCT(&mca_btl_openib_component.send_free_coalesced, ompi_free_list_t);
     OBJ_CONSTRUCT(&mca_btl_openib_component.send_user_free, ompi_free_list_t);
     OBJ_CONSTRUCT(&mca_btl_openib_component.recv_user_free, ompi_free_list_t);

ompi/mca/mpool/base/base.h

@@ -9,6 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
+ * Copyright (c) 2008      Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
  * 
  * Additional copyrights may follow
@@ -95,15 +96,12 @@ OMPI_DECLSPEC extern uint32_t mca_mpool_base_page_size;
 OMPI_DECLSPEC extern uint32_t mca_mpool_base_page_size_log;
 
 /* only used within base -- no need to DECLSPEC */
-extern int mca_mpool_base_use_mem_hooks; 
-extern int mca_mpool_base_mallopt_disable_free;
-
+extern int mca_mpool_base_used_mem_hooks;
+extern int mca_mpool_base_use_mem_hooks_index;
+extern int mca_mpool_base_disable_mallopt_index;
     
 #if defined(c_plusplus) || defined(__cplusplus)
 }
 #endif
 
 #endif /* MCA_MEM_BASE_H */
-
-
-

ompi/mca/mpool/base/mpool_base_close.c

@@ -10,6 +10,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
+ * Copyright (c) 2008      Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
  * 
  * Additional copyrights may follow
@@ -66,7 +67,7 @@ int mca_mpool_base_close(void)
                             &mca_mpool_base_components, NULL);
 
   /* deregister memory free callback */
-  if( (modules_length > 0) && mca_mpool_base_use_mem_hooks && 
+  if( (modules_length > 0) && mca_mpool_base_used_mem_hooks && 
      0 != (OPAL_MEMORY_FREE_SUPPORT & opal_mem_hooks_support_level())) {
       opal_mem_hooks_unregister_release(mca_mpool_base_mem_cb);
       OBJ_DESTRUCT(&mca_mpool_base_mem_cb_array);

ompi/mca/mpool/base/mpool_base_lookup.c

@@ -11,6 +11,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2006-2007 Mellanox Technologies. All rights reserved.
+ * Copyright (c) 2008      Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
  * 
  * Additional copyrights may follow
@@ -35,6 +36,7 @@
 #include "orte/util/proc_info.h"
 #include "orte/runtime/orte_globals.h"
 #include "ompi/mca/mpool/mpool.h"
+#include "ompi/runtime/params.h"
 #include "ompi/mca/mpool/base/base.h"
 #include "mpool_base_mem_cb.h"
 
@@ -65,11 +67,11 @@ mca_mpool_base_module_t* mca_mpool_base_module_create(
     void* user_data,
     struct mca_mpool_base_resources_t* resources) 
 {
-    
     mca_mpool_base_component_t* component = NULL; 
     mca_mpool_base_module_t* module = NULL; 
     opal_list_item_t* item;
     mca_mpool_base_selected_module_t *sm;
+    int use_mem_hooks, disable_mallopt;
 
     for (item = opal_list_get_first(&mca_mpool_base_components);
          item != opal_list_get_end(&mca_mpool_base_components);
@@ -102,11 +104,27 @@ mca_mpool_base_module_t* mca_mpool_base_module_create(
        still need to register a callback to handle the case of the
        user calling mmap/munmap on his own. */ 
     if (opal_list_get_size(&mca_mpool_base_modules) == 1) { 
-        if (mca_mpool_base_use_mem_hooks) {
+
+        /* Lookup the current value of the MCA params and see if any
+           other entity in the code base requested mem hooks */
+        mca_base_param_lookup_int(mca_mpool_base_use_mem_hooks_index,
+                                  &use_mem_hooks);
+        mca_base_param_lookup_int(mca_mpool_base_disable_mallopt_index,
+                                  &disable_mallopt);
+
+        /* force mem hooks if leave_pinned or leave_pinned_pipeline is
+           enabled (note that either of these leave_pinned variables
+           may have been set by a user MCA param or elsewhere in the
+           code base) */
+        if (ompi_mpi_leave_pinned || ompi_mpi_leave_pinned_pipeline) {
+            use_mem_hooks = 1;
+        }
+
+        if (use_mem_hooks) {
             if (0 != (OPAL_MEMORY_FREE_SUPPORT & opal_mem_hooks_support_level())) {
                 opal_mem_hooks_register_release(mca_mpool_base_mem_cb, NULL);
                 OBJ_CONSTRUCT(&mca_mpool_base_mem_cb_array, opal_pointer_array_t);
-            } else if (mca_mpool_base_mallopt_disable_free &&
+            } else if (!disable_mallopt &&
                        0 != (OPAL_MEMORY_MUNMAP_SUPPORT & opal_mem_hooks_support_level())) {
                 opal_mem_hooks_register_release(mca_mpool_base_mem_cb, NULL);
                 OBJ_CONSTRUCT(&mca_mpool_base_mem_cb_array, opal_pointer_array_t);
@@ -122,6 +140,10 @@ mca_mpool_base_module_t* mca_mpool_base_module_create(
                                orte_process_info.nodename);
                 return NULL;
             }
+
+            /* Set this to true so that mpool_base_close knows to
+               cleanup */
+            mca_mpool_base_used_mem_hooks = 1;
         }
     }
     return module; 

ompi/mca/mpool/base/mpool_base_open.c

@@ -9,7 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2007-2008 Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
  * 
  * Additional copyrights may follow
@@ -31,7 +31,6 @@
 #include "opal/mca/mca.h"
 #include "opal/mca/base/base.h"
 #include "opal/memoryhooks/memory.h"
-#include "ompi/runtime/params.h"
 #include "ompi/mca/mpool/mpool.h"
 #include "ompi/mca/mpool/base/base.h"
 #include "ompi/constants.h"
@@ -49,11 +48,13 @@
  */
 int mca_mpool_base_output = -1;
 
+/* whether we actually used the mem hooks or not */
+int mca_mpool_base_used_mem_hooks = 0;
 /* should we attempt to use the available memory hooks */
-int mca_mpool_base_use_mem_hooks = 0; 
+int mca_mpool_base_use_mem_hooks_index;
 /* should we attempt to use mallopt to disable free() returning memory
    to OS? */
-int mca_mpool_base_mallopt_disable_free = 0;
+int mca_mpool_base_disable_mallopt_index;
 
 uint32_t mca_mpool_base_page_size; 
 uint32_t mca_mpool_base_page_size_log;
@@ -70,8 +71,7 @@ int mca_mpool_base_open(void)
     /* Open up all available components - and populate the
        mca_mpool_base_components list */
     
-    int use_mem_hooks;
-    int no_mallopt;
+    int i;
     
     if (OMPI_SUCCESS != 
         mca_base_components_open("mpool", 0, mca_mpool_base_static_components, 
@@ -88,53 +88,32 @@ int mca_mpool_base_open(void)
      * check for use_mem_hooks (for diagnostics/testing) 
      * however if leave_pinned is set we force this to be enabled
      */
-    mca_base_param_reg_int_name("mpool", 
-                                "base_use_mem_hooks", 
-                                "use memory hooks for deregistering freed memory",
-                                false, 
-                                false, 
-                                0,
-                                &mca_mpool_base_use_mem_hooks); 
-    
-    mca_base_param_reg_int_name("mpool", 
-                                "use_mem_hooks", 
-                                "(deprecated, use mpool_base_use_mem_hooks)",
-                                false, 
-                                false, 
-                                0,
-                                &use_mem_hooks); 
-    
-    mca_mpool_base_use_mem_hooks = use_mem_hooks || mca_mpool_base_use_mem_hooks;
+    mca_mpool_base_use_mem_hooks_index = 
+        mca_base_param_reg_int_name("mpool", 
+                                    "base_use_mem_hooks", 
+                                    "Use memory hooks for deregistering freed memory",
+                                    false, 
+                                    false, 
+                                    0,
+                                    NULL);
+    mca_base_param_reg_syn_name(i, "mpool", "use_mem_hooks", true);
 
-    
+    mca_mpool_base_disable_mallopt_index =
+        mca_base_param_reg_int_name("mpool", 
+                                    "base_disable_mallopt",
+                                    "Do not use mallopt to disable returning memory to "
+                                    "the OS when leave_pinned is active and no memory "
+                                    "components are found (this value is only changable on Linux systems that support mallopt()).",
+                                    false, 
 #if OMPI_MPOOL_BASE_HAVE_LINUX_MALLOPT
-    mca_base_param_reg_int_name("mpool", 
-                                "base_disable_mallopt",
-                                "do not use mallopt to disable returning memory to "
-                                "the OS when leave_pinned is active and no memory "
-                                "components are found.",
-                                false, 
-                                false, 
-                                0,
-                                &no_mallopt);
+                                    false, 
+                                    0,
 #else
-    no_mallopt = 1;
+                                    true,
+                                    1,
 #endif
+                                    NULL);
 
-    /* force mem hooks if leave_pinned or leave_pinned_pipeline is enabled */
-    if (ompi_mpi_leave_pinned || ompi_mpi_leave_pinned_pipeline) {
-        mca_mpool_base_use_mem_hooks = 1;
-    }
-
-    /* enable mallopt if we're using leave pinned, there is support
-       for intercepting munmap, and the user didn't tell us not to use
-       mallopt */
-    if ((ompi_mpi_leave_pinned || ompi_mpi_leave_pinned_pipeline) &&
-        (0 != (OPAL_MEMORY_MUNMAP_SUPPORT & opal_mem_hooks_support_level())) &&
-        0 == no_mallopt) {
-        mca_mpool_base_mallopt_disable_free = 1;        
-    }
-    
     /* get the page size for this architecture*/ 
     mca_mpool_base_page_size = sysconf(_SC_PAGESIZE); 
     mca_mpool_base_page_size_log = my_log2(mca_mpool_base_page_size); 

ompi/mca/mpool/rdma/mpool_rdma_component.c

@@ -10,7 +10,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2006      Voltaire. All rights reserved.
- * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2007-2008 Cisco Systems, Inc.  All rights reserved.
  *
  * $COPYRIGHT$
  *
@@ -26,7 +26,6 @@
 #include "opal/mca/base/mca_base_param.h"
 #include "mpool_rdma.h"
 #include "orte/util/proc_info.h"
-#include "ompi/runtime/params.h"
 #ifdef HAVE_UNISTD_H
 #include <unistd.h>
 #endif
@@ -97,9 +96,6 @@ static int mca_mpool_rdma_open(void)
 
     mca_mpool_rdma_component.print_stats = val?true:false;
 
-    mca_mpool_rdma_component.leave_pinned = (int) 
-        (ompi_mpi_leave_pinned || ompi_mpi_leave_pinned_pipeline);
-
     return OMPI_SUCCESS;
 }
 

ompi/mca/mpool/rdma/mpool_rdma_module.c

@@ -10,7 +10,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2006-2008 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2006      Voltaire. All rights reserved.
  * Copyright (c) 2007      Mellanox Technologies. All rights reserved.
  *
@@ -37,6 +37,7 @@
 #include "ompi/mca/rcache/rcache.h"
 #include "ompi/mca/rcache/base/base.h"
 #include "ompi/mca/mpool/base/base.h"
+#include "ompi/runtime/params.h"
 
 /*
  *  Initializes the mpool module.
@@ -67,6 +68,12 @@ void mca_mpool_rdma_module_init(mca_mpool_rdma_module_t* mpool)
     OBJ_CONSTRUCT(&mpool->gc_list, opal_list_t);
     mpool->stat_cache_hit = mpool->stat_cache_miss = mpool->stat_evicted = 0;
     mpool->stat_cache_found = mpool->stat_cache_notfound = 0;
+
+    /* Set this here (vs in component.c) because
+       ompi_mpi_leave_pinned* may have been set after MCA params were
+       read (e.g., by the openib btl) */
+    mca_mpool_rdma_component.leave_pinned = (int) 
+        (ompi_mpi_leave_pinned || ompi_mpi_leave_pinned_pipeline);
 }
 
 static inline int dereg_mem(mca_mpool_base_module_t *mpool,

ompi/mca/pml/ob1/pml_ob1_component.c

@@ -9,7 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2007-2008 Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
  * 
  * Additional copyrights may follow
@@ -141,12 +141,8 @@ static int mca_pml_ob1_component_open(void)
         return OMPI_ERROR;
     }
 
-    mca_pml_ob1.leave_pinned = ompi_mpi_leave_pinned;
-    mca_pml_ob1.leave_pinned_pipeline = (int) ompi_mpi_leave_pinned_pipeline;
-
     mca_pml_ob1.enabled = false; 
     return mca_bml_base_open(); 
-    
 }
 
 
@@ -180,6 +176,12 @@ mca_pml_ob1_component_init( int* priority,
         return NULL;
     }
 
+    /* Set this here (vs in component_open()) because
+       ompi_mpi_leave_pinned* may have been set after MCA params were
+       read (e.g., by the openib btl) */
+    mca_pml_ob1.leave_pinned = ompi_mpi_leave_pinned;
+    mca_pml_ob1.leave_pinned_pipeline = (int) ompi_mpi_leave_pinned_pipeline;
+
     /* As our own progress function does nothing except calling the BML
      * progress, let's modify the progress function pointer in our structure
      * to avoid useless functions calls. The event library will instead call