opal/memory: move initialization to first rcache creation

Because of the removal of the linux memory component it is no longer
necessary to initialize the memory component in opal_init(). This
commit moves the initialization to the creation of the first rcache
component.

Signed-off-by: Nathan Hjelm <hjelmn@lanl.gov>

opal/mca/btl/openib/btl_openib_component.c

@@ -2557,39 +2557,6 @@ btl_openib_component_init(int *num_btl_modules,
         goto no_btls;
     }
 
-    /* If we are using ptmalloc2 and there are no posix threads
-       available, this will cause memory corruption.  Refuse to run.
-       Right now, ptmalloc2 is the only memory manager that we have on
-       OS's that support OpenFabrics that provide both FREE and MUNMAP
-       support, so the following test is [currently] good enough... */
-    value = opal_mem_hooks_support_level();
-
-    /* If we have a memory manager available, and
-       opal_leave_pinned==-1, then unless the user explicitly set
-       opal_leave_pinned_pipeline==0, then set opal_leave_pinned to 1.
-
-       We have a memory manager if we have both FREE and MUNMAP
-       support */
-    if ((OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT) ==
-        ((OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT) & value)) {
-        if (0 == opal_leave_pinned_pipeline &&
-            -1 == opal_leave_pinned) {
-            opal_leave_pinned = 1;
-        }
-    } else {
-        opal_leave_pinned = 0;
-        opal_leave_pinned_pipeline = 0;
-    }
-
-#if OPAL_CUDA_SUPPORT
-    if (mca_btl_openib_component.cuda_want_gdr && (0 == opal_leave_pinned)) {
-        opal_show_help("help-mpi-btl-openib.txt",
-                       "CUDA_gdr_and_nopinned", true,
-                       opal_process_info.nodename);
-        goto no_btls;
-    }
-#endif /* OPAL_CUDA_SUPPORT */
-
     index = mca_base_var_find("ompi", "btl", "openib", "max_inline_data");
     if (index >= 0) {
         if (OPAL_SUCCESS == mca_base_var_get_value(index, NULL, &source, NULL)) {
@@ -2926,6 +2893,22 @@ btl_openib_component_init(int *num_btl_modules,
         mca_btl_openib_component.if_exclude_list = NULL;
     }
 
+    /* If we are using ptmalloc2 and there are no posix threads
+       available, this will cause memory corruption.  Refuse to run.
+       Right now, ptmalloc2 is the only memory manager that we have on
+       OS's that support OpenFabrics that provide both FREE and MUNMAP
+       support, so the following test is [currently] good enough... */
+    value = opal_mem_hooks_support_level();
+
+#if OPAL_CUDA_SUPPORT
+   if (mca_btl_openib_component.cuda_want_gdr && (0 == opal_leave_pinned)) {
+        opal_show_help("help-mpi-btl-openib.txt",
+                       "CUDA_gdr_and_nopinned", true,
+                       opal_process_info.nodename);
+        goto no_btls;
+    }
+#endif /* OPAL_CUDA_SUPPORT */
+
     mca_btl_openib_component.memory_registration_verbose = opal_output_open(NULL);
     opal_output_set_verbosity (mca_btl_openib_component.memory_registration_verbose,
                                mca_btl_openib_component.memory_registration_verbose_level);

opal/mca/btl/ugni/btl_ugni_component.c

@@ -331,26 +331,6 @@ btl_ugni_component_close(void)
     return OPAL_SUCCESS;
 }
 
-static void mca_btl_ugni_autoset_leave_pinned (void) {
-    if (MCA_BTL_UGNI_RCACHE_UDREG != mca_btl_ugni_component.rcache_type) {
-        int value = opal_mem_hooks_support_level();
-        if ((OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT) ==
-            ((OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT) & value)) {
-            /* Set leave pinned to 1 if leave pinned pipeline is not set */
-            if (-1 == opal_leave_pinned) {
-                opal_leave_pinned = !opal_leave_pinned_pipeline;
-            }
-        } else {
-            opal_leave_pinned = 0;
-            opal_leave_pinned_pipeline = 0;
-        }
-    } else if (-1 == opal_leave_pinned) {
-        /* if udreg is in use we can set leave pinned without checking for the
-         * memory hooks. */
-        opal_leave_pinned = !opal_leave_pinned_pipeline;
-    }
-}
-
 static mca_btl_base_module_t **
 mca_btl_ugni_component_init (int *num_btl_modules,
                              bool enable_progress_threads,
@@ -409,8 +389,6 @@ mca_btl_ugni_component_init (int *num_btl_modules,
         }
     }
 
-    mca_btl_ugni_autoset_leave_pinned ();
-
     mca_btl_ugni_module.super.btl_rdma_pipeline_send_length = mca_btl_ugni_module.super.btl_eager_limit;
 
     for (i = 0 ; i < mca_btl_ugni_component.ugni_num_btls ; ++i) {

opal/mca/rcache/base/base.h

@@ -29,6 +29,7 @@
 #include "opal/class/opal_list.h"
 #include "opal/mca/mca.h"
 #include "opal/mca/rcache/rcache.h"
+#include "opal/mca/memory/base/base.h"
 
 BEGIN_C_DECLS
 

opal/mca/rcache/base/rcache_base_create.c

@@ -43,6 +43,45 @@ mca_rcache_base_module_t* mca_rcache_base_module_create (const char* name, void
     mca_base_component_list_item_t *cli;
     mca_rcache_base_selected_module_t *sm;
 
+    /* on the very first creation of a module we init the memory
+       callback */
+    if (!mca_rcache_base_used_mem_hooks) {
+        /* Use the memory hooks if leave_pinned or
+         * leave_pinned_pipeline is enabled (note that either of these
+         * leave_pinned variables may have been set by a user MCA
+         * param or elsewhere in the code base).  Yes, we could havexc
+         * coded this more succinctly, but this is more clear. Do not
+         * check memory hooks if the rcache does not provide an
+         * range invalidation function.. */
+        if (opal_leave_pinned != 0 || opal_leave_pinned_pipeline) {
+            /* open the memory manager components.  Memory hooks may be
+               triggered before this (any time after mem_free_init(),
+               actually).  This is a hook available for memory manager hooks
+               without good initialization routine support */
+            (void) mca_base_framework_open (&opal_memory_base_framework, 0);
+        }
+
+        if (opal_leave_pinned != 0 || opal_leave_pinned_pipeline) {
+            if ((OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT) ==
+                ((OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT) &
+                 opal_mem_hooks_support_level())) {
+                if (-1 == opal_leave_pinned) {
+                    opal_leave_pinned = !opal_leave_pinned_pipeline;
+                }
+                opal_mem_hooks_register_release(mca_rcache_base_mem_cb, NULL);
+            } else {
+                opal_show_help("help-rcache-base.txt", "leave pinned failed",
+                               true, name, OPAL_NAME_PRINT(OPAL_PROC_MY_NAME),
+                               opal_proc_local_get()->proc_hostname);
+                return NULL;
+            }
+
+            /* Set this to true so that rcache_base_close knows to
+               cleanup */
+            mca_rcache_base_used_mem_hooks = 1;
+        }
+    }
+
     OPAL_LIST_FOREACH(cli, &opal_rcache_base_framework.framework_components, mca_base_component_list_item_t) {
          component = (mca_rcache_base_component_t *) cli->cli_component;
          if(0 == strcmp(component->rcache_version.mca_component_name, name)) {
@@ -61,35 +100,6 @@ mca_rcache_base_module_t* mca_rcache_base_module_create (const char* name, void
     sm->user_data = user_data;
     opal_list_append(&mca_rcache_base_modules, (opal_list_item_t*) sm);
 
-    /* on the very first creation of a module we init the memory
-       callback */
-    if (!mca_rcache_base_used_mem_hooks) {
-        /* Use the memory hooks if leave_pinned or
-         * leave_pinned_pipeline is enabled (note that either of these
-         * leave_pinned variables may have been set by a user MCA
-         * param or elsewhere in the code base).  Yes, we could havexc
-         * coded this more succinctly, but this is more clear. Do not
-         * check memory hooks if the rcache does not provide an
-         * range invalidation function.. */
-        if ((opal_leave_pinned > 0 || opal_leave_pinned_pipeline) &&
-            module->rcache_invalidate_range) {
-            if ((OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT) ==
-                ((OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT) &
-                 opal_mem_hooks_support_level())) {
-                opal_mem_hooks_register_release(mca_rcache_base_mem_cb, NULL);
-            } else {
-                opal_show_help("help-rcache-base.txt", "leave pinned failed",
-                               true, name, OPAL_NAME_PRINT(OPAL_PROC_MY_NAME),
-                               opal_proc_local_get()->proc_hostname);
-                return NULL;
-            }
-
-            /* Set this to true so that rcache_base_close knows to
-               cleanup */
-            mca_rcache_base_used_mem_hooks = 1;
-        }
-    }
-
     return module;
 }
 

opal/mca/rcache/base/rcache_base_frame.c

@@ -10,7 +10,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2012-2013 Los Alamos National Security, LLC.
+ * Copyright (c) 2012-2016 Los Alamos National Security, LLC.
  *                         All rights reserved
  * Copyright (c) 2015-2016 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
@@ -92,6 +92,13 @@ static int mca_rcache_base_close(void)
     /* deregister memory free callback */
     if (mca_rcache_base_used_mem_hooks) {
         opal_mem_hooks_unregister_release(mca_rcache_base_mem_cb);
+
+        /* close the memory manager components.  Registered hooks can
+           still be fired any time between now and the call to
+           opal_mem_free_finalize(), and callbacks from the memory manager
+           hooks to the bowels of the mem_free code can still occur any
+           time between now and end of application (even post main()!) */
+        (void) mca_base_framework_close (&opal_memory_base_framework);
     }
     /* All done */
 

opal/memoryhooks/memory.c

@@ -106,7 +106,7 @@ opal_mem_hooks_set_support(int support)
 void
 opal_mem_hooks_release_hook(void *buf, size_t length, bool from_alloc)
 {
-    opal_list_item_t *item;
+    callback_list_item_t *cbitem, *next;
 
     if (!release_run_callbacks) return;
 
@@ -121,12 +121,7 @@ opal_mem_hooks_release_hook(void *buf, size_t length, bool from_alloc)
      */
 
     opal_atomic_lock(&release_lock);
-    item = opal_list_get_first(&release_cb_list);
-    while(item != opal_list_get_end(&release_cb_list)) {
-        opal_list_item_t* next = opal_list_get_next(item);
-        callback_list_item_t *cbitem = (callback_list_item_t*) item;
-        item = next;
-
+    OPAL_LIST_FOREACH_SAFE(cbitem, next, &release_cb_list, callback_list_item_t) {
         opal_atomic_unlock(&release_lock);
         cbitem->cbfunc(buf, length, cbitem->cbdata, (bool) from_alloc);
         opal_atomic_lock(&release_lock);

opal/runtime/opal_finalize.c

@@ -40,7 +40,6 @@
 #include "opal/mca/installdirs/base/base.h"
 #include "opal/mca/memchecker/base/base.h"
 #include "opal/mca/memcpy/base/base.h"
-#include "opal/mca/memory/base/base.h"
 #include "opal/mca/patcher/base/base.h"
 #include "opal/mca/backtrace/base/base.h"
 #include "opal/mca/sec/base/base.h"
@@ -154,13 +153,6 @@ opal_finalize(void)
 
     (void) mca_base_framework_close(&opal_backtrace_base_framework);
     (void) mca_base_framework_close(&opal_memchecker_base_framework);
-
-    /* close the memory manager components.  Registered hooks can
-       still be fired any time between now and the call to
-       opal_mem_free_finalize(), and callbacks from the memory manager
-       hooks to the bowels of the mem_free code can still occur any
-       time between now and end of application (even post main()!) */
-    (void) mca_base_framework_close(&opal_memory_base_framework);
     (void) mca_base_framework_close(&opal_patcher_base_framework);
 
     /* close the memcpy framework */

opal/runtime/opal_init.c

@@ -439,15 +439,6 @@ opal_init(int* pargc, char*** pargv)
     /* select a patcher module. if a patcher module can not be found it is not an error. */
     (void) opal_patcher_base_select ();
 
-    /* open the memory manager components.  Memory hooks may be
-       triggered before this (any time after mem_free_init(),
-       actually).  This is a hook available for memory manager hooks
-       without good initialization routine support */
-    if (OPAL_SUCCESS != (ret = mca_base_framework_open(&opal_memory_base_framework, 0))) {
-        error = "opal_memory_base_open";
-        goto return_error;
-    }
-
     /* initialize the memory manager / tracker */
     if (OPAL_SUCCESS != (ret = opal_mem_hooks_init())) {
         error = "opal_mem_hooks_init";