opal/memory: update component structure

This commit makes it possible to set relative priorities for
components. Before the addition of the patched component there was
only one component that would run on any system but that is no longer
the case. When determining which component to open each component's
query function is called and the one that returns the highest priority
is opened. The default priority of the patcher component is set
slightly higher than the old ptmalloc2/ummunotify component.

This commit fixes a long-standing break in the abstration of the
memory components. ompi_mpi_init.c was referencing the linux malloc
hook initilize function to ensure the hooks are initialized for
libmpi.so. The abstraction break has been fixed by adding a memory
base function that calls the open memory component's malloc hook init
function if it has one. The code is not yet complete but is intended
to support ptmalloc in 2.0.0. In that case the base function will
always call the ptmalloc hook init if exists.

Signed-off-by: Nathan Hjelm <hjelmn@lanl.gov>

ompi/runtime/ompi_mpi_init.c

@@ -98,15 +98,13 @@
 #endif
 #include "ompi/runtime/ompi_cr.h"
 
-#if defined(MEMORY_LINUX_PTMALLOC2) && MEMORY_LINUX_PTMALLOC2
-#include "opal/mca/memory/linux/memory_linux.h"
+#include "opal/mca/memory/base/base.h"
 /* So this sucks, but with OPAL in its own library that is brought in
    implicity from libmpi, there are times when the malloc initialize
    hook in the memory component doesn't work.  So we have to do it
    from here, since any MPI code is going to call MPI_Init... */
 OPAL_DECLSPEC void (*__malloc_initialize_hook) (void) =
-    opal_memory_linux_malloc_init_hook;
-#endif /* defined(MEMORY_LINUX_PTMALLOC2) && MEMORY_LINUX_PTMALLOC2 */
+    opal_memory_base_malloc_init_hook;
 
 /* This is required for the boundaries of the hash tables used to store
  * the F90 types returned by the MPI_Type_create_f90_XXX functions.

opal/mca/memory/base/base.h

@@ -32,5 +32,7 @@ BEGIN_C_DECLS
  */
 OPAL_DECLSPEC extern mca_base_framework_t opal_memory_base_framework;
 
+OPAL_DECLSPEC void opal_memory_base_malloc_init_hook (void);
+
 END_C_DECLS
 #endif /* OPAL_BASE_MEMORY_H */

opal/mca/memory/base/memory_base_open.c

@@ -13,6 +13,8 @@
  * Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2016      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016      Los Alamos National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -43,20 +45,22 @@ static int empty_process(void)
     return OPAL_SUCCESS;
 }
 
+static int empty_query (int *priority)
+{
+    *priority = 0;
+    return OPAL_SUCCESS;
+}
 
 /*
  * Local variables
  */
 static opal_memory_base_component_2_0_0_t empty_component = {
-    /* Don't care about the version info */
-    { 0, },
-    /* Don't care about the data */
-    { 0, },
     /* Empty / safe functions to call if no memory componet is selected */
-    empty_process,
-    opal_memory_base_component_register_empty,
-    opal_memory_base_component_deregister_empty,
-    opal_memory_base_component_set_alignment_empty,
+    .memoryc_query = empty_query,
+    .memoryc_process = empty_process,
+    .memoryc_register = opal_memory_base_component_register_empty,
+    .memoryc_deregister = opal_memory_base_component_deregister_empty,
+    .memoryc_set_alignment = opal_memory_base_component_set_alignment_empty,
 };
 
 
@@ -66,6 +70,12 @@ static opal_memory_base_component_2_0_0_t empty_component = {
 opal_memory_base_component_2_0_0_t *opal_memory = &empty_component;
 
 
+void opal_memory_base_malloc_init_hook (void)
+{
+    if (opal_memory->memoryc_init_hook) {
+        opal_memory->memoryc_init_hook ();
+    }
+}
 
 /*
  * Function for finding and opening either all MCA components, or the one
@@ -73,23 +83,36 @@ opal_memory_base_component_2_0_0_t *opal_memory = &empty_component;
  */
 static int opal_memory_base_open(mca_base_open_flag_t flags)
 {
+    mca_base_component_list_item_t *item, *next;
+    opal_memory_base_component_2_0_0_t *tmp;
+    int priority, highest_priority = 0;
     int ret;
 
-    /* Open up all available components */
+    /* can only be zero or one */
+    OPAL_LIST_FOREACH(item, &opal_memory_base_framework.framework_components, mca_base_component_list_item_t) {
+        tmp = (opal_memory_base_component_2_0_0_t *) item->cli_component;
+        ret = tmp->memoryc_query (&priority);
+        if (OPAL_SUCCESS != ret || priority < highest_priority) {
+            continue;
+        }
+
+        highest_priority = priority;
+        opal_memory = tmp;
+    }
+
+    OPAL_LIST_FOREACH_SAFE(item, next, &opal_memory_base_framework.framework_components, mca_base_component_list_item_t) {
+        if ((void *) opal_memory != (void *) item->cli_component) {
+            mca_base_component_unload (item->cli_component, opal_memory_base_framework.framework_output);
+            opal_list_remove_item (&opal_memory_base_framework.framework_components, &item->super);
+        }
+    }
+
+    /* open remaining component */
     ret = mca_base_framework_components_open (&opal_memory_base_framework, flags);
     if (ret != OPAL_SUCCESS) {
         return ret;
     }
 
-    /* can only be zero or one */
-    if (opal_list_get_size(&opal_memory_base_framework.framework_components) == 1) {
-        mca_base_component_list_item_t *item;
-        item = (mca_base_component_list_item_t*)
-            opal_list_get_first(&opal_memory_base_framework.framework_components);
-        opal_memory = (opal_memory_base_component_2_0_0_t*)
-            item->cli_component;
-    }
-
     /* All done */
     return OPAL_SUCCESS;
 }

opal/mca/memory/linux/hooks.c

@@ -744,6 +744,10 @@ void opal_memory_linux_malloc_init_hook(void)
     check_result_t r1, lp, lpp;
     bool want_rcache = false, found_driver = false;
 
+    if (!opal_memory_linux_opened) {
+        return;
+    }
+
     /* First, check for a FAKEROOT environment.  If we're in a
        fakeroot, then access() (and likely others) have been replaced
        and are not safe to call here in this pre-main environment.  So

opal/mca/memory/linux/memory_linux.h

@@ -83,6 +83,8 @@ OPAL_DECLSPEC void opal_memory_linux_malloc_init_hook(void);
 OPAL_DECLSPEC void opal_memory_linux_malloc_set_alignment(int use_memalign, size_t memalign_threshold);
 #endif /* MEMORY_LINUX_MALLOC_ALIGN_ENABLED */
 
+extern bool opal_memory_linux_opened;
+
 END_C_DECLS
 
 #endif

opal/mca/memory/linux/memory_linux_component.c

@@ -11,7 +11,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2009-2014 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2013-2015 Los Alamos National Security, LLC. All rights
+ * Copyright (c) 2013-2016 Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2016      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
@@ -60,6 +60,7 @@
 static int linux_open(void);
 static int linux_close(void);
 static int linux_register(void);
+static int linux_query(int *);
 
 #if MEMORY_LINUX_UMMUNOTIFY
 static bool ummunotify_opened = false;
@@ -69,6 +70,9 @@ static bool ptmalloc2_opened = false;
 #endif
 
 bool opal_memory_linux_disable = false;
+static int mca_memory_linux_priority;
+
+bool opal_memory_linux_opened = false;
 
 opal_memory_linux_component_t mca_memory_linux_component = {
     /* First, the opal_memory_base_component_2_0_0_t */
@@ -96,6 +100,8 @@ opal_memory_linux_component_t mca_memory_linux_component = {
         /* Memory framework functions.  These function pointer values
            are replaced by memory_linux_ummunotify.c at run time if we
            end up using ummunotify support. */
+        .memoryc_init_hook = opal_memory_linux_malloc_init_hook,
+        .memoryc_query = linux_query,
         .memoryc_register = opal_memory_base_component_register_empty,
         .memoryc_deregister = opal_memory_base_component_deregister_empty,
 #if MEMORY_LINUX_MALLOC_ALIGN_ENABLED
@@ -243,11 +249,25 @@ static int linux_register(void)
     if (0 > ret) {
         return ret;
     }
+
+    mca_memory_linux_priority = 50;
+    ret = mca_base_component_var_register (&mca_memory_linux_component.super.memoryc_version,
+                                           "priority", "Priority of the linux memory hook component",
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_5,
+                                           MCA_BASE_VAR_SCOPE_CONSTANT, &mca_memory_linux_priority);
+    if (0 > ret) {
+        return ret;
+    }
 #endif /* MEMORY_LINUX_MALLOC_ALIGN_ENABLED */
 
     return (0 > ret) ? ret : OPAL_SUCCESS;
 }
 
+static int linux_query (int *priority)
+{
+    *priority = mca_memory_linux_priority;
+    return OPAL_SUCCESS;
+}
 
 static int linux_open(void)
 {
@@ -318,6 +338,7 @@ done:
         __malloc_hook = _opal_memory_linux_malloc_align_hook;
     }
 #endif /* MEMORY_LINUX_MALLOC_ALIGN_ENABLED */
+    opal_memory_linux_opened = true;
 
     return OPAL_SUCCESS;
 }

opal/mca/memory/malloc_solaris/memory_malloc_solaris_component.c

@@ -12,7 +12,7 @@
  *                         All rights reserved.
  * Copyright (c) 2007-2011 Oracle and/or its affiliates.  All rights reserved.
  * Copyright (c) 2009-2011 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Los Alamos National Security, LLC.  All rights
+ * Copyright (c) 2015-2016 Los Alamos National Security, LLC.  All rights
  *                         reserved.
  * Copyright (c) 2016      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
@@ -46,6 +46,7 @@ int  __munmap(caddr_t addr, size_t len);
 #endif
 
 static int opal_memory_malloc_open(void);
+static int opal_memory_malloc_query(int *);
 
 const opal_memory_base_component_2_0_0_t mca_memory_malloc_solaris_component = {
     /* First, the mca_component_t struct containing meta information
@@ -68,6 +69,7 @@ const opal_memory_base_component_2_0_0_t mca_memory_malloc_solaris_component = {
 
     /* This component doesn't need these functions, but need to
        provide safe/empty register/deregister functions to call */
+    .memoryc_query = opal_memory_malloc_query,
     .memoryc_register = opal_memory_base_component_register_empty,
     .memoryc_deregister = opal_memory_base_component_deregister_empty,
     .memoryc_set_alignment = opal_memory_base_component_set_alignment_empty,
@@ -93,6 +95,11 @@ opal_memory_malloc_open(void)
     return OPAL_SUCCESS;
 }
 
+static int opal_memory_malloc_query (int *priority)
+{
+    *priority = 79;
+    return OPAL_SUCCESS;
+}
 
 /*
  * Three ways to call munmap.  Prefered is to call __munmap, which

opal/mca/memory/memory.h

@@ -11,7 +11,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
+ * Copyright (c) 2015-2016 Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2016      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
@@ -78,6 +78,12 @@ BEGIN_C_DECLS
  */
 typedef int (*opal_memory_base_component_process_fn_t)(void);
 
+/**
+ * Prototype for a function that is invoked when the memory base is
+ * trying to select a component. This funtionality is required.
+ */
+typedef int (*opal_memory_base_component_query_fn_t)(int *priority);
+
 /**
  * Prototype for a function that is invoked when Open MPI starts to
  * "care" about a specific memory region.  That is, Open MPI declares
@@ -119,6 +125,11 @@ typedef int (*opal_memory_base_component_deregister_fn_t)(void *base,
 typedef void (*opal_memory_base_component_set_alignment_fn_t)(int use_memalign,
                                                               size_t memalign_threshold);
 
+/**
+ * Function to be called when initializing malloc hooks
+ */
+typedef void (*opal_memory_base_component_init_hook_fn_t)(void);
+
 /**
  * Structure for memory components.
  */
@@ -128,6 +139,12 @@ typedef struct opal_memory_base_component_2_0_0_t {
     /** MCA base data */
     mca_base_component_data_t memoryc_data;
 
+    opal_memory_base_component_query_fn_t memoryc_query;
+
+    /** This function will be called when the malloc hooks are
+     * initialized. It may be NULL if no hooks are needed. */
+    opal_memory_base_component_init_hook_fn_t memoryc_init_hook;
+
     /** Function to call when something has changed, as indicated by
         opal_memory_changed().  Will be ignored if the component does
         not provide an opal_memory_changed() macro that returns

opal/mca/memory/patcher/memory_patcher_component.c

@@ -47,6 +47,9 @@
 static int patcher_open(void);
 static int patcher_close(void);
 static int patcher_register(void);
+static int patcher_query (int *);
+
+static int mca_memory_patcher_priority;
 
 opal_memory_patcher_component_t mca_memory_patcher_component = {
     .super = {
@@ -69,6 +72,7 @@ opal_memory_patcher_component_t mca_memory_patcher_component = {
         },
 
         /* Memory framework functions. */
+        .memoryc_query = patcher_query,
         .memoryc_register = opal_memory_base_component_register_empty,
         .memoryc_deregister = opal_memory_base_component_deregister_empty,
         .memoryc_set_alignment = opal_memory_base_component_set_alignment_empty,
@@ -226,6 +230,18 @@ static int intercept_brk (void *addr)
 
 static int patcher_register (void)
 {
+    mca_memory_patcher_priority = 80;
+    mca_base_component_var_register (&mca_memory_patcher_component.super.memoryc_version,
+                                     "priority", "Priority of the patcher memory hook component",
+                                     MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_5,
+                                     MCA_BASE_VAR_SCOPE_CONSTANT, &mca_memory_patcher_priority);
+
+    return OPAL_SUCCESS;
+}
+
+static int patcher_query (int *priority)
+{
+    *priority = mca_memory_patcher_priority;
     return OPAL_SUCCESS;
 }
 
@@ -240,6 +256,9 @@ static int patcher_open (void)
 
     was_executed_already = 1;
 
+    /* set memory hooks support level */
+    opal_mem_hooks_set_support (OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT);
+
     rc = opal_patch_symbol ("mmap", (uintptr_t) intercept_mmap);
     if (OPAL_SUCCESS != rc) {
         return rc;