Merge pull request #5441 from hoopoepg/topic/ucx-memhooks-to-common-module

MCA/COMMON/UCX: shift opal memhooks into common UCX

ompi/mca/pml/ucx/pml_ucx.h

@@ -47,7 +47,6 @@ struct mca_pml_ucx_module {
     ompi_request_t            completed_send_req;
     size_t                    request_size;
     int                       num_disconnect;
-    bool                      opal_mem_hooks;
 
     /* Converters pool */
     mca_pml_ucx_freelist_t    convs;

ompi/mca/pml/ucx/pml_ucx_component.c

@@ -9,11 +9,8 @@
 
 #include "pml_ucx.h"
 
-#include "opal/memoryhooks/memory.h"
 #include "opal/mca/memory/base/base.h"
 
-#include <ucm/api/ucm.h>
-
 
 static int mca_pml_ucx_component_register(void);
 static int mca_pml_ucx_component_open(void);
@@ -67,38 +64,13 @@ static int mca_pml_ucx_component_register(void)
                                            OPAL_INFO_LVL_3,
                                            MCA_BASE_VAR_SCOPE_LOCAL,
                                            &ompi_pml_ucx.num_disconnect);
-
-    ompi_pml_ucx.opal_mem_hooks = 0;
-    (void) mca_base_component_var_register(&mca_pml_ucx_component.pmlm_version, "opal_mem_hooks",
-                                           "Use OPAL memory hooks, instead of UCX internal memory hooks",
-                                           MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
-                                           OPAL_INFO_LVL_3,
-                                           MCA_BASE_VAR_SCOPE_LOCAL,
-                                           &ompi_pml_ucx.opal_mem_hooks);
     return 0;
 }
 
-static void mca_pml_ucx_mem_release_cb(void *buf, size_t length,
-                                       void *cbdata, bool from_alloc)
-{
-    ucm_vm_munmap(buf, length);
-}
-
 static int mca_pml_ucx_component_open(void)
 {
     opal_common_ucx_mca_register();
 
-    /* Set memory hooks */
-    if (ompi_pml_ucx.opal_mem_hooks &&
-        (OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT) ==
-        ((OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT) &
-         opal_mem_hooks_support_level()))
-    {
-        PML_UCX_VERBOSE(1, "%s", "using OPAL memory hooks as external events");
-        ucm_set_external_event(UCM_EVENT_VM_UNMAPPED);
-        opal_mem_hooks_register_release(mca_pml_ucx_mem_release_cb, NULL);
-    }
-
     return mca_pml_ucx_open();
 }
 
@@ -111,7 +83,6 @@ static int mca_pml_ucx_component_close(void)
         return rc;
     }
 
-    opal_mem_hooks_unregister_release(mca_pml_ucx_mem_release_cb);
     opal_common_ucx_mca_deregister();
     return 0;
 }

opal/mca/common/ucx/Makefile.am

@@ -59,7 +59,8 @@ lib@OPAL_LIB_PREFIX@mca_common_ucx_la_LDFLAGS = \
 		$(common_ucx_LDFLAGS)
 lib@OPAL_LIB_PREFIX@mca_common_ucx_la_LIBADD = \
 		$(common_ucx_LIBS) \
-		$(OMPI_TOP_BUILDDIR)/opal/lib@OPAL_LIB_PREFIX@open-pal.la
+		$(OMPI_TOP_BUILDDIR)/opal/lib@OPAL_LIB_PREFIX@open-pal.la \
+		$(OMPI_TOP_BUILDDIR)/opal/mca/memory/libmca_memory.la
 lib@OPAL_LIB_PREFIX@mca_common_ucx_noinst_la_SOURCES = \
         $(headers) $(sources)
 lib@OPAL_LIB_PREFIX@mca_common_ucx_noinst_la_CFLAGS = \

opal/mca/common/ucx/common_ucx.c

@@ -11,16 +11,29 @@
 
 #include "common_ucx.h"
 #include "opal/mca/base/mca_base_var.h"
+#include "opal/mca/base/mca_base_framework.h"
 #include "opal/mca/pmix/pmix.h"
+#include "opal/memoryhooks/memory.h"
+
+#include <ucm/api/ucm.h>
 
 /***********************************************************************/
 
+extern mca_base_framework_t opal_memory_base_framework;
+
 opal_common_ucx_module_t opal_common_ucx = {
     .verbose             = 0,
     .progress_iterations = 100,
-    .registered          = 0
+    .registered          = 0,
+    .opal_mem_hooks      = 0
 };
 
+static void opal_common_ucx_mem_release_cb(void *buf, size_t length,
+                                           void *cbdata, bool from_alloc)
+{
+    ucm_vm_munmap(buf, length);
+}
+
 OPAL_DECLSPEC void opal_common_ucx_mca_register(void)
 {
     opal_common_ucx.registered++;
@@ -39,9 +52,28 @@ OPAL_DECLSPEC void opal_common_ucx_mca_register(void)
                           MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
                           OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_LOCAL,
                           &opal_common_ucx.progress_iterations);
+    mca_base_var_register("opal", "opal_common", "ucx", "opal_mem_hooks",
+                          "Use OPAL memory hooks, instead of UCX internal memory hooks",
+                          MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
+                          OPAL_INFO_LVL_3,
+                          MCA_BASE_VAR_SCOPE_LOCAL,
+                          &opal_common_ucx.opal_mem_hooks);
 
     opal_common_ucx.output = opal_output_open(NULL);
     opal_output_set_verbosity(opal_common_ucx.output, opal_common_ucx.verbose);
+
+    mca_base_framework_open(&opal_memory_base_framework, 0);
+
+    /* Set memory hooks */
+    if (opal_common_ucx.opal_mem_hooks &&
+        (OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT) ==
+        ((OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT) &
+         opal_mem_hooks_support_level()))
+    {
+        MCA_COMMON_UCX_VERBOSE(1, "%s", "using OPAL memory hooks as external events");
+        ucm_set_external_event(UCM_EVENT_VM_UNMAPPED);
+        opal_mem_hooks_register_release(opal_common_ucx_mem_release_cb, NULL);
+    }
 }
 
 OPAL_DECLSPEC void opal_common_ucx_mca_deregister(void)
@@ -52,6 +84,7 @@ OPAL_DECLSPEC void opal_common_ucx_mca_deregister(void)
     if (opal_common_ucx.registered) {
         return;
     }
+    opal_mem_hooks_unregister_release(opal_common_ucx_mem_release_cb);
     opal_output_close(opal_common_ucx.output);
 }
 

opal/mca/common/ucx/common_ucx.h

@@ -53,10 +53,11 @@ BEGIN_C_DECLS
     }
 
 typedef struct opal_common_ucx_module {
-    int output;
+    int  output;
-    int verbose;
+    int  verbose;
-    int progress_iterations;
+    int  progress_iterations;
-    int registered;
+    int  registered;
+    bool opal_mem_hooks;
 } opal_common_ucx_module_t;
 
 extern opal_common_ucx_module_t opal_common_ucx;