diff --git a/ompi/mca/pml/ucx/pml_ucx.h b/ompi/mca/pml/ucx/pml_ucx.h index ed7d0746a4..da1b3ef0c5 100644 --- a/ompi/mca/pml/ucx/pml_ucx.h +++ b/ompi/mca/pml/ucx/pml_ucx.h @@ -47,7 +47,6 @@ struct mca_pml_ucx_module { ompi_request_t completed_send_req; size_t request_size; int num_disconnect; - bool opal_mem_hooks; /* Converters pool */ mca_pml_ucx_freelist_t convs; diff --git a/ompi/mca/pml/ucx/pml_ucx_component.c b/ompi/mca/pml/ucx/pml_ucx_component.c index 5bc7a1f892..27e8997731 100644 --- a/ompi/mca/pml/ucx/pml_ucx_component.c +++ b/ompi/mca/pml/ucx/pml_ucx_component.c @@ -9,11 +9,8 @@ #include "pml_ucx.h" -#include "opal/memoryhooks/memory.h" #include "opal/mca/memory/base/base.h" -#include - static int mca_pml_ucx_component_register(void); static int mca_pml_ucx_component_open(void); @@ -67,38 +64,13 @@ static int mca_pml_ucx_component_register(void) OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_LOCAL, &ompi_pml_ucx.num_disconnect); - - ompi_pml_ucx.opal_mem_hooks = 0; - (void) mca_base_component_var_register(&mca_pml_ucx_component.pmlm_version, "opal_mem_hooks", - "Use OPAL memory hooks, instead of UCX internal memory hooks", - MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, - OPAL_INFO_LVL_3, - MCA_BASE_VAR_SCOPE_LOCAL, - &ompi_pml_ucx.opal_mem_hooks); return 0; } -static void mca_pml_ucx_mem_release_cb(void *buf, size_t length, - void *cbdata, bool from_alloc) -{ - ucm_vm_munmap(buf, length); -} - static int mca_pml_ucx_component_open(void) { opal_common_ucx_mca_register(); - /* Set memory hooks */ - if (ompi_pml_ucx.opal_mem_hooks && - (OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT) == - ((OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT) & - opal_mem_hooks_support_level())) - { - PML_UCX_VERBOSE(1, "%s", "using OPAL memory hooks as external events"); - ucm_set_external_event(UCM_EVENT_VM_UNMAPPED); - opal_mem_hooks_register_release(mca_pml_ucx_mem_release_cb, NULL); - } - return mca_pml_ucx_open(); } @@ -111,7 +83,6 @@ static int mca_pml_ucx_component_close(void) return rc; } - opal_mem_hooks_unregister_release(mca_pml_ucx_mem_release_cb); opal_common_ucx_mca_deregister(); return 0; } diff --git a/opal/mca/common/ucx/Makefile.am b/opal/mca/common/ucx/Makefile.am index d225dabf76..f1b2b06a36 100644 --- a/opal/mca/common/ucx/Makefile.am +++ b/opal/mca/common/ucx/Makefile.am @@ -59,7 +59,8 @@ lib@OPAL_LIB_PREFIX@mca_common_ucx_la_LDFLAGS = \ $(common_ucx_LDFLAGS) lib@OPAL_LIB_PREFIX@mca_common_ucx_la_LIBADD = \ $(common_ucx_LIBS) \ - $(OMPI_TOP_BUILDDIR)/opal/lib@OPAL_LIB_PREFIX@open-pal.la + $(OMPI_TOP_BUILDDIR)/opal/lib@OPAL_LIB_PREFIX@open-pal.la \ + $(OMPI_TOP_BUILDDIR)/opal/mca/memory/libmca_memory.la lib@OPAL_LIB_PREFIX@mca_common_ucx_noinst_la_SOURCES = \ $(headers) $(sources) lib@OPAL_LIB_PREFIX@mca_common_ucx_noinst_la_CFLAGS = \ diff --git a/opal/mca/common/ucx/common_ucx.c b/opal/mca/common/ucx/common_ucx.c index 86f527567a..cd54490e4d 100644 --- a/opal/mca/common/ucx/common_ucx.c +++ b/opal/mca/common/ucx/common_ucx.c @@ -11,16 +11,29 @@ #include "common_ucx.h" #include "opal/mca/base/mca_base_var.h" +#include "opal/mca/base/mca_base_framework.h" #include "opal/mca/pmix/pmix.h" +#include "opal/memoryhooks/memory.h" + +#include /***********************************************************************/ +extern mca_base_framework_t opal_memory_base_framework; + opal_common_ucx_module_t opal_common_ucx = { .verbose = 0, .progress_iterations = 100, - .registered = 0 + .registered = 0, + .opal_mem_hooks = 0 }; +static void opal_common_ucx_mem_release_cb(void *buf, size_t length, + void *cbdata, bool from_alloc) +{ + ucm_vm_munmap(buf, length); +} + OPAL_DECLSPEC void opal_common_ucx_mca_register(void) { opal_common_ucx.registered++; @@ -39,9 +52,28 @@ OPAL_DECLSPEC void opal_common_ucx_mca_register(void) MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_LOCAL, &opal_common_ucx.progress_iterations); + mca_base_var_register("opal", "opal_common", "ucx", "opal_mem_hooks", + "Use OPAL memory hooks, instead of UCX internal memory hooks", + MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, + OPAL_INFO_LVL_3, + MCA_BASE_VAR_SCOPE_LOCAL, + &opal_common_ucx.opal_mem_hooks); opal_common_ucx.output = opal_output_open(NULL); opal_output_set_verbosity(opal_common_ucx.output, opal_common_ucx.verbose); + + mca_base_framework_open(&opal_memory_base_framework, 0); + + /* Set memory hooks */ + if (opal_common_ucx.opal_mem_hooks && + (OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT) == + ((OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT) & + opal_mem_hooks_support_level())) + { + MCA_COMMON_UCX_VERBOSE(1, "%s", "using OPAL memory hooks as external events"); + ucm_set_external_event(UCM_EVENT_VM_UNMAPPED); + opal_mem_hooks_register_release(opal_common_ucx_mem_release_cb, NULL); + } } OPAL_DECLSPEC void opal_common_ucx_mca_deregister(void) @@ -52,6 +84,7 @@ OPAL_DECLSPEC void opal_common_ucx_mca_deregister(void) if (opal_common_ucx.registered) { return; } + opal_mem_hooks_unregister_release(opal_common_ucx_mem_release_cb); opal_output_close(opal_common_ucx.output); } diff --git a/opal/mca/common/ucx/common_ucx.h b/opal/mca/common/ucx/common_ucx.h index 1587f24054..13a03000e8 100644 --- a/opal/mca/common/ucx/common_ucx.h +++ b/opal/mca/common/ucx/common_ucx.h @@ -53,10 +53,11 @@ BEGIN_C_DECLS } typedef struct opal_common_ucx_module { - int output; - int verbose; - int progress_iterations; - int registered; + int output; + int verbose; + int progress_iterations; + int registered; + bool opal_mem_hooks; } opal_common_ucx_module_t; extern opal_common_ucx_module_t opal_common_ucx;