From c10bbfded6da9478308757f940f8525864fec75c Mon Sep 17 00:00:00 2001 From: Joshua Hursey Date: Fri, 20 Jan 2017 16:36:31 -0600 Subject: [PATCH] ompi/hook: Add the hook/license framework * Include a 'demo' component that shows some of the features. * Currently has hooks for: - MPI_Initialized - top, bottom - MPI_Init_thread - top, bottom - MPI_Finalized - top, bottom - MPI_Init - top (pre-opal_init), top (post-opal_init), error, bottom - MPI_Finalize - top, bottom * Other places in ompi can 'register' to hook into any one of these places by passing back a component structure filled with function pointers. * Add a `MCA_BASE_COMPONENT_FLAG_REQUIRED` flag to the MCA structure that is checked by the `hook` framework. If a required, static component has been excluded then the `hook` framework will fail to initialize. - See note in `opal/mca/mca.h` as to why this is checked in the `hook` framework and not in `opal/mca/base/mca_base_component_find.c` Signed-off-by: Joshua Hursey --- ompi/mca/hook/Makefile.am | 27 ++ ompi/mca/hook/base/Makefile.am | 16 ++ ompi/mca/hook/base/base.h | 58 +++++ ompi/mca/hook/base/help-mca-hook-base.txt | 19 ++ ompi/mca/hook/base/hook_base.c | 297 ++++++++++++++++++++++ ompi/mca/hook/base/owner.txt | 7 + ompi/mca/hook/configure.m4 | 14 + ompi/mca/hook/demo/.opal_ignore | 0 ompi/mca/hook/demo/Makefile.am | 20 ++ ompi/mca/hook/demo/configure.m4 | 25 ++ ompi/mca/hook/demo/hook_demo.h | 43 ++++ ompi/mca/hook/demo/hook_demo_component.c | 107 ++++++++ ompi/mca/hook/demo/hook_demo_fns.c | 65 +++++ ompi/mca/hook/demo/owner.txt | 7 + ompi/mca/hook/hook.h | 219 ++++++++++++++++ ompi/mpi/c/finalized.c | 5 + ompi/mpi/c/init_thread.c | 5 + ompi/mpi/c/initialized.c | 5 + ompi/runtime/ompi_mpi_finalize.c | 10 + ompi/runtime/ompi_mpi_init.c | 16 ++ opal/mca/mca.h | 21 +- 21 files changed, 984 insertions(+), 2 deletions(-) create mode 100644 ompi/mca/hook/Makefile.am create mode 100644 ompi/mca/hook/base/Makefile.am create mode 100644 ompi/mca/hook/base/base.h create mode 100644 ompi/mca/hook/base/help-mca-hook-base.txt create mode 100644 ompi/mca/hook/base/hook_base.c create mode 100644 ompi/mca/hook/base/owner.txt create mode 100644 ompi/mca/hook/configure.m4 create mode 100644 ompi/mca/hook/demo/.opal_ignore create mode 100644 ompi/mca/hook/demo/Makefile.am create mode 100644 ompi/mca/hook/demo/configure.m4 create mode 100644 ompi/mca/hook/demo/hook_demo.h create mode 100644 ompi/mca/hook/demo/hook_demo_component.c create mode 100644 ompi/mca/hook/demo/hook_demo_fns.c create mode 100644 ompi/mca/hook/demo/owner.txt create mode 100644 ompi/mca/hook/hook.h diff --git a/ompi/mca/hook/Makefile.am b/ompi/mca/hook/Makefile.am new file mode 100644 index 0000000000..af0f9e28a3 --- /dev/null +++ b/ompi/mca/hook/Makefile.am @@ -0,0 +1,27 @@ +# +# Copyright (c) 2017 IBM Corporation. All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +# main library setup +noinst_LTLIBRARIES = libmca_hook.la +libmca_hook_la_SOURCES = + +# local files +headers = hook.h +libmca_hook_la_SOURCES += $(headers) + +# Conditionally install the header files +if WANT_INSTALL_HEADERS +ompidir = $(ompiincludedir)/$(subdir) +nobase_ompi_HEADERS = $(headers) +endif + +include base/Makefile.am + +distclean-local: + rm -f base/static-components.h diff --git a/ompi/mca/hook/base/Makefile.am b/ompi/mca/hook/base/Makefile.am new file mode 100644 index 0000000000..29ac5246be --- /dev/null +++ b/ompi/mca/hook/base/Makefile.am @@ -0,0 +1,16 @@ +# +# Copyright (c) 2017 IBM Corporation. All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +dist_ompidata_DATA = base/help-mca-hook-base.txt + +headers += \ + base/base.h + +libmca_hook_la_SOURCES += \ + base/hook_base.c diff --git a/ompi/mca/hook/base/base.h b/ompi/mca/hook/base/base.h new file mode 100644 index 0000000000..675f4bc495 --- /dev/null +++ b/ompi/mca/hook/base/base.h @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2017 IBM Corporation. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef OMPI_HOOK_BASE_H +#define OMPI_HOOK_BASE_H + +#include "ompi_config.h" + +#include "ompi/mca/mca.h" +#include "opal/mca/base/mca_base_framework.h" + +#include "ompi/mca/hook/hook.h" + +BEGIN_C_DECLS + +/** + * Framework struct declaration for this framework + */ +OMPI_DECLSPEC extern mca_base_framework_t ompi_hook_base_framework; + + +/** + * Dynamically register function pointers to be called from outside of the hook + * framework. For example, a collective component could register a callback + * at the bottom of init to perform some action. + */ +OMPI_DECLSPEC int ompi_hook_base_register_callbacks(ompi_hook_base_component_t *comp); +OMPI_DECLSPEC int ompi_hook_base_deregister_callbacks(ompi_hook_base_component_t *comp); + +/** + * Wrapper functions matching the interface functions + */ +OMPI_DECLSPEC void ompi_hook_base_mpi_initialized_top(int *flag); +OMPI_DECLSPEC void ompi_hook_base_mpi_initialized_bottom(int *flag); + +OMPI_DECLSPEC void ompi_hook_base_mpi_init_thread_top(int *argc, char ***argv, int required, int *provided); +OMPI_DECLSPEC void ompi_hook_base_mpi_init_thread_bottom(int *argc, char ***argv, int required, int *provided); + +OMPI_DECLSPEC void ompi_hook_base_mpi_finalized_top(int *flag); +OMPI_DECLSPEC void ompi_hook_base_mpi_finalized_bottom(int *flag); + +OMPI_DECLSPEC void ompi_hook_base_mpi_init_top(int argc, char **argv, int requested, int *provided); +OMPI_DECLSPEC void ompi_hook_base_mpi_init_top_post_opal(int argc, char **argv, int requested, int *provided); +OMPI_DECLSPEC void ompi_hook_base_mpi_init_bottom(int argc, char **argv, int requested, int *provided); +OMPI_DECLSPEC void ompi_hook_base_mpi_init_error(int argc, char **argv, int requested, int *provided); + +OMPI_DECLSPEC void ompi_hook_base_mpi_finalize_top(void); +OMPI_DECLSPEC void ompi_hook_base_mpi_finalize_bottom(void); + +END_C_DECLS + +#endif /* OMPI_BASE_HOOK_H */ diff --git a/ompi/mca/hook/base/help-mca-hook-base.txt b/ompi/mca/hook/base/help-mca-hook-base.txt new file mode 100644 index 0000000000..c364d22956 --- /dev/null +++ b/ompi/mca/hook/base/help-mca-hook-base.txt @@ -0,0 +1,19 @@ +# -*- text -*- +# +# Copyright (c) 2017 IBM Corporation. All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# +# This is the US/English help file for Open MPI MCA hook-specific +# error messages. +# +[hook:missing-required-component] +Error: A request was made to exclude a hook component from consideration that +is required to be included. This component (noted below) can -not- be excluded +from consideration. The program will fail at this time. + +Framework: %s +Component: %s diff --git a/ompi/mca/hook/base/hook_base.c b/ompi/mca/hook/base/hook_base.c new file mode 100644 index 0000000000..6a5723a7a4 --- /dev/null +++ b/ompi/mca/hook/base/hook_base.c @@ -0,0 +1,297 @@ +/* + * Copyright (c) 2017 IBM Corporation. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "ompi_config.h" + +#include +#include + +#include "ompi/mca/mca.h" +#include "opal/mca/base/base.h" + +#include "opal/runtime/opal.h" +#include "opal/util/output.h" +#include "opal/util/show_help.h" +#include "opal/class/opal_list.h" +#include "opal/include/opal/prefetch.h" + +#include "ompi/constants.h" +#include "ompi/mca/hook/hook.h" +#include "ompi/mca/hook/base/base.h" + +/* + * The following file was created by configure. It contains extern + * statements and the definition of an array of pointers to each + * component's public mca_base_component_t struct. + */ +#include "ompi/mca/hook/base/static-components.h" + + +// Is the framework open - or has it been closed and we need to reopen it. +static bool ompi_hook_is_framework_open = false; + +static opal_list_t *additional_callback_components = NULL; + + +static int ompi_hook_base_register( mca_base_register_flag_t flags ) +{ + return OMPI_SUCCESS; +} + +static int ompi_hook_base_open( mca_base_open_flag_t flags ) +{ + int ret; + const mca_base_component_t **static_components = ompi_hook_base_framework.framework_static_components; + mca_base_component_list_item_t *cli = NULL; + mca_base_component_t *component = NULL; + bool found = false; + + additional_callback_components = OBJ_NEW(opal_list_t); + + /* Open up all available components */ + ret = mca_base_framework_components_open( &ompi_hook_base_framework, flags ); + if (ret != OMPI_SUCCESS) { + return ret; + } + + /* + * Make sure that the `MCA_BASE_COMPONENT_FLAG_REQUIRED` components defined + * as static are loaded. If we find one that was avoided then error out. + */ + if( NULL != static_components ) { + for (int i = 0 ; NULL != static_components[i]; ++i) { + if( static_components[i]->mca_component_flags & MCA_BASE_COMPONENT_FLAG_REQUIRED ) { + // Make sure that this component is in the list of components that + // were included in the earlier framework_components_open() call. + found = false; + OPAL_LIST_FOREACH(cli, &ompi_hook_base_framework.framework_components, mca_base_component_list_item_t) { + component = (mca_base_component_t*)cli->cli_component; + if( component == static_components[i] ) { + found = true; + break; + } + } + if( !found ) { + opal_show_help("help-mca-hook-base.txt", "hook:missing-required-component", true, + ompi_hook_base_framework.framework_name, + static_components[i]->mca_component_name); + return OPAL_ERR_NOT_SUPPORTED; + } + } + } + } + + /* Assume that if the component is present then it wants to be used. + * It has the option to have NULL as the function pointer for any + * functions call hook locations they do not want to hear about + */ + + /* + * There are three classes of components - we want them processed in this order + * 1) static components + * 2) dynamic components + * 3) internal 'component' hooks (from other places in the code) + * + * The ordering of (1) and (2) is managed by mca_base_component_find(). + * We keep a separate list for the 'internal' hooks. + */ + + ompi_hook_is_framework_open = true; + + /* All done */ + return OMPI_SUCCESS; +} + +static int ompi_hook_base_close( void ) +{ + int ret; + + /* + * Close our components + */ + ret = mca_base_framework_components_close( &ompi_hook_base_framework, NULL ); + if( OMPI_SUCCESS != ret ) { + return ret; + } + ompi_hook_is_framework_open = false; + + return OMPI_SUCCESS; +} + + +int ompi_hook_base_register_callbacks(ompi_hook_base_component_t *comp) +{ + mca_base_component_list_item_t *cli; + + // Check if it is already there + OPAL_LIST_FOREACH(cli, additional_callback_components, mca_base_component_list_item_t) { + if( cli->cli_component == (mca_base_component_t*)comp ) { + return OMPI_SUCCESS; + } + } + + // Not found, so add it to the list + cli = OBJ_NEW(mca_base_component_list_item_t); + cli->cli_component = (mca_base_component_t*)comp; + opal_list_append(additional_callback_components, (opal_list_item_t*) cli); + + return OMPI_SUCCESS; +} + +int ompi_hook_base_deregister_callbacks(ompi_hook_base_component_t *comp) +{ + mca_base_component_list_item_t *cli; + + // Check if it is already there + OPAL_LIST_FOREACH(cli, additional_callback_components, mca_base_component_list_item_t) { + if( cli->cli_component == (mca_base_component_t*)comp ) { + opal_list_remove_item(additional_callback_components, (opal_list_item_t*) cli); + OBJ_RELEASE(cli); + return OMPI_SUCCESS; + } + } + + return OMPI_ERR_NOT_FOUND; +} + +MCA_BASE_FRAMEWORK_DECLARE(ompi, hook, "hook hooks", + ompi_hook_base_register, + ompi_hook_base_open, + ompi_hook_base_close, + mca_hook_base_static_components, 0); + + +/* *********************************************************************** + * *********************************************************************** + * *********************************************************************** */ + +/* + * If the framework has not been opened, then we can only use the static components. + * + * Otherwise we would need to initialize opal outside of ompi_mpi_init and possibly + * after ompi_mpi_finalize which gets messy (especially when trying to cleanup). + */ +#define HOOK_CALL_COMMON_HOOK_NOT_INITIALIZED(fn_name, ...) \ + do { \ + ompi_hook_base_component_t *component; \ + int idx; \ + \ + for(idx = 0; NULL != mca_hook_base_static_components[idx]; ++idx ) { \ + component = (ompi_hook_base_component_t*)mca_hook_base_static_components[idx]; \ + if( NULL != component->hookm_ ## fn_name && \ + ompi_hook_base_ ## fn_name != component->hookm_ ## fn_name ) { \ + component->hookm_ ## fn_name ( __VA_ARGS__ ); \ + } \ + } \ + } while(0) + +/* + * Once the framework is open then call all available components with + * the approprate function pointer. Call order: + * 1) static components + * 2) dynamic components + * 3) 'registered' components (those registered by ompi_hook_base_register_callbacks) + */ +#define HOOK_CALL_COMMON_HOOK_INITIALIZED(fn_name, ...) \ + do { \ + mca_base_component_list_item_t *cli; \ + ompi_hook_base_component_t *component; \ + \ + OPAL_LIST_FOREACH(cli, &ompi_hook_base_framework.framework_components, mca_base_component_list_item_t) { \ + component = (ompi_hook_base_component_t*)cli->cli_component; \ + if( NULL != component->hookm_ ## fn_name && \ + ompi_hook_base_ ## fn_name != component->hookm_ ## fn_name ) { \ + component->hookm_ ## fn_name ( __VA_ARGS__ ); \ + } \ + } \ + \ + OPAL_LIST_FOREACH(cli, additional_callback_components, mca_base_component_list_item_t) { \ + component = (ompi_hook_base_component_t*)cli->cli_component; \ + if( NULL != component->hookm_ ## fn_name && \ + ompi_hook_base_ ## fn_name != component->hookm_ ## fn_name ) { \ + component->hookm_ ## fn_name ( __VA_ARGS__ ); \ + } \ + } \ + } while(0) + +#define HOOK_CALL_COMMON(fn_name, ...) \ + do { \ + if( OPAL_LIKELY(ompi_hook_is_framework_open) ) { \ + HOOK_CALL_COMMON_HOOK_INITIALIZED(fn_name, __VA_ARGS__); \ + } \ + else { \ + HOOK_CALL_COMMON_HOOK_NOT_INITIALIZED(fn_name, __VA_ARGS__); \ + } \ + } while(0) + + + +void ompi_hook_base_mpi_initialized_top(int *flag) +{ + HOOK_CALL_COMMON( mpi_initialized_top, flag ); +} + +void ompi_hook_base_mpi_initialized_bottom(int *flag) +{ + HOOK_CALL_COMMON( mpi_initialized_bottom, flag ); +} + + +void ompi_hook_base_mpi_init_thread_top(int *argc, char ***argv, int required, int *provided) +{ + HOOK_CALL_COMMON( mpi_init_thread_top, argc, argv, required, provided ); +} + +void ompi_hook_base_mpi_init_thread_bottom(int *argc, char ***argv, int required, int *provided) +{ + HOOK_CALL_COMMON( mpi_init_thread_bottom, argc, argv, required, provided ); +} + + +void ompi_hook_base_mpi_finalized_top(int *flag) +{ + HOOK_CALL_COMMON( mpi_finalized_top, flag ); +} + +void ompi_hook_base_mpi_finalized_bottom(int *flag) +{ + HOOK_CALL_COMMON( mpi_finalized_bottom, flag ); +} + + +void ompi_hook_base_mpi_init_top(int argc, char **argv, int requested, int *provided) +{ + HOOK_CALL_COMMON( mpi_init_top, argc, argv, requested, provided); +} + +void ompi_hook_base_mpi_init_top_post_opal(int argc, char **argv, int requested, int *provided) +{ + HOOK_CALL_COMMON( mpi_init_top_post_opal, argc, argv, requested, provided); +} + +void ompi_hook_base_mpi_init_bottom(int argc, char **argv, int requested, int *provided) +{ + HOOK_CALL_COMMON( mpi_init_bottom, argc, argv, requested, provided); +} + +void ompi_hook_base_mpi_init_error(int argc, char **argv, int requested, int *provided) +{ + HOOK_CALL_COMMON( mpi_init_error, argc, argv, requested, provided); +} + + +void ompi_hook_base_mpi_finalize_top(void) +{ + HOOK_CALL_COMMON( mpi_finalize_top, ); +} + +void ompi_hook_base_mpi_finalize_bottom(void) +{ + HOOK_CALL_COMMON( mpi_finalize_bottom, ); +} diff --git a/ompi/mca/hook/base/owner.txt b/ompi/mca/hook/base/owner.txt new file mode 100644 index 0000000000..e6150b6b0f --- /dev/null +++ b/ompi/mca/hook/base/owner.txt @@ -0,0 +1,7 @@ +# +# owner/status file +# owner: institution that is responsible for this package +# status: e.g. active, maintenance, unmaintained +# +owner: project +status: active diff --git a/ompi/mca/hook/configure.m4 b/ompi/mca/hook/configure.m4 new file mode 100644 index 0000000000..890da477aa --- /dev/null +++ b/ompi/mca/hook/configure.m4 @@ -0,0 +1,14 @@ +# +# Copyright (c) 2017 IBM Corporation. All rights reserved. +# +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +AC_DEFUN([MCA_ompi_hook_CONFIG],[ + # configure all the components + MCA_CONFIGURE_FRAMEWORK($1, $2, 1) +]) diff --git a/ompi/mca/hook/demo/.opal_ignore b/ompi/mca/hook/demo/.opal_ignore new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ompi/mca/hook/demo/Makefile.am b/ompi/mca/hook/demo/Makefile.am new file mode 100644 index 0000000000..6a7c5ddf29 --- /dev/null +++ b/ompi/mca/hook/demo/Makefile.am @@ -0,0 +1,20 @@ +# +# Copyright (c) 2017 IBM Corporation. All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +sources = \ + hook_demo.h \ + hook_demo_component.c \ + hook_demo_fns.c + +# This component will only ever be built statically -- never as a DSO. + +noinst_LTLIBRARIES = libmca_hook_demo.la + +libmca_hook_demo_la_SOURCES = $(sources) +libmca_hook_demo_la_LDFLAGS = -module -avoid-version diff --git a/ompi/mca/hook/demo/configure.m4 b/ompi/mca/hook/demo/configure.m4 new file mode 100644 index 0000000000..0531ae0449 --- /dev/null +++ b/ompi/mca/hook/demo/configure.m4 @@ -0,0 +1,25 @@ +# +# Copyright (c) 2017 IBM Corporation. All rights reserved. +# +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +# Make this a static component +AC_DEFUN([MCA_ompi_hook_demo_COMPILE_MODE], [ + AC_MSG_CHECKING([for MCA component $2:$3 compile mode]) + $4="static" + AC_MSG_RESULT([$$4]) +]) + +# MCA_hook_demo_CONFIG([action-if-can-compile], +# [action-if-cant-compile]) +# ------------------------------------------------ +AC_DEFUN([MCA_ompi_hook_demo_CONFIG],[ + AC_CONFIG_FILES([ompi/mca/hook/demo/Makefile]) + + $1 +]) diff --git a/ompi/mca/hook/demo/hook_demo.h b/ompi/mca/hook/demo/hook_demo.h new file mode 100644 index 0000000000..2022df4938 --- /dev/null +++ b/ompi/mca/hook/demo/hook_demo.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2017 IBM Corporation. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ +#ifndef MCA_HOOK_DEMO_H +#define MCA_HOOK_DEMO_H + +#include "ompi_config.h" + +#include "ompi/constants.h" + +#include "opal/util/output.h" + +#include "ompi/mca/hook/hook.h" +#include "ompi/mca/hook/base/base.h" + +BEGIN_C_DECLS + +OMPI_MODULE_DECLSPEC extern const ompi_hook_base_component_1_0_0_t mca_hook_demo_component; + +void ompi_hook_demo_mpi_initialized_top(int *flag); +void ompi_hook_demo_mpi_initialized_bottom(int *flag); + +void ompi_hook_demo_mpi_finalized_top(int *flag); +void ompi_hook_demo_mpi_finalized_bottom(int *flag); + +void ompi_hook_demo_mpi_init_top(int argc, char **argv, int requested, int *provided); +void ompi_hook_demo_mpi_init_top_post_opal(int argc, char **argv, int requested, int *provided); +void ompi_hook_demo_mpi_init_bottom(int argc, char **argv, int requested, int *provided); +void ompi_hook_demo_mpi_init_error(int argc, char **argv, int requested, int *provided); + +void ompi_hook_demo_mpi_finalize_top(void); +void ompi_hook_demo_mpi_finalize_bottom(void); + +void ompi_hook_demo_extra_mpi_init_bottom(int argc, char **argv, int requested, int *provided); + +END_C_DECLS + +#endif /* MCA_HOOK_DEMO_H */ diff --git a/ompi/mca/hook/demo/hook_demo_component.c b/ompi/mca/hook/demo/hook_demo_component.c new file mode 100644 index 0000000000..919b57cf90 --- /dev/null +++ b/ompi/mca/hook/demo/hook_demo_component.c @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2017 IBM Corporation. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "ompi_config.h" + +#include "hook_demo.h" + +static int ompi_hook_demo_component_open(void); +static int ompi_hook_demo_component_close(void); +static int ompi_hook_demo_component_register(void); + +/* + * Public string showing the component version number + */ +const char *mca_hook_demo_component_version_string = + "Open MPI 'demo' hook MCA component version " OMPI_VERSION; + +/* + * Instantiate the public struct with all of our public information + * and pointers to our public functions in it + */ +const ompi_hook_base_component_1_0_0_t mca_hook_demo_component = { + + /* First, the mca_component_t struct containing meta information + * about the component itself */ + .hookm_version = { + OMPI_HOOK_BASE_VERSION_1_0_0, + + /* Component name and version */ + .mca_component_name = "demo", + MCA_BASE_MAKE_VERSION(component, OMPI_MAJOR_VERSION, OMPI_MINOR_VERSION, + OMPI_RELEASE_VERSION), + + /* Component open and close functions */ + .mca_open_component = ompi_hook_demo_component_open, + .mca_close_component = ompi_hook_demo_component_close, + .mca_register_component_params = ompi_hook_demo_component_register, + + // Force this component to always be considered - component must be static + .mca_component_flags = MCA_BASE_COMPONENT_FLAG_REQUIRED, + }, + .hookm_data = { + /* The component is checkpoint ready */ + MCA_BASE_METADATA_PARAM_CHECKPOINT + }, + + /* Component functions */ + .hookm_mpi_initialized_top = ompi_hook_demo_mpi_initialized_top, + .hookm_mpi_initialized_bottom = ompi_hook_demo_mpi_initialized_bottom, + + .hookm_mpi_finalized_top = ompi_hook_demo_mpi_finalized_top, + .hookm_mpi_finalized_bottom = ompi_hook_demo_mpi_finalized_bottom, + + .hookm_mpi_init_top = ompi_hook_demo_mpi_init_top, + .hookm_mpi_init_top_post_opal = ompi_hook_demo_mpi_init_top_post_opal, + .hookm_mpi_init_bottom = ompi_hook_demo_mpi_init_bottom, + .hookm_mpi_init_error = ompi_hook_demo_mpi_init_error, + + .hookm_mpi_finalize_top = ompi_hook_demo_mpi_finalize_top, + .hookm_mpi_finalize_bottom = ompi_hook_demo_mpi_finalize_bottom, +}; + +/* + * Example 'extra' component with an additional callback that is dynamically + * registered at runtime in addition to the callbacks that are a part of the + * component structure. + */ +ompi_hook_base_component_1_0_0_t hook_demo_extra_component = { + /* Component functions */ + .hookm_mpi_init_bottom = ompi_hook_demo_extra_mpi_init_bottom, +}; + +static int ompi_hook_demo_component_open(void) +{ + opal_output(0, "hook/demo: component_open()"); + + // Register the 'extra' callback(s) to be called the next time those + // functions are encountered. + ompi_hook_base_register_callbacks( &hook_demo_extra_component ); + + return OMPI_SUCCESS; +} + +static int ompi_hook_demo_component_close(void) +{ + opal_output(0, "hook/demo: component_close()"); + + // Deregister the 'extra' callback(s) so that they are no longer called. + // Must pass in the same 'component' structure that was passed to the + // ompi_hook_base_register_callbacks() earlier. + ompi_hook_base_deregister_callbacks( &hook_demo_extra_component ); + + return OMPI_SUCCESS; +} + +static int ompi_hook_demo_component_register(void) +{ + opal_output(0, "hook/demo: component_register()"); + return OMPI_SUCCESS; +} + diff --git a/ompi/mca/hook/demo/hook_demo_fns.c b/ompi/mca/hook/demo/hook_demo_fns.c new file mode 100644 index 0000000000..c700eef24e --- /dev/null +++ b/ompi/mca/hook/demo/hook_demo_fns.c @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2017 IBM Corporation. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "ompi_config.h" + +#include "hook_demo.h" + +#define DEBUG_OUTPUT( fn_name ) \ + do { \ + opal_output(0, "hook:demo: %s", __func__ ); \ + } while(0) + +void ompi_hook_demo_mpi_initialized_top(int *flag) { + DEBUG_OUTPUT(); +} + +void ompi_hook_demo_mpi_initialized_bottom(int *flag) { + DEBUG_OUTPUT(); +} + + +void ompi_hook_demo_mpi_finalized_top(int *flag) { + DEBUG_OUTPUT(); +} + +void ompi_hook_demo_mpi_finalized_bottom(int *flag) { + DEBUG_OUTPUT(); +} + + +void ompi_hook_demo_mpi_init_top(int argc, char **argv, int requested, int *provided) { + DEBUG_OUTPUT(); +} + +void ompi_hook_demo_mpi_init_top_post_opal(int argc, char **argv, int requested, int *provided) { + DEBUG_OUTPUT(); +} + +void ompi_hook_demo_mpi_init_bottom(int argc, char **argv, int requested, int *provided) { + DEBUG_OUTPUT(); +} + +void ompi_hook_demo_mpi_init_error(int argc, char **argv, int requested, int *provided) { + DEBUG_OUTPUT(); +} + + +void ompi_hook_demo_mpi_finalize_top(void) { + DEBUG_OUTPUT(); +} + +void ompi_hook_demo_mpi_finalize_bottom(void) { + DEBUG_OUTPUT(); +} + + +void ompi_hook_demo_extra_mpi_init_bottom(int argc, char **argv, int requested, int *provided) { + DEBUG_OUTPUT(); +} diff --git a/ompi/mca/hook/demo/owner.txt b/ompi/mca/hook/demo/owner.txt new file mode 100644 index 0000000000..2d23c9be65 --- /dev/null +++ b/ompi/mca/hook/demo/owner.txt @@ -0,0 +1,7 @@ +# +# owner/status file +# owner: institution that is responsible for this package +# status: e.g. active, maintenance, unmaintained +# +owner: project +status: maintenance diff --git a/ompi/mca/hook/hook.h b/ompi/mca/hook/hook.h new file mode 100644 index 0000000000..4625b17f43 --- /dev/null +++ b/ompi/mca/hook/hook.h @@ -0,0 +1,219 @@ +/* + * Copyright (c) 2017 IBM Corporation. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +/** + * @file + * + * The hook framework is designed to allow a component of the hook framework + * to be called a designated points in the MPI process lifecycle. + * + * There is not module structure in this framework since the components apply to + * the whole process. Further, the component might need to active before mca_init. + * + * Why not a PMPI library? + * - The desire is to allow a component to be built statically into the library + * for licensing and disgnostic purposes. As such we need the ability for + * the component to be statically linked into the libmpi library. + * + * Why is there not a hook for location XYZ? + * - The number of possible places for hooks is quite large. This framework + * supports a set that we think will provide the best coverage for the + * common use cases. If you have another use case we are always open to + * discussing extensions on the mailing list. + */ +#ifndef MCA_HOOK_H +#define MCA_HOOK_H + +#include "ompi_config.h" + +#include "mpi.h" +#include "ompi/mca/mca.h" +#include "opal/mca/base/base.h" + +BEGIN_C_DECLS + +/* + * Some functions are specially marked. See decoder below. + * + * *** Static Only (Always) *** + * Only static components will ever see this callback triggered due to the + * position of the hook relative to ompi_mpi_init and ompi_mpi_finalize. + * + * *** Static Only (Outside MPI), Everyone (Inside MPI) *** + * Only static components will ever see this callback triggered when outside + * of the ompi_mpi_init and ompi_mpi_finalize region. Others may see this + * triggered if the user's program makes additional calls while inside + * that region. + * + * *** Everyone (Inside MPI) *** + * Everyone registered will see this callback triggered. It is only triggered + * between ompi_mpi_init and ompi_mpi_finalize. + * + */ +/* ******************************************************************** */ + +/** + * *** Static Only (Outside MPI), Everyone (Inside MPI) *** + * + * Location: mpi/c/initialized.c + * At the top of function (outside of mutex) + */ +typedef void (*ompi_hook_base_component_mpi_initialized_top_fn_t)(int *flag); + +/** + * *** Static Only (Outside MPI), Everyone (Inside MPI) *** + * + * Location: mpi/c/initialized.c + * At the bottom of function (outside of mutex) + */ +typedef void (*ompi_hook_base_component_mpi_initialized_bottom_fn_t)(int *flag); + +/** + * *** Static Only (Outside MPI), Everyone (Inside MPI) *** + * + * Location: mpi/c/init_thread.c + * At the top of function + */ +typedef void (*ompi_hook_base_component_mpi_init_thread_top_fn_t)(int *argc, char ***argv, int required, int *provided); + +/** + * *** Static Only (Outside MPI), Everyone (Inside MPI) *** + * + * Location: mpi/c/init_thread.c + * At the bottom of function + */ +typedef void (*ompi_hook_base_component_mpi_init_thread_bottom_fn_t)(int *argc, char ***argv, int required, int *provided); + +/** + * *** Static Only (Outside MPI), Everyone (Inside MPI) *** + * + * Location: mpi/c/finalized.c + * At the top of function (outside of mutex) + */ +typedef void (*ompi_hook_base_component_mpi_finalized_top_fn_t)(int *flag); + +/** + * *** Static Only (Outside MPI), Everyone (Inside MPI) *** + * + * Location: mpi/c/finalized.c + * At the bottom of function (outside of mutex) + */ +typedef void (*ompi_hook_base_component_mpi_finalized_bottom_fn_t)(int *flag); + +/** + * *** Static Only (Always) *** + * + * Location: runtime/ompi_mpi_init.c + * At top of function (outside of mutex) + */ +typedef void (*ompi_hook_base_component_mpi_init_top_fn_t)(int argc, char **argv, int requested, int *provided); + +/** + * *** Everyone (Inside MPI) *** + * + * Location: runtime/ompi_mpi_init.c + * Just after opal_init_util and MCA initialization. (inside mutex) + * Notes: + * This framework has been opened as have its components. + */ +typedef void (*ompi_hook_base_component_mpi_init_top_post_opal_fn_t)(int argc, char **argv, int requested, int *provided); + +/** + * *** Everyone (Inside MPI) *** + * + * Location: runtime/ompi_mpi_init.c + * At the bottom of the function. (outside mutex) + * Notes: + * This framework has been opened as have its components. + * Can safely use all MPI functionality. + */ +typedef void (*ompi_hook_base_component_mpi_init_bottom_fn_t)(int argc, char **argv, int requested, int *provided); + +/** + * *** Everyone (Inside MPI) *** + * + * Location: runtime/ompi_mpi_init.c + * At the bottom of the error path. (outside mutex) + * Notes: + * This framework has been opened as have its components. + */ +typedef void (*ompi_hook_base_component_mpi_init_error_fn_t)(int argc, char **argv, int requested, int *provided); + +/** + * *** Everyone (Inside MPI) *** + * + * Location: runtime/ompi_mpi_finalize.c + * At the top of the function. (outside mutex) + * Notes: + * This framework has been opened as have its components. + * Can safely use all MPI functionality. + */ +typedef void (*ompi_hook_base_component_mpi_finalize_top_fn_t)(void); + +/** + * *** Static Only (Always) *** + * + * Location: runtime/ompi_mpi_finalize.c + * At top of function (outside of mutex) + * Notes: + * This framework has been closed. + */ +typedef void (*ompi_hook_base_component_mpi_finalize_bottom_fn_t)(void); + + +/* ******************************************************************** */ + +/** + * Hook component version and interface functions. + */ +struct ompi_hook_base_component_1_0_0_t { + mca_base_component_t hookm_version; + mca_base_component_data_t hookm_data; + + /* MPI_Initialized */ + ompi_hook_base_component_mpi_initialized_top_fn_t hookm_mpi_initialized_top; + ompi_hook_base_component_mpi_initialized_bottom_fn_t hookm_mpi_initialized_bottom; + + /* MPI_Init_thread */ + ompi_hook_base_component_mpi_init_thread_top_fn_t hookm_mpi_init_thread_top; + ompi_hook_base_component_mpi_init_thread_bottom_fn_t hookm_mpi_init_thread_bottom; + + /* MPI_Finalized */ + ompi_hook_base_component_mpi_finalized_top_fn_t hookm_mpi_finalized_top; + ompi_hook_base_component_mpi_finalized_bottom_fn_t hookm_mpi_finalized_bottom; + + /* ompi_mpi_init */ + ompi_hook_base_component_mpi_init_top_fn_t hookm_mpi_init_top; + ompi_hook_base_component_mpi_init_top_post_opal_fn_t hookm_mpi_init_top_post_opal; + ompi_hook_base_component_mpi_init_bottom_fn_t hookm_mpi_init_bottom; + ompi_hook_base_component_mpi_init_error_fn_t hookm_mpi_init_error; + + /* ompi_mpi_finalize */ + ompi_hook_base_component_mpi_finalize_top_fn_t hookm_mpi_finalize_top; + ompi_hook_base_component_mpi_finalize_bottom_fn_t hookm_mpi_finalize_bottom; +}; +typedef struct ompi_hook_base_component_1_0_0_t ompi_hook_base_component_1_0_0_t; +typedef ompi_hook_base_component_1_0_0_t ompi_hook_base_component_t; +/* + * Note: We do -not- expose a component object for this framework. + * All interation with the component should go through the base/base.h interfaces. + * See that header for more information on calling functions. + */ + +/* ******************************************************************** */ + +/* + * Macro for use in components that are of type hook + */ +#define OMPI_HOOK_BASE_VERSION_1_0_0 \ + OMPI_MCA_BASE_VERSION_2_1_0("hook", 1, 0, 0) + +END_C_DECLS + +#endif /* MCA_HOOK_H */ diff --git a/ompi/mpi/c/finalized.c b/ompi/mpi/c/finalized.c index dc6e8a6bab..7720a84d80 100644 --- a/ompi/mpi/c/finalized.c +++ b/ompi/mpi/c/finalized.c @@ -26,6 +26,7 @@ #include "ompi/runtime/params.h" #include "ompi/communicator/communicator.h" #include "ompi/errhandler/errhandler.h" +#include "ompi/mca/hook/base/base.h" #if OMPI_BUILD_MPI_PROFILING #if OPAL_HAVE_WEAK_SYMBOLS @@ -41,6 +42,8 @@ int MPI_Finalized(int *flag) { OPAL_CR_NOOP_PROGRESS(); + ompi_hook_base_mpi_finalized_top(flag); + /* We must obtain the lock to guarnatee consistent values of ompi_mpi_initialized and ompi_mpi_finalized. Note, too, that this lock is held for the bulk of the duration of @@ -74,5 +77,7 @@ int MPI_Finalized(int *flag) *flag = ompi_mpi_finalized; opal_mutex_unlock(&ompi_mpi_bootstrap_mutex); + ompi_hook_base_mpi_finalized_bottom(flag); + return MPI_SUCCESS; } diff --git a/ompi/mpi/c/init_thread.c b/ompi/mpi/c/init_thread.c index bbb7806107..38c6d7b7a8 100644 --- a/ompi/mpi/c/init_thread.c +++ b/ompi/mpi/c/init_thread.c @@ -31,6 +31,7 @@ #include "ompi/communicator/communicator.h" #include "ompi/errhandler/errhandler.h" #include "ompi/constants.h" +#include "ompi/mca/hook/base/base.h" #if OMPI_BUILD_MPI_PROFILING #if OPAL_HAVE_WEAK_SYMBOLS @@ -47,6 +48,8 @@ int MPI_Init_thread(int *argc, char ***argv, int required, { int err; + ompi_hook_base_mpi_init_thread_top(argc, argv, required, provided); + if ( MPI_PARAM_CHECK ) { if (required < MPI_THREAD_SINGLE || required > MPI_THREAD_MULTIPLE) { ompi_mpi_errors_are_fatal_comm_handler(NULL, NULL, FUNC_NAME); @@ -78,5 +81,7 @@ int MPI_Init_thread(int *argc, char ***argv, int required, OPAL_CR_INIT_LIBRARY(); + ompi_hook_base_mpi_init_thread_bottom(argc, argv, required, provided); + return MPI_SUCCESS; } diff --git a/ompi/mpi/c/initialized.c b/ompi/mpi/c/initialized.c index 459b764af1..3b163279a8 100644 --- a/ompi/mpi/c/initialized.c +++ b/ompi/mpi/c/initialized.c @@ -26,6 +26,7 @@ #include "ompi/runtime/params.h" #include "ompi/communicator/communicator.h" #include "ompi/errhandler/errhandler.h" +#include "ompi/mca/hook/base/base.h" #if OMPI_BUILD_MPI_PROFILING #if OPAL_HAVE_WEAK_SYMBOLS @@ -41,6 +42,8 @@ int MPI_Initialized(int *flag) { OPAL_CR_NOOP_PROGRESS(); + ompi_hook_base_mpi_initialized_top(flag); + /* We must obtain the lock to guarnatee consistent values of ompi_mpi_initialized and ompi_mpi_finalized. Note, too, that this lock is held for the bulk of the duration of @@ -74,5 +77,7 @@ int MPI_Initialized(int *flag) *flag = ompi_mpi_initialized; opal_mutex_unlock(&ompi_mpi_bootstrap_mutex); + ompi_hook_base_mpi_initialized_bottom(flag); + return MPI_SUCCESS; } diff --git a/ompi/runtime/ompi_mpi_finalize.c b/ompi/runtime/ompi_mpi_finalize.c index 2ec6abbab2..bfbefd5867 100644 --- a/ompi/runtime/ompi_mpi_finalize.c +++ b/ompi/runtime/ompi_mpi_finalize.c @@ -82,6 +82,7 @@ #include "ompi/runtime/params.h" #include "ompi/dpm/dpm.h" #include "ompi/mpiext/mpiext.h" +#include "ompi/mca/hook/base/base.h" #if OPAL_ENABLE_FT_CR == 1 #include "ompi/mca/crcp/crcp.h" @@ -110,6 +111,8 @@ int ompi_mpi_finalize(void) OPAL_TIMING_DECLARE(tm); OPAL_TIMING_INIT_EXT(&tm, OPAL_TIMING_GET_TIME_OF_DAY); + ompi_hook_base_mpi_finalize_top(); + /* Be a bit social if an erroneous program calls MPI_FINALIZE in two different threads, otherwise we may deadlock in ompi_comm_free() (or run into other nasty lions, tigers, or @@ -487,6 +490,12 @@ int ompi_mpi_finalize(void) goto done; } + /* Now close the hook framework */ + if (OMPI_SUCCESS != (ret = mca_base_framework_close(&ompi_hook_base_framework) ) ) { + OMPI_ERROR_LOG(ret); + goto done; + } + if (OPAL_SUCCESS != (ret = opal_finalize_util())) { goto done; } @@ -506,6 +515,7 @@ int ompi_mpi_finalize(void) done: opal_mutex_unlock(&ompi_mpi_bootstrap_mutex); + ompi_hook_base_mpi_finalize_bottom(); return ret; } diff --git a/ompi/runtime/ompi_mpi_init.c b/ompi/runtime/ompi_mpi_init.c index 34616380f3..a39424ff80 100644 --- a/ompi/runtime/ompi_mpi_init.c +++ b/ompi/runtime/ompi_mpi_init.c @@ -92,6 +92,7 @@ #include "ompi/mca/pml/base/pml_base_bsend.h" #include "ompi/dpm/dpm.h" #include "ompi/mpiext/mpiext.h" +#include "ompi/mca/hook/base/base.h" #if OPAL_ENABLE_FT_CR == 1 #include "ompi/mca/crcp/crcp.h" @@ -388,6 +389,8 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided) * for the modex in order to work in heterogeneous environments. */ uint8_t threadlevel_bf; + ompi_hook_base_mpi_init_top(argc, argv, requested, provided); + /* Ensure that we were not already initialized or finalized. This lock is held for the duration of ompi_mpi_init() and @@ -474,6 +477,15 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided) mca_base_var_set_value(ret, allvalue, 4, MCA_BASE_VAR_SOURCE_DEFAULT, NULL); } + /* open the ompi hook framework */ + if (OMPI_SUCCESS != (ret = mca_base_framework_open(&ompi_hook_base_framework, 0))) { + error = "ompi_hook_base_open() failed"; + goto error; + } + + ompi_hook_base_mpi_init_top_post_opal(argc, argv, requested, provided); + + OPAL_TIMING_MSTART((&tm,"time from start to completion of rte_init")); /* if we were not externally started, then we need to setup @@ -939,6 +951,7 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided) "MPI_INIT", "MPI_INIT", error, err_msg, ret); } opal_mutex_unlock(&ompi_mpi_bootstrap_mutex); + ompi_hook_base_mpi_init_error(argc, argv, requested, provided); return ret; } @@ -969,5 +982,8 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided) OPAL_TIMING_RELEASE(&tm); opal_mutex_unlock(&ompi_mpi_bootstrap_mutex); + + ompi_hook_base_mpi_init_bottom(argc, argv, requested, provided); + return MPI_SUCCESS; } diff --git a/opal/mca/mca.h b/opal/mca/mca.h index 7a54574365..c8b142a063 100644 --- a/opal/mca/mca.h +++ b/opal/mca/mca.h @@ -11,8 +11,9 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2008-2012 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2015 Los Alamos National Security, LLC. All rights + * Copyright (c) 2015-2016 Los Alamos National Security, LLC. All rights * reserved. + * Copyright (c) 2016-2017 IBM Corporation. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -256,6 +257,19 @@ typedef int (*mca_base_register_component_params_2_0_0_fn_t)(void); */ #define MCA_BASE_MAX_COMPONENT_NAME_LEN 63 +/** + * Component flags (mca_component_flags field) + */ +enum { + /** Always consider this component for selection. For this flag to + * work properly the component must always be built statically. + * + * Enforecement of this flag is left to the individual frameworks + * so as to limit its use. See discussion from the Open MPI + * face-to-face meeting Jan. 2017 */ + MCA_BASE_COMPONENT_FLAG_REQUIRED = 1, +}; + /** * Common type for all MCA components. * @@ -315,9 +329,12 @@ struct mca_base_component_2_1_0_t { mca_base_register_component_params_2_0_0_fn_t mca_register_component_params; /**< Method for registering the component's MCA parameters */ + int32_t mca_component_flags; + /**< flags for this component */ + /** Extra space to allow for expansion in the future without breaking older components. */ - char reserved[32]; + char reserved[28]; }; /** Unversioned convenience typedef; use this name in frameworks/components to stay forward source-compatible */