Merge pull request #2773 from jjhursey/topic/hook-fwk

Add a 'hook' framework

ompi/mca/hook/Makefile.am

@@ -0,0 +1,27 @@
+#
+# Copyright (c) 2017      IBM Corporation.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# main library setup
+noinst_LTLIBRARIES = libmca_hook.la
+libmca_hook_la_SOURCES =
+
+# local files
+headers = hook.h
+libmca_hook_la_SOURCES += $(headers)
+
+# Conditionally install the header files
+if WANT_INSTALL_HEADERS
+ompidir = $(ompiincludedir)/$(subdir)
+nobase_ompi_HEADERS = $(headers)
+endif
+
+include base/Makefile.am
+
+distclean-local:
+	rm -f base/static-components.h

ompi/mca/hook/base/Makefile.am

@@ -0,0 +1,16 @@
+#
+# Copyright (c) 2017      IBM Corporation.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+dist_ompidata_DATA = base/help-mca-hook-base.txt
+
+headers += \
+        base/base.h
+
+libmca_hook_la_SOURCES += \
+        base/hook_base.c

ompi/mca/hook/base/base.h

@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2017      IBM Corporation.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef OMPI_HOOK_BASE_H
+#define OMPI_HOOK_BASE_H
+
+#include "ompi_config.h"
+
+#include "ompi/mca/mca.h"
+#include "opal/mca/base/mca_base_framework.h"
+
+#include "ompi/mca/hook/hook.h"
+
+BEGIN_C_DECLS
+
+/**
+ * Framework struct declaration for this framework
+ */
+OMPI_DECLSPEC extern mca_base_framework_t ompi_hook_base_framework;
+
+
+/**
+ * Dynamically register function pointers to be called from outside of the hook
+ * framework. For example, a collective component could register a callback
+ * at the bottom of init to perform some action.
+ */
+OMPI_DECLSPEC int ompi_hook_base_register_callbacks(ompi_hook_base_component_t *comp);
+OMPI_DECLSPEC int ompi_hook_base_deregister_callbacks(ompi_hook_base_component_t *comp);
+
+/**
+ * Wrapper functions matching the interface functions
+ */
+OMPI_DECLSPEC void ompi_hook_base_mpi_initialized_top(int *flag);
+OMPI_DECLSPEC void ompi_hook_base_mpi_initialized_bottom(int *flag);
+
+OMPI_DECLSPEC void ompi_hook_base_mpi_init_thread_top(int *argc, char ***argv, int required, int *provided);
+OMPI_DECLSPEC void ompi_hook_base_mpi_init_thread_bottom(int *argc, char ***argv, int required, int *provided);
+
+OMPI_DECLSPEC void ompi_hook_base_mpi_finalized_top(int *flag);
+OMPI_DECLSPEC void ompi_hook_base_mpi_finalized_bottom(int *flag);
+
+OMPI_DECLSPEC void ompi_hook_base_mpi_init_top(int argc, char **argv, int requested, int *provided);
+OMPI_DECLSPEC void ompi_hook_base_mpi_init_top_post_opal(int argc, char **argv, int requested, int *provided);
+OMPI_DECLSPEC void ompi_hook_base_mpi_init_bottom(int argc, char **argv, int requested, int *provided);
+OMPI_DECLSPEC void ompi_hook_base_mpi_init_error(int argc, char **argv, int requested, int *provided);
+
+OMPI_DECLSPEC void ompi_hook_base_mpi_finalize_top(void);
+OMPI_DECLSPEC void ompi_hook_base_mpi_finalize_bottom(void);
+
+END_C_DECLS
+
+#endif /* OMPI_BASE_HOOK_H */

ompi/mca/hook/base/help-mca-hook-base.txt

@@ -0,0 +1,19 @@
+# -*- text -*-
+#
+# Copyright (c) 2017      IBM Corporation.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+# This is the US/English help file for Open MPI MCA hook-specific
+# error messages.
+#
+[hook:missing-required-component]
+Error: A request was made to exclude a hook component from consideration that
+is required to be included. This component (noted below) can -not- be excluded
+from consideration. The program will fail at this time.
+ 
+Framework: %s
+Component: %s

ompi/mca/hook/base/hook_base.c

@@ -0,0 +1,297 @@
+/*
+ * Copyright (c) 2017      IBM Corporation.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include <sys/types.h>
+#include <unistd.h>
+
+#include "ompi/mca/mca.h"
+#include "opal/mca/base/base.h"
+
+#include "opal/runtime/opal.h"
+#include "opal/util/output.h"
+#include "opal/util/show_help.h"
+#include "opal/class/opal_list.h"
+#include "opal/include/opal/prefetch.h"
+
+#include "ompi/constants.h"
+#include "ompi/mca/hook/hook.h"
+#include "ompi/mca/hook/base/base.h"
+
+/*
+ * The following file was created by configure.  It contains extern
+ * statements and the definition of an array of pointers to each
+ * component's public mca_base_component_t struct.
+ */
+#include "ompi/mca/hook/base/static-components.h"
+
+
+// Is the framework open - or has it been closed and we need to reopen it.
+static bool ompi_hook_is_framework_open = false;
+
+static opal_list_t *additional_callback_components = NULL;
+
+
+static int ompi_hook_base_register( mca_base_register_flag_t flags )
+{
+    return OMPI_SUCCESS;
+}
+
+static int ompi_hook_base_open( mca_base_open_flag_t flags )
+{
+    int ret;
+    const mca_base_component_t **static_components = ompi_hook_base_framework.framework_static_components;
+    mca_base_component_list_item_t *cli = NULL;
+    mca_base_component_t *component = NULL;
+    bool found = false;
+
+    additional_callback_components = OBJ_NEW(opal_list_t);
+
+    /* Open up all available components */
+    ret = mca_base_framework_components_open( &ompi_hook_base_framework, flags );
+    if (ret != OMPI_SUCCESS) {
+        return ret;
+    }
+
+    /*
+     * Make sure that the `MCA_BASE_COMPONENT_FLAG_REQUIRED` components defined
+     * as static are loaded. If we find one that was avoided then error out.
+     */
+    if( NULL != static_components ) {
+        for (int i = 0 ; NULL != static_components[i]; ++i) {
+            if( static_components[i]->mca_component_flags & MCA_BASE_COMPONENT_FLAG_REQUIRED ) {
+                // Make sure that this component is in the list of components that
+                // were included in the earlier framework_components_open() call.
+                found = false;
+                OPAL_LIST_FOREACH(cli, &ompi_hook_base_framework.framework_components, mca_base_component_list_item_t) {
+                    component = (mca_base_component_t*)cli->cli_component;
+                    if( component == static_components[i] ) {
+                        found = true;
+                        break;
+                    }
+                }
+                if( !found ) {
+                    opal_show_help("help-mca-hook-base.txt", "hook:missing-required-component", true,
+                                   ompi_hook_base_framework.framework_name,
+                                   static_components[i]->mca_component_name);
+                    return OPAL_ERR_NOT_SUPPORTED;
+                }
+            }
+        }
+    }
+
+    /* Assume that if the component is present then it wants to be used.
+     * It has the option to have NULL as the function pointer for any
+     * functions call hook locations they do not want to hear about
+     */
+
+    /*
+     * There are three classes of components - we want them processed in this order
+     * 1) static components
+     * 2) dynamic components
+     * 3) internal 'component' hooks (from other places in the code)
+     *
+     * The ordering of (1) and (2) is managed by mca_base_component_find().
+     * We keep a separate list for the 'internal' hooks.
+     */
+
+    ompi_hook_is_framework_open = true;
+
+    /* All done */
+    return OMPI_SUCCESS;
+}
+
+static int ompi_hook_base_close( void )
+{
+    int ret;
+
+    /*
+     * Close our components
+     */
+    ret = mca_base_framework_components_close( &ompi_hook_base_framework, NULL );
+    if( OMPI_SUCCESS != ret ) {
+        return ret;
+    }
+    ompi_hook_is_framework_open = false;
+
+    return OMPI_SUCCESS;
+}
+
+
+int ompi_hook_base_register_callbacks(ompi_hook_base_component_t *comp)
+{
+    mca_base_component_list_item_t *cli;
+
+    // Check if it is already there
+    OPAL_LIST_FOREACH(cli, additional_callback_components, mca_base_component_list_item_t) {
+        if( cli->cli_component == (mca_base_component_t*)comp ) {
+            return OMPI_SUCCESS;
+        }
+    }
+
+    // Not found, so add it to the list
+    cli = OBJ_NEW(mca_base_component_list_item_t);
+    cli->cli_component = (mca_base_component_t*)comp;
+    opal_list_append(additional_callback_components, (opal_list_item_t*) cli);
+
+    return OMPI_SUCCESS;
+}
+
+int ompi_hook_base_deregister_callbacks(ompi_hook_base_component_t *comp)
+{
+    mca_base_component_list_item_t *cli;
+
+    // Check if it is already there
+    OPAL_LIST_FOREACH(cli, additional_callback_components, mca_base_component_list_item_t) {
+        if( cli->cli_component == (mca_base_component_t*)comp ) {
+            opal_list_remove_item(additional_callback_components, (opal_list_item_t*) cli);
+            OBJ_RELEASE(cli);
+            return OMPI_SUCCESS;
+        }
+    }
+
+    return OMPI_ERR_NOT_FOUND;
+}
+
+MCA_BASE_FRAMEWORK_DECLARE(ompi, hook, "hook hooks",
+                           ompi_hook_base_register,
+                           ompi_hook_base_open,
+                           ompi_hook_base_close,
+                           mca_hook_base_static_components, 0);
+
+
+/* ***********************************************************************
+ * ***********************************************************************
+ * *********************************************************************** */
+
+/*
+ * If the framework has not been opened, then we can only use the static components.
+ *
+ * Otherwise we would need to initialize opal outside of ompi_mpi_init and possibly
+ * after ompi_mpi_finalize which gets messy (especially when trying to cleanup).
+ */
+#define HOOK_CALL_COMMON_HOOK_NOT_INITIALIZED(fn_name, ...)             \
+    do {                                                                \
+        ompi_hook_base_component_t *component;                          \
+        int idx;                                                        \
+                                                                        \
+        for(idx = 0; NULL != mca_hook_base_static_components[idx]; ++idx ) { \
+            component = (ompi_hook_base_component_t*)mca_hook_base_static_components[idx]; \
+            if( NULL != component->hookm_ ## fn_name &&                 \
+                ompi_hook_base_ ## fn_name != component->hookm_ ## fn_name ) { \
+                component->hookm_ ## fn_name ( __VA_ARGS__ );           \
+            }                                                           \
+        }                                                               \
+    } while(0)
+
+/*
+ * Once the framework is open then call all available components with
+ * the approprate function pointer. Call order:
+ * 1) static components
+ * 2) dynamic components
+ * 3) 'registered' components (those registered by ompi_hook_base_register_callbacks)
+ */
+#define HOOK_CALL_COMMON_HOOK_INITIALIZED(fn_name, ...)                 \
+    do {                                                                \
+        mca_base_component_list_item_t *cli;                            \
+        ompi_hook_base_component_t *component;                          \
+                                                                        \
+        OPAL_LIST_FOREACH(cli, &ompi_hook_base_framework.framework_components, mca_base_component_list_item_t) { \
+            component = (ompi_hook_base_component_t*)cli->cli_component; \
+            if( NULL != component->hookm_ ## fn_name &&                 \
+                ompi_hook_base_ ## fn_name != component->hookm_ ## fn_name ) { \
+                component->hookm_ ## fn_name ( __VA_ARGS__ );           \
+            }                                                           \
+        }                                                               \
+                                                                        \
+        OPAL_LIST_FOREACH(cli, additional_callback_components, mca_base_component_list_item_t) { \
+            component = (ompi_hook_base_component_t*)cli->cli_component; \
+            if( NULL != component->hookm_ ## fn_name &&                 \
+                ompi_hook_base_ ## fn_name != component->hookm_ ## fn_name ) { \
+                component->hookm_ ## fn_name ( __VA_ARGS__ );           \
+            }                                                           \
+        }                                                               \
+    } while(0)
+
+#define HOOK_CALL_COMMON(fn_name, ...)                                  \
+    do {                                                                \
+        if( OPAL_LIKELY(ompi_hook_is_framework_open) ) {                \
+            HOOK_CALL_COMMON_HOOK_INITIALIZED(fn_name, __VA_ARGS__);    \
+        }                                                               \
+        else {                                                          \
+            HOOK_CALL_COMMON_HOOK_NOT_INITIALIZED(fn_name, __VA_ARGS__); \
+        }                                                               \
+    } while(0)
+
+
+
+void ompi_hook_base_mpi_initialized_top(int *flag)
+{
+    HOOK_CALL_COMMON( mpi_initialized_top, flag );
+}
+
+void ompi_hook_base_mpi_initialized_bottom(int *flag)
+{
+    HOOK_CALL_COMMON( mpi_initialized_bottom, flag );
+}
+
+
+void ompi_hook_base_mpi_init_thread_top(int *argc, char ***argv, int required, int *provided)
+{
+    HOOK_CALL_COMMON( mpi_init_thread_top, argc, argv, required, provided );
+}
+
+void ompi_hook_base_mpi_init_thread_bottom(int *argc, char ***argv, int required, int *provided)
+{
+    HOOK_CALL_COMMON( mpi_init_thread_bottom, argc, argv, required, provided );
+}
+
+
+void ompi_hook_base_mpi_finalized_top(int *flag)
+{
+    HOOK_CALL_COMMON( mpi_finalized_top, flag );
+}
+
+void ompi_hook_base_mpi_finalized_bottom(int *flag)
+{
+    HOOK_CALL_COMMON( mpi_finalized_bottom, flag );
+}
+
+
+void ompi_hook_base_mpi_init_top(int argc, char **argv, int requested, int *provided)
+{
+    HOOK_CALL_COMMON( mpi_init_top, argc, argv, requested, provided);
+}
+
+void ompi_hook_base_mpi_init_top_post_opal(int argc, char **argv, int requested, int *provided)
+{
+    HOOK_CALL_COMMON( mpi_init_top_post_opal, argc, argv, requested, provided);
+}
+
+void ompi_hook_base_mpi_init_bottom(int argc, char **argv, int requested, int *provided)
+{
+    HOOK_CALL_COMMON( mpi_init_bottom, argc, argv, requested, provided);
+}
+
+void ompi_hook_base_mpi_init_error(int argc, char **argv, int requested, int *provided)
+{
+    HOOK_CALL_COMMON( mpi_init_error, argc, argv, requested, provided);
+}
+
+
+void ompi_hook_base_mpi_finalize_top(void)
+{
+    HOOK_CALL_COMMON( mpi_finalize_top, );
+}
+
+void ompi_hook_base_mpi_finalize_bottom(void)
+{
+    HOOK_CALL_COMMON( mpi_finalize_bottom, );
+}

ompi/mca/hook/base/owner.txt

@@ -0,0 +1,7 @@
+#
+# owner/status file
+# owner: institution that is responsible for this package
+# status: e.g. active, maintenance, unmaintained
+#
+owner: project
+status: active

ompi/mca/hook/configure.m4

@@ -0,0 +1,14 @@
+#
+# Copyright (c) 2017      IBM Corporation.  All rights reserved.
+#
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+AC_DEFUN([MCA_ompi_hook_CONFIG],[
+    # configure all the components
+    MCA_CONFIGURE_FRAMEWORK($1, $2, 1)
+])

ompi/mca/hook/demo/Makefile.am

@@ -0,0 +1,20 @@
+#
+# Copyright (c) 2017      IBM Corporation.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+sources = \
+	hook_demo.h \
+	hook_demo_component.c \
+	hook_demo_fns.c
+
+# This component will only ever be built statically -- never as a DSO.
+
+noinst_LTLIBRARIES = libmca_hook_demo.la
+
+libmca_hook_demo_la_SOURCES = $(sources)
+libmca_hook_demo_la_LDFLAGS = -module -avoid-version

ompi/mca/hook/demo/configure.m4

@@ -0,0 +1,25 @@
+#
+# Copyright (c) 2017      IBM Corporation.  All rights reserved.
+#
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# Make this a static component
+AC_DEFUN([MCA_ompi_hook_demo_COMPILE_MODE], [
+    AC_MSG_CHECKING([for MCA component $2:$3 compile mode])
+    $4="static"
+    AC_MSG_RESULT([$$4])
+])
+
+# MCA_hook_demo_CONFIG([action-if-can-compile],
+#                      [action-if-cant-compile])
+# ------------------------------------------------
+AC_DEFUN([MCA_ompi_hook_demo_CONFIG],[
+    AC_CONFIG_FILES([ompi/mca/hook/demo/Makefile])
+
+    $1
+])

ompi/mca/hook/demo/hook_demo.h

@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2017      IBM Corporation.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#ifndef MCA_HOOK_DEMO_H
+#define MCA_HOOK_DEMO_H
+
+#include "ompi_config.h"
+
+#include "ompi/constants.h"
+
+#include "opal/util/output.h"
+
+#include "ompi/mca/hook/hook.h"
+#include "ompi/mca/hook/base/base.h"
+
+BEGIN_C_DECLS
+
+OMPI_MODULE_DECLSPEC extern const ompi_hook_base_component_1_0_0_t mca_hook_demo_component;
+
+void ompi_hook_demo_mpi_initialized_top(int *flag);
+void ompi_hook_demo_mpi_initialized_bottom(int *flag);
+
+void ompi_hook_demo_mpi_finalized_top(int *flag);
+void ompi_hook_demo_mpi_finalized_bottom(int *flag);
+
+void ompi_hook_demo_mpi_init_top(int argc, char **argv, int requested, int *provided);
+void ompi_hook_demo_mpi_init_top_post_opal(int argc, char **argv, int requested, int *provided);
+void ompi_hook_demo_mpi_init_bottom(int argc, char **argv, int requested, int *provided);
+void ompi_hook_demo_mpi_init_error(int argc, char **argv, int requested, int *provided);
+
+void ompi_hook_demo_mpi_finalize_top(void);
+void ompi_hook_demo_mpi_finalize_bottom(void);
+
+void ompi_hook_demo_extra_mpi_init_bottom(int argc, char **argv, int requested, int *provided);
+
+END_C_DECLS
+
+#endif /* MCA_HOOK_DEMO_H */

ompi/mca/hook/demo/hook_demo_component.c

@@ -0,0 +1,107 @@
+/*
+ * Copyright (c) 2017      IBM Corporation.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "hook_demo.h"
+
+static int ompi_hook_demo_component_open(void);
+static int ompi_hook_demo_component_close(void);
+static int ompi_hook_demo_component_register(void);
+
+/*
+ * Public string showing the component version number
+ */
+const char *mca_hook_demo_component_version_string =
+    "Open MPI 'demo' hook MCA component version " OMPI_VERSION;
+
+/*
+ * Instantiate the public struct with all of our public information
+ * and pointers to our public functions in it
+ */
+const ompi_hook_base_component_1_0_0_t mca_hook_demo_component = {
+
+    /* First, the mca_component_t struct containing meta information
+     * about the component itself */
+    .hookm_version = {
+        OMPI_HOOK_BASE_VERSION_1_0_0,
+
+        /* Component name and version */
+        .mca_component_name = "demo",
+        MCA_BASE_MAKE_VERSION(component, OMPI_MAJOR_VERSION, OMPI_MINOR_VERSION,
+                              OMPI_RELEASE_VERSION),
+
+        /* Component open and close functions */
+        .mca_open_component = ompi_hook_demo_component_open,
+        .mca_close_component = ompi_hook_demo_component_close,
+        .mca_register_component_params = ompi_hook_demo_component_register,
+
+        // Force this component to always be considered - component must be static
+        .mca_component_flags = MCA_BASE_COMPONENT_FLAG_REQUIRED,
+    },
+    .hookm_data = {
+        /* The component is checkpoint ready */
+        MCA_BASE_METADATA_PARAM_CHECKPOINT
+    },
+
+    /* Component functions */
+    .hookm_mpi_initialized_top = ompi_hook_demo_mpi_initialized_top,
+    .hookm_mpi_initialized_bottom = ompi_hook_demo_mpi_initialized_bottom,
+
+    .hookm_mpi_finalized_top = ompi_hook_demo_mpi_finalized_top,
+    .hookm_mpi_finalized_bottom = ompi_hook_demo_mpi_finalized_bottom,
+
+    .hookm_mpi_init_top = ompi_hook_demo_mpi_init_top,
+    .hookm_mpi_init_top_post_opal = ompi_hook_demo_mpi_init_top_post_opal,
+    .hookm_mpi_init_bottom = ompi_hook_demo_mpi_init_bottom,
+    .hookm_mpi_init_error = ompi_hook_demo_mpi_init_error,
+
+    .hookm_mpi_finalize_top = ompi_hook_demo_mpi_finalize_top,
+    .hookm_mpi_finalize_bottom = ompi_hook_demo_mpi_finalize_bottom,
+};
+
+/*
+ * Example 'extra' component with an additional callback that is dynamically
+ * registered at runtime in addition to the callbacks that are a part of the
+ * component structure.
+ */
+ompi_hook_base_component_1_0_0_t hook_demo_extra_component = {
+    /* Component functions */
+    .hookm_mpi_init_bottom = ompi_hook_demo_extra_mpi_init_bottom,
+};
+
+static int ompi_hook_demo_component_open(void)
+{
+    opal_output(0, "hook/demo: component_open()");
+
+    // Register the 'extra' callback(s) to be called the next time those
+    // functions are encountered.
+    ompi_hook_base_register_callbacks( &hook_demo_extra_component );
+
+    return OMPI_SUCCESS;
+}
+
+static int ompi_hook_demo_component_close(void)
+{
+    opal_output(0, "hook/demo: component_close()");
+
+    // Deregister the 'extra' callback(s) so that they are no longer called.
+    // Must pass in the same 'component' structure that was passed to the
+    // ompi_hook_base_register_callbacks() earlier.
+    ompi_hook_base_deregister_callbacks( &hook_demo_extra_component );
+
+    return OMPI_SUCCESS;
+}
+
+static int ompi_hook_demo_component_register(void)
+{
+    opal_output(0, "hook/demo: component_register()");
+    return OMPI_SUCCESS;
+}
+

ompi/mca/hook/demo/hook_demo_fns.c

@@ -0,0 +1,65 @@
+/*
+ * Copyright (c) 2017      IBM Corporation.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "hook_demo.h"
+
+#define DEBUG_OUTPUT( fn_name )                     \
+    do {                                            \
+        opal_output(0, "hook:demo: %s", __func__ ); \
+    } while(0)
+
+void ompi_hook_demo_mpi_initialized_top(int *flag) {
+    DEBUG_OUTPUT();
+}
+
+void ompi_hook_demo_mpi_initialized_bottom(int *flag) {
+    DEBUG_OUTPUT();
+}
+
+
+void ompi_hook_demo_mpi_finalized_top(int *flag) {
+    DEBUG_OUTPUT();
+}
+
+void ompi_hook_demo_mpi_finalized_bottom(int *flag) {
+    DEBUG_OUTPUT();
+}
+
+
+void ompi_hook_demo_mpi_init_top(int argc, char **argv, int requested, int *provided) {
+    DEBUG_OUTPUT();
+}
+
+void ompi_hook_demo_mpi_init_top_post_opal(int argc, char **argv, int requested, int *provided) {
+    DEBUG_OUTPUT();
+}
+
+void ompi_hook_demo_mpi_init_bottom(int argc, char **argv, int requested, int *provided) {
+    DEBUG_OUTPUT();
+}
+
+void ompi_hook_demo_mpi_init_error(int argc, char **argv, int requested, int *provided) {
+    DEBUG_OUTPUT();
+}
+
+
+void ompi_hook_demo_mpi_finalize_top(void) {
+    DEBUG_OUTPUT();
+}
+
+void ompi_hook_demo_mpi_finalize_bottom(void) {
+    DEBUG_OUTPUT();
+}
+
+
+void ompi_hook_demo_extra_mpi_init_bottom(int argc, char **argv, int requested, int *provided) {
+    DEBUG_OUTPUT();
+}

ompi/mca/hook/demo/owner.txt

@@ -0,0 +1,7 @@
+#
+# owner/status file
+# owner: institution that is responsible for this package
+# status: e.g. active, maintenance, unmaintained
+#
+owner: project
+status: maintenance

ompi/mca/hook/hook.h

@@ -0,0 +1,219 @@
+/*
+ * Copyright (c) 2017      IBM Corporation.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+/**
+ * @file
+ *
+ * The hook framework is designed to allow a component of the hook framework
+ * to be called a designated points in the MPI process lifecycle.
+ *
+ * There is not module structure in this framework since the components apply to
+ * the whole process. Further, the component might need to active before mca_init.
+ *
+ * Why not a PMPI library?
+ *  - The desire is to allow a component to be built statically into the library
+ *    for licensing and disgnostic purposes. As such we need the ability for
+ *    the component to be statically linked into the libmpi library.
+ *
+ * Why is there not a hook for location XYZ?
+ *  - The number of possible places for hooks is quite large. This framework
+ *    supports a set that we think will provide the best coverage for the
+ *    common use cases. If you have another use case we are always open to
+ *    discussing extensions on the mailing list.
+ */
+#ifndef MCA_HOOK_H
+#define MCA_HOOK_H
+
+#include "ompi_config.h"
+
+#include "mpi.h"
+#include "ompi/mca/mca.h"
+#include "opal/mca/base/base.h"
+
+BEGIN_C_DECLS
+
+/*
+ * Some functions are specially marked. See decoder below.
+ *
+ * *** Static Only (Always) ***
+ *  Only static components will ever see this callback triggered due to the
+ *  position of the hook relative to ompi_mpi_init and ompi_mpi_finalize.
+ *
+ * *** Static Only (Outside MPI), Everyone (Inside MPI) ***
+ *  Only static components will ever see this callback triggered when outside
+ *  of the ompi_mpi_init and ompi_mpi_finalize region. Others may see this
+ *  triggered if the user's program makes additional calls while inside
+ *  that region.
+ *
+ * *** Everyone (Inside MPI) ***
+ *  Everyone registered will see this callback triggered. It is only triggered
+ *  between ompi_mpi_init and ompi_mpi_finalize.
+ * 
+ */
+/* ******************************************************************** */
+
+/**
+ * *** Static Only (Outside MPI), Everyone (Inside MPI) ***
+ *
+ * Location: mpi/c/initialized.c
+ *           At the top of function (outside of mutex)
+ */
+typedef void (*ompi_hook_base_component_mpi_initialized_top_fn_t)(int *flag);
+
+/**
+ * *** Static Only (Outside MPI), Everyone (Inside MPI) ***
+ *
+ * Location: mpi/c/initialized.c
+ *           At the bottom of function (outside of mutex)
+ */
+typedef void (*ompi_hook_base_component_mpi_initialized_bottom_fn_t)(int *flag);
+
+/**
+ * *** Static Only (Outside MPI), Everyone (Inside MPI) ***
+ *
+ * Location: mpi/c/init_thread.c
+ *           At the top of function
+ */
+typedef void (*ompi_hook_base_component_mpi_init_thread_top_fn_t)(int *argc, char ***argv, int required, int *provided);
+
+/**
+ * *** Static Only (Outside MPI), Everyone (Inside MPI) ***
+ *
+ * Location: mpi/c/init_thread.c
+ *           At the bottom of function
+ */
+typedef void (*ompi_hook_base_component_mpi_init_thread_bottom_fn_t)(int *argc, char ***argv, int required, int *provided);
+
+/**
+ * *** Static Only (Outside MPI), Everyone (Inside MPI) ***
+ *
+ * Location: mpi/c/finalized.c
+ *           At the top of function (outside of mutex)
+ */
+typedef void (*ompi_hook_base_component_mpi_finalized_top_fn_t)(int *flag);
+
+/**
+ * *** Static Only (Outside MPI), Everyone (Inside MPI) ***
+ *
+ * Location: mpi/c/finalized.c
+ *           At the bottom of function (outside of mutex)
+ */
+typedef void (*ompi_hook_base_component_mpi_finalized_bottom_fn_t)(int *flag);
+
+/**
+ * *** Static Only (Always) ***
+ *
+ * Location: runtime/ompi_mpi_init.c
+ *           At top of function (outside of mutex)
+ */
+typedef void (*ompi_hook_base_component_mpi_init_top_fn_t)(int argc, char **argv, int requested, int *provided);
+
+/**
+ * *** Everyone (Inside MPI) ***
+ *
+ * Location: runtime/ompi_mpi_init.c
+ *           Just after opal_init_util and MCA initialization. (inside mutex)
+ * Notes:
+ *           This framework has been opened as have its components.
+ */
+typedef void (*ompi_hook_base_component_mpi_init_top_post_opal_fn_t)(int argc, char **argv, int requested, int *provided);
+
+/**
+ * *** Everyone (Inside MPI) ***
+ *
+ * Location: runtime/ompi_mpi_init.c
+ *           At the bottom of the function. (outside mutex)
+ * Notes:
+ *           This framework has been opened as have its components.
+ *           Can safely use all MPI functionality.
+ */
+typedef void (*ompi_hook_base_component_mpi_init_bottom_fn_t)(int argc, char **argv, int requested, int *provided);
+
+/**
+ * *** Everyone (Inside MPI) ***
+ *
+ * Location: runtime/ompi_mpi_init.c
+ *           At the bottom of the error path. (outside mutex)
+ * Notes:
+ *           This framework has been opened as have its components.
+ */
+typedef void (*ompi_hook_base_component_mpi_init_error_fn_t)(int argc, char **argv, int requested, int *provided);
+
+/**
+ * *** Everyone (Inside MPI) ***
+ *
+ * Location: runtime/ompi_mpi_finalize.c
+ *           At the top of the function. (outside mutex)
+ * Notes:
+ *           This framework has been opened as have its components.
+ *           Can safely use all MPI functionality.
+ */
+typedef void (*ompi_hook_base_component_mpi_finalize_top_fn_t)(void);
+
+/**
+ * *** Static Only (Always) ***
+ *
+ * Location: runtime/ompi_mpi_finalize.c
+ *           At top of function (outside of mutex)
+ * Notes:
+ *           This framework has been closed.
+ */
+typedef void (*ompi_hook_base_component_mpi_finalize_bottom_fn_t)(void);
+
+
+/* ******************************************************************** */
+
+/**
+ * Hook component version and interface functions.
+ */
+struct ompi_hook_base_component_1_0_0_t {
+    mca_base_component_t hookm_version;
+    mca_base_component_data_t hookm_data;
+
+    /* MPI_Initialized */
+    ompi_hook_base_component_mpi_initialized_top_fn_t hookm_mpi_initialized_top;
+    ompi_hook_base_component_mpi_initialized_bottom_fn_t hookm_mpi_initialized_bottom;
+
+    /* MPI_Init_thread */
+    ompi_hook_base_component_mpi_init_thread_top_fn_t hookm_mpi_init_thread_top;
+    ompi_hook_base_component_mpi_init_thread_bottom_fn_t hookm_mpi_init_thread_bottom;
+
+    /* MPI_Finalized */
+    ompi_hook_base_component_mpi_finalized_top_fn_t hookm_mpi_finalized_top;
+    ompi_hook_base_component_mpi_finalized_bottom_fn_t hookm_mpi_finalized_bottom;
+
+    /* ompi_mpi_init */
+    ompi_hook_base_component_mpi_init_top_fn_t hookm_mpi_init_top;
+    ompi_hook_base_component_mpi_init_top_post_opal_fn_t hookm_mpi_init_top_post_opal;
+    ompi_hook_base_component_mpi_init_bottom_fn_t hookm_mpi_init_bottom;
+    ompi_hook_base_component_mpi_init_error_fn_t hookm_mpi_init_error;
+
+    /* ompi_mpi_finalize */
+    ompi_hook_base_component_mpi_finalize_top_fn_t hookm_mpi_finalize_top;
+    ompi_hook_base_component_mpi_finalize_bottom_fn_t hookm_mpi_finalize_bottom;
+};
+typedef struct ompi_hook_base_component_1_0_0_t ompi_hook_base_component_1_0_0_t;
+typedef ompi_hook_base_component_1_0_0_t ompi_hook_base_component_t;
+/* 
+ * Note: We do -not- expose a component object for this framework.
+ * All interation with the component should go through the base/base.h interfaces.
+ * See that header for more information on calling functions.
+ */
+
+/* ******************************************************************** */
+
+/*
+ * Macro for use in components that are of type hook
+ */
+#define OMPI_HOOK_BASE_VERSION_1_0_0 \
+    OMPI_MCA_BASE_VERSION_2_1_0("hook", 1, 0, 0)
+
+END_C_DECLS
+
+#endif /* MCA_HOOK_H */

ompi/mpi/c/finalized.c

@@ -26,6 +26,7 @@
 #include "ompi/runtime/params.h"
 #include "ompi/communicator/communicator.h"
 #include "ompi/errhandler/errhandler.h"
+#include "ompi/mca/hook/base/base.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -41,6 +42,8 @@ int MPI_Finalized(int *flag)
 {
     OPAL_CR_NOOP_PROGRESS();
 
+    ompi_hook_base_mpi_finalized_top(flag);
+
     /* We must obtain the lock to guarnatee consistent values of
        ompi_mpi_initialized and ompi_mpi_finalized.  Note, too, that
        this lock is held for the bulk of the duration of
@@ -74,5 +77,7 @@ int MPI_Finalized(int *flag)
     *flag = ompi_mpi_finalized;
     opal_mutex_unlock(&ompi_mpi_bootstrap_mutex);
 
+    ompi_hook_base_mpi_finalized_bottom(flag);
+
     return MPI_SUCCESS;
 }

ompi/mpi/c/init_thread.c

@@ -31,6 +31,7 @@
 #include "ompi/communicator/communicator.h"
 #include "ompi/errhandler/errhandler.h"
 #include "ompi/constants.h"
+#include "ompi/mca/hook/base/base.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -47,6 +48,8 @@ int MPI_Init_thread(int *argc, char ***argv, int required,
 {
     int err;
 
+    ompi_hook_base_mpi_init_thread_top(argc, argv, required, provided);
+
     if ( MPI_PARAM_CHECK ) {
         if (required < MPI_THREAD_SINGLE || required > MPI_THREAD_MULTIPLE) {
             ompi_mpi_errors_are_fatal_comm_handler(NULL, NULL, FUNC_NAME);
@@ -78,5 +81,7 @@ int MPI_Init_thread(int *argc, char ***argv, int required,
 
     OPAL_CR_INIT_LIBRARY();
 
+    ompi_hook_base_mpi_init_thread_bottom(argc, argv, required, provided);
+
     return MPI_SUCCESS;
 }

ompi/mpi/c/initialized.c

@@ -26,6 +26,7 @@
 #include "ompi/runtime/params.h"
 #include "ompi/communicator/communicator.h"
 #include "ompi/errhandler/errhandler.h"
+#include "ompi/mca/hook/base/base.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -41,6 +42,8 @@ int MPI_Initialized(int *flag)
 {
     OPAL_CR_NOOP_PROGRESS();
 
+    ompi_hook_base_mpi_initialized_top(flag);
+
     /* We must obtain the lock to guarnatee consistent values of
        ompi_mpi_initialized and ompi_mpi_finalized.  Note, too, that
        this lock is held for the bulk of the duration of
@@ -74,5 +77,7 @@ int MPI_Initialized(int *flag)
     *flag = ompi_mpi_initialized;
     opal_mutex_unlock(&ompi_mpi_bootstrap_mutex);
 
+    ompi_hook_base_mpi_initialized_bottom(flag);
+
     return MPI_SUCCESS;
 }

ompi/runtime/ompi_mpi_finalize.c

@@ -82,6 +82,7 @@
 #include "ompi/runtime/params.h"
 #include "ompi/dpm/dpm.h"
 #include "ompi/mpiext/mpiext.h"
+#include "ompi/mca/hook/base/base.h"
 
 #if OPAL_ENABLE_FT_CR == 1
 #include "ompi/mca/crcp/crcp.h"
@@ -110,6 +111,8 @@ int ompi_mpi_finalize(void)
     OPAL_TIMING_DECLARE(tm);
     OPAL_TIMING_INIT_EXT(&tm, OPAL_TIMING_GET_TIME_OF_DAY);
 
+    ompi_hook_base_mpi_finalize_top();
+
     /* Be a bit social if an erroneous program calls MPI_FINALIZE in
        two different threads, otherwise we may deadlock in
        ompi_comm_free() (or run into other nasty lions, tigers, or
@@ -487,6 +490,12 @@ int ompi_mpi_finalize(void)
         goto done;
     }
 
+    /* Now close the hook framework */
+    if (OMPI_SUCCESS != (ret = mca_base_framework_close(&ompi_hook_base_framework) ) ) {
+        OMPI_ERROR_LOG(ret);
+        goto done;
+    }
+
     if (OPAL_SUCCESS != (ret = opal_finalize_util())) {
         goto done;
     }
@@ -506,6 +515,7 @@ int ompi_mpi_finalize(void)
 
  done:
     opal_mutex_unlock(&ompi_mpi_bootstrap_mutex);
+    ompi_hook_base_mpi_finalize_bottom();
 
     return ret;
 }

ompi/runtime/ompi_mpi_init.c

@@ -92,6 +92,7 @@
 #include "ompi/mca/pml/base/pml_base_bsend.h"
 #include "ompi/dpm/dpm.h"
 #include "ompi/mpiext/mpiext.h"
+#include "ompi/mca/hook/base/base.h"
 
 #if OPAL_ENABLE_FT_CR == 1
 #include "ompi/mca/crcp/crcp.h"
@@ -388,6 +389,8 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided)
      * for the modex in order to work in heterogeneous environments. */
     uint8_t threadlevel_bf;
 
+    ompi_hook_base_mpi_init_top(argc, argv, requested, provided);
+
     /* Ensure that we were not already initialized or finalized.
 
        This lock is held for the duration of ompi_mpi_init() and
@@ -474,6 +477,15 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided)
         mca_base_var_set_value(ret, allvalue, 4, MCA_BASE_VAR_SOURCE_DEFAULT, NULL);
     }
 
+    /* open the ompi hook framework */
+    if (OMPI_SUCCESS != (ret = mca_base_framework_open(&ompi_hook_base_framework, 0))) {
+        error = "ompi_hook_base_open() failed";
+        goto error;
+    }
+
+    ompi_hook_base_mpi_init_top_post_opal(argc, argv, requested, provided);
+
+
     OPAL_TIMING_MSTART((&tm,"time from start to completion of rte_init"));
 
     /* if we were not externally started, then we need to setup
@@ -939,6 +951,7 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided)
                            "MPI_INIT", "MPI_INIT", error, err_msg, ret);
         }
         opal_mutex_unlock(&ompi_mpi_bootstrap_mutex);
+        ompi_hook_base_mpi_init_error(argc, argv, requested, provided);
         return ret;
     }
 
@@ -969,5 +982,8 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided)
     OPAL_TIMING_RELEASE(&tm);
 
     opal_mutex_unlock(&ompi_mpi_bootstrap_mutex);
+
+    ompi_hook_base_mpi_init_bottom(argc, argv, requested, provided);
+
     return MPI_SUCCESS;
 }

opal/mca/mca.h

@@ -11,8 +11,9 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2008-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
+ * Copyright (c) 2015-2016 Los Alamos National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2016-2017 IBM Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -256,6 +257,19 @@ typedef int (*mca_base_register_component_params_2_0_0_fn_t)(void);
  */
 #define MCA_BASE_MAX_COMPONENT_NAME_LEN 63
 
+/**
+ * Component flags (mca_component_flags field)
+ */
+enum {
+    /** Always consider this component for selection. For this flag to
+     * work properly the component must always be built statically.
+     *
+     * Enforecement of this flag is left to the individual frameworks
+     * so as to limit its use. See discussion from the Open MPI
+     * face-to-face meeting Jan. 2017 */
+    MCA_BASE_COMPONENT_FLAG_REQUIRED = 1,
+};
+
 /**
  * Common type for all MCA components.
  *
@@ -315,9 +329,12 @@ struct mca_base_component_2_1_0_t {
   mca_base_register_component_params_2_0_0_fn_t mca_register_component_params;
   /**< Method for registering the component's MCA parameters */
 
+  int32_t mca_component_flags;
+  /**< flags for this component */
+
   /** Extra space to allow for expansion in the future without
       breaking older components. */
-  char reserved[32];
+  char reserved[28];
 };
 /** Unversioned convenience typedef; use this name in
     frameworks/components to stay forward source-compatible */