dl framework: new dynamic loader framework

Embedding libltdl without the use of Libtool bootstrapping has
proven... difficult.  Instead, create a new simple "dl" framework.  It
only provides 4 functions:

- open a DSO (very similar to lt_dlopenadvise())
- lookup a symbol in a previously-opened DSO (very similar to lt_dlsym())
- close a previously-opened DSO (very similar to lt_dlclose())
- iterate over all files in a directory (very similar to ld_dlforeachfile())

There will be follow-on commits with a simple dlopen-based component
(nowhere near as complete/functional as libltdl, but good enough for
Linux and OS X), and a libltdl-based component for all other
platforms.

The intent is that the dlopen-based component can be built by default
in almost all cases.  But if libltdl is available, that component will
be built.  End result: we still get DSO-based functionality by default
in (almost?) all cases.  Without embedding libltdl.  Which is what we
want.

README

@@ -8,7 +8,7 @@ Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
                         University of Stuttgart.  All rights reserved.
 Copyright (c) 2004-2007 The Regents of the University of California.
                         All rights reserved.
-Copyright (c) 2006-2014 Cisco Systems, Inc.  All rights reserved.
+Copyright (c) 2006-2015 Cisco Systems, Inc.  All rights reserved.
 Copyright (c) 2006-2011 Mellanox Technologies. All rights reserved.
 Copyright (c) 2006-2012 Oracle and/or its affiliates.  All rights reserved.
 Copyright (c) 2007      Myricom, Inc.  All rights reserved.
@@ -1977,6 +1977,7 @@ backtrace   - Debugging call stack backtrace support
 compress    - Compression algorithms
 crs         - Checkpoint and restart service
 db          - Internal database support
+dl          - Dynamic loading library interface
 event       - Event library (libevent) versioning support
 hwloc       - Hardware locality (hwloc) versioning support
 if          - OS IP interface support

opal/mca/dl/Makefile.am

@@ -0,0 +1,36 @@
+#
+# Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2010-2015 Cisco Systems, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# main library setup
+noinst_LTLIBRARIES = libmca_dl.la
+libmca_dl_la_SOURCES =
+
+# local files
+headers = dl.h
+libmca_dl_la_SOURCES += $(headers)
+
+# Ensure that the man pages are rebuilt if the opal_config.h file
+# changes; a "good enough" way to know if configure was run again (and
+# therefore the release date or version may have changed)
+$(nodist_man_MANS): $(top_builddir)/opal/include/opal_config.h
+
+# Conditionally install the header files
+if WANT_INSTALL_HEADERS
+opaldir = $(opalincludedir)/$(subdir)
+nobase_opal_HEADERS = $(headers)
+endif
+
+include base/Makefile.am
+
+distclean-local:
+	rm -f base/static-components.h
+	rm -f $(nodist_man_MANS)

opal/mca/dl/base/Makefile.am

@@ -0,0 +1,17 @@
+#
+# Copyright (c) 2015 Cisco Systems, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+headers += \
+        base/base.h
+
+libmca_dl_la_SOURCES += \
+        base/dl_base_open.c \
+        base/dl_base_close.c \
+        base/dl_base_select.c \
+        base/dl_base_fns.c

opal/mca/dl/base/base.h

@@ -0,0 +1,106 @@
+/*
+ * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2015 Cisco Systems, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef OPAL_DL_BASE_H
+#define OPAL_DL_BASE_H
+
+#include "opal_config.h"
+#include "opal/mca/dl/dl.h"
+#include "opal/util/opal_environ.h"
+#include "opal/runtime/opal_cr.h"
+
+#include "opal/mca/base/base.h"
+
+
+BEGIN_C_DECLS
+
+/**
+ * Globals
+ */
+OPAL_DECLSPEC extern mca_base_framework_t opal_dl_base_framework;
+OPAL_DECLSPEC extern opal_dl_base_component_t
+*opal_dl_base_selected_component;
+OPAL_DECLSPEC extern opal_dl_base_module_t *opal_dl;
+
+
+/**
+ * Initialize the DL MCA framework
+ *
+ * @retval OPAL_SUCCESS Upon success
+ * @retval OPAL_ERROR   Upon failures
+ *
+ * This function is invoked during opal_init();
+ */
+OPAL_DECLSPEC int opal_dl_base_open(mca_base_open_flag_t flags);
+
+/**
+ * Select an available component.
+ *
+ * @retval OPAL_SUCCESS Upon Success
+ * @retval OPAL_NOT_FOUND If no component can be selected
+ * @retval OPAL_ERROR Upon other failure
+ *
+ */
+OPAL_DECLSPEC int opal_dl_base_select(void);
+
+/**
+ * Finalize the DL MCA framework
+ *
+ * @retval OPAL_SUCCESS Upon success
+ * @retval OPAL_ERROR   Upon failures
+ *
+ * This function is invoked during opal_finalize();
+ */
+OPAL_DECLSPEC int opal_dl_base_close(void);
+
+/**
+ * Open a DSO
+ *
+ * (see opal_dl_base_module_open_ft_t in opal/mca/dl/dl.h for
+ * documentation of this function)
+ */
+OPAL_DECLSPEC int opal_dl_open(const char *fname,
+                               bool use_ext, bool private_namespace,
+                               opal_dl_handle_t **handle, char **err_msg);
+
+/**
+ * Lookup a symbol in a DSO
+ *
+ * (see opal_dl_base_module_lookup_ft_t in opal/mca/dl/dl.h for
+ * documentation of this function)
+ */
+OPAL_DECLSPEC int opal_dl_lookup(opal_dl_handle_t *handle,
+                                 const char *symbol,
+                                 void **ptr, char **err_msg);
+
+/**
+ * Close a DSO
+ *
+ * (see opal_dl_base_module_close_ft_t in opal/mca/dl/dl.h for
+ * documentation of this function)
+ */
+OPAL_DECLSPEC int opal_dl_close(opal_dl_handle_t *handle);
+
+/**
+ * Iterate over files in a path
+ *
+ * (see opal_dl_base_module_foreachfile_ft_t in opal/mca/dl/dl.h for
+ * documentation of this function)
+ */
+OPAL_DECLSPEC int opal_dl_foreachfile(const char *search_path,
+                                      int (*cb_func)(const char *filename,
+                                                     void *context),
+                                      void *context);
+
+END_C_DECLS
+
+#endif /* OPAL_DL_BASE_H */

opal/mca/dl/base/dl_base_close.c

@@ -0,0 +1,25 @@
+/*
+ * Copyright (c) 2004-2010 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2015 Cisco Systems, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "opal_config.h"
+
+#include "opal/mca/mca.h"
+#include "opal/mca/base/base.h"
+
+#include "opal/mca/dl/dl.h"
+#include "opal/mca/dl/base/base.h"
+
+
+int opal_dl_base_close(void)
+{
+    /* Close all available modules that are open */
+    return mca_base_framework_components_close(&opal_dl_base_framework, NULL);
+}

opal/mca/dl/base/dl_base_fns.c

@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2004-2010 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2015 Cisco Systems, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+/**
+ * This file is a simple set of wrappers around the selected OPAL DL
+ * component (it's a compile-time framework with, at most, a single
+ * component; see dl.h for details).
+ */
+
+#include "opal_config.h"
+
+#include "opal/include/opal/constants.h"
+
+#include "opal/mca/dl/base/base.h"
+
+
+int opal_dl_open(const char *fname,
+                 bool use_ext, bool private_namespace,
+                 opal_dl_handle_t **handle, char **err_msg)
+{
+    *handle = NULL;
+
+    if (NULL != opal_dl && NULL != opal_dl->open) {
+        return opal_dl->open(fname, use_ext, private_namespace,
+                             handle, err_msg);
+    }
+
+    return OPAL_ERR_NOT_SUPPORTED;
+}
+
+int opal_dl_lookup(opal_dl_handle_t *handle,
+                   const char *symbol,
+                   void **ptr, char **err_msg)
+{
+    if (NULL != opal_dl && NULL != opal_dl->lookup) {
+        return opal_dl->lookup(handle, symbol, ptr, err_msg);
+    }
+
+    return OPAL_ERR_NOT_SUPPORTED;
+}
+
+int opal_dl_close(opal_dl_handle_t *handle)
+{
+    if (NULL != opal_dl && NULL != opal_dl->close) {
+        return opal_dl->close(handle);
+    }
+
+    return OPAL_ERR_NOT_SUPPORTED;
+}
+
+int opal_dl_foreachfile(const char *search_path,
+                        int (*cb_func)(const char *filename, void *context),
+                        void *context)
+{
+    if (NULL != opal_dl && NULL != opal_dl->foreachfile) {
+        return opal_dl->foreachfile(search_path, cb_func, context);
+    }
+
+    return OPAL_ERR_NOT_SUPPORTED;
+}

opal/mca/dl/base/dl_base_open.c

@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2004-2010 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2013 Los Alamos National Security, LLC.
+ *                         All rights reserved.
+ * Copyright (c) 2015 Cisco Systems, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "opal_config.h"
+
+#include "opal/mca/dl/base/base.h"
+
+#include "opal/mca/dl/base/static-components.h"
+
+
+/*
+ * Globals
+ */
+opal_dl_base_module_t *opal_dl = NULL;
+opal_dl_base_component_t *opal_dl_base_selected_component = NULL;
+
+
+/*
+ * Function for finding and opening either all MCA components,
+ * or the one that was specifically requested via a MCA parameter.
+ *
+ * Note that we really don't need this function -- we could specify a
+ * NULL pointer in the framework declare and the base would do this
+ * exact same thing.  However, we need to have at least some
+ * executable code in this file, or some linkers (cough cough OS X
+ * cough cough) may not actually link in this .o file.
+ */
+int opal_dl_base_open(mca_base_open_flag_t flags)
+{
+    /* Open up all available components */
+    return mca_base_framework_components_open(&opal_dl_base_framework, flags);
+}
+
+/* VERY IMPORTANT: This framework is static, and is opened before any
+   other dyanmic frameworks are opened (which makes sense, of course).
+   But we must mark this framework is NO_DSO so that the MCA framework
+   base doesn't try to open any dynamic components in this
+   framework. */
+MCA_BASE_FRAMEWORK_DECLARE(opal, dl, "Dynamic loader framework",
+                           NULL /* register */,
+                           opal_dl_base_open /* open */,
+                           NULL /* close */,
+                           mca_dl_base_static_components,
+                           MCA_BASE_FRAMEWORK_FLAG_NO_DSO);

opal/mca/dl/base/dl_base_select.c

@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2004-2010 The Trustees of Indiana University.
+ *                         All rights reserved.
+ *
+ * Copyright (c) 2015 Cisco Systems, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "opal_config.h"
+
+#ifdef HAVE_UNISTD_H
+#include "unistd.h"
+#endif
+
+#include "opal/include/opal/constants.h"
+#include "opal/util/output.h"
+#include "opal/mca/mca.h"
+#include "opal/mca/base/base.h"
+#include "opal/mca/dl/dl.h"
+#include "opal/mca/dl/base/base.h"
+
+
+int opal_dl_base_select(void)
+{
+    int exit_status = OPAL_SUCCESS;
+    opal_dl_base_component_t *best_component = NULL;
+    opal_dl_base_module_t *best_module = NULL;
+
+    /*
+     * Select the best component
+     */
+    if (OPAL_SUCCESS != mca_base_select("dl",
+                                        opal_dl_base_framework.framework_output,
+                                        &opal_dl_base_framework.framework_components,
+                                        (mca_base_module_t **) &best_module,
+                                        (mca_base_component_t **) &best_component) ) {
+        /* This will only happen if no component was selected */
+        exit_status = OPAL_ERROR;
+        goto cleanup;
+    }
+
+    /* Save the winner */
+    opal_dl_base_selected_component = best_component;
+    opal_dl = best_module;
+
+ cleanup:
+    return exit_status;
+}

opal/mca/dl/configure.m4

@@ -0,0 +1,77 @@
+dnl -*- shell-script -*-
+dnl
+dnl Copyright (c) 2010-2015 Cisco Systems, Inc.  All rights reserved.
+dnl $COPYRIGHT$
+dnl
+dnl Additional copyrights may follow
+dnl
+dnl $HEADER$
+dnl
+
+dnl There will only be one component used in this framework, and it will
+dnl be selected at configure time by priority.  Components must set
+dnl their priorities in their configure.m4 file.
+
+dnl We only want one winning component (vs. STOP_AT_FIRST_PRIORITY,
+dnl which will allow all components of the same priority who succeed to
+dnl win)
+m4_define(MCA_opal_dl_CONFIGURE_MODE, STOP_AT_FIRST)
+
+AC_DEFUN([MCA_opal_dl_CONFIG],[
+    OPAL_HAVE_DL_SUPPORT=0
+
+    # If --disable-dlopen was used, then have all the components fail
+    # (we still need to configure them all so that things like "make
+    # dist" work", but we just want the MCA system to (artificially)
+    # conclude that it can't build any of the components.
+    AS_IF([test "$enable_dlopen" = "no"],
+          [want_dl=0], [want_dl=1])
+
+    MCA_CONFIGURE_FRAMEWORK([opal], [dl], [$want_dl])
+
+    # If we found no suitable static dl component and dlopen support
+    # was not specifically disabled, this is an error.
+    AS_IF([test "$MCA_opal_dl_STATIC_COMPONENTS" = "" && \
+           test "$enable_dlopen" != "no"],
+          [AC_MSG_WARN([Did not find a suitable static opal dl component])
+           AC_MSG_WARN([You might need to install libltld (and its headers) or])
+           AC_MSG_WARN([specify --disable-dlopen to configure.])
+           AC_MSG_ERROR([Cannot continue])])
+
+    # If we have a winning component (which, per above, will only
+    # happen if --disable-dlopen was *not* specified), do some more
+    # logic.
+    AS_IF([test "$MCA_opal_dl_STATIC_COMPONENTS" != ""],
+       [ # We had a winner -- w00t!
+
+        OPAL_HAVE_DL_SUPPORT=1
+        # If we added any -L flags to ADD_LDFLAGS, then we (might)
+        # need to add those directories to LD_LIBRARY_PATH.
+        # Otherwise, if we try to AC RUN_IFELSE anything here in
+        # configure, it might die because it can't find the libraries
+        # we just linked against.
+        OPAL_VAR_SCOPE_PUSH([opal_dl_base_found_l opal_dl_base_token opal_dl_base_tmp opal_dl_base_dir])
+        opal_dl_base_found_l=0
+        eval "opal_dl_base_tmp=\$opal_dl_${opal_dl_winner}_ADD_LIBS"
+        for opal_dl_base_token in $opal_dl_base_tmp; do
+            case $opal_dl_base_token in
+            -l*) opal_dl_base_found_l=1 ;;
+            esac
+        done
+        AS_IF([test $opal_dl_base_found_l -eq 1],
+              [eval "opal_dl_base_tmp=\$opal_dl_${opal_dl_winner}_ADD_LDFLAGS"
+               for opal_dl_base_token in $opal_dl_base_tmp; do
+                   case $opal_dl_base_token in
+                   -L*)
+                       opal_dl_base_dir=`echo $opal_dl_base_token | cut -c3-`
+                       export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$opal_dl_base_dir
+                       AC_MSG_WARN([Adding to LD_LIBRARY_PATH: $opal_dl_base_dir])
+                       ;;
+                   esac
+               done])
+        OPAL_VAR_SCOPE_POP
+    ])
+
+    AC_DEFINE_UNQUOTED([OPAL_HAVE_DL_SUPPORT], [$OPAL_HAVE_DL_SUPPORT],
+                       [Whether the OPAL DL framework is functional or not])
+])

opal/mca/dl/dl.h

@@ -0,0 +1,184 @@
+/*
+ * Copyright (c) 2015 Cisco Systems, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+/**
+ * @file
+ *
+ * Dynamic library framework
+ *
+ * General Description:
+ *
+ * This framework provides portable access to dlopen- and dlsym-like
+ * functionality, very similar to Libtool's libltdl.  Indeed, one of
+ * the components in this framework will use libltdl, if it is
+ * present/available.  However, on some common types systems where
+ * libltdl headers and libraries are *not* available, we can support
+ * plugins via this simple framework.
+ *
+ * This is a compile-time framework: a single component will be
+ * selected by the priority that its configure.m4 provides.  All other
+ * components will be ignored (i.e., not built/not part of the
+ * installation).  Meaning: the static_components of the dl framework
+ * will always contain 0 or 1 components.
+ *
+ * SIDENOTE: Open MPI used to embed libltdl.  However, as of early
+ * 2015, this became problematic, for a variety of complex and
+ * uninteresting reasons (see the following if you care about the
+ * details: https://github.com/open-mpi/ompi/issues/311,
+ * http://debbugs.gnu.org/cgi/bugreport.cgi?bug=19370,
+ * https://github.com/open-mpi/ompi/pull/366,
+ * https://github.com/open-mpi/ompi/pull/390).  That being said, we,
+ * as a developer community, still wanted to be able to natively use
+ * DSOs by default.  A small/simple framework for DL functionality,
+ * along with a simple component that supports dlopen/dlsym on POSIX
+ * platforms and another component that natively uses libltdl seemed
+ * like a good solution.
+ */
+
+#ifndef MCA_DLOPEN_H
+#define MCA_DLOPEN_H
+
+#include "opal_config.h"
+
+#include "opal/mca/mca.h"
+#include "opal/mca/base/base.h"
+
+BEGIN_C_DECLS
+
+/**
+ * Handle for an opened file
+ */
+struct opal_dl_handle_t;
+typedef struct opal_dl_handle_t opal_dl_handle_t;
+
+/**
+ * Dynamically open the file specified.
+ *
+ * Arguments:
+ *   fname   = Base filename to open.  If NULL, open this process.
+ *   use_ext = If true, try various filename suffixes that are
+ *       relevant on this platform (e.g., .so, .dll, .dylib).  If
+ *       false, just use exactly whatever was passed as fname.
+ *   private = If true, open the file in a private namespace.
+ *       Otherwise, open the file in a global namespace.
+ *   handle = Upon successful open, a handle to the opened file will
+ *       be returned.
+ *   err_msg= if non-NULL and !=OPAL_SUCCESS is returned, will point to a
+ *       string error message
+ *
+ * Returns:
+ *   OPAL_SUCCESS on success, or OPAL_ERROR
+ *
+ * Space for the handle must be allocated by the module (it can be
+ * freed during the call to opal_dl_base_module_dlclose_fn_t).
+ *
+ * The err_msg points to an internal string and should not be altered
+ * or freed by the caller.  The contents of the err_msg string may
+ * change after successive calls to opal_dl API calls.
+ */
+typedef int (*opal_dl_base_module_open_fn_t)
+    (const char *fname, bool use_ext, bool private_namespace,
+     opal_dl_handle_t **handle, char **err_msg);
+
+/**
+ * Lookup a symbol in an opened file.
+ *
+ * Arguments:
+ *   handle = handle of a previously dynamically opened file
+ *   symbol = name of the symbol to lookup
+ *   ptr    = if found, a pointer to the symbol.  Otherwise, NULL.
+ *   err_msg= if non-NULL and !=OPAL_SUCCESS is returned, will point to a
+ *            string error message
+ * Returns:
+ *   OPAL_SUCCESS on success, or OPAL_ERROR
+ *
+ *
+ * The err_msg points to an internal string and should not be altered
+ * or freed by the caller.  The contents of the err_msg string may
+ * change after successive calls to opal_dl API calls.
+ */
+typedef int (*opal_dl_base_module_lookup_fn_t)
+    (opal_dl_handle_t *handle, const char *symbol, void **ptr, char **err_msg);
+
+/**
+ * Dynamically close a previously dynamically-opened file.
+ *
+ * Arguments:
+ *   handle = handle of a previously dynamically opened file.
+ * Returns:
+ *   OPAL_SUCCESS on success, or OPAL_ERROR
+ *
+ * This function should close the file and free and resources
+ * associated with it (e.g., whatever is cached on the handle).
+ */
+typedef int (*opal_dl_base_module_close_fn_t)
+    (opal_dl_handle_t *handle);
+
+/**
+ * Search through a path of directories, invoking a callback on each
+ * unique regular (non-Libtool) file basename found (e.g., will only
+ * be invoked once for the files "foo.la" and "foo.so", with the
+ * parameter "foo").
+ *
+ * Arguments:
+ *   path   = OPAL_ENV_SEP-delimited list of directories
+ *   cb_func= function to invoke on each filename found
+ *   data   = context for callback function
+ * Returns:
+ *   OPAL_SUCESS on success, OPAL_ERR* otherwise
+ */
+typedef int (*opal_dl_base_module_foreachfile_fn_t)
+    (const char *search_path,
+     int (*cb_func)(const char *filename, void *context),
+     void *context);
+
+/**
+ * Structure for DL components.
+ */
+struct opal_dl_base_component_1_0_0_t {
+    /** MCA base component */
+    mca_base_component_t base_version;
+    /** MCA base data */
+    mca_base_component_data_t base_data;
+
+    /** Default priority */
+    int priority;
+};
+typedef struct opal_dl_base_component_1_0_0_t opal_dl_base_component_1_0_0_t;
+typedef struct opal_dl_base_component_1_0_0_t opal_dl_base_component_t;
+
+/**
+ * Structure for DL modules
+ */
+struct opal_dl_base_module_1_0_0_t {
+    mca_base_module_2_0_0_t                 super;
+
+    /** Open / close */
+    opal_dl_base_module_open_fn_t           open;
+    opal_dl_base_module_close_fn_t          close;
+
+    /** Lookup a symbol */
+    opal_dl_base_module_lookup_fn_t         lookup;
+
+    /** Iterate looking for files */
+    opal_dl_base_module_foreachfile_fn_t    foreachfile;
+};
+typedef struct opal_dl_base_module_1_0_0_t opal_dl_base_module_1_0_0_t;
+typedef struct opal_dl_base_module_1_0_0_t opal_dl_base_module_t;
+
+/**
+ * Macro for use in components that are of type DL
+ */
+#define OPAL_DL_BASE_VERSION_1_0_0              \
+    MCA_BASE_VERSION_2_0_0,                     \
+    "dl", 1, 0, 0
+
+END_C_DECLS
+
+#endif /* OPAL_DL_H */