Merge pull request #1070 from jsquyres/pr/move-usnic-fake-ibv-driver-to-its-own-static-common-component

common_usnic: move fake IBV provider to libopen-pal

opal/mca/common/verbs/Makefile.am

@@ -17,7 +17,6 @@ headers = \
 sources = \
 	common_verbs_basics.c \
 	common_verbs_devlist.c \
-	common_verbs_fake.c \
 	common_verbs_find_max_inline.c \
 	common_verbs_find_ports.c \
 	common_verbs_mca.c \

opal/mca/common/verbs/common_verbs.h

@@ -180,11 +180,6 @@ OPAL_DECLSPEC int opal_common_verbs_qp_test(struct ibv_context *device_context,
  */
 int opal_common_verbs_fork_test(void);
 
-/*
- * Register fake verbs drivers
- */
-void opal_common_verbs_register_fake_drivers(void);
-
 END_C_DECLS
 
 #endif

opal/mca/common/verbs/common_verbs_basics.c

@@ -21,6 +21,8 @@
 #include <unistd.h>
 #endif
 
+#include "opal/mca/common/verbs_usnic/common_verbs_usnic.h"
+
 /* This is crummy, but <infiniband/driver.h> doesn't work on all
    platforms with all compilers.  Specifically, trying to include it
    on RHEL4U3 with the PGI 32 bit compiler will cause problems because
@@ -89,10 +91,12 @@ int opal_common_verbs_fork_test(void)
     }
 #endif
 
-    /* Now rgister any necessary fake libibverbs drivers.  We
+    /* Now register any necessary fake libibverbs drivers.  We
        piggyback loading these fake drivers on the fork test because
-       they must be loaded before ibv_get_device_list() is invoked. */
-    opal_common_verbs_register_fake_drivers();
+       they must be loaded before ibv_get_device_list() is invoked.
+       Note that this routine is in a different common component (see
+       comments over there for an explanation why).  */
+    opal_common_verbs_usnic_register_fake_drivers();
 
     return ret;
 }

opal/mca/common/verbs_usnic/Makefile.am

@@ -0,0 +1,40 @@
+#
+# Copyright (c) 2009-2012 Mellanox Technologies.  All rights reserved.
+# Copyright (c) 2009-2012 Oak Ridge National Laboratory.  All rights reserved.
+# Copyright (c) 2012-2015 Cisco Systems, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+headers = common_verbs_usnic.h
+
+sources = common_verbs_usnic_fake.c
+
+# This component is always linked statically.  It has code that is
+# registered as a driver for libibverbs.  There is no corresponding
+# *un*register API in libibverbs, so this code can never be dlclosed.
+# And therefore it must be in the libopen-pal library, not a DSO or
+# dependent library.
+
+noinst_LTLIBRARIES = lib@OPAL_LIB_PREFIX@mca_common_verbs_usnic.la
+
+lib@OPAL_LIB_PREFIX@mca_common_verbs_usnic_la_SOURCES = \
+        $(headers) $(sources)
+lib@OPAL_LIB_PREFIX@mca_common_verbs_usnic_la_CPPFLAGS = \
+        $(common_verbs_usnic_CPPFLAGS)
+lib@OPAL_LIB_PREFIX@mca_common_verbs_usnic_la_LDFLAGS =  \
+        $(common_verbs_usnic_LDFLAGS)
+lib@OPAL_LIB_PREFIX@mca_common_verbs_usnic_la_LIBADD = \
+        $(common_verbs_usnic_LIBS)
+
+# Conditionally install the header files
+
+if WANT_INSTALL_HEADERS
+opaldir = $(opalincludedir)/opal/mca/common/verbs_usnic
+opal_HEADERS = $(headers)
+else
+opaldir = $(includedir)
+endif

opal/mca/common/verbs_usnic/common_verbs_usnic.h

@@ -0,0 +1,27 @@
+/*
+ * Copyright (c) 2015 Cisco Systems, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef _COMMON_VERBS_USNIC_H_
+#define _COMMON_VERBS_USNIC_H_
+
+#include "opal_config.h"
+
+#include <stdint.h>
+#include <infiniband/verbs.h>
+
+BEGIN_C_DECLS
+
+/*
+ * Register fake verbs drivers
+ */
+void opal_common_verbs_usnic_register_fake_drivers(void);
+
+END_C_DECLS
+
+#endif

opal/mca/common/verbs_usnic/common_verbs_usnic_fake.c

@@ -27,6 +27,13 @@
  * More specifically: the userspace side of usNIC is exposed through
  * libfabric; we don't need libibverbs warnings about not being able
  * to find a usnic driver.
+ *
+ * Note: this code is statically linked into libopen-pal.  It is
+ * registered via ibv_register_driver(), and there is no corresponding
+ * *un*register IBV API.  Hence, we cannot allow this code to be
+ * dlclosed (e.g., if it is a DSO or a dependent common library) -- it
+ * must be in libopen-pal itself, which will stay resident in the MPI
+ * application.
  */
 
 #include "opal_config.h"
@@ -39,7 +46,7 @@
 #include <infiniband/driver.h>
 #endif
 
-#include "common_verbs.h"
+#include "common_verbs_usnic.h"
 
 /***********************************************************************/
 
@@ -95,7 +102,7 @@ static struct ibv_device *fake_driver_init(const char *uverbs_sys_path,
 }
 
 
-void opal_common_verbs_register_fake_drivers(void)
+void opal_common_verbs_usnic_register_fake_drivers(void)
 {
     /* No need to do this more than once */
     static bool already_done = false;

opal/mca/common/verbs_usnic/configure.m4

@@ -0,0 +1,54 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2007-2015 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2009-2012 Mellanox Technologies.  All rights reserved.
+# Copyright (c) 2009-2012 Oak Ridge National Laboratory.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+#
+# This component must be linked statically into libopen-pal because it
+# registers a provider for libibverbs at run time, and there's no
+# libibverbs API to *un*register a plugin.  Hence, we can't allow this
+# code to be dlclosed/removed from the process.  Hence: it must be
+# compiled statically into libopen-pal.
+#
+AC_DEFUN([MCA_opal_common_verbs_usnic_COMPILE_MODE], [
+    AC_MSG_CHECKING([for MCA component $2:$3 compile mode])
+    $4="static"
+    AC_MSG_RESULT([$$4])
+])
+
+# MCA_opal_common_verbs_usnic_CONFIG([action-if-can-compile],
+#                      [action-if-cant-compile])
+# ------------------------------------------------
+AC_DEFUN([MCA_opal_common_verbs_usnic_CONFIG],[
+    AC_CONFIG_FILES([opal/mca/common/verbs_usnic/Makefile])
+    common_verbs_usnic_happy="no"
+
+    OPAL_CHECK_OPENFABRICS([common_verbs_usnic],
+                           [common_verbs_usnic_happy="yes"])
+
+    AS_IF([test "$common_verbs_usnic_happy" = "yes"],
+          [$1],
+          [$2])
+
+    # substitute in the things needed to build openib
+    AC_SUBST([common_verbs_usnic_CPPFLAGS])
+    AC_SUBST([common_verbs_usnic_LDFLAGS])
+    AC_SUBST([common_verbs_usnic_LIBS])
+])dnl

opal/mca/common/verbs_usnic/owner.txt

@@ -0,0 +1,7 @@
+#
+# owner/status file
+# owner: institution that is responsible for this package
+# status: e.g. active, maintenance, unmaintained
+#
+owner: Cisco
+status: maintenance