ugni thread safety fixes

ompi/mca/osc/rdma/Makefile.am

@@ -0,0 +1,52 @@
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University.
+#                         All rights reserved.
+# Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+#                         All rights reserved.
+# Copyright (c) 2004-2009 High Performance Computing Center Stuttgart, 
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+
+rdma_sources = \
+	osc_rdma.h \
+	osc_rdma_module.c \
+	osc_rdma_comm.h \
+	osc_rdma_comm.c \
+        osc_rdma_component.c \
+	osc_rdma_frag.h \
+	osc_rdma_frag.c \
+	osc_rdma_header.h \
+	osc_rdma_request.h \
+	osc_rdma_request.c \
+	osc_rdma_active_target.h \
+	osc_rdma_active_target.c \
+	osc_rdma_passive_target.c
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if MCA_BUILD_ompi_osc_rdma_DSO
+component_noinst =
+component_install = mca_osc_rdma.la
+else
+component_noinst = libmca_osc_rdma.la
+component_install =
+endif
+
+mcacomponentdir = $(ompilibdir)
+mcacomponent_LTLIBRARIES = $(component_install)
+mca_osc_rdma_la_SOURCES = $(rdma_sources)
+mca_osc_rdma_la_LDFLAGS = -module -avoid-version
+
+noinst_LTLIBRARIES = $(component_noinst)
+libmca_osc_rdma_la_SOURCES = $(rdma_sources)
+libmca_osc_rdma_la_LDFLAGS = -module -avoid-version

ompi/mca/osc/rdma/configure.m4

@@ -0,0 +1,26 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2013      Sandia National Laboratories.  All rights reserved.
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+
+# MCA_ompi_osc_rdma_POST_CONFIG(will_build)
+# ----------------------------------------
+# Only require the tag if we're actually going to be built, since bml
+# is one of the ones frequently disabled for large installs.
+AC_DEFUN([MCA_ompi_osc_rdma_POST_CONFIG], [
+    AS_IF([test "$1" = "1"], [OMPI_REQUIRE_ENDPOINT_TAG([BML])])
+])dnl
+
+# MCA_ompi_osc_rdma_CONFIG(action-if-can-compile, 
+#                        [action-if-cant-compile])
+# ------------------------------------------------
+# We can always build, unless we were explicitly disabled.
+AC_DEFUN([MCA_ompi_osc_rdma_CONFIG],[
+    AC_CONFIG_FILES([ompi/mca/osc/rdma/Makefile])
+    [$1]
+])dnl

opal/mca/btl/ugni/btl_ugni_atomic.c

@@ -50,7 +50,9 @@ int mca_btl_ugni_aop (struct mca_btl_base_module_t *btl, struct mca_btl_base_end
 
     post_desc->desc.base.first_operand = operand;
 
+    OPAL_THREAD_LOCK(&endpoint->btl->device->dev_lock);
     rc = GNI_PostFma (endpoint->rdma_ep_handle, &post_desc->desc.base);
+    OPAL_THREAD_UNLOCK(&endpoint->btl->device->dev_lock);
     if (GNI_RC_SUCCESS != rc) {
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
@@ -84,7 +86,9 @@ int mca_btl_ugni_afop (struct mca_btl_base_module_t *btl, struct mca_btl_base_en
 
     post_desc->desc.base.first_operand = operand;
 
+    OPAL_THREAD_LOCK(&endpoint->btl->device->dev_lock);
     rc = GNI_PostFma (endpoint->rdma_ep_handle, &post_desc->desc.base);
+    OPAL_THREAD_UNLOCK(&endpoint->btl->device->dev_lock);
     if (GNI_RC_SUCCESS != rc) {
         mca_btl_ugni_return_post_descriptor (endpoint->btl, post_desc);
         return OPAL_ERR_OUT_OF_RESOURCE;
@@ -119,7 +123,9 @@ int mca_btl_ugni_acswap (struct mca_btl_base_module_t *btl, struct mca_btl_base_
     post_desc->desc.base.first_operand = compare;
     post_desc->desc.base.second_operand = value;
 
+    OPAL_THREAD_LOCK(&endpoint->btl->device->dev_lock);
     rc = GNI_PostFma (endpoint->rdma_ep_handle, &post_desc->desc.base);
+    OPAL_THREAD_UNLOCK(&endpoint->btl->device->dev_lock);
     if (GNI_RC_SUCCESS != rc) {
         mca_btl_ugni_return_post_descriptor (endpoint->btl, post_desc);
         return OPAL_ERR_OUT_OF_RESOURCE;

opal/mca/btl/ugni/btl_ugni_endpoint.c

@@ -86,14 +86,14 @@ int mca_btl_ugni_ep_disconnect (mca_btl_base_endpoint_t *ep, bool send_disconnec
 static inline int mca_btl_ugni_ep_connect_start (mca_btl_base_endpoint_t *ep) {
     int rc;
 
-    BTL_VERBOSE(("initiaiting connection to remote peer with address: %u id: %u proc: %p",
-                 ep->common->ep_rem_addr, ep->common->ep_rem_id, (void *)ep->peer_proc));
-
     rc = mca_btl_ugni_ep_connect_rdma (ep);
     if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
         return rc;
     }
 
+    BTL_VERBOSE(("initiaiting connection to remote peer with address: %u id: %u proc: %p",
+                 ep->common->ep_rem_addr, ep->common->ep_rem_id, (void *)ep->peer_proc));
+
     /* bind endpoint to remote address */
     /* we bind two endpoints to seperate out local smsg completion and local fma completion */
     rc = opal_common_ugni_ep_create (ep->common, ep->btl->smsg_local_cq, &ep->smsg_ep_handle);