on Solaris, when IBV_ACCESS_SO is available, use strong ordered memory region for eager rdma connection

This commit was SVN r24395.

ompi/mca/btl/openib/btl_openib_component.c

@@ -15,7 +15,7 @@
  * Copyright (c) 2006-2007 Los Alamos National Security, LLC.  All rights
  *                         reserved.
  * Copyright (c) 2006-2007 Voltaire All rights reserved.
- * Copyright (c) 2009-2010 Oracle and/or its affiliates.  All rights reserved.
+ * Copyright (c) 2009-2011 Oracle and/or its affiliates.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -527,9 +527,16 @@ static int openib_reg_mr(void *reg_data, void *base, size_t size,
 {
     mca_btl_openib_device_t *device = (mca_btl_openib_device_t*)reg_data;
     mca_btl_openib_reg_t *openib_reg = (mca_btl_openib_reg_t*)reg;
+    enum ibv_access_flags access_flag = IBV_ACCESS_LOCAL_WRITE |
+        IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ;
 
-    openib_reg->mr = ibv_reg_mr(device->ib_pd, base, size, IBV_ACCESS_LOCAL_WRITE |
-            IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ);
+#if defined(HAVE_IBV_ACCESS_SO)
+    if (reg->flags & MCA_MPOOL_FLAGS_SO_MEM) {
+        access_flag |= IBV_ACCESS_SO;
+    }
+#endif
+
+    openib_reg->mr = ibv_reg_mr(device->ib_pd, base, size, access_flag);
 
     if (NULL == openib_reg->mr) {
         return OMPI_ERR_OUT_OF_RESOURCE;

ompi/mca/btl/openib/btl_openib_endpoint.c

@@ -16,7 +16,7 @@
  * Copyright (c) 2006-2007 Voltaire All rights reserved.
  * Copyright (c) 2006-2009 Mellanox Technologies, Inc.  All rights reserved.
  * Copyright (c) 2010      IBM Corporation.  All rights reserved.
- * Copyright (c) 2010      Oracle and/or its affiliates.  All rights reserved
+ * Copyright (c) 2010-2011 Oracle and/or its affiliates.  All rights reserved
  *
  * $COPYRIGHT$
  *
@@ -911,6 +911,7 @@ void mca_btl_openib_endpoint_connect_eager_rdma(
     char *buf;
     mca_btl_openib_recv_frag_t *headers_buf;
     int i;
+    uint32_t flag = MCA_MPOOL_FLAGS_CACHE_BYPASS;
 
     /* Set local rdma pointer to 1 temporarily so other threads will not try
      * to enter the function */
@@ -925,11 +926,25 @@ void mca_btl_openib_endpoint_connect_eager_rdma(
     if(NULL == headers_buf)
        goto unlock_rdma_local;
 
+#if defined(HAVE_IBV_ACCESS_SO)
+    /* Solaris implements the Relaxed Ordering feature defined in the
+       PCI Specification. With this in mind any memory region which
+       relies on a buffer being written in a specific order, for
+       example the eager rdma connections created in this routinue,
+       must set a strong order flag when registering the memory for
+       rdma operations.
+
+       The following flag will be interpreted and the appropriate
+       steps will be taken when the memory is registered in
+       openib_reg_mr(). */
+    flag |= MCA_MPOOL_FLAGS_SO_MEM;
+#endif
+
     buf = (char *) openib_btl->super.btl_mpool->mpool_alloc(openib_btl->super.btl_mpool,
             openib_btl->eager_rdma_frag_size *
             mca_btl_openib_component.eager_rdma_num,
             mca_btl_openib_component.buffer_alignment,
-            MCA_MPOOL_FLAGS_CACHE_BYPASS,
+            flag,
             (mca_mpool_base_registration_t**)&endpoint->eager_rdma_local.reg);
 
     if(!buf)

ompi/mca/btl/openib/configure.m4

@@ -12,6 +12,7 @@
 #                         All rights reserved.
 # Copyright (c) 2007-2010 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2008      Mellanox Technologies.  All rights reserved.
+# Copyright (c) 2011      Oracle and/or its affiliates.  All rights reserved.
 # $COPYRIGHT$
 # 
 # Additional copyrights may follow
@@ -34,7 +35,7 @@ AC_DEFUN([MCA_ompi_btl_openib_POST_CONFIG], [
 AC_DEFUN([MCA_ompi_btl_openib_CONFIG],[
     AC_CONFIG_FILES([ompi/mca/btl/openib/Makefile])
 
-    OPAL_VAR_SCOPE_PUSH([cpcs have_threads])
+    OPAL_VAR_SCOPE_PUSH([cpcs have_threads have_ibv_access_so])
     cpcs="oob"
 
     OMPI_CHECK_OPENIB([btl_openib],
@@ -78,6 +79,19 @@ AC_DEFUN([MCA_ompi_btl_openib_CONFIG],[
           AC_MSG_CHECKING([which openib btl cpcs will be built])
           AC_MSG_RESULT([$cpcs])])
 
+    # check for Solaris specific memory access flag
+    AS_IF([test "$btl_openib_happy" = "yes"],
+          [if test "`echo $build_os | $GREP solaris`"; then
+              AC_TRY_COMPILE([#include <infiniband/verbs.h>],
+                  [int flag = IBV_ACCESS_SO;],
+                  [have_ibv_access_so="yes"
+                      AC_DEFINE_UNQUOTED([HAVE_IBV_ACCESS_SO],
+                      1,[openib define HAVE_IBV_ACCESS_SO])],
+                  [have_ibv_access_so="no"])
+              AC_MSG_CHECKING([for IBV_ACCESS_SO in Solaris])
+              AC_MSG_RESULT([$have_ibv_access_so])
+          fi])
+
     # Enable openib device failover.  It is disabled by default.
     AC_ARG_ENABLE([btl-openib-failover],
        [AC_HELP_STRING([--enable-btl-openib-failover],