performance fix: openib use memalign for malloc

This commit was SVN r26409.

ompi/mca/btl/openib/btl_openib.h

@@ -297,6 +297,11 @@ struct mca_btl_openib_component_t {
 #if BTL_OPENIB_FAILOVER_ENABLED
     int verbose_failover;
 #endif
+#if BTL_OPENIB_MALLOC_HOOKS_ENABLED
+    int use_memalign;
+    size_t memalign_threshold;
+    void* (*previous_malloc_hook)(size_t __size, const void*);
+#endif
 }; typedef struct mca_btl_openib_component_t mca_btl_openib_component_t;
 
 OMPI_MODULE_DECLSPEC extern mca_btl_openib_component_t mca_btl_openib_component;

ompi/mca/btl/openib/btl_openib_component.c

@@ -47,6 +47,9 @@ const char *ibv_get_sysfs_path(void);
 #include <sys/stat.h>
 #include <stdlib.h>
 #include <stddef.h>
+#if BTL_OPENIB_MALLOC_HOOKS_ENABLED
+#include <malloc.h>
+#endif
 
 #include "opal/mca/event/event.h"
 #include "opal/align.h"
@@ -136,6 +139,26 @@ mca_btl_openib_component_t mca_btl_openib_component = {
     }
 };
 
+#if BTL_OPENIB_MALLOC_HOOKS_ENABLED
+/* This is a memory allocator hook. The purpose of this is to make
+ * every malloc aligned since this speeds up IB HCA work.
+ * There two basic cases here:
+ * 1. Memory manager for Open MPI is enabled. Then memalign below will be
+ * overridden by __memalign_hook which is set to opal_memory_linux_memalign_hook.
+ * Thus, _malloc_hook is going to use opal_memory_linux_memalign_hook.
+ * 2. No memory manager support. The memalign below is just regular glibc
+ * memalign which will be called through __malloc_hook instead of malloc.
+ */
+static void *btl_openib_malloc_hook(size_t sz, const void* caller)
+{
+    if (sz < mca_btl_openib_component.memalign_threshold) {
+        return mca_btl_openib_component.previous_malloc_hook(sz, caller);
+    } else {
+        return memalign(mca_btl_openib_component.use_memalign, sz);
+    }
+}
+#endif
+
 static int btl_openib_component_register(void)
 {
     int ret;
@@ -162,6 +185,13 @@ static int btl_openib_component_register(void)
         return OMPI_ERR_NOT_AVAILABLE;
     }
 
+#if BTL_OPENIB_MALLOC_HOOKS_ENABLED
+    mca_btl_openib_component.previous_malloc_hook = __malloc_hook;
+    if (mca_btl_openib_component.use_memalign > 0) {
+        __malloc_hook = btl_openib_malloc_hook; 
+    }
+#endif
+
     return OMPI_SUCCESS;
 }
 
@@ -178,6 +208,7 @@ static int btl_openib_component_open(void)
     OBJ_CONSTRUCT(lock, opal_mutex_t);
     OBJ_CONSTRUCT(srq_addr_table, opal_hash_table_t);
 #endif
+
     /* initialize state */
     mca_btl_openib_component.ib_num_btls = 0;
     mca_btl_openib_component.openib_btls = NULL;
@@ -238,6 +269,11 @@ static int btl_openib_component_close(void)
     if (NULL != mca_btl_openib_component.default_recv_qps) {
         free(mca_btl_openib_component.default_recv_qps);
     }
+    
+#if BTL_OPENIB_MALLOC_HOOKS_ENABLED
+    __malloc_hook = mca_btl_openib_component.previous_malloc_hook;
+#endif
+
     return rc;
 }
 
@@ -3829,3 +3865,4 @@ int mca_btl_openib_post_srr(mca_btl_openib_module_t* openib_btl, const int qp)
     OPAL_THREAD_UNLOCK(&openib_btl->ib_lock);
     return OMPI_ERROR;
 }
+

ompi/mca/btl/openib/btl_openib_mca.c

@@ -608,6 +608,36 @@ int btl_openib_register_mca_params(void)
                   0, &mca_btl_openib_component.gid_index,
                   REGINT_GE_ZERO));
 
+#if OPENIB_MALLOC_HOOKS_ENABLED
+    CHECK(reg_int("memalign", NULL,
+                  "[64 | 32 | 0] - Enable (64bit or 32bit)/Disable(0) memory"
+                  "alignment for all malloc calls if btl openib is used.",
+                  32, &mca_btl_openib_component.use_memalign,
+                  REGINT_GE_ZERO));
+    
+    if (mca_btl_openib_component.use_memalign != 32  
+        && mca_btl_openib_component.use_memalign != 64
+        && mca_btl_openib_component.use_memalign != 0){ 
+        orte_show_help("help-mpi-btl-openib.txt", "invalid mca param value",
+                       true, "Wrong btl_openib_memalign parameter value. Allowed values: 64, 32, 0.",
+                       "btl_openib_memalign is reset to 32");
+        mca_btl_openib_component.use_memalign = 32; 
+    }
+    reg_int("memalign_threshold", NULL,
+            "Allocating memory more than btl_openib_memalign_threshhold"
+            "bytes will automatically be algined to the value of btl_openib_memalign bytes."
+            "memalign_threshhold defaults to the same value as mca_btl_openib_eager_limit.",
+            mca_btl_openib_component.eager_limit,
+            &ival,
+            REGINT_GE_ZERO);
+    if (ival < 0){
+        orte_show_help("help-mpi-btl-openib.txt", "invalid mca param value",
+                       true, "btl_openib_memalign_threshold must be positive",
+                       "btl_openib_memalign_threshold is reset to btl_openib_eager_limit");
+        ival = mca_btl_openib_component.eager_limit;
+    }
+    mca_btl_openib_component.memalign_threshold = (size_t)ival;
+#endif
     /* Register any MCA params for the connect pseudo-components */
     if (OMPI_SUCCESS == ret) {
         ret = ompi_btl_openib_connect_base_register();

ompi/mca/btl/openib/configure.m4

@@ -81,6 +81,7 @@ AC_DEFUN([MCA_ompi_btl_openib_CONFIG],[
           AC_MSG_RESULT([$cpcs])])
 
     # Enable openib device failover.  It is disabled by default.
+    AC_MSG_CHECKING([whether openib failover is enabled])
     AC_ARG_ENABLE([btl-openib-failover],
        [AC_HELP_STRING([--enable-btl-openib-failover],
            [enable openib BTL failover (default: disabled)])])
@@ -95,6 +96,29 @@ AC_DEFUN([MCA_ompi_btl_openib_CONFIG],[
                        [enable openib BTL failover])
     AM_CONDITIONAL([MCA_btl_openib_enable_failover], [test "x$btl_openib_failover_enabled" = "x1"])
 
+
+    # Check for __malloc_hook availability
+    AC_ARG_ENABLE(btl-openib-malloc-alignment,
+    	AC_HELP_STRING([--enable-btl-openib-malloc-alignment], [Enable support for allocated memory alignment. Default: enabled if supported, disabled otherwise.]))
+
+    btl_openib_malloc_hooks_enabled=0
+    AS_IF([test "$enable_btl_openib_malloc_alignment" != "no"],
+        [AC_CHECK_HEADER([malloc.h],
+             [AC_CHECK_FUNC([__malloc_hook],
+                  [AC_CHECK_FUNC([__realloc_hook],
+                       [AC_CHECK_FUNC([__free_hook],
+                            [btl_openib_malloc_hooks_enabled=1])])])])])
+
+    AS_IF([test "$enable_btl_openib_malloc_alignment" = "yes" -a "$btl_openib_malloc_hooks_enabled" = "0"],
+          [AC_MSG_ERROR([openib malloc alignment is requested but __malloc_hook is not available])])
+    AC_MSG_CHECKING([whether the openib BTL will use malloc hooks])
+    AS_IF([test "$btl_openib_malloc_hooks_enabled" = "0"],
+          [AC_MSG_RESULT([no])],
+          [AC_MSG_RESULT([yes])])
+
+    AC_DEFINE_UNQUOTED(BTL_OPENIB_MALLOC_HOOKS_ENABLED, [$btl_openib_malloc_hooks_enabled],
+                       [Whether the openib BTL malloc hooks are enabled]) 
+   
     # substitute in the things needed to build openib
     AC_SUBST([btl_openib_CFLAGS])
     AC_SUBST([btl_openib_CPPFLAGS])