1. Adding 2 new components:

ofacm - generic connection manager for IB interconnects. ofautils - IB common utilities and compatibility code 2. Updating OpenIB configure code - ORNL & Mellanox Teams This commit was SVN r26707.
2012-07-02 15:20:12 +00:00 · 2012-07-02 15:20:12 +00:00 · f7664b3814
--- a/ompi/config/ompi_check_openfabrics.m4
+++ b/ompi/config/ompi_check_openfabrics.m4
@ -15,6 +15,7 @@
 #                         reserved.
 # Copyright (c) 2006-2009 Mellanox Technologies. All rights reserved.
 # Copyright (c) 2010-2012 Oracle and/or its affiliates.  All rights reserved.
+# Copyright (c) 2009-2012 Oak Ridge National Laboratory.  All rights reserved.
 # $COPYRIGHT$
 # 
 # Additional copyrights may follow
@ -23,12 +24,12 @@
 #


-# OMPI_CHECK_OPENIB(prefix, [action-if-found], [action-if-not-found])
+# OMPI_CHECK_OPENFABRICS(prefix, [action-if-found], [action-if-not-found])
 # --------------------------------------------------------
 # check if OPENIB support can be found.  sets prefix_{CPPFLAGS, 
 # LDFLAGS, LIBS} as needed and runs action-if-found if there is
 # support, otherwise executes action-if-not-found
-AC_DEFUN([OMPI_CHECK_OPENIB],[
+AC_DEFUN([OMPI_CHECK_OPENFABRICS],[
    OPAL_VAR_SCOPE_PUSH([$1_msg])

    # Setup the --with switches to allow users to specify where
@ -69,29 +70,6 @@ AC_DEFUN([OMPI_CHECK_OPENIB],[
    AC_DEFINE_UNQUOTED([OMPI_OPENIB_PAD_HDR], [$ompi_openib_pad_hdr],
                       [Add padding bytes to the openib BTL control header])

-    #
-    # ConnectX XRC support
-    #
-    AC_ARG_ENABLE([openib-connectx-xrc],
-        [AC_HELP_STRING([--enable-openib-connectx-xrc],
-                        [Enable ConnectX XRC support in the openib BTL. If you do not have InfiniBand ConnectX adapters, you may disable the ConnectX XRC support. If you do not know which InfiniBand adapter is installed on your cluster, leave this option enabled (default: enabled)])],
-                        [enable_connectx_xrc="$enableval"], [enable_connectx_xrc="yes"])
-
-    #
-    # Unconnect Datagram (UD) based connection manager
-    #
-    AC_ARG_ENABLE([openib-udcm],
-        [AC_HELP_STRING([--enable-openib-udcm],
-                        [Enable datagram connection support in openib BTL (default: enabled)])], 
-                        [enable_openib_udcm="$enableval"], [enable_openib_udcm="yes"])
-
-    #
-    # Openfabrics RDMACM
-    #
-    AC_ARG_ENABLE([openib-rdmacm],
-        [AC_HELP_STRING([--enable-openib-rdmacm],
-                        [Enable Open Fabrics RDMACM support in openib BTL (default: enabled)])])
-
    AS_IF([test "$opal_want_verbs" = "no"],
          [ompi_check_openib_happy="no"],
          [ompi_check_openib_happy="yes"])
@ -170,8 +148,6 @@ AC_DEFUN([OMPI_CHECK_OPENIB],[
    # Set these up so that we can do an AC_DEFINE below
    # (unconditionally)
    $1_have_xrc=0
-    $1_have_udcm=0
-    $1_have_rdmacm=0
    $1_have_opensm_devel=0

    # If we have the openib stuff available, find out what we've got
@ -189,11 +165,6 @@ AC_DEFUN([OMPI_CHECK_OPENIB],[
               AC_CHECK_FUNCS([ibv_create_xrc_rcv_qp], [$1_have_xrc=1])
           fi

-           # is udcm enabled
-           if test "$enable_openib_udcm" = "yes"; then
-               $1_have_udcm=1
-           fi
-
           if test "no" != "$enable_openib_dynamic_sl"; then
               # We need ib_types.h file, which is installed with opensm-devel
               # package. However, ib_types.h has a bad include directive,
@ -219,30 +190,6 @@ AC_DEFUN([OMPI_CHECK_OPENIB],[
                             AC_MSG_ERROR([Cannot continue])])])
           fi

-           # Do we have a recent enough RDMA CM?  Need to have the
-           # rdma_get_peer_addr (inline) function (originally appeared
-           # in OFED v1.3).
-           if test "$enable_openib_rdmacm" != "no"; then
-                 AC_CHECK_HEADERS([rdma/rdma_cma.h],
-                     [AC_CHECK_LIB([rdmacm], [rdma_create_id],
-                         [AC_MSG_CHECKING([for rdma_get_peer_addr])
-                         $1_msg=no
-                         AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include "rdma/rdma_cma.h"
-                                 ]], [[void *ret = (void*) rdma_get_peer_addr((struct rdma_cm_id*)0);]])],
-                             [$1_have_rdmacm=1 
-                             $1_msg=yes])
-                         AC_MSG_RESULT([$$1_msg])])])
-
-                 if test "1" = "$$1_have_rdmacm"; then
-                     $1_LIBS="-lrdmacm $$1_LIBS"
-                 else
-                     AS_IF([test "$enable_openib_rdmacm" = "yes"],
-                           [AC_MSG_WARN([--enable-openib-rdmacm was specified but the])
-                            AC_MSG_WARN([appropriate files could not be found])
-                            AC_MSG_WARN([Please install librdmacm and librdmacm-devel or disable rdmacm support])
-                            AC_MSG_ERROR([Cannot continue])])
-                 fi
-           fi

           # Check support for RDMAoE devices
           $1_have_rdmaoe=0
@ -291,24 +238,6 @@ AC_DEFUN([OMPI_CHECK_OPENIB],[
        AC_MSG_RESULT([no])
    fi

-    AC_MSG_CHECKING([if UD CM is enabled])
-    AC_DEFINE_UNQUOTED([OMPI_HAVE_UDCM], [$$1_have_udcm],
-        [Whether UD CM is available or not])
-    if test "1" = "$$1_have_udcm"; then
-        AC_MSG_RESULT([yes])
-    else
-        AC_MSG_RESULT([no])
-    fi
-
-    AC_MSG_CHECKING([if OpenFabrics RDMACM support is enabled])
-    AC_DEFINE_UNQUOTED([OMPI_HAVE_RDMACM], [$$1_have_rdmacm],
-        [Whether RDMA CM is available or not])
-    if test "1" = "$$1_have_rdmacm"; then
-        AC_MSG_RESULT([yes])
-    else
-        AC_MSG_RESULT([no])
-    fi
-
    AS_IF([test -z "$opal_verbs_dir"],
          [openib_include_dir="/usr/include"],
          [openib_include_dir="$opal_verbs_dir/include"])
@ -327,3 +256,148 @@ AC_DEFUN([OMPI_CHECK_OPENIB],[

     OPAL_VAR_SCOPE_POP
 ])
+
+AC_DEFUN([OMPI_CHECK_OPENFABRICS_CM_ARGS],[
+    #
+    # ConnectX XRC support
+    #
+    AC_ARG_ENABLE([openib-connectx-xrc],
+        [AC_HELP_STRING([--enable-openib-connectx-xrc],
+                        [Enable ConnectX XRC support in the openib BTL. If you do not have InfiniBand ConnectX adapters, you may disable the ConnectX XRC support. If you do not know which InfiniBand adapter is installed on your cluster, leave this option enabled (default: enabled)])],
+                        [enable_connectx_xrc="$enableval"], [enable_connectx_xrc="yes"])
+    #
+    # Unconnect Datagram (UD) based connection manager
+    #
+    AC_ARG_ENABLE([openib-udcm],
+        [AC_HELP_STRING([--enable-openib-udcm],
+                        [Enable datagram connection support in openib BTL (default: enabled)])], 
+                        [enable_openib_udcm="$enableval"], [enable_openib_udcm="yes"])
+    
+    #
+    # Openfabrics RDMACM
+    #
+    AC_ARG_ENABLE([openib-rdmacm],
+        [AC_HELP_STRING([--enable-openib-rdmacm],
+                        [Enable Open Fabrics RDMACM support in openib BTL (default: enabled)])])
+])dnl
+
+AC_DEFUN([OMPI_CHECK_OPENFABRICS_CM],[
+    AC_REQUIRE([OMPI_CHECK_OPENFABRICS_CM_ARGS])
+    $1_have_udcm=0
+    $1_have_rdmacm=0
+
+    AS_IF([test "$ompi_check_openib_happy" = "yes"],
+          [# Do we have a recent enough RDMA CM?  Need to have the
+           # rdma_get_peer_addr (inline) function (originally appeared
+           # in OFED v1.3).
+           if test "$enable_openib_rdmacm" != "no"; then
+                 AC_CHECK_HEADERS([rdma/rdma_cma.h],
+                     [AC_CHECK_LIB([rdmacm], [rdma_create_id],
+                         [AC_MSG_CHECKING([for rdma_get_peer_addr])
+                         $1_msg=no
+                         AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include "rdma/rdma_cma.h"
+                                 ]], [[void *ret = (void*) rdma_get_peer_addr((struct rdma_cm_id*)0);]])],
+                             [$1_have_rdmacm=1 
+                             $1_msg=yes])
+                         AC_MSG_RESULT([$$1_msg])])])
+
+                 if test "1" = "$$1_have_rdmacm"; then
+                     $1_LIBS="-lrdmacm $$1_LIBS"
+                 else
+                     AS_IF([test "$enable_openib_rdmacm" = "yes"],
+                           [AC_MSG_WARN([--enable-openib-rdmacm was specified but the])
+                            AC_MSG_WARN([appropriate files could not be found])
+                            AC_MSG_WARN([Please install librdmacm and librdmacm-devel or disable rdmacm support])
+                            AC_MSG_ERROR([Cannot continue])])
+                 fi
+           fi
+
+           # is udcm enabled
+           if test "$enable_openib_udcm" = "yes"; then
+               $1_have_udcm=1
+           fi
+           ])
+
+    AC_MSG_CHECKING([if UD CM is enabled])
+    AC_DEFINE_UNQUOTED([OMPI_HAVE_UDCM], [$$1_have_udcm],
+        [Whether UD CM is available or not])
+    if test "1" = "$$1_have_udcm"; then
+        AC_MSG_RESULT([yes])
+    else
+        AC_MSG_RESULT([no])
+    fi
+
+    AC_MSG_CHECKING([if OpenFabrics RDMACM support is enabled])
+    AC_DEFINE_UNQUOTED([OMPI_HAVE_RDMACM], [$$1_have_rdmacm],
+        [Whether RDMA CM is available or not])
+    if test "1" = "$$1_have_rdmacm"; then
+        AC_MSG_RESULT([yes])
+    else
+        AC_MSG_RESULT([no])
+    fi
+])dnl
+
+AC_DEFUN([OMPI_CHECK_MLNX_OPENFABRICS],[
+     $1_have_mverbs=0
+     $1_have_mqe=0
+
+    AS_IF([test "$ompi_check_openib_happy" = "yes"], 
+           [OMPI_CHECK_PACKAGE([$1],
+                               [infiniband/mverbs.h],
+                               [mverbs],
+                               [ibv_m_query_device],
+                               ["$$1_LIBS"],
+                               [$opal_verbs_dir],
+                               [$opal_verbs_libdir],
+                               [$1_have_mverbs=1],
+                               [])])
+
+    AS_IF([test "$ompi_check_openib_happy" = "yes"], 
+           [OMPI_CHECK_PACKAGE([$1],
+                               [infiniband/mqe.h],
+                               [mqe],
+                               [mqe_context_create],
+                               ["$$1_LIBS"],
+                               [$opal_verbs_dir],
+                               [$opal_verbs_libdir],
+                               [$1_have_mqe=1],
+                               [])])
+
+    AC_MSG_CHECKING([if Mellanox OpenFabrics VERBS is enabled])
+    AC_DEFINE_UNQUOTED([OMPI_HAVE_MVERBS], [$$1_have_mverbs],
+        [Whether MVERBS is available or not])
+    AS_IF([test "1" = "$$1_have_mverbs"],
+          [AC_MSG_RESULT([yes])],
+          [AC_MSG_RESULT([no])])
+
+    # save the CPPFLAGS since we would have to update it for next test
+    ompi_check_mellanox_openfabrics_$1_save_CPPFLAGS="$CPPFLAGS"
+
+    # If openfabrics custom directory have been defined, we have
+    # to use it for MACRO test that uses mverbs.h file.
+    #
+    if test ! -z "$ompi_check_verbs_dir" ; then
+        CPPFLAGS="-I${opal_verbs_dir}/include $CPPFLAGS"
+    fi
+
+    AS_IF([test "1" = "$$1_have_mverbs"],
+          [AC_CHECK_DECLS([IBV_M_WR_CALC_RDMA_WRITE_WITH_IMM], 
+                          [AC_DEFINE_UNQUOTED([OMPI_HAVE_IBOFFLOAD_CALC_RDMA], [1],
+                                              [Whether IBV_M_WR_CALC_SEND is defined or not])], 
+                          [AC_DEFINE_UNQUOTED([OMPI_HAVE_IBOFFLOAD_CALC_RDMA], [0],
+                                              [Whether IBV_M_WR_CALC_SEND is defined or not])], 
+                          [#include <infiniband/mverbs.h>])])
+
+    # restoring the CPPFLAGS
+    CPPFLAGS="$ompi_check_mellanox_openfabrics_$1_save_CPPFLAGS"
+
+    AC_MSG_CHECKING([if Mellanox OpenFabrics MQE is enabled])
+    AC_DEFINE_UNQUOTED([OMPI_HAVE_MQE], [$$1_have_mqe],
+        [Whether MQE is available or not])
+    AS_IF([test "1" = "$$1_have_mqe"],
+          [AC_MSG_RESULT([yes])],
+          [AC_MSG_RESULT([no])])
+
+    AS_IF([test "1" = "$$1_have_mverbs" -a "1" = $$1_have_mqe],
+            [$2], [$3])
+])dnl
--- a/ompi/mca/btl/ofud/configure.m4
+++ b/ompi/mca/btl/ofud/configure.m4
@ -27,7 +27,7 @@
 AC_DEFUN([MCA_ompi_btl_ofud_CONFIG],[
    AC_CONFIG_FILES([ompi/mca/btl/ofud/Makefile])

-    OMPI_CHECK_OPENIB([btl_ofud],
+    OMPI_CHECK_OPENFABRICS([btl_ofud],
                        [btl_ofud_happy="yes"],
                        [btl_ofud_happy="no"])

--- a/ompi/mca/btl/openib/Makefile.am
+++ b/ompi/mca/btl/openib/Makefile.am
@ -13,6 +13,7 @@
 # Copyright (c) 2010      Oracle and/or its affiliates.  All rights reserved.
 # Copyright (c) 2011      NVIDIA Corporation.  All rights reserved.
 # Copyright (c) 2011      Mellanox Technologies.  All rights reserved.
+# Copyright (c) 2012      Oak Ridge National Laboratory.  All rights reserved
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@ -126,7 +127,8 @@ mcacomponentdir = $(pkglibdir)
 mcacomponent_LTLIBRARIES = $(component)
 mca_btl_openib_la_SOURCES = $(component_sources)
 mca_btl_openib_la_LDFLAGS = -module -avoid-version $(btl_openib_LDFLAGS)
-mca_btl_openib_la_LIBADD = $(btl_openib_LIBS)
+mca_btl_openib_la_LIBADD = $(btl_openib_LIBS) \
+    $(top_ompi_builddir)/ompi/mca/common/ofautils/libmca_common_ofautils.la
 if MCA_ompi_cuda_support
 mca_btl_openib_la_LIBADD += \
    $(top_ompi_builddir)/ompi/mca/common/cuda/libmca_common_cuda.la
--- a/ompi/mca/btl/openib/btl_openib_component.c
+++ b/ompi/mca/btl/openib/btl_openib_component.c
@ -17,6 +17,7 @@
 * Copyright (c) 2006-2007 Voltaire All rights reserved.
 * Copyright (c) 2009-2012 Oracle and/or its affiliates.  All rights reserved.
 * Copyright (c) 2011      NVIDIA Corporation.  All rights reserved.
+ * Copyright (c) 2012      Oak Ridge National Laboratory.  All rights reserved
 * $COPYRIGHT$
 *
 * Additional copyrights may follow
@ -60,6 +61,7 @@ const char *ibv_get_sysfs_path(void);
 #include "opal/util/argv.h"
 #include "opal/memoryhooks/memory.h"
 #include "opal/mca/base/mca_base_param.h"
+#include "ompi/mca/common/ofautils/common_ofautils.h"
 /* Define this before including hwloc.h so that we also get the hwloc
   verbs helper header file, too.  We have to do this level of
   indirection because the hwloc subsystem is a component -- we don't
@ -2341,53 +2343,6 @@ static int finish_btl_init(mca_btl_openib_module_t *openib_btl)
    return OMPI_SUCCESS;
 }

-static struct ibv_device **ibv_get_device_list_compat(int *num_devs)
-{
-    struct ibv_device **ib_devs;
-
-#ifdef HAVE_IBV_GET_DEVICE_LIST
-    ib_devs = ibv_get_device_list(num_devs);
-#else
-    struct dlist *dev_list;
-    struct ibv_device *ib_dev;
-    *num_devs = 0;
-
-    /* Determine the number of device's available on the host */
-    dev_list = ibv_get_devices();
-    if (NULL == dev_list)
-        return NULL;
-
-    dlist_start(dev_list);
-
-    dlist_for_each_data(dev_list, ib_dev, struct ibv_device)
-        (*num_devs)++;
-
-    /* Allocate space for the ib devices */
-    ib_devs = (struct ibv_device**)malloc(*num_devs * sizeof(struct ibv_dev*));
-    if(NULL == ib_devs) {
-        *num_devs = 0;
-        BTL_ERROR(("Failed malloc: %s:%d", __FILE__, __LINE__));
-        return NULL;
-    }
-
-    dlist_start(dev_list);
-
-    dlist_for_each_data(dev_list, ib_dev, struct ibv_device)
-        *(++ib_devs) =  ib_dev;
-#endif
-
-    return ib_devs;
-}
-
-static void ibv_free_device_list_compat(struct ibv_device **ib_devs)
-{
-#ifdef HAVE_IBV_GET_DEVICE_LIST
-    ibv_free_device_list(ib_devs);
-#else
-    free(ib_devs);
-#endif
-}
-
 struct dev_distance {
    struct ibv_device *ib_dev;
    int distance;
--- a/ompi/mca/btl/openib/configure.m4
+++ b/ompi/mca/btl/openib/configure.m4
@ -39,8 +39,9 @@ AC_DEFUN([MCA_ompi_btl_openib_CONFIG],[
    OPAL_VAR_SCOPE_PUSH([cpcs have_threads])
    cpcs="oob"

-    OMPI_CHECK_OPENIB([btl_openib],
-                     [btl_openib_happy="yes"],
+    OMPI_CHECK_OPENFABRICS([btl_openib],
+                     [btl_openib_happy="yes"]
+                     OMPI_CHECK_OPENFABRICS_CM([btl_openib]),
                     [btl_openib_happy="no"])

    AS_IF([test "$btl_openib_happy" = "yes"],
--- a/ompi/mca/common/ofacm/Makefile.am
+++ b/ompi/mca/common/ofacm/Makefile.am
@ -0,0 +1,76 @@
+#
+# Copyright (c) 2009-2012 Mellanox Technologies.  All rights reserved.
+# Copyright (c) 2009 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2009-2012 Oak Ridge National Laboratory.  All rights reserved.
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+
+AM_CPPFLAGS = $(common_ofacm_CPPFLAGS)
+
+dist_pkgdata_DATA = \
+        help-mpi-common-ofacm-base.txt \
+        help-mpi-common-ofacm-oob.txt
+headers = \
+	base.h \
+	common_ofacm_oob.h \
+	common_ofacm_empty.h \
+	connect.h
+
+sources = \
+    common_ofacm_base.c \
+    common_ofacm_oob.c \
+    common_ofacm_empty.c
+
+# If we have XRC support, build XOOB connection module
+if MCA_common_ofacm_have_xrc
+sources += \
+    common_ofacm_xoob.c \
+    common_ofacm_xoob.h
+endif
+
+# See ompi/mca/common/sm/Makefile.am for an explanation of the
+# LTLIBRARIES values listed below.
+
+lib_LTLIBRARIES =
+noinst_LTLIBRARIES =
+comp_inst = libmca_common_ofacm.la
+comp_noinst = libmca_common_ofacm_noinst.la
+
+if MCA_BUILD_ompi_common_ofacm_DSO
+lib_LTLIBRARIES += $(comp_inst)
+else
+noinst_LTLIBRARIES += $(comp_noinst)
+endif
+
+libmca_common_ofacm_la_SOURCES = $(headers) $(sources)
+libmca_common_ofacm_la_CPPFLAGS = $(common_ofacm_CPPFLAGS)
+libmca_common_ofacm_la_LDFLAGS = $(common_ofacm_LDFLAGS)
+libmca_common_ofacm_la_LIBADD = $(common_ofacm_LIBS)
+libmca_common_ofacm_noinst_la_SOURCES = $(libmca_common_ofacm_la_SOURCES)
+
+# Conditionally install the header files
+
+if WANT_INSTALL_HEADERS
+ompidir = $(includedir)/openmpi/ompi/mca/common/ofacm
+ompi_HEADERS = $(headers)
+else
+ompidir = $(includedir)
+endif
+
+# See ompi/mca/common/sm/Makefile.am for an explanation of the
+# the *-local rules, below.
+
+all-local:
+	if test -z "$(lib_LTLIBRARIES)"; then \
+	  rm -f "$(comp_inst)"; \
+	  $(LN_S) "$(comp_noinst)" "$(comp_inst)"; \
+	fi
+
+clean-local:
+	if test -z "$(lib_LTLIBRARIES)"; then \
+	  rm -f "$(comp_inst)"; \
+	fi
--- a/ompi/mca/common/ofacm/base.h
+++ b/ompi/mca/common/ofacm/base.h
@ -0,0 +1,193 @@
+/*
+ * Copyright (c) 2007-2008 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2009-2012 Mellanox Technologies.  All rights reserved.
+ *
+ * Copyright (c) 2009-2012 Oak Ridge National Laboratory.  All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#ifndef OMPI_COMMON_OFACM_BASE_H
+#define OMPI_COMMON_OFACM_BASE_H
+#include "ompi_config.h"
+
+#include <stdio.h>
+#include <stdarg.h>
+
+#include "orte/runtime/orte_globals.h"
+#include "orte/util/proc_info.h"
+#include "orte/util/name_fns.h"
+#include "connect.h"
+
+BEGIN_C_DECLS
+
+#define HAVE_XRC (1 == OMPI_HAVE_CONNECTX_XRC)
+
+extern int ompi_common_ofacm_base_output;
+extern int ompi_common_ofacm_base_verbose; /* disabled by default */
+/* File for sl data produced only for a 3D-Torus Cluster */
+extern char* ompi_common_ofacm_three_dim_torus; 
+
+static inline int ompi_common_ofacm_base_err(const char* fmt, ...)
+{
+    va_list list;
+    int ret;
+
+    va_start(list, fmt);
+    ret = vfprintf(stderr, fmt, list);
+    va_end(list);
+    return ret;
+}
+
+#define OFACM_ERROR(args)                                  \
+    do {                                                   \
+        ompi_common_ofacm_base_err("[%s]%s[%s:%d:%s] ",     \
+            orte_process_info.nodename,                     \
+            ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),             \
+            __FILE__, __LINE__, __func__);                  \
+        ompi_common_ofacm_base_err args;                    \
+        ompi_common_ofacm_base_err("\n");                   \
+    } while(0);
+
+#if OPAL_ENABLE_DEBUG
+#define OFACM_VERBOSE(args)                                    \
+    do {                                                         \
+        if(ompi_common_ofacm_base_verbose > 0) {                            \
+            ompi_common_ofacm_base_err("[%s]%s[%s:%d:%s] ",                \
+                    orte_process_info.nodename,                   \
+                    ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),  \
+                    __FILE__, __LINE__, __func__);               \
+            ompi_common_ofacm_base_err args;                               \
+            ompi_common_ofacm_base_err("\n");                              \
+        }                                                         \
+    } while(0); 
+#else
+#define OFACM_VERBOSE(args) 
+#endif
+
+/* 
+ * PUBLIC functions
+ * ****************
+ */
+
+/*
+ * Open function
+ */
+OMPI_DECLSPEC int ompi_common_ofacm_base_register(mca_base_component_t *base);
+
+/*
+ * Query CPCs to see if they want to run on a specific port.
+ * Input:
+ *  port - port information
+ * Output:
+ *  cpcs - list of availible cpcs
+ *  num_cpcs - number of cpcs
+ */
+OMPI_DECLSPEC int ompi_common_ofacm_base_select_for_local_port
+    (ompi_common_ofacm_base_dev_desc_t *dev,
+     ompi_common_ofacm_base_module_t ***cpcs, int *num_cpcs);
+
+/*
+ * Select function
+ * Input:
+ * local_cpcs - local cpc modules
+ * num_local_cpcs - number of local cpc modules
+ * remote_cpc_data - cpc information from remote peer
+ * remote_cpc_data_count - num of remote information from remote peer
+ * Output:
+ * ret_local_cpc - matched cpc module
+ * ret_remote_cpc_data - matched remote cpc data
+ */
+OMPI_DECLSPEC int ompi_common_ofacm_base_find_match
+    (ompi_common_ofacm_base_module_t **local_cpcs, int num_local_cpcs,
+     ompi_common_ofacm_base_module_data_t *remote_cpc_data, int remote_cpc_data_count,
+     ompi_common_ofacm_base_module_t **ret_local_cpc,
+     ompi_common_ofacm_base_module_data_t **ret_remote_cpc_data);
+
+/*
+ * Find a CPC's index so that we can send it in the modex
+ */
+OMPI_DECLSPEC int ompi_common_ofacm_base_get_cpc_index
+    (ompi_common_ofacm_base_component_t *cpc);
+
+/*
+ * Start a new connection to an endpoint
+ */
+OMPI_DECLSPEC int ompi_common_ofacm_base_start_connect
+        (struct ompi_common_ofacm_base_local_connection_context_t *context);
+
+/*
+ * Component-wide CPC finalize
+ */
+OMPI_DECLSPEC void ompi_common_ofacm_base_finalize(void);
+
+/*
+ * Component-wide CPC init
+ */
+OMPI_DECLSPEC int ompi_common_ofacm_base_init(void);
+
+/*
+ * Lookup a CPC by its index (received from the modex)
+ */
+OMPI_DECLSPEC ompi_common_ofacm_base_component_t *
+    ompi_common_ofacm_base_get_cpc_byindex(uint8_t index);
+
+/* 
+ * PRIVATE functions (called only by cpcs)
+ * ***************************************
+ */
+
+/*
+ * Proc initialization function
+ */ 
+void ompi_common_ofacm_base_proc_setup
+    (ompi_common_ofacm_base_proc_t *proc, 
+     ompi_common_ofacm_base_local_connection_context_t *context,
+     ompi_proc_t *proc_ompi);
+/*
+ * Context initialization function
+ */ 
+int ompi_common_ofacm_base_context_init
+    (ompi_common_ofacm_base_local_connection_context_t *context,
+     ompi_common_ofacm_base_module_t *cpc,
+     ompi_common_ofacm_base_context_connect_cb_fn_t connect_cb,
+     ompi_common_ofacm_base_context_error_cb_fn_t error_cb,
+     ompi_common_ofacm_base_context_prepare_recv_cb_fn_t prepare_recv_cb,
+     ompi_common_ofacm_base_proc_t *proc,
+     ompi_common_ofacm_base_qp_config_t *qp_config,
+     struct ibv_pd *pd, uint64_t subnet_id, int cpc_type, 
+     uint16_t lid, uint16_t rem_lid, 
+     int32_t user_context_index, void *user_context);
+
+/*
+ *  Remote context initialization.
+ *  Returns operation status
+ */ 
+int ompi_common_ofacm_base_remote_context_init
+    (ompi_common_ofacm_base_remote_connection_context_t *context,
+     int num_qps, int num_srqs);
+
+/* Find OFACM proc on specific component */
+ompi_common_ofacm_base_proc_t* ompi_common_ofacm_base_find_proc
+        (ompi_common_ofacm_base_component_t *component, ompi_proc_t *proc);
+
+#if 0
+/* 
+ * Allocate a CTS frag
+ */
+int ompi_common_ofacm_base_alloc_cts(
+        struct mca_btl_base_endpoint_t *endpoint);
+
+/* 
+ * Free a CTS frag
+ */
+int ompi_common_ofacm_base_free_cts(
+        struct mca_btl_base_endpoint_t *endpoint);
+#endif
+
+END_C_DECLS
+
+#endif
--- a/ompi/mca/common/ofacm/common_ofacm_base.c
+++ b/ompi/mca/common/ofacm/common_ofacm_base.c
@ -0,0 +1,678 @@
+/*
+ * Copyright (c) 2007-2009 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2007-2012 Mellanox Technologies.  All rights reserved.
+ *
+ * Copyright (c) 2009-2012 Oak Ridge National Laboratory.  All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+#include "ompi_config.h"
+
+#include <string.h>
+#include "base.h"
+#include "common_ofacm_oob.h"
+#include "common_ofacm_empty.h"
+#if HAVE_XRC
+#include "common_ofacm_xoob.h"
+#endif
+
+#include "ompi/constants.h"
+#include "orte/util/show_help.h"
+#include "opal/class/opal_list.h"
+#include "opal/util/argv.h"
+#include "opal/util/output.h"
+#include "infiniband/verbs.h"
+
+/* Global variables */
+static ompi_common_ofacm_base_component_t **available = NULL;
+static int num_available = 0;
+int ompi_common_ofacm_base_verbose = 0; /* disabled by default */
+char* ompi_common_ofacm_three_dim_torus = NULL;
+bool cpc_explicitly_defined = false;
+int ompi_common_ofacm_base_output = 1;
+bool ompi_common_ofacm_base_register_was_called = false;
+bool ompi_common_ofacm_base_init_was_called = false;
+/*
+ * Array of all possible connection functions
+ */
+static ompi_common_ofacm_base_component_t *all[] = {
+    &ompi_common_ofacm_oob,
+
+    /* Always have an entry here so that the CP indexes will always be
+       the same: if XRC is not available, use the "empty" CPC */
+#if HAVE_XRC
+    &ompi_common_ofacm_xoob,
+#else
+    &ompi_common_ofacm_empty,
+#endif
+    NULL
+};
+
+static void ofacm_base_proc_contructor (ompi_common_ofacm_base_proc_t *proc)
+{
+    proc->proc_ompi = NULL;
+    OBJ_CONSTRUCT(&proc->all_contexts, opal_list_t);
+}
+
+static void ofacm_base_proc_destructor (ompi_common_ofacm_base_proc_t *proc)
+{
+    OBJ_DESTRUCT(&proc->all_contexts);
+}
+
+void ompi_common_ofacm_base_proc_setup(ompi_common_ofacm_base_proc_t *proc, 
+        ompi_common_ofacm_base_local_connection_context_t *context,
+        ompi_proc_t *proc_ompi) 
+{
+    if (NULL == proc->proc_ompi) {
+        /* first init for the proc, lets set ompi proc */
+        proc->proc_ompi = proc_ompi;
+    }
+    /* put the context on the proc list */
+    opal_list_append(&proc->all_contexts, (opal_list_item_t *)context);
+}
+
+OBJ_CLASS_INSTANCE(ompi_common_ofacm_base_proc_t,
+                   opal_list_item_t,
+                   ofacm_base_proc_contructor,
+                   ofacm_base_proc_destructor);
+
+/* Constructors / Destructors */
+static void base_local_context_constructor
+    (ompi_common_ofacm_base_local_connection_context_t *context)
+{
+    context->proc = NULL;
+    context->state = MCA_COMMON_OFACM_CLOSED;
+    context->subnet_id = 0;
+    context->lid = 0;
+    context->num_of_qps = 0;
+    context->init_attr = NULL;
+    context->attr = NULL;
+    context->ib_pd = NULL;
+    context->qps = NULL;
+    context->user_context = NULL;
+    context->initiator = 0;
+    context->index = 0;
+    context->xrc_recv_qp_num = 0;
+    /* remote info we will allocate and fill during qp creation */
+    memset(&context->remote_info, 0, sizeof(context->remote_info));
+    OBJ_CONSTRUCT(&context->context_lock, opal_mutex_t);
+}
+
+static void base_local_context_destructor
+    (ompi_common_ofacm_base_local_connection_context_t *context)
+{
+    /* Release remote data arrays */
+    if (NULL != context->remote_info.rem_qps) {
+        free(context->remote_info.rem_qps);
+    }
+    if (NULL != context->remote_info.rem_srqs) {
+        free(context->remote_info.rem_srqs);
+    }
+    OBJ_DESTRUCT(&context->context_lock);
+}
+
+OBJ_CLASS_INSTANCE(ompi_common_ofacm_base_local_connection_context_t,
+                   opal_list_item_t,
+                   base_local_context_constructor,
+                   base_local_context_destructor);
+
+int ompi_common_ofacm_base_context_init(ompi_common_ofacm_base_local_connection_context_t *context,
+                                         ompi_common_ofacm_base_module_t *cpc,
+                                         ompi_common_ofacm_base_context_connect_cb_fn_t connect_cb,
+                                         ompi_common_ofacm_base_context_error_cb_fn_t error_cb,
+                                         ompi_common_ofacm_base_context_prepare_recv_cb_fn_t prepare_recv_cb,
+                                         ompi_common_ofacm_base_proc_t *proc,
+                                         ompi_common_ofacm_base_qp_config_t *qp_config,
+                                         struct ibv_pd *pd, uint64_t subnet_id, int cpc_type, 
+                                         uint16_t lid, uint16_t rem_lid, 
+                                         int32_t user_context_index, void *user_context)
+{
+    context->proc = proc;
+    context->cpc = cpc;
+    context->subnet_id = subnet_id;
+    context->cpc_type = cpc_type;
+    context->lid = lid;
+    context->rem_lid = rem_lid;
+    context->num_of_qps = qp_config->num_qps;
+    /* If upper layer defines the QPs we do not want to overwrite it */
+    if (NULL == context->qps) {
+        context->qps = calloc(context->num_of_qps, sizeof(ompi_common_ofacm_base_qp_t));
+        if(NULL == context->qps) {
+            OFACM_ERROR(("Failed to allocate memory for qps"));
+            return OMPI_ERR_OUT_OF_RESOURCE;
+        }
+    }
+
+    context->num_of_srqs = qp_config->num_srqs;
+    context->srq_num = qp_config->srq_num;
+    context->init_attr = qp_config->init_attr;
+    context->attr = qp_config->attr;
+    context->custom_init_attr_mask = qp_config->init_attr_mask;
+    context->custom_rtr_attr_mask = qp_config->rtr_attr_mask;
+    context->custom_rts_attr_mask = qp_config->rts_attr_mask;
+    context->ib_pd = pd;
+    context->connect_cb = connect_cb;
+    context->error_cb = error_cb;
+    context->prepare_recv_cb = prepare_recv_cb ;
+    context->index = user_context_index;
+    context->user_context = user_context;
+    return OMPI_SUCCESS;
+}
+
+int ompi_common_ofacm_base_remote_context_init(ompi_common_ofacm_base_remote_connection_context_t *context,
+                                                int num_qps, int num_srqs)
+{
+    context->rem_qps = (ompi_common_ofacm_base_rem_qp_info_t *)
+        calloc(num_qps, sizeof(ompi_common_ofacm_base_rem_qp_info_t));
+    if (NULL == context->rem_qps) {
+        return OMPI_ERROR;
+    }
+
+    context->rem_srqs = (ompi_common_ofacm_base_rem_srq_info_t *)
+        calloc(num_qps, sizeof(ompi_common_ofacm_base_rem_srq_info_t));
+    if (NULL == context->rem_srqs) {
+        return OMPI_ERROR;
+    }
+
+    return OMPI_SUCCESS; 
+}
+
+ompi_common_ofacm_base_proc_t* ompi_common_ofacm_base_find_proc
+        (ompi_common_ofacm_base_component_t *component, ompi_proc_t *proc)
+{
+    ompi_common_ofacm_base_proc_t *ret = NULL;
+    opal_list_item_t *item;
+    opal_list_t *list = &component->all_procs;
+
+    for (item = opal_list_get_first(list);
+            item != opal_list_get_end(list);
+            item = opal_list_get_next(item)) {
+        if (proc == ((ompi_common_ofacm_base_proc_t *)item)->proc_ompi){
+            ret = (ompi_common_ofacm_base_proc_t *)item;
+        }
+    }
+    return ret;
+}
+/*
+ * Register MCA parameters
+ */
+int ompi_common_ofacm_base_register(mca_base_component_t *base)
+{
+    int i, j, save;
+    char **temp = NULL, *string = NULL, *all_cpc_names = NULL;
+    char *cpc_include = NULL, *cpc_exclude = NULL;
+
+    if (ompi_common_ofacm_base_register_was_called) {
+        return OMPI_SUCCESS;
+    }
+
+    ompi_common_ofacm_base_register_was_called = true;
+
+    /* Make an MCA parameter to select which connect module to use */
+    for (i = 0; NULL != all[i]; ++i) {
+        /* The CPC name "empty" is reserved for "fake" CPC modules */
+        if (0 != strcmp(all[i]->cbc_name, "empty")) {
+            opal_argv_append_nosize(&temp, all[i]->cbc_name);
+        }
+    }
+    all_cpc_names = opal_argv_join(temp, ',');
+    opal_argv_free(temp);
+    asprintf(&string,
+             "Method used to select OpenFabrics connections (valid values: %s)",
+             all_cpc_names);
+
+    mca_base_param_reg_string(base, "ofacm_cpc_include", string, false, false,
+            NULL, &cpc_include);
+    free(string);
+
+    asprintf(&string,
+             "Method used to exclude OpenFabrics connections (valid values: %s)",
+             all_cpc_names);
+
+    mca_base_param_reg_string(base, "ofacm_cpc_exclude", string, false, false, 
+            NULL, &cpc_exclude);
+    free(string);
+
+    /* Register the name of the file containing the fabric's Service Levels (SL) */
+    mca_base_param_reg_string_name("common", "ofacm_three_dim_torus",
+                    "The name of the file contating Service Level (SL) data for 3D-Torus cluster",
+                    false, false, NULL, &ompi_common_ofacm_three_dim_torus);
+
+
+    mca_base_param_reg_int_name("common", 
+                                "ofacm_base_verbose", 
+                                "Verbosity level of the OFACM framework", 
+                                false, false, 
+                                0, 
+                                &ompi_common_ofacm_base_verbose);
+
+
+    /* Parse the if_[in|ex]clude paramters to come up with a list of
+       CPCs that are available */
+    available = calloc(1, sizeof(all));
+
+    /* If we have an "include" list, then find all those CPCs and put
+       them in available[] */
+    if (NULL != cpc_include) {
+        cpc_explicitly_defined = true;
+        temp = opal_argv_split(cpc_include, ',');
+        for (save = j = 0; NULL != temp[j]; ++j) {
+            for (i = 0; NULL != all[i]; ++i) {
+                if (0 == strcmp(temp[j], all[i]->cbc_name)) { 
+                    OFACM_VERBOSE(("include: saving %s", all[i]->cbc_name));
+                    available[save++] = all[i];
+                    ++num_available;
+                    break;
+                }
+            }
+            if (NULL == all[i]) {
+                orte_show_help("help-mpi-common-ofacm-cpc-base.txt",
+                               "cpc name not found", true,
+                               "include", orte_process_info.nodename,
+                               "include", cpc_include, temp[j], 
+                               all_cpc_names);
+                opal_argv_free(temp);
+                free(all_cpc_names);
+                return OMPI_ERR_NOT_FOUND;
+            }
+        }
+        opal_argv_free(temp);
+    }
+
+    /* Otherwise, if we have an "exclude" list, take all the CPCs that
+       are not in that list and put them in available[] */
+    else if (NULL != cpc_exclude) {
+        cpc_explicitly_defined = true;
+        temp = opal_argv_split(cpc_exclude, ',');
+        /* First: error check -- ensure that all the names are valid */
+        for (j = 0; NULL != temp[j]; ++j) {
+            for (i = 0; NULL != all[i]; ++i) {
+                if (0 == strcmp(temp[j], all[i]->cbc_name)) { 
+                    break;
+                }
+            }
+            if (NULL == all[i]) {
+                orte_show_help("help-mpi-common-ofacm-cpc-base.txt",
+                               "cpc name not found", true,
+                               "exclude", orte_process_info.nodename,
+                               "exclude", cpc_exclude, temp[j], 
+                               all_cpc_names);
+                opal_argv_free(temp);
+                free(all_cpc_names);
+                return OMPI_ERR_NOT_FOUND;
+            }
+        }
+
+        /* Now do the exclude */
+        for (save = i = 0; NULL != all[i]; ++i) {
+            for (j = 0; NULL != temp[j]; ++j) {
+                if (0 == strcmp(temp[j], all[i]->cbc_name)) {
+                    break;
+                }
+            }
+            if (NULL == temp[j]) {
+                OFACM_VERBOSE(("exclude: saving %s", all[i]->cbc_name));
+                available[save++] = all[i];
+                ++num_available;
+            }
+        }
+        opal_argv_free(temp);
+    } 
+
+    /* If there's no include/exclude list, copy all[] into available[] */
+    else {
+        OFACM_VERBOSE(("no include or exclude: saving all"));
+        memcpy(available, all, sizeof(all));
+        num_available = (sizeof(all) / 
+                         sizeof(ompi_common_ofacm_base_module_t *)) - 1;
+    }
+
+    /* Call the register function on all the CPCs so that they may
+       setup any MCA params specific to the connection type */
+    for (i = 0; NULL != available[i]; ++i) {
+        if (NULL != available[i]->cbc_register) {
+            available[i]->cbc_register();
+        }
+    }
+
+    return OMPI_SUCCESS;
+}
+
+/*
+ * Called once during openib BTL component initialization to allow CPC
+ * components to initialize.
+ */
+int ompi_common_ofacm_base_init(void)
+{
+    int i, rc;
+
+    if (ompi_common_ofacm_base_init_was_called) {
+        return OMPI_SUCCESS;
+    }
+
+    ompi_common_ofacm_base_init_was_called = true;
+
+    /* Call each available CPC component's open function, if it has
+       one.  If the CPC component open function returns OMPI_SUCCESS,
+       keep it.  If it returns ERR_NOT_SUPPORTED, remove it from the
+       available[] array.  If it returns something else, return that
+       error upward. */
+    for (i = num_available = 0; NULL != available[i]; ++i) {
+        if (NULL == available[i]->cbc_init) {
+            available[num_available++] = available[i];
+            OFACM_VERBOSE(("found available cpc (NULL init): %s",
+                        all[i]->cbc_name));
+            continue;
+        }
+
+        rc = available[i]->cbc_init();
+        if (OMPI_SUCCESS == rc) {
+            available[num_available++] = available[i];
+            OFACM_VERBOSE(("found available cpc (SUCCESS init): %s",
+                        all[i]->cbc_name));
+            continue;
+        } else if (OMPI_ERR_NOT_SUPPORTED == rc) {
+            continue;
+        } else {
+            return rc;
+        }
+    }
+    available[num_available] = NULL;
+
+    return (num_available > 0) ? OMPI_SUCCESS : OMPI_ERR_NOT_AVAILABLE;
+}
+
+
+/*
+ * Find all the CPCs that are eligible for a single local port (i.e.,
+ * openib module).
+ */
+int ompi_common_ofacm_base_select_for_local_port(ompi_common_ofacm_base_dev_desc_t *dev,
+        ompi_common_ofacm_base_module_t ***cpcs, int *num_cpcs)
+{
+    char *msg = NULL;
+    int i, rc, cpc_index, len;
+    ompi_common_ofacm_base_module_t **tmp_cpcs;
+
+    tmp_cpcs = calloc(num_available, 
+                  sizeof(ompi_common_ofacm_base_module_t *));
+    if (NULL == tmp_cpcs) {
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+
+    /* Go through all available CPCs and query them to see if they
+       want to run on this module.  If they do, save them to a running
+       array. */
+    for (len = 1, i = 0; NULL != available[i]; ++i) {
+        len += strlen(available[i]->cbc_name) + 2;
+    }
+    msg = malloc(len);
+    if (NULL == msg) {
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+    msg[0] = '\0';
+    for (cpc_index = i = 0; NULL != available[i]; ++i) {
+        if (i > 0) {
+            strcat(msg, ", ");
+        }
+        strcat(msg, available[i]->cbc_name);
+
+        rc = available[i]->cbc_query(dev, &tmp_cpcs[cpc_index]);
+        if (OMPI_ERR_NOT_SUPPORTED == rc || OMPI_ERR_UNREACH == rc) {
+            continue;
+        } else if (OMPI_SUCCESS != rc) {
+            free(tmp_cpcs);
+            free(msg);
+            return rc;
+        }
+        OFACM_VERBOSE(("match cpc for local port: %s",
+                    available[i]->cbc_name));
+
+        /* If the CPC wants to use the CTS protocol, check to ensure
+           that QP 0 is PP; if it's not, we can't use this CPC (or the
+           CTS protocol) */
+        /* Pasha: Wrong place to check qp type, should be moved to CMs 
+        if (cpcs[cpc_index]->cbm_uses_cts &&
+            !BTL_OPENIB_QP_TYPE_PP(0)) {
+            OFACM_VERBOSE(("this CPC only supports when the first btl_openib_receive_queues QP is a PP QP"));
+            continue;
+        }
+        */
+
+        /* This CPC has indicated that it wants to run on this openib
+           BTL module.  Woo hoo! */
+        ++cpc_index;
+    }
+
+    /* If we got an empty array, then no CPCs were eligible.  Doh! */
+    if (0 == cpc_index) {
+        orte_show_help("help-mpi-common-ofacm-cpc-base.txt",
+                       "no cpcs for port", true,
+                       orte_process_info.nodename,
+                       ibv_get_device_name(dev->ib_dev),
+                       msg);
+        free(tmp_cpcs);
+        free(msg);
+        return OMPI_ERR_NOT_SUPPORTED;
+    }
+    free(msg);
+
+    /* We got at least one eligible CPC; save the array into the
+       module's port_info */
+    *num_cpcs = cpc_index;
+    *cpcs = tmp_cpcs;
+
+    return OMPI_SUCCESS;
+}
+
+/* 
+ * This function is invoked when determining whether we have a CPC in
+ * common with a specific remote port.  We already know that the
+ * subnet ID is the same between a specific local port and the target
+ * remote port; now we need to know if we can find a CPC in common
+ * between the two.
+ *
+ * If yes, be sure to find the *same* CPC on both sides.  We know
+ * which CPCs are available on each side, and we know the priorities
+ * that were assigned on both sides.  So find a CPC that is common to
+ * both sides and has the highest overall priority (between both
+ * sides).
+ *
+ * Return the matching CPC, or NULL if not found.
+ */
+int
+ompi_common_ofacm_base_find_match(ompi_common_ofacm_base_module_t **local_cpcs, int num_local_cpcs,
+        ompi_common_ofacm_base_module_data_t *remote_cpc_data, int remote_cpc_data_count,
+        ompi_common_ofacm_base_module_t **ret_local_cpc,
+        ompi_common_ofacm_base_module_data_t **ret_remote_cpc_data)
+{
+    int i, j, max = -1;
+    ompi_common_ofacm_base_module_t *local_cpc, *local_selected = NULL;
+    ompi_common_ofacm_base_module_data_t *local_cpcd, *remote_cpcd,
+        *remote_selected = NULL;
+
+    /* Iterate over all the CPCs on the local module */
+    for (i = 0; i < num_local_cpcs; ++i) {
+        local_cpc = local_cpcs[i];
+        local_cpcd = &(local_cpc->data);
+
+        /* Iterate over all the CPCs on the remote port */
+        for (j = 0; j < remote_cpc_data_count; ++j) {
+            remote_cpcd = &(remote_cpc_data[j]);
+
+            /* Are the components the same? */
+            if (local_cpcd->cbm_component == remote_cpcd->cbm_component) {
+                /* If so, update the max priority found so far */
+                if (max < local_cpcd->cbm_priority) {
+                    max = local_cpcd->cbm_priority;
+                    local_selected = local_cpc;
+                    remote_selected = remote_cpcd;
+                }
+                if (max < remote_cpcd->cbm_priority) {
+                    max = remote_cpcd->cbm_priority;
+                    local_selected = local_cpc;
+                    remote_selected = remote_cpcd;
+                }
+            }
+        }
+    }
+
+    /* All done! */
+    if (NULL != local_selected) {
+        *ret_local_cpc = local_selected;
+        *ret_remote_cpc_data = remote_selected;
+        OFACM_VERBOSE(("find_match: found match!"));
+        return OMPI_SUCCESS;
+    } else {
+        OFACM_VERBOSE(("find_match: did NOT find match!"));
+        return OMPI_ERR_NOT_FOUND;
+    }
+}
+
+/*
+ * Lookup a CPC component's index in the all[] array so that we can
+ * send it int the modex
+ */
+int ompi_common_ofacm_base_get_cpc_index(ompi_common_ofacm_base_component_t *cpc)
+{
+    int i;
+    for (i = 0; NULL != all[i]; ++i) {
+        if (all[i] == cpc) {
+            return i;
+        }
+    }
+
+    /* Not found */
+    return -1;
+}
+
+/*
+ * Lookup a CPC by its index (received from the modex)
+ */
+ompi_common_ofacm_base_component_t *
+ompi_common_ofacm_base_get_cpc_byindex(uint8_t index)
+{
+    return (index >= (sizeof(all) / 
+                      sizeof(ompi_common_ofacm_base_module_t *))) ?
+        NULL : all[index];
+}
+
+/*
+ * This function we never call from BTL - so it is no reason to expose it 
+ * in base.
+ */
+#if 0
+int ompi_common_ofacm_base_alloc_cts(mca_btl_base_endpoint_t *endpoint)
+{
+    ompi_free_list_item_t *fli;
+    int length = sizeof(mca_btl_openib_header_t) +
+        sizeof(mca_btl_openib_header_coalesced_t) +
+        sizeof(mca_btl_openib_control_header_t) +
+        sizeof(mca_btl_openib_footer_t) +
+        mca_btl_openib_component.qp_infos[mca_btl_openib_component.credits_qp].size;
+
+    /* Explicitly don't use the mpool registration */
+    fli = &(endpoint->endpoint_cts_frag.super.super.base.super);
+    fli->registration = NULL;
+    fli->ptr = malloc(length);
+    if (NULL == fli->ptr) {
+        BTL_ERROR(("malloc failed"));
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+
+    endpoint->endpoint_cts_mr = 
+        ibv_reg_mr(endpoint->endpoint_btl->device->ib_pd, 
+                   fli->ptr, length,
+                   IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE |
+                   IBV_ACCESS_REMOTE_READ);
+    OPAL_OUTPUT((-1, "registered memory %p, length %d", fli->ptr, length));
+    if (NULL == endpoint->endpoint_cts_mr) {
+        free(fli->ptr);
+        BTL_ERROR(("Failed to reg mr!"));
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+
+    /* Copy the lkey where it needs to go */
+    endpoint->endpoint_cts_frag.super.sg_entry.lkey = 
+        endpoint->endpoint_cts_frag.super.super.segment.seg_key.key32[0] = 
+        endpoint->endpoint_cts_mr->lkey;
+    endpoint->endpoint_cts_frag.super.sg_entry.length = length;
+
+    /* Construct the rest of the recv_frag_t */
+    OBJ_CONSTRUCT(&(endpoint->endpoint_cts_frag), mca_btl_openib_recv_frag_t);
+    endpoint->endpoint_cts_frag.super.super.base.order = 
+        mca_btl_openib_component.credits_qp;
+    endpoint->endpoint_cts_frag.super.endpoint = endpoint;
+    OPAL_OUTPUT((-1, "Got a CTS frag for peer %s, addr %p, length %d, lkey %d",
+                 endpoint->endpoint_proc->proc_ompi->proc_hostname,
+                 (void*) endpoint->endpoint_cts_frag.super.sg_entry.addr,
+                 endpoint->endpoint_cts_frag.super.sg_entry.length,
+                 endpoint->endpoint_cts_frag.super.sg_entry.lkey));
+
+    return OMPI_SUCCESS;
+}
+#endif
+/* This function is needed for CTS packet release on completion..
+ * and it is bad idea...it is 2 possible solutions:
+ * - make the send operation blocking (simple and not optimal).
+ * - rdmacm should add own progress function (best but not trivial).
+ */
+#if 0
+int ompi_common_ofacm_base_free_cts(mca_btl_base_endpoint_t *endpoint)
+{
+    if (NULL != endpoint->endpoint_cts_mr) {
+        ibv_dereg_mr(endpoint->endpoint_cts_mr);
+        endpoint->endpoint_cts_mr = NULL;
+    }
+    if (NULL != endpoint->endpoint_cts_frag.super.super.base.super.ptr) {
+        free(endpoint->endpoint_cts_frag.super.super.base.super.ptr);
+        endpoint->endpoint_cts_frag.super.super.base.super.ptr = NULL;
+        OPAL_OUTPUT((-1, "Freeing CTS frag"));
+    }
+
+    return OMPI_SUCCESS;
+}
+#endif
+
+/*
+ * Called to start a connection
+ */
+int ompi_common_ofacm_base_start_connect(
+        ompi_common_ofacm_base_local_connection_context_t *context)
+{
+#if 0
+    /* If the CPC uses the CTS protocol, provide a frag buffer for the
+       CPC to post.  Must allocate these frags up here in the main
+       thread because the FREE_LIST_WAIT is not thread safe. */
+    if (cpc->cbm_uses_cts) {
+        int rc;
+        rc = ompi_common_ofacm_base_alloc_cts(context);
+        if (OMPI_SUCCESS != rc) {
+            return rc;
+        }
+    }
+#endif
+
+    return context->cpc->cbm_start_connect(context);
+}
+
+/*
+ * Called during openib btl component close
+ */
+void ompi_common_ofacm_base_finalize(void)
+{
+    int i;
+
+    if (NULL != available) {
+        for (i = 0; NULL != available[i]; ++i) {
+            if (NULL != available[i]->cbc_finalize) {
+                available[i]->cbc_finalize();
+            }
+        }
+        free(available);
+        available = NULL;
+    }
+}
--- a/ompi/mca/common/ofacm/common_ofacm_empty.c
+++ b/ompi/mca/common/ofacm/common_ofacm_empty.c
@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2008 Cisco Systems, Inc.  All rights reserved.
+ *
+ * Copyright (c) 2009-2012 Oak Ridge National Laboratory.  All rights reserved.
+ * Copyright (c) 2009-2012 Mellanox Technologies.  All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "base.h"
+#include "connect.h"
+#include "ompi/constants.h"
+
+static void empty_component_register(void);
+static int empty_component_init(void);
+static int empty_component_query(ompi_common_ofacm_base_dev_desc_t *dev, 
+                                 ompi_common_ofacm_base_module_t **cpc);
+
+ompi_common_ofacm_base_component_t ompi_common_ofacm_empty = {
+    "empty",
+    empty_component_register,
+    empty_component_init,
+    empty_component_query,
+    NULL
+};
+
+static void empty_component_register(void)
+{
+    /* Nothing to do */
+}
+
+static int empty_component_init(void)
+{
+    /* Never let this CPC run */
+    return OMPI_ERR_NOT_SUPPORTED;
+}
+
+static int empty_component_query(ompi_common_ofacm_base_dev_desc_t *dev, 
+                                 ompi_common_ofacm_base_module_t **cpc)
+{
+    /* Never let this CPC run */
+    return OMPI_ERR_NOT_SUPPORTED;
+}
--- a/ompi/mca/common/ofacm/common_ofacm_empty.h
+++ b/ompi/mca/common/ofacm/common_ofacm_empty.h
@ -0,0 +1,22 @@
+/*
+ * Copyright (c) 2007-2008 Cisco Systems, Inc.  All rights reserved.
+ *
+ * Copyright (c) 2009-2012 Oak Ridge National Laboratory.  All rights reserved.
+ * Copyright (c) 2009-2012 Mellanox Technologies.  All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#ifndef BTL_OPENIB_CONNECT_EMPTY_H
+#define BTL_OPENIB_CONNECT_EMPTY_H
+
+#include "ompi_config.h"
+
+#include "connect.h"
+
+extern ompi_common_ofacm_base_component_t ompi_common_ofacm_empty;
+
+#endif
--- a/ompi/mca/common/ofacm/common_ofacm_oob.c
+++ b/ompi/mca/common/ofacm/common_ofacm_oob.c
--- a/ompi/mca/common/ofacm/common_ofacm_oob.h
+++ b/ompi/mca/common/ofacm/common_ofacm_oob.h
@ -0,0 +1,20 @@
+/*
+ * Copyright (c) 2007-2008 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2009-2012 Mellanox Technologies.  All rights reserved.
+ *
+ * Copyright (c) 2009-2012 Oak Ridge National Laboratory.  All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#ifndef COMMON_OFACM_OOB_H
+#define COMMON_OFACM_OOB_H
+
+#include "connect.h"
+
+extern ompi_common_ofacm_base_component_t ompi_common_ofacm_oob;
+
+#endif
--- a/ompi/mca/common/ofacm/common_ofacm_xoob.c
+++ b/ompi/mca/common/ofacm/common_ofacm_xoob.c
--- a/ompi/mca/common/ofacm/common_ofacm_xoob.h
+++ b/ompi/mca/common/ofacm/common_ofacm_xoob.h
@ -0,0 +1,69 @@
+/*
+ * Copyright (c) 2007-2012 Mellanox Technologies.  All rights reserved.
+ * Copyright (c) 2008 Cisco Systems, Inc.  All rights reserved.
+ *
+ * Copyright (c) 2009-2012 Oak Ridge National Laboratory.  All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#ifndef COMMON_OFACM_XOOB_H
+#define COMMON_OFACM_XOOB_H
+
+#include "opal/class/opal_hash_table.h"
+#include "connect.h"
+
+extern ompi_common_ofacm_base_component_t ompi_common_ofacm_xoob;
+
+typedef enum {
+    XOOB_ADDR_CONNECTING = 100,
+    XOOB_ADDR_CONNECTED,
+    XOOB_ADDR_CLOSED
+} ompi_common_ofacm_ib_addr_state_t;
+
+struct ib_address_t {
+    opal_list_item_t super;
+    void *key;                                /* the key with size 80bit - [subnet(64) LID(16bit)] */
+    uint64_t subnet_id;                       /* caching subnet_id  */
+    uint16_t lid;                             /* caching lid */
+    opal_list_t pending_contexts;             /* list of endpoints that use this ib_address */
+    struct ompi_common_ofacm_base_qp_t *qps;  /* pointer to qp that will be used
+                                                 for communication with the
+                                                 destination */
+    uint32_t remote_xrc_rcv_qp_num;           /* remote xrc qp number */
+    opal_mutex_t addr_lock;                   /* protection */
+    ompi_common_ofacm_ib_addr_state_t status; /* ib port status */
+};
+typedef struct ib_address_t 
+               ib_address_t;
+
+struct ompi_common_ofacm_xoob_local_connection_context_t {
+    ompi_common_ofacm_base_local_connection_context_t super;
+    ib_address_t *addr;
+    uint32_t xrc_recv_qp_num; /* in xrc we will use it as recv qp */
+    uint32_t xrc_recv_psn;
+};
+typedef struct ompi_common_ofacm_xoob_local_connection_context_t 
+               ompi_common_ofacm_xoob_local_connection_context_t;
+OMPI_DECLSPEC OBJ_CLASS_DECLARATION(ompi_common_ofacm_xoob_local_connection_context_t);
+
+struct ompi_common_ofacm_xoob_module_t {
+    ompi_common_ofacm_base_module_t super;
+    opal_hash_table_t ib_addr_table; /**< used only for xrc.hash-table that
+                                       keeps table of all lids/subnets */
+};
+typedef struct ompi_common_ofacm_xoob_module_t
+               ompi_common_ofacm_xoob_module_t;
+
+struct pending_context_t {
+    opal_list_item_t super;
+    ompi_common_ofacm_xoob_local_connection_context_t *xcontext;
+};
+typedef struct pending_context_t
+               pending_context_t;
+OMPI_DECLSPEC OBJ_CLASS_DECLARATION(pending_context_t);
+
+#endif
--- a/ompi/mca/common/ofacm/configure.m4
+++ b/ompi/mca/common/ofacm/configure.m4
@ -0,0 +1,63 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2007-2009 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2009-2012 Mellanox Technologies.  All rights reserved.
+# Copyright (c) 2009-2012 Oak Ridge National Laboratory.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# MCA_ompi_common_ofacm_CONFIG([should_build])
+# ------------------------------------------
+AC_DEFUN([MCA_ompi_common_ofacm_POST_CONFIG], [
+    AM_CONDITIONAL([MCA_common_ofacm_have_xrc], [test $1 -eq 1 -a "x$common_ofacm_have_xrc" = "x1"])
+])
+
+
+# MCA_ompi_common_ofacm_CONFIG([action-if-can-compile], 
+#                      [action-if-cant-compile])
+# ------------------------------------------------
+AC_DEFUN([MCA_ompi_common_ofacm_CONFIG],[
+    AC_CONFIG_FILES([ompi/mca/common/ofacm/Makefile])
+    OPAL_VAR_SCOPE_PUSH([modules ofacm_have_threads])
+    modules="oob"
+
+    common_ofacm_happy="no"
+    OMPI_CHECK_OPENFABRICS([common_ofacm],
+                           [common_ofacm_happy="yes"
+                            OMPI_CHECK_OPENFABRICS_CM([common_ofacm])])
+
+    AS_IF([test "$common_ofacm_happy" = "yes"],
+          [common_ofacm_WRAPPER_EXTRA_LDFLAGS="$common_ofacm_LDFLAGS"
+           common_ofacm_WRAPPER_EXTRA_LIBS="$common_ofacm_LIBS"
+           $1],
+          [$2])
+
+    AS_IF([test "$common_ofacm_happy" = "yes"],
+          [if test "x$common_ofacm_have_xrc" = "x1"; then
+              modules="$modules xoob"
+          fi
+          AC_MSG_CHECKING([which OpenFabrics CM modules will be built])
+          AC_MSG_RESULT([$modules])])
+
+    # substitute in the things needed to build openib
+    AC_SUBST([common_ofacm_CFLAGS])
+    AC_SUBST([common_ofacm_CPPFLAGS])
+    AC_SUBST([common_ofacm_LDFLAGS])
+    AC_SUBST([common_ofacm_LIBS])
+
+    OPAL_VAR_SCOPE_POP
+])dnl
--- a/ompi/mca/common/ofacm/configure.params
+++ b/ompi/mca/common/ofacm/configure.params
@ -0,0 +1,26 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2007      Los Alamos National Security, LLC.  All rights
+#                         reserved. 
+# Copyright (c) 2009-2012 Oak Ridge National Laboratory.  All rights reserved.
+# Copyright (c) 2009-2012 Mellanox Technologies.  All rights reserved.
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+
+# Specific to this module
+
+PARAM_CONFIG_FILES="Makefile"
--- a/ompi/mca/common/ofacm/connect.h
+++ b/ompi/mca/common/ofacm/connect.h
@ -0,0 +1,541 @@
+/*
+ * Copyright (c) 2007-2008 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2009 Mellanox Technogies, Inc.  All rights reserved.
+ *
+ * Copyright (c) 2009-2012 Oak Ridge National Laboratory.  All rights reserved.
+ * Copyright (c) 2009-2012 Mellanox Technologies.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+/**
+ * @file
+ *
+ * This interface is designed to hide the back-end details of how IB
+ * RC connections are made from the rest of the openib BTL.  There are
+ * module-like instances of the implemented functionality (dlopen and
+ * friends are not used, but all the functionality is accessed through
+ * struct's of function pointers, so you can swap between multiple
+ * different implementations at run time, just like real components).
+ * Hence, these entities are referred to as "Connect
+ * Pseudo-Components" (CPCs).
+ *
+ * The CPCs are referenced by their names (e.g., "oob", "rdma_cm").
+ *
+ * CPCs are split into components and modules, similar to all other
+ * MCA frameworks in this code base.
+ *
+ * Before diving into the CPC interface, let's discuss some
+ * terminology and mappings of data structures:
+ *
+ * - a BTL module represents a network port (in the case of the openib
+ *   BTL, a LID)
+ * - a CPC module represents one way to make connections to a BTL module
+ * - hence, a BTL module has potentially multiple CPC modules
+ *   associated with it
+ * - an endpoint represnts a connection between a local BTL module and
+ *   a remote BTL module (in the openib BTL, because of BSRQ, an
+ *   endpoint can contain multiple QPs)
+ * - when an endpoint is created, one of the CPC modules associated
+ *   with the local BTL is selected and associated with the endpoint
+ *   (obviously, it is a CPC module that is common between the local
+ *   and remote BTL modules)
+ * - endpoints may be created and destroyed during the MPI job
+ * - endpoints are created lazily, during the first communication
+ *   between two peers
+ * - endpoints are destroyed when two MPI processes become
+ *   disconnected (e.g., MPI-2 dynamics or MPI_FINALIZE)
+ * - hence, BTL modules and CPC modules outlive endpoints.
+ *   Specifically, BTL modules and CPC modules live from MPI_INIT to
+ *   MPI_FINALIZE. endpoints come and go as MPI semantics demand it.
+ * - therefore, CPC modules need to cache information on endpoints that
+ *   are specific to that connection.
+ *
+ * Component interface:
+ *
+ * - component_register(): The openib BTL's component_open() function
+ * calls the connect_base_register() function, which scans all
+ * compiled-in CPC's.  If they have component_register() functions,
+ * they are called (component_register() functions are only allowed to
+ * register MCA parameters).
+ *
+ * NOTE: The connect_base_register() function will process the
+ * btl_openib_cpc_include and btl_openib_cpc_exclude MCA parameters
+ * and automatically include/exclude CPCs as relevant.  If a CPC is
+ * excluded, none of its other interface functions will be invoked for
+ * the duration of the process.
+ *
+ * - component_init(): The openib BTL's component_init() function
+ * calls connect_base_init(), which will invoke this query function on
+ * each CPC to see if it wants to run at all.  CPCs can gracefully
+ * remove themselves from consideration in this process by returning
+ * OMPI_ERR_NOT_SUPPORTED.
+ *
+ * - component_query(): The openib BTL's init_one_port() calls the
+ * connect_base_select_for_local_port() function, which, for each LID
+ * on that port, calls the component_query() function on every
+ * available CPC on that LID.  This function is intended to see if a
+ * CPC can run on a sepcific openib BTL module (i.e., LID).  If it
+ * can, the CPC is supposed to create a CPC module that is specific to
+ * that BTL/LID and return it.  If it cannot, it should return
+ * OMPI_ERR_NOT_SUPPORTED and be gracefully skipped for this
+ * OpenFabrics port.
+ *
+ * component_finalize(): The openib BTL's component_close() function
+ * calls connect_base_finalize(), which, in turn, calls the
+ * component_finalize() function on all available CPCs.  Note that all
+ * CPC modules will have been finalized by this point; the CPC
+ * component_finalize() function is a chance for the CPC to clean up
+ * any component-specific resources.
+ *
+ * Module interface:
+ *
+ * cbm_component member: A pointer pointing to the single, global
+ * instance of the CPC component.  This member is used for creating a
+ * unique index representing the modules' component so that it can be
+ * shared with remote peer processes.
+ *
+ * cbm_priority member: An integer between 0 and 100, inclusive,
+ * representing the priority of this CPC.
+ *
+ * cbm_modex_message member: A pointer to a blob buffer that will be
+ * included in the modex message for this port for this CPC (it is
+ * assumed that this blob is a) only understandable by the
+ * corresponding CPC in the peer process, and b) contains specific
+ * addressing/contact information for *this* port's CPC module).
+ *
+ * cbm_modex_message_len member: The length of the cbm_modex_message
+ * blob, in bytes.
+ *
+ * cbm_endpoint_init(): Called during endpoint creation, allowing a
+ * CPC module to cache information on the endpoint.  A pointer to the
+ * endpoint's CPC module is already cached on the endpoint.
+ *
+ * cbm_start_connect(): initiate a connection to a remote peer.  The
+ * CPC is responsible for setting itself up for asyncronous operation
+ * for progressing the outgoing connection request.
+ *
+ * cbm_endpoint_finalize(): Called during the endpoint destrouction,
+ * allowing the CPC module to destroy anything that it cached on the
+ * endpoint.
+ *
+ * cbm_finalize(): shut down all asynchronous handling and clean up
+ * any state that was setup for this CPC module/BTL.  Some CPCs setup
+ * asynchronous support on a per-HCA/NIC basis (vs. per-port/LID).  It
+ * is the reponsibility of the CPC to figure out such issues (e.g.,
+ * via reference counting) -- there is no notification from the
+ * upper-level BTL about when an entire HCA/NIC is no longer being
+ * used.  There is only this function, which tells when a specific
+ * CPC/BTL module is no longer being used.
+ *
+ * cbm_uses_cts: a bool that indicates whether the CPC will use the
+ * CTS protocol or not.
+ *   - if true: the CPC will post the fragment on
+ *     endpoint->endpoint_cts_frag as a receive buffer and will *not*
+ *     call ompi_btl_openib_post_recvs().
+ *   - if false: the CPC will call ompi_btl_openib_post_recvs() before
+ *     calling ompi_btl_openib_cpc_complete().
+ *
+ * There are two functions in the main openib BTL that the CPC may
+ * call:
+ *
+ * - ompi_btl_openib_post_recvs(endpoint): once a QP is locally
+ * connected to the remote side (but we don't know if the remote side
+ * is connected to us yet), this function is invoked to post buffers
+ * on the QP, setup credits for the endpoint, etc.  This function is
+ * *only* invoked if the CPC's cbm_uses_cts is false.
+ *
+ * - ompi_btl_openib_cpc_complete(endpoint): once that a CPC knows
+ * that a QP is connected on *both* sides, this function is invoked to
+ * tell the main openib BTL "ok, you can use this connection now."
+ * (e.g., the main openib BTL will either invoke the CTS protocol or
+ * start sending out fragments that were queued while the connection
+ * was establishing, etc.).
+ */
+#ifndef OMPI_COMMON_OFACM_CONNECT_H
+#define OMPI_COMMON_OFACM_CONNECT_H
+
+/* System includes */
+#include <infiniband/verbs.h>
+
+/* Open MPI includes */
+#include "ompi/proc/proc.h"
+
+BEGIN_C_DECLS
+
+#define BCF_MAX_NAME 64
+
+/**
+ * Must forward declare these structs to avoid include file loops.
+ */
+
+/**
+ * This is struct is defined below
+ */
+struct ompi_common_ofacm_base_module_t;
+
+/* special capabilities */
+#define OMPI_COMMON_OFACM_XRC_ONLY        1
+#define OMPI_COMMON_OFACM_IWARP_ONLY      1 << 1
+
+/**
+ * State of OFACM connection.
+ */
+
+typedef enum {
+    /* Defines the state in which this BTL instance
+     * has started the process of connection */
+    MCA_COMMON_OFACM_CONNECTING,
+
+    /* Waiting for ack from endpoint */
+    MCA_COMMON_OFACM_CONNECT_ACK,
+
+    /*Waiting for final connection ACK from endpoint */
+    MCA_COMMON_OFACM_WAITING_ACK,
+
+    /* Connected ... both sender & receiver have
+     * buffers associated with this connection */
+    MCA_COMMON_OFACM_CONNECTED,
+
+    /* Connection is closed, there are no resources
+     * associated with this */
+    MCA_COMMON_OFACM_CLOSED,
+
+    /* Maximum number of retries have been used.
+     * Report failure on send to upper layer */
+    MCA_COMMON_OFACM_FAILED,
+
+    /* We found is useful to have one more
+     * state that maybe utilized for user needs */
+    MCA_COMMON_OFACM_USER_CUSTOM
+} ompi_common_ofacm_connection_state_t;
+
+typedef enum {
+    MCA_COMMON_OFACM_BTL = 0,
+    MCA_COMMON_OFACM_COLL = 100
+} ompi_common_ofacm_type;
+
+typedef struct ompi_common_ofacm_base_dev_desc_t {
+    struct ibv_device* ib_dev;          /* device */
+    struct ibv_context* ib_dev_context; /* device context */
+    int capabilities;                   /* Special capabilities like: XRC, Iwarp, etc.. */
+} ompi_common_ofacm_base_dev_desc_t;
+
+/* QPs configuration container that should be filled by
+ * upper layer, for example - btl */
+typedef struct ompi_common_ofacm_base_qp_config_t {
+    int num_qps;
+    int num_srqs;
+    struct ibv_qp_init_attr *init_attr;
+    struct ibv_qp_attr *attr;
+    uint32_t *srq_num;
+    uint32_t *init_attr_mask;
+    uint32_t *rtr_attr_mask;
+    uint32_t *rts_attr_mask;
+} ompi_common_ofacm_base_qp_config_t;
+
+/* QP base data */
+typedef struct ompi_common_ofacm_base_qp_t {
+    struct ibv_qp *lcl_qp;
+    size_t ib_inline_max; /**< max size of IB inline send */
+    uint32_t lcl_psn;
+    int32_t  sd_wqe;      /**< number of available send wqe entries */
+    int users;
+    opal_mutex_t lock;
+} ompi_common_ofacm_base_qp_t;
+
+/* Remote QP info */
+typedef struct ompi_common_ofacm_base_rem_qp_info_t {
+    uint32_t                    rem_qp_num;
+    /* Remote QP number */
+    uint32_t                    rem_psn;
+    /* Remote processes port sequence number */
+} ompi_common_ofacm_base_rem_qp_info_t;
+
+/* Remote SRQ info */
+typedef struct ompi_common_ofacm_base_rem_srq_info_t {
+    /* Remote SRQ number */
+    uint32_t                    rem_srq_num;
+} ompi_common_ofacm_base_rem_srq_info_t;
+
+/* Remote connection context */
+typedef struct ompi_common_ofacm_base_remote_connection_context_t {
+    opal_object_t               super;
+    /* Local identifier of the remote process */
+    uint16_t                    rem_lid;
+    /* subnet id of remote process */
+    uint64_t                    rem_subnet_id;
+    /* MTU of remote process */
+    uint32_t                    rem_mtu; /* TBD: not sure that i need this one */
+    /* index of remote endpoint in endpoint array */
+    uint32_t                    rem_index; /* TBD: the index we use as immidiate data */
+    /* Remote QPs */
+    ompi_common_ofacm_base_rem_qp_info_t *rem_qps;
+    /* Remote xrc_srq info, used only with XRC connections */
+    ompi_common_ofacm_base_rem_srq_info_t *rem_srqs;
+} ompi_common_ofacm_base_remote_connection_context_t;
+
+typedef struct ompi_common_ofacm_base_proc_t {
+    opal_list_item_t super;
+    ompi_proc_t *proc_ompi; /* target proc */
+    opal_list_t all_contexts; /* list of all contexts connected to
+                                 this endpoint*/
+} ompi_common_ofacm_base_proc_t;
+OMPI_DECLSPEC OBJ_CLASS_DECLARATION(ompi_common_ofacm_base_proc_t);
+
+/* Connection call back function that is called on connection setup */
+typedef void (*ompi_common_ofacm_base_context_connect_cb_fn_t)(void *);
+
+/* Connection call back function that is called on context error */
+typedef void (*ompi_common_ofacm_base_context_error_cb_fn_t)(void *);
+/* Prepare recive call back function that is when recv side should be prepared,
+ * for example recv packet prepost */
+typedef int (*ompi_common_ofacm_base_context_prepare_recv_cb_fn_t)(void *);
+
+/* Basic connection context
+ * ========================
+ * The initial connection contxet is created during endpoint initialazation call.
+ * Each CPC will return ompi_common_ofacm_base_local_connection_context_t that
+ * is based on CPC connection context.
+ *
+ * As Input for context creation user must provide:
+ * ================================================
+ * number of QPs
+ * qp init atributes
+ * qp standart attribute
+ * pointer to protection domain
+ * pointer to user context (for example pointer to endpoint in case of btl)
+ */
+typedef struct ompi_common_ofacm_base_local_connection_context_t {
+    opal_list_item_t super;
+    struct ompi_common_ofacm_base_proc_t *proc;         /* target proc */
+    struct ompi_common_ofacm_base_module_t *cpc;        /* Pointer to context cpc */
+    ompi_common_ofacm_connection_state_t state;         /* Connection context status */
+    uint64_t subnet_id;                                 /* caching subnet_id */
+    int cpc_type;                                       /* connection manager family: openib, coll, etc..*/
+    uint16_t lid;                                       /* caching lid */
+    uint16_t rem_lid;                                   /* remote lid */
+    uint8_t num_of_qps;                                 /* Number of qps that we want to open */
+    struct ompi_common_ofacm_base_qp_t *qps;            /* qps data */
+    uint8_t num_of_srqs;                                /* Number of qps that we want to open */
+    uint32_t *srq_num;                                  /* srq numbers for recv on this context */
+    struct ibv_qp_init_attr *init_attr;                 /* list of initial attr for each qp   */
+    struct ibv_qp_attr *attr;                           /* qp attributes                      */
+    struct ibv_pd* ib_pd;                               /* protection domain                  */
+    uint32_t *custom_init_attr_mask;                    /* in additional to standard attr_mask we want allow to user
+                                                           specify special custom masks for init */
+    uint32_t *custom_rtr_attr_mask;                     /* in additional to standard attr_mask we want allow to user
+                                                          specify special custom masks for rtr */
+    uint32_t *custom_rts_attr_mask;                     /* in additional to standard attr_mask we want allow to user
+                                                          specify special custom masks for rts */
+    void *user_context;                                 /* back pointer to endpoint */
+    ompi_common_ofacm_base_context_connect_cb_fn_t connect_cb; /* Connection callback function */
+    ompi_common_ofacm_base_context_error_cb_fn_t error_cb; /* Error callback function */
+    ompi_common_ofacm_base_context_prepare_recv_cb_fn_t prepare_recv_cb; /* Prepare recv side
+                                                                            (prepost) callback function */
+    /* TBD: Need to check when we can update the index. I think during endpoint creation we do not
+     * have it. It mean that BTL should some how to update it later ...*/
+    int32_t index;                                      /* user context index */
+    bool initiator;                                     /* initiator of connection ? */
+    ompi_common_ofacm_base_remote_connection_context_t remote_info; /* data about remote side of this
+                                                                       connection*/
+    uint32_t xrc_recv_qp_num ;                          /* in xrc we will use it as recv qp */
+
+    opal_mutex_t context_lock;                          /* protection */
+} ompi_common_ofacm_base_local_connection_context_t;
+OMPI_DECLSPEC OBJ_CLASS_DECLARATION(ompi_common_ofacm_base_local_connection_context_t);
+/* Constructor and destructor are located in common_ofacm_base.c */
+
+/************************************************************************/
+
+/**
+ * Function to register MCA params in the connect functions.  It
+ * returns no value, so it cannot fail.
+ */
+typedef void (*ompi_common_ofacm_base_component_register_fn_t)(void);
+
+/**
+ * This function is invoked once by the openib BTL component during
+ * startup.  It is intended to have CPC component-wide startup.
+ *
+ * Return value:
+ *
+ * - OMPI_SUCCESS: this CPC component will be used in selection during
+ *   this process.
+ *
+ * - OMPI_ERR_NOT_SUPPORTED: this CPC component will be silently
+ *   ignored in this process.
+ *
+ * - Other OMPI_ERR_* values: the error will be propagated upwards,
+ *   likely causing a fatal error (and/or the openib BTL component
+ *   being ignored).
+ */
+typedef int (*ompi_common_ofacm_base_component_init_fn_t)(void);
+
+/**
+ * Query the CPC to see if it wants to run on a specific port (i.e., a
+ * specific BTL module).  If the component init function previously
+ * returned OMPI_SUCCESS, this function is invoked once per BTL module
+ * creation (i.e., for each port found by an MPI process).  If this
+ * CPC wants to be used on this BTL module, it returns a CPC module
+ * that is specific to this BTL module.
+ *
+ * The BTL module in question is passed to the function; all of its
+ * attributes can be used to query to see if it's eligible for this
+ * CPC.
+ *
+ * If it is eligible, the CPC is responsible for creating a
+ * corresponding CPC module, filling in all the relevant fields on the
+ * modules, and for setting itself up to run (per above) and returning
+ * a CPC module (this is effectively the "module_init" function).
+ * Note that the module priority must be between 0 and 100
+ * (inclusive).  When multiple CPCs are eligible for a single module,
+ * the CPC with the highest priority will be used.
+ *
+ * Return value:
+ *
+ * - OMPI_SUCCESS if this CPC is eligible for and was able to be setup
+ * for this BTL module.  It is assumed that the CPC is now completely
+ * setup to run on this openib module (per description above).
+ *
+ * - OMPI_ERR_NOT_SUPPORTED if this CPC cannot support this BTL
+ * module.  This is not an error; it's just the CPC saying "sorry, I
+ * cannot support this BTL module."
+ *
+ * - Other OMPI_ERR_* code: an error occurred.
+ */
+typedef int (*ompi_common_ofacm_base_func_component_query_t)
+    (struct ompi_common_ofacm_base_dev_desc_t *dev,
+     struct ompi_common_ofacm_base_module_t **cpc);
+
+/**
+ * This function is invoked once by the openib BTL component during
+ * shutdown.  It is intended to have CPC component-wide shutdown.
+ */
+typedef int (*ompi_common_ofacm_base_component_finalize_fn_t)(void);
+
+/**
+ * CPC component struct
+ */
+typedef struct ompi_common_ofacm_base_component_t {
+    /** Name of this set of connection functions */
+    char cbc_name[BCF_MAX_NAME];
+
+    /** Register function.  Can be NULL. */
+    ompi_common_ofacm_base_component_register_fn_t cbc_register;
+
+    /** CPC component init function.  Can be NULL. */
+    ompi_common_ofacm_base_component_init_fn_t cbc_init;
+
+    /** Query the CPC component to get a CPC module corresponding to
+        an openib BTL module.  Cannot be NULL. */
+    ompi_common_ofacm_base_func_component_query_t cbc_query;
+
+    /** CPC component finalize function.  Can be NULL. */
+    ompi_common_ofacm_base_component_finalize_fn_t cbc_finalize;
+    /** All connection contexts that are using this CPC **/
+    opal_list_t all_procs;
+} ompi_common_ofacm_base_component_t;
+
+/************************************************************************/
+
+/**
+ * Function called when an endpoint has been created and has been
+ * associated with a CPC.
+ */
+typedef ompi_common_ofacm_base_local_connection_context_t*
+    (*ompi_common_ofacm_base_module_endpoint_init_fn_t)
+    (ompi_proc_t *proc,
+     ompi_common_ofacm_base_qp_config_t *qp_config, struct ibv_pd *pd,
+     uint64_t subnet_id, int cpc_type, uint16_t lid, uint16_t rem_lid,
+     int32_t user_context_index, void *user_context,
+     struct ompi_common_ofacm_base_module_t *cpc,
+     ompi_common_ofacm_base_context_connect_cb_fn_t connect_cb,
+     ompi_common_ofacm_base_context_error_cb_fn_t error_cb,
+     ompi_common_ofacm_base_context_prepare_recv_cb_fn_t prepare_recv_cb);
+
+/**
+ * Function to initiate a connection to a remote process.
+ */
+typedef int (*ompi_common_ofacm_base_module_start_connect_fn_t)
+    (struct ompi_common_ofacm_base_local_connection_context_t *context);
+
+/**
+ * Function called when an endpoint is being destroyed.
+ */
+typedef int (*ompi_common_ofacm_base_module_endpoint_finalize_fn_t)
+     (struct ompi_common_ofacm_base_local_connection_context_t *context);
+
+/**
+ * Function to finalize the CPC module.  It is called once when the
+ * CPC module's corresponding openib BTL module is being finalized.
+ */
+typedef int (*ompi_common_ofacm_base_module_finalize_fn_t)(void);
+
+/**
+ * Error callback that is called by cpc module on error.
+ * The callback should be set on upper layer (for example BTL)
+ */
+typedef int (*ompi_common_ofacm_base_module_error_cb_fn_t)(void *);
+/**
+ * Meta data about a CPC module.  This is in a standalone struct
+ * because it is used in both the CPC module struct and the
+ * openib_btl_proc_t struct to hold information received from the
+ * modex.
+ */
+typedef struct ompi_common_ofacm_base_module_data_t {
+    /** Pointer back to the component.  Used by the base and openib
+        btl to calculate this module's index for the modex. */
+    ompi_common_ofacm_base_component_t *cbm_component;
+
+    /** Priority of the CPC module (must be >=0 and <=100) */
+    uint8_t cbm_priority;
+
+    /** Blob that the CPC wants to include in the openib modex message
+        for a specific port, or NULL if the CPC does not want to
+        include a message in the modex.  */
+    void *cbm_modex_message;
+
+    /** Length of the cbm_modex_message blob (0 if
+        cbm_modex_message==NULL).  The message is intended to be short
+        (because the size of the modex broadcast is a function of
+        sum(cbm_modex_message_len[i]) for
+        i=(0...total_num_ports_in_MPI_job) -- e.g., IBCM imposes its
+        own [very short] limits (per IBTA volume 1, chapter 12). */
+    uint8_t cbm_modex_message_len;
+} ompi_common_ofacm_base_module_data_t;
+
+/**
+ * Struct for holding CPC module and associated meta data
+ */
+typedef struct ompi_common_ofacm_base_module_t {
+    /** Meta data about the module */
+    ompi_common_ofacm_base_module_data_t data;
+
+    /** Endpoint initialization function */
+    ompi_common_ofacm_base_module_endpoint_init_fn_t cbm_endpoint_init;
+
+    /** Connect function */
+    ompi_common_ofacm_base_module_start_connect_fn_t cbm_start_connect;
+
+    /** Endpoint finalization function */
+    ompi_common_ofacm_base_module_endpoint_finalize_fn_t cbm_endpoint_finalize;
+
+    /** Finalize the cpc module */
+    ompi_common_ofacm_base_module_finalize_fn_t cbm_finalize;
+
+    /** Whether this module will use the CTS protocol or not.  This
+        directly states whether this module will call
+        mca_btl_openib_endpoint_post_recvs() or not: true = this
+        module will *not* call _post_recvs() and instead will post the
+        receive buffer provided at endpoint->endpoint_cts_frag on qp
+        0. */
+    bool cbm_uses_cts;
+} ompi_common_ofacm_base_module_t;
+
+END_C_DECLS
+
+#endif
--- a/ompi/mca/common/ofacm/help-mpi-common-ofacm-base.txt
+++ b/ompi/mca/common/ofacm/help-mpi-common-ofacm-base.txt
@ -0,0 +1,41 @@
+# -*- text -*-
+#
+# Copyright (c) 2008 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2009-2012 Oak Ridge National Laboratory.  All rights reserved.
+# Copyright (c) 2009-2012 Mellanox Technologies.  All rights reserved.
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+# This is the US/English help file for Open MPI's OpenFabrics IB CPC
+# support.
+#
+[no cpcs for port]
+No OpenFabrics connection schemes reported that they were able to be
+used on a specific port.  As such, the openib BTL (OpenFabrics
+support) will be disabled for this port.
+
+  Local host:           %s
+  Local device:         %s
+  CPCs attempted:       %s
+#
+[cpc name not found]
+An invalid CPC name was specified via the btl_openib_cpc_%s MCA 
+parameter.
+
+  Local host:                   %s
+  btl_openib_cpc_%s value: %s
+  Invalid name:                 %s
+  All possible valid names:     %s
+#
+[inline truncated]
+WARNING: The btl_openib_max_inline_data MCA parameter was used to
+specify how much inline data should be used, but a device reduced this
+value.  This is not an error; it simply means that your run will use
+a smaller inline data value than was requested.
+
+  Local host:           %s
+  Requested value:      %d
+  Value used by device: %d
--- a/ompi/mca/common/ofacm/help-mpi-common-ofacm-oob.txt
+++ b/ompi/mca/common/ofacm/help-mpi-common-ofacm-oob.txt
@ -0,0 +1,20 @@
+# -*- text -*-
+#
+# Copyright (c) 2009-2012 Mellanox Technologies.  All rights reserved.
+# Copyright (c) 2009-2012 Oak Ridge National Laboratory.  All rights reserved.
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+[ofacm oob fatal error]
+The OOB OpenFabrics Connection Manager module tried to raise fatal error,
+but failed. 
+
+  Local host: %s
+  Source file: %s
+  Source line: %d
+
+Your job is now going to abort, sorry.
+#
--- a/ompi/mca/common/ofautils/Makefile.am
+++ b/ompi/mca/common/ofautils/Makefile.am
@ -0,0 +1,68 @@
+#
+# Copyright (c) 2009-2012 Mellanox Technologies.  All rights reserved.
+# Copyright (c) 2009-2012 Oak Ridge National Laboratory.  All rights reserved.
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+
+AM_CPPFLAGS = $(common_ofautils_CPPFLAGS)
+
+headers = \
+	common_ofautils.h
+
+sources = \
+	common_ofautils.c
+
+# To simplify components that link to this library, we will *always*
+# have an output libtool library named libmca_<type>_<name>.la -- even
+# for case 2) described above (i.e., so there's no conditional logic
+# necessary in component Makefile.am's that link to this library).
+# Hence, if we're creating a noinst version of this library (i.e.,
+# case 2), we sym link it to the libmca_<type>_<name>.la name
+# (libtool will do the Right Things under the covers).  See the
+# all-local and clean-local rules, below, for how this is effected.
+
+lib_LTLIBRARIES =
+noinst_LTLIBRARIES =
+comp_inst = libmca_common_ofautils.la
+comp_noinst = libmca_common_ofautils_noinst.la
+
+if MCA_BUILD_ompi_common_ofautils_DSO
+lib_LTLIBRARIES += $(comp_inst)
+else
+noinst_LTLIBRARIES += $(comp_noinst)
+endif
+
+libmca_common_ofautils_la_SOURCES = $(headers) $(sources)
+# TBD: create own config file 
+libmca_common_ofautils_la_CPPFLAGS = $(common_ofautils_CPPFLAGS)
+libmca_common_ofautils_la_LDFLAGS =  $(common_ofautils_LDFLAGS)
+libmca_common_ofautils_la_LIBADD =   $(common_ofautils_LIBS)
+libmca_common_ofautils_noinst_la_SOURCES = $(libmca_common_ofautils_la_SOURCES)
+
+# Conditionally install the header files
+
+if WANT_INSTALL_HEADERS
+ompidir = $(includedir)/openmpi/ompi/mca/common/ofautils
+ompi_HEADERS = $(headers)
+else
+ompidir = $(includedir)
+endif
+
+# These two rules will sym link the "noinst" libtool library filename
+# to the installable libtool library filename in the case where we are
+# compiling this component statically (case 2), described above).
+
+all-local:
+	if test -z "$(lib_LTLIBRARIES)"; then \
+	  rm -f "$(comp_inst)"; \
+	  $(LN_S) "$(comp_noinst)" "$(comp_inst)"; \
+	fi
+
+clean-local:
+	if test -z "$(lib_LTLIBRARIES)"; then \
+	  rm -f "$(comp_inst)"; \
+	fi
--- a/ompi/mca/common/ofautils/common_ofautils.c
+++ b/ompi/mca/common/ofautils/common_ofautils.c
@ -0,0 +1,89 @@
+/* -*- Mode: C; c-basic-offset:4 ; -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2008 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2009 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2006-2012 Mellanox Technologies.  All rights reserved.
+ * Copyright (c) 2006-2007 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2006-2007 Voltaire All rights reserved.
+ * Copyright (c) 2009      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2009-2012 Oak Ridge National Laboratory.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include <infiniband/verbs.h>
+/* This is crummy, but <infiniband/driver.h> doesn't work on all
+   platforms with all compilers.  Specifically, trying to include it
+   on RHEL4U3 with the PGI 32 bit compiler will cause problems because
+   certain 64 bit types are not defined.  Per advice from Roland D.,
+   just include the one prototype that we need in this case
+   (ibv_get_sysfs_path()). */
+#ifdef HAVE_INFINIBAND_DRIVER_H
+#include <infiniband/driver.h>
+#else
+const char *ibv_get_sysfs_path(void);
+#endif
+#include "opal/util/output.h"
+#include "common_ofautils.h"
+
+struct ibv_device **ibv_get_device_list_compat(int *num_devs)
+{
+    struct ibv_device **ib_devs;
+
+#ifdef HAVE_IBV_GET_DEVICE_LIST
+    ib_devs = ibv_get_device_list(num_devs);
+#else
+    struct dlist *dev_list;
+    struct ibv_device *ib_dev;
+    *num_devs = 0;
+
+    /* Determine the number of device's available on the host */
+    dev_list = ibv_get_devices();
+    if (NULL == dev_list)
+        return NULL;
+
+    dlist_start(dev_list);
+
+    dlist_for_each_data(dev_list, ib_dev, struct ibv_device)
+        (*num_devs)++;
+
+    /* Allocate space for the ib devices */
+    ib_devs = (struct ibv_device**)malloc(*num_devs * sizeof(struct ibv_dev*));
+    if(NULL == ib_devs) {
+        *num_devs = 0;
+        opal_output("Failed malloc: %s:%d", __FILE__, __LINE__);
+        return NULL;
+    }
+
+    dlist_start(dev_list);
+
+    dlist_for_each_data(dev_list, ib_dev, struct ibv_device)
+        *(++ib_devs) =  ib_dev;
+#endif
+
+    return ib_devs;
+}
+
+void ibv_free_device_list_compat(struct ibv_device **ib_devs)
+{
+#ifdef HAVE_IBV_GET_DEVICE_LIST
+    ibv_free_device_list(ib_devs);
+#else
+    free(ib_devs);
+#endif
+}
--- a/ompi/mca/common/ofautils/common_ofautils.h
+++ b/ompi/mca/common/ofautils/common_ofautils.h
@ -0,0 +1,26 @@
+/*
+ * Copyright (c) 2009-2012 Mellanox Technologies.  All rights reserved.
+ *                         All rights reserved.
+ * Copyright (c) 2009-2012 Oak Ridge National Laboratory.  All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#ifndef _COMMON_OFAUTILS_H_
+#define _COMMON_OFAUTILS_H_
+
+#include "ompi_config.h"
+
+OMPI_DECLSPEC extern
+struct ibv_device **ibv_get_device_list_compat(int *num_devs);
+
+OMPI_DECLSPEC extern
+void ibv_free_device_list_compat(struct ibv_device **ib_devs);
+
+END_C_DECLS
+
+#endif
+
--- a/ompi/mca/common/ofautils/configure.m4
+++ b/ompi/mca/common/ofautils/configure.m4
@ -0,0 +1,43 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2007-2009 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2009-2012 Mellanox Technologies.  All rights reserved.
+# Copyright (c) 2009-2012 Oak Ridge National Laboratory.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# MCA_ompi_common_ofautils_CONFIG([action-if-can-compile], 
+#                      [action-if-cant-compile])
+# ------------------------------------------------
+AC_DEFUN([MCA_ompi_common_ofautils_CONFIG],[
+    AC_CONFIG_FILES([ompi/mca/common/ofautils/Makefile])
+    common_ofautils_happy="no"
+    OMPI_CHECK_OPENFABRICS([common_ofautils],
+                           [common_ofautils_happy="yes"])
+
+    AS_IF([test "$common_ofautils_happy" = "yes"],
+          [common_ofautils_WRAPPER_EXTRA_LDFLAGS="$common_ofautils_LDFLAGS"
+           common_ofautils_WRAPPER_EXTRA_LIBS="$common_ofautils_LIBS"
+           $1],
+          [$2])
+
+    # substitute in the things needed to build openib
+    AC_SUBST([common_ofautils_CFLAGS])
+    AC_SUBST([common_ofautils_CPPFLAGS])
+    AC_SUBST([common_ofautils_LDFLAGS])
+    AC_SUBST([common_ofautils_LIBS])
+])dnl
--- a/ompi/mca/common/ofautils/configure.params
+++ b/ompi/mca/common/ofautils/configure.params
@ -0,0 +1,26 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2007      Los Alamos National Security, LLC.  All rights
+#                         reserved. 
+# Copyright (c) 2009-2012 Oak Ridge National Laboratory.  All rights reserved.
+# Copyright (c) 2009-2012 Mellanox Technologies.  All rights reserved.
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+
+# Specific to this module
+
+PARAM_CONFIG_FILES="Makefile"
--- a/ompi/mca/dpm/dpm.h
+++ b/ompi/mca/dpm/dpm.h
@ -56,6 +56,10 @@ BEGIN_C_DECLS
 /* common sm component query result index */
 #define OMPI_RML_TAG_COMMON_SM_COMP_INDEX           OMPI_RML_TAG_BASE+10

+/* OFACM RML TAGs */
+#define OMPI_RML_TAG_OFACM                          OMPI_RML_TAG_BASE+11
+#define OMPI_RML_TAG_XOFACM                         OMPI_RML_TAG_BASE+12
+
 #define OMPI_RML_TAG_DYNAMIC                        OMPI_RML_TAG_BASE+200