usnic: use the new usnic verbs extensions

If they exist, call the usnic verbs extensions to both enable UDP
support and get the UD receiver header length that should be used
(rather than assume 40/struct GRH).

This commit was SVN r30912.

ompi/mca/btl/usnic/Makefile.am

@@ -55,6 +55,8 @@ sources = \
     btl_usnic_cagent.c \
     btl_usnic_endpoint.c \
     btl_usnic_endpoint.h \
+    btl_usnic_ext.h \
+    btl_usnic_ext.c \
     btl_usnic_frag.c \
     btl_usnic_frag.h \
     btl_usnic_graph.h \

ompi/mca/btl/usnic/btl_usnic_component.c

@@ -74,6 +74,7 @@
 #include "btl_usnic_send.h"
 #include "btl_usnic_recv.h"
 #include "btl_usnic_proc.h"
+#include "btl_usnic_ext.h"
 #include "btl_usnic_test.h"
 
 #define OMPI_BTL_USNIC_NUM_WC       500
@@ -518,6 +519,11 @@ static mca_btl_base_module_t** usnic_component_init(int* num_btl_modules,
         }
     }
 
+    /* Initialize the table of usnic extension function pointers */
+    item = opal_list_get_first(port_list);
+    port = (ompi_common_verbs_port_item_t*) item;
+    ompi_btl_usnic_ext_init(port->device->context);
+
     /* Setup an array of pointers to point to each module (which we'll
        return upstream) */
     mca_btl_usnic_component.num_modules = opal_list_get_size(port_list);
@@ -624,6 +630,37 @@ static mca_btl_base_module_t** usnic_component_init(int* num_btl_modules,
             continue;
         }
 
+        /* Tell this device's context that we are aware that we need
+           to request the UD header length.  If it fails, just skip
+           this device. */
+        if (NULL != ompi_btl_usnic_ext.enable_udp) {
+            opal_output_verbose(5, USNIC_OUT,
+                                "btl:usnic: enabling UDP support for %s",
+                                ibv_get_device_name(module->device));
+            if (0 !=
+                ompi_btl_usnic_ext.enable_udp(port->device->context)) {
+                --mca_btl_usnic_component.num_modules;
+                opal_output_verbose(5, USNIC_OUT,
+                                    "btl:usnic: UDP support unexpectedly failed for %s; ignoring this device", 
+                                    ibv_get_device_name(module->device));
+                continue;
+            }
+
+            int len =
+                ompi_btl_usnic_ext.get_ud_header_len(port->device->context,
+                                                     port->port_num);
+            /* Sanity check: the len we get back should be 42.  If
+               it's not, skip this device. */
+            if (OMPI_BTL_USNIC_UDP_HDR_SZ != len) {
+                opal_output_verbose(5, USNIC_OUT,
+                                    "btl:usnic: unexpected UD header length for %s reported by extension (%d); ignoring this device", 
+                                    ibv_get_device_name(module->device),
+                                    len);
+                --mca_btl_usnic_component.num_modules;
+                continue;
+            }
+        }
+
         /* How many xQ entries do we want? */
         if (-1 == mca_btl_usnic_component.sd_num) {
             module->sd_num = device_attr.max_qp_wr;

ompi/mca/btl/usnic/btl_usnic_ext.c

@@ -0,0 +1,66 @@
+/*
+ * Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "opal/util/output.h"
+
+#include "ompi/mca/btl/base/base.h"
+
+#include "btl_usnic_compat.h"
+#include "btl_usnic_ext.h"
+
+/*
+ * Global variable of usnic extension function pointers
+ */
+ompi_btl_usnic_ext_fns_t ompi_btl_usnic_ext;
+
+
+/*
+ * Initialize ompi_btl_usnic_ext
+ */
+void ompi_btl_usnic_ext_init(struct ibv_context *context)
+{
+    memset(&ompi_btl_usnic_ext, 0, sizeof(ompi_btl_usnic_ext));
+
+    /* See if this context supports the usnic extensions.  Do the
+       magic query port on port number 42 (which is THE ANSWER) */
+    int rc;
+    rc = ibv_query_port(context, 42,
+                        (struct ibv_port_attr*) &ompi_btl_usnic_ext.qpt);
+    if (0 != rc) {
+        opal_output_verbose(5, USNIC_OUT,
+                            "btl:usnic: verbs plugin does not support extensions");
+        return;
+    }
+
+    /* If the libusnic_verbs plugin under the verbs API supporting
+       this context supports the usnic extensions, it'll return 0==rc
+       and give us a function that we can use to look up other usnic
+       verb extension function pointers.  If the lookup_version is one
+       that we understand, use it to look up the extensions we care
+       about. */
+    opal_output_verbose(5, USNIC_OUT,
+                        "btl:usnic: verbs plugin has extension lookup ABI version %d",
+                        ompi_btl_usnic_ext.qpt.lookup_version);
+    if (1 != ompi_btl_usnic_ext.qpt.lookup_version) {
+        opal_output_verbose(5, USNIC_OUT,
+                            "btl:usnic: unrecognized lookup ABI version"
+                            " (I only recognize version 1) "
+                            " -- extensions ignored");
+        return;
+    }
+    opal_output_verbose(5, USNIC_OUT,
+                        "btl:usnic: BTL recognizes this lookup ABI -- yay!");
+
+    *(void **) (&ompi_btl_usnic_ext.enable_udp) =
+        ompi_btl_usnic_ext.qpt.lookup("enable_udp");
+    *(void **) (&ompi_btl_usnic_ext.get_ud_header_len) =
+        ompi_btl_usnic_ext.qpt.lookup("get_ud_header_len");
+}

ompi/mca/btl/usnic/btl_usnic_ext.h

@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef OMPI_BTL_USNIC_EXT_H
+#define OMPI_BTL_USNIC_EXT_H
+
+#include "ompi_config.h"
+
+#include <infiniband/verbs.h>
+
+typedef void *(*ompi_btl_usnic_dlsym_fn_t)(const char *name);
+
+typedef struct {
+    int lookup_version;
+    ompi_btl_usnic_dlsym_fn_t lookup;
+} ompi_btl_usnic_query_port_table_t;
+
+/*
+ * Tells libusnic_verbs to enable UDP support.
+ */
+typedef int (*ompi_btl_usnic_enable_udp_fn_t)(struct ibv_context *context);
+
+/*
+ * Find out what the UD header length is
+ */
+typedef int (*ompi_btl_usnic_get_ud_header_len_fn_t)(struct ibv_context *context,
+                                                     uint8_t port_num);
+
+/*
+ * Struct usnic extension function pointers
+ */
+typedef struct {
+    ompi_btl_usnic_query_port_table_t qpt;
+
+    ompi_btl_usnic_enable_udp_fn_t enable_udp;
+    ompi_btl_usnic_get_ud_header_len_fn_t get_ud_header_len;
+} ompi_btl_usnic_ext_fns_t;
+
+/*
+ * Global variable of usnic extension function pointers
+ */
+extern ompi_btl_usnic_ext_fns_t ompi_btl_usnic_ext;
+
+/*
+ * Function to initialze the global variable of usnic extension
+ * function pointers
+ */
+void ompi_btl_usnic_ext_init(struct ibv_context *ctx);
+
+#endif /* OMPI_BTL_USNIC_EXT_H */
+