common/verbs: update usnic transport probe

RHEL 7 has shipped with kernel support for the RDMA_TRANSPORT_USNIC
enum, but ''not'' the RDMA_TRANSPORT_USNIC_UDP enum.  This means that
when you install usNIC drivers from cisco.com, the kernel will report
IBV_TRANSPORT_USNIC, even though the transport is actually using UDP.

Therefore, we have to modify the logic in common/verbs to do the
additional magic probe if the device reports either an
IBV_TRANSPORT_IWARP or IBV_TRANSPORT_USNIC (because both of those might
be lies -- do the probe to figure out the real transport).

The code changed by this patch is fairly trivial; it simply moves the
logic of the magic probe to its own short function, and then calls that
short function in both the IBV_TRANSPORT_(IWARP|USNIC) cases.  It looks
longer because several lengthy comments were also updated.

Authored-by: Jeff Squyres <jsquyres@cisco.com>
Reviewed-by: Dave Goodell <dgoodell@cisco.com>

cmr=v1.8.2:reviewer=ompi-rm1.8

This commit was SVN r32098.

ompi/mca/common/verbs/common_verbs_find_ports.c

@@ -231,43 +231,26 @@ typedef union {
 #define USNIC_PORT_QUERY_MAGIC (0x43494e7375534355ULL)
 
 /*
- * usNIC devices will always return one of three
- * device->transport_type values:
+ * Probe for the magic number to see if the userspace side of verbs is
+ * new enough to include UDP transport support.
  *
- * 1. TRANSPORT_IWARP: for older kernels (e.g., on systems such as
- * RHEL 6.x (x>=4) with the drivers downloaded from cisco.com) where
- * the cisco.com drivers could not modify verbs.h to include
- * TRANSPORT_USNIC*.  In this case, it is unknown whether the
- * transport is usNIC/L2 or usNIC/UDP -- you have to do an additional
- * probe to figure it out.
+ * If the userspace side is too old to include UDP support, then it
+ * will fail the magic probe.  If somehow we eneded up with a "new"
+ * userspace (e.g., that supports UDP) and an "old" kernel module
+ * (e.g., that does not support UDP), then the userspace will fail the
+ * ABI check with the kernel module and we won't get this far at all.
  *
- * 2. TRANSPORT_USNIC: should probably never be returned on a customer
- * system.  In this case, the transport is guaranteed to be usNIC/L2.
- *
- * 3. TRANSPORT_USNIC_UDP: on systems with new kernels and new
- * libibverbs.  In this case, the transport is guaranteed to be
- * usNIC/UDP.
+ * NB: it will be complicated if we ever need to extend this scheme
+ * (e.g., if we support something other than UDP someday), because the
+ * real way to know what the actual transport is will be to call a
+ * usnic verbs extension, and that code is all currently over in the
+ * usnic BTL, which we can't call from here.
  */
-static int usnic_transport(struct ibv_device *device,
-                           struct ibv_context *context)
+static int usnic_magic_probe(struct ibv_context *context)
 {
-    if (!device_is_usnic(device)) {
-        return USNIC_UNKNOWN;
-    }
-
-    /* If we got a transport type of IWARP, see if the magic query
-       exists.  If it does, the transport type is UDP.  If it does
-       not, the transport type is L2.
-
-       NB: it will be complicated if we ever need to extend this
-       scheme (e.g., if we support something other than UDP someday),
-       because the real way to know what the actual transport is will
-       be to call a usnic verbs extension, and that code is all
-       currently over in the usnic BTL, which we can't call from
-       here. */
-    if (IBV_TRANSPORT_IWARP == device->transport_type) {
     int rc;
     port_query_u u;
+
     rc = ibv_query_port(context, 42, &u.attr);
     /* See comment in btl_usnic_ext.c about why we have to check
        for rc==0 *and* the magic number. */
@@ -284,6 +267,39 @@ static int usnic_transport(struct ibv_device *device,
     }
 }
 
+/*
+ * usNIC devices will always return one of three
+ * device->transport_type values:
+ *
+ * 1. TRANSPORT_IWARP: for older kernels (e.g., on systems such as
+ * RHEL 6.x (x>=4) with the drivers downloaded from cisco.com) where
+ * the cisco.com drivers could not modify verbs.h to include
+ * TRANSPORT_USNIC*.  In this case, it is unknown whether the
+ * transport is usNIC/L2 or usNIC/UDP -- you have to do an additional
+ * probe to figure it out.
+ *
+ * 2. TRANSPORT_USNIC: for some systems that updated to include the
+ * RDMA_TRANSPORT_USNIC constant, but not the RDMA_TRANSPORT_USNIC_UDP
+ * constant, with the drivers downloaded from cisco.com (e.g., RHEL
+ * 7.0).  This is just like the TRANSPORT_IWARP case: we have to do an
+ * additional probe to figure out whether the transport is usNIC/L2 or
+ * usNIC/UDP.
+ *
+ * 3. TRANSPORT_USNIC_UDP: on systems with new kernels and new
+ * libibverbs.  In this case, the transport is guaranteed to be
+ * usNIC/UDP.
+*/
+static int usnic_transport(struct ibv_device *device,
+                           struct ibv_context *context)
+{
+    if (!device_is_usnic(device)) {
+        return USNIC_UNKNOWN;
+    }
+
+    if (IBV_TRANSPORT_IWARP == device->transport_type) {
+        return usnic_magic_probe(context);
+    }
+
 #if HAVE_DECL_IBV_TRANSPORT_USNIC_UDP
     /* If we got the transport type of USNIC_UDP, then it's definitely
        the UDP transport. */
@@ -293,10 +309,8 @@ static int usnic_transport(struct ibv_device *device,
 #endif
 
 #if HAVE_DECL_IBV_TRANSPORT_USNIC
-    /* If we got the transport type of USNIC, then it's definitely
-       the L2 transport. */
     if (IBV_TRANSPORT_USNIC == device->transport_type) {
-        return USNIC_L2;
+        return usnic_magic_probe(context);
     }
 #endif