Add support for routable ROCE where different subnet_id is a valid to proceed with MPI routing.

(can happen in the same LAN)
developed by vasily, reviewed by miked
cmr=v1.7.4:reviewer=ompi-gk1.7

This commit was SVN r29479.

ompi/mca/btl/openib/btl_openib.h

@@ -315,6 +315,9 @@ struct mca_btl_openib_component_t {
     bool cuda_async_send;
     bool cuda_async_recv;
 #endif /* OMPI_CUDA_SUPPORT */
+#if HAVE_DECL_IBV_LINK_LAYER_ETHERNET
+    int rroce_enable;
+#endif
 }; typedef struct mca_btl_openib_component_t mca_btl_openib_component_t;
 
 OMPI_MODULE_DECLSPEC extern mca_btl_openib_component_t mca_btl_openib_component;

ompi/mca/btl/openib/btl_openib_component.c

@@ -731,8 +731,8 @@ static int init_one_port(opal_list_t *btl_list, mca_btl_openib_device_t *device,
 
 #if HAVE_DECL_IBV_LINK_LAYER_ETHERNET
         if (IBV_LINK_LAYER_ETHERNET == ib_port_attr->link_layer) {
-            subnet_id = mca_btl_openib_get_ip_subnet_id(device->ib_dev,
+            subnet_id = mca_btl_openib_component.rroce_enable ? 0 :
-                                                           port_num);
+                   mca_btl_openib_get_ip_subnet_id(device->ib_dev, port_num);
         } else {
             subnet_id = ntoh64(gid.global.subnet_prefix);
         }

ompi/mca/btl/openib/btl_openib_mca.c

@@ -499,6 +499,13 @@ int btl_openib_register_mca_params(void)
     mca_btl_openib_component.enable_srq_resize = false;
 #endif
 
+#if HAVE_DECL_IBV_LINK_LAYER_ETHERNET
+    CHECK(reg_bool("rroce_enable", NULL,
+                   "Enable/Disable routing between different subnets"
+                   "(0 = disable, nonzero = enable)", false,
+                   &mca_btl_openib_component.rroce_enable));
+#endif
+
     CHECK(reg_uint("buffer_alignment", NULL,
                    "Preferred communication buffer alignment, in bytes "
                    "(must be > 0 and power of two)",