From d6ead2a3a5203ab98a117208f8d00a2cf9f8ea00 Mon Sep 17 00:00:00 2001 From: Mike Dubman Date: Wed, 23 Oct 2013 06:08:54 +0000 Subject: [PATCH] Add support for routable ROCE where different subnet_id is a valid to proceed with MPI routing. (can happen in the same LAN) developed by vasily, reviewed by miked cmr=v1.7.4:reviewer=ompi-gk1.7 This commit was SVN r29479. --- ompi/mca/btl/openib/btl_openib.h | 3 +++ ompi/mca/btl/openib/btl_openib_component.c | 4 ++-- ompi/mca/btl/openib/btl_openib_mca.c | 7 +++++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/ompi/mca/btl/openib/btl_openib.h b/ompi/mca/btl/openib/btl_openib.h index 82756bc0b3..09de43e722 100644 --- a/ompi/mca/btl/openib/btl_openib.h +++ b/ompi/mca/btl/openib/btl_openib.h @@ -315,6 +315,9 @@ struct mca_btl_openib_component_t { bool cuda_async_send; bool cuda_async_recv; #endif /* OMPI_CUDA_SUPPORT */ +#if HAVE_DECL_IBV_LINK_LAYER_ETHERNET + int rroce_enable; +#endif }; typedef struct mca_btl_openib_component_t mca_btl_openib_component_t; OMPI_MODULE_DECLSPEC extern mca_btl_openib_component_t mca_btl_openib_component; diff --git a/ompi/mca/btl/openib/btl_openib_component.c b/ompi/mca/btl/openib/btl_openib_component.c index 670fa435fc..d8e346ba39 100644 --- a/ompi/mca/btl/openib/btl_openib_component.c +++ b/ompi/mca/btl/openib/btl_openib_component.c @@ -731,8 +731,8 @@ static int init_one_port(opal_list_t *btl_list, mca_btl_openib_device_t *device, #if HAVE_DECL_IBV_LINK_LAYER_ETHERNET if (IBV_LINK_LAYER_ETHERNET == ib_port_attr->link_layer) { - subnet_id = mca_btl_openib_get_ip_subnet_id(device->ib_dev, - port_num); + subnet_id = mca_btl_openib_component.rroce_enable ? 0 : + mca_btl_openib_get_ip_subnet_id(device->ib_dev, port_num); } else { subnet_id = ntoh64(gid.global.subnet_prefix); } diff --git a/ompi/mca/btl/openib/btl_openib_mca.c b/ompi/mca/btl/openib/btl_openib_mca.c index 4e07dfebdb..6e7319c0f9 100644 --- a/ompi/mca/btl/openib/btl_openib_mca.c +++ b/ompi/mca/btl/openib/btl_openib_mca.c @@ -499,6 +499,13 @@ int btl_openib_register_mca_params(void) mca_btl_openib_component.enable_srq_resize = false; #endif +#if HAVE_DECL_IBV_LINK_LAYER_ETHERNET + CHECK(reg_bool("rroce_enable", NULL, + "Enable/Disable routing between different subnets" + "(0 = disable, nonzero = enable)", false, + &mca_btl_openib_component.rroce_enable)); +#endif + CHECK(reg_uint("buffer_alignment", NULL, "Preferred communication buffer alignment, in bytes " "(must be > 0 and power of two)",