diff --git a/ompi/mca/btl/openib/btl_openib.h b/ompi/mca/btl/openib/btl_openib.h index bd24172396..0590605213 100644 --- a/ompi/mca/btl/openib/btl_openib.h +++ b/ompi/mca/btl/openib/btl_openib.h @@ -184,7 +184,7 @@ struct mca_btl_openib_component_t { uint32_t ib_rnr_retry; uint32_t ib_max_rdma_dst_ops; uint32_t ib_service_level; - uint32_t use_eager_rdma; + int32_t use_eager_rdma; int32_t eager_rdma_threshold; /**< After this number of msg, use RDMA for short messages, always */ int32_t eager_rdma_num; int32_t max_eager_rdma; diff --git a/ompi/mca/btl/openib/btl_openib_component.c b/ompi/mca/btl/openib/btl_openib_component.c index 7dc981b0dd..56ec3cf24e 100644 --- a/ompi/mca/btl/openib/btl_openib_component.c +++ b/ompi/mca/btl/openib/btl_openib_component.c @@ -877,9 +877,8 @@ static int prepare_hca_for_use(mca_btl_openib_hca_t *hca) hca->endpoints = OBJ_NEW(opal_pointer_array_t); opal_pointer_array_init(hca->endpoints, 10, INT_MAX, 10); opal_pointer_array_add(&mca_btl_openib_component.hcas, hca); - if(mca_btl_openib_component.max_eager_rdma > 0 && - mca_btl_openib_component.use_eager_rdma && - hca->use_eager_rdma) { + if (mca_btl_openib_component.max_eager_rdma > 0 && + hca->use_eager_rdma) { hca->eager_rdma_buffers = calloc(mca_btl_openib_component.max_eager_rdma * hca->btls, sizeof(mca_btl_openib_endpoint_t*)); @@ -1558,10 +1557,20 @@ static int init_one_hca(opal_list_t *btl_list, struct ibv_device* ib_dev) } } - /* If "use eager rdma" was set, then enable it on this HCA */ - if (values.use_eager_rdma_set) { + /* Should we use RDMA for short / eager messages? First check MCA + param, then check INI file values. */ + if (mca_btl_openib_component.use_eager_rdma >= 0) { + hca->use_eager_rdma = mca_btl_openib_component.use_eager_rdma; + } else if (values.use_eager_rdma_set) { hca->use_eager_rdma = values.use_eager_rdma; } + /* Eager RDMA is not currently supported with progress threads */ + if (hca->use_eager_rdma && OMPI_ENABLE_PROGRESS_THREADS) { + hca->use_eager_rdma = 0; + orte_show_help("help-mpi-btl-openib.txt", + "eager RDMA and progress threads", true); + } + opal_output(0, "Using eager rdma: %d\n", hca->use_eager_rdma); #if HAVE_XRC /* if user configured to run with XRC qp and the device doesn't diff --git a/ompi/mca/btl/openib/btl_openib_mca.c b/ompi/mca/btl/openib/btl_openib_mca.c index 3340b5f4e1..ea0d05f279 100644 --- a/ompi/mca/btl/openib/btl_openib_mca.c +++ b/ompi/mca/btl/openib/btl_openib_mca.c @@ -223,7 +223,8 @@ int btl_openib_register_mca_params(void) mca_btl_openib_component.ib_cq_size[BTL_OPENIB_HP_CQ] = (uint32_t) ival; CHECK(reg_int("ib_max_inline_data", "Maximum size of inline data segment " - "(-1 = use per-device devaults, 0 = run-time probe to discover max value, " + "(-1 = use device default, " + "0 = run-time probe to discover max value, " "otherwise must be >= 1)", -1, &ival, REGINT_NEG_ONE_OK | REGINT_GE_ZERO)); mca_btl_openib_component.ib_max_inline_data = (int32_t) ival; @@ -365,13 +366,11 @@ int btl_openib_register_mca_params(void) } mca_btl_openib_component.ib_service_level = (uint32_t) ival; - CHECK(reg_int("use_eager_rdma", "Use RDMA for eager messages", - 1, &ival, 0)); - mca_btl_openib_component.use_eager_rdma = (uint32_t) (ival != 0); -#if OMPI_ENABLE_PROGRESS_THREADS == 1 - /* Fast rdma path isn't supported by PROGRESS_THREAD */ - mca_btl_openib_component.use_eager_rdma = 0; -#endif + CHECK(reg_int("use_eager_rdma", "Use RDMA for eager messages " + "(-1 = use device default, 0 = do not use eager RDMA, " + "1 = use eager RDMA)", + -1, &ival, 0)); + mca_btl_openib_component.use_eager_rdma = (int32_t) ival; CHECK(reg_int("eager_rdma_threshold", "Use RDMA for short messages after this number of " diff --git a/ompi/mca/btl/openib/help-mpi-btl-openib.txt b/ompi/mca/btl/openib/help-mpi-btl-openib.txt index 665d7f6a62..22efce4313 100644 --- a/ompi/mca/btl/openib/help-mpi-btl-openib.txt +++ b/ompi/mca/btl/openib/help-mpi-btl-openib.txt @@ -537,3 +537,11 @@ support) will be disabled. Host: %s Device: %s +# +[eager RDMA and progress threads] +WARNING: The openib BTL was directed to use "eager RDMA" for short +messages, but the openib BTL was compiled with progress threads +support. Short eager RDMA is not yet supported with progress threads; +its use has been disabled in this job. + +This is a warning only; you job will attempt to continue.