diff --git a/ompi/mca/btl/openib/connect/btl_openib_connect_ibcm.c b/ompi/mca/btl/openib/connect/btl_openib_connect_ibcm.c index c1ca750d4f..35839ef34e 100644 --- a/ompi/mca/btl/openib/connect/btl_openib_connect_ibcm.c +++ b/ompi/mca/btl/openib/connect/btl_openib_connect_ibcm.c @@ -822,9 +822,15 @@ static int qp_create_one(mca_btl_base_endpoint_t* endpoint, int qp, return OMPI_ERROR; } endpoint->qps[qp].qp->lcl_qp = my_qp; - endpoint->qps[qp].ib_inline_max = - init_attr.cap.max_inline_data < req_inline ? - init_attr.cap.max_inline_data : req_inline; + if (init_attr.cap.max_inline_data < req_inline) { + endpoint->qps[qp].ib_inline_max = init_attr.cap.max_inline_data; + orte_show_help("help-mpi-btl-openib-cpc-base.txt", + "inline truncated", orte_process_info.nodename, + ibv_get_device_name(openib_btl->hca->ib_dev), + req_inline, init_attr.cap.max_inline_data); + } else { + endpoint->qps[qp].ib_inline_max = req_inline; + } /* Setup meta data on the endpoint */ endpoint->qps[qp].qp->lcl_psn = lrand48() & 0xffffff; endpoint->qps[qp].credit_frag = NULL; diff --git a/ompi/mca/btl/openib/connect/btl_openib_connect_oob.c b/ompi/mca/btl/openib/connect/btl_openib_connect_oob.c index 41dc9b9e37..27de8e8ff4 100644 --- a/ompi/mca/btl/openib/connect/btl_openib_connect_oob.c +++ b/ompi/mca/btl/openib/connect/btl_openib_connect_oob.c @@ -37,6 +37,7 @@ #include "btl_openib_endpoint.h" #include "btl_openib_proc.h" #include "connect/connect.h" +#include "orte/util/show_help.h" typedef enum { ENDPOINT_CONNECT_REQUEST, @@ -458,9 +459,16 @@ static int qp_create_one(mca_btl_base_endpoint_t* endpoint, int qp, return OMPI_ERROR; } endpoint->qps[qp].qp->lcl_qp = my_qp; - endpoint->qps[qp].ib_inline_max = - init_attr.cap.max_inline_data < req_inline ? - init_attr.cap.max_inline_data : req_inline; + + if (init_attr.cap.max_inline_data < req_inline) { + endpoint->qps[qp].ib_inline_max = init_attr.cap.max_inline_data; + orte_show_help("help-mpi-btl-openib-cpc-base.txt", + "inline truncated", orte_process_info.nodename, + ibv_get_device_name(openib_btl->hca->ib_dev), + req_inline, init_attr.cap.max_inline_data); + } else { + endpoint->qps[qp].ib_inline_max = req_inline; + } attr.qp_state = IBV_QPS_INIT; attr.pkey_index = openib_btl->pkey_index; diff --git a/ompi/mca/btl/openib/connect/btl_openib_connect_rdmacm.c b/ompi/mca/btl/openib/connect/btl_openib_connect_rdmacm.c index 907c276a7f..a144952768 100644 --- a/ompi/mca/btl/openib/connect/btl_openib_connect_rdmacm.c +++ b/ompi/mca/btl/openib/connect/btl_openib_connect_rdmacm.c @@ -37,6 +37,7 @@ #include "btl_openib_endpoint.h" #include "connect/connect.h" #include "btl_openib_iwarp.h" +#include "orte/util/show_help.h" /* JMS to be removed: see #1264 */ #undef event @@ -264,9 +265,15 @@ static int rdmacm_setup_qp(rdmacm_contents_t *local, } endpoint->qps[qpnum].qp->lcl_qp = qp; - endpoint->qps[qpnum].ib_inline_max = - attr.cap.max_inline_data < req_inline ? - attr.cap.max_inline_data : req_inline; + if (attr.cap.max_inline_data < req_inline) { + endpoint->qps[qpnum].ib_inline_max = attr.cap.max_inline_data; + orte_show_help("help-mpi-btl-openib-cpc-base.txt", + "inline truncated", orte_process_info.nodename, + ibv_get_device_name(local->openib_btl->hca->ib_dev), + req_inline, attr.cap.max_inline_data); + } else { + endpoint->qps[qpnum].ib_inline_max = req_inline; + } id->qp = qp; return 0; diff --git a/ompi/mca/btl/openib/connect/btl_openib_connect_xoob.c b/ompi/mca/btl/openib/connect/btl_openib_connect_xoob.c index 4223ac5cd0..6db068e005 100644 --- a/ompi/mca/btl/openib/connect/btl_openib_connect_xoob.c +++ b/ompi/mca/btl/openib/connect/btl_openib_connect_xoob.c @@ -25,6 +25,7 @@ #include "btl_openib_xrc.h" #include "btl_openib_async.h" #include "connect/connect.h" +#include "orte/util/show_help.h" static void xoob_component_register(void); static int xoob_component_query(mca_btl_openib_module_t *openib_btl, @@ -400,14 +401,21 @@ static int xoob_send_qp_create (mca_btl_base_endpoint_t* endpoint) qp_init_attr.qp_type = IBV_QPT_XRC; qp_init_attr.xrc_domain = openib_btl->hca->xrc_domain; *qp = ibv_create_qp(openib_btl->hca->ib_pd, &qp_init_attr); - endpoint->qps[0].ib_inline_max = - qp_init_attr.cap.max_inline_data < req_inline ? - qp_init_attr.cap.max_inline_data : req_inline; if (NULL == *qp) { BTL_ERROR(("Error creating QP, errno says: %s", strerror(errno))); return OMPI_ERROR; } + if (qp_init_attr.cap.max_inline_data < req_inline) { + endpoint->qps[0].ib_inline_max = qp_init_attr.cap.max_inline_data; + orte_show_help("help-mpi-btl-openib-cpc-base.txt", + "inline truncated", orte_process_info.nodename, + ibv_get_device_name(openib_btl->hca->ib_dev), + req_inline, qp_init_attr.cap.max_inline_data); + } else { + endpoint->qps[0].ib_inline_max = req_inline; + } + attr.qp_state = IBV_QPS_INIT; attr.pkey_index = openib_btl->pkey_index; attr.port_num = openib_btl->port_num; diff --git a/ompi/mca/btl/openib/connect/help-mpi-btl-openib-cpc-base.txt b/ompi/mca/btl/openib/connect/help-mpi-btl-openib-cpc-base.txt index 820588c234..4ef931dc81 100644 --- a/ompi/mca/btl/openib/connect/help-mpi-btl-openib-cpc-base.txt +++ b/ompi/mca/btl/openib/connect/help-mpi-btl-openib-cpc-base.txt @@ -27,3 +27,14 @@ Host: %s btl_openib_cpc_%s value: %s Invalid name: %s All possible valid names: %s +# +[inline truncated] +WARNING: The btl_openib_ib_max_inline_data MCA parameter was used to +specify how much inline data should be used, but a device reduced this +value. This is not an error; it simply means that your run will use +a smaller inline data value than was requested. + + Local host: %s + Local device: %s + Requested value: %d + Value used by device: %d