From 77f8282d51d8f40f6ae988ef84c9c852de75c625 Mon Sep 17 00:00:00 2001 From: Gilles Gouaillardet Date: Fri, 10 Jul 2015 13:44:11 +0900 Subject: [PATCH] configury: fix (again) XRC detection on OFED < 3.12 since ibv_create_xrc_rcv_qp is now deprecated, and in order to be "future-proof", we have to consider the case in which only XRC Domains are supported. Thanks Paul Hargrove for the detailled report. --- config/opal_check_openfabrics.m4 | 18 +++++++++++------- opal/mca/btl/openib/btl_openib.h | 4 ++-- opal/mca/btl/openib/btl_openib_async.c | 11 ++++------- opal/mca/btl/openib/btl_openib_async.h | 2 +- opal/mca/btl/openib/btl_openib_atomic.c | 7 ++++++- opal/mca/btl/openib/btl_openib_put.c | 7 +++++-- 6 files changed, 29 insertions(+), 20 deletions(-) diff --git a/config/opal_check_openfabrics.m4 b/config/opal_check_openfabrics.m4 index 9a05080bfe..d38d0bf3e8 100644 --- a/config/opal_check_openfabrics.m4 +++ b/config/opal_check_openfabrics.m4 @@ -154,13 +154,17 @@ AC_DEFUN([OPAL_CHECK_OPENFABRICS],[ $1_have_xrc=1 AC_CHECK_FUNCS([ibv_create_xrc_rcv_qp], [], [$1_have_xrc=0]) - AC_CHECK_DECLS([IBV_SRQT_XRC], - [], [$1_have_xrc=0], - [#include ]) - fi - if test "$enable_connectx_xrc" = "yes" \ - && test $$1_have_xrc -eq 1; then - AC_CHECK_FUNCS([ibv_cmd_open_xrcd], [$1_have_xrc_domains=1]) + $1_have_xrc_domains=1 + AC_CHECK_FUNCS([ibv_cmd_open_xrcd], + [AC_CHECK_DECLS([IBV_SRQT_XRC], + [], [$1_have_xrc_domains=0], + [#include ])], + [$1_have_xrc_domains=0]) + # XRC and XRC Domains should be considered as exclusive + if test "$$1_have_xrc" -eq 1 && \ + test "$$1_have_xrc_domains" -eq 1; then + $1_have_xrc=0 + fi fi diff --git a/opal/mca/btl/openib/btl_openib.h b/opal/mca/btl/openib/btl_openib.h index f7741e5cd9..92506bc5d0 100644 --- a/opal/mca/btl/openib/btl_openib.h +++ b/opal/mca/btl/openib/btl_openib.h @@ -53,8 +53,8 @@ BEGIN_C_DECLS -#define HAVE_XRC (1 == OPAL_HAVE_CONNECTX_XRC) -#define ENABLE_DYNAMIC_SL (1 == OPAL_ENABLE_DYNAMIC_SL) +#define HAVE_XRC (OPAL_HAVE_CONNECTX_XRC || OPAL_HAVE_CONNECTX_XRC_DOMAINS) +#define ENABLE_DYNAMIC_SL OPAL_ENABLE_DYNAMIC_SL #define MCA_BTL_IB_LEAVE_PINNED 1 #define IB_DEFAULT_GID_PREFIX 0xfe80000000000000ll diff --git a/opal/mca/btl/openib/btl_openib_async.c b/opal/mca/btl/openib/btl_openib_async.c index 7f6ec499d9..d5cc5cc303 100644 --- a/opal/mca/btl/openib/btl_openib_async.c +++ b/opal/mca/btl/openib/btl_openib_async.c @@ -122,7 +122,7 @@ static mca_btl_openib_endpoint_t * qp2endpoint(struct ibv_qp *qp, mca_btl_openib return NULL; } -#if HAVE_XRC && !OPAL_HAVE_CONNECTX_XRC_DOMAINS +#if OPAL_HAVE_CONNECTX_XRC /* XRC recive QP to endpoint */ static mca_btl_openib_endpoint_t * xrc_qp2endpoint(uint32_t qp_num, mca_btl_openib_device_t *device) { @@ -352,24 +352,21 @@ static int btl_openib_async_deviceh(struct mca_btl_openib_async_poll *devices_po } event_type = event.event_type; -#if HAVE_XRC +#if OPAL_HAVE_CONNECTX_XRC /* is it XRC event ?*/ -#if OPAL_HAVE_CONNECTX_XRC_DOMAINS -#else bool xrc_event = false; if (IBV_XRC_QP_EVENT_FLAG & event.event_type) { xrc_event = true; /* Clean the bitnd handel as usual */ event_type ^= IBV_XRC_QP_EVENT_FLAG; } -#endif #endif switch(event_type) { case IBV_EVENT_PATH_MIG: BTL_ERROR(("Alternative path migration event reported")); if (APM_ENABLED) { BTL_ERROR(("Trying to find additional path...")); -#if HAVE_XRC && !OPAL_HAVE_CONNECTX_XRC_DOMAINS +#if OPAL_HAVE_CONNECTX_XRC if (xrc_event) mca_btl_openib_load_apm_xrc_rcv(event.element.xrc_qp_num, xrc_qp2endpoint(event.element.xrc_qp_num, device)); @@ -653,7 +650,7 @@ void mca_btl_openib_load_apm(struct ibv_qp *qp, mca_btl_openib_endpoint_t *ep) qp->qp_num, strerror(errno), errno)); } -#if HAVE_XRC && ! OPAL_HAVE_CONNECTX_XRC_DOMAINS +#if OPAL_HAVE_CONNECTX_XRC void mca_btl_openib_load_apm_xrc_rcv(uint32_t qp_num, mca_btl_openib_endpoint_t *ep) { struct ibv_qp_init_attr qp_init_attr; diff --git a/opal/mca/btl/openib/btl_openib_async.h b/opal/mca/btl/openib/btl_openib_async.h index c9243821d3..33137546a4 100644 --- a/opal/mca/btl/openib/btl_openib_async.h +++ b/opal/mca/btl/openib/btl_openib_async.h @@ -19,7 +19,7 @@ int start_async_event_thread(void); void mca_btl_openib_load_apm(struct ibv_qp *qp, mca_btl_openib_endpoint_t *ep); int btl_openib_async_command_done(int exp); -#if HAVE_XRC && ! OPAL_HAVE_CONNECTX_XRC_DOMAINS +#if OPAL_HAVE_CONNECTX_XRC void mca_btl_openib_load_apm_xrc_rcv(uint32_t qp_num, mca_btl_openib_endpoint_t *ep); #endif diff --git a/opal/mca/btl/openib/btl_openib_atomic.c b/opal/mca/btl/openib/btl_openib_atomic.c index f96ce4c681..bc15c66569 100644 --- a/opal/mca/btl/openib/btl_openib_atomic.c +++ b/opal/mca/btl/openib/btl_openib_atomic.c @@ -73,7 +73,12 @@ static int mca_btl_openib_atomic_internal (struct mca_btl_base_module_t *btl, st #if HAVE_XRC if (MCA_BTL_XRC_ENABLED && BTL_OPENIB_QP_TYPE_XRC(qp)) { - frag->sr_desc.xrc_remote_srq_num=endpoint->rem_info.rem_srqs[qp].rem_srq_num; +#if OPAL_HAVE_CONNECTX_XRC_DOMAINS + frag->sr_desc.qp_type.xrc.remote_srqn = endpoint->rem_info.rem_srqs[qp].rem_srq_num; +#else + frag->sr_desc.xrc_remote_srq_num = endpoint->rem_info.rem_srqs[qp].rem_srq_num; +#endif + } #endif diff --git a/opal/mca/btl/openib/btl_openib_put.c b/opal/mca/btl/openib/btl_openib_put.c index d7839c1c2c..25b5d3f532 100644 --- a/opal/mca/btl/openib/btl_openib_put.c +++ b/opal/mca/btl/openib/btl_openib_put.c @@ -103,10 +103,13 @@ int mca_btl_openib_put (mca_btl_base_module_t *btl, struct mca_btl_base_endpoint #if HAVE_XRC if (MCA_BTL_XRC_ENABLED && BTL_OPENIB_QP_TYPE_XRC(qp)) { -#if OPAL_HAVE_CONNECTX_XRC_DOMAINS + +#if OPAL_HAVE_CONNECTX_XRC + to_out_frag(frag)->sr_desc.xrc_remote_srq_num = ep->rem_info.rem_srqs[qp].rem_srq_num; +#elif OPAL_HAVE_CONNECTX_XRC_DOMAINS to_out_frag(frag)->sr_desc.qp_type.xrc.remote_srqn = ep->rem_info.rem_srqs[qp].rem_srq_num; #else - to_out_frag(frag)->sr_desc.xrc_remote_srq_num = ep->rem_info.rem_srqs[qp].rem_srq_num; +#error "that should never happen" #endif } #endif