From 135ecce0eb80251da0ef7982b206fdf53a30b0a2 Mon Sep 17 00:00:00 2001 From: Gilles Gouaillardet Date: Wed, 7 Jan 2015 13:27:25 +0900 Subject: [PATCH] btl/openib: rename OPAL_HAVE_XRCD macro into OPAL_HAVE_CONNECTX_XRC_DOMAINS --- config/opal_check_openfabrics.m4 | 8 ++--- opal/mca/btl/openib/btl_openib.c | 6 ++-- opal/mca/btl/openib/btl_openib.h | 2 +- opal/mca/btl/openib/btl_openib_async.c | 8 ++--- opal/mca/btl/openib/btl_openib_async.h | 2 +- opal/mca/btl/openib/btl_openib_endpoint.c | 4 +-- opal/mca/btl/openib/btl_openib_endpoint.h | 4 +-- opal/mca/btl/openib/btl_openib_xrc.c | 14 ++++---- .../openib/connect/btl_openib_connect_udcm.c | 36 +++++++++---------- 9 files changed, 40 insertions(+), 44 deletions(-) diff --git a/config/opal_check_openfabrics.m4 b/config/opal_check_openfabrics.m4 index 50292fb406..6a1a92bf50 100644 --- a/config/opal_check_openfabrics.m4 +++ b/config/opal_check_openfabrics.m4 @@ -151,7 +151,7 @@ AC_DEFUN([OPAL_CHECK_OPENFABRICS],[ # Set these up so that we can do an AC_DEFINE below # (unconditionally) $1_have_xrc=0 - $1_have_xrcd=0 + $1_have_xrc_domains=0 $1_have_opensm_devel=0 # If we have the openib stuff available, find out what we've got @@ -170,7 +170,7 @@ AC_DEFUN([OPAL_CHECK_OPENFABRICS],[ AC_CHECK_FUNCS([ibv_create_xrc_rcv_qp ibv_cmd_open_xrcd], [$1_have_xrc=1]) fi if test "$enable_connectx_xrc" = "yes"; then - AC_CHECK_FUNCS([ibv_cmd_open_xrcd], [$1_have_xrcd=1]) + AC_CHECK_FUNCS([ibv_cmd_open_xrcd], [$1_have_xrc_domains=1]) fi @@ -238,9 +238,9 @@ AC_DEFUN([OPAL_CHECK_OPENFABRICS],[ fi AC_MSG_CHECKING([if ConnectIB XRC support is enabled]) - AC_DEFINE_UNQUOTED([OPAL_HAVE_XRCD], [$$1_have_xrcd], + AC_DEFINE_UNQUOTED([OPAL_HAVE_CONNECTX_XRC_DOMAINS], [$$1_have_xrc_domains], [Enable features required for XRC domains support]) - if test "1" = "$$1_have_xrcd"; then + if test "1" = "$$1_have_xrc_domains"; then AC_MSG_RESULT([yes]) else AC_MSG_RESULT([no]) diff --git a/opal/mca/btl/openib/btl_openib.c b/opal/mca/btl/openib/btl_openib.c index fb949d6a8e..2527ddbcdb 100644 --- a/opal/mca/btl/openib/btl_openib.c +++ b/opal/mca/btl/openib/btl_openib.c @@ -325,7 +325,7 @@ static int create_srq(mca_btl_openib_module_t *openib_btl) openib_btl->qps[qp].u.srq_qp.rd_posted = 0; #if HAVE_XRC if(BTL_OPENIB_QP_TYPE_XRC(qp)) { -#if OPAL_HAVE_XRCD +#if OPAL_HAVE_CONNECTX_XRC_DOMAINS struct ibv_srq_init_attr_ex attr_ex; memset(&attr_ex, 0, sizeof(struct ibv_srq_init_attr_ex)); attr_ex.attr.max_wr = attr.attr.max_wr; @@ -1963,7 +1963,7 @@ int mca_btl_openib_put( mca_btl_base_module_t* btl, to_com_frag(frag)->endpoint = ep; #if HAVE_XRC if (MCA_BTL_XRC_ENABLED && BTL_OPENIB_QP_TYPE_XRC(qp)) -#if OPAL_HAVE_XRCD +#if OPAL_HAVE_CONNECTX_XRC_DOMAINS frag->sr_desc.qp_type.xrc.remote_srqn=ep->rem_info.rem_srqs[qp].rem_srq_num; #else frag->sr_desc.xrc_remote_srq_num=ep->rem_info.rem_srqs[qp].rem_srq_num; @@ -2056,7 +2056,7 @@ int mca_btl_openib_get(mca_btl_base_module_t* btl, #if HAVE_XRC if (MCA_BTL_XRC_ENABLED && BTL_OPENIB_QP_TYPE_XRC(qp)) -#if OPAL_HAVE_XRCD +#if OPAL_HAVE_CONNECTX_XRC_DOMAINS frag->sr_desc.qp_type.xrc.remote_srqn=ep->rem_info.rem_srqs[qp].rem_srq_num; #else frag->sr_desc.xrc_remote_srq_num=ep->rem_info.rem_srqs[qp].rem_srq_num; diff --git a/opal/mca/btl/openib/btl_openib.h b/opal/mca/btl/openib/btl_openib.h index a6f8bfc467..36863351d3 100644 --- a/opal/mca/btl/openib/btl_openib.h +++ b/opal/mca/btl/openib/btl_openib.h @@ -401,7 +401,7 @@ typedef struct mca_btl_openib_device_t { volatile bool got_port_event; #endif #if HAVE_XRC -#if OPAL_HAVE_XRCD +#if OPAL_HAVE_CONNECTX_XRC_DOMAINS struct ibv_xrcd *xrcd; #else struct ibv_xrc_domain *xrc_domain; diff --git a/opal/mca/btl/openib/btl_openib_async.c b/opal/mca/btl/openib/btl_openib_async.c index ca4743d3d3..e7fe7e8625 100644 --- a/opal/mca/btl/openib/btl_openib_async.c +++ b/opal/mca/btl/openib/btl_openib_async.c @@ -131,7 +131,7 @@ static mca_btl_openib_endpoint_t * xrc_qp2endpoint(uint32_t qp_num, mca_btl_open int ep_i; for(ep_i = 0; ep_i < opal_pointer_array_get_size(device->endpoints); ep_i++) { ep = opal_pointer_array_get_item(device->endpoints, ep_i); -#if OPAL_HAVE_XRCD +#if OPAL_HAVE_CONNECTX_XRC_DOMAINS if (qp_num == ep->xrc_recv_qp->qp_num) #else if (qp_num == ep->xrc_recv_qp_num) @@ -359,7 +359,7 @@ static int btl_openib_async_deviceh(struct mca_btl_openib_async_poll *devices_po event_type = event.event_type; #if HAVE_XRC /* is it XRC event ?*/ -#if OPAL_HAVE_XRCD +#if OPAL_HAVE_CONNECTX_XRC_DOMAINS #else if (IBV_XRC_QP_EVENT_FLAG & event.event_type) { xrc_event = true; @@ -377,7 +377,7 @@ static int btl_openib_async_deviceh(struct mca_btl_openib_async_poll *devices_po mca_btl_openib_load_apm(event.element.qp, qp2endpoint(event.element.qp, device)); #if HAVE_XRC -#if OPAL_HAVE_XRCD +#if OPAL_HAVE_CONNECTX_XRC_DOMAINS #else else mca_btl_openib_load_apm_xrc_rcv(event.element.xrc_qp_num, @@ -661,7 +661,7 @@ void mca_btl_openib_load_apm(struct ibv_qp *qp, mca_btl_openib_endpoint_t *ep) qp->qp_num, strerror(errno), errno)); } -#if HAVE_XRC && ! OPAL_HAVE_XRCD +#if HAVE_XRC && ! OPAL_HAVE_CONNECTX_XRC_DOMAINS void mca_btl_openib_load_apm_xrc_rcv(uint32_t qp_num, mca_btl_openib_endpoint_t *ep) { struct ibv_qp_init_attr qp_init_attr; diff --git a/opal/mca/btl/openib/btl_openib_async.h b/opal/mca/btl/openib/btl_openib_async.h index 3021e66d27..c9243821d3 100644 --- a/opal/mca/btl/openib/btl_openib_async.h +++ b/opal/mca/btl/openib/btl_openib_async.h @@ -19,7 +19,7 @@ int start_async_event_thread(void); void mca_btl_openib_load_apm(struct ibv_qp *qp, mca_btl_openib_endpoint_t *ep); int btl_openib_async_command_done(int exp); -#if HAVE_XRC && ! OPAL_HAVE_XRCD +#if HAVE_XRC && ! OPAL_HAVE_CONNECTX_XRC_DOMAINS void mca_btl_openib_load_apm_xrc_rcv(uint32_t qp_num, mca_btl_openib_endpoint_t *ep); #endif diff --git a/opal/mca/btl/openib/btl_openib_endpoint.c b/opal/mca/btl/openib/btl_openib_endpoint.c index 45849d456f..5787d11544 100644 --- a/opal/mca/btl/openib/btl_openib_endpoint.c +++ b/opal/mca/btl/openib/btl_openib_endpoint.c @@ -352,7 +352,7 @@ static void mca_btl_openib_endpoint_construct(mca_btl_base_endpoint_t* endpoint) } endpoint->ib_addr = NULL; -#if OPAL_HAVE_XRCD +#if OPAL_HAVE_CONNECTX_XRC_DOMAINS endpoint->xrc_recv_qp = NULL; #else endpoint->xrc_recv_qp_num = 0; @@ -467,7 +467,7 @@ static void mca_btl_openib_endpoint_destruct(mca_btl_base_endpoint_t* endpoint) /* unregister xrc recv qp */ #if HAVE_XRC -#if OPAL_HAVE_XRCD +#if OPAL_HAVE_CONNECTX_XRC_DOMAINS if (NULL != endpoint->xrc_recv_qp) { if(ibv_destroy_qp(endpoint->xrc_recv_qp)) { BTL_ERROR(("Failed to unregister XRC recv QP:%d\n", endpoint->xrc_recv_qp->qp_num)); diff --git a/opal/mca/btl/openib/btl_openib_endpoint.h b/opal/mca/btl/openib/btl_openib_endpoint.h index 4db2c16a6a..6d1baf0457 100644 --- a/opal/mca/btl/openib/btl_openib_endpoint.h +++ b/opal/mca/btl/openib/btl_openib_endpoint.h @@ -211,7 +211,7 @@ struct mca_btl_base_endpoint_t { opal_list_t pending_lazy_frags; mca_btl_openib_endpoint_qp_t *qps; -#if OPAL_HAVE_XRCD +#if OPAL_HAVE_CONNECTX_XRC_DOMAINS struct ibv_qp *xrc_recv_qp; #else uint32_t xrc_recv_qp_num; /* in xrc we will use it as recv qp */ @@ -606,7 +606,7 @@ static inline int post_send(mca_btl_openib_endpoint_t *ep, } #if HAVE_XRC -#if OPAL_HAVE_XRCD +#if OPAL_HAVE_CONNECTX_XRC_DOMAINS if(BTL_OPENIB_QP_TYPE_XRC(qp)) sr_desc->qp_type.xrc.remote_srqn = ep->rem_info.rem_srqs[qp].rem_srq_num; #else diff --git a/opal/mca/btl/openib/btl_openib_xrc.c b/opal/mca/btl/openib/btl_openib_xrc.c index 9118a05c01..8a837817aa 100644 --- a/opal/mca/btl/openib/btl_openib_xrc.c +++ b/opal/mca/btl/openib/btl_openib_xrc.c @@ -2,11 +2,9 @@ * Copyright (c) 2007-2008 Mellanox Technologies. All rights reserved. * Copyright (c) 2009 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2014 NVIDIA Corporation. All rights reserved. - * Copyright (c) 2014 Research Organization for Information Science + * Copyright (c) 2014-2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2014 Bull SAS. All rights reserved. - * Copyright (c) 2015 Research Organization for Information Science - * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -50,7 +48,7 @@ bool mca_btl_openib_xrc_check_api() return false; } -#if OPAL_HAVE_XRCD +#if OPAL_HAVE_CONNECTX_XRC_DOMAINS if (NULL != dlsym(lib, "ibv_open_xrcd")) { BTL_ERROR(("XRC error: bad XRC API (require XRC from OFED 3.12+)")); return false; @@ -70,7 +68,7 @@ int mca_btl_openib_open_xrc_domain(struct mca_btl_openib_device_t *device) int len; char *xrc_file_name; const char *dev_name; -#if OPAL_HAVE_XRCD +#if OPAL_HAVE_CONNECTX_XRC_DOMAINS struct ibv_xrcd_init_attr xrcd_attr; #endif @@ -91,7 +89,7 @@ int mca_btl_openib_open_xrc_domain(struct mca_btl_openib_device_t *device) free(xrc_file_name); return OPAL_ERROR; } -#if OPAL_HAVE_XRCD +#if OPAL_HAVE_CONNECTX_XRC_DOMAINS memset(&xrcd_attr, 0, sizeof xrcd_attr); xrcd_attr.comp_mask = IBV_XRCD_INIT_ATTR_FD | IBV_XRCD_INIT_ATTR_OFLAGS; xrcd_attr.fd = device->xrc_fd; @@ -114,7 +112,7 @@ int mca_btl_openib_open_xrc_domain(struct mca_btl_openib_device_t *device) /* This func. closes XRC domain */ int mca_btl_openib_close_xrc_domain(struct mca_btl_openib_device_t *device) { -#if OPAL_HAVE_XRCD +#if OPAL_HAVE_CONNECTX_XRC_DOMAINS if (NULL == device->xrcd) { #else if (NULL == device->xrc_domain) { @@ -122,7 +120,7 @@ int mca_btl_openib_close_xrc_domain(struct mca_btl_openib_device_t *device) /* No XRC domain, just exit */ return OPAL_SUCCESS; } -#if OPAL_HAVE_XRCD +#if OPAL_HAVE_CONNECTX_XRC_DOMAINS if (ibv_close_xrcd(device->xrcd)) { #else if (ibv_close_xrc_domain(device->xrc_domain)) { diff --git a/opal/mca/btl/openib/connect/btl_openib_connect_udcm.c b/opal/mca/btl/openib/connect/btl_openib_connect_udcm.c index c22e95e6cc..5f24701d6a 100644 --- a/opal/mca/btl/openib/connect/btl_openib_connect_udcm.c +++ b/opal/mca/btl/openib/connect/btl_openib_connect_udcm.c @@ -5,12 +5,10 @@ * Copyright (c) 2009 IBM Corporation. All rights reserved. * Copyright (c) 2011-2014 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2014 Research Organization for Information Science + * Copyright (c) 2014-2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2014 Intel, Inc. All rights reserved. * Copyright (c) 2014 Bull SAS. All rights reserved. - * Copyright (c) 2014-2015 Research Organization for Information Science - * and Technology (RIST). All rights reserved. * * $COPYRIGHT$ * @@ -332,7 +330,7 @@ static int udcm_xrc_start_connect (opal_btl_openib_connect_base_module_t *cpc, static int udcm_xrc_restart_connect (mca_btl_base_endpoint_t *lcl_ep); static int udcm_xrc_send_qp_connect (mca_btl_openib_endpoint_t *lcl_ep, udcm_msg_hdr_t *msg_hdr); static int udcm_xrc_send_qp_create (mca_btl_base_endpoint_t *lcl_ep); -#if OPAL_HAVE_XRCD +#if OPAL_HAVE_CONNECTX_XRC_DOMAINS static int udcm_xrc_recv_qp_connect (mca_btl_openib_endpoint_t *lcl_ep, uint32_t qp_num); #else static int udcm_xrc_recv_qp_connect (mca_btl_openib_endpoint_t *lcl_ep); @@ -1970,7 +1968,7 @@ static int udcm_process_messages (struct ibv_cq *event_cq, udcm_module_t *m) if (UDCM_MESSAGE_XCONNECT2 == message->hdr.type) { /* save the qp number for unregister */ -#if ! OPAL_HAVE_XRCD +#if ! OPAL_HAVE_CONNECTX_XRC_DOMAINS lcl_ep->xrc_recv_qp_num = message->hdr.data.xreq.rem_qp_num; #endif @@ -2413,7 +2411,7 @@ static int udcm_xrc_send_qp_create (mca_btl_base_endpoint_t *lcl_ep) uint32_t send_wr; struct ibv_qp **qp; uint32_t *psn; -#if OPAL_HAVE_XRCD +#if OPAL_HAVE_CONNECTX_XRC_DOMAINS struct ibv_qp_init_attr_ex qp_init_attr; #else struct ibv_qp_init_attr qp_init_attr; @@ -2434,7 +2432,7 @@ static int udcm_xrc_send_qp_create (mca_btl_base_endpoint_t *lcl_ep) send_wr = lcl_ep->ib_addr->qp->sd_wqe + (mca_btl_openib_component.use_eager_rdma ? mca_btl_openib_component.max_eager_rdma : 0); -#if OPAL_HAVE_XRCD +#if OPAL_HAVE_CONNECTX_XRC_DOMAINS memset(&qp_init_attr, 0, sizeof(struct ibv_qp_init_attr_ex)); #else memset(&qp_init_attr, 0, sizeof(struct ibv_qp_init_attr)); @@ -2451,7 +2449,7 @@ static int udcm_xrc_send_qp_create (mca_btl_base_endpoint_t *lcl_ep) qp_init_attr.cap.max_send_sge = 1; /* this one is ignored by driver */ qp_init_attr.cap.max_recv_sge = 1; /* we do not use SG list */ -#if OPAL_HAVE_XRCD +#if OPAL_HAVE_CONNECTX_XRC_DOMAINS qp_init_attr.qp_type = IBV_QPT_XRC_SEND; qp_init_attr.comp_mask = IBV_QP_INIT_ATTR_PD; qp_init_attr.pd = openib_btl->device->ib_pd; @@ -2507,7 +2505,7 @@ static int udcm_xrc_send_qp_create (mca_btl_base_endpoint_t *lcl_ep) /* mark: xrc receive qp */ /* Recv qp connect */ -#if OPAL_HAVE_XRCD +#if OPAL_HAVE_CONNECTX_XRC_DOMAINS static int udcm_xrc_recv_qp_connect (mca_btl_openib_endpoint_t *lcl_ep, uint32_t qp_num) #else static int udcm_xrc_recv_qp_connect (mca_btl_openib_endpoint_t *lcl_ep) @@ -2515,7 +2513,7 @@ static int udcm_xrc_recv_qp_connect (mca_btl_openib_endpoint_t *lcl_ep) { mca_btl_openib_module_t *openib_btl = lcl_ep->endpoint_btl; -#if OPAL_HAVE_XRCD +#if OPAL_HAVE_CONNECTX_XRC_DOMAINS struct ibv_qp_open_attr attr; memset(&attr, 0, sizeof(struct ibv_qp_open_attr)); attr.comp_mask = IBV_QP_OPEN_ATTR_NUM | IBV_QP_OPEN_ATTR_XRCD | IBV_QP_OPEN_ATTR_TYPE; @@ -2553,7 +2551,7 @@ static int udcm_xrc_recv_qp_connect (mca_btl_openib_endpoint_t *lcl_ep) static int udcm_xrc_recv_qp_create (mca_btl_openib_endpoint_t *lcl_ep, udcm_msg_hdr_t *msg_hdr) { mca_btl_openib_module_t* openib_btl = lcl_ep->endpoint_btl; -#if OPAL_HAVE_XRCD +#if OPAL_HAVE_CONNECTX_XRC_DOMAINS struct ibv_qp_init_attr_ex qp_init_attr; #else struct ibv_qp_init_attr qp_init_attr; @@ -2563,7 +2561,7 @@ static int udcm_xrc_recv_qp_create (mca_btl_openib_endpoint_t *lcl_ep, udcm_msg_ BTL_VERBOSE(("creating xrc receive qp")); -#if OPAL_HAVE_XRCD +#if OPAL_HAVE_CONNECTX_XRC_DOMAINS memset(&qp_init_attr, 0, sizeof(struct ibv_qp_init_attr_ex)); qp_init_attr.qp_type = IBV_QPT_XRC_RECV; qp_init_attr.comp_mask = IBV_QP_INIT_ATTR_XRCD; @@ -2592,7 +2590,7 @@ static int udcm_xrc_recv_qp_create (mca_btl_openib_endpoint_t *lcl_ep, udcm_msg_ attr.pkey_index = openib_btl->pkey_index; attr.port_num = openib_btl->port_num; attr.qp_access_flags = IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ; -#if OPAL_HAVE_XRCD +#if OPAL_HAVE_CONNECTX_XRC_DOMAINS ret = ibv_modify_qp(lcl_ep->xrc_recv_qp, &attr, IBV_QP_STATE| @@ -2636,7 +2634,7 @@ static int udcm_xrc_recv_qp_create (mca_btl_openib_endpoint_t *lcl_ep, udcm_msg_ /* if user enabled dynamic SL, get it from PathRecord */ if (0 != mca_btl_openib_component.ib_path_record_service_level) { int rc = btl_openib_connect_get_pathrecord_sl( -#if OPAL_HAVE_XRCD +#if OPAL_HAVE_CONNECTX_XRC_DOMAINS openib_btl->device->xrcd->context, #else openib_btl->device->xrc_domain->context, @@ -2651,7 +2649,7 @@ static int udcm_xrc_recv_qp_create (mca_btl_openib_endpoint_t *lcl_ep, udcm_msg_ } #endif -#if OPAL_HAVE_XRCD +#if OPAL_HAVE_CONNECTX_XRC_DOMAINS ret = ibv_modify_qp(lcl_ep->xrc_recv_qp, &attr, IBV_QP_STATE| @@ -2685,7 +2683,7 @@ static int udcm_xrc_recv_qp_create (mca_btl_openib_endpoint_t *lcl_ep, udcm_msg_ #endif #if OPAL_HAVE_THREADS if (APM_ENABLED) { -#if OPAL_HAVE_XRCD +#if OPAL_HAVE_CONNECTX_XRC_DOMAINS mca_btl_openib_load_apm(lcl_ep->xrc_recv_qp, lcl_ep); #else mca_btl_openib_load_apm_xrc_rcv(lcl_ep->xrc_recv_qp_num, lcl_ep); @@ -2754,7 +2752,7 @@ static int udcm_xrc_send_xresponse (mca_btl_base_endpoint_t *lcl_ep, mca_btl_bas msg->data->hdr.data.xres.rem_ep_index = htonl(lcl_ep->index); if (UDCM_MESSAGE_XRESPONSE == msg_type) { -#if OPAL_HAVE_XRCD +#if OPAL_HAVE_CONNECTX_XRC_DOMAINS BTL_VERBOSE(("Sending qp: %d, psn: %d", lcl_ep->xrc_recv_qp->qp_num, lcl_ep->xrc_recv_psn)); msg->data->hdr.data.xres.rem_qp_num = htonl(lcl_ep->xrc_recv_qp->qp_num); msg->data->hdr.data.xres.rem_psn = htonl(lcl_ep->xrc_recv_psn); @@ -2766,7 +2764,7 @@ static int udcm_xrc_send_xresponse (mca_btl_base_endpoint_t *lcl_ep, mca_btl_bas } for (int i = 0; i < mca_btl_openib_component.num_xrc_qps; ++i) { -#if OPAL_HAVE_XRCD +#if OPAL_HAVE_CONNECTX_XRC_DOMAINS uint32_t srq_num; if (ibv_get_srq_num(lcl_ep->endpoint_btl->qps[i].u.srq_qp.srq, &srq_num)) { BTL_ERROR(("BTL openib XOOB internal error: can't get srq num")); @@ -2816,7 +2814,7 @@ static int udcm_xrc_handle_xconnect (mca_btl_openib_endpoint_t *lcl_ep, udcm_msg if (UDCM_MESSAGE_XCONNECT2 == msg_hdr->type) { response_type = UDCM_MESSAGE_XRESPONSE2; -#if OPAL_HAVE_XRCD +#if OPAL_HAVE_CONNECTX_XRC_DOMAINS rc = udcm_xrc_recv_qp_connect (lcl_ep, msg_hdr->data.xreq.rem_qp_num); #else rc = udcm_xrc_recv_qp_connect (lcl_ep);