ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

btl/openib: rename OPAL_HAVE_XRCD macro into OPAL_HAVE_CONNECTX_XRC_DOMAINS

Просмотр исходного кода
Этот коммит содержится в:
Gilles Gouaillardet 2015-01-07 13:27:25 +09:00
родитель e0927895db
Коммит 135ecce0eb
9 изменённых файлов: 40 добавлений и 44 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

8
config/opal_check_openfabrics.m4
Просмотреть файл

@ -151,7 +151,7 @@ AC_DEFUN([OPAL_CHECK_OPENFABRICS],[
# Set these up so that we can do an AC_DEFINE below # Set these up so that we can do an AC_DEFINE below
# (unconditionally) # (unconditionally)
$1_have_xrc=0 $1_have_xrc=0
$1_have_xrcd=0 $1_have_xrc_domains=0
$1_have_opensm_devel=0 $1_have_opensm_devel=0
# If we have the openib stuff available, find out what we've got # If we have the openib stuff available, find out what we've got
@ -170,7 +170,7 @@ AC_DEFUN([OPAL_CHECK_OPENFABRICS],[
AC_CHECK_FUNCS([ibv_create_xrc_rcv_qp ibv_cmd_open_xrcd], [$1_have_xrc=1]) AC_CHECK_FUNCS([ibv_create_xrc_rcv_qp ibv_cmd_open_xrcd], [$1_have_xrc=1])
fi fi
if test "$enable_connectx_xrc" = "yes"; then if test "$enable_connectx_xrc" = "yes"; then
AC_CHECK_FUNCS([ibv_cmd_open_xrcd], [$1_have_xrcd=1]) AC_CHECK_FUNCS([ibv_cmd_open_xrcd], [$1_have_xrc_domains=1])
fi fi
@ -238,9 +238,9 @@ AC_DEFUN([OPAL_CHECK_OPENFABRICS],[
fi fi
AC_MSG_CHECKING([if ConnectIB XRC support is enabled]) AC_MSG_CHECKING([if ConnectIB XRC support is enabled])
AC_DEFINE_UNQUOTED([OPAL_HAVE_XRCD], [$$1_have_xrcd], AC_DEFINE_UNQUOTED([OPAL_HAVE_CONNECTX_XRC_DOMAINS], [$$1_have_xrc_domains],
[Enable features required for XRC domains support]) [Enable features required for XRC domains support])
if test "1" = "$$1_have_xrcd"; then if test "1" = "$$1_have_xrc_domains"; then
AC_MSG_RESULT([yes]) AC_MSG_RESULT([yes])
else else
AC_MSG_RESULT([no]) AC_MSG_RESULT([no])

6
opal/mca/btl/openib/btl_openib.c
Просмотреть файл

@ -325,7 +325,7 @@ static int create_srq(mca_btl_openib_module_t *openib_btl)
openib_btl->qps[qp].u.srq_qp.rd_posted = 0; openib_btl->qps[qp].u.srq_qp.rd_posted = 0;
#if HAVE_XRC #if HAVE_XRC
if(BTL_OPENIB_QP_TYPE_XRC(qp)) { if(BTL_OPENIB_QP_TYPE_XRC(qp)) {
#if OPAL_HAVE_XRCD #if OPAL_HAVE_CONNECTX_XRC_DOMAINS
struct ibv_srq_init_attr_ex attr_ex; struct ibv_srq_init_attr_ex attr_ex;
memset(&attr_ex, 0, sizeof(struct ibv_srq_init_attr_ex)); memset(&attr_ex, 0, sizeof(struct ibv_srq_init_attr_ex));
attr_ex.attr.max_wr = attr.attr.max_wr; attr_ex.attr.max_wr = attr.attr.max_wr;
@ -1963,7 +1963,7 @@ int mca_btl_openib_put( mca_btl_base_module_t* btl,
to_com_frag(frag)->endpoint = ep; to_com_frag(frag)->endpoint = ep;
#if HAVE_XRC #if HAVE_XRC
if (MCA_BTL_XRC_ENABLED && BTL_OPENIB_QP_TYPE_XRC(qp)) if (MCA_BTL_XRC_ENABLED && BTL_OPENIB_QP_TYPE_XRC(qp))
#if OPAL_HAVE_XRCD #if OPAL_HAVE_CONNECTX_XRC_DOMAINS
frag->sr_desc.qp_type.xrc.remote_srqn=ep->rem_info.rem_srqs[qp].rem_srq_num; frag->sr_desc.qp_type.xrc.remote_srqn=ep->rem_info.rem_srqs[qp].rem_srq_num;
#else #else
frag->sr_desc.xrc_remote_srq_num=ep->rem_info.rem_srqs[qp].rem_srq_num; frag->sr_desc.xrc_remote_srq_num=ep->rem_info.rem_srqs[qp].rem_srq_num;
@ -2056,7 +2056,7 @@ int mca_btl_openib_get(mca_btl_base_module_t* btl,
#if HAVE_XRC #if HAVE_XRC
if (MCA_BTL_XRC_ENABLED && BTL_OPENIB_QP_TYPE_XRC(qp)) if (MCA_BTL_XRC_ENABLED && BTL_OPENIB_QP_TYPE_XRC(qp))
#if OPAL_HAVE_XRCD #if OPAL_HAVE_CONNECTX_XRC_DOMAINS
frag->sr_desc.qp_type.xrc.remote_srqn=ep->rem_info.rem_srqs[qp].rem_srq_num; frag->sr_desc.qp_type.xrc.remote_srqn=ep->rem_info.rem_srqs[qp].rem_srq_num;
#else #else
frag->sr_desc.xrc_remote_srq_num=ep->rem_info.rem_srqs[qp].rem_srq_num; frag->sr_desc.xrc_remote_srq_num=ep->rem_info.rem_srqs[qp].rem_srq_num;

2
opal/mca/btl/openib/btl_openib.h
Просмотреть файл

@ -401,7 +401,7 @@ typedef struct mca_btl_openib_device_t {
volatile bool got_port_event; volatile bool got_port_event;
#endif #endif
#if HAVE_XRC #if HAVE_XRC
#if OPAL_HAVE_XRCD #if OPAL_HAVE_CONNECTX_XRC_DOMAINS
struct ibv_xrcd *xrcd; struct ibv_xrcd *xrcd;
#else #else
struct ibv_xrc_domain *xrc_domain; struct ibv_xrc_domain *xrc_domain;

8
opal/mca/btl/openib/btl_openib_async.c
Просмотреть файл

@ -131,7 +131,7 @@ static mca_btl_openib_endpoint_t * xrc_qp2endpoint(uint32_t qp_num, mca_btl_open
int ep_i; int ep_i;
for(ep_i = 0; ep_i < opal_pointer_array_get_size(device->endpoints); ep_i++) { for(ep_i = 0; ep_i < opal_pointer_array_get_size(device->endpoints); ep_i++) {
ep = opal_pointer_array_get_item(device->endpoints, ep_i); ep = opal_pointer_array_get_item(device->endpoints, ep_i);
#if OPAL_HAVE_XRCD #if OPAL_HAVE_CONNECTX_XRC_DOMAINS
if (qp_num == ep->xrc_recv_qp->qp_num) if (qp_num == ep->xrc_recv_qp->qp_num)
#else #else
if (qp_num == ep->xrc_recv_qp_num) if (qp_num == ep->xrc_recv_qp_num)
@ -359,7 +359,7 @@ static int btl_openib_async_deviceh(struct mca_btl_openib_async_poll *devices_po
event_type = event.event_type; event_type = event.event_type;
#if HAVE_XRC #if HAVE_XRC
/* is it XRC event ?*/ /* is it XRC event ?*/
#if OPAL_HAVE_XRCD #if OPAL_HAVE_CONNECTX_XRC_DOMAINS
#else #else
if (IBV_XRC_QP_EVENT_FLAG & event.event_type) { if (IBV_XRC_QP_EVENT_FLAG & event.event_type) {
xrc_event = true; xrc_event = true;
@ -377,7 +377,7 @@ static int btl_openib_async_deviceh(struct mca_btl_openib_async_poll *devices_po
mca_btl_openib_load_apm(event.element.qp, mca_btl_openib_load_apm(event.element.qp,
qp2endpoint(event.element.qp, device)); qp2endpoint(event.element.qp, device));
#if HAVE_XRC #if HAVE_XRC
#if OPAL_HAVE_XRCD #if OPAL_HAVE_CONNECTX_XRC_DOMAINS
#else #else
else else
mca_btl_openib_load_apm_xrc_rcv(event.element.xrc_qp_num, mca_btl_openib_load_apm_xrc_rcv(event.element.xrc_qp_num,
@ -661,7 +661,7 @@ void mca_btl_openib_load_apm(struct ibv_qp *qp, mca_btl_openib_endpoint_t *ep)
qp->qp_num, strerror(errno), errno)); qp->qp_num, strerror(errno), errno));
} }
#if HAVE_XRC && ! OPAL_HAVE_XRCD #if HAVE_XRC && ! OPAL_HAVE_CONNECTX_XRC_DOMAINS
void mca_btl_openib_load_apm_xrc_rcv(uint32_t qp_num, mca_btl_openib_endpoint_t *ep) void mca_btl_openib_load_apm_xrc_rcv(uint32_t qp_num, mca_btl_openib_endpoint_t *ep)
{ {
struct ibv_qp_init_attr qp_init_attr; struct ibv_qp_init_attr qp_init_attr;

2
opal/mca/btl/openib/btl_openib_async.h
Просмотреть файл

@ -19,7 +19,7 @@
int start_async_event_thread(void); int start_async_event_thread(void);
void mca_btl_openib_load_apm(struct ibv_qp *qp, mca_btl_openib_endpoint_t *ep); void mca_btl_openib_load_apm(struct ibv_qp *qp, mca_btl_openib_endpoint_t *ep);
int btl_openib_async_command_done(int exp); int btl_openib_async_command_done(int exp);
#if HAVE_XRC && ! OPAL_HAVE_XRCD #if HAVE_XRC && ! OPAL_HAVE_CONNECTX_XRC_DOMAINS
void mca_btl_openib_load_apm_xrc_rcv(uint32_t qp_num, mca_btl_openib_endpoint_t *ep); void mca_btl_openib_load_apm_xrc_rcv(uint32_t qp_num, mca_btl_openib_endpoint_t *ep);
#endif #endif

4
opal/mca/btl/openib/btl_openib_endpoint.c
Просмотреть файл

@ -352,7 +352,7 @@ static void mca_btl_openib_endpoint_construct(mca_btl_base_endpoint_t* endpoint)
} }
endpoint->ib_addr = NULL; endpoint->ib_addr = NULL;
#if OPAL_HAVE_XRCD #if OPAL_HAVE_CONNECTX_XRC_DOMAINS
endpoint->xrc_recv_qp = NULL; endpoint->xrc_recv_qp = NULL;
#else #else
endpoint->xrc_recv_qp_num = 0; endpoint->xrc_recv_qp_num = 0;
@ -467,7 +467,7 @@ static void mca_btl_openib_endpoint_destruct(mca_btl_base_endpoint_t* endpoint)
/* unregister xrc recv qp */ /* unregister xrc recv qp */
#if HAVE_XRC #if HAVE_XRC
#if OPAL_HAVE_XRCD #if OPAL_HAVE_CONNECTX_XRC_DOMAINS
if (NULL != endpoint->xrc_recv_qp) { if (NULL != endpoint->xrc_recv_qp) {
if(ibv_destroy_qp(endpoint->xrc_recv_qp)) { if(ibv_destroy_qp(endpoint->xrc_recv_qp)) {
BTL_ERROR(("Failed to unregister XRC recv QP:%d\n", endpoint->xrc_recv_qp->qp_num)); BTL_ERROR(("Failed to unregister XRC recv QP:%d\n", endpoint->xrc_recv_qp->qp_num));

4
opal/mca/btl/openib/btl_openib_endpoint.h
Просмотреть файл

@ -211,7 +211,7 @@ struct mca_btl_base_endpoint_t {
opal_list_t pending_lazy_frags; opal_list_t pending_lazy_frags;
mca_btl_openib_endpoint_qp_t *qps; mca_btl_openib_endpoint_qp_t *qps;
#if OPAL_HAVE_XRCD #if OPAL_HAVE_CONNECTX_XRC_DOMAINS
struct ibv_qp *xrc_recv_qp; struct ibv_qp *xrc_recv_qp;
#else #else
uint32_t xrc_recv_qp_num; /* in xrc we will use it as recv qp */ uint32_t xrc_recv_qp_num; /* in xrc we will use it as recv qp */
@ -606,7 +606,7 @@ static inline int post_send(mca_btl_openib_endpoint_t *ep,
} }
#if HAVE_XRC #if HAVE_XRC
#if OPAL_HAVE_XRCD #if OPAL_HAVE_CONNECTX_XRC_DOMAINS
if(BTL_OPENIB_QP_TYPE_XRC(qp)) if(BTL_OPENIB_QP_TYPE_XRC(qp))
sr_desc->qp_type.xrc.remote_srqn = ep->rem_info.rem_srqs[qp].rem_srq_num; sr_desc->qp_type.xrc.remote_srqn = ep->rem_info.rem_srqs[qp].rem_srq_num;
#else #else

14
opal/mca/btl/openib/btl_openib_xrc.c
Просмотреть файл

@ -2,11 +2,9 @@
* Copyright (c) 2007-2008 Mellanox Technologies. All rights reserved. * Copyright (c) 2007-2008 Mellanox Technologies. All rights reserved.
* Copyright (c) 2009 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2009 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2014 NVIDIA Corporation. All rights reserved. * Copyright (c) 2014 NVIDIA Corporation. All rights reserved.
* Copyright (c) 2014 Research Organization for Information Science * Copyright (c) 2014-2015 Research Organization for Information Science
* and Technology (RIST). All rights reserved. * and Technology (RIST). All rights reserved.
* Copyright (c) 2014 Bull SAS. All rights reserved. * Copyright (c) 2014 Bull SAS. All rights reserved.
* Copyright (c) 2015 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* $COPYRIGHT$ * $COPYRIGHT$
* *
* Additional copyrights may follow * Additional copyrights may follow
@ -50,7 +48,7 @@ bool mca_btl_openib_xrc_check_api()
return false; return false;
} }
#if OPAL_HAVE_XRCD #if OPAL_HAVE_CONNECTX_XRC_DOMAINS
if (NULL != dlsym(lib, "ibv_open_xrcd")) { if (NULL != dlsym(lib, "ibv_open_xrcd")) {
BTL_ERROR(("XRC error: bad XRC API (require XRC from OFED 3.12+)")); BTL_ERROR(("XRC error: bad XRC API (require XRC from OFED 3.12+)"));
return false; return false;
@ -70,7 +68,7 @@ int mca_btl_openib_open_xrc_domain(struct mca_btl_openib_device_t *device)
int len; int len;
char *xrc_file_name; char *xrc_file_name;
const char *dev_name; const char *dev_name;
#if OPAL_HAVE_XRCD #if OPAL_HAVE_CONNECTX_XRC_DOMAINS
struct ibv_xrcd_init_attr xrcd_attr; struct ibv_xrcd_init_attr xrcd_attr;
#endif #endif
@ -91,7 +89,7 @@ int mca_btl_openib_open_xrc_domain(struct mca_btl_openib_device_t *device)
free(xrc_file_name); free(xrc_file_name);
return OPAL_ERROR; return OPAL_ERROR;
} }
#if OPAL_HAVE_XRCD #if OPAL_HAVE_CONNECTX_XRC_DOMAINS
memset(&xrcd_attr, 0, sizeof xrcd_attr); memset(&xrcd_attr, 0, sizeof xrcd_attr);
xrcd_attr.comp_mask = IBV_XRCD_INIT_ATTR_FD | IBV_XRCD_INIT_ATTR_OFLAGS; xrcd_attr.comp_mask = IBV_XRCD_INIT_ATTR_FD | IBV_XRCD_INIT_ATTR_OFLAGS;
xrcd_attr.fd = device->xrc_fd; xrcd_attr.fd = device->xrc_fd;
@ -114,7 +112,7 @@ int mca_btl_openib_open_xrc_domain(struct mca_btl_openib_device_t *device)
/* This func. closes XRC domain */ /* This func. closes XRC domain */
int mca_btl_openib_close_xrc_domain(struct mca_btl_openib_device_t *device) int mca_btl_openib_close_xrc_domain(struct mca_btl_openib_device_t *device)
{ {
#if OPAL_HAVE_XRCD #if OPAL_HAVE_CONNECTX_XRC_DOMAINS
if (NULL == device->xrcd) { if (NULL == device->xrcd) {
#else #else
if (NULL == device->xrc_domain) { if (NULL == device->xrc_domain) {
@ -122,7 +120,7 @@ int mca_btl_openib_close_xrc_domain(struct mca_btl_openib_device_t *device)
/* No XRC domain, just exit */ /* No XRC domain, just exit */
return OPAL_SUCCESS; return OPAL_SUCCESS;
} }
#if OPAL_HAVE_XRCD #if OPAL_HAVE_CONNECTX_XRC_DOMAINS
if (ibv_close_xrcd(device->xrcd)) { if (ibv_close_xrcd(device->xrcd)) {
#else #else
if (ibv_close_xrc_domain(device->xrc_domain)) { if (ibv_close_xrc_domain(device->xrc_domain)) {

36
opal/mca/btl/openib/connect/btl_openib_connect_udcm.c
Просмотреть файл

@ -5,12 +5,10 @@
* Copyright (c) 2009 IBM Corporation. All rights reserved. * Copyright (c) 2009 IBM Corporation. All rights reserved.
* Copyright (c) 2011-2014 Los Alamos National Security, LLC. All rights * Copyright (c) 2011-2014 Los Alamos National Security, LLC. All rights
* reserved. * reserved.
* Copyright (c) 2014 Research Organization for Information Science * Copyright (c) 2014-2015 Research Organization for Information Science
* and Technology (RIST). All rights reserved. * and Technology (RIST). All rights reserved.
* Copyright (c) 2014 Intel, Inc. All rights reserved. * Copyright (c) 2014 Intel, Inc. All rights reserved.
* Copyright (c) 2014 Bull SAS. All rights reserved. * Copyright (c) 2014 Bull SAS. All rights reserved.
* Copyright (c) 2014-2015 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* *
* $COPYRIGHT$ * $COPYRIGHT$
* *
@ -332,7 +330,7 @@ static int udcm_xrc_start_connect (opal_btl_openib_connect_base_module_t *cpc,
static int udcm_xrc_restart_connect (mca_btl_base_endpoint_t *lcl_ep); static int udcm_xrc_restart_connect (mca_btl_base_endpoint_t *lcl_ep);
static int udcm_xrc_send_qp_connect (mca_btl_openib_endpoint_t *lcl_ep, udcm_msg_hdr_t *msg_hdr); static int udcm_xrc_send_qp_connect (mca_btl_openib_endpoint_t *lcl_ep, udcm_msg_hdr_t *msg_hdr);
static int udcm_xrc_send_qp_create (mca_btl_base_endpoint_t *lcl_ep); static int udcm_xrc_send_qp_create (mca_btl_base_endpoint_t *lcl_ep);
#if OPAL_HAVE_XRCD #if OPAL_HAVE_CONNECTX_XRC_DOMAINS
static int udcm_xrc_recv_qp_connect (mca_btl_openib_endpoint_t *lcl_ep, uint32_t qp_num); static int udcm_xrc_recv_qp_connect (mca_btl_openib_endpoint_t *lcl_ep, uint32_t qp_num);
#else #else
static int udcm_xrc_recv_qp_connect (mca_btl_openib_endpoint_t *lcl_ep); static int udcm_xrc_recv_qp_connect (mca_btl_openib_endpoint_t *lcl_ep);
@ -1970,7 +1968,7 @@ static int udcm_process_messages (struct ibv_cq *event_cq, udcm_module_t *m)
if (UDCM_MESSAGE_XCONNECT2 == message->hdr.type) { if (UDCM_MESSAGE_XCONNECT2 == message->hdr.type) {
/* save the qp number for unregister */ /* save the qp number for unregister */
#if ! OPAL_HAVE_XRCD #if ! OPAL_HAVE_CONNECTX_XRC_DOMAINS
lcl_ep->xrc_recv_qp_num = message->hdr.data.xreq.rem_qp_num; lcl_ep->xrc_recv_qp_num = message->hdr.data.xreq.rem_qp_num;
#endif #endif
@ -2413,7 +2411,7 @@ static int udcm_xrc_send_qp_create (mca_btl_base_endpoint_t *lcl_ep)
uint32_t send_wr; uint32_t send_wr;
struct ibv_qp **qp; struct ibv_qp **qp;
uint32_t *psn; uint32_t *psn;
#if OPAL_HAVE_XRCD #if OPAL_HAVE_CONNECTX_XRC_DOMAINS
struct ibv_qp_init_attr_ex qp_init_attr; struct ibv_qp_init_attr_ex qp_init_attr;
#else #else
struct ibv_qp_init_attr qp_init_attr; struct ibv_qp_init_attr qp_init_attr;
@ -2434,7 +2432,7 @@ static int udcm_xrc_send_qp_create (mca_btl_base_endpoint_t *lcl_ep)
send_wr = lcl_ep->ib_addr->qp->sd_wqe + send_wr = lcl_ep->ib_addr->qp->sd_wqe +
(mca_btl_openib_component.use_eager_rdma ? (mca_btl_openib_component.use_eager_rdma ?
mca_btl_openib_component.max_eager_rdma : 0); mca_btl_openib_component.max_eager_rdma : 0);
#if OPAL_HAVE_XRCD #if OPAL_HAVE_CONNECTX_XRC_DOMAINS
memset(&qp_init_attr, 0, sizeof(struct ibv_qp_init_attr_ex)); memset(&qp_init_attr, 0, sizeof(struct ibv_qp_init_attr_ex));
#else #else
memset(&qp_init_attr, 0, sizeof(struct ibv_qp_init_attr)); memset(&qp_init_attr, 0, sizeof(struct ibv_qp_init_attr));
@ -2451,7 +2449,7 @@ static int udcm_xrc_send_qp_create (mca_btl_base_endpoint_t *lcl_ep)
qp_init_attr.cap.max_send_sge = 1; qp_init_attr.cap.max_send_sge = 1;
/* this one is ignored by driver */ /* this one is ignored by driver */
qp_init_attr.cap.max_recv_sge = 1; /* we do not use SG list */ qp_init_attr.cap.max_recv_sge = 1; /* we do not use SG list */
#if OPAL_HAVE_XRCD #if OPAL_HAVE_CONNECTX_XRC_DOMAINS
qp_init_attr.qp_type = IBV_QPT_XRC_SEND; qp_init_attr.qp_type = IBV_QPT_XRC_SEND;
qp_init_attr.comp_mask = IBV_QP_INIT_ATTR_PD; qp_init_attr.comp_mask = IBV_QP_INIT_ATTR_PD;
qp_init_attr.pd = openib_btl->device->ib_pd; qp_init_attr.pd = openib_btl->device->ib_pd;
@ -2507,7 +2505,7 @@ static int udcm_xrc_send_qp_create (mca_btl_base_endpoint_t *lcl_ep)
/* mark: xrc receive qp */ /* mark: xrc receive qp */
/* Recv qp connect */ /* Recv qp connect */
#if OPAL_HAVE_XRCD #if OPAL_HAVE_CONNECTX_XRC_DOMAINS
static int udcm_xrc_recv_qp_connect (mca_btl_openib_endpoint_t *lcl_ep, uint32_t qp_num) static int udcm_xrc_recv_qp_connect (mca_btl_openib_endpoint_t *lcl_ep, uint32_t qp_num)
#else #else
static int udcm_xrc_recv_qp_connect (mca_btl_openib_endpoint_t *lcl_ep) static int udcm_xrc_recv_qp_connect (mca_btl_openib_endpoint_t *lcl_ep)
@ -2515,7 +2513,7 @@ static int udcm_xrc_recv_qp_connect (mca_btl_openib_endpoint_t *lcl_ep)
{ {
mca_btl_openib_module_t *openib_btl = lcl_ep->endpoint_btl; mca_btl_openib_module_t *openib_btl = lcl_ep->endpoint_btl;
#if OPAL_HAVE_XRCD #if OPAL_HAVE_CONNECTX_XRC_DOMAINS
struct ibv_qp_open_attr attr; struct ibv_qp_open_attr attr;
memset(&attr, 0, sizeof(struct ibv_qp_open_attr)); memset(&attr, 0, sizeof(struct ibv_qp_open_attr));
attr.comp_mask = IBV_QP_OPEN_ATTR_NUM | IBV_QP_OPEN_ATTR_XRCD | IBV_QP_OPEN_ATTR_TYPE; attr.comp_mask = IBV_QP_OPEN_ATTR_NUM | IBV_QP_OPEN_ATTR_XRCD | IBV_QP_OPEN_ATTR_TYPE;
@ -2553,7 +2551,7 @@ static int udcm_xrc_recv_qp_connect (mca_btl_openib_endpoint_t *lcl_ep)
static int udcm_xrc_recv_qp_create (mca_btl_openib_endpoint_t *lcl_ep, udcm_msg_hdr_t *msg_hdr) static int udcm_xrc_recv_qp_create (mca_btl_openib_endpoint_t *lcl_ep, udcm_msg_hdr_t *msg_hdr)
{ {
mca_btl_openib_module_t* openib_btl = lcl_ep->endpoint_btl; mca_btl_openib_module_t* openib_btl = lcl_ep->endpoint_btl;
#if OPAL_HAVE_XRCD #if OPAL_HAVE_CONNECTX_XRC_DOMAINS
struct ibv_qp_init_attr_ex qp_init_attr; struct ibv_qp_init_attr_ex qp_init_attr;
#else #else
struct ibv_qp_init_attr qp_init_attr; struct ibv_qp_init_attr qp_init_attr;
@ -2563,7 +2561,7 @@ static int udcm_xrc_recv_qp_create (mca_btl_openib_endpoint_t *lcl_ep, udcm_msg_
BTL_VERBOSE(("creating xrc receive qp")); BTL_VERBOSE(("creating xrc receive qp"));
#if OPAL_HAVE_XRCD #if OPAL_HAVE_CONNECTX_XRC_DOMAINS
memset(&qp_init_attr, 0, sizeof(struct ibv_qp_init_attr_ex)); memset(&qp_init_attr, 0, sizeof(struct ibv_qp_init_attr_ex));
qp_init_attr.qp_type = IBV_QPT_XRC_RECV; qp_init_attr.qp_type = IBV_QPT_XRC_RECV;
qp_init_attr.comp_mask = IBV_QP_INIT_ATTR_XRCD; qp_init_attr.comp_mask = IBV_QP_INIT_ATTR_XRCD;
@ -2592,7 +2590,7 @@ static int udcm_xrc_recv_qp_create (mca_btl_openib_endpoint_t *lcl_ep, udcm_msg_
attr.pkey_index = openib_btl->pkey_index; attr.pkey_index = openib_btl->pkey_index;
attr.port_num = openib_btl->port_num; attr.port_num = openib_btl->port_num;
attr.qp_access_flags = IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ; attr.qp_access_flags = IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ;
#if OPAL_HAVE_XRCD #if OPAL_HAVE_CONNECTX_XRC_DOMAINS
ret = ibv_modify_qp(lcl_ep->xrc_recv_qp, ret = ibv_modify_qp(lcl_ep->xrc_recv_qp,
&attr, &attr,
IBV_QP_STATE| IBV_QP_STATE|
@ -2636,7 +2634,7 @@ static int udcm_xrc_recv_qp_create (mca_btl_openib_endpoint_t *lcl_ep, udcm_msg_
/* if user enabled dynamic SL, get it from PathRecord */ /* if user enabled dynamic SL, get it from PathRecord */
if (0 != mca_btl_openib_component.ib_path_record_service_level) { if (0 != mca_btl_openib_component.ib_path_record_service_level) {
int rc = btl_openib_connect_get_pathrecord_sl( int rc = btl_openib_connect_get_pathrecord_sl(
#if OPAL_HAVE_XRCD #if OPAL_HAVE_CONNECTX_XRC_DOMAINS
openib_btl->device->xrcd->context, openib_btl->device->xrcd->context,
#else #else
openib_btl->device->xrc_domain->context, openib_btl->device->xrc_domain->context,
@ -2651,7 +2649,7 @@ static int udcm_xrc_recv_qp_create (mca_btl_openib_endpoint_t *lcl_ep, udcm_msg_
} }
#endif #endif
#if OPAL_HAVE_XRCD #if OPAL_HAVE_CONNECTX_XRC_DOMAINS
ret = ibv_modify_qp(lcl_ep->xrc_recv_qp, ret = ibv_modify_qp(lcl_ep->xrc_recv_qp,
&attr, &attr,
IBV_QP_STATE| IBV_QP_STATE|
@ -2685,7 +2683,7 @@ static int udcm_xrc_recv_qp_create (mca_btl_openib_endpoint_t *lcl_ep, udcm_msg_
#endif #endif
#if OPAL_HAVE_THREADS #if OPAL_HAVE_THREADS
if (APM_ENABLED) { if (APM_ENABLED) {
#if OPAL_HAVE_XRCD #if OPAL_HAVE_CONNECTX_XRC_DOMAINS
mca_btl_openib_load_apm(lcl_ep->xrc_recv_qp, lcl_ep); mca_btl_openib_load_apm(lcl_ep->xrc_recv_qp, lcl_ep);
#else #else
mca_btl_openib_load_apm_xrc_rcv(lcl_ep->xrc_recv_qp_num, lcl_ep); mca_btl_openib_load_apm_xrc_rcv(lcl_ep->xrc_recv_qp_num, lcl_ep);
@ -2754,7 +2752,7 @@ static int udcm_xrc_send_xresponse (mca_btl_base_endpoint_t *lcl_ep, mca_btl_bas
msg->data->hdr.data.xres.rem_ep_index = htonl(lcl_ep->index); msg->data->hdr.data.xres.rem_ep_index = htonl(lcl_ep->index);
if (UDCM_MESSAGE_XRESPONSE == msg_type) { if (UDCM_MESSAGE_XRESPONSE == msg_type) {
#if OPAL_HAVE_XRCD #if OPAL_HAVE_CONNECTX_XRC_DOMAINS
BTL_VERBOSE(("Sending qp: %d, psn: %d", lcl_ep->xrc_recv_qp->qp_num, lcl_ep->xrc_recv_psn)); BTL_VERBOSE(("Sending qp: %d, psn: %d", lcl_ep->xrc_recv_qp->qp_num, lcl_ep->xrc_recv_psn));
msg->data->hdr.data.xres.rem_qp_num = htonl(lcl_ep->xrc_recv_qp->qp_num); msg->data->hdr.data.xres.rem_qp_num = htonl(lcl_ep->xrc_recv_qp->qp_num);
msg->data->hdr.data.xres.rem_psn = htonl(lcl_ep->xrc_recv_psn); msg->data->hdr.data.xres.rem_psn = htonl(lcl_ep->xrc_recv_psn);
@ -2766,7 +2764,7 @@ static int udcm_xrc_send_xresponse (mca_btl_base_endpoint_t *lcl_ep, mca_btl_bas
} }
for (int i = 0; i < mca_btl_openib_component.num_xrc_qps; ++i) { for (int i = 0; i < mca_btl_openib_component.num_xrc_qps; ++i) {
#if OPAL_HAVE_XRCD #if OPAL_HAVE_CONNECTX_XRC_DOMAINS
uint32_t srq_num; uint32_t srq_num;
if (ibv_get_srq_num(lcl_ep->endpoint_btl->qps[i].u.srq_qp.srq, &srq_num)) { if (ibv_get_srq_num(lcl_ep->endpoint_btl->qps[i].u.srq_qp.srq, &srq_num)) {
BTL_ERROR(("BTL openib XOOB internal error: can't get srq num")); BTL_ERROR(("BTL openib XOOB internal error: can't get srq num"));
@ -2816,7 +2814,7 @@ static int udcm_xrc_handle_xconnect (mca_btl_openib_endpoint_t *lcl_ep, udcm_msg
if (UDCM_MESSAGE_XCONNECT2 == msg_hdr->type) { if (UDCM_MESSAGE_XCONNECT2 == msg_hdr->type) {
response_type = UDCM_MESSAGE_XRESPONSE2; response_type = UDCM_MESSAGE_XRESPONSE2;
#if OPAL_HAVE_XRCD #if OPAL_HAVE_CONNECTX_XRC_DOMAINS
rc = udcm_xrc_recv_qp_connect (lcl_ep, msg_hdr->data.xreq.rem_qp_num); rc = udcm_xrc_recv_qp_connect (lcl_ep, msg_hdr->data.xreq.rem_qp_num);
#else #else
rc = udcm_xrc_recv_qp_connect (lcl_ep); rc = udcm_xrc_recv_qp_connect (lcl_ep);