btl/openib: rename OPAL_HAVE_XRCD macro into OPAL_HAVE_CONNECTX_XRC_DOMAINS

config/opal_check_openfabrics.m4

@@ -151,7 +151,7 @@ AC_DEFUN([OPAL_CHECK_OPENFABRICS],[
     # Set these up so that we can do an AC_DEFINE below
     # (unconditionally)
     $1_have_xrc=0
-    $1_have_xrcd=0
+    $1_have_xrc_domains=0
     $1_have_opensm_devel=0
 
     # If we have the openib stuff available, find out what we've got
@@ -170,7 +170,7 @@ AC_DEFUN([OPAL_CHECK_OPENFABRICS],[
                AC_CHECK_FUNCS([ibv_create_xrc_rcv_qp ibv_cmd_open_xrcd], [$1_have_xrc=1])
            fi
            if test "$enable_connectx_xrc" = "yes"; then
-               AC_CHECK_FUNCS([ibv_cmd_open_xrcd], [$1_have_xrcd=1])
+               AC_CHECK_FUNCS([ibv_cmd_open_xrcd], [$1_have_xrc_domains=1])
            fi
 
 
@@ -238,9 +238,9 @@ AC_DEFUN([OPAL_CHECK_OPENFABRICS],[
     fi
 
     AC_MSG_CHECKING([if ConnectIB XRC support is enabled])
-    AC_DEFINE_UNQUOTED([OPAL_HAVE_XRCD], [$$1_have_xrcd],
+    AC_DEFINE_UNQUOTED([OPAL_HAVE_CONNECTX_XRC_DOMAINS], [$$1_have_xrc_domains],
         [Enable features required for XRC domains support])
-    if test "1" = "$$1_have_xrcd"; then
+    if test "1" = "$$1_have_xrc_domains"; then
         AC_MSG_RESULT([yes])
     else
         AC_MSG_RESULT([no])

opal/mca/btl/openib/btl_openib.c

@@ -325,7 +325,7 @@ static int create_srq(mca_btl_openib_module_t *openib_btl)
             openib_btl->qps[qp].u.srq_qp.rd_posted = 0;
 #if HAVE_XRC
             if(BTL_OPENIB_QP_TYPE_XRC(qp)) {
-#if OPAL_HAVE_XRCD
+#if OPAL_HAVE_CONNECTX_XRC_DOMAINS
                 struct ibv_srq_init_attr_ex attr_ex;
                 memset(&attr_ex, 0, sizeof(struct ibv_srq_init_attr_ex));
                 attr_ex.attr.max_wr = attr.attr.max_wr;
@@ -1963,7 +1963,7 @@ int mca_btl_openib_put( mca_btl_base_module_t* btl,
     to_com_frag(frag)->endpoint = ep;
 #if HAVE_XRC
     if (MCA_BTL_XRC_ENABLED && BTL_OPENIB_QP_TYPE_XRC(qp))
-#if OPAL_HAVE_XRCD
+#if OPAL_HAVE_CONNECTX_XRC_DOMAINS
         frag->sr_desc.qp_type.xrc.remote_srqn=ep->rem_info.rem_srqs[qp].rem_srq_num;
 #else
         frag->sr_desc.xrc_remote_srq_num=ep->rem_info.rem_srqs[qp].rem_srq_num;
@@ -2056,7 +2056,7 @@ int mca_btl_openib_get(mca_btl_base_module_t* btl,
 
 #if HAVE_XRC
     if (MCA_BTL_XRC_ENABLED && BTL_OPENIB_QP_TYPE_XRC(qp))
-#if OPAL_HAVE_XRCD
+#if OPAL_HAVE_CONNECTX_XRC_DOMAINS
         frag->sr_desc.qp_type.xrc.remote_srqn=ep->rem_info.rem_srqs[qp].rem_srq_num;
 #else
         frag->sr_desc.xrc_remote_srq_num=ep->rem_info.rem_srqs[qp].rem_srq_num;

opal/mca/btl/openib/btl_openib.h

@@ -401,7 +401,7 @@ typedef struct mca_btl_openib_device_t {
     volatile bool got_port_event;
 #endif
 #if HAVE_XRC
-#if OPAL_HAVE_XRCD
+#if OPAL_HAVE_CONNECTX_XRC_DOMAINS
     struct ibv_xrcd *xrcd;
 #else
     struct ibv_xrc_domain *xrc_domain;

opal/mca/btl/openib/btl_openib_async.c

@@ -131,7 +131,7 @@ static mca_btl_openib_endpoint_t * xrc_qp2endpoint(uint32_t qp_num, mca_btl_open
     int  ep_i;
     for(ep_i = 0; ep_i < opal_pointer_array_get_size(device->endpoints); ep_i++) {
         ep = opal_pointer_array_get_item(device->endpoints, ep_i);
-#if OPAL_HAVE_XRCD
+#if OPAL_HAVE_CONNECTX_XRC_DOMAINS
         if (qp_num == ep->xrc_recv_qp->qp_num)
 #else
         if (qp_num == ep->xrc_recv_qp_num)
@@ -359,7 +359,7 @@ static int btl_openib_async_deviceh(struct mca_btl_openib_async_poll *devices_po
         event_type = event.event_type;
 #if HAVE_XRC
         /* is it XRC event ?*/
-#if OPAL_HAVE_XRCD
+#if OPAL_HAVE_CONNECTX_XRC_DOMAINS
 #else
         if (IBV_XRC_QP_EVENT_FLAG & event.event_type) {
             xrc_event = true;
@@ -377,7 +377,7 @@ static int btl_openib_async_deviceh(struct mca_btl_openib_async_poll *devices_po
                         mca_btl_openib_load_apm(event.element.qp,
                                 qp2endpoint(event.element.qp, device));
 #if HAVE_XRC
-#if OPAL_HAVE_XRCD
+#if OPAL_HAVE_CONNECTX_XRC_DOMAINS
 #else
                     else
                         mca_btl_openib_load_apm_xrc_rcv(event.element.xrc_qp_num,
@@ -661,7 +661,7 @@ void mca_btl_openib_load_apm(struct ibv_qp *qp, mca_btl_openib_endpoint_t *ep)
                    qp->qp_num, strerror(errno), errno));
 }
 
-#if HAVE_XRC && ! OPAL_HAVE_XRCD
+#if HAVE_XRC && ! OPAL_HAVE_CONNECTX_XRC_DOMAINS
 void mca_btl_openib_load_apm_xrc_rcv(uint32_t qp_num, mca_btl_openib_endpoint_t *ep)
 {
     struct ibv_qp_init_attr qp_init_attr;

opal/mca/btl/openib/btl_openib_async.h

@@ -19,7 +19,7 @@
 int        start_async_event_thread(void);
 void       mca_btl_openib_load_apm(struct ibv_qp *qp, mca_btl_openib_endpoint_t *ep);
 int        btl_openib_async_command_done(int exp);
-#if HAVE_XRC && ! OPAL_HAVE_XRCD
+#if HAVE_XRC && ! OPAL_HAVE_CONNECTX_XRC_DOMAINS
 void       mca_btl_openib_load_apm_xrc_rcv(uint32_t qp_num, mca_btl_openib_endpoint_t *ep);
 #endif
 

opal/mca/btl/openib/btl_openib_endpoint.c

@@ -352,7 +352,7 @@ static void mca_btl_openib_endpoint_construct(mca_btl_base_endpoint_t* endpoint)
     }
 
     endpoint->ib_addr = NULL;
-#if OPAL_HAVE_XRCD
+#if OPAL_HAVE_CONNECTX_XRC_DOMAINS
     endpoint->xrc_recv_qp = NULL;
 #else
     endpoint->xrc_recv_qp_num = 0;
@@ -467,7 +467,7 @@ static void mca_btl_openib_endpoint_destruct(mca_btl_base_endpoint_t* endpoint)
 
     /* unregister xrc recv qp */
 #if HAVE_XRC
-#if OPAL_HAVE_XRCD
+#if OPAL_HAVE_CONNECTX_XRC_DOMAINS
     if (NULL != endpoint->xrc_recv_qp) {
         if(ibv_destroy_qp(endpoint->xrc_recv_qp)) {
             BTL_ERROR(("Failed to unregister XRC recv QP:%d\n", endpoint->xrc_recv_qp->qp_num));

opal/mca/btl/openib/btl_openib_endpoint.h

@@ -211,7 +211,7 @@ struct mca_btl_base_endpoint_t {
     opal_list_t                 pending_lazy_frags;
 
     mca_btl_openib_endpoint_qp_t *qps;
-#if OPAL_HAVE_XRCD
+#if OPAL_HAVE_CONNECTX_XRC_DOMAINS
     struct ibv_qp *xrc_recv_qp;
 #else
     uint32_t xrc_recv_qp_num; /* in xrc we will use it as recv qp */
@@ -606,7 +606,7 @@ static inline int post_send(mca_btl_openib_endpoint_t *ep,
     }
 
 #if HAVE_XRC
-#if OPAL_HAVE_XRCD
+#if OPAL_HAVE_CONNECTX_XRC_DOMAINS
     if(BTL_OPENIB_QP_TYPE_XRC(qp))
         sr_desc->qp_type.xrc.remote_srqn = ep->rem_info.rem_srqs[qp].rem_srq_num;
 #else

opal/mca/btl/openib/btl_openib_xrc.c

@@ -2,11 +2,9 @@
  * Copyright (c) 2007-2008 Mellanox Technologies. All rights reserved.
  * Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2014      NVIDIA Corporation.  All rights reserved.
- * Copyright (c) 2014      Research Organization for Information Science
+ * Copyright (c) 2014-2015 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2014      Bull SAS.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -50,7 +48,7 @@ bool mca_btl_openib_xrc_check_api()
         return false;
     }
 
-#if OPAL_HAVE_XRCD
+#if OPAL_HAVE_CONNECTX_XRC_DOMAINS
     if (NULL != dlsym(lib, "ibv_open_xrcd")) {
         BTL_ERROR(("XRC error: bad XRC API (require XRC from OFED 3.12+)"));
         return false;
@@ -70,7 +68,7 @@ int mca_btl_openib_open_xrc_domain(struct mca_btl_openib_device_t *device)
     int len;
     char *xrc_file_name;
     const char *dev_name;
-#if OPAL_HAVE_XRCD
+#if OPAL_HAVE_CONNECTX_XRC_DOMAINS
     struct ibv_xrcd_init_attr xrcd_attr;
 #endif
 
@@ -91,7 +89,7 @@ int mca_btl_openib_open_xrc_domain(struct mca_btl_openib_device_t *device)
         free(xrc_file_name);
         return OPAL_ERROR;
     }
-#if OPAL_HAVE_XRCD
+#if OPAL_HAVE_CONNECTX_XRC_DOMAINS
     memset(&xrcd_attr, 0, sizeof xrcd_attr);
     xrcd_attr.comp_mask = IBV_XRCD_INIT_ATTR_FD | IBV_XRCD_INIT_ATTR_OFLAGS;
     xrcd_attr.fd = device->xrc_fd;
@@ -114,7 +112,7 @@ int mca_btl_openib_open_xrc_domain(struct mca_btl_openib_device_t *device)
 /* This func. closes XRC domain */
 int mca_btl_openib_close_xrc_domain(struct mca_btl_openib_device_t *device)
 {
-#if OPAL_HAVE_XRCD
+#if OPAL_HAVE_CONNECTX_XRC_DOMAINS
     if (NULL == device->xrcd) {
 #else
     if (NULL == device->xrc_domain) {
@@ -122,7 +120,7 @@ int mca_btl_openib_close_xrc_domain(struct mca_btl_openib_device_t *device)
         /* No XRC domain, just exit */
         return OPAL_SUCCESS;
     }
-#if OPAL_HAVE_XRCD
+#if OPAL_HAVE_CONNECTX_XRC_DOMAINS
     if (ibv_close_xrcd(device->xrcd)) {
 #else
     if (ibv_close_xrc_domain(device->xrc_domain)) {

opal/mca/btl/openib/connect/btl_openib_connect_udcm.c

@@ -5,12 +5,10 @@
  * Copyright (c) 2009      IBM Corporation.  All rights reserved.
  * Copyright (c) 2011-2014 Los Alamos National Security, LLC.  All rights
  *                         reserved.
- * Copyright (c) 2014      Research Organization for Information Science
+ * Copyright (c) 2014-2015 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2014      Intel, Inc. All rights reserved.
  * Copyright (c) 2014      Bull SAS.  All rights reserved.
- * Copyright (c) 2014-2015 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
  *
  * $COPYRIGHT$
  * 
@@ -332,7 +330,7 @@ static int udcm_xrc_start_connect (opal_btl_openib_connect_base_module_t *cpc,
 static int udcm_xrc_restart_connect (mca_btl_base_endpoint_t *lcl_ep);
 static int udcm_xrc_send_qp_connect (mca_btl_openib_endpoint_t *lcl_ep, udcm_msg_hdr_t *msg_hdr);
 static int udcm_xrc_send_qp_create (mca_btl_base_endpoint_t *lcl_ep);
-#if OPAL_HAVE_XRCD
+#if OPAL_HAVE_CONNECTX_XRC_DOMAINS
 static int udcm_xrc_recv_qp_connect (mca_btl_openib_endpoint_t *lcl_ep, uint32_t qp_num);
 #else
 static int udcm_xrc_recv_qp_connect (mca_btl_openib_endpoint_t *lcl_ep);
@@ -1970,7 +1968,7 @@ static int udcm_process_messages (struct ibv_cq *event_cq, udcm_module_t *m)
 
             if (UDCM_MESSAGE_XCONNECT2 == message->hdr.type) {
                 /* save the qp number for unregister */
-#if ! OPAL_HAVE_XRCD
+#if ! OPAL_HAVE_CONNECTX_XRC_DOMAINS
                 lcl_ep->xrc_recv_qp_num = message->hdr.data.xreq.rem_qp_num;
 #endif
 
@@ -2413,7 +2411,7 @@ static int udcm_xrc_send_qp_create (mca_btl_base_endpoint_t *lcl_ep)
     uint32_t send_wr;
     struct ibv_qp **qp;
     uint32_t *psn;
-#if OPAL_HAVE_XRCD
+#if OPAL_HAVE_CONNECTX_XRC_DOMAINS
     struct ibv_qp_init_attr_ex qp_init_attr;
 #else
     struct ibv_qp_init_attr qp_init_attr;
@@ -2434,7 +2432,7 @@ static int udcm_xrc_send_qp_create (mca_btl_base_endpoint_t *lcl_ep)
     send_wr = lcl_ep->ib_addr->qp->sd_wqe +
         (mca_btl_openib_component.use_eager_rdma ?
          mca_btl_openib_component.max_eager_rdma : 0);
-#if OPAL_HAVE_XRCD
+#if OPAL_HAVE_CONNECTX_XRC_DOMAINS
     memset(&qp_init_attr, 0, sizeof(struct ibv_qp_init_attr_ex));
 #else
     memset(&qp_init_attr, 0, sizeof(struct ibv_qp_init_attr));
@@ -2451,7 +2449,7 @@ static int udcm_xrc_send_qp_create (mca_btl_base_endpoint_t *lcl_ep)
     qp_init_attr.cap.max_send_sge = 1;
     /* this one is ignored by driver */
     qp_init_attr.cap.max_recv_sge = 1; /* we do not use SG list */
-#if OPAL_HAVE_XRCD
+#if OPAL_HAVE_CONNECTX_XRC_DOMAINS
     qp_init_attr.qp_type = IBV_QPT_XRC_SEND;
     qp_init_attr.comp_mask = IBV_QP_INIT_ATTR_PD;
     qp_init_attr.pd = openib_btl->device->ib_pd;
@@ -2507,7 +2505,7 @@ static int udcm_xrc_send_qp_create (mca_btl_base_endpoint_t *lcl_ep)
 /* mark: xrc receive qp */
 
 /* Recv qp connect */
-#if OPAL_HAVE_XRCD
+#if OPAL_HAVE_CONNECTX_XRC_DOMAINS
 static int udcm_xrc_recv_qp_connect (mca_btl_openib_endpoint_t *lcl_ep, uint32_t qp_num)
 #else
 static int udcm_xrc_recv_qp_connect (mca_btl_openib_endpoint_t *lcl_ep)
@@ -2515,7 +2513,7 @@ static int udcm_xrc_recv_qp_connect (mca_btl_openib_endpoint_t *lcl_ep)
 {
     mca_btl_openib_module_t *openib_btl = lcl_ep->endpoint_btl;
 
-#if OPAL_HAVE_XRCD
+#if OPAL_HAVE_CONNECTX_XRC_DOMAINS
     struct ibv_qp_open_attr attr;
     memset(&attr, 0, sizeof(struct ibv_qp_open_attr));
     attr.comp_mask = IBV_QP_OPEN_ATTR_NUM | IBV_QP_OPEN_ATTR_XRCD | IBV_QP_OPEN_ATTR_TYPE;
@@ -2553,7 +2551,7 @@ static int udcm_xrc_recv_qp_connect (mca_btl_openib_endpoint_t *lcl_ep)
 static int udcm_xrc_recv_qp_create (mca_btl_openib_endpoint_t *lcl_ep, udcm_msg_hdr_t *msg_hdr)
 {
     mca_btl_openib_module_t* openib_btl = lcl_ep->endpoint_btl;
-#if OPAL_HAVE_XRCD
+#if OPAL_HAVE_CONNECTX_XRC_DOMAINS
     struct ibv_qp_init_attr_ex qp_init_attr;
 #else
     struct ibv_qp_init_attr qp_init_attr;
@@ -2563,7 +2561,7 @@ static int udcm_xrc_recv_qp_create (mca_btl_openib_endpoint_t *lcl_ep, udcm_msg_
 
     BTL_VERBOSE(("creating xrc receive qp"));
 
-#if OPAL_HAVE_XRCD
+#if OPAL_HAVE_CONNECTX_XRC_DOMAINS
     memset(&qp_init_attr, 0, sizeof(struct ibv_qp_init_attr_ex));
     qp_init_attr.qp_type = IBV_QPT_XRC_RECV;
     qp_init_attr.comp_mask = IBV_QP_INIT_ATTR_XRCD;
@@ -2592,7 +2590,7 @@ static int udcm_xrc_recv_qp_create (mca_btl_openib_endpoint_t *lcl_ep, udcm_msg_
     attr.pkey_index = openib_btl->pkey_index;
     attr.port_num = openib_btl->port_num;
     attr.qp_access_flags = IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ;
-#if OPAL_HAVE_XRCD
+#if OPAL_HAVE_CONNECTX_XRC_DOMAINS
     ret = ibv_modify_qp(lcl_ep->xrc_recv_qp,
             &attr,
             IBV_QP_STATE|
@@ -2636,7 +2634,7 @@ static int udcm_xrc_recv_qp_create (mca_btl_openib_endpoint_t *lcl_ep, udcm_msg_
     /* if user enabled dynamic SL, get it from PathRecord */
     if (0 != mca_btl_openib_component.ib_path_record_service_level) {
         int rc = btl_openib_connect_get_pathrecord_sl(
-#if OPAL_HAVE_XRCD
+#if OPAL_HAVE_CONNECTX_XRC_DOMAINS
                                                       openib_btl->device->xrcd->context,
 #else
                                                       openib_btl->device->xrc_domain->context,
@@ -2651,7 +2649,7 @@ static int udcm_xrc_recv_qp_create (mca_btl_openib_endpoint_t *lcl_ep, udcm_msg_
     }
 #endif
 
-#if OPAL_HAVE_XRCD
+#if OPAL_HAVE_CONNECTX_XRC_DOMAINS
     ret = ibv_modify_qp(lcl_ep->xrc_recv_qp,
             &attr,
             IBV_QP_STATE|
@@ -2685,7 +2683,7 @@ static int udcm_xrc_recv_qp_create (mca_btl_openib_endpoint_t *lcl_ep, udcm_msg_
 #endif
 #if OPAL_HAVE_THREADS
     if (APM_ENABLED) {
-#if OPAL_HAVE_XRCD
+#if OPAL_HAVE_CONNECTX_XRC_DOMAINS
         mca_btl_openib_load_apm(lcl_ep->xrc_recv_qp, lcl_ep);
 #else
         mca_btl_openib_load_apm_xrc_rcv(lcl_ep->xrc_recv_qp_num, lcl_ep);
@@ -2754,7 +2752,7 @@ static int udcm_xrc_send_xresponse (mca_btl_base_endpoint_t *lcl_ep, mca_btl_bas
     msg->data->hdr.data.xres.rem_ep_index = htonl(lcl_ep->index);
 
     if (UDCM_MESSAGE_XRESPONSE == msg_type) {
-#if OPAL_HAVE_XRCD
+#if OPAL_HAVE_CONNECTX_XRC_DOMAINS
         BTL_VERBOSE(("Sending qp: %d, psn: %d", lcl_ep->xrc_recv_qp->qp_num, lcl_ep->xrc_recv_psn));
         msg->data->hdr.data.xres.rem_qp_num = htonl(lcl_ep->xrc_recv_qp->qp_num);
         msg->data->hdr.data.xres.rem_psn    = htonl(lcl_ep->xrc_recv_psn);
@@ -2766,7 +2764,7 @@ static int udcm_xrc_send_xresponse (mca_btl_base_endpoint_t *lcl_ep, mca_btl_bas
     }
 
     for (int i = 0; i < mca_btl_openib_component.num_xrc_qps; ++i) {
-#if OPAL_HAVE_XRCD
+#if OPAL_HAVE_CONNECTX_XRC_DOMAINS
         uint32_t srq_num;
         if (ibv_get_srq_num(lcl_ep->endpoint_btl->qps[i].u.srq_qp.srq, &srq_num)) {
             BTL_ERROR(("BTL openib XOOB internal error: can't get srq num"));
@@ -2816,7 +2814,7 @@ static int udcm_xrc_handle_xconnect (mca_btl_openib_endpoint_t *lcl_ep, udcm_msg
 
         if (UDCM_MESSAGE_XCONNECT2 == msg_hdr->type) {
             response_type = UDCM_MESSAGE_XRESPONSE2;
-#if OPAL_HAVE_XRCD
+#if OPAL_HAVE_CONNECTX_XRC_DOMAINS
             rc = udcm_xrc_recv_qp_connect (lcl_ep, msg_hdr->data.xreq.rem_qp_num);
 #else
             rc = udcm_xrc_recv_qp_connect (lcl_ep);