XRC - replacing the new old API with new one.

This commit was SVN r17369.

config/ompi_check_openib.m4

@@ -108,8 +108,8 @@ AC_DEFUN([OMPI_CHECK_OPENIB],[
            AC_CHECK_MEMBERS([struct ibv_device.transport_type], [], [],
                             [#include <infiniband/verbs.h>])
 
-           # ibv_open_xrc_domain was added in OFED 1.3
-           AC_CHECK_FUNCS([ibv_open_xrc_domain], [$1_have_xrc=1])
+           # ibv_create_xrc_rcv_qp was added in OFED 1.3
+           AC_CHECK_FUNCS([ibv_create_xrc_rcv_qp], [$1_have_xrc=1])
           ])
 
     CPPFLAGS="$ompi_check_openib_$1_save_CPPFLAGS"

ompi/mca/btl/openib/btl_openib.h

@@ -50,7 +50,7 @@
 
 BEGIN_C_DECLS
 
-#define HAVE_XRC (defined(HAVE_IBV_OPEN_XRC_DOMAIN) && (1 == OMPI_ENABLE_CONNECTX_XRC_SUPPORT))
+#define HAVE_XRC (defined(HAVE_IBV_CREATE_XRC_RCV_QP) && (1 == OMPI_ENABLE_CONNECTX_XRC_SUPPORT))
 
 #define MCA_BTL_IB_LEAVE_PINNED 1
 #define IB_DEFAULT_GID_PREFIX 0xfe80000000000000ll

ompi/mca/btl/openib/btl_openib_async.c

@@ -186,6 +186,8 @@ static int btl_openib_async_hcah(struct mca_btl_openib_async_poll *hcas_poll, in
     int j;
     mca_btl_openib_hca_t *hca = NULL;
     struct ibv_async_event event;
+    bool xrc_event = false;
+    int event_type;
 
     /* We need to find correct hca and process this event */
     for (j=0; j < mca_btl_openib_component.ib_num_btls; j++) {
@@ -205,7 +207,17 @@ static int btl_openib_async_hcah(struct mca_btl_openib_async_poll *hcas_poll, in
                 return OMPI_ERROR;
             }
         }
-        switch(event.event_type) {
+
+        event_type = event.event_type;
+#if HAVE_XRC
+        /* is it XRC event ?*/
+        if (IBV_XRC_QP_EVENT_FLAG & event.event_type) {
+            xrc_event = true;
+            /* Clean the bitnd handel as usual */
+            event_type ^= IBV_XRC_QP_EVENT_FLAG;
+        }
+#endif
+        switch(event_type) {
             case IBV_EVENT_PATH_MIG:
                 if (0 != mca_btl_openib_component.apm) {
                     BTL_ERROR(("APM: Alternative path migration reported."));
@@ -227,7 +239,8 @@ static int btl_openib_async_hcah(struct mca_btl_openib_async_poll *hcas_poll, in
             case IBV_EVENT_PORT_ERR:
                 opal_show_help("help-mpi-btl-openib.txt", "of error event",
                     true,orte_system_info.nodename, orte_process_info.pid,
-                    event.event_type, openib_event_to_str(event.event_type));
+                    event.event_type, openib_event_to_str(event.event_type),
+                    xrc_event ? "true" : "false");
                 break;
             case IBV_EVENT_COMM_EST:
             case IBV_EVENT_PORT_ACTIVE:
@@ -244,7 +257,7 @@ static int btl_openib_async_hcah(struct mca_btl_openib_async_poll *hcas_poll, in
             default:
                 opal_show_help("help-mpi-btl-openib.txt", "of unknown event",
                         true,orte_system_info.nodename, orte_process_info.pid,
-                        event.event_type);
+                        event.event_type, xrc_event ? "true" : "false");
         }
         ibv_ack_async_event(&event);
     } else {

ompi/mca/btl/openib/btl_openib_endpoint.c

@@ -396,7 +396,7 @@ static void mca_btl_openib_endpoint_construct(mca_btl_base_endpoint_t* endpoint)
     }
 
     endpoint->ib_addr = NULL;
-    endpoint->xrc_recv_qp = NULL;
+    endpoint->xrc_recv_qp_num = 0;
     endpoint->endpoint_btl = 0;
     endpoint->endpoint_proc = 0;
     endpoint->endpoint_tstamp = 0.0;
@@ -483,12 +483,15 @@ static void mca_btl_openib_endpoint_destruct(mca_btl_base_endpoint_t* endpoint)
     /* free the qps */
     free(endpoint->qps);
 
-    /* destroy recv qp */
-    if (NULL != endpoint->xrc_recv_qp) {
-        if(ibv_destroy_qp(endpoint->xrc_recv_qp)) {
-            BTL_ERROR(("Failed to destroy XRC recv QP:%d\n", qp));
+    /* unregister xrc recv qp */
+#if HAVE_XRC
+    if (0 != endpoint->xrc_recv_qp_num) {
+        if(ibv_unreg_xrc_rcv_qp(endpoint->endpoint_btl->hca->xrc_domain,
+                    endpoint->xrc_recv_qp_num)) {
+            BTL_ERROR(("Failed to unregister XRC recv QP:%d\n", endpoint->xrc_recv_qp_num));
         }
     }
+#endif
 
     OBJ_DESTRUCT(&endpoint->endpoint_lock);
     /* Clean pending lists */

ompi/mca/btl/openib/btl_openib_endpoint.h

@@ -176,7 +176,7 @@ struct mca_btl_base_endpoint_t {
      */
 
     mca_btl_openib_endpoint_qp_t *qps;
-    struct ibv_qp *xrc_recv_qp; /* in xrc we will use it as recv qp */
+    uint32_t xrc_recv_qp_num; /* in xrc we will use it as recv qp */
     uint32_t xrc_recv_psn;
 
 

ompi/mca/btl/openib/btl_openib_xrc.h

@@ -35,7 +35,8 @@ struct ib_address_t {
     mca_btl_openib_qp_t *qp;               /* pointer to qp that will be used
                                               for communication with the
                                               destination */
-    opal_mutex_t addr_lock;             /* protection */
+    uint32_t remote_xrc_rcv_qp_num;        /* remote xrc qp number */
+    opal_mutex_t addr_lock;                /* protection */
     mca_btl_openib_ib_addr_state_t status; /* ib port status */
 };
 typedef struct ib_address_t ib_address_t;