Adding support for APM over different ports

This commit was SVN r17521.

ompi/mca/btl/openib/btl_openib.h

@@ -169,7 +169,8 @@ struct mca_btl_openib_component_t {
     int32_t max_eager_rdma;
     uint32_t btls_per_lid;
     uint32_t max_lmc;
-    int32_t apm;
+    int32_t apm_lmc;
+    int32_t apm_ports;
     uint32_t buffer_alignment;    /**< Preferred communication buffer alignment in Bytes (must be power of two) */
 #if OMPI_HAVE_THREADS
     int32_t fatal_counter;           /**< Counts number on fatal events that we got on all hcas */
@@ -229,9 +230,8 @@ struct mca_btl_openib_port_info {
     uint8_t padding[4];
 #endif
     uint64_t subnet_id;
-#if HAVE_XRC
     uint16_t lid; /* used only in xrc */
-#endif
+    uint16_t apm_lid; /* the lid is used for APM to different port */
     char *cpclist;
 };
 typedef struct mca_btl_openib_port_info mca_btl_openib_port_info_t;
@@ -285,6 +285,7 @@ typedef struct mca_btl_openib_hca_t {
     uint8_t use_eager_rdma;
     uint8_t btls;              /** < number of btls using this HCA */
     opal_pointer_array_t *endpoints;
+    opal_pointer_array_t *hca_btls;
     uint16_t hp_cq_polls;
     uint16_t eager_rdma_polls;
     bool pollme;
@@ -335,7 +336,7 @@ struct mca_btl_openib_module_t {
     uint16_t pkey_index;
     struct ibv_port_attr ib_port_attr;
     uint16_t lid;                      /**< lid that is actually used (for LMC) */
-    uint16_t apm_lmc_max;                  /**< the maximal lmc that can be used for apm */
+    int apm_port;                      /**< Alternative port that may be used for APM */
     uint8_t src_path_bits;             /**< offset from base lid (for LMC) */
 
     int32_t num_peers;

ompi/mca/btl/openib/btl_openib_async.c

@@ -27,6 +27,8 @@
 #include "btl_openib.h"
 #include "btl_openib_mca.h"
 #include "btl_openib_async.h"
+#include "btl_openib_proc.h"
+#include "btl_openib_endpoint.h"
 
 struct mca_btl_openib_async_poll {
     int active_poll_size;
@@ -90,6 +92,33 @@ static const char *openib_event_to_str (enum ibv_event_type event)
         return "UNKNOWN";
     }
 }
+/* QP to endpoint */
+static mca_btl_openib_endpoint_t * qp2endpoint(struct ibv_qp *qp, mca_btl_openib_hca_t *hca)
+{
+    mca_btl_openib_endpoint_t *ep;
+    int  ep_i, qp_i;
+    for(ep_i = 0; ep_i < opal_pointer_array_get_size(hca->endpoints); ep_i++) {
+        ep = opal_pointer_array_get_item(hca->endpoints, ep_i);
+        for(qp_i = 0; qp_i < mca_btl_openib_component.num_qps; qp_i++) {
+            if (qp == ep->qps[qp_i].qp->lcl_qp)
+                return ep;
+        }
+    }
+    return NULL;
+}
+
+/* XRC recive QP to endpoint */
+static mca_btl_openib_endpoint_t * xrc_qp2endpoint(uint32_t qp_num, mca_btl_openib_hca_t *hca)
+{
+    mca_btl_openib_endpoint_t *ep;
+    int  ep_i;
+    for(ep_i = 0; ep_i < opal_pointer_array_get_size(hca->endpoints); ep_i++) {
+        ep = opal_pointer_array_get_item(hca->endpoints, ep_i);
+        if (qp_num == ep->xrc_recv_qp_num)
+            return ep;
+    }
+    return NULL;
+}
 
 /* Function inits mca_btl_openib_async_poll */
 static int btl_openib_async_poll_init(struct mca_btl_openib_async_poll *hcas_poll)
@@ -194,6 +223,7 @@ static int btl_openib_async_hcah(struct mca_btl_openib_async_poll *hcas_poll, in
         if (mca_btl_openib_component.openib_btls[j]->hca->ib_dev_context->async_fd ==
                 hcas_poll->async_pollfd[index].fd ) {
             hca = mca_btl_openib_component.openib_btls[j]->hca;
+            break;
         }
     }
     if (NULL != hca) {
@@ -219,15 +249,16 @@ static int btl_openib_async_hcah(struct mca_btl_openib_async_poll *hcas_poll, in
 #endif
         switch(event_type) {
             case IBV_EVENT_PATH_MIG:
-                if (0 != mca_btl_openib_component.apm) {
+                BTL_ERROR(("Alternative path migration event reported"));
-                    BTL_ERROR(("APM: Alternative path migration reported."));
+                if (APM_ENABLED) {
+                    BTL_ERROR(("Trying to find additional path..."));
                     if (!xrc_event) 
                         mca_btl_openib_load_apm(event.element.qp,
-                                mca_btl_openib_component.openib_btls[j]);
+                                qp2endpoint(event.element.qp, hca));
 #if HAVE_XRC
                     else
                         mca_btl_openib_load_apm_xrc_rcv(event.element.xrc_qp_num,
-                                mca_btl_openib_component.openib_btls[j]);
+                                xrc_qp2endpoint(event.element.xrc_qp_num, hca));
 #endif
                 }
                 break;
@@ -345,55 +376,126 @@ static void apm_update_attr(struct ibv_qp_attr *attr, enum ibv_qp_attr_mask *mas
 {
     *mask = IBV_QP_ALT_PATH|IBV_QP_PATH_MIG_STATE;
     attr->alt_ah_attr.dlid = attr->ah_attr.dlid + 1;
+    attr->alt_ah_attr.src_path_bits = attr->ah_attr.src_path_bits + 1;
     attr->alt_ah_attr.static_rate = attr->ah_attr.static_rate;
     attr->alt_ah_attr.sl = attr->ah_attr.sl;
     attr->alt_pkey_index = attr->pkey_index;
     attr->alt_port_num = attr->port_num;
+    attr->alt_timeout = attr->timeout;
+    attr->path_mig_state = IBV_MIG_REARM;
+    BTL_VERBOSE(("New APM LMC loaded: alt_src_port:%d, dlid: %d, src_bits %d, old_src_bits: %d, old_dlid %d",
+                attr->alt_port_num, attr->alt_ah_attr.dlid,
+                attr->alt_ah_attr.src_path_bits, attr->ah_attr.src_path_bits, attr->ah_attr.dlid));
+}
+
+static int apm_update_port(mca_btl_openib_endpoint_t *ep,
+        struct ibv_qp_attr *attr, enum ibv_qp_attr_mask *mask)
+{
+    size_t port_i;
+    uint16_t apm_lid = 0;
+
+    if (attr->port_num == ep->endpoint_btl->apm_port) {
+        /* all ports were used */
+        BTL_ERROR(("APM: already all ports were used port_num %d apm_port %d",
+                    attr->port_num, ep->endpoint_btl->apm_port));
+        return OMPI_ERROR;
+    }
+    /* looking for alternatve lid on remote site */
+    for(port_i = 0; port_i < ep->endpoint_proc->proc_port_count; port_i++) {
+        if (ep->endpoint_proc->proc_ports[port_i].lid == attr->ah_attr.dlid - mca_btl_openib_component.apm_lmc) {
+            apm_lid = ep->endpoint_proc->proc_ports[port_i].apm_lid;
+        }
+    }
+    if (0 == apm_lid) {
+        /* APM was disabled on one of site ? */
+        BTL_VERBOSE(("APM: Was disabled ? dlid %d %d %d", attr->ah_attr.dlid, attr->ah_attr.src_path_bits, ep->endpoint_btl->src_path_bits));
+        return OMPI_ERROR;
+    }
+    /* We guess cthat the LMC is the same on all ports */
+    attr->alt_ah_attr.static_rate = attr->ah_attr.static_rate;
+    attr->alt_ah_attr.sl = attr->ah_attr.sl;
+    attr->alt_pkey_index = attr->pkey_index;
+    attr->alt_timeout = attr->timeout;
+    attr->path_mig_state = IBV_MIG_REARM;
+    *mask = IBV_QP_ALT_PATH|IBV_QP_PATH_MIG_STATE;
+
+    attr->alt_port_num = ep->endpoint_btl->apm_port;
+    attr->alt_ah_attr.src_path_bits = ep->endpoint_btl->src_path_bits;
+    attr->alt_ah_attr.dlid = apm_lid;
+
+    BTL_VERBOSE(("New APM port loaded: alt_src_port:%d, dlid: %d, src_bits: %d:%d, old_dlid %d",
+                attr->alt_port_num, attr->alt_ah_attr.dlid,
+                attr->ah_attr.src_path_bits, attr->alt_ah_attr.src_path_bits,
+                attr->ah_attr.dlid));
+    return OMPI_SUCCESS;
 }
 
 /* Load new dlid to the QP */
-void mca_btl_openib_load_apm(struct ibv_qp *qp, struct mca_btl_openib_module_t *btl)
+void mca_btl_openib_load_apm(struct ibv_qp *qp, mca_btl_openib_endpoint_t *ep)
 {
     struct ibv_qp_init_attr qp_init_attr;
     struct ibv_qp_attr attr;
     enum ibv_qp_attr_mask mask = 0;
+    struct mca_btl_openib_module_t *btl;
 
     BTL_VERBOSE(("APM: Loading alternative path"));
+    assert (NULL != ep);
+    btl = ep->endpoint_btl;
 
     if (ibv_query_qp(qp, &attr, mask, &qp_init_attr))
         BTL_ERROR(("Failed to ibv_query_qp, qp num: %d", qp->qp_num));
 
-    if (attr.ah_attr.src_path_bits - btl->src_path_bits < btl->apm_lmc_max) {
+    if (mca_btl_openib_component.apm_lmc &&
-        attr.alt_ah_attr.src_path_bits = attr.ah_attr.src_path_bits + 1;
+            attr.ah_attr.src_path_bits - btl->src_path_bits < mca_btl_openib_component.apm_lmc) {
+        BTL_VERBOSE(("APM LMC: src: %d btl_src: %d lmc_max: %d",
+                    attr.ah_attr.src_path_bits,
+                    btl->src_path_bits,
+                    mca_btl_openib_component.apm_lmc));
+        apm_update_attr(&attr, &mask);
     } else {
-        BTL_ERROR(("Failed to load alternative path, all %d were used",
+        if (mca_btl_openib_component.apm_ports) {
-                    attr.ah_attr.src_path_bits - btl->src_path_bits));
+            /* Try to migrate to next port */
+            if (OMPI_SUCCESS != apm_update_port(ep, &attr, &mask))
+                return;
+        } else {
+            BTL_ERROR(("Failed to load alternative path, all %d were used",
+                        attr.ah_attr.src_path_bits - btl->src_path_bits));
+        }
     }
-    apm_update_attr(&attr, &mask);
 
     if (ibv_modify_qp(qp, &attr, mask))
-        BTL_ERROR(("Failed to ibv_query_qp, qp num: %d", qp->qp_num));
+        BTL_ERROR(("Failed to ibv_query_qp, qp num: %p, errno says: %s (%d)"
+                    ,qp->qp_num ,strerror(errno), errno));
 }
 
-void mca_btl_openib_load_apm_xrc_rcv(uint32_t qp_num, struct mca_btl_openib_module_t *btl)
+void mca_btl_openib_load_apm_xrc_rcv(uint32_t qp_num, mca_btl_openib_endpoint_t *ep)
 {
 #if HAVE_XRC
     struct ibv_qp_init_attr qp_init_attr;
     struct ibv_qp_attr attr;
     enum ibv_qp_attr_mask mask = 0;
+    struct mca_btl_openib_module_t *btl;
 
     BTL_VERBOSE(("APM XRC: Loading alternative path"));
+    assert (NULL != ep);
+    btl = ep->endpoint_btl;
 
     if (ibv_query_xrc_rcv_qp(btl->hca->xrc_domain, qp_num, &attr, mask, &qp_init_attr))
         BTL_ERROR(("Failed to ibv_query_qp, qp num: %d", qp_num));
 
-    if (attr.ah_attr.src_path_bits - btl->src_path_bits < btl->apm_lmc_max) {
+    if (mca_btl_openib_component.apm_lmc &&
-        attr.alt_ah_attr.src_path_bits = attr.ah_attr.src_path_bits + 1;
+            attr.ah_attr.src_path_bits - btl->src_path_bits < mca_btl_openib_component.apm_lmc) {
+        apm_update_attr(&attr, &mask);
     } else {
-        BTL_ERROR(("Failed to load alternative path, all %d were used",
+        if (mca_btl_openib_component.apm_ports) {
-                    attr.ah_attr.src_path_bits - btl->src_path_bits));
+            /* Try to migrate to next port */
+            if (OMPI_SUCCESS != apm_update_port(ep, &attr, &mask))
+                return;
+        } else {
+            BTL_ERROR(("Failed to load alternative path, all %d were used",
+                        attr.ah_attr.src_path_bits - btl->src_path_bits));
+        }
     }
-    apm_update_attr(&attr, &mask);
 
     ibv_modify_xrc_rcv_qp(btl->hca->xrc_domain, qp_num, &attr, mask);
     /* Maybe the qp already was modified by other process - ignoring error */

ompi/mca/btl/openib/btl_openib_async.h

@@ -11,8 +11,12 @@
 
 #ifndef MCA_BTL_OPENIB_ASYNC_H
 #define MCA_BTL_OPENIB_ASYNC_H
+#include "btl_openib_endpoint.h"
 
 void*      btl_openib_async_thread(void *one_hca);
-void       mca_btl_openib_load_apm(struct ibv_qp *qp, struct mca_btl_openib_module_t *btl);
+void       mca_btl_openib_load_apm(struct ibv_qp *qp, mca_btl_openib_endpoint_t *ep);
-void       mca_btl_openib_load_apm_xrc_rcv(uint32_t qp_num, struct mca_btl_openib_module_t *btl);
+void       mca_btl_openib_load_apm_xrc_rcv(uint32_t qp_num, mca_btl_openib_endpoint_t *ep);
+
+#define APM_ENABLED (0 != mca_btl_openib_component.apm_lmc || 0 != mca_btl_openib_component.apm_ports)
+
 #endif

ompi/mca/btl/openib/btl_openib_component.c

@@ -402,19 +402,20 @@ static int init_one_port(opal_list_t *btl_list, mca_btl_openib_hca_t *hca,
 #if OMPI_HAVE_THREADS
     /* APM support */
     if (lmc > 1){
-        if (-1 == mca_btl_openib_component.apm) {
+        if (-1 == mca_btl_openib_component.apm_lmc) {
             lmc_step = lmc;
-        } else if (0 == lmc % (mca_btl_openib_component.apm + 1)) {
+            mca_btl_openib_component.apm_lmc = lmc - 1;
-            lmc_step = mca_btl_openib_component.apm + 1;
+        } else if (0 == lmc % (mca_btl_openib_component.apm_lmc + 1)) {
+            lmc_step = mca_btl_openib_component.apm_lmc + 1;
         } else {
             opal_show_help("help-mpi-btl-openib.txt", "apm with wrong lmc",true,
-                    mca_btl_openib_component.apm, lmc);
+                    mca_btl_openib_component.apm_lmc, lmc);
             return OMPI_ERROR;
         }
     } else {
-        if (mca_btl_openib_component.apm) {
+        if (mca_btl_openib_component.apm_lmc) {
             /* Disable apm and report warning */
-            mca_btl_openib_component.apm = 0;
+            mca_btl_openib_component.apm_lmc = 0;
             opal_show_help("help-mpi-btl-openib.txt", "apm without lmc",true);
         }
     }
@@ -441,18 +442,14 @@ static int init_one_port(opal_list_t *btl_list, mca_btl_openib_hca_t *hca,
             openib_btl->port_num = (uint8_t) port_num;
             openib_btl->pkey_index = pkey_index;
             openib_btl->lid = lid;
-            openib_btl->apm_lmc_max = lmc_step ;
+            openib_btl->apm_port = 0;
             openib_btl->src_path_bits = lid - ib_port_attr->lid;
             /* store the subnet for multi-nic support */
             openib_btl->port_info.subnet_id = subnet_id;
             openib_btl->port_info.mtu = hca->mtu;
-#if HAVE_XRC
             /* This code is protected with ifdef because we don't want to send
              * extra bytes during OOB */
-            if(MCA_BTL_XRC_ENABLED) {
+            openib_btl->port_info.lid = lid;
-                openib_btl->port_info.lid = lid;
-            }
-#endif
             rc = ompi_btl_openib_connect_base_query(&openib_btl->port_info.cpclist, hca);
             if (OMPI_SUCCESS != rc) {
                 continue;
@@ -549,6 +546,7 @@ static int init_one_port(opal_list_t *btl_list, mca_btl_openib_hca_t *hca,
                 }
             }
             opal_list_append(btl_list, (opal_list_item_t*) ib_selected);
+            opal_pointer_array_add(hca->hca_btls, (void*) openib_btl);
             hca->btls++;
             ++mca_btl_openib_component.ib_num_btls;
             if (-1 != mca_btl_openib_component.ib_max_btls &&
@@ -868,6 +866,22 @@ static bool inline is_credit_message(const mca_btl_openib_recv_frag_t *frag)
         (MCA_BTL_OPENIB_CONTROL_CREDITS == chdr->type);
 }
 
+static void init_apm_port(mca_btl_openib_hca_t *hca, int port, uint16_t lid)
+{
+    int index;
+    struct mca_btl_openib_module_t *btl;
+    for(index = 0; index < hca->btls; index++) {
+        btl = opal_pointer_array_get_item(hca->hca_btls, index);
+        /* Ok, we already have btl for the fist port,
+         * second one will be used for APM */
+        btl->apm_port = port;
+        btl->port_info.apm_lid = lid + btl->src_path_bits;
+        mca_btl_openib_component.apm_ports++;
+        BTL_VERBOSE(("APM-PORT: Setting alternative port - %d, lid - %d"
+                    ,port ,lid));
+    }
+}
+
 static int init_one_hca(opal_list_t *btl_list, struct ibv_device* ib_dev)
 {
     struct mca_mpool_base_resources_t mpool_resources;
@@ -885,6 +899,11 @@ static int init_one_hca(opal_list_t *btl_list, struct ibv_device* ib_dev)
 
     hca->ib_dev = ib_dev;
     hca->ib_dev_context = ibv_open_device(ib_dev);
+    hca->hca_btls = OBJ_NEW(opal_pointer_array_t);
+    if (OPAL_SUCCESS != opal_pointer_array_init(hca->hca_btls, 2, INT_MAX, 2)) {
+        BTL_ERROR(("Failed to initialize hca_btls array: %s:%d\n", __FILE__, __LINE__));
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
 
     if(NULL == hca->ib_dev_context){
         BTL_ERROR(("error obtaining device context for %s errno says %s\n",
@@ -1033,13 +1052,15 @@ static int init_one_hca(opal_list_t *btl_list, struct ibv_device* ib_dev)
                         ibv_get_device_name(ib_dev), i, strerror(errno)));
             break;
         }
-        if(IBV_PORT_ACTIVE == ib_port_attr.state){
+        if(IBV_PORT_ACTIVE == ib_port_attr.state) {
-
+            if (mca_btl_openib_component.apm_ports && hca->btls > 0) {
+                init_apm_port(hca, i, ib_port_attr.lid);
+                break;
+            }
             if (0 == mca_btl_openib_component.ib_pkey_val) {
                 ret = init_one_port(btl_list, hca, i, mca_btl_openib_component.ib_pkey_ix,
                                     &ib_port_attr);
-            }
+            } else {
-            else {
                 uint16_t pkey,j;
                 for (j=0; j < hca->ib_dev_attr.max_pkeys; j++) {
                     ibv_query_pkey(hca->ib_dev_context, i, j, &pkey);
@@ -1063,6 +1084,11 @@ static int init_one_hca(opal_list_t *btl_list, struct ibv_device* ib_dev)
     /* If we made a BTL, we're done.  Otherwise, fall through and
        destroy everything */
     if (hca->btls > 0) {
+        /* if apm was enabled it should be > 1 */
+        if (1 == mca_btl_openib_component.apm_ports) {
+            opal_show_help("help-mpi-btl-openib.txt", "apm not enough ports", true);
+            mca_btl_openib_component.apm_ports = 0;
+        }
         ret = prepare_hca_for_use(hca);
         if(OMPI_SUCCESS == ret)
             return OMPI_SUCCESS;

ompi/mca/btl/openib/btl_openib_endpoint.c

@@ -551,15 +551,15 @@ void mca_btl_openib_endpoint_connected(mca_btl_openib_endpoint_t *endpoint)
 
     /* Run over all qps and load alternative path */
 #if OMPI_HAVE_THREADS
-    if (0 != mca_btl_openib_component.apm) {
+    if (APM_ENABLED) {
         int i;
         if (MCA_BTL_XRC_ENABLED) {
             if (master) {
-                mca_btl_openib_load_apm(endpoint->ib_addr->qp->lcl_qp, endpoint->endpoint_btl);
+                mca_btl_openib_load_apm(endpoint->ib_addr->qp->lcl_qp, endpoint);
             }
         } else {
             for(i = 0; i < mca_btl_openib_component.num_qps; i++) {
-                mca_btl_openib_load_apm(endpoint->qps[i].qp->lcl_qp, endpoint->endpoint_btl);
+                mca_btl_openib_load_apm(endpoint->qps[i].qp->lcl_qp, endpoint);
             }
         }
     }

ompi/mca/btl/openib/btl_openib_mca.c

@@ -403,10 +403,14 @@ int btl_openib_register_mca_params(void)
     mca_btl_openib_component.max_lmc = (uint32_t) ival;
 
 #if OMPI_HAVE_THREADS
-    CHECK(reg_int("enable_apm", "Maximum number of alterative paths for each HCA port "
+    CHECK(reg_int("enable_apm_over_lmc", "Maximum number of alterative paths for each HCA port "
                   "(must be >= -1, where 0 = disable apm, -1 = all availible alternative paths )",
                   0, &ival, REGINT_NEG_ONE_OK|REGINT_GE_ZERO));
-    mca_btl_openib_component.apm = (uint32_t) ival;
+    mca_btl_openib_component.apm_lmc = (uint32_t) ival;
+    CHECK(reg_int("enable_apm_over_ports", "Enable alterative path migration over different ports of the same HCA"
+                  "(must be >= 0, where 0 = disable apm over ports , 1 = enable apm over ports of the same hca )",
+                  0, &ival, REGINT_GE_ZERO));
+    mca_btl_openib_component.apm_ports = (uint32_t) ival;
 
     CHECK(reg_int("use_async_event_thread",
                 "If nonzero, use the thread that will handle InfiniBand asyncihronous events ",

ompi/mca/btl/openib/connect/btl_openib_connect_xoob.c

@@ -570,8 +570,8 @@ static int xoob_recv_qp_create(mca_btl_openib_endpoint_t *endpoint, mca_btl_open
                     endpoint->xrc_recv_qp_num, strerror(ret), ret));
         return OMPI_ERROR;
     }
-    if (0 != mca_btl_openib_component.apm) {
+    if (APM_ENABLED) {
-        mca_btl_openib_load_apm_xrc_rcv(endpoint->xrc_recv_qp_num, openib_btl);
+        mca_btl_openib_load_apm_xrc_rcv(endpoint->xrc_recv_qp_num, endpoint);
     }
 
     return OMPI_SUCCESS;

ompi/mca/btl/openib/help-mpi-btl-openib.txt

@@ -444,3 +444,7 @@ APM support will be disabled.
 #
 [apm with wrong lmc]
 Can not provide %d alternative paths with LMC bit configured to %d.
+#
+[apm not enough ports]
+WARNING: For APM over ports ompi require at least 2 active ports and only single
+active port was found. Disabling APM over ports