Balance RDMA operation in round robin fashion between all available RDMA BTLs.

OB1 always use first element from array of BTLs available for RDMA. The patch
change the array creation algorithm, it puts different BTL in the first element
in round robin fashion.

This commit was SVN r13174.

ompi/mca/bml/bml.h

@@ -244,6 +244,7 @@ struct mca_bml_base_endpoint_t {
     size_t                   bml_max_send_length;
     size_t                   bml_max_rdma_length;
     mca_bml_base_btl_array_t btl_rdma;          /**< array of btls that support (prefer) rdma */
+    size_t btl_rdma_index;                      /**< index of last used BTL for RDMA */
     uint32_t btl_flags_or;                      /**< the bitwise OR of the btl flags */
     uint32_t btl_flags_and;                     /**< the bitwise AND of the btl flags */
 };

ompi/mca/bml/r2/bml_r2.c

@@ -378,6 +378,7 @@ int mca_bml_r2_add_procs(
         n_size = mca_bml_base_btl_array_get_size(&bml_endpoint->btl_send); 
         bml_endpoint->bml_max_send_length = 0;
         bml_endpoint->bml_max_rdma_length = 0;
+        bml_endpoint->btl_rdma_index = 0;
         for(n_index = 0; n_index < n_size; n_index++) {
             mca_bml_base_btl_t* bml_btl = 
                 mca_bml_base_btl_array_get_index(&bml_endpoint->btl_send, n_index);

ompi/mca/pml/ob1/pml_ob1_rdma.c

@@ -54,10 +54,11 @@ size_t mca_pml_ob1_rdma_btls(
     for(n = 0; n < num_btls && num_btls_used < MCA_PML_OB1_MAX_RDMA_PER_REQUEST;
             n++) {
         mca_bml_base_btl_t* bml_btl =
-            mca_bml_base_btl_array_get_index(&bml_endpoint->btl_rdma, n); 
+            mca_bml_base_btl_array_get_index(&bml_endpoint->btl_rdma,
+                    (bml_endpoint->btl_rdma_index + n) % num_btls); 
         mca_mpool_base_registration_t* reg = NULL;
         mca_mpool_base_module_t *btl_mpool = bml_btl->btl_mpool;
- 
+
         /* btl is rdma capable and registration is not required */
         if(NULL == btl_mpool) {
             reg = NULL;
@@ -80,5 +81,6 @@ size_t mca_pml_ob1_rdma_btls(
             num_btls_used++;
         }
     }
+    bml_endpoint->btl_rdma_index = (bml_endpoint->btl_rdma_index + 1) % num_btls;
     return num_btls_used;
 }

ompi/mca/pml/ob1/pml_ob1_sendreq.c

@@ -632,8 +632,10 @@ int mca_pml_ob1_send_request_start_rdma(
     size_t size)
 {
     /*
-     * FIX - to get the basics working - schedule on the
-     * first BTL only
+     * When req_rdma array is constructed the firs element of the array always
+     * assigned different btl in round robin fashion (if there are more than
+     * one RDMA capable BTLs). This way round robin distribution of RDMA
+     * operation is achieved.
      */
 
     mca_mpool_base_registration_t* reg = sendreq->req_rdma[0].btl_reg;