* fix for ticket #141. If we are going to shortcut out of polling the

send/receive queues if there is something available in the short message
  rdma queues, then we have to poll *ALL* the rdma queues before exiting,
  or we aren't fair about frag reception and fall into degenerate matching
  cases.

This commit was SVN r10410.

ompi/mca/btl/mvapi/btl_mvapi_component.c

@@ -727,10 +727,8 @@ int mca_btl_mvapi_component_progress( void )
     
     mca_btl_mvapi_frag_t* frag;
     mca_btl_mvapi_endpoint_t* endpoint; 
-    /* Poll for completions */
+    /* Poll for RDMA completions - if any succeed, we don't process the slower queues */
     for(i = 0; i < mca_btl_mvapi_component.ib_num_btls; i++) {
-        VAPI_ret_t ret; 
-        VAPI_wc_desc_t comp; 
         mca_btl_mvapi_module_t* mvapi_btl = &mca_btl_mvapi_component.mvapi_btls[i];
 
         OPAL_THREAD_LOCK(&mvapi_btl->eager_rdma_lock);
@@ -774,10 +772,13 @@ int mca_btl_mvapi_component_progress( void )
             } else
                 OPAL_THREAD_UNLOCK(&endpoint->eager_rdma_local.lock);
         }
+    }
+    if (count) return count;
 
-        if(count)
-           break;
-
+    for(i = 0; i < mca_btl_mvapi_component.ib_num_btls; i++) {
+        VAPI_ret_t ret; 
+        VAPI_wc_desc_t comp; 
+        mca_btl_mvapi_module_t* mvapi_btl = &mca_btl_mvapi_component.mvapi_btls[i];
         /* we have two completion queues, one for "high" priority and one for "low". 
          *   we will check the high priority and process them until there are none left. 
          *   note that low priority messages are only processed one per progress call. 

ompi/mca/btl/openib/btl_openib_component.c

@@ -812,17 +812,10 @@ int mca_btl_openib_component_progress()
     mca_btl_openib_frag_t* frag; 
     mca_btl_openib_endpoint_t* endpoint; 
 
-    /* Poll for completions */
+    /* Poll for RDMA completions - if any succeed, we don't process the slower queues */
     for(i = 0; i < mca_btl_openib_component.ib_num_btls; i++) {
-        
-        struct ibv_wc wc; 
         mca_btl_openib_module_t* openib_btl = &mca_btl_openib_component.openib_btls[i];
 
-        /* we have two completion queues, one for "high" priority and one for "low". 
-         *   we will check the high priority and process them until there are none left. 
-         *   note that low priority messages are only processed one per progress call. 
-         */
-
         OPAL_THREAD_LOCK(&openib_btl->eager_rdma_lock);
         c = openib_btl->eager_rdma_buffers_count;
         OPAL_THREAD_UNLOCK(&openib_btl->eager_rdma_lock);
@@ -864,9 +857,17 @@ int mca_btl_openib_component_progress()
             } else
                 OPAL_THREAD_UNLOCK(&endpoint->eager_rdma_local.lock);
         }
+    }
+    if(count) return count;
 
-        if(count)
-           break;
+    for(i = 0; i < mca_btl_openib_component.ib_num_btls; i++) {
+        struct ibv_wc wc; 
+        mca_btl_openib_module_t* openib_btl = &mca_btl_openib_component.openib_btls[i];
+        
+        /* we have two completion queues, one for "high" priority and one for "low". 
+         *   we will check the high priority and process them until there are none left. 
+         *   note that low priority messages are only processed one per progress call. 
+         */
 
         ne=ibv_poll_cq(openib_btl->ib_cq_hp, 1, &wc );
         if(ne < 0 ){