Added sender side per qp send tokens to limit the number of outstanding

sends. 

This commit was SVN r7112.

ompi/mca/btl/mvapi/btl_mvapi.h

@@ -176,14 +176,6 @@ struct mca_btl_mvapi_module_t {
     /**< an array to allow posting of rr in one swoop */ 
     size_t ib_inline_max; /**< max size of inline send*/ 
 
-    
-    /* number of outstanding send's */ 
-    uint32_t send_tokens; 
-    
-    opal_list_t                 pending_send_frags;
-    /**< list of pending send frags for this endpoint */
-
-
 }; typedef struct mca_btl_mvapi_module_t mca_btl_mvapi_module_t;
     
 

ompi/mca/btl/mvapi/btl_mvapi_component.c

@@ -413,7 +413,6 @@ mca_btl_base_module_t** mca_btl_mvapi_component_init(int *num_btl_modules,
         OBJ_CONSTRUCT(&mvapi_btl->repost, opal_list_t);
         OBJ_CONSTRUCT(&mvapi_btl->reg_mru_list, opal_list_t); 
         
-        OBJ_CONSTRUCT(&mvapi_btl->pending_send_frags, opal_list_t);
 
         if(mca_btl_mvapi_module_init(mvapi_btl) != OMPI_SUCCESS) {
             free(hca_ids);
@@ -502,10 +501,8 @@ mca_btl_base_module_t** mca_btl_mvapi_component_init(int *num_btl_modules,
         
         /* Initialize the rr_desc_post array for posting of rr*/ 
         mvapi_btl->rr_desc_post = (VAPI_rr_desc_t*) malloc((mca_btl_mvapi_component.ib_rr_buf_max * sizeof(VAPI_rr_desc_t))); 
-        
-        mvapi_btl->send_tokens = mca_btl_mvapi_component.max_send_tokens; 
-
         btls[i] = &mvapi_btl->super;
+    
     }
 
     /* Post OOB receive to support dynamic connection setup */
@@ -554,22 +551,21 @@ int mca_btl_mvapi_component_progress()
                 return OMPI_ERROR; 
            
             case VAPI_CQE_SQ_SEND_DATA :
-                mvapi_btl->send_tokens++;
+                frag = (mca_btl_mvapi_frag_t*) comp.id; 
+                frag->endpoint->send_tokens++;
                 
                 /* fall through */ 
             case VAPI_CQE_SQ_RDMA_READ:
             case VAPI_CQE_SQ_RDMA_WRITE:
            
-                
                 /* Process a completed send or an rdma write  */
-                frag = (mca_btl_mvapi_frag_t*) comp.id; 
                 frag->rc = OMPI_SUCCESS; 
                 frag->base.des_cbfunc(&mvapi_btl->super, frag->endpoint, &frag->base, frag->rc); 
                 count++;
                 /* check and see if we need to progress pending sends */ 
-                if(mvapi_btl->send_tokens && !opal_list_is_empty(&(mvapi_btl->pending_send_frags))) { 
+                if(frag->endpoint->send_tokens && !opal_list_is_empty(&(frag->endpoint->pending_send_frags))) { 
                     opal_list_item_t *frag_item;
-                    frag_item = opal_list_remove_first(&(mvapi_btl->pending_send_frags));
+                    frag_item = opal_list_remove_first(&(frag->endpoint->pending_send_frags));
                     frag = (mca_btl_mvapi_frag_t *) frag_item;
                     
                     if(OMPI_SUCCESS !=  mca_btl_mvapi_endpoint_send(frag->endpoint, frag)) { 

ompi/mca/btl/mvapi/btl_mvapi_endpoint.c

@@ -123,7 +123,8 @@ static void mca_btl_mvapi_endpoint_construct(mca_btl_base_endpoint_t* endpoint)
     
     endpoint->rr_posted_high = 0;
     endpoint->rr_posted_low = 0; 
-
+    endpoint->send_tokens = mca_btl_mvapi_component.max_send_tokens; 
+    
 }
 
 /*
@@ -568,16 +569,16 @@ int mca_btl_mvapi_endpoint_send(
                 
                 mvapi_btl = endpoint->endpoint_btl;
                 
-                if(0 == mvapi_btl->send_tokens) { 
+                if(0 == endpoint->send_tokens) { 
                     BTL_VERBOSE(("Queing because no send tokens \n"));
                     
-                    opal_list_append(&mvapi_btl->pending_send_frags,
+                    opal_list_append(&endpoint->pending_send_frags,
                                      (opal_list_item_t *)frag);
                     
                     rc = OMPI_SUCCESS;
                 } else { 
                 
-                    mvapi_btl->send_tokens--; 
+                    endpoint->send_tokens--; 
                     
                     BTL_VERBOSE(("Send to : %d, len : %d, frag : %p", 
                                  endpoint->endpoint_proc->proc_guid.vpid,
@@ -688,8 +689,8 @@ int mca_btl_mvapi_endpoint_create_qp(
     switch(transport_type) {
 
     case VAPI_TS_RC: /* Set up RC qp parameters */
-        qp_init_attr.cap.max_oust_wr_rq = mca_btl_mvapi_component.ib_wq_size;
-        qp_init_attr.cap.max_oust_wr_sq = mca_btl_mvapi_component.ib_wq_size;
+        qp_init_attr.cap.max_oust_wr_rq = mca_btl_mvapi_component.ib_rr_buf_max;
+        qp_init_attr.cap.max_oust_wr_sq = mca_btl_mvapi_component.max_send_tokens;
         qp_init_attr.cap.max_sg_size_rq = mca_btl_mvapi_component.ib_sg_list_size;
         qp_init_attr.cap.max_sg_size_sq = mca_btl_mvapi_component.ib_sg_list_size;
         qp_init_attr.pd_hndl            = ptag;

ompi/mca/btl/mvapi/btl_mvapi_endpoint.h

@@ -105,7 +105,10 @@ struct mca_btl_base_endpoint_t {
 
     opal_list_t                 pending_send_frags;
     /**< list of pending send frags for this endpoint */
-
+    
+    uint32_t                    send_tokens; 
+    /**< number of sends that  can be outstanding (down counter) */ 
+    
     VAPI_qp_num_t               rem_qp_num_high;
     /* High priority remote side QP number */