Scale the SRQ based on the log base 2 of the number of peers,

this assumes that the peers have all been added via add_procs up front. 
Bad things will happen if add_procs is called again later on a new set of
 procs to fix this we need to modify the srq which may wreck things.. looking
 into this deeper.. 

This commit was SVN r7142.

ompi/mca/btl/mvapi/btl_mvapi.c

@@ -31,6 +31,8 @@
 #include "mca/mpool/mvapi/mpool_mvapi.h" 
 #include "mca/btl/base/btl_base_error.h" 
 #include <vapi_types.h> 
+#include <math.h> /* for log2 */ 
+
 mca_btl_mvapi_module_t mca_btl_mvapi_module = {
     {
         &mca_btl_mvapi_component.super,
@@ -115,7 +117,12 @@ int mca_btl_mvapi_add_procs(
         peers[i] = ib_peer;
     }
     
-    return OMPI_SUCCESS;
+    mvapi_btl->num_peers += nprocs; 
+    if(mca_btl_mvapi_component.use_srq) { 
+        mvapi_btl->rd_buf_max = mca_btl_mvapi_component.ib_rr_buf_max + log2(nprocs) * mca_btl_mvapi_component.rd_per_peer; 
+    }
+
+   return OMPI_SUCCESS;
 }
 
 /* 

ompi/mca/btl/mvapi/btl_mvapi.h

@@ -95,6 +95,9 @@ struct mca_btl_mvapi_component_t {
     uint32_t ib_rr_buf_min; 
     /**< the minimum number of posted rr */ 
     
+    uint32_t rd_per_peer; 
+    /**< the number of recv desc posted per log(peer) in SRQ mode */ 
+
     size_t eager_limit; 
     size_t max_send_size; 
 
@@ -176,6 +179,10 @@ struct mca_btl_mvapi_module_t {
     /**< an array to allow posting of rr in one swoop */ 
     size_t ib_inline_max; /**< max size of inline send*/ 
     
+    uint32_t num_peers; 
+    uint32_t rd_buf_max; 
+
+    
 }; typedef struct mca_btl_mvapi_module_t mca_btl_mvapi_module_t;
     
 

ompi/mca/btl/mvapi/btl_mvapi_component.c

@@ -178,9 +178,13 @@ int mca_btl_mvapi_component_open(void)
         mca_btl_mvapi_param_register_int("ib_src_path_bits", 
                                       0); 
 
+    mca_btl_mvapi_component.rd_per_peer = 
+        mca_btl_mvapi_param_register_int("rd_per_peer", 
+                                         16); 
+    
 
     mca_btl_mvapi_module.super.btl_exclusivity =
-        mca_btl_mvapi_param_register_int ("exclusivity", MCA_BTL_EXCLUSIVITY_DEFAULT);
+                                         mca_btl_mvapi_param_register_int ("exclusivity", MCA_BTL_EXCLUSIVITY_DEFAULT);
     mca_btl_mvapi_module.super.btl_eager_limit = 
         mca_btl_mvapi_param_register_int ("eager_limit", (64*1024)) 
         - sizeof(mca_btl_mvapi_header_t); 
@@ -203,7 +207,6 @@ int mca_btl_mvapi_component_open(void)
         mca_btl_mvapi_param_register_int("flags", 
                                       MCA_BTL_FLAGS_PUT|MCA_BTL_FLAGS_GET); 
     
-    
     param = mca_base_param_find("mpi", NULL, "leave_pinned"); 
     mca_base_param_lookup_int(param, &value); 
     mca_btl_mvapi_component.leave_pinned = value; 
@@ -215,6 +218,7 @@ int mca_btl_mvapi_component_open(void)
                            16, 
                            &(mca_btl_mvapi_component.max_wr_sq_tokens)); 
     
+   
     mca_btl_mvapi_component.max_send_size = mca_btl_mvapi_module.super.btl_max_send_size; 
     mca_btl_mvapi_component.eager_limit = mca_btl_mvapi_module.super.btl_eager_limit; 
     
@@ -395,9 +399,8 @@ mca_btl_base_module_t** mca_btl_mvapi_component_init(int *num_btl_modules,
         free(mvapi_btl); 
 
         mvapi_btl = &mca_btl_mvapi_component.mvapi_btls[i];
-        
+        mvapi_btl->rd_buf_max = mca_btl_mvapi_component.ib_rr_buf_max; 
-        /* Initialize the modules function pointers */
+        mvapi_btl->num_peers = 0; 
-        
 
         /* Initialize module state */
 
@@ -442,7 +445,7 @@ mca_btl_base_module_t** mca_btl_mvapi_component_init(int *num_btl_modules,
         ompi_free_list_init(&mvapi_btl->send_free_eager,
                             length, 
                             OBJ_CLASS(mca_btl_mvapi_send_frag_eager_t),
-                            2*mca_btl_mvapi_component.ib_rr_buf_max,
+                            2*mvapi_btl->rd_buf_max,
                             mca_btl_mvapi_component.ib_free_list_max,
                             mca_btl_mvapi_component.ib_free_list_inc,
                             mvapi_btl->ib_pool);
@@ -450,7 +453,7 @@ mca_btl_base_module_t** mca_btl_mvapi_component_init(int *num_btl_modules,
         ompi_free_list_init(&mvapi_btl->recv_free_eager,
                             length, 
                             OBJ_CLASS(mca_btl_mvapi_recv_frag_eager_t),
-                            2*mca_btl_mvapi_component.ib_rr_buf_max,
+                            2*mvapi_btl->rd_buf_max,
                             mca_btl_mvapi_component.ib_free_list_max,
                             mca_btl_mvapi_component.ib_free_list_inc,
                             mvapi_btl->ib_pool);
@@ -466,7 +469,7 @@ mca_btl_base_module_t** mca_btl_mvapi_component_init(int *num_btl_modules,
         ompi_free_list_init(&mvapi_btl->send_free_max,
                             length, 
                             OBJ_CLASS(mca_btl_mvapi_send_frag_max_t),
-                            2*mca_btl_mvapi_component.ib_rr_buf_max,
+                            2*mvapi_btl->rd_buf_max,
                             mca_btl_mvapi_component.ib_free_list_max,
                             mca_btl_mvapi_component.ib_free_list_inc,
                             mvapi_btl->ib_pool);
@@ -477,7 +480,7 @@ mca_btl_base_module_t** mca_btl_mvapi_component_init(int *num_btl_modules,
         ompi_free_list_init (&mvapi_btl->recv_free_max, 
                              length, 
                              OBJ_CLASS (mca_btl_mvapi_recv_frag_max_t),
-                             2*mca_btl_mvapi_component.ib_rr_buf_max,
+                             2*mvapi_btl->rd_buf_max,
                              mca_btl_mvapi_component.ib_free_list_max,
                              mca_btl_mvapi_component.ib_free_list_inc, mvapi_btl->ib_pool);
 
@@ -500,7 +503,7 @@ mca_btl_base_module_t** mca_btl_mvapi_component_init(int *num_btl_modules,
         
         
         /* Initialize the rr_desc_post array for posting of rr*/ 
-        mvapi_btl->rr_desc_post = (VAPI_rr_desc_t*) malloc((mca_btl_mvapi_component.ib_rr_buf_max * sizeof(VAPI_rr_desc_t))); 
+        mvapi_btl->rr_desc_post = (VAPI_rr_desc_t*) malloc((mvapi_btl->rd_buf_max * sizeof(VAPI_rr_desc_t))); 
         btls[i] = &mvapi_btl->super;
     
     }

ompi/mca/btl/mvapi/btl_mvapi_endpoint.h

@@ -174,8 +174,8 @@ void mca_btl_mvapi_progress_send_frags(mca_btl_mvapi_endpoint_t*);
     mca_btl_mvapi_module_t * post_rr_high_mvapi_btl = post_rr_high_endpoint->endpoint_btl; \
     OPAL_THREAD_LOCK(&post_rr_high_mvapi_btl->ib_lock); \
     if(post_rr_high_endpoint->rr_posted_high <= mca_btl_mvapi_component.ib_rr_buf_min+post_rr_high_additional && \
-       post_rr_high_endpoint->rr_posted_high < mca_btl_mvapi_component.ib_rr_buf_max){ \
+       post_rr_high_endpoint->rr_posted_high < post_rr_high_mvapi_btl->rd_buf_max){ \
-        MCA_BTL_MVAPI_ENDPOINT_POST_RR_SUB(mca_btl_mvapi_component.ib_rr_buf_max -  \
+        MCA_BTL_MVAPI_ENDPOINT_POST_RR_SUB(post_rr_high_mvapi_btl->rd_buf_max -  \
                                             post_rr_high_endpoint->rr_posted_high, \
                                             post_rr_high_endpoint, \
                                             &post_rr_high_mvapi_btl->recv_free_eager, \
@@ -192,8 +192,8 @@ void mca_btl_mvapi_progress_send_frags(mca_btl_mvapi_endpoint_t*);
     mca_btl_mvapi_module_t * post_rr_low_mvapi_btl = post_rr_low_endpoint->endpoint_btl; \
     OPAL_THREAD_LOCK(&post_rr_low_mvapi_btl->ib_lock); \
     if(post_rr_low_endpoint->rr_posted_low <= mca_btl_mvapi_component.ib_rr_buf_min+post_rr_low_additional && \
-       post_rr_low_endpoint->rr_posted_low < mca_btl_mvapi_component.ib_rr_buf_max){ \
+       post_rr_low_endpoint->rr_posted_low < post_rr_low_mvapi_btl->rd_buf_max){ \
-        MCA_BTL_MVAPI_ENDPOINT_POST_RR_SUB(mca_btl_mvapi_component.ib_rr_buf_max -  \
+        MCA_BTL_MVAPI_ENDPOINT_POST_RR_SUB(post_rr_low_mvapi_btl->rd_buf_max -  \
                                             post_rr_low_endpoint->rr_posted_low, \
                                             post_rr_low_endpoint, \
                                             &post_rr_low_mvapi_btl->recv_free_max, \