srq cleanup

This commit was SVN r8106.

ompi/mca/btl/openib/btl_openib.c

@@ -125,9 +125,11 @@ int mca_btl_openib_add_procs(
         openib_btl->num_peers += nprocs; 
         if(mca_btl_openib_component.use_srq) { 
             openib_btl->rd_num = mca_btl_openib_component.rd_num + log2(nprocs) * mca_btl_openib_component.srq_rd_per_peer; 
+            if(openib_btl->rd_num > mca_btl_openib_component.srq_rd_max)
+               openib_btl->rd_num = mca_btl_openib_component.srq_rd_max;
+            openib_btl->rd_low = openib_btl->rd_num - 1;
             free(openib_btl->rd_desc_post); 
             openib_btl->rd_desc_post = (struct ibv_recv_wr*) malloc((openib_btl->rd_num * sizeof(struct ibv_recv_wr))); 
-            openib_btl->rd_low = openib_btl->rd_num * 0.75;
        
         }
     }
@@ -743,7 +745,7 @@ int mca_btl_openib_module_init(mca_btl_openib_module_t *openib_btl)
     if(mca_btl_openib_component.use_srq) { 
         
         struct ibv_srq_init_attr attr; 
-        attr.attr.max_wr = openib_btl->rd_num;
+        attr.attr.max_wr = mca_btl_openib_component.srq_rd_max;
         attr.attr.max_sge = mca_btl_openib_component.ib_sg_list_size;
 
         openib_btl->srd_posted_hp = 0; 

ompi/mca/btl/openib/btl_openib.h

@@ -88,8 +88,9 @@ struct mca_btl_openib_component_t {
     int32_t rd_win; /**< ack credits when window size exceeded */
     int32_t rd_rsv; /**< descriptors held in reserve for control messages */
 
+    int32_t srq_rd_max; /* maximum number of receive descriptors posted */
     int32_t srq_rd_per_peer; /* number of receive descriptors to post per log2(peers) in SRQ mode */
-    int32_t srq_sd_per_proc; /* maximum number of send descriptors posted */
+    int32_t srq_sd_max; /* maximum number of send descriptors posted */
 
     size_t eager_limit; 
     size_t max_send_size; 

ompi/mca/btl/openib/btl_openib_component.c

@@ -177,10 +177,12 @@ int mca_btl_openib_component_open(void)
                                       8,  (int*) &mca_btl_openib_component.rd_win);
     mca_btl_openib_component.rd_rsv = ((mca_btl_openib_component.rd_num<<1)-1) / mca_btl_openib_component.rd_win;
 
+    mca_btl_openib_param_register_int("srq_rd_max", "Max number of receive descriptors posted per SRQ.",
+                                      1000, (int*) &mca_btl_openib_component.srq_rd_max);
     mca_btl_openib_param_register_int("srq_rd_per_peer", "Number of receive descriptors posted per peer. (SRQ)",
                                       16, (int*) &mca_btl_openib_component.srq_rd_per_peer);
-    mca_btl_openib_param_register_int("srq_sd_per_proc", "Maximum number of send descriptors posted. (SRQ)",
-                                      16,  &mca_btl_openib_component.srq_sd_per_proc);
+    mca_btl_openib_param_register_int("srq_sd_max", "Maximum number of send descriptors posted. (SRQ)",
+                                      8,  &mca_btl_openib_component.srq_sd_max);
 
     mca_btl_openib_param_register_int ("eager_limit", "eager send limit", 
                                        (64*1024),(int*) &mca_btl_openib_module.super.btl_eager_limit);  
@@ -429,7 +431,7 @@ mca_btl_base_module_t** mca_btl_openib_component_init(int *num_btl_modules,
         openib_btl->rd_num = mca_btl_openib_component.rd_num + mca_btl_openib_component.rd_rsv;
         openib_btl->rd_low = mca_btl_openib_component.rd_low;
         openib_btl->num_peers = 0; 
-        openib_btl->sd_tokens_hp = openib_btl->sd_tokens_lp = mca_btl_openib_component.srq_sd_per_proc;
+        openib_btl->sd_tokens_hp = openib_btl->sd_tokens_lp = mca_btl_openib_component.srq_sd_max;
 
         /* Initialize module state */