Fixed up sending large fragments and preposting of receive descriptors..

This commit was SVN r6023.

src/mca/bmi/ib/bmi_ib.c

@@ -211,7 +211,7 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_src(
     mca_bmi_ib_module_t* ib_bmi; 
     mca_bmi_ib_frag_t* frag; 
     struct iovec iov; 
-    int32_t out_size = 0; 
+    int32_t iov_count = 1; 
     size_t max_data = *size; 
     int32_t free_after; 
     int rc; 
@@ -230,7 +230,7 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_src(
         iov.iov_len = max_data; 
         iov.iov_base = frag->segment.seg_addr.pval + reserve; 
         
-        rc = ompi_convertor_pack(convertor, &iov, &out_size, &max_data, &free_after); 
+        rc = ompi_convertor_pack(convertor, &iov, &iov_count, &max_data, &free_after); 
         if( rc < 0 ) { 
             MCA_BMI_IB_FRAG_RETURN_EAGER(bmi, frag); 
             return NULL; 
@@ -249,9 +249,9 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_src(
             max_data = frag->size - reserve; 
         }
         iov.iov_len = max_data; 
-        iov.iov_base = frag->segment.seg_addr.pval + reserve; 
+        iov.iov_base = (unsigned char*) frag->segment.seg_addr.pval + reserve; 
         
-        rc = ompi_convertor_pack(convertor, &iov, &out_size, &max_data, &free_after); 
+        rc = ompi_convertor_pack(convertor, &iov, &iov_count, &max_data, &free_after); 
         if( rc < 0 ) { 
             MCA_BMI_IB_FRAG_RETURN_MAX(bmi, frag); 
             return NULL; 
@@ -277,15 +277,15 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_src(
           mr_in.r_key = 0;
           mr_in.pd_hndl = ib_bmi->ptag; 
           mr_in.type = VAPI_MR;
+          MCA_BMI_IB_FRAG_ALLOC_FRAG(bmi, frag, rc); 
+          if(NULL == frag){
+              return NULL; 
+          } 
                 
-          
-          frag = (mca_bmi_ib_send_frag_frag_t*) ib_bmi->ib_pool->mpool_alloc(ib_bmi->ib_pool,  sizeof(frag) + sizeof(mca_bmi_ib_header_t) + *size ,0, &user_out);
-          frag->base.super.user_data = user_out;
-          OBJ_CONSTRUCT(frag, mca_bmi_ib_send_frag_frag_t);
           iov.iov_len = max_data; 
           iov.iov_base = NULL; 
           
-          ompi_convertor_pack(convertor, &iov, &out_size, &max_data, &free_after); 
+          ompi_convertor_pack(convertor, &iov, &iov_count, &max_data, &free_after); 
           frag->segment.seg_len = max_data; 
           frag->segment.seg_addr.pval = iov.iov_base; 
   
@@ -307,6 +307,9 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_src(
           mem_hndl.l_key = mr_out.l_key; 
           mem_hndl.r_key = mr_out.r_key; 
  
+          frag->mem_hndl = mem_hndl.hndl; 
+          frag->segment.seg_key.key32[1] = (uint32_t) mem_hndl.l_key; 
+  
           return &frag->base; 
 
     }

src/mca/bmi/ib/bmi_ib_component.c

@@ -123,7 +123,8 @@ int mca_bmi_ib_component_open(void)
         mca_bmi_ib_param_register_string("mpool", "ib"); 
     mca_bmi_ib_component.ib_rr_buf_max = 
         mca_bmi_ib_param_register_int("rr_buf_max", 16); 
-    
+    mca_bmi_ib_component.ib_rr_buf_min = 
+        mca_bmi_ib_param_register_int("rr_buf_min", 8); 
     mca_bmi_ib_module.super.bmi_exclusivity =
         mca_bmi_ib_param_register_int ("exclusivity", 0);
     mca_bmi_ib_module.super.bmi_eager_limit = 
@@ -135,8 +136,10 @@ int mca_bmi_ib_component_open(void)
                                        (64*1024 
                                         - sizeof(mca_bmi_ib_header_t)));
     mca_bmi_ib_module.super.bmi_max_send_size =
-        mca_bmi_ib_param_register_int ("max_send_size", 128*1024);
+        mca_bmi_ib_param_register_int ("max_send_size", 128*1024 -
-    
+                                       sizeof(mca_bmi_ib_header_t));
+    mca_bmi_ib_module.ib_pin_min = 
+        mca_bmi_ib_param_register_int("ib_pin_min", 128*1024);                                    
     mca_bmi_ib_module.ib_cq_size = 
         mca_bmi_ib_param_register_int("ib_cq_size", 
                                       40000); 
@@ -528,7 +531,6 @@ int mca_bmi_ib_component_progress()
                                                 ((mca_bmi_ib_frag_t*)comp.id)->endpoint); 
                 
                 
-                               
                 count++; 
                 break;
                 

src/mca/bmi/ib/bmi_ib_endpoint.h

@@ -125,7 +125,7 @@ static inline int mca_bmi_ib_endpoint_post_rr(int cnt, mca_bmi_ib_endpoint_t *en
         OMPI_FREE_LIST_WAIT(&ib_bmi->recv_free, item, rc); 
         frag = (mca_bmi_ib_frag_t*) item; 
         frag->endpoint = endpoint; 
-        frag->sg_entry.len = frag->size;     
+        frag->sg_entry.len = frag->size + sizeof(mca_bmi_ib_header_t);     
         rr_desc_post[i] = frag->rr_desc; 
         
     }

src/mca/bmi/ib/bmi_ib_frag.c

@@ -68,7 +68,7 @@ static void mca_bmi_ib_send_frag_max_constructor(mca_bmi_ib_frag_t* frag)
 
 static void mca_bmi_ib_recv_frag_constructor(mca_bmi_ib_frag_t* frag) 
 {
-    frag->size = mca_bmi_ib_component.max_send_size; 
+    frag->size = mca_bmi_ib_component.max_send_size+100; 
     mca_bmi_ib_recv_frag_common_constructor(frag); 
     
 }