More mpool bmi changes - changed mca_mpool_base_registration_t to an ompi

object. Modified bmi_ib to use OBJ_RETAIN and OBJ_RELEASE with the
mca_mpool_vapi_registration_t object. 

NOTE!: bmi_ib fails with OMPI_MCA_leave_pinned=1 when the memory region must
be reallocated due to size limitations. This appears to be caused by
OBJ_RELEASE being called when the object is already freed. Should be fixed by
the end of this weekend. 

This commit was SVN r6180.

src/mca/bmi/ib/bmi_ib.c

@@ -184,26 +184,12 @@ int mca_bmi_ib_free(
                     mca_bmi_base_descriptor_t* des) 
 {
     mca_bmi_ib_frag_t* frag = (mca_bmi_ib_frag_t*)des; 
-    mca_bmi_ib_module_t * ib_bmi = (mca_bmi_ib_module_t*) bmi; 
 
     if(frag->size == 0) {
         MCA_BMI_IB_FRAG_RETURN_FRAG(bmi, frag); 
-        
-        /* we also need to unregister the associated memory  iff 
-            the memory wasn't allocated via MPI_Alloc_mem */
-
-        if(frag->base.des_flags & MCA_BMI_DES_FLAGS_DEREGISTER){ 
-            frag->ret = VAPI_deregister_mr(
-                                           ib_bmi->nic, 
-                                           frag->mem_hndl
-                                           ); 
-            if(frag->ret!=VAPI_OK){ 
-                ompi_output(0, "%s:error deregistering memory region", __func__); 
-                return OMPI_ERROR; 
+        OBJ_RELEASE(frag->vapi_reg); 
     } 
-        } 
-
-    } else if(frag->size == mca_bmi_ib_component.max_send_size){ 
+    else if(frag->size == mca_bmi_ib_component.max_send_size){ 
         MCA_BMI_IB_FRAG_RETURN_MAX(bmi, frag); 
     } else if(frag->size == mca_bmi_ib_component.eager_limit){ 
         MCA_BMI_IB_FRAG_RETURN_EAGER(bmi, frag); 
@@ -267,18 +253,17 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_src(
         size_t reg_len; 
         reg_len = (unsigned char*)vapi_reg->bound - (unsigned char*)iov.iov_base + 1; 
         if(frag->segment.seg_len > reg_len) { 
+            size_t new_len = vapi_reg->bound - vapi_reg->base + 1 
+                + frag->segment.seg_len - reg_len; 
+            void * base_addr = vapi_reg->base; 
 
             mca_mpool_base_remove((void*) vapi_reg->base); 
-            ompi_list_remove_item(&ib_bmi->reg_mru_list, (ompi_list_item_t*) vapi_reg); 
-            ib_bmi->ib_pool->mpool_deregister(
-                                              ib_bmi->ib_pool, 
-                                              vapi_reg->base, 
-                                              0, 
-                                              (mca_mpool_base_registration_t*) vapi_reg); 
+            /* ompi_list_remove_item(&ib_bmi->reg_mru_list, (ompi_list_item_t*) vapi_reg); */
+            OBJ_RELEASE(vapi_reg); 
             
             ib_bmi->ib_pool->mpool_register(ib_bmi->ib_pool, 
-                                            vapi_reg->base, 
-                                            vapi_reg->bound - vapi_reg->base + 1 + frag->segment.seg_len - reg_len, 
+                                            base_addr, 
+                                            new_len, 
                                             (mca_mpool_base_registration_t**) &vapi_reg);
                 
             rc = mca_mpool_base_insert(iov.iov_base, 
@@ -287,10 +272,12 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_src(
                                        (void*) (&ib_bmi->super), 
                                        (mca_mpool_base_registration_t*) vapi_reg); 
             
+            
+
             if(rc != OMPI_SUCCESS) 
                 return NULL; 
         
-            ompi_list_append(&ib_bmi->reg_mru_list, (ompi_list_item_t*) vapi_reg); 
+            /* ompi_list_append(&ib_bmi->reg_mru_list, (ompi_list_item_t*) vapi_reg);  */
             
         }   
         
@@ -306,7 +293,8 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_src(
         frag->base.des_dst = NULL;
         frag->base.des_dst_cnt = 0;
         frag->base.des_flags = 0; 
-
+        frag->vapi_reg = vapi_reg; 
+        OBJ_RETAIN(vapi_reg); 
         return &frag->base;
         
     } else if((mca_bmi_ib_component.leave_pinned || max_data > bmi->bmi_max_send_size) && 
@@ -342,9 +330,7 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_src(
                                        (mca_mpool_base_registration_t*) vapi_reg); 
             if(rc != OMPI_SUCCESS) 
                 return NULL; 
-            
-        } else { 
-            frag->base.des_flags |= MCA_BMI_DES_FLAGS_DEREGISTER; 
+            OBJ_RETAIN(vapi_reg); 
         } 
         frag->mem_hndl = vapi_reg->hndl; 
         frag->sg_entry.len = max_data; 
@@ -357,8 +343,10 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_src(
         frag->base.des_src_cnt = 1;
         frag->base.des_dst = NULL;
         frag->base.des_dst_cnt = 0;
-
+        frag->vapi_reg = vapi_reg; 
+        OBJ_RETAIN(vapi_reg); 
         return &frag->base;
+
     } else if (max_data+reserve <=  bmi->bmi_eager_limit) { 
            
         MCA_BMI_IB_FRAG_ALLOC_EAGER(bmi, frag, rc); 
@@ -460,17 +448,16 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_dst(
     if(NULL!= vapi_reg){ 
         reg_len = (unsigned char*)vapi_reg->bound - (unsigned char*)frag->segment.seg_addr.pval + 1; 
         if(frag->segment.seg_len > reg_len) { 
-            ib_bmi->ib_pool->mpool_deregister(
-                                              ib_bmi->ib_pool, 
-                                              vapi_reg->base, 
-                                              0, 
-                                              (mca_mpool_base_registration_t*) vapi_reg); 
+            size_t new_len = vapi_reg->bound - vapi_reg->base + 1 
+                + frag->segment.seg_len - reg_len; 
+            void * base_addr = vapi_reg->base; 
 
             mca_mpool_base_remove((void*) vapi_reg->base); 
+            OBJ_RELEASE(vapi_reg); 
             
             ib_bmi->ib_pool->mpool_register(ib_bmi->ib_pool, 
-                                            vapi_reg->base, 
-                                            vapi_reg->bound - vapi_reg->base + 1 + frag->segment.seg_len - reg_len, 
+                                            base_addr, 
+                                            new_len,
                                             (mca_mpool_base_registration_t**) &vapi_reg);
             
         
@@ -480,6 +467,7 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_dst(
                                        (void*) (&ib_bmi->super), 
                                        (mca_mpool_base_registration_t*) vapi_reg); 
             
+            
         } 
                 
     }  else { 
@@ -496,9 +484,7 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_dst(
                                        (mca_mpool_base_registration_t*)  vapi_reg); 
             if(rc != OMPI_SUCCESS) 
                 return NULL;
-        }
-        else { 
-            frag->base.des_flags |= MCA_BMI_DES_FLAGS_DEREGISTER; 
+            OBJ_RETAIN(vapi_reg); 
         }
     }
 
@@ -514,7 +500,8 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_dst(
     frag->base.des_dst_cnt = 1; 
     frag->base.des_src = NULL; 
     frag->base.des_src_cnt = 0; 
-    
+    frag->vapi_reg = vapi_reg; 
+    OBJ_RETAIN(vapi_reg); 
     return &frag->base; 
     
 }

src/mca/bmi/ib/bmi_ib_frag.h

@@ -25,6 +25,7 @@
 #include <vapi.h> 
 #include <mtl_common.h> 
 #include <vapi_common.h> 
+#include "mca/mpool/vapi/mpool_vapi.h" 
 
 #if defined(c_plusplus) || defined(__cplusplus)
 extern "C" {
@@ -61,6 +62,7 @@ struct mca_bmi_ib_frag_t {
     VAPI_mr_hndl_t mem_hndl; 
     VAPI_ret_t ret;
     mca_bmi_ib_header_t *hdr;
+    mca_mpool_vapi_registration_t * vapi_reg; 
 }; 
 typedef struct mca_bmi_ib_frag_t mca_bmi_ib_frag_t; 
 OBJ_CLASS_DECLARATION(mca_bmi_ib_frag_t); 

src/mca/mpool/mpool.h

@@ -23,6 +23,7 @@
 #include "info/info.h"
 #include "class/ompi_list.h" 
 
+
 struct mca_mpool_base_resources_t;
 
 struct mca_mpool_base_registration_t{ 
@@ -30,7 +31,8 @@ struct mca_mpool_base_registration_t{
     struct mca_mpool_base_module_t *mpool; 
 };  
 typedef struct mca_mpool_base_registration_t mca_mpool_base_registration_t; 
-OBJ_CLASS_DECLARATION(mca_mpool_base_registration_t); 
+
+OMPI_DECLSPEC OBJ_CLASS_DECLARATION(mca_mpool_base_registration_t); 
 
 /**
  * component initialize

src/mca/mpool/vapi/mpool_vapi.h

@@ -86,7 +86,7 @@ struct mca_mpool_vapi_registration_t {
 
 };
 typedef struct mca_mpool_vapi_registration_t mca_mpool_vapi_registration_t;
-OBJ_CLASS_DECLARATION(mca_mpool_vapi_registration_t); 
+OMPI_DECLSPEC OBJ_CLASS_DECLARATION(mca_mpool_vapi_registration_t); 
 
 
 

src/mca/mpool/vapi/mpool_vapi_component.c

@@ -65,7 +65,7 @@ mca_mpool_vapi_component_t mca_mpool_vapi_component = {
 
 static void mca_mpool_vapi_registration_constructor( mca_mpool_vapi_registration_t * registration ) 
 { 
-    memset(registration, 0, sizeof(mca_mpool_base_registration_t)); 
+    
 }
 
 static void mca_mpool_vapi_registration_destructor( mca_mpool_vapi_registration_t * registration )