More mpool and bmi restructuring

misc compiler warning fixes. 

This commit was SVN r6173.

src/class/ompi_free_list.c

@@ -75,7 +75,7 @@ int ompi_free_list_grow(ompi_free_list_t* flist, size_t num_elements)
     unsigned char* ptr;
     size_t i;
     size_t mod;
-    struct mca_mpool_base_registration_t* user_out; 
+    mca_mpool_base_registration_t* user_out; 
 
     if (flist->fl_max_to_alloc > 0 && flist->fl_num_allocated + num_elements > flist->fl_max_to_alloc)
         return OMPI_ERR_TEMP_OUT_OF_RESOURCE;

src/class/ompi_rb_tree.c

@@ -188,27 +188,6 @@ void * ompi_rb_tree_find(ompi_rb_tree_t *tree, void *key)
     return(NULL);
 }
 
-/* Finds the node in the tree based on the key */
-void * ompi_rb_tree_find_with_cmp(ompi_rb_tree_t *tree, 
-                                  void *key, 
-                                  ompi_rb_tree_comp_fn_t* comp)
-{
-    ompi_rb_tree_node_t * node;
-    int compvalue;
-
-    node = tree->root_ptr->left;
-    while (node != tree->nill) {
-        compvalue = (*comp)(key, node->key);
-        /* if the result of the comparison function is 0, we found it */
-        if (compvalue == 0) {
-            return(node->value);
-        }
-        /* else if it is less than 0, go left, else right */
-        (compvalue < 0) ? (node = node->left) : (node = node->right);
-    }
-    /* if we didn't find anything, return NULL */
-    return(NULL);
-}
 /* Finds the node in the tree based on the key and returns a pointer
  * to the node. This is a bit a code duplication, but this has to be fast
  * so we go ahead with the duplication */

src/mca/allocator/allocator.h

@@ -20,8 +20,8 @@
 #ifndef MCA_ALLOCATOR_H
 #define MCA_ALLOCATOR_H
 #include "mca/mca.h"
+#include "mca/mpool/mpool.h" 
 
-struct mca_mpool_base_registration_t;
 struct mca_mpool_base_resources_t;
 
 
@@ -38,7 +38,7 @@ typedef void* (*mca_allocator_base_module_alloc_fn_t)(
     struct mca_allocator_base_module_t*, 
     size_t size, 
     size_t align, 
-    struct mca_mpool_base_registration_t** registration);
+    mca_mpool_base_registration_t** registration);
  
 /**
   * The realloc function typedef
@@ -46,7 +46,7 @@ typedef void* (*mca_allocator_base_module_alloc_fn_t)(
 typedef void* (*mca_allocator_base_module_realloc_fn_t)(
     struct mca_allocator_base_module_t*, 
     void*, size_t, 
-    struct mca_mpool_base_registration_t** registration);
+    mca_mpool_base_registration_t** registration);
 
 /**
   * Free function typedef
@@ -103,7 +103,7 @@ typedef struct mca_allocator_base_module_t mca_allocator_base_module_t;
 typedef void* (*mca_allocator_base_component_segment_alloc_fn_t)(
     struct mca_mpool_base_module_t* module,
     size_t* size, 
-    struct mca_mpool_base_registration_t** registration);
+    mca_mpool_base_registration_t** registration);
 
 /**
   * A function to free memory from the control of the allocator framework 

src/mca/allocator/basic/allocator_basic.c

@@ -161,7 +161,7 @@ void *mca_allocator_basic_alloc(
     mca_allocator_base_module_t * base, 
     size_t size,
     size_t align, 
-    struct mca_mpool_base_registration_t** registration)
+    mca_mpool_base_registration_t** registration)
 {
     mca_allocator_basic_module_t* module = (mca_allocator_basic_module_t*)base;
     mca_allocator_basic_segment_t* seg;
@@ -241,7 +241,7 @@ void * mca_allocator_basic_realloc(
     mca_allocator_base_module_t * base, 
     void * ptr, 
     size_t size, 
-    struct mca_mpool_base_registration_t** registration)
+    mca_mpool_base_registration_t** registration)
 {
     unsigned char* addr = ((unsigned char*)ptr) - sizeof(size_t);
     size_t alloc_size = *(size_t*)addr;

src/mca/allocator/basic/allocator_basic.h

@@ -92,7 +92,7 @@ mca_allocator_base_module_t* mca_allocator_basic_component_init(
         mca_allocator_base_module_t * mem, 
         size_t size, 
         size_t align, 
-        struct mca_mpool_base_registration_t** registration);
+        mca_mpool_base_registration_t** registration);
 
 /**
    * Attempts to resize the passed region of memory into a larger or a smaller
@@ -112,7 +112,7 @@ mca_allocator_base_module_t* mca_allocator_basic_component_init(
         mca_allocator_base_module_t * mem, 
         void * ptr, 
         size_t size, 
-        struct mca_mpool_base_registration_t** registration);
+        mca_mpool_base_registration_t** registration);
 
 /**
    * Frees the passed region of memory

src/mca/allocator/bucket/allocator_bucket.c

@@ -19,6 +19,7 @@
 #include "include/constants.h"
 #include "mca/base/mca_base_param.h"
 #include "mca/allocator/bucket/allocator_bucket_alloc.h"
+#include "mca/mpool/mpool.h" 
 
 struct mca_allocator_base_module_t* mca_allocator_bucket_module_init(
     bool enable_mpi_threads,
@@ -34,7 +35,7 @@ int mca_allocator_bucket_module_close(void);
 void * mca_allocator_bucket_alloc_wrapper(
     struct mca_allocator_base_module_t* allocator,
     size_t size, size_t align, 
-    struct mca_mpool_base_registration_t** registration);
+    mca_mpool_base_registration_t** registration);
 
 static int mca_allocator_num_buckets;
 
@@ -91,7 +92,7 @@ void * mca_allocator_bucket_alloc_wrapper(
     struct mca_allocator_base_module_t* allocator,
     size_t size, 
     size_t align, 
-    struct mca_mpool_base_registration_t** registration)
+    mca_mpool_base_registration_t** registration)
 {
     if(0 == align){
         return(mca_allocator_bucket_alloc(allocator, size, registration));

src/mca/allocator/bucket/allocator_bucket_alloc.c

@@ -71,7 +71,7 @@ mca_allocator_bucket_t * mca_allocator_bucket_init(
 void * mca_allocator_bucket_alloc(
     mca_allocator_base_module_t * mem,
     size_t size, 
-    struct mca_mpool_base_registration_t** registration)
+    mca_mpool_base_registration_t** registration)
 {
     mca_allocator_bucket_t * mem_options = (mca_allocator_bucket_t *) mem;
     /* initialize for the later bit shifts */
@@ -154,7 +154,7 @@ void * mca_allocator_bucket_alloc_align(
     mca_allocator_base_module_t * mem, 
     size_t size, 
     size_t alignment, 
-    struct mca_mpool_base_registration_t** registration)
+    mca_mpool_base_registration_t** registration)
 {
     mca_allocator_bucket_t * mem_options = (mca_allocator_bucket_t *) mem; 
     int bucket_num = 1;
@@ -237,7 +237,7 @@ void * mca_allocator_bucket_realloc(
     mca_allocator_base_module_t * mem,
     void * ptr, 
     size_t size, 
-    struct mca_mpool_base_registration_t** registration)
+    mca_mpool_base_registration_t** registration)
 {
     mca_allocator_bucket_t * mem_options = (mca_allocator_bucket_t *) mem;
     /* initialize for later bit shifts */

src/mca/allocator/bucket/allocator_bucket_alloc.h

@@ -125,7 +125,7 @@ typedef struct mca_allocator_bucket_t mca_allocator_bucket_t;
     void * mca_allocator_bucket_alloc(
         mca_allocator_base_module_t * mem, 
         size_t size, 
-        struct mca_mpool_base_registration_t** registration);
+        mca_mpool_base_registration_t** registration);
 
 /**
    * Accepts a request for memory in a specific region defined by the
@@ -146,7 +146,7 @@ typedef struct mca_allocator_bucket_t mca_allocator_bucket_t;
         mca_allocator_base_module_t * mem,
         size_t size, 
         size_t alignment, 
-        struct mca_mpool_base_registration_t** registration);
+        mca_mpool_base_registration_t** registration);
 
 /**
    * Attempts to resize the passed region of memory into a larger or a smaller
@@ -166,7 +166,7 @@ typedef struct mca_allocator_bucket_t mca_allocator_bucket_t;
         mca_allocator_base_module_t * mem, 
         void * ptr, 
         size_t size, 
-        struct mca_mpool_base_registration_t** registration);
+        mca_mpool_base_registration_t** registration);
 
 /**
    * Frees the passed region of memory

src/mca/bmi/bmi.h

@@ -119,7 +119,6 @@
 struct mca_bmi_base_module_t;
 struct mca_bmi_base_endpoint_t;
 struct mca_bmi_base_descriptor_t;
-struct mca_mpool_base_registration_t;
 struct mca_mpool_base_resources_t;
 
 
@@ -184,9 +183,8 @@ typedef struct mca_bmi_base_descriptor_t mca_bmi_base_descriptor_t;
 OBJ_CLASS_DECLARATION(mca_bmi_base_descriptor_t);
 
                                                                                                                
-#define MCA_BMI_DES_FLAGS_PINNED   0x0001
+#define MCA_BMI_DES_FLAGS_DEREGISTER   0x0001
 #define MCA_BMI_DES_FLAGS_PRIORITY 0x0002
-#define MCA_BMI_DES_FLAGS_LEAVE_PINNED 0x003
 
 /**
  * Maximum number of allowed segments in src/dst fields of a descriptor.
@@ -410,7 +408,7 @@ typedef int (*mca_bmi_base_module_free_fn_t)(
 typedef struct mca_bmi_base_descriptor_t* (*mca_bmi_base_module_prepare_fn_t)(
     struct mca_bmi_base_module_t* bmi,
     struct mca_bmi_base_endpoint_t* endpoint,
-    struct mca_mpool_base_registration_t* registration,
+    mca_mpool_base_registration_t* registration,
     struct ompi_convertor_t* convertor,
     size_t reserve,
     size_t* size

src/mca/bmi/ib/bmi_ib.c

@@ -29,7 +29,7 @@
 #include "mca/common/vapi/vapi_mem_reg.h" 
 #include "mca/mpool/base/base.h" 
 #include "mca/mpool/mpool.h" 
-
+#include "mca/mpool/vapi/mpool_vapi.h" 
 
 mca_bmi_ib_module_t mca_bmi_ib_module = {
     {
@@ -192,7 +192,7 @@ int mca_bmi_ib_free(
         /* we also need to unregister the associated memory  iff 
             the memory wasn't allocated via MPI_Alloc_mem */
 
-        if(!(frag->base.des_flags && MCA_BMI_DES_FLAGS_PINNED)){ 
+        if(frag->base.des_flags & MCA_BMI_DES_FLAGS_DEREGISTER){ 
             frag->ret = VAPI_deregister_mr(
                                            ib_bmi->nic, 
                                            frag->mem_hndl
@@ -209,7 +209,7 @@ int mca_bmi_ib_free(
         MCA_BMI_IB_FRAG_RETURN_EAGER(bmi, frag); 
     } 
     
-    return frag->rc; 
+    return OMPI_SUCCESS; 
 }
 
 
@@ -223,7 +223,7 @@ int mca_bmi_ib_free(
 mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_src(
     struct mca_bmi_base_module_t* bmi,
     struct mca_bmi_base_endpoint_t* endpoint,
-    struct mca_mpool_base_registration_t* registration, 
+    mca_mpool_base_registration_t* registration, 
     struct ompi_convertor_t* convertor,
     size_t reserve,
     size_t* size
@@ -231,6 +231,7 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_src(
 {
     mca_bmi_ib_module_t* ib_bmi; 
     mca_bmi_ib_frag_t* frag; 
+    mca_mpool_vapi_registration_t * vapi_reg; 
     struct iovec iov; 
     int32_t iov_count = 1; 
     size_t max_data = *size; 
@@ -239,18 +240,127 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_src(
     
     
     ib_bmi = (mca_bmi_ib_module_t*) bmi; 
-    
+    vapi_reg = (mca_mpool_vapi_registration_t*) registration; 
+
     /** if the data fits in the eager limit and we aren't told to pinn then we 
         simply pack, if the data fits in the eager limit and the data is non contiguous 
         then we pack **/ 
 
-    if( (max_data+reserve <=  bmi->bmi_eager_limit  
+    
-         && NULL == registration 
+    if(NULL != vapi_reg &&  0 == ompi_convertor_need_buffers(convertor)){ 
-         && !mca_bmi_ib_component.leave_pinned) 
+        
-        || 
+        MCA_BMI_IB_FRAG_ALLOC_FRAG(bmi, frag, rc); 
-        (max_data+reserve <= bmi->bmi_eager_limit 
+        if(NULL == frag){
-         && 1 == ompi_convertor_need_buffers( convertor ))) { 
+            return NULL; 
+        } 
+        
+        iov.iov_len = max_data; 
+        iov.iov_base = NULL; 
+        
+        ompi_convertor_pack(convertor, &iov, &iov_count, &max_data, &free_after); 
+        
+        /* first we will try to find this address in the memory tree (from MPI_Alloc_mem) */ 
+        
+        frag->segment.seg_len = max_data; 
+        frag->segment.seg_addr.pval = iov.iov_base; 
+        
+        size_t reg_len; 
+        reg_len = (unsigned char*)vapi_reg->bound - (unsigned char*)iov.iov_base + 1; 
+        if(frag->segment.seg_len > reg_len) { 
+            
+            mca_mpool_base_remove((void*) vapi_reg->base); 
+            ompi_list_remove_item(&ib_bmi->reg_mru_list, (ompi_list_item_t*) vapi_reg); 
+            ib_bmi->ib_pool->mpool_deregister(
+                                              ib_bmi->ib_pool, 
+                                              vapi_reg->base, 
+                                              0, 
+                                              (mca_mpool_base_registration_t*) vapi_reg); 
+            
+            ib_bmi->ib_pool->mpool_register(ib_bmi->ib_pool, 
+                                            vapi_reg->base, 
+                                            vapi_reg->bound - vapi_reg->base + 1 + frag->segment.seg_len - reg_len, 
+                                            (mca_mpool_base_registration_t**) &vapi_reg);
+                
+            rc = mca_mpool_base_insert(iov.iov_base, 
+                                       iov.iov_len, 
+                                       ib_bmi->ib_pool, 
+                                       (void*) (&ib_bmi->super), 
+                                       (mca_mpool_base_registration_t*) vapi_reg); 
+            
+            if(rc != OMPI_SUCCESS) 
+                return NULL; 
+        
+            ompi_list_append(&ib_bmi->reg_mru_list, (ompi_list_item_t*) vapi_reg); 
+            
+        }   
+        
+        frag->mem_hndl = vapi_reg->hndl; 
+        frag->sg_entry.len = max_data; 
+        frag->sg_entry.lkey = vapi_reg->l_key; 
+        frag->sg_entry.addr = (VAPI_virt_addr_t) (MT_virt_addr_t) iov.iov_base; 
+        
+        frag->segment.seg_key.key32[0] = (uint32_t) vapi_reg->l_key; 
+        
+        frag->base.des_src = &frag->segment;
+        frag->base.des_src_cnt = 1;
+        frag->base.des_dst = NULL;
+        frag->base.des_dst_cnt = 0;
+        frag->base.des_flags = 0; 
 
+        return &frag->base;
+        
+    } else if((mca_bmi_ib_component.leave_pinned || max_data > bmi->bmi_max_send_size) && 
+               ompi_convertor_need_buffers(convertor) == 0 && 
+               reserve == 0)
+    {
+        MCA_BMI_IB_FRAG_ALLOC_FRAG(bmi, frag, rc); 
+        if(NULL == frag){
+            return NULL; 
+        } 
+       
+        iov.iov_len = max_data; 
+        iov.iov_base = NULL; 
+        
+        ompi_convertor_pack(convertor, &iov, &iov_count, &max_data, &free_after); 
+        
+        
+        frag->segment.seg_len = max_data; 
+        frag->segment.seg_addr.pval = iov.iov_base; 
+        frag->base.des_flags = 0; 
+
+        ib_bmi->ib_pool->mpool_register(ib_bmi->ib_pool,
+                                        iov.iov_base, 
+                                        max_data, 
+                                        (mca_mpool_base_registration_t**) &vapi_reg); 
+        
+        
+        if(mca_bmi_ib_component.leave_pinned) { 
+            rc = mca_mpool_base_insert(iov.iov_base, 
+                                       iov.iov_len, 
+                                       ib_bmi->ib_pool, 
+                                       (void*) (&ib_bmi->super), 
+                                       (mca_mpool_base_registration_t*) vapi_reg); 
+            if(rc != OMPI_SUCCESS) 
+                return NULL; 
+            
+        } else { 
+            frag->base.des_flags |= MCA_BMI_DES_FLAGS_DEREGISTER; 
+        } 
+        frag->mem_hndl = vapi_reg->hndl; 
+        frag->sg_entry.len = max_data; 
+        frag->sg_entry.lkey = vapi_reg->l_key; 
+        frag->sg_entry.addr = (VAPI_virt_addr_t) (MT_virt_addr_t) iov.iov_base; 
+        
+        frag->segment.seg_key.key32[0] = (uint32_t) vapi_reg->l_key; 
+            
+        frag->base.des_src = &frag->segment;
+        frag->base.des_src_cnt = 1;
+        frag->base.des_dst = NULL;
+        frag->base.des_dst_cnt = 0;
+
+        return &frag->base;
+    } else if (max_data+reserve <=  bmi->bmi_eager_limit) { 
+           
         MCA_BMI_IB_FRAG_ALLOC_EAGER(bmi, frag, rc); 
         if(NULL == frag) { 
             return NULL; 
@@ -277,14 +387,11 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_src(
         return &frag->base; 
         
     }
-    /** if the data fits in the max limit and we aren't told to pinn then we 
+       /** if the data fits in the max limit and we aren't told to pinn then we 
-        simply pack, if the data  is non contiguous then we pack **/ 
+           simply pack, if the data  is non contiguous then we pack **/ 
-
+       
-    else if((max_data + reserve <= ib_bmi->super.bmi_max_send_size 
+       else if(max_data + reserve <= ib_bmi->super.bmi_max_send_size) { 
-             && NULL == registration 
+           
-               && !mca_bmi_ib_component.leave_pinned)
-              || 1 == ompi_convertor_need_buffers( convertor)){ 
-        
         MCA_BMI_IB_FRAG_ALLOC_MAX(bmi, frag, rc); 
         if(NULL == frag) { 
             return NULL; 
@@ -312,94 +419,7 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_src(
         frag->base.des_flags=0; 
         
         return &frag->base; 
-    } else { 
-        
-        
-        VAPI_mrw_t mr_in, mr_out;
-        mca_mpool_base_chunk_t * mpool_chunk; 
-          
-          
-        memset(&mr_in, 0, sizeof(VAPI_mrw_t)); 
-        memset(&mr_out, 0, sizeof(VAPI_mrw_t)); 
-        
-        mpool_chunk = NULL; 
-
-
-        mr_in.acl = VAPI_EN_LOCAL_WRITE | VAPI_EN_REMOTE_WRITE;
-        mr_in.l_key = 0;
-        mr_in.r_key = 0;
-        mr_in.pd_hndl = ib_bmi->ptag; 
-        mr_in.type = VAPI_MR;
-        MCA_BMI_IB_FRAG_ALLOC_FRAG(bmi, frag, rc); 
-        if(NULL == frag){
-            return NULL; 
-        } 
-        
-        iov.iov_len = max_data; 
-        iov.iov_base = NULL; 
-        
-        ompi_convertor_pack(convertor, &iov, &iov_count, &max_data, &free_after); 
-        
-        /* first we will try to find this address in the memory tree (from MPI_Alloc_mem) */ 
-        
-        frag->segment.seg_len = max_data; 
-        frag->segment.seg_addr.pval = iov.iov_base; 
-                        
-        if(NULL != registration) { 
-            size_t reg_len; 
-            reg_len = (unsigned char*)registration->bound - (unsigned char*)iov.iov_base + 1; 
-            if(frag->segment.seg_len > reg_len) { 
-                
-                ib_bmi->ib_pool->mpool_deregister(
-                                                  ib_bmi->ib_pool, 
-                                                  registration->base, 
-                                                  0, 
-                                                  registration); 
-
-                mca_mpool_base_remove((void*) registration->base); 
-            
-                ib_bmi->ib_pool->mpool_register(ib_bmi->ib_pool, 
-                                                registration->base, 
-                                                registration->bound - registration->base + 1 + frag->segment.seg_len - reg_len, 
-                                                &registration); 
-            } 
-     
-            frag->base.des_flags |= MCA_BMI_DES_FLAGS_PINNED; 
-        }
-        else { 
-            ib_bmi->ib_pool->mpool_register(ib_bmi->ib_pool,
-                                            iov.iov_base, 
-                                            max_data, 
-                                            &registration); 
-            
-            if(mca_bmi_ib_component.leave_pinned) { 
-                rc = mca_mpool_base_insert(iov.iov_base, 
-                                           iov.iov_len, 
-                                           ib_bmi->ib_pool, 
-                                           (void*) (&ib_bmi->super), 
-                                           registration); 
-                if(rc != OMPI_SUCCESS) 
-                    return NULL; 
-                frag->base.des_flags |= MCA_BMI_DES_FLAGS_PINNED; 
-            }
-            
-        }
-        frag->mem_hndl = registration->hndl; 
-        frag->sg_entry.len = max_data; 
-        frag->sg_entry.lkey = registration->l_key; 
-        frag->sg_entry.addr = (VAPI_virt_addr_t) (MT_virt_addr_t) iov.iov_base; 
-        
-        frag->segment.seg_key.key32[0] = (uint32_t) registration->l_key; 
-          
-        frag->base.des_src = &frag->segment;
-        frag->base.des_src_cnt = 1;
-        frag->base.des_dst = NULL;
-        frag->base.des_dst_cnt = 0;
-        return &frag->base; 
-        
     }
-    
-    
     return NULL; 
 }
 
@@ -412,18 +432,20 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_src(
 mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_dst(
     struct mca_bmi_base_module_t* bmi,
     struct mca_bmi_base_endpoint_t* endpoint,
-    struct mca_mpool_base_registration_t* registration, 
+    mca_mpool_base_registration_t* registration, 
     struct ompi_convertor_t* convertor,
     size_t reserve,
     size_t* size)
 {
     mca_bmi_ib_module_t* ib_bmi; 
     mca_bmi_ib_frag_t* frag; 
+    mca_mpool_vapi_registration_t * vapi_reg; 
     int rc; 
     size_t reg_len; 
-    ib_bmi = (mca_bmi_ib_module_t*) bmi; 
-    
 
+    ib_bmi = (mca_bmi_ib_module_t*) bmi; 
+    vapi_reg = (mca_mpool_vapi_registration_t*) registration; 
+    
     MCA_BMI_IB_FRAG_ALLOC_FRAG(bmi, frag, rc); 
 
     if(NULL == frag){
@@ -433,51 +455,60 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_dst(
     
     frag->segment.seg_len = *size; 
     frag->segment.seg_addr.pval = convertor->pBaseBuf + convertor->bConverted; 
-    if(NULL!= registration){ 
+    frag->base.des_flags = 0; 
-        reg_len = (unsigned char*)registration->bound - (unsigned char*)frag->segment.seg_addr.pval + 1; 
+
+    if(NULL!= vapi_reg){ 
+        reg_len = (unsigned char*)vapi_reg->bound - (unsigned char*)frag->segment.seg_addr.pval + 1; 
         if(frag->segment.seg_len > reg_len) { 
             ib_bmi->ib_pool->mpool_deregister(
                                               ib_bmi->ib_pool, 
-                                              registration->base, 
+                                              vapi_reg->base, 
                                               0, 
-                                              registration); 
+                                              (mca_mpool_base_registration_t*) vapi_reg); 
-
+            
-            mca_mpool_base_remove((void*) registration->base); 
+            mca_mpool_base_remove((void*) vapi_reg->base); 
             
             ib_bmi->ib_pool->mpool_register(ib_bmi->ib_pool, 
-                                            registration->base, 
+                                            vapi_reg->base, 
-                                            registration->bound - registration->base + 1 + frag->segment.seg_len - reg_len, 
+                                            vapi_reg->bound - vapi_reg->base + 1 + frag->segment.seg_len - reg_len, 
-                                            &registration); 
+                                            (mca_mpool_base_registration_t**) &vapi_reg);
-        } 
+            
-     
-        frag->base.des_flags |= MCA_BMI_DES_FLAGS_PINNED; 
         
+            rc = mca_mpool_base_insert(frag->segment.seg_addr.pval, 
+                                       frag->segment.seg_len, 
+                                       ib_bmi->ib_pool, 
+                                       (void*) (&ib_bmi->super), 
+                                       (mca_mpool_base_registration_t*) vapi_reg); 
+            
+        } 
+                
     }  else { 
         ib_bmi->ib_pool->mpool_register(ib_bmi->ib_pool,
                                         frag->segment.seg_addr.pval,
                                         *size, 
-                                        &registration); 
+                                        (mca_mpool_base_registration_t**) &vapi_reg);
-        
+            
         if(mca_bmi_ib_component.leave_pinned) { 
             rc = mca_mpool_base_insert(frag->segment.seg_addr.pval,  
                                        *size, 
                                        ib_bmi->ib_pool, 
                                        (void*) (&ib_bmi->super), 
-                                       (void*) registration); 
+                                       (mca_mpool_base_registration_t*)  vapi_reg); 
             if(rc != OMPI_SUCCESS) 
                 return NULL; 
-            frag->base.des_flags |= MCA_BMI_DES_FLAGS_PINNED; 
         }
-            
+        else { 
+            frag->base.des_flags |= MCA_BMI_DES_FLAGS_DEREGISTER; 
+        }
     }
-    
+
-    frag->mem_hndl = registration->hndl; 
+    frag->mem_hndl = vapi_reg->hndl; 
     
     frag->sg_entry.len = *size; 
-    frag->sg_entry.lkey = registration->l_key; 
+    frag->sg_entry.lkey = vapi_reg->l_key; 
     frag->sg_entry.addr = (VAPI_virt_addr_t) (MT_virt_addr_t) frag->segment.seg_addr.pval; 
     
-    frag->segment.seg_key.key32[0] = (uint32_t) registration->l_key; 
+    frag->segment.seg_key.key32[0] = (uint32_t) vapi_reg->l_key; 
     
     frag->base.des_dst = &frag->segment; 
     frag->base.des_dst_cnt = 1; 

src/mca/bmi/ib/bmi_ib.h

@@ -93,7 +93,8 @@ struct mca_bmi_ib_component_t {
     size_t max_send_size; 
 
     uint32_t leave_pinned; 
-
+    uint32_t reg_mru_len; 
+    
     
 }; typedef struct mca_bmi_ib_component_t mca_bmi_ib_component_t;
 
@@ -129,6 +130,8 @@ struct mca_bmi_ib_module_t {
     ompi_free_list_t recv_free_eager;    /**< High priority free list of buffer descriptors */
     ompi_free_list_t recv_free_max;      /**< Low priority free list of buffer descriptors */ 
 
+    ompi_list_t reg_mru_list;   /**< a most recently used list of mca_mpool_vapi_registration_t 
+                                       entries, this allows us to keep a working set of memory pinned */ 
     
     ompi_list_t repost;            /**< list of buffers to repost */
     ompi_mutex_t ib_lock;          /**< module level lock */ 
@@ -342,7 +345,7 @@ extern int mca_bmi_ib_free(
 mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_src(
     struct mca_bmi_base_module_t* bmi,
     struct mca_bmi_base_endpoint_t* peer,
-    struct mca_mpool_base_registration_t* registration, 
+    mca_mpool_base_registration_t* registration, 
     struct ompi_convertor_t* convertor,
     size_t reserve,
     size_t* size
@@ -355,13 +358,12 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_src(
  * @param peer (IN)     BMI peer addressing
  */
 extern mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_dst( 
-    struct mca_bmi_base_module_t* bmi, 
+                                                         struct mca_bmi_base_module_t* bmi, 
-    struct mca_bmi_base_endpoint_t* peer,
+                                                         struct mca_bmi_base_endpoint_t* peer,
-    struct mca_mpool_base_registration_t* registration, 
+                                                         mca_mpool_base_registration_t* registration, 
-    struct ompi_convertor_t* convertor,
+                                                         struct ompi_convertor_t* convertor,
-    size_t reserve,
+                                                         size_t reserve,
-    size_t* size); 
+                                                         size_t* size); 
-
 /**
  * Return a send fragment to the modules free list.
  *

src/mca/bmi/ib/bmi_ib_component.c

@@ -36,6 +36,8 @@
 #include <vapi.h> 
 #include <vapi_common.h> 
 #include "datatype/convertor.h" 
+#include "mca/mpool/vapi/mpool_vapi.h" 
+
 mca_bmi_ib_component_t mca_bmi_ib_component = {
     {
         /* First, the mca_base_component_t struct containing meta information
@@ -126,6 +128,9 @@ int mca_bmi_ib_component_open(void)
         mca_bmi_ib_param_register_int("rr_buf_max", 16); 
     mca_bmi_ib_component.ib_rr_buf_min = 
         mca_bmi_ib_param_register_int("rr_buf_min", 8); 
+    mca_bmi_ib_component.reg_mru_len = 
+        mca_bmi_ib_param_register_int("reg_mru_len",  16); 
+    
     mca_bmi_ib_module.super.bmi_exclusivity =
         mca_bmi_ib_param_register_int ("exclusivity", 0);
     mca_bmi_ib_module.super.bmi_eager_limit = 
@@ -198,6 +203,7 @@ int mca_bmi_ib_component_open(void)
         mca_bmi_ib_param_register_int("flags", 
                                       MCA_BMI_FLAGS_RDMA); 
     
+    
     param = mca_base_param_find("mpi", NULL, "leave_pinned"); 
     mca_base_param_lookup_int(param, &value); 
     mca_bmi_ib_component.leave_pinned = value; 
@@ -244,7 +250,7 @@ mca_bmi_base_module_t** mca_bmi_ib_component_init(int *num_bmi_modules,
     uint32_t num_hcas; 
     mca_bmi_base_module_t** bmis;
     uint32_t i,j, length;
-    mca_mpool_base_resources_t hca_pd; 
+    struct mca_mpool_base_resources_t hca_pd; 
     ompi_list_t bmi_list; 
     mca_bmi_ib_module_t * ib_bmi; 
     mca_bmi_base_selected_module_t* ib_selected; 
@@ -366,7 +372,7 @@ mca_bmi_base_module_t** mca_bmi_ib_component_init(int *num_bmi_modules,
         
 
         OBJ_CONSTRUCT(&ib_bmi->repost, ompi_list_t);
-
+        OBJ_CONSTRUCT(&ib_bmi->reg_mru_list, ompi_list_t); 
         
       
 

src/mca/bmi/ib/bmi_ib_endpoint.c

@@ -561,15 +561,6 @@ int mca_bmi_ib_endpoint_send(
                 
                 rc = mca_bmi_ib_endpoint_post_send(ib_bmi, endpoint, frag); 
                 
-                
-                
-/*                 rc = mca_bmi_ib_post_send(endpoint->endpoint_bmi, endpoint, */
-/*                                           &frag->ib_buf, (void*) frag); */
-/*                 while(NULL != (item = ompi_list_remove_first(&ib_bmi->repost))) { */
-/*                     mca_bmi_ib_buffer_repost(ib_bmi->nic, item); */
-/*                 } */
-/*                 break; */
-
                 break; 
             }
 

src/mca/bmi/ib/bmi_ib_frag.c

@@ -1,12 +1,12 @@
 #include "bmi_ib_frag.h" 
 #include "mca/common/vapi/vapi_mem_reg.h"
-
+#include "mca/mpool/vapi/mpool_vapi.h" 
 
 
 
 static void mca_bmi_ib_frag_common_constructor( mca_bmi_ib_frag_t* frag) 
 {
-    mca_mpool_base_registration_t* mem_hndl = frag->base.super.user_data; 
+    mca_mpool_vapi_registration_t* mem_hndl = (mca_mpool_vapi_registration_t*) frag->base.super.user_data; 
     frag->hdr = (mca_bmi_ib_header_t*) (frag+1);    /* initialize the bmi header to point to start at end of frag */ 
 #if 0   
     mod = (unsigned long) frag->hdr % MCA_BMI_IB_FRAG_ALIGN; 
@@ -31,9 +31,6 @@ static void mca_bmi_ib_frag_common_constructor( mca_bmi_ib_frag_t* frag)
     frag->sg_entry.lkey = mem_hndl->l_key; 
     frag->sg_entry.addr = (VAPI_virt_addr_t) (MT_virt_addr_t) frag->hdr; 
     frag->base.des_flags = 0; 
-    if(mca_bmi_ib_component.leave_pinned == MCA_BMI_IB_LEAVE_PINNED) { 
-        frag->base.des_flags |= MCA_BMI_DES_FLAGS_LEAVE_PINNED; 
-    } 
 
 }
 

src/mca/bmi/self/bmi_self.c

@@ -186,7 +186,7 @@ extern int mca_bmi_self_free(
 struct mca_bmi_base_descriptor_t* mca_bmi_self_prepare_src(
     struct mca_bmi_base_module_t* bmi,
     struct mca_bmi_base_endpoint_t* endpoint,
-    struct mca_mpool_base_registration_t* registration,
+    mca_mpool_base_registration_t* registration,
     struct ompi_convertor_t* convertor,
     size_t reserve,
     size_t* size)
@@ -252,7 +252,7 @@ struct mca_bmi_base_descriptor_t* mca_bmi_self_prepare_src(
 struct mca_bmi_base_descriptor_t* mca_bmi_self_prepare_dst(
     struct mca_bmi_base_module_t* bmi,
     struct mca_bmi_base_endpoint_t* endpoint,
-    struct mca_mpool_base_registration_t* registration,
+    mca_mpool_base_registration_t* registration,
     struct ompi_convertor_t* convertor,
     size_t reserve,
     size_t* size)

src/mca/bmi/sm/bmi_sm.c

@@ -776,7 +776,7 @@ extern int mca_bmi_sm_free(
 struct mca_bmi_base_descriptor_t* mca_bmi_sm_prepare_src(
     struct mca_bmi_base_module_t* bmi,
     struct mca_bmi_base_endpoint_t* endpoint,
-    struct mca_mpool_base_registration_t* registration,
+    mca_mpool_base_registration_t* registration,
     struct ompi_convertor_t* convertor,
     size_t reserve,
     size_t* size)

src/mca/bmi/sm/bmi_sm.h

@@ -300,7 +300,7 @@ extern int mca_bmi_sm_free(
 struct mca_bmi_base_descriptor_t* mca_bmi_sm_prepare_src(
     struct mca_bmi_base_module_t* bmi,
     struct mca_bmi_base_endpoint_t* endpoint,
-    struct mca_mpool_base_registration_t* registration,
+    mca_mpool_base_registration_t* registration,
     struct ompi_convertor_t* convertor,
     size_t reserve,
     size_t* size

src/mca/common/sm/common_sm_mmap.c

@@ -216,7 +216,7 @@ mca_common_sm_mmap_t* mca_common_sm_mmap_init(size_t size, char *file_name,
 void* mca_common_sm_mmap_seg_alloc(
     struct mca_mpool_base_module_t* mpool,
     size_t* size,
-    struct mca_mpool_base_registration_t** registration)
+    mca_mpool_base_registration_t** registration)
 {
     mca_common_sm_mmap_t* map = mca_common_sm_mmap;
     mca_common_sm_file_header_t* seg = map->map_seg;

src/mca/common/sm/common_sm_mmap.h

@@ -22,12 +22,13 @@
 #include "class/ompi_object.h"
 #include "class/ompi_list.h"
 #include "include/sys/atomic.h"
+#include "mca/mpool/mpool.h" 
+
 #if defined(c_plusplus) || defined(__cplusplus)
 extern "C" {
 #endif
 
 struct mca_mpool_base_module_t;
-struct mca_mpool_base_registration_t;
 
 struct mca_common_sm_file_header_t {
 
@@ -99,7 +100,7 @@ extern mca_common_sm_mmap_t* mca_common_sm_mmap_init(
 extern void* mca_common_sm_mmap_seg_alloc(
     struct mca_mpool_base_module_t* mpool, 
     size_t* size, 
-    struct mca_mpool_base_registration_t** registration);
+    mca_mpool_base_registration_t** registration);
 
 /*
  * Instance that is shared between components that use shared memory

src/mca/common/vapi/vapi_mem_reg.h

@@ -26,31 +26,6 @@
 extern "C" {
 #endif
 
-
-struct mca_mpool_base_resources_t {
-  VAPI_hca_hndl_t hca;   /* the hca (nic) */ 
-  VAPI_pd_hndl_t pd_tag; /* the protection domain */ 
-}; 
-typedef struct mca_mpool_base_resources_t mca_mpool_base_resources_t;  
-
-
-struct mca_mpool_base_registration_t {
-    VAPI_mr_hndl_t                  hndl;
-    /* Memory region handle */
-    
-    VAPI_lkey_t                     l_key;
-    /* Local key to registered memory, needed for
-     * posting send/recv requests */
-    
-    VAPI_rkey_t                     r_key;
-    /* Remote key to registered memory, need to send this
-     * to remote processes for incoming RDMA ops */
-    void * base; 
-    void * bound; 
-
-};
-typedef struct mca_mpool_base_registration_t mca_mpool_base_registration_t;
-
 #if defined(c_plusplus) || defined(__cplusplus)
 }
 #endif

src/mca/mpool/base/base.h

@@ -69,7 +69,7 @@ struct mca_mpool_base_reg_mpool_t
 {
     mca_mpool_base_module_t * mpool; /**< the registered memory pool */
     void* user_data; /**< user data */
-    struct mca_mpool_base_registration_t* mpool_registration; /**< mpool specific info associated w/ registration */
+    mca_mpool_base_registration_t* mpool_registration; /**< mpool specific info associated w/ registration */
 };
 typedef struct mca_mpool_base_reg_mpool_t mca_mpool_base_reg_mpool_t;
 

src/mca/mpool/base/mpool_base_alloc.c

@@ -29,6 +29,10 @@ ompi_rb_tree_t mca_mpool_base_tree;
 ompi_free_list_t mca_mpool_base_mem_list;
 ompi_mutex_t mca_mpool_base_tree_lock; 
 
+OBJ_CLASS_INSTANCE(mca_mpool_base_registration_t, 
+                   ompi_list_item_t, 
+                   NULL, 
+                   NULL); 
 
 /**
  * Function for the red black tree to compare 2 keys
@@ -59,9 +63,9 @@ int mca_mpool_base_tree_node_compare(void * key1, void * key2)
 }
 
 int mca_mpool_base_insert(void * addr, size_t size, 
-    mca_mpool_base_module_t* mpool, 
+                          mca_mpool_base_module_t* mpool, 
-    void* user_data,
+                          void* user_data,
-    struct mca_mpool_base_registration_t* registration)
+                          mca_mpool_base_registration_t* registration)
 {
     ompi_list_item_t *item; 
     int rc; 
@@ -143,7 +147,7 @@ void * mca_mpool_base_alloc(size_t size, ompi_info_t * info)
     mca_mpool_base_selected_module_t * no_reg_function = NULL;
     mca_mpool_base_selected_module_t ** has_reg_function = (mca_mpool_base_selected_module_t **) 
                            malloc(num_modules * sizeof(mca_mpool_base_module_t *));
-    struct mca_mpool_base_registration_t * registration;
+    mca_mpool_base_registration_t * registration;
     void * mem = NULL;
     char * key;
     bool match_found;

src/mca/mpool/mpool.h

@@ -21,9 +21,16 @@
 #define MCA_MPOOL_H
 #include "mca/mca.h"
 #include "info/info.h"
+#include "class/ompi_list.h" 
 
 struct mca_mpool_base_resources_t;
-struct mca_mpool_base_registration_t;
+
+struct mca_mpool_base_registration_t{ 
+    ompi_list_item_t super; 
+    struct mca_mpool_base_module_t *mpool; 
+};  
+typedef struct mca_mpool_base_registration_t mca_mpool_base_registration_t; 
+OBJ_CLASS_DECLARATION(mca_mpool_base_registration_t); 
 
 /**
  * component initialize
@@ -38,7 +45,7 @@ typedef void* (*mca_mpool_base_module_alloc_fn_t)(
     struct mca_mpool_base_module_t* mpool,
     size_t size,
     size_t align,
-    struct mca_mpool_base_registration_t** registration);
+    mca_mpool_base_registration_t** registration);
                                                                                                                                    
 /**
   * realloc function typedef
@@ -47,7 +54,7 @@ typedef void* (*mca_mpool_base_module_realloc_fn_t)(
     struct mca_mpool_base_module_t* mpool,
     void* addr,
     size_t size,
-    struct mca_mpool_base_registration_t** registration);
+    mca_mpool_base_registration_t** registration);
                                                                                                                                    
 /**
   * free function typedef
@@ -55,7 +62,7 @@ typedef void* (*mca_mpool_base_module_realloc_fn_t)(
 typedef void (*mca_mpool_base_module_free_fn_t)(
     struct mca_mpool_base_module_t* mpool,
     void *addr, 
-    struct mca_mpool_base_registration_t* registration);
+    mca_mpool_base_registration_t* registration);
                                                                                                                                    
 /**
   * register memory
@@ -64,7 +71,7 @@ typedef int (*mca_mpool_base_module_register_fn_t)(
     struct mca_mpool_base_module_t* mpool,
     void * addr,
     size_t size,
-    struct mca_mpool_base_registration_t** registration);
+    mca_mpool_base_registration_t** registration);
     
 /**
   * deregister memory
@@ -73,7 +80,7 @@ typedef int (*mca_mpool_base_module_deregister_fn_t)(
     struct mca_mpool_base_module_t* mpool,
     void * addr,
     size_t size, 
-    struct mca_mpool_base_registration_t* registration);
+    mca_mpool_base_registration_t* registration);
 
 /**
   * if appropriate - returns base address of memory pool
@@ -125,6 +132,7 @@ struct mca_mpool_base_module_t {
  */
 typedef struct mca_mpool_base_module_t mca_mpool_base_module_t;
 
+
 /**
  * Function to allocate special memory according to what the user requests in
  * the info object.
@@ -145,7 +153,7 @@ typedef struct mca_mpool_base_module_t mca_mpool_base_module_t;
  * @retval pointer to the allocated memory
  * @retval NULL on failure
  */
-OMPI_DECLSPEC void * mca_mpool_base_alloc(size_t size, ompi_info_t * info);
+OMPI_DECLSPEC void * mca_mpool_base_alloc(size_t size, struct ompi_info_t * info);
 
 /**
  * Function to free memory previously allocated by mca_mpool_base_alloc
@@ -186,7 +194,7 @@ OMPI_DECLSPEC int mca_mpool_base_insert(
     size_t size, 
     mca_mpool_base_module_t* mpool, 
     void* user_in, 
-    struct mca_mpool_base_registration_t* registration); 
+    mca_mpool_base_registration_t* registration); 
 
 OMPI_DECLSPEC int mca_mpool_base_remove(void * base); 
 

src/mca/mpool/sm/mpool_sm.h

@@ -66,7 +66,7 @@ void* mca_mpool_sm_alloc(
     mca_mpool_base_module_t* mpool, 
     size_t size, 
     size_t align, 
-    struct mca_mpool_base_registration_t** registration);
+    mca_mpool_base_registration_t** registration);
 
 /**
   * realloc function typedef
@@ -75,7 +75,7 @@ void* mca_mpool_sm_realloc(
     mca_mpool_base_module_t* mpool, 
     void* addr, 
     size_t size, 
-    struct mca_mpool_base_registration_t** registration);
+    mca_mpool_base_registration_t** registration);
 
 /**
   * free function typedef
@@ -83,7 +83,7 @@ void* mca_mpool_sm_realloc(
 void mca_mpool_sm_free(
     mca_mpool_base_module_t* mpool, 
     void * addr, 
-    struct mca_mpool_base_registration_t* registration);
+    mca_mpool_base_registration_t* registration);
 
 
 #if defined(c_plusplus) || defined(__cplusplus)

src/mca/mpool/sm/mpool_sm_module.c

@@ -52,7 +52,7 @@ void* mca_mpool_sm_alloc(
     mca_mpool_base_module_t* mpool, 
     size_t size, 
     size_t align, 
-    struct mca_mpool_base_registration_t** registration)
+    mca_mpool_base_registration_t** registration)
 {
     mca_mpool_sm_module_t* mpool_sm = (mca_mpool_sm_module_t*)mpool; 
     return mpool_sm->sm_allocator->alc_alloc(mpool_sm->sm_allocator, size, align, registration);
@@ -65,7 +65,7 @@ void* mca_mpool_sm_realloc(
     mca_mpool_base_module_t* mpool, 
     void* addr, 
     size_t size, 
-    struct mca_mpool_base_registration_t** registration)
+    mca_mpool_base_registration_t** registration)
 {
     mca_mpool_sm_module_t* mpool_sm = (mca_mpool_sm_module_t*)mpool; 
     return mpool_sm->sm_allocator->alc_realloc(mpool_sm->sm_allocator, addr, size, registration);
@@ -75,7 +75,7 @@ void* mca_mpool_sm_realloc(
   * free function 
   */
 void mca_mpool_sm_free(mca_mpool_base_module_t* mpool, void * addr, 
-                       struct mca_mpool_base_registration_t* registration)
+                       mca_mpool_base_registration_t* registration)
 {
     mca_mpool_sm_module_t* mpool_sm = (mca_mpool_sm_module_t*)mpool; 
     mpool_sm->sm_allocator->alc_free(mpool_sm->sm_allocator, addr);

src/mca/mpool/vapi/mpool_vapi.h

@@ -52,12 +52,45 @@ typedef struct mca_mpool_vapi_component_t mca_mpool_vapi_component_t;
 
 OMPI_COMP_EXPORT extern mca_mpool_vapi_component_t mca_mpool_vapi_component;
 
+
+
+
+
+struct mca_mpool_base_resources_t {
+  VAPI_hca_hndl_t hca;   /* the hca (nic) */ 
+  VAPI_pd_hndl_t pd_tag; /* the protection domain */ 
+}; 
+typedef struct mca_mpool_base_resources_t mca_mpool_base_resources_t;  
+
 struct mca_mpool_vapi_module_t {
     mca_mpool_base_module_t super;
     mca_allocator_base_module_t * vapi_allocator; 
-    mca_mpool_base_resources_t  hca_pd;
+    struct mca_mpool_base_resources_t  hca_pd;
 }; typedef struct mca_mpool_vapi_module_t mca_mpool_vapi_module_t; 
 
+
+struct mca_mpool_vapi_registration_t {
+    mca_mpool_base_registration_t base_reg; 
+    VAPI_mr_hndl_t                  hndl;
+    /* Memory region handle */
+    
+    VAPI_lkey_t                     l_key;
+    /* Local key to registered memory, needed for
+     * posting send/recv requests */
+    
+    VAPI_rkey_t                     r_key;
+    /* Remote key to registered memory, need to send this
+     * to remote processes for incoming RDMA ops */
+    void * base; 
+    void * bound; 
+
+};
+typedef struct mca_mpool_vapi_registration_t mca_mpool_vapi_registration_t;
+OBJ_CLASS_DECLARATION(mca_mpool_vapi_registration_t); 
+
+
+
+
 /* 
  *  Initializes the mpool module. 
  */ 
@@ -76,7 +109,7 @@ void* mca_mpool_vapi_alloc(
     mca_mpool_base_module_t* mpool, 
     size_t size, 
     size_t align, 
-    struct mca_mpool_base_registration_t** registration);
+    mca_mpool_base_registration_t** registration);
 
 /**
   * realloc function typedef
@@ -85,7 +118,7 @@ void* mca_mpool_vapi_realloc(
     mca_mpool_base_module_t* mpool, 
     void* addr, 
     size_t size, 
-    struct mca_mpool_base_registration_t** registration);
+    mca_mpool_base_registration_t** registration);
 
 /**
   * register function typedef
@@ -94,26 +127,26 @@ int mca_mpool_vapi_register(
     mca_mpool_base_module_t* mpool, 
     void *addr, 
     size_t size, 
-    struct mca_mpool_base_registration_t** registration);
+    mca_mpool_base_registration_t** registration);
 
 int mca_mpool_vapi_deregister(
     mca_mpool_base_module_t* mpool, 
     void *addr, 
     size_t size, 
-    struct mca_mpool_base_registration_t* );
+    mca_mpool_base_registration_t* );
 
 
 /**
   * free function typedef
   */
 void mca_mpool_vapi_free(mca_mpool_base_module_t* mpool, 
-    void * addr, 
+                         void * addr, 
-    struct mca_mpool_base_registration_t* registration);
+                         mca_mpool_base_registration_t* registration);
 
 void* mca_common_vapi_segment_alloc(
-    struct mca_mpool_base_module_t* module, 
+                                    struct mca_mpool_base_module_t* module, 
-    size_t* size, 
+                                    size_t* size, 
-    struct mca_mpool_base_registration_t** registration);
+                                    mca_mpool_base_registration_t** registration);
 
 #if defined(c_plusplus) || defined(__cplusplus)
 }

src/mca/mpool/vapi/mpool_vapi_component.c

@@ -61,6 +61,38 @@ mca_mpool_vapi_component_t mca_mpool_vapi_component = {
     }
 };
 
+
+
+static void mca_mpool_vapi_registration_constructor( mca_mpool_vapi_registration_t * registration ) 
+{ 
+    memset(registration, 0, sizeof(mca_mpool_base_registration_t)); 
+}
+
+static void mca_mpool_vapi_registration_destructor( mca_mpool_vapi_registration_t * registration ) 
+{ 
+    mca_mpool_base_remove((void*) registration); 
+    registration->base_reg.mpool->mpool_deregister(
+                                                   registration->base_reg.mpool, 
+                                                   registration->base, 
+                                                   0, 
+                                                   (mca_mpool_base_registration_t*) registration); 
+    
+    registration->base = NULL; 
+    registration->bound = NULL; 
+
+} 
+
+
+OBJ_CLASS_INSTANCE( 
+                   mca_mpool_vapi_registration_t, 
+                   mca_mpool_base_registration_t, 
+                   mca_mpool_vapi_registration_constructor, 
+                   mca_mpool_vapi_registration_destructor
+                   ); 
+
+
+
+
 static char* mca_mpool_vapi_param_register_string(
     const char* param_name,
     const char* default_value)
@@ -91,7 +123,7 @@ static int mca_mpool_vapi_open(void)
 void* mca_common_vapi_segment_alloc(
     struct mca_mpool_base_module_t* mpool,
     size_t* size, 
-    struct mca_mpool_base_registration_t** registration)
+    mca_mpool_base_registration_t** registration)
 {
     void* addr_malloc = (void*)malloc((*size) + mca_mpool_vapi_component.page_size); 
     void* addr = (void*)  ALIGN_ADDR(addr_malloc, mca_mpool_vapi_component.page_size_log); 

src/mca/mpool/vapi/mpool_vapi_module.c

@@ -44,7 +44,7 @@ void* mca_mpool_vapi_alloc(
     mca_mpool_base_module_t* mpool, 
     size_t size, 
     size_t align, 
-    struct mca_mpool_base_registration_t** registration)
+    mca_mpool_base_registration_t** registration)
 {
     mca_mpool_vapi_module_t* mpool_vapi = (mca_mpool_vapi_module_t*)mpool; 
     return mpool_vapi->vapi_allocator->alc_alloc(mpool_vapi->vapi_allocator, size, align, registration);
@@ -54,22 +54,27 @@ void* mca_mpool_vapi_alloc(
 /* 
  * register memory 
  */ 
-int mca_mpool_vapi_register(mca_mpool_base_module_t* mpool, void *addr, size_t size, 
+int mca_mpool_vapi_register(mca_mpool_base_module_t* mpool, 
-    struct mca_mpool_base_registration_t** registration){
+                            void *addr, 
+                            size_t size, 
+                            mca_mpool_base_registration_t** registration){
+    
     
     mca_mpool_vapi_module_t * mpool_module = (mca_mpool_vapi_module_t*) mpool; 
+    mca_mpool_vapi_registration_t * vapi_reg; 
     VAPI_mrw_t mr_in, mr_out;
-  
+    
     VAPI_ret_t ret; 
-    mca_mpool_base_registration_t* mem_hndl; 
+    
     memset(&mr_in, 0, sizeof(VAPI_mrw_t)); 
     memset(&mr_out, 0, sizeof(VAPI_mrw_t)); 
     
-
+    *registration = (mca_mpool_base_registration_t*) OBJ_NEW(mca_mpool_vapi_registration_t);    /* (void*) malloc(sizeof(mca_mpool_base_registration_t));  */
-    *registration = (void*) malloc(sizeof(mca_mpool_base_registration_t)); 
+    vapi_reg = (mca_mpool_vapi_registration_t*) *registration; 
-    mem_hndl = (mca_mpool_base_registration_t*) *registration;  
+    vapi_reg->base_reg.mpool = mpool;
-    memset(mem_hndl, 0, sizeof(mca_mpool_base_registration_t*)); 
+    
-    mem_hndl->hndl = VAPI_INVAL_HNDL; 
+    
+    vapi_reg->hndl = VAPI_INVAL_HNDL; 
     
     
     mr_in.acl = VAPI_EN_LOCAL_WRITE | VAPI_EN_REMOTE_WRITE;
@@ -84,7 +89,7 @@ int mca_mpool_vapi_register(mca_mpool_base_module_t* mpool, void *addr, size_t s
     ret = VAPI_register_mr(
                            mpool_module->hca_pd.hca, 
                            &mr_in, 
-                           &mem_hndl->hndl, 
+                           &vapi_reg->hndl, 
                            &mr_out
                            ); 
     
@@ -93,10 +98,10 @@ int mca_mpool_vapi_register(mca_mpool_base_module_t* mpool, void *addr, size_t s
         return OMPI_ERROR; 
     }
     
-    mem_hndl->l_key = mr_out.l_key; 
+    vapi_reg->l_key = mr_out.l_key; 
-    mem_hndl->r_key = mr_out.r_key; 
+    vapi_reg->r_key = mr_out.r_key; 
-    mem_hndl->base = addr; 
+    vapi_reg->base = addr; 
-    mem_hndl->bound = (void*) ((char*) addr + size - 1); 
+    vapi_reg->bound = (void*) ((char*) addr + size - 1); 
     
     return OMPI_SUCCESS; 
 }
@@ -106,14 +111,15 @@ int mca_mpool_vapi_register(mca_mpool_base_module_t* mpool, void *addr, size_t s
  * deregister memory 
  */ 
 int mca_mpool_vapi_deregister(mca_mpool_base_module_t* mpool, void *addr, size_t size, 
-                              struct mca_mpool_base_registration_t* registration){
+                              mca_mpool_base_registration_t* registration){
     
     VAPI_ret_t ret; 
     mca_mpool_vapi_module_t * mpool_vapi = (mca_mpool_vapi_module_t*) mpool; 
-
+    mca_mpool_vapi_registration_t * vapi_reg; 
+    vapi_reg = (mca_mpool_vapi_registration_t*) registration; 
     ret = VAPI_deregister_mr(
                              mpool_vapi->hca_pd.hca, 
-                             registration->hndl 
+                             vapi_reg->hndl 
                              ); 
     
     if(VAPI_OK != ret){ 
@@ -131,7 +137,7 @@ void* mca_mpool_vapi_realloc(
     mca_mpool_base_module_t* mpool, 
     void* addr, 
     size_t size, 
-    struct mca_mpool_base_registration_t** registration)
+    mca_mpool_base_registration_t** registration)
 {
     mca_mpool_vapi_module_t* mpool_vapi = (mca_mpool_vapi_module_t*)mpool; 
     return mpool_vapi->vapi_allocator->alc_realloc( mpool_vapi->vapi_allocator, addr, size, registration);
@@ -141,7 +147,7 @@ void* mca_mpool_vapi_realloc(
   * free function 
   */
 void mca_mpool_vapi_free(mca_mpool_base_module_t* mpool, void * addr,
-                         struct mca_mpool_base_registration_t* registration)
+                         mca_mpool_base_registration_t* registration)
 {
     
     mca_mpool_vapi_module_t* mpool_vapi = (mca_mpool_vapi_module_t*)mpool; 

src/mca/pml/base/pml_base_bsend.c

@@ -24,7 +24,9 @@
 #include "mca/pml/base/pml_base_request.h"
 #include "mca/pml/base/pml_base_sendreq.h"
 #include "mca/pml/base/pml_base_bsend.h"
- 
+#include "mca/mpool/mpool.h" 
+
+
 static ompi_mutex_t     mca_pml_bsend_mutex;      /* lock for thread safety */
 static ompi_condition_t mca_pml_bsend_condition;  /* condition variable to block on detach */
 static mca_allocator_base_component_t* mca_pml_bsend_allocator_component;  
@@ -45,7 +47,7 @@ static int32_t          mca_pml_bsend_init = 0;
 static void* mca_pml_bsend_alloc_segment(
     struct mca_mpool_base_module_t* module, 
     size_t* size_inout, 
-    struct mca_mpool_base_registration_t** registration)
+    mca_mpool_base_registration_t** registration)
 {
     void *addr;
     size_t size = *size_inout;

src/mca/pml/ob1/pml_ob1_recvreq.c

@@ -18,6 +18,7 @@
 
 #include "mca/pml/pml.h"
 #include "mca/bmi/bmi.h"
+#include "mca/mpool/mpool.h" 
 #include "pml_ob1_comm.h"
 #include "pml_ob1_recvreq.h"
 #include "pml_ob1_recvfrag.h"
@@ -388,7 +389,7 @@ void mca_pml_ob1_recv_request_schedule(mca_pml_ob1_recv_request_t* recvreq)
                     recvreq->pin2[recvreq->pin_index] = get_profiler_timestamp();
 #endif
                 } else {
-                    struct mca_mpool_base_registration_t * reg; 
+                    mca_mpool_base_registration_t * reg; 
                     size = bytes_remaining;
 
                     /* prepare a descriptor for RDMA */

src/mca/pml/ob1/pml_ob1_sendreq.c

@@ -23,6 +23,7 @@
 #include "mca/pml/pml.h"
 #include "mca/bmi/bmi.h"
 #include "mca/errmgr/errmgr.h"
+#include "mca/mpool/mpool.h" 
 #include "pml_ob1.h"
 #include "pml_ob1_hdr.h"
 #include "pml_ob1_proc.h"
@@ -593,7 +594,7 @@ void mca_pml_ob1_send_request_put(
 { 
     mca_pml_ob1_proc_t* proc = sendreq->req_proc;
     mca_pml_ob1_endpoint_t* ep = mca_pml_ob1_ep_array_find(&proc->bmi_rdma,bmi);
-    struct mca_mpool_base_registration_t* reg = NULL;
+    mca_mpool_base_registration_t* reg = NULL;
     mca_bmi_base_descriptor_t* des;
     mca_pml_ob1_rdma_frag_t* frag;
     size_t offset = hdr->hdr_rdma_offset;

src/mca/ptl/sm/src/ptl_sm.c

@@ -795,7 +795,8 @@ int mca_ptl_sm_send(
     /* if needed, pack data in payload buffer */
     if( 0 < size ) {
         ompi_convertor_t *convertor;
-        unsigned int iov_count, max_data;
+        unsigned int iov_count;
+        size_t max_data; 
         int free_after=0;
         struct iovec address;
 
@@ -809,7 +810,7 @@ int mca_ptl_sm_send(
         iov_count=1;
         max_data=address.iov_len;
         return_status=ompi_convertor_pack(convertor,&address,&iov_count,
-                &max_data, &free_after);
+                                          &max_data, &free_after);
         if( 0 > return_status ) {
             return OMPI_ERROR;
         }
@@ -913,7 +914,8 @@ int mca_ptl_sm_send_continue(
     mca_ptl_sm_second_frag_t *send_frag;
     ompi_convertor_t *convertor;
     struct iovec address;
-    unsigned int max_data,iov_count;
+    unsigned int iov_count;
+    size_t max_data; 
 
     /* cast to shared memory send descriptor */
     sm_request=(mca_ptl_sm_send_request_t *)sendreq;

src/mca/ptl/sm/src/ptl_sm_send.c

@@ -71,9 +71,9 @@ void mca_ptl_sm_matched(
     mca_ptl_sm_frag_t *sm_frag_desc;
     struct iovec iov; 
     ompi_convertor_t frag_convertor;
-    ompi_proc_t *proc;
     int  free_after,my_local_smp_rank,peer_local_smp_rank, return_status;
-    unsigned int iov_count, max_data;
+    unsigned int iov_count; 
+    size_t max_data;
     ompi_fifo_t *send_fifo;
 
     /* copy data from shared memory buffer to user buffer */
@@ -106,7 +106,6 @@ void mca_ptl_sm_matched(
         iov_count = 1;
         max_data = iov.iov_len;
         ompi_convertor_unpack( &frag_convertor, &iov, &iov_count, &max_data, &free_after ); 
-        OBJ_DESTRUCT(&frag_convertor);
     }
 
     /* update receive request information */