Removed allocator from mpool_mvapi, moved is_leave_pinned to mpool_base,

corrected free and realloc in mpool. Added alloc_base to
mca_mpool_base_registration_t to be used as the actual alloc'd base address,
which may be different from the reported base address due to page allignment. 

This commit was SVN r6524.

ompi/mca/btl/mvapi/btl_mvapi.c

@@ -234,7 +234,7 @@ mca_btl_base_descriptor_t* mca_btl_mvapi_prepare_src(
 
     
     if(NULL != vapi_reg &&  0 == ompi_convertor_need_buffers(convertor)){ 
-        bool is_leave_pinned = vapi_reg->is_leave_pinned; 
+        bool is_leave_pinned = vapi_reg->base_reg.is_leave_pinned; 
         MCA_BTL_IB_FRAG_ALLOC_FRAG(btl, frag, rc); 
         if(NULL == frag){
             return NULL; 
@@ -292,7 +292,7 @@ mca_btl_base_descriptor_t* mca_btl_mvapi_prepare_src(
 
             OBJ_RETAIN(vapi_reg); 
             if(is_leave_pinned) {
-                vapi_reg->is_leave_pinned = is_leave_pinned; 
+                vapi_reg->base_reg.is_leave_pinned = is_leave_pinned; 
                 opal_list_append(&mvapi_btl->reg_mru_list, (opal_list_item_t*) vapi_reg);
             } 
         }   
@@ -378,7 +378,7 @@ mca_btl_base_descriptor_t* mca_btl_mvapi_prepare_src(
                 return NULL; 
             OBJ_RETAIN(vapi_reg); 
             
-            vapi_reg->is_leave_pinned = true; 
+            vapi_reg->base_reg.is_leave_pinned = true; 
                     
             opal_list_append(&mvapi_btl->reg_mru_list, (opal_list_item_t*) vapi_reg);
             
@@ -388,7 +388,7 @@ mca_btl_base_descriptor_t* mca_btl_mvapi_prepare_src(
                                             max_data, 
                                             (mca_mpool_base_registration_t**) &vapi_reg); 
             
-            vapi_reg->is_leave_pinned = false; 
+            vapi_reg->base_reg.is_leave_pinned = false; 
         } 
         /* frag->mem_hndl = vapi_reg->hndl;  */
         frag->sg_entry.len = max_data; 
@@ -505,7 +505,7 @@ mca_btl_base_descriptor_t* mca_btl_mvapi_prepare_dst(
 
     if(NULL!= vapi_reg){ 
         reg_len = (unsigned char*)vapi_reg->base_reg.bound - (unsigned char*)frag->segment.seg_addr.pval + 1; 
-        bool is_leave_pinned = vapi_reg->is_leave_pinned; 
+        bool is_leave_pinned = vapi_reg->base_reg.is_leave_pinned; 
 
         if(frag->segment.seg_len > reg_len ) { 
             size_t new_len = vapi_reg->base_reg.bound - vapi_reg->base_reg.base + 1 
@@ -545,7 +545,7 @@ mca_btl_base_descriptor_t* mca_btl_mvapi_prepare_dst(
             OBJ_RETAIN(vapi_reg); 
             
             if(is_leave_pinned) { 
-                vapi_reg->is_leave_pinned = is_leave_pinned; 
+                vapi_reg->base_reg.is_leave_pinned = is_leave_pinned; 
                 opal_list_append(&mvapi_btl->reg_mru_list, (opal_list_item_t*) vapi_reg);
             } 
 
@@ -587,7 +587,7 @@ mca_btl_base_descriptor_t* mca_btl_mvapi_prepare_dst(
                                         *size, 
                                         (mca_mpool_base_registration_t**) &vapi_reg);
             
-            vapi_reg->is_leave_pinned = true;
+            vapi_reg->base_reg.is_leave_pinned = true;
             
             rc = mca_mpool_base_insert(vapi_reg->base_reg.base,  
                                        vapi_reg->base_reg.bound - vapi_reg->base_reg.base + 1, 
@@ -607,7 +607,7 @@ mca_btl_base_descriptor_t* mca_btl_mvapi_prepare_dst(
                                             frag->segment.seg_addr.pval,
                                             *size, 
                                             (mca_mpool_base_registration_t**) &vapi_reg);
-            vapi_reg->is_leave_pinned=false; 
+            vapi_reg->base_reg.is_leave_pinned=false; 
         }
         
         

ompi/mca/mpool/mpool.h

@@ -32,6 +32,7 @@ struct mca_mpool_base_registration_t {
     bool is_leave_pinned;
     unsigned char* base;
     unsigned char* bound; 
+    unsigned char* alloc_base; 
 };  
 typedef struct mca_mpool_base_registration_t mca_mpool_base_registration_t; 
 

ompi/mca/mpool/mvapi/mpool_mvapi.h

@@ -23,7 +23,6 @@
 #include "class/ompi_free_list.h"
 #include "opal/event/event.h"
 #include "mca/mpool/mpool.h"
-#include "mca/allocator/allocator.h"
 #include "mca/common/vapi/vapi_mem_reg.h" 
 
 #if defined(c_plusplus) || defined(__cplusplus)
@@ -43,7 +42,6 @@ static inline void*  ALIGN_ADDR(void* addr, uint32_t cnt ) {
 
 struct mca_mpool_mvapi_component_t {
     mca_mpool_base_component_t super;
-    char*  vapi_allocator_name;
     long page_size; 
     long page_size_log; 
 };
@@ -64,7 +62,6 @@ typedef struct mca_mpool_base_resources_t mca_mpool_base_resources_t;
 
 struct mca_mpool_mvapi_module_t {
     mca_mpool_base_module_t super;
-    mca_allocator_base_module_t * vapi_allocator; 
     struct mca_mpool_base_resources_t  hca_pd;
 }; typedef struct mca_mpool_mvapi_module_t mca_mpool_mvapi_module_t; 
 
@@ -81,7 +78,6 @@ struct mca_mpool_mvapi_registration_t {
     VAPI_rkey_t                     r_key;
     /* Remote key to registered memory, need to send this
      * to remote processes for incoming RDMA ops */
-    bool is_leave_pinned; 
   
 };
 typedef struct mca_mpool_mvapi_registration_t mca_mpool_mvapi_registration_t;
@@ -142,11 +138,6 @@ void mca_mpool_mvapi_free(mca_mpool_base_module_t* mpool,
                          void * addr, 
                          mca_mpool_base_registration_t* registration);
 
-void* mca_common_vapi_segment_alloc(
-                                    struct mca_mpool_base_module_t* module, 
-                                    size_t* size, 
-                                    mca_mpool_base_registration_t** registration);
-
 #if defined(c_plusplus) || defined(__cplusplus)
 }
 #endif

ompi/mca/mpool/mvapi/mpool_mvapi_component.c

@@ -18,7 +18,6 @@
 #include "opal/util/output.h"
 #include "mca/base/base.h"
 #include "mca/base/mca_base_param.h"
-#include "mca/allocator/base/base.h"
 #include "mpool_mvapi.h"
 #include "util/proc_info.h"
 #include "util/sys_info.h"
@@ -65,14 +64,14 @@ mca_mpool_mvapi_component_t mca_mpool_mvapi_component = {
 
 static void mca_mpool_mvapi_registration_constructor( mca_mpool_mvapi_registration_t * registration ) 
 { 
-    registration->is_leave_pinned = false; 
+    registration->base_reg.is_leave_pinned = false; 
 }
 
 static void mca_mpool_mvapi_registration_destructor( mca_mpool_mvapi_registration_t * registration ) 
 { 
     registration->base_reg.base = NULL; 
     registration->base_reg.bound = NULL; 
-    registration->is_leave_pinned=false; 
+    registration->base_reg.is_leave_pinned=false; 
 
 } 
 
@@ -105,35 +104,16 @@ static int mca_mpool_mvapi_open(void)
 {
     /* register VAPI component parameters */
     
-    mca_mpool_mvapi_component.vapi_allocator_name =
-        mca_mpool_mvapi_param_register_string("allocator", "bucket");
     /* get the page size for this architecture*/ 
     mca_mpool_mvapi_component.page_size = sysconf(_SC_PAGESIZE); 
 
     return OMPI_SUCCESS;
 }
 
-/* Allocates a segment of memory and registers with IB, user_out returns the memory handle. */ 
-void* mca_common_vapi_segment_alloc(
-    struct mca_mpool_base_module_t* mpool,
-    size_t* size, 
-    mca_mpool_base_registration_t** registration)
-{
-    void* addr_malloc = (void*)malloc((*size) + mca_mpool_mvapi_component.page_size); 
-    void* addr = (void*)  ALIGN_ADDR(addr_malloc, mca_mpool_mvapi_component.page_size_log); 
-    if(OMPI_SUCCESS !=  mpool->mpool_register(mpool, addr, *size, registration)) { 
-        free(addr_malloc);
-        return NULL; 
-    } 
-    return addr; 
-}
-
-/* Allocates a segment of memory and registers with IB, user_out returns the memory handle. */ 
 static mca_mpool_base_module_t* mca_mpool_mvapi_init(
      struct mca_mpool_base_resources_t* resources)
 {
     mca_mpool_mvapi_module_t* mpool_module; 
-    mca_allocator_base_component_t* allocator_component;
     long page_size = mca_mpool_mvapi_component.page_size; 
 
     mca_mpool_mvapi_component.page_size_log = 0; 
@@ -142,33 +122,10 @@ static mca_mpool_base_module_t* mca_mpool_mvapi_init(
         mca_mpool_mvapi_component.page_size_log++; 
     }    
     
-    /* if specified allocator cannout be loaded - look for an alternative */
-    allocator_component = mca_allocator_component_lookup(mca_mpool_mvapi_component.vapi_allocator_name);
-    if(NULL == allocator_component) {
-        if(opal_list_get_size(&mca_allocator_base_components) == 0) {
-            mca_base_component_list_item_t* item = (mca_base_component_list_item_t*)
-                opal_list_get_first(&mca_allocator_base_components);
-            allocator_component = (mca_allocator_base_component_t*)item->cli_component;
-            opal_output(0, "mca_mpool_mvapi_init: unable to locate allocator: %s - using %s\n",
-                mca_mpool_mvapi_component.vapi_allocator_name, allocator_component->allocator_version.mca_component_name);
-        } else {
-            opal_output(0, "mca_mpool_mvapi_init: unable to locate allocator: %s\n",
-                mca_mpool_mvapi_component.vapi_allocator_name);
-            return NULL;
-        }
-    }
-    
     mpool_module = (mca_mpool_mvapi_module_t*)malloc(sizeof(mca_mpool_mvapi_module_t)); 
     mca_mpool_mvapi_module_init(mpool_module); 
     
-    /* setup allocator  TODO fix up */
     mpool_module->hca_pd = *resources;
-    mpool_module->vapi_allocator = 
-        allocator_component->allocator_init(true, mca_common_vapi_segment_alloc, NULL, &mpool_module->super);
-    if(NULL == mpool_module->vapi_allocator) {
-      opal_output(0, "mca_mpool_mvapi_init: unable to initialize allocator");
-      return NULL;
-    }
     return &mpool_module->super;
 }
 

ompi/mca/mpool/mvapi/mpool_mvapi_module.c

@@ -45,8 +45,15 @@ void* mca_mpool_mvapi_alloc(
     size_t align, 
     mca_mpool_base_registration_t** registration)
 {
-    mca_mpool_mvapi_module_t* mpool_mvapi = (mca_mpool_mvapi_module_t*)mpool; 
-    return mpool_mvapi->vapi_allocator->alc_alloc(mpool_mvapi->vapi_allocator, size, align, registration);
+    
+    void* addr_malloc = (void*)malloc(size + mca_mpool_mvapi_component.page_size); 
+    void* addr = (void*)  ALIGN_ADDR(addr_malloc, mca_mpool_mvapi_component.page_size_log); 
+    if(OMPI_SUCCESS !=  mpool->mpool_register(mpool, addr, size, registration)) { 
+        free(addr_malloc);
+        return NULL; 
+    }
+    (*registration)->alloc_base = addr_malloc; 
+    return addr; 
 }
 
 
@@ -139,7 +146,12 @@ void* mca_mpool_mvapi_realloc(
     mca_mpool_base_registration_t** registration)
 {
     mca_mpool_mvapi_module_t* mpool_mvapi = (mca_mpool_mvapi_module_t*)mpool; 
-    return mpool_mvapi->vapi_allocator->alc_realloc( mpool_mvapi->vapi_allocator, addr, size, registration);
+    mca_mpool_base_registration_t* old_reg  = *registration; 
+    
+    void* new_mem = mpool->mpool_alloc(mpool, size, 0, registration); 
+    memcpy(new_mem, addr, old_reg->bound - old_reg->base); 
+    mpool->mpool_free(mpool, addr, &old_reg); 
+    return new_mem; 
 }
 
 /**
@@ -151,7 +163,7 @@ void mca_mpool_mvapi_free(mca_mpool_base_module_t* mpool, void * addr,
     
     mca_mpool_mvapi_module_t* mpool_mvapi = (mca_mpool_mvapi_module_t*)mpool; 
     mpool_mvapi->super.mpool_deregister(mpool, addr, 0, registration); 
-    mpool_mvapi->vapi_allocator->alc_free(mpool_mvapi->vapi_allocator, addr);
+    free(registration->alloc_base); 
     
     
 }