slight change to allocator/memory pool interface

This commit was SVN r6115.

src/mca/allocator/allocator.h

@@ -21,6 +21,10 @@
 #define MCA_ALLOCATOR_H
 #include "mca/mca.h"
 
+struct mca_bmi_base_registration_t;
+struct mca_bmi_base_resources_t;
+
+
 #if defined(c_plusplus) || defined(__cplusplus)
 extern "C" {
 #endif
@@ -30,17 +34,25 @@ struct mca_allocator_base_module_t;
 /**
   * The allocate function typedef for the function to be provided by the component.
   */
-typedef void* (*mca_allocator_base_module_alloc_fn_t)(struct mca_allocator_base_module_t*, size_t size, size_t align, void** user_out);
+typedef void* (*mca_allocator_base_module_alloc_fn_t)(
+    struct mca_allocator_base_module_t*, 
+    size_t size, 
+    size_t align, 
+    struct mca_bmi_base_registration_t** registration);
  
 /**
   * The realloc function typedef
   */
-typedef void* (*mca_allocator_base_module_realloc_fn_t)(struct mca_allocator_base_module_t*, void*, size_t, void** user_out);
+typedef void* (*mca_allocator_base_module_realloc_fn_t)(
+    struct mca_allocator_base_module_t*, 
+    void*, size_t, 
+    struct mca_bmi_base_registration_t** registration);
 
 /**
   * Free function typedef
   */
-typedef void(*mca_allocator_base_module_free_fn_t)(struct mca_allocator_base_module_t*, void *);
+typedef void(*mca_allocator_base_module_free_fn_t)(
+    struct mca_allocator_base_module_t*, void *);
 
 
 /**
@@ -72,9 +84,10 @@ struct mca_allocator_base_module_t {
     /**< Free memory */
     mca_allocator_base_module_compact_fn_t alc_compact;   
     /**< Return memory */
-  mca_allocator_base_module_finalize_fn_t alc_finalize; 
+    mca_allocator_base_module_finalize_fn_t alc_finalize; 
     /**< Finalize and free everything */
-  void* user_in; 
+    /* memory pool and resources */
+    struct mca_mpool_base_module_t* alc_mpool;
 };
 /**
  * Convenience typedef.
@@ -87,14 +100,19 @@ typedef struct mca_allocator_base_module_t mca_allocator_base_module_t;
   * provided by the module to the allocator framework.
   */
 
-typedef void* (*mca_allocator_base_component_segment_alloc_fn_t)(size_t* size, void* user_in, void** user_out);
+typedef void* (*mca_allocator_base_component_segment_alloc_fn_t)(
+    struct mca_mpool_base_module_t* module,
+    size_t* size, 
+    struct mca_bmi_base_registration_t** registration);
 
 /**
   * A function to free memory from the control of the allocator framework 
   * back to the system. This function is to be provided by the module to the
   * allocator frmaework.
   */
-typedef void* (*mca_allocator_base_component_segment_free_fn_t)(void* segment);
+typedef void* (*mca_allocator_base_component_segment_free_fn_t)(
+    struct mca_mpool_base_module_t* module,
+    void* segment);
 
 
 /**
@@ -105,7 +123,7 @@ typedef struct mca_allocator_base_module_t*
     bool enable_mpi_threads,
     mca_allocator_base_component_segment_alloc_fn_t segment_alloc,
     mca_allocator_base_component_segment_free_fn_t segment_free, 
-    void* user_in
+    struct mca_mpool_base_module_t* mpool
 );
 
 /**

src/mca/allocator/basic/allocator_basic.c

@@ -77,7 +77,7 @@ mca_allocator_base_module_t* mca_allocator_basic_component_init(
     bool enable_mpi_threads,
     mca_allocator_base_component_segment_alloc_fn_t segment_alloc,
     mca_allocator_base_component_segment_free_fn_t segment_free, 
-    void* user_in)
+    struct mca_mpool_base_module_t* mpool)
 {
     mca_allocator_basic_module_t *module = (mca_allocator_basic_module_t *)
                                             malloc(sizeof(mca_allocator_basic_module_t));
@@ -90,7 +90,7 @@ mca_allocator_base_module_t* mca_allocator_basic_component_init(
     module->super.alc_free = mca_allocator_basic_free;
     module->super.alc_compact = mca_allocator_basic_compact;
     module->super.alc_finalize = mca_allocator_basic_finalize;
-    module->super.user_in = user_in;
+    module->super.alc_mpool = mpool;
     module->seg_alloc = segment_alloc;
     module->seg_free = segment_free;
     OBJ_CONSTRUCT(&module->seg_list, ompi_list_t);
@@ -161,7 +161,7 @@ void *mca_allocator_basic_alloc(
     mca_allocator_base_module_t * base, 
     size_t size,
     size_t align, 
-    void** user_out)
+    struct mca_bmi_base_registration_t** registration)
 {
     mca_allocator_basic_module_t* module = (mca_allocator_basic_module_t*)base;
     mca_allocator_basic_segment_t* seg;
@@ -197,7 +197,7 @@ void *mca_allocator_basic_alloc(
 
     /* request additional block */
     allocated_size = (unsigned char)size;
-    if(NULL == (addr = (unsigned char *)module->seg_alloc(&allocated_size, module->super.user_in, user_out))) {
+    if(NULL == (addr = (unsigned char *)module->seg_alloc(module->super.alc_mpool, &allocated_size, registration))) {
         OMPI_THREAD_UNLOCK(&module->seg_lock);
         return NULL;
     }
@@ -241,13 +241,13 @@ void * mca_allocator_basic_realloc(
     mca_allocator_base_module_t * base, 
     void * ptr, 
     size_t size, 
-    void** user_out)
+    struct mca_bmi_base_registration_t** registration)
 {
     unsigned char* addr = ((unsigned char*)ptr) - sizeof(size_t);
     size_t alloc_size = *(size_t*)addr;
     if(size <= alloc_size)
         return ptr;
-    addr = (unsigned char *)mca_allocator_basic_alloc(base,size,0, user_out);
+    addr = (unsigned char *)mca_allocator_basic_alloc(base,size,0,registration);
     if(addr == NULL)
         return addr;
     memcpy(addr,ptr,alloc_size);

src/mca/allocator/basic/allocator_basic.h

@@ -74,7 +74,7 @@ mca_allocator_base_module_t* mca_allocator_basic_component_init(
     bool enable_mpi_threads,
     mca_allocator_base_component_segment_alloc_fn_t segment_alloc,
     mca_allocator_base_component_segment_free_fn_t segment_free, 
-    void* user_in
+    struct mca_mpool_base_module_t* module
 );
                                                                                                  
 /**
@@ -92,7 +92,7 @@ mca_allocator_base_module_t* mca_allocator_basic_component_init(
         mca_allocator_base_module_t * mem, 
         size_t size, 
         size_t align, 
-        void** user_out);
+        struct mca_bmi_base_registration_t** registration);
 
 /**
    * Attempts to resize the passed region of memory into a larger or a smaller
@@ -112,7 +112,7 @@ mca_allocator_base_module_t* mca_allocator_basic_component_init(
         mca_allocator_base_module_t * mem, 
         void * ptr, 
         size_t size, 
-        void** user_out);
+        struct mca_bmi_base_registration_t** registration);
 
 /**
    * Frees the passed region of memory

src/mca/allocator/bucket/allocator_bucket.c

@@ -24,15 +24,18 @@ struct mca_allocator_base_module_t* mca_allocator_bucket_module_init(
     bool enable_mpi_threads,
     mca_allocator_base_component_segment_alloc_fn_t segment_alloc,
     mca_allocator_base_component_segment_free_fn_t segment_free, 
-    void* user
+    struct mca_mpool_base_module_t* mpool
     );
 
 int mca_allocator_bucket_module_open(void);
 
 int mca_allocator_bucket_module_close(void);
 
-void * mca_allocator_bucket_alloc_wrapper(struct mca_allocator_base_module_t* allocator,
+void * mca_allocator_bucket_alloc_wrapper(
-                                          size_t size, size_t align, void** user_out);
+    struct mca_allocator_base_module_t* allocator,
+    size_t size, size_t align, 
+    struct mca_bmi_base_registration_t** registration);
+
 static int mca_allocator_num_buckets;
 
 
@@ -48,7 +51,7 @@ struct mca_allocator_base_module_t* mca_allocator_bucket_module_init(
     bool enable_mpi_threads,
     mca_allocator_base_component_segment_alloc_fn_t segment_alloc,
     mca_allocator_base_component_segment_free_fn_t segment_free, 
-    void* user_in)
+    struct mca_mpool_base_module_t* mpool)
 {
     size_t alloc_size = sizeof(mca_allocator_bucket_t);
     mca_allocator_bucket_t * retval;
@@ -56,8 +59,10 @@ struct mca_allocator_base_module_t* mca_allocator_bucket_module_init(
     if(NULL == allocator) {
         return(NULL);
     }
-    retval = mca_allocator_bucket_init((mca_allocator_base_module_t *) allocator, mca_allocator_num_buckets, 
+    retval = mca_allocator_bucket_init((mca_allocator_base_module_t *) allocator, 
-                                        segment_alloc, segment_free);
+        mca_allocator_num_buckets, 
+        segment_alloc, 
+        segment_free);
     if(NULL == retval) {
         free(allocator);
         return(NULL);
@@ -67,7 +72,7 @@ struct mca_allocator_base_module_t* mca_allocator_bucket_module_init(
     allocator->super.alc_free =  mca_allocator_bucket_free;
     allocator->super.alc_compact = mca_allocator_bucket_cleanup;
     allocator->super.alc_finalize = mca_allocator_bucket_finalize;
-    allocator->super.user_in = user_in; 
+    allocator->super.alc_mpool = mpool;
     return((mca_allocator_base_module_t *) allocator);
 }
 
@@ -82,13 +87,16 @@ int mca_allocator_bucket_module_close(void) {
     return(OMPI_SUCCESS);
 }
 
-void * mca_allocator_bucket_alloc_wrapper(struct mca_allocator_base_module_t* allocator,
+void * mca_allocator_bucket_alloc_wrapper(
-                                          size_t size, size_t align, void** user_out)
+    struct mca_allocator_base_module_t* allocator,
+    size_t size, 
+    size_t align, 
+    struct mca_bmi_base_registration_t** registration)
 {
     if(0 == align){
-        return(mca_allocator_bucket_alloc(allocator, size, user_out));
+        return(mca_allocator_bucket_alloc(allocator, size, registration));
     }
-    return(mca_allocator_bucket_alloc_align(allocator, size, align, user_out));
+    return(mca_allocator_bucket_alloc_align(allocator, size, align, registration));
 }    
 
 

src/mca/allocator/bucket/allocator_bucket_alloc.c

@@ -32,10 +32,11 @@
    * Initializes the mca_allocator_bucket_options_t data structure for the passed
    * parameters.
    */
-mca_allocator_bucket_t * mca_allocator_bucket_init(mca_allocator_base_module_t * mem,
+mca_allocator_bucket_t * mca_allocator_bucket_init(
-                                   int num_buckets,
+    mca_allocator_base_module_t * mem,
-                                   mca_allocator_base_component_segment_alloc_fn_t get_mem_funct,
+    int num_buckets,
-                                   mca_allocator_base_component_segment_free_fn_t free_mem_funct)
+    mca_allocator_base_component_segment_alloc_fn_t get_mem_funct,
+    mca_allocator_base_component_segment_free_fn_t free_mem_funct)
 {
     mca_allocator_bucket_t * mem_options = (mca_allocator_bucket_t *) mem;
     int i;
@@ -67,8 +68,10 @@ mca_allocator_bucket_t * mca_allocator_bucket_init(mca_allocator_base_module_t *
    * region or NULL if there was an error
    *
    */
-void * mca_allocator_bucket_alloc(mca_allocator_base_module_t * mem,
+void * mca_allocator_bucket_alloc(
-                                  size_t size, void** user_out)
+    mca_allocator_base_module_t * mem,
+    size_t size, 
+    struct mca_bmi_base_registration_t** registration)
 {
     mca_allocator_bucket_t * mem_options = (mca_allocator_bucket_t *) mem;
     /* initialize for the later bit shifts */
@@ -107,7 +110,7 @@ void * mca_allocator_bucket_alloc(mca_allocator_base_module_t * mem,
     allocated_size += sizeof(mca_allocator_bucket_segment_head_t);
     /* attempt to get the memory */
     segment_header = (mca_allocator_bucket_segment_head_t *)
-                   mem_options->get_mem_fn(&allocated_size, mem_options->super.user_in, user_out);
+                   mem_options->get_mem_fn(mem_options->super.alc_mpool, &allocated_size, registration);
     if(NULL == segment_header) {
         /* release the lock */
         OMPI_THREAD_UNLOCK(&(mem_options->buckets[bucket_num].lock)); 
@@ -147,8 +150,11 @@ void * mca_allocator_bucket_alloc(mca_allocator_base_module_t * mem,
 /*
   * allocates an aligned region of memory
   */
-void * mca_allocator_bucket_alloc_align(mca_allocator_base_module_t * mem, 
+void * mca_allocator_bucket_alloc_align(
-                                        size_t size, size_t alignment, void** user_out)
+    mca_allocator_base_module_t * mem, 
+    size_t size, 
+    size_t alignment, 
+    struct mca_bmi_base_registration_t** registration)
 {
     mca_allocator_bucket_t * mem_options = (mca_allocator_bucket_t *) mem; 
     int bucket_num = 1;
@@ -168,7 +174,7 @@ void * mca_allocator_bucket_alloc_align(mca_allocator_base_module_t * mem,
     bucket_size = size + sizeof(mca_allocator_bucket_chunk_header_t);
     allocated_size = aligned_max_size; 
     /* get some memory */ 
-    ptr = mem_options->get_mem_fn(&allocated_size, mem_options->super.user_in, user_out);
+    ptr = mem_options->get_mem_fn(mem_options->super.alc_mpool, &allocated_size, registration);
     if(NULL == ptr) {
         return(NULL);
     }
@@ -227,8 +233,11 @@ void * mca_allocator_bucket_alloc_align(mca_allocator_base_module_t * mem,
 /*
   * function to reallocate the segment of memory
   */
-void * mca_allocator_bucket_realloc(mca_allocator_base_module_t * mem,
+void * mca_allocator_bucket_realloc(
-                                    void * ptr, size_t size, void** user_out)
+    mca_allocator_base_module_t * mem,
+    void * ptr, 
+    size_t size, 
+    struct mca_bmi_base_registration_t** registration)
 {
     mca_allocator_bucket_t * mem_options = (mca_allocator_bucket_t *) mem;
     /* initialize for later bit shifts */
@@ -249,7 +258,7 @@ void * mca_allocator_bucket_realloc(mca_allocator_base_module_t * mem,
         return(ptr);
     }
     /* we need a new space in memory, so let's get it */
-    ret_ptr = mca_allocator_bucket_alloc((mca_allocator_base_module_t *) mem_options, size, user_out);
+    ret_ptr = mca_allocator_bucket_alloc((mca_allocator_base_module_t *) mem_options, size, registration);
     if(NULL == ret_ptr) {
         /* we were unable to get a larger area of memory */
         return(NULL);
@@ -329,7 +338,7 @@ int mca_allocator_bucket_cleanup(mca_allocator_base_module_t * mem)
                 *segment_header = segment->next_segment;
                 /* free the memory */
                 if(mem_options->free_mem_fn)
-                    mem_options->free_mem_fn(segment);
+                    mem_options->free_mem_fn(mem->alc_mpool, segment);
             } else {
                 /* go to next segment */
                 segment_header = &((*segment_header)->next_segment);

src/mca/allocator/bucket/allocator_bucket_alloc.h

@@ -122,7 +122,10 @@ typedef struct mca_allocator_bucket_t mca_allocator_bucket_t;
    * @retval Pointer to the area of memory if the allocation was successful
    * @retval NULL if the allocation was unsuccessful
    */
-    void * mca_allocator_bucket_alloc(mca_allocator_base_module_t * mem, size_t size, void** user_out);
+    void * mca_allocator_bucket_alloc(
+        mca_allocator_base_module_t * mem, 
+        size_t size, 
+        struct mca_bmi_base_registration_t** registration);
 
 /**
    * Accepts a request for memory in a specific region defined by the
@@ -139,8 +142,11 @@ typedef struct mca_allocator_bucket_t mca_allocator_bucket_t;
    * @retval NULL if the allocation was unsuccessful
    *
    */
-    void * mca_allocator_bucket_alloc_align(mca_allocator_base_module_t * mem,
+    void * mca_allocator_bucket_alloc_align(
-                                            size_t size, size_t alignment, void** user_out);
+        mca_allocator_base_module_t * mem,
+        size_t size, 
+        size_t alignment, 
+        struct mca_bmi_base_registration_t** registration);
 
 /**
    * Attempts to resize the passed region of memory into a larger or a smaller
@@ -156,8 +162,11 @@ typedef struct mca_allocator_bucket_t mca_allocator_bucket_t;
    * @retval NULL if the allocation was unsuccessful
    *
    */
-    void * mca_allocator_bucket_realloc(mca_allocator_base_module_t * mem, 
+    void * mca_allocator_bucket_realloc(
-                                        void * ptr, size_t size, void** user_out);
+        mca_allocator_base_module_t * mem, 
+        void * ptr, 
+        size_t size, 
+        struct mca_bmi_base_registration_t** registration);
 
 /**
    * Frees the passed region of memory

src/mca/bmi/bmi.h

@@ -119,6 +119,8 @@
 struct mca_bmi_base_module_t;
 struct mca_bmi_base_endpoint_t;
 struct mca_bmi_base_descriptor_t;
+struct mca_bmi_base_registration_t;
+struct mca_bmi_base_resources_t;
 
 
 /* send/recv operations require tag matching */
@@ -417,6 +419,7 @@ typedef int (*mca_bmi_base_module_free_fn_t)(
 typedef struct mca_bmi_base_descriptor_t* (*mca_bmi_base_module_prepare_fn_t)(
     struct mca_bmi_base_module_t* bmi,
     struct mca_bmi_base_endpoint_t* endpoint,
+    struct mca_bmi_base_registration_t* registration,
     struct ompi_convertor_t* convertor,
     size_t reserve,
     size_t* size

src/mca/bmi/ib/.ompi_unignore

@@ -1,2 +1,3 @@
+ompi
 twoodall
-gshipman
+gshipman

src/mca/bmi/ib/bmi_ib.c

@@ -616,7 +616,6 @@ int mca_bmi_ib_put( mca_bmi_base_module_t* bmi,
     frag->endpoint = endpoint;
     frag->sr_desc.opcode = VAPI_RDMA_WRITE; 
     
-    
     frag->sr_desc.remote_qp = endpoint->rem_qp_num_low; 
     frag->sr_desc.remote_addr = (VAPI_virt_addr_t) (MT_virt_addr_t) frag->base.des_dst->seg_addr.pval; 
     frag->sr_desc.r_key = frag->base.des_dst->seg_key.key32[0]; 

src/mca/bmi/self/.ompi_unignore

@@ -1,2 +1,3 @@
+ompi
 twoodall
-gshipman
+gshipman

src/mca/bmi/sm/.ompi_unignore

@@ -1,3 +1,4 @@
+ompi
 twoodall
 gshipman
 bosilca

src/mca/common/sm/common_sm_mmap.c

@@ -32,10 +32,10 @@
 #endif
 
 #include "include/constants.h"
+#include "common_sm_mmap.h"
 #include "util/output.h"
 #include "util/sys_info.h"
 #include "util/proc_info.h"
-#include "common_sm_mmap.h"
 
 
 OBJ_CLASS_INSTANCE(
@@ -212,7 +212,11 @@ mca_common_sm_mmap_t* mca_common_sm_mmap_init(size_t size, char *file_name,
  * 
  *  @retval addr virtual address
  */
-void* mca_common_sm_mmap_alloc(size_t* size, void* user_in, void** user_out)
+
+void* mca_common_sm_mmap_seg_alloc(
+    struct mca_mpool_base_module_t* mpool,
+    size_t* size,
+    struct mca_bmi_base_registration_t** registration)
 {
     mca_common_sm_mmap_t* map = mca_common_sm_mmap;
     mca_common_sm_file_header_t* seg = map->map_seg;
@@ -226,8 +230,8 @@ void* mca_common_sm_mmap_alloc(size_t* size, void* user_in, void** user_out)
         addr = map->data_addr + seg->seg_offset;
         seg->seg_offset += *size;
     }
+    *registration = NULL;
     ompi_atomic_unlock(&seg->seg_lock);
     return addr;
 }
 
-

src/mca/common/sm/common_sm_mmap.h

@@ -25,6 +25,10 @@
 #if defined(c_plusplus) || defined(__cplusplus)
 extern "C" {
 #endif
+
+struct mca_mpool_base_module_t;
+struct mca_bmi_base_registration_t;
+
 struct mca_common_sm_file_header_t {
 
     /* lock to control atomic access */
@@ -86,10 +90,16 @@ OBJ_CLASS_DECLARATION(mca_common_sm_mmap_t);
  *  @returnvalue pointer to control structure at head of file.
  */
 
-mca_common_sm_mmap_t* mca_common_sm_mmap_init(size_t size, char *file_name,
+extern mca_common_sm_mmap_t* mca_common_sm_mmap_init(
-                size_t size_ctl_structure, size_t data_seg_alignment);
+    size_t size, 
-void* mca_common_sm_mmap_alloc(size_t* size, void* user_in, void** user_out);
+    char *file_name,
-void  mca_common_sm_mmap_free(void* addr);
+    size_t size_ctl_structure, 
+    size_t data_seg_alignment);
+
+extern void* mca_common_sm_mmap_seg_alloc(
+    struct mca_mpool_base_module_t* mpool, 
+    size_t* size, 
+    struct mca_bmi_base_registration_t** registration);
 
 /*
  * Instance that is shared between components that use shared memory

src/mca/common/vapi/vapi_mem_reg.h

@@ -27,12 +27,14 @@ extern "C" {
 #endif
 
 
-struct mca_common_vapi_hca_pd_t{
+struct mca_bmi_base_resources_t {
   VAPI_hca_hndl_t hca;   /* the hca (nic) */ 
   VAPI_pd_hndl_t pd_tag; /* the protection domain */ 
-}; typedef struct mca_common_vapi_hca_pd_t mca_common_vapi_hca_pd_t;  
+}; 
+typedef struct mca_bmi_base_resources_t mca_bmi_base_resources_t;  
 
-struct mca_common_vapi_memhandle_t {
+
+struct mca_bmi_base_registration_t {
   VAPI_mr_hndl_t                  hndl;
   /* Memory region handle */
   
@@ -44,7 +46,7 @@ struct mca_common_vapi_memhandle_t {
   /* Remote key to registered memory, need to send this
    * to remote processes for incoming RDMA ops */
 };
-typedef struct mca_common_vapi_memhandle_t mca_common_vapi_memhandle_t;
+typedef struct mca_bmi_base_registration_t mca_bmi_base_registration_t;
 
 #if defined(c_plusplus) || defined(__cplusplus)
 }

src/mca/mpool/base/base.h

@@ -34,6 +34,8 @@ struct mca_mpool_base_selected_module_t {
   ompi_list_item_t super;
   mca_mpool_base_component_t *mpool_component;
   mca_mpool_base_module_t *mpool_module;
+  struct mca_bmi_base_module_t *mpool_bmi;
+  struct mca_bmi_base_resources_t *mpool_resources;
 };
 typedef struct mca_mpool_base_selected_module_t mca_mpool_base_selected_module_t;
 
@@ -65,7 +67,8 @@ typedef struct mca_mpool_base_key_t mca_mpool_base_key_t;
 struct mca_mpool_base_reg_mpool_t
 {
     mca_mpool_base_module_t * mpool; /**< the registered memory pool */
-    void * user;                     /**< user specific information */
+    struct mca_bmi_base_module_t *bmi_module; /**< bmi that registered the memory */
+    struct mca_bmi_base_registration_t* bmi_registration; /**< bmi specific info associated w/ registration */
 };
 typedef struct mca_mpool_base_reg_mpool_t mca_mpool_base_reg_mpool_t;
 
@@ -87,14 +90,14 @@ typedef struct mca_mpool_base_chunk_t mca_mpool_base_chunk_t;
  */
 
 OMPI_DECLSPEC int mca_mpool_base_open(void);
-OMPI_DECLSPEC int mca_mpool_base_init(bool enable_progress_threads,
+OMPI_DECLSPEC int mca_mpool_base_init(bool enable_progress_threads, bool enable_mpi_threads);
-                                      bool enable_mpi_threads);
 OMPI_DECLSPEC int mca_mpool_base_close(void);
 OMPI_DECLSPEC mca_mpool_base_component_t* mca_mpool_base_component_lookup(const char* name);
-OMPI_DECLSPEC mca_mpool_base_module_t* mca_mpool_base_module_lookup(const char* name);
+OMPI_DECLSPEC mca_mpool_base_module_t* mca_mpool_base_module_create(
-OMPI_DECLSPEC mca_mpool_base_module_t* mca_mpool_base_module_init(const char* name);
+    const char* name, 
-OMPI_DECLSPEC mca_mpool_base_module_t* mca_mpool_base_module_create(const char* name, void* user); 
+    struct mca_bmi_base_module_t* bmi,
-
+    struct mca_bmi_base_resources_t* bmi_resources);
+ 
 /*
  * Globals
  */

src/mca/mpool/base/mpool_base_alloc.c

@@ -55,7 +55,10 @@ int mca_mpool_base_tree_node_compare(void * key1, void * key2)
     }
 }
 
-int mca_mpool_base_insert(void * addr, size_t size, mca_mpool_base_module_t* mpool, void*  user_data)
+int mca_mpool_base_insert(void * addr, size_t size, 
+    mca_mpool_base_module_t* mpool, 
+    struct mca_bmi_base_module_t* bmi,
+    struct mca_bmi_base_registration_t* registration)
 {
     ompi_list_item_t *item; 
     int rc; 
@@ -70,7 +73,8 @@ int mca_mpool_base_insert(void * addr, size_t size, mca_mpool_base_module_t* mpo
     if(rc != OMPI_SUCCESS) 
         return rc; 
     ((mca_mpool_base_chunk_t *) item)->mpools[0].mpool = mpool; 
-    ((mca_mpool_base_chunk_t *) item)->mpools[0].user = user_data;
+    ((mca_mpool_base_chunk_t *) item)->mpools[0].bmi_module = bmi;
+    ((mca_mpool_base_chunk_t *) item)->mpools[0].bmi_registration = registration;
     return OMPI_SUCCESS; 
 }
 
@@ -122,12 +126,12 @@ void * mca_mpool_base_alloc(size_t size, ompi_info_t * info)
     int num_modules = ompi_list_get_size(&mca_mpool_base_modules);
     int reg_module_num = 0;
     int i, num_keys;
-    mca_mpool_base_module_t * current;
+    mca_mpool_base_selected_module_t * current;
-    mca_mpool_base_module_t * no_reg_function = NULL;
+    mca_mpool_base_selected_module_t * no_reg_function = NULL;
-    mca_mpool_base_module_t ** has_reg_function = (mca_mpool_base_module_t **) 
+    mca_mpool_base_selected_module_t ** has_reg_function = (mca_mpool_base_selected_module_t **) 
                            malloc(num_modules * sizeof(mca_mpool_base_module_t *));
+    struct mca_bmi_base_registration_t * registration;
     void * mem = NULL;
-    void * user;
     char * key;
     bool match_found;
 
@@ -137,8 +141,8 @@ void * mca_mpool_base_alloc(size_t size, ompi_info_t * info)
             item != ompi_list_get_end(&mca_mpool_base_modules);
             item = ompi_list_get_next(item)) 
         {
-            current = ((mca_mpool_base_selected_module_t *) item)->mpool_module;
+            current = ((mca_mpool_base_selected_module_t *) item);
-            if(NULL == current->mpool_register)
+            if(NULL == current->mpool_module->mpool_register)
             {
                 no_reg_function = current;
             }
@@ -160,12 +164,12 @@ void * mca_mpool_base_alloc(size_t size, ompi_info_t * info)
                 item != ompi_list_get_end(&mca_mpool_base_modules);
                 item = ompi_list_get_next(item)) 
             {
-                current = ((mca_mpool_base_selected_module_t *)item)->mpool_module;
+                current = ((mca_mpool_base_selected_module_t *)item);
                 if(0 == strcmp(key, 
-                       current->mpool_component->mpool_version.mca_component_name))
+                       current->mpool_module->mpool_component->mpool_version.mca_component_name))
                 {
                     match_found = true;
-                    if(NULL == current->mpool_register)
+                    if(NULL == current->mpool_module->mpool_register)
                     {
                         if(NULL != no_reg_function)
                         {
@@ -227,29 +231,32 @@ void * mca_mpool_base_alloc(size_t size, ompi_info_t * info)
     num_modules = 0;
     if(NULL != no_reg_function)
     {
-        mem = no_reg_function->mpool_alloc(no_reg_function, size, 0, &user);
+        mca_mpool_base_module_t* mpool = no_reg_function->mpool_module;
+        mem = mpool->mpool_alloc(mpool, size, 0, &registration);
         ((mca_mpool_base_chunk_t *) item)->key.bottom = mem;
-        ((mca_mpool_base_chunk_t *) item)->key.top = (void *) 
+        ((mca_mpool_base_chunk_t *) item)->key.top = (void *)((char *) mem + size - 1);
-                                                     ((char *) mem + size - 1);
+        ((mca_mpool_base_chunk_t *) item)->mpools[num_modules].mpool = mpool;
-        ((mca_mpool_base_chunk_t *) item)->mpools[num_modules].mpool = 
+        ((mca_mpool_base_chunk_t *) item)->mpools[num_modules].bmi_module = no_reg_function->mpool_bmi;
-                                                                  no_reg_function;
+        ((mca_mpool_base_chunk_t *) item)->mpools[num_modules++].bmi_registration = registration;
-        ((mca_mpool_base_chunk_t *) item)->mpools[num_modules++].user = user;
         num_modules++;
     }
     else
     {
-        mem = has_reg_function[i]->mpool_alloc(has_reg_function[i], size, 0, &user);
+        mca_mpool_base_module_t* mpool = has_reg_function[i]->mpool_module;
+        mem = mpool->mpool_alloc(mpool, size, 0, &registration);
         ((mca_mpool_base_chunk_t *) item)->key.bottom = mem;
         ((mca_mpool_base_chunk_t *) item)->key.top = (void *) ((char *) mem + size - 1);
-        ((mca_mpool_base_chunk_t *) item)->mpools[num_modules].mpool = has_reg_function[i++];
+        ((mca_mpool_base_chunk_t *) item)->mpools[num_modules].mpool = mpool;
-        ((mca_mpool_base_chunk_t *) item)->mpools[num_modules++].user = user;
+        ((mca_mpool_base_chunk_t *) item)->mpools[num_modules].bmi_module = has_reg_function[i]->mpool_bmi;
+        ((mca_mpool_base_chunk_t *) item)->mpools[num_modules++].bmi_registration = registration;
+        i++;
         num_modules++;
     }
     
     while(i < reg_module_num && MCA_MPOOL_BASE_MAX_REG > num_modules)
     {
-        if(OMPI_SUCCESS != has_reg_function[i]->mpool_register(has_reg_function[i],
+        mca_mpool_base_module_t* mpool = has_reg_function[i]->mpool_module;
-                                                               mem, size, &user))
+        if(OMPI_SUCCESS != mpool->mpool_register(mpool, mem, size, &registration))
         {
             if(info == &ompi_mpi_info_null)
             {
@@ -261,8 +268,11 @@ void * mca_mpool_base_alloc(size_t size, ompi_info_t * info)
             free(has_reg_function);
             return NULL;
         }
-        ((mca_mpool_base_chunk_t *) item)->mpools[num_modules].mpool = has_reg_function[i++];
+        ((mca_mpool_base_chunk_t *) item)->mpools[num_modules].mpool = mpool;
-        ((mca_mpool_base_chunk_t *) item)->mpools[num_modules++].user = user;
+        ((mca_mpool_base_chunk_t *) item)->mpools[num_modules].bmi_module = has_reg_function[i]->mpool_bmi;
+        ((mca_mpool_base_chunk_t *) item)->mpools[num_modules].bmi_registration = registration;
+        num_modules++;
+        i++;
     }
 
     if(MCA_MPOOL_BASE_MAX_REG > num_modules)

src/mca/mpool/base/mpool_base_init.c

@@ -50,62 +50,3 @@ int mca_mpool_base_init(bool enable_progress_threads, bool enable_mpi_threads)
     return ompi_rb_tree_init(&mca_mpool_base_tree, mca_mpool_base_tree_node_compare);
 }
 
-mca_mpool_base_module_t* mca_mpool_base_module_init(const char* name)
-{
-  ompi_list_item_t *item;
-  mca_base_component_list_item_t *cli;
-  mca_mpool_base_component_t *component;
-  mca_mpool_base_module_t *module;
-  mca_mpool_base_selected_module_t *sm;
-
-  /* Traverse the list of available modules; call their init
-     functions. */
-
-  for (item = ompi_list_get_first(&mca_mpool_base_components);
-       ompi_list_get_end(&mca_mpool_base_components) != item;
-       item = ompi_list_get_next(item)) {
-    cli = (mca_base_component_list_item_t *) item;
-    component = (mca_mpool_base_component_t *) cli->cli_component;
-    if(strcmp(component->mpool_version.mca_component_name,name) != 0)
-        continue;
-
-    ompi_output_verbose(10, mca_mpool_base_output, 
-                        "select: initializing %s module %s",
-                        component->mpool_version.mca_type_name,
-                        component->mpool_version.mca_component_name);
-    if (NULL == component->mpool_init) {
-      ompi_output_verbose(10, mca_mpool_base_output,
-                          "select: no init function; ignoring module");
-    } else {
-      module = component->mpool_init(NULL);
-
-      /* If the module didn't initialize, unload it */
-
-      if (NULL == module) {
-        ompi_output_verbose(10, mca_mpool_base_output,
-                            "select: init returned failure");
-
-        mca_base_component_repository_release((mca_base_component_t *) component);
-        ompi_output_verbose(10, mca_mpool_base_output,
-                            "select: component %s unloaded",
-                            component->mpool_version.mca_component_name);
-      } 
-
-      /* Otherwise, it initialized properly.  Save it. */
-
-      else {
-          ompi_output_verbose(10, mca_mpool_base_output,
-                           "select: init returned success");
-
-          sm = OBJ_NEW(mca_mpool_base_selected_module_t);
-          sm->mpool_component = component;
-          sm->mpool_module = module;
-          ompi_list_append(&mca_mpool_base_modules, (ompi_list_item_t*) sm);
-          return module;
-        }
-      }
-  }
-  return NULL;
-}
-
-

src/mca/mpool/base/mpool_base_lookup.c

@@ -44,25 +44,10 @@ mca_mpool_base_component_t* mca_mpool_base_component_lookup(const char* name)
 }
 
 
-mca_mpool_base_module_t* mca_mpool_base_module_lookup(const char* name)
+mca_mpool_base_module_t* mca_mpool_base_module_create(
-{
+    const char* name, 
-    /* does the module already exist? */
+    struct mca_bmi_base_module_t* bmi,
-    ompi_list_item_t *item;
+    struct mca_bmi_base_resources_t* resources) 
-    for(item =  ompi_list_get_first(&mca_mpool_base_modules);
-        item != ompi_list_get_end(&mca_mpool_base_modules);
-        item = ompi_list_get_next(item)) {
-        mca_mpool_base_selected_module_t *sm = (mca_mpool_base_selected_module_t *) item;
-        if(strcmp(sm->mpool_component->mpool_version.mca_component_name,
-                  name) == 0) {
-          return sm->mpool_module;
-        }
-    }
-    /* if not create it */
-    return mca_mpool_base_module_init(name);
-}
-
-
-mca_mpool_base_module_t* mca_mpool_base_module_create(const char* name, void* user) 
 {
     
     mca_mpool_base_component_t* component = NULL; 
@@ -84,7 +69,7 @@ mca_mpool_base_module_t* mca_mpool_base_module_create(const char* name, void* us
 
     if(NULL == component)  
         return NULL;
-    module = component->mpool_init(user); 
+    module = component->mpool_init(bmi,resources); 
     sm = OBJ_NEW(mca_mpool_base_selected_module_t); 
     sm->mpool_component = component; 
     sm->mpool_module = module; 

src/mca/mpool/mpool.h

@@ -22,49 +22,64 @@
 #include "mca/mca.h"
 #include "info/info.h"
 struct mca_mpool_t;
-
+struct mca_bmi_base_module_t;
+struct mca_bmi_base_resources_t;
+struct mca_bmi_base_registration_t;
 
 /**
  * component initialize
  */
-typedef struct mca_mpool_base_module_t* (*mca_mpool_base_component_init_fn_t)
+typedef struct mca_mpool_base_module_t* (*mca_mpool_base_component_init_fn_t)(
-  (void* user_in);
+    struct mca_bmi_base_module_t* bmi, 
+    struct mca_bmi_base_resources_t*);
+
+/**
+  * allocate function typedef
+  */
+typedef void* (*mca_mpool_base_module_alloc_fn_t)(
+    struct mca_mpool_base_module_t* mpool,
+    size_t size,
+    size_t align,
+    struct mca_bmi_base_registration_t** registration);
+                                                                                                                                   
+/**
+  * realloc function typedef
+  */
+typedef void* (*mca_mpool_base_module_realloc_fn_t)(
+    struct mca_mpool_base_module_t* mpool,
+    void* addr,
+    size_t size,
+    struct mca_bmi_base_registration_t** registration);
+                                                                                                                                   
+/**
+  * free function typedef
+  */
+typedef void (*mca_mpool_base_module_free_fn_t)(
+    struct mca_mpool_base_module_t* mpool,
+    void *);
+                                                                                                                                   
+/**
+  * register memory
+  */
+typedef int (*mca_mpool_base_module_register_fn_t)(
+    struct mca_mpool_base_module_t* mpool,
+    void * addr,
+    size_t size,
+    struct mca_bmi_base_registration_t** registration);
+                                                                                                                                   
+/**
+  * deregister memory
+  */
+typedef int (*mca_mpool_base_module_deregister_fn_t)(
+    struct mca_mpool_base_module_t* mpool,
+    void * addr,
+    size_t size);
 
 /**
   * if appropriate - returns base address of memory pool
   */
 typedef void* (*mca_mpool_base_module_address_fn_t)(struct mca_mpool_base_module_t* mpool);
 
-/**
-  * allocate function typedef
-  */
-typedef void* (*mca_mpool_base_module_alloc_fn_t)(struct mca_mpool_base_module_t* mpool, size_t size, size_t align, void** user_out);
-
-/**
-  * allocate function typedef
-  */
-typedef void* (*mca_mpool_base_module_alloc_and_register_fn_t)(struct mca_mpool_base_module_t* mpool,size_t size, size_t align, void** user_out);
-
-/**
-  * realloc function typedef
-  */
-typedef void* (*mca_mpool_base_module_realloc_fn_t)(struct mca_mpool_base_module_t* mpool, void* addr, size_t size, void** user_out);
-
-/**
-  * free function typedef
-  */
-typedef void (*mca_mpool_base_module_free_fn_t)(struct mca_mpool_base_module_t* mpool, void *);
-
-/**
-  * register memory
-  */
-typedef int (*mca_mpool_base_module_register_fn_t)(struct mca_mpool_base_module_t* mpool, void * addr, size_t size, void** user_out);
-
-/**
-  * deregister memory
-  */
-typedef int (*mca_mpool_base_module_deregister_fn_t)(struct mca_mpool_base_module_t* mpool, void * addr, size_t size);
-
 /**
   * finalize
   */
@@ -166,10 +181,12 @@ OMPI_DECLSPEC int mca_mpool_base_tree_node_compare(void * key1, void * key2);
 OMPI_DECLSPEC struct mca_mpool_base_chunk_t * mca_mpool_base_find(void * base);
 
 
-OMPI_DECLSPEC int mca_mpool_base_insert(void * addr, 
+OMPI_DECLSPEC int mca_mpool_base_insert(
-                                        size_t size, 
+    void * addr, 
-                                        mca_mpool_base_module_t* mpool, 
+    size_t size, 
-                                        void* user_data); 
+    mca_mpool_base_module_t* mpool, 
+    struct mca_bmi_base_module_t* bmi,
+    struct mca_bmi_base_registration_t* registration); 
 
 OMPI_DECLSPEC int mca_mpool_base_remove(void * base); 
 

src/mca/mpool/sm/mpool_sm.h

@@ -62,17 +62,27 @@ void* mca_mpool_sm_base(mca_mpool_base_module_t*);
 /**
   *  Allocate block of shared memory.
   */
-void* mca_mpool_sm_alloc( mca_mpool_base_module_t* mpool, size_t size, size_t align, void** user_out);
+void* mca_mpool_sm_alloc( 
+    mca_mpool_base_module_t* mpool, 
+    size_t size, 
+    size_t align, 
+    struct mca_bmi_base_registration_t** registration);
 
 /**
   * realloc function typedef
   */
-void* mca_mpool_sm_realloc(mca_mpool_base_module_t* mpool, void* addr, size_t size, void** user_out);
+void* mca_mpool_sm_realloc(
+    mca_mpool_base_module_t* mpool, 
+    void* addr, 
+    size_t size, 
+    struct mca_bmi_base_registration_t** registration);
 
 /**
   * free function typedef
   */
-void mca_mpool_sm_free(mca_mpool_base_module_t* mpool, void *);
+void mca_mpool_sm_free(
+    mca_mpool_base_module_t* mpool, 
+    void *);
 
 
 #if defined(c_plusplus) || defined(__cplusplus)

src/mca/mpool/sm/mpool_sm_component.c

@@ -28,7 +28,9 @@
  * Local functions
  */
 static int mca_mpool_sm_open(void);
-static mca_mpool_base_module_t* mca_mpool_sm_init(void* user);
+static mca_mpool_base_module_t* mca_mpool_sm_init(
+    struct mca_bmi_base_module_t* bmi,
+    struct mca_bmi_base_resources_t* resources);
 
 mca_mpool_sm_component_t mca_mpool_sm_component = {
     {
@@ -95,7 +97,9 @@ static int mca_mpool_sm_open(void)
 }
 
 
-static mca_mpool_base_module_t* mca_mpool_sm_init(void * user_in)
+static mca_mpool_base_module_t* mca_mpool_sm_init(
+    struct mca_bmi_base_module_t* bmi,
+    struct mca_bmi_base_resources_t* resources)
 {
     char *file_name;
     int len;
@@ -149,7 +153,7 @@ static mca_mpool_base_module_t* mca_mpool_sm_init(void * user_in)
     /* setup allocator  TODO fix up */
     mpool_module->sm_allocator = 
       allocator_component->allocator_init(true,
-                                          mca_common_sm_mmap_alloc, NULL, NULL);
+                                          mca_common_sm_mmap_seg_alloc, NULL, NULL);
     if(NULL == mpool_module->sm_allocator) {
       ompi_output(0, "mca_mpool_sm_init: unable to initialize allocator");
         return NULL;

src/mca/mpool/sm/mpool_sm_module.c

@@ -36,16 +36,6 @@ void mca_mpool_sm_module_init(mca_mpool_sm_module_t* mpool)
   mpool->super.mpool_finalize = NULL; 
 
 }
-/*  mca_mpool_base_module_t mca_mpool_sm_module = { */
-/*      &mca_mpool_sm_component.super, */
-/*      mca_mpool_sm_base, */
-/*      mca_mpool_sm_alloc, */
-/*      mca_mpool_sm_realloc, */
-/*      mca_mpool_sm_free, */
-/*      NULL, */
-/*      NULL */
-/*  }; */
-
 
 /*
  * base address of shared memory mapping
@@ -58,19 +48,27 @@ void* mca_mpool_sm_base(mca_mpool_base_module_t* mpool)
 /**
   * allocate function 
   */
-void* mca_mpool_sm_alloc(mca_mpool_base_module_t* mpool, size_t size, size_t align, void** user_out)
+void* mca_mpool_sm_alloc(
+    mca_mpool_base_module_t* mpool, 
+    size_t size, 
+    size_t align, 
+    struct mca_bmi_base_registration_t** registration)
 {
-  mca_mpool_sm_module_t* mpool_sm = (mca_mpool_sm_module_t*)mpool; 
+    mca_mpool_sm_module_t* mpool_sm = (mca_mpool_sm_module_t*)mpool; 
-  return mpool_sm->sm_allocator->alc_alloc(mpool_sm->sm_allocator, size, align, user_out);
+    return mpool_sm->sm_allocator->alc_alloc(mpool_sm->sm_allocator, size, align, registration);
 }
 
 /**
   * realloc function 
   */
-void* mca_mpool_sm_realloc(mca_mpool_base_module_t* mpool, void* addr, size_t size, void** user_out)
+void* mca_mpool_sm_realloc(
+    mca_mpool_base_module_t* mpool, 
+    void* addr, 
+    size_t size, 
+    struct mca_bmi_base_registration_t** registration)
 {
-  mca_mpool_sm_module_t* mpool_sm = (mca_mpool_sm_module_t*)mpool; 
+    mca_mpool_sm_module_t* mpool_sm = (mca_mpool_sm_module_t*)mpool; 
-  return mpool_sm->sm_allocator->alc_realloc(mpool_sm->sm_allocator, addr, size, user_out);
+    return mpool_sm->sm_allocator->alc_realloc(mpool_sm->sm_allocator, addr, size, registration);
 }
 
 /**
@@ -78,6 +76,6 @@ void* mca_mpool_sm_realloc(mca_mpool_base_module_t* mpool, void* addr, size_t si
   */
 void mca_mpool_sm_free(mca_mpool_base_module_t* mpool, void * addr)
 {
-  mca_mpool_sm_module_t* mpool_sm = (mca_mpool_sm_module_t*)mpool; 
+    mca_mpool_sm_module_t* mpool_sm = (mca_mpool_sm_module_t*)mpool; 
-  mpool_sm->sm_allocator->alc_free(mpool_sm->sm_allocator, addr);
+    mpool_sm->sm_allocator->alc_free(mpool_sm->sm_allocator, addr);
 }

src/mca/mpool/vapi/.ompi_unignore

@@ -1,2 +1,3 @@
+ompi
 gshipman
 twoodall

src/mca/mpool/vapi/mpool_vapi.h

@@ -32,11 +32,12 @@ extern "C" {
 
 
 static inline  void * DOWN_ALIGN_ADDR(void * addr, uint32_t cnt) { 
-    return (MT_virt_addr_t)(addr) & (~((MT_virt_addr_t)0) << (cnt)); 
+    return (void*)((MT_virt_addr_t)(addr) & (~((MT_virt_addr_t)0) << (cnt))); 
 }
 
 static inline void*  ALIGN_ADDR(void* addr, uint32_t cnt ) { 
     DOWN_ALIGN_ADDR(((addr) +  ~(~((MT_virt_addr_t)0) << (cnt))), (cnt)); 
+    return addr;
 } 
 
 
@@ -54,8 +55,8 @@ OMPI_COMP_EXPORT extern mca_mpool_vapi_component_t mca_mpool_vapi_component;
 struct mca_mpool_vapi_module_t {
   mca_mpool_base_module_t super;
   mca_allocator_base_module_t * vapi_allocator; 
-  mca_common_vapi_hca_pd_t  hca_pd;
+  mca_bmi_base_resources_t  hca_pd;
-  mca_common_vapi_memhandle_t mem_hndl; 
+  mca_bmi_base_registration_t mem_hndl; 
 }; typedef struct mca_mpool_vapi_module_t mca_mpool_vapi_module_t; 
 
 /* 
@@ -72,17 +73,34 @@ void* mca_mpool_vapi_base(mca_mpool_base_module_t*);
 /**
   *  Allocate block of shared memory.
   */
-void* mca_mpool_vapi_alloc( mca_mpool_base_module_t* mpool, size_t size, size_t align, void** user_out);
+void* mca_mpool_vapi_alloc( 
+    mca_mpool_base_module_t* mpool, 
+    size_t size, 
+    size_t align, 
+    struct mca_bmi_base_registration_t** registration);
 
 /**
   * realloc function typedef
   */
-void* mca_mpool_vapi_realloc(mca_mpool_base_module_t* mpool, void* addr, size_t size, void** user_out);
+void* mca_mpool_vapi_realloc(
+    mca_mpool_base_module_t* mpool, 
+    void* addr, 
+    size_t size, 
+    struct mca_bmi_base_registration_t** registration);
 
 /**
   * register function typedef
   */
-    int mca_mpool_vapi_register(mca_mpool_base_module_t* mpool, void *addr, size_t size, void** user_out); 
+int mca_mpool_vapi_register(
+    mca_mpool_base_module_t* mpool, 
+    void *addr, 
+    size_t size, 
+    struct mca_bmi_base_registration_t** registration);
+
+int mca_mpool_vapi_deregister(
+    mca_mpool_base_module_t* mpool, 
+    void *addr, 
+    size_t size);
 
 
 /**
@@ -90,7 +108,10 @@ void* mca_mpool_vapi_realloc(mca_mpool_base_module_t* mpool, void* addr, size_t
   */
 void mca_mpool_vapi_free(mca_mpool_base_module_t* mpool, void *);
 
-void* mca_common_vapi_segment_alloc(size_t* size, void* user_in, void** user_out);
+void* mca_common_vapi_segment_alloc(
+    struct mca_mpool_base_module_t* module, 
+    size_t* size, 
+    struct mca_bmi_base_registration_t** registration);
 
 #if defined(c_plusplus) || defined(__cplusplus)
 }

src/mca/mpool/vapi/mpool_vapi_component.c

@@ -28,7 +28,9 @@
  * Local functions
  */
 static int mca_mpool_vapi_open(void);
-static mca_mpool_base_module_t* mca_mpool_vapi_init(void* user);
+static mca_mpool_base_module_t* mca_mpool_vapi_init(
+    struct mca_bmi_base_module_t* module,
+    struct mca_bmi_base_resources_t* resources);
 
 mca_mpool_vapi_component_t mca_mpool_vapi_component = {
     {
@@ -70,16 +72,6 @@ static char* mca_mpool_vapi_param_register_string(
     return param_value;
 }
 
-static  int mca_mpool_vapi_param_register_int(
-    const char* param_name,
-    int default_value)
-{
-    int id = mca_base_param_register_int("mpool","vapi",param_name,NULL,default_value);
-    int param_value = default_value;
-    mca_base_param_lookup_int(id,&param_value);
-    return param_value;
-}
-
 
 /**
   * component open/close/init function
@@ -97,38 +89,37 @@ static int mca_mpool_vapi_open(void)
 }
 
 /* Allocates a segment of memory and registers with IB, user_out returns the memory handle. */ 
-void* mca_common_vapi_segment_alloc(size_t* size, void* user_in, void** user_out){
+void* mca_common_vapi_segment_alloc(
-    int rc; 
+    struct mca_mpool_base_module_t* mpool,
-    mca_mpool_base_module_t* mpool = (mca_mpool_base_module_t*) user_in; 
+    size_t* size, 
-    
+    struct mca_bmi_base_registration_t** registration)
-    void* addr = (void*)malloc((*size) + mca_mpool_vapi_component.page_size); 
+{
-    addr = (void*)  ALIGN_ADDR(addr, mca_mpool_vapi_component.page_size_log); 
+    void* addr_malloc = (void*)malloc((*size) + mca_mpool_vapi_component.page_size); 
-    if(OMPI_SUCCESS !=  mpool->mpool_register(mpool, addr, *size, user_out)) { 
+    void* addr = (void*)  ALIGN_ADDR(addr_malloc, mca_mpool_vapi_component.page_size_log); 
+    if(OMPI_SUCCESS !=  mpool->mpool_register(mpool, addr, *size, registration)) { 
+        free(addr_malloc);
         return NULL; 
     } 
     return addr; 
 }
 
-
 /* Allocates a segment of memory and registers with IB, user_out returns the memory handle. */ 
-
+static mca_mpool_base_module_t* mca_mpool_vapi_init(
-static mca_mpool_base_module_t* mca_mpool_vapi_init(void * user_in)
+    struct mca_bmi_base_module_t* bmi,
+    struct mca_bmi_base_resources_t* resources)
 {
- 
+    mca_mpool_vapi_module_t* mpool_module; 
-   
+    mca_allocator_base_component_t* allocator_component;
-    
+    long page_size = mca_mpool_vapi_component.page_size; 
-    long page_size; 
+
-    page_size = mca_mpool_vapi_component.page_size; 
     mca_mpool_vapi_component.page_size_log = 0; 
     while(page_size > 1){ 
         page_size = page_size >> 1; 
         mca_mpool_vapi_component.page_size_log++; 
     }    
-    mca_allocator_base_component_t* allocator_component = mca_allocator_component_lookup( 
-                                                                                         mca_mpool_vapi_component.vapi_allocator_name);
     
-    
+    /* if specified allocator cannout be loaded - look for an alternative */
-  /* if specified allocator cannout be loaded - look for an alternative */
+    allocator_component = mca_allocator_component_lookup(mca_mpool_vapi_component.vapi_allocator_name);
     if(NULL == allocator_component) {
         if(ompi_list_get_size(&mca_allocator_base_components) == 0) {
             mca_base_component_list_item_t* item = (mca_base_component_list_item_t*)
@@ -143,34 +134,18 @@ static mca_mpool_base_module_t* mca_mpool_vapi_init(void * user_in)
         }
     }
     
-    mca_mpool_vapi_module_t* mpool_module; 
     mpool_module = (mca_mpool_vapi_module_t*)malloc(sizeof(mca_mpool_vapi_module_t)); 
     mca_mpool_vapi_module_init(mpool_module); 
     
     /* setup allocator  TODO fix up */
-    mpool_module->hca_pd = *(mca_common_vapi_hca_pd_t*) user_in; 
+    mpool_module->hca_pd = *resources;
     mpool_module->vapi_allocator = 
-      allocator_component->allocator_init(true,
+      allocator_component->allocator_init(true, mca_common_vapi_segment_alloc, NULL, &mpool_module->super);
-                                          mca_common_vapi_segment_alloc, NULL, mpool_module);
     if(NULL == mpool_module->vapi_allocator) {
       ompi_output(0, "mca_mpool_vapi_init: unable to initialize allocator");
       return NULL;
     }
-   
-    
     return &mpool_module->super;
 }
 
 
-
-
-
-
-
-
-
-
-
-
-
-

src/mca/mpool/vapi/mpool_vapi_module.c

@@ -32,7 +32,7 @@ void mca_mpool_vapi_module_init(mca_mpool_vapi_module_t* mpool)
     mpool->super.mpool_realloc = mca_mpool_vapi_realloc; 
     mpool->super.mpool_free = mca_mpool_vapi_free; 
     mpool->super.mpool_register = mca_mpool_vapi_register; 
-    mpool->super.mpool_deregister = NULL; 
+    mpool->super.mpool_deregister = mca_mpool_vapi_deregister; 
     mpool->super.mpool_finalize = NULL; 
 }
 
@@ -40,32 +40,35 @@ void mca_mpool_vapi_module_init(mca_mpool_vapi_module_t* mpool)
 /**
   * allocate function 
   */
-void* mca_mpool_vapi_alloc(mca_mpool_base_module_t* mpool, size_t size, size_t align, void** user_out)
+void* mca_mpool_vapi_alloc(
+    mca_mpool_base_module_t* mpool, 
+    size_t size, 
+    size_t align, 
+    struct mca_bmi_base_registration_t** registration)
 {
-  mca_mpool_vapi_module_t* mpool_vapi = (mca_mpool_vapi_module_t*)mpool; 
+    mca_mpool_vapi_module_t* mpool_vapi = (mca_mpool_vapi_module_t*)mpool; 
-  return  mpool_vapi->vapi_allocator->alc_alloc(mpool_vapi->vapi_allocator, size, align, user_out);
+    return mpool_vapi->vapi_allocator->alc_alloc(mpool_vapi->vapi_allocator, size, align, registration);
-  
 }
 
 
 /* 
  * register memory 
  */ 
-int mca_mpool_vapi_register(mca_mpool_base_module_t* mpool, void *addr, size_t size, void** user_out){
+int mca_mpool_vapi_register(mca_mpool_base_module_t* mpool, void *addr, size_t size, 
+    struct mca_bmi_base_registration_t** registration){
     
     mca_mpool_vapi_module_t * mpool_module = (mca_mpool_vapi_module_t*) mpool; 
     VAPI_mrw_t mr_in, mr_out;
   
     VAPI_ret_t ret; 
-    mca_common_vapi_memhandle_t* mem_hndl; 
+    mca_bmi_base_registration_t* mem_hndl; 
     memset(&mr_in, 0, sizeof(VAPI_mrw_t)); 
     memset(&mr_out, 0, sizeof(VAPI_mrw_t)); 
     
 
-    *user_out = (void*) malloc(sizeof(mca_common_vapi_memhandle_t)); 
+    *registration = (void*) malloc(sizeof(mca_bmi_base_registration_t)); 
-    
+    mem_hndl = (mca_bmi_base_registration_t*) *registration;  
-    mem_hndl = (mca_common_vapi_memhandle_t*) *user_out;  
+    memset(mem_hndl, 0, sizeof(mca_bmi_base_registration_t*)); 
-    memset(mem_hndl, 0, sizeof(mca_common_vapi_memhandle_t*)); 
     mem_hndl->hndl = VAPI_INVAL_HNDL; 
     
     
@@ -119,10 +122,14 @@ int mca_mpool_vapi_deregister(mca_mpool_base_module_t* mpool, void *addr, size_t
 /**
   * realloc function 
   */
-void* mca_mpool_vapi_realloc(mca_mpool_base_module_t* mpool, void* addr, size_t size, void** user_out)
+void* mca_mpool_vapi_realloc(
+    mca_mpool_base_module_t* mpool, 
+    void* addr, 
+    size_t size, 
+    struct mca_bmi_base_registration_t** registration)
 {
     mca_mpool_vapi_module_t* mpool_vapi = (mca_mpool_vapi_module_t*)mpool; 
-    return mpool_vapi->vapi_allocator->alc_realloc(mpool_vapi->vapi_allocator, addr, size, user_out);
+    return mpool_vapi->vapi_allocator->alc_realloc( mpool_vapi->vapi_allocator, addr, size, registration);
 }
 
 /**

src/mca/pml/base/pml_base_bsend.c

@@ -42,7 +42,10 @@ static int32_t          mca_pml_bsend_init = 0;
 /*
  * Routine to return pages to sub-allocator as needed 
  */
-static void* mca_pml_bsend_alloc_segment(size_t* size_inout, void* user_in, void** user_out)
+static void* mca_pml_bsend_alloc_segment(
+    struct mca_mpool_base_module_t* module, 
+    size_t* size_inout, 
+    struct mca_bmi_base_registration_t** registration)
 {
     void *addr;
     size_t size = *size_inout;
@@ -54,6 +57,7 @@ static void* mca_pml_bsend_alloc_segment(size_t* size_inout, void* user_in, void
     addr = mca_pml_bsend_addr;
     mca_pml_bsend_addr += size;
     *size_inout = size;
+    *registration = NULL;
     return addr;
 }
 

src/mca/pml/ob1/.ompi_unignore

@@ -1,3 +1,4 @@
+ompi
 twoodall
 gshipman
 bosilca

src/mca/pml/ob1/pml_ob1_recvreq.c

@@ -346,6 +346,7 @@ void mca_pml_ob1_recv_request_schedule(mca_pml_ob1_recv_request_t* recvreq)
                 dst = ep->bmi_prepare_dst(
                     ep->bmi,
                     ep->bmi_endpoint,
+                    NULL,
                     &recvreq->req_recv.req_convertor,
                     0,
                     &size);

src/mca/pml/ob1/pml_ob1_sendreq.c

@@ -403,6 +403,7 @@ int mca_pml_ob1_send_request_schedule(mca_pml_ob1_send_request_t* sendreq)
                 des = ep->bmi_prepare_src(
                     ep->bmi,
                     ep->bmi_endpoint,
+                    NULL,
                     &sendreq->req_send.req_convertor,
                     sizeof(mca_pml_ob1_frag_hdr_t),
                     &size);
@@ -613,6 +614,7 @@ void mca_pml_ob1_send_request_put(
     des = bmi->bmi_prepare_src(
         bmi, 
         ep->bmi_endpoint,
+        NULL,
         &sendreq->req_send.req_convertor, 
         0,
         &size);