Updated the mpool interfaces for user specified data

This commit was SVN r5953.

src/mca/mpool/base/base.h

@@ -93,7 +93,7 @@ OMPI_DECLSPEC int mca_mpool_base_close(void);
 OMPI_DECLSPEC mca_mpool_base_component_t* mca_mpool_base_component_lookup(const char* name);
 OMPI_DECLSPEC mca_mpool_base_module_t* mca_mpool_base_module_lookup(const char* name);
 OMPI_DECLSPEC mca_mpool_base_module_t* mca_mpool_base_module_init(const char* name);
-
+OMPI_DECLSPEC mca_mpool_base_module_t* mca_mpool_base_module_create(const char* name, void* user); 
 
 /*
  * Globals

src/mca/mpool/base/mpool_base_lookup.c

@@ -62,7 +62,31 @@ mca_mpool_base_module_t* mca_mpool_base_module_lookup(const char* name)
 }
 
 
+mca_mpool_base_module_t* mca_mpool_base_module_create(const char* name, void* user) 
+{
+    
+    mca_mpool_base_component_t* component = NULL; 
+    mca_mpool_base_module_t* module = NULL; 
+    ompi_list_item_t* item;
+    for (item = ompi_list_get_first(&mca_mpool_base_components);
+         item != ompi_list_get_end(&mca_mpool_base_components);
+         item = ompi_list_get_next(item)) {
+         mca_base_component_list_item_t *cli = 
+           (mca_base_component_list_item_t *) item;
+         component = 
+           (mca_mpool_base_component_t *) cli->cli_component;
+         if(strcmp(component->mpool_version.mca_component_name, name) == 0) {
+             break;
+         }
+    }
 
-
-
-
+    if(NULL == component)  
+        return NULL;
+    module = component->mpool_init(user); 
+    mca_mpool_base_selected_module_t *sm;
+    sm = OBJ_NEW(mca_mpool_base_selected_module_t); 
+    sm->mpool_component = component; 
+    sm->mpool_module = module; 
+    ompi_list_append(&mca_mpool_base_modules, (ompi_list_item_t*) sm); 
+    return module; 
+}

src/mca/mpool/mpool.h

@@ -58,7 +58,7 @@ typedef void (*mca_mpool_base_module_free_fn_t)(struct mca_mpool_base_module_t*
 /**
   * register memory
   */
-typedef int (*mca_mpool_base_module_register_fn_t)(struct mca_mpool_base_module_t* mpool, void * addr, size_t size, void** user_out);
+typedef int (*mca_mpool_base_module_register_fn_t)(struct mca_mpool_base_module_t* mpool, void * addr, size_t size, void* user_out);
 
 /**
   * deregister memory

src/mca/mpool/sm/mpool_sm.h

@@ -62,12 +62,12 @@ void* mca_mpool_sm_base(mca_mpool_base_module_t*);
 /**
   *  Allocate block of shared memory.
   */
-void* mca_mpool_sm_alloc( mca_mpool_base_module_t* mpool, size_t size, size_t align, void* user_out);
+void* mca_mpool_sm_alloc( mca_mpool_base_module_t* mpool, size_t size, size_t align, void** user_out);
 
 /**
   * realloc function typedef
   */
-void* mca_mpool_sm_realloc(mca_mpool_base_module_t* mpool, void* addr, size_t size, void* user_out);
+void* mca_mpool_sm_realloc(mca_mpool_base_module_t* mpool, void* addr, size_t size, void** user_out);
 
 /**
   * free function typedef

src/mca/mpool/sm/mpool_sm_module.c

@@ -58,7 +58,7 @@ void* mca_mpool_sm_base(mca_mpool_base_module_t* mpool)
 /**
   * allocate function 
   */
-void* mca_mpool_sm_alloc(mca_mpool_base_module_t* mpool, size_t size, size_t align, void* user_out)
+void* mca_mpool_sm_alloc(mca_mpool_base_module_t* mpool, size_t size, size_t align, void** user_out)
 {
   mca_mpool_sm_module_t* mpool_sm = (mca_mpool_sm_module_t*)mpool; 
   return mpool_sm->sm_allocator->alc_alloc(mpool_sm->sm_allocator, size, align, user_out);
@@ -67,7 +67,7 @@ void* mca_mpool_sm_alloc(mca_mpool_base_module_t* mpool, size_t size, size_t ali
 /**
   * realloc function 
   */
-void* mca_mpool_sm_realloc(mca_mpool_base_module_t* mpool, void* addr, size_t size, void* user_out)
+void* mca_mpool_sm_realloc(mca_mpool_base_module_t* mpool, void* addr, size_t size, void** user_out)
 {
   mca_mpool_sm_module_t* mpool_sm = (mca_mpool_sm_module_t*)mpool; 
   return mpool_sm->sm_allocator->alc_realloc(mpool_sm->sm_allocator, addr, size, user_out);

src/mca/mpool/vapi/configure.params

@@ -17,7 +17,7 @@
 
 # Specific to this module
 
-PARAM_INIT_FILE=mpool_vapi.c
+PARAM_INIT_FILE=mpool_vapi_module.c
 PARAM_CONFIG_HEADER_FILE="vapi_config.h"
 PARAM_CONFIG_FILES="Makefile"
 

src/mca/mpool/vapi/mpool_vapi.h

@@ -42,7 +42,7 @@ OMPI_COMP_EXPORT extern mca_mpool_vapi_component_t mca_mpool_vapi_component;
 struct mca_mpool_vapi_module_t {
   mca_mpool_base_module_t super;
   mca_allocator_base_module_t * vapi_allocator; 
-  mca_common_vapi_mem_reg_t mem_reg;
+  mca_common_vapi_hca_pd_t  hca_pd;
   mca_common_vapi_memhandle_t mem_hndl; 
 }; typedef struct mca_mpool_vapi_module_t mca_mpool_vapi_module_t; 
 

src/mca/mpool/vapi/mpool_vapi_component.c

@@ -86,38 +86,52 @@ static  int mca_mpool_vapi_param_register_int(
 static int mca_mpool_vapi_open(void)
 {
     /* register VAPI component parameters */
-    mca_mpool_vapi_component.vapi_size =
-        mca_mpool_vapi_param_register_int("size", 512*1024*1024);
+    
     mca_mpool_vapi_component.vapi_allocator_name =
         mca_mpool_vapi_param_register_string("allocator", "bucket");
     return OMPI_SUCCESS;
 }
 
 /* Allocates a segment of memory and registers with IB, user_out returns the memory handle. */ 
-void* mca_common_vapi_segment_alloc(size_t* size, void* user_in, void* user_out){
+void* mca_common_vapi_segment_alloc(size_t* size, void* user_in, void** user_out){
   
   mca_mpool_vapi_module_t * mpool_module = (mca_mpool_vapi_module_t*)user_in; 
-  void* addr = (void*)malloc(size); 
+  void* addr = (void*)malloc((*size)); 
   
   VAPI_mrw_t mr_in, mr_out;
-  mca_common_vapi_memhandle_t* mem_hndl = (mca_common_vapi_memhandle_t*) user_out;  
-  mr_in = *mpool_module->mem_reg.req_mrw_p; 
+  memset(&mr_in, 0, sizeof(VAPI_mrw_t)); 
+  memset(&mr_out, 0, sizeof(VAPI_mrw_t)); 
+  
+  VAPI_ret_t ret; 
+  mca_common_vapi_memhandle_t* mem_hndl; 
+  *user_out = (void*) malloc(sizeof(mca_common_vapi_memhandle_t)); 
+  
+  mem_hndl = (mca_common_vapi_memhandle_t*) *user_out;  
+  memset(mem_hndl, 0, sizeof(mca_common_vapi_memhandle_t*)); 
+  mem_hndl->hndl = VAPI_INVAL_HNDL; 
+  
+  
   mr_in.acl = VAPI_EN_LOCAL_WRITE | VAPI_EN_REMOTE_WRITE;
   mr_in.l_key = 0;
   mr_in.r_key = 0;
-  mr_in.pd_hndl = mpool_module->mem_reg.pd_tag;
-  mr_in.size = size;
+  mr_in.pd_hndl = mpool_module->hca_pd.pd_tag;
+  mr_in.size = *size;
   mr_in.start = (VAPI_virt_addr_t) (MT_virt_addr_t) addr;
   mr_in.type = VAPI_MR;
   
 
-  VAPI_register_mr(
-                    mpool_module->mem_reg.hca, 
+  ret = VAPI_register_mr(
+                    mpool_module->hca_pd.hca, 
                     &mr_in, 
-                    &mem_hndl->hdnl, 
+                    &mem_hndl->hndl, 
                     &mr_out
                     ); 
   
+  if(VAPI_OK != ret){ 
+      ompi_output(0, "error pinning vapi memory\n"); 
+      return NULL; 
+  }
+  
   mem_hndl->l_key = mr_out.l_key; 
   mem_hndl->r_key = mr_out.r_key; 
   return addr; 
@@ -149,9 +163,10 @@ static mca_mpool_base_module_t* mca_mpool_vapi_init(void * user_in)
     mca_mpool_vapi_module_init(mpool_module); 
     
     /* setup allocator  TODO fix up */
+    mpool_module->hca_pd = *(mca_common_vapi_hca_pd_t*) user_in; 
     mpool_module->vapi_allocator = 
       allocator_component->allocator_init(true,
-                                          mca_common_vapi_segment_alloc, NULL, user_in);
+                                          mca_common_vapi_segment_alloc, NULL, mpool_module);
     if(NULL == mpool_module->vapi_allocator) {
       ompi_output(0, "mca_mpool_vapi_init: unable to initialize allocator");
       return NULL;

src/mca/mpool/vapi/mpool_vapi_module.c

@@ -18,7 +18,7 @@
 #include <string.h>
 #include "util/output.h"
 #include "mca/mpool/vapi/mpool_vapi.h"
-#include "mca/common/vapi/common/vapi_mem_reg.h"
+#include "mca/common/vapi/vapi_mem_reg.h"
 
 
 /* 
@@ -43,7 +43,7 @@ void mca_mpool_vapi_module_init(mca_mpool_vapi_module_t* mpool)
 void* mca_mpool_vapi_alloc(mca_mpool_base_module_t* mpool, size_t size, size_t align, void** user_out)
 {
   mca_mpool_vapi_module_t* mpool_vapi = (mca_mpool_vapi_module_t*)mpool; 
-  return mpool_vapi->vapi_allocator->alc_alloc(mpool_vapi->vapi_allocator, size, align);
+  return mpool_vapi->vapi_allocator->alc_alloc(mpool_vapi->vapi_allocator, size, align, user_out);
 }
 
 /**
@@ -52,7 +52,7 @@ void* mca_mpool_vapi_alloc(mca_mpool_base_module_t* mpool, size_t size, size_t a
 void* mca_mpool_vapi_realloc(mca_mpool_base_module_t* mpool, void* addr, size_t size, void** user_out)
 {
   mca_mpool_vapi_module_t* mpool_vapi = (mca_mpool_vapi_module_t*)mpool; 
-  return mpool_vapi->vapi_allocator->alc_realloc(mpool_vapi->vapi_allocator, addr, size);
+  return mpool_vapi->vapi_allocator->alc_realloc(mpool_vapi->vapi_allocator, addr, size, user_out);
 }
 
 /**