moving mvapi to vapi.. You should all still be able to build, let me know if

you are not able to. 

This commit was SVN r6226.

src/mca/mpool/mvapi/.ompi_unignore

@@ -0,0 +1 @@
+gshipman

src/mca/mpool/mvapi/Makefile.am

@@ -0,0 +1,50 @@
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University.
+#                         All rights reserved.
+# Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+#                         All rights reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+
+# Use the top-level Makefile.options
+
+include $(top_ompi_srcdir)/config/Makefile.options
+
+sources = \
+	mpool_vapi.h \
+	mpool_vapi_module.c \
+	mpool_vapi_component.c
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if OMPI_BUILD_mpool_vapi_DSO
+component_noinst =
+component_install = mca_mpool_vapi.la
+else
+component_noinst = libmca_mpool_vapi.la
+component_install =
+endif
+
+# See src/mca/ptl/vapi/Makefile.am for an explanation of
+# libmca_common_vapi.la.
+
+mcacomponentdir = $(libdir)/openmpi
+mcacomponent_LTLIBRARIES = $(component_install)
+mca_mpool_vapi_la_SOURCES = $(sources)
+mca_mpool_vapi_la_LDFLAGS = -module -avoid-version
+
+noinst_LTLIBRARIES = $(component_noinst)
+libmca_mpool_vapi_la_SOURCES = $(sources)
+libmca_mpool_vapi_la_LDFLAGS = -module -avoid-version
+
+

src/mca/mpool/mvapi/configure.params

@@ -0,0 +1,22 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University.
+#                         All rights reserved.
+# Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+#                         All rights reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+
+# Specific to this module
+
+PARAM_INIT_FILE=mpool_vapi_component.c
+PARAM_CONFIG_HEADER_FILE="vapi_config.h"
+PARAM_CONFIG_FILES="Makefile"

src/mca/mpool/mvapi/configure.stub

@@ -0,0 +1,157 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University.
+#                         All rights reserved.
+# Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+#                         All rights reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+
+#
+# Main function.  This will be invoked in the middle of the templated
+# configure script.
+#
+AC_DEFUN([MCA_CONFIGURE_STUB],[
+
+    # Additional --with flags that can be specified
+
+    AC_ARG_WITH(btl-mvapi, 
+        AC_HELP_STRING([--with-btl-mvapi=IBDIR],
+                       [Specify the installation directory of IB (should enable the correct automatic determination of using the 32 or 64 bit library, if both are present under IBDIR/lib and IBDIR/lib64)]))
+    AC_ARG_WITH(btl-mvapi-libdir,
+        AC_HELP_STRING([--with-btl-mvapi-libdir=IBLIBDIR],
+                       [directory where the IB library can be found, if it is not in $IBDIR/lib or $IBDIR/lib64]))
+
+    # Quick sanity check
+
+    if test "$with_btl_mvapi" = "no"; then
+        AC_MSG_WARN([*** --without-btl-mvapi specified -- aborting])
+        AC_MSG_ERROR([Will not continue])
+    fi
+
+    # Find the right IBDIR
+
+    if test "$with_btl_mvapi" != "" -a "$with_btl_mvapi" != "yes" ; then
+        IBDIR="$with_btl_mvapi"
+        IBLIBDIR="$with_btl_mvapi"
+    fi
+    if test "$with_btl_mvapi_libdir" != "" -a "$with_btl_mvapi_libdir" != "yes" -a \
+        "$with_btl_mvapi_libdir" != "no"; then
+        IBLIBDIR="$with_btl_mvapi_libdir"
+    fi
+
+    # Add to CPPFLAGS if necessary
+
+    EXTRA_CPPFLAGS=
+    if test "$IBDIR" != ""; then
+        if test -d "$IBDIR/include"; then
+            EXTRA_CPPFLAGS="-I$IBDIR/include"
+        else
+            AC_MSG_WARN([*** Warning: cannot find $IBDIR/include])
+            AC_MSG_WARN([*** Will still try to configure ib btl anyway...])
+        fi
+        if test "$IBDIR" != "" -a -d "$IBDIR/wrap"; then
+            EXTRA_CPPFLAGS="-I$IBDIR/wrap $EXTRA_CPPFLAGS"
+        else
+            AC_MSG_WARN([*** Warning: cannot find $IBDIR/wrap])
+            AC_MSG_WARN([*** Will still try to configure ib btl anyway...])
+        fi
+    fi
+
+    # See if we can find vapi.h
+
+    CPPFLAGS="$CPPFLAGS $EXTRA_CPPFLAGS"
+    AC_CHECK_HEADERS(vapi.h,,
+        AC_MSG_ERROR([*** Cannot find working vapi.h]))
+
+    # Note that it is possible to find the library even if -L is not
+    # specified, if the LD_LIBRARY_PATH includes the directory where
+    # the shared ib library is kept.  Hence, we unset LD_LIBRARY_PATH
+    # before running this test.
+
+    LD_LIBRARY_PATH_save="$LD_LIBRARY_PATH"
+    unset LD_LIBRARY_PATH
+
+    # Helpfer function to try to find libvapi (called from below).  In
+    # some versions of Mellanox (v3.1), we need to expliitly link in
+    # the thread libraries.  #$%#@$%@%#$!!!
+
+    
+    # Many vapi.h's have horrid semantics and don't obey ISOC99
+    # standards.  So we have to turn off flags like -pedantic.  Sigh.
+
+    CFLAGS="`echo $CFLAGS | sed 's/-pedantic//g'`"
+
+
+
+mca_btl_mvapi_try_find_libvapi() {
+    func1=[$]1
+    func2=[$]2
+
+    LDFLAGS="$LDFLAGS $EXTRA_LDFLAGS"
+    vapi_badness=
+    AC_CHECK_LIB([vapi], [$func1], [], [vapi_badness=true],
+                 [-lmtl_common -lmpga -lmosal])
+    if test "$vapi_badness" != ""; then
+         AC_CHECK_LIB([pthread], [pthread_create], 
+		      [pthread=yes LIBS="$LIBS -lpthread"],
+		      [pthread=no])
+	 if test "$pthread" = "yes"; then
+	     AC_CHECK_LIB([vapi], [$func2], [], [],
+                          [-lmtl_common -lmpga -lmosal])
+	fi
+    fi    
+}
+
+    # The libraries may be in $IBDIR/lib or $IBDIR/lib64.  Try them
+    # both.
+
+    LIBS_save="$LIBS"
+    LDFLAGS_save="$LDFLAGS"
+    LIBS="$LIBS -lmosal -lmpga -lmtl_common"
+    LIBS_orig="$LIBS"
+
+    EXTRA_LDFLAGS=
+    if test -d "$IBLIBDIR/lib"; then
+        EXTRA_LDFLAGS="-L$IBLIBDIR/lib"
+        LDFLAGS="$LDFLAGS $EXTRA_LDFLAGS"
+        mca_btl_mvapi_try_find_libvapi VAPI_open_hca VAPI_query_hca_cap
+        if test "$LIBS" != "$LIBS_orig"; then
+            echo "--> found libvapi in $IBLIBDIR/lib"
+        fi
+    fi
+
+    if test "$LIBS" = "$LIBS_orig" -a -d "$IBLIBDIR/lib64"; then
+        EXTRA_LDFLAGS="-L$IBLIBDIR/lib64"
+        LDFLAGS="$LDFLAGS_save $EXTRA_LDFLAGS"
+        mca_btl_mvapi_try_find_libvapi EVAPI_list_hcas EVAPI_open_hca
+        if test "$LIBS" != "$LIBS_orig"; then
+            echo "--> found libvapi in $IBLIBDIR/lib64"
+        fi
+    fi
+
+    if test "$LIBS" = "$LIBS_orig"; then
+      AC_MSG_ERROR([*** Cannot find working libvapi.])
+    fi
+    LD_LIBRARY_PATH="$LD_LIBRARY_PATH_save"
+    LIBS="$LIBS -lmtl_common -lmpga"
+
+    #
+    # Save extra compiler/linker flags so that they can be added in
+    # the wrapper compilers, if necessary
+    #
+
+    WRAPPER_EXTRA_LDFLAGS="$EXTRA_LDFLAGS"
+    WRAPPER_EXTRA_LIBS="-lvapi -lmtl_common -lmpga -lmosal"
+])dnl
+
+
+

src/mca/mpool/mvapi/mpool_vapi.h

@@ -0,0 +1,169 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+/**
+ * @file
+ */
+#ifndef MCA_MPOOL_VAPI_H
+#define MCA_MPOOL_VAPI_H
+
+#include "class/ompi_list.h"
+#include "class/ompi_free_list.h"
+#include "event/event.h"
+#include "mca/mpool/mpool.h"
+#include "mca/allocator/allocator.h"
+#include "mca/common/vapi/vapi_mem_reg.h" 
+
+#if defined(c_plusplus) || defined(__cplusplus)
+extern "C" {
+#endif
+
+
+static inline  void * DOWN_ALIGN_ADDR(void * addr, uint32_t cnt) { 
+    return (void*)((MT_virt_addr_t)(addr) & (~((MT_virt_addr_t)0) << (cnt))); 
+}
+
+static inline void*  ALIGN_ADDR(void* addr, uint32_t cnt ) { 
+    DOWN_ALIGN_ADDR(((addr) +  ~(~((MT_virt_addr_t)0) << (cnt))), (cnt)); 
+    return addr;
+} 
+
+
+struct mca_mpool_vapi_component_t {
+    mca_mpool_base_component_t super;
+    char*  vapi_allocator_name;
+    long page_size; 
+    long page_size_log; 
+};
+
+typedef struct mca_mpool_vapi_component_t mca_mpool_vapi_component_t;
+
+OMPI_COMP_EXPORT extern mca_mpool_vapi_component_t mca_mpool_vapi_component;
+
+
+
+
+
+struct mca_mpool_base_resources_t {
+  VAPI_hca_hndl_t hca;   /* the hca (nic) */ 
+  VAPI_pd_hndl_t pd_tag; /* the protection domain */ 
+}; 
+typedef struct mca_mpool_base_resources_t mca_mpool_base_resources_t;  
+
+struct mca_mpool_vapi_module_t {
+    mca_mpool_base_module_t super;
+    mca_allocator_base_module_t * vapi_allocator; 
+    struct mca_mpool_base_resources_t  hca_pd;
+}; typedef struct mca_mpool_vapi_module_t mca_mpool_vapi_module_t; 
+
+
+struct mca_mpool_vapi_registration_t {
+    mca_mpool_base_registration_t base_reg; 
+    VAPI_mr_hndl_t                  hndl;
+    /* Memory region handle */
+    
+    VAPI_lkey_t                     l_key;
+    /* Local key to registered memory, needed for
+     * posting send/recv requests */
+    
+    VAPI_rkey_t                     r_key;
+    /* Remote key to registered memory, need to send this
+     * to remote processes for incoming RDMA ops */
+    bool is_leave_pinned; 
+
+};
+typedef struct mca_mpool_vapi_registration_t mca_mpool_vapi_registration_t;
+OMPI_DECLSPEC OBJ_CLASS_DECLARATION(mca_mpool_vapi_registration_t); 
+
+
+
+
+/* 
+ *  Initializes the mpool module. 
+ */ 
+void mca_mpool_vapi_module_init(mca_mpool_vapi_module_t* mpool); 
+
+
+/*
+ *  Returns base address of shared memory mapping.
+ */
+void* mca_mpool_vapi_base(mca_mpool_base_module_t*);
+
+/**
+  *  Allocate block of shared memory.
+  */
+void* mca_mpool_vapi_alloc( 
+    mca_mpool_base_module_t* mpool, 
+    size_t size, 
+    size_t align, 
+    mca_mpool_base_registration_t** registration);
+
+/**
+  * realloc function typedef
+  */
+void* mca_mpool_vapi_realloc(
+    mca_mpool_base_module_t* mpool, 
+    void* addr, 
+    size_t size, 
+    mca_mpool_base_registration_t** registration);
+
+/**
+  * register function typedef
+  */
+int mca_mpool_vapi_register(
+    mca_mpool_base_module_t* mpool, 
+    void *addr, 
+    size_t size, 
+    mca_mpool_base_registration_t** registration);
+
+int mca_mpool_vapi_deregister(
+    mca_mpool_base_module_t* mpool, 
+    void *addr, 
+    size_t size, 
+    mca_mpool_base_registration_t* );
+
+
+/**
+  * free function typedef
+  */
+void mca_mpool_vapi_free(mca_mpool_base_module_t* mpool, 
+                         void * addr, 
+                         mca_mpool_base_registration_t* registration);
+
+void* mca_common_vapi_segment_alloc(
+                                    struct mca_mpool_base_module_t* module, 
+                                    size_t* size, 
+                                    mca_mpool_base_registration_t** registration);
+
+#if defined(c_plusplus) || defined(__cplusplus)
+}
+#endif
+
+#endif
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

src/mca/mpool/mvapi/mpool_vapi_component.c

@@ -0,0 +1,182 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include "util/output.h"
+#include "mca/base/base.h"
+#include "mca/base/mca_base_param.h"
+#include "mca/allocator/base/base.h"
+#include "mpool_vapi.h"
+#include "util/proc_info.h"
+#include "util/sys_info.h"
+#include <unistd.h> 
+
+/*
+ * Local functions
+ */
+static int mca_mpool_vapi_open(void);
+static mca_mpool_base_module_t* mca_mpool_vapi_init(
+    struct mca_mpool_base_resources_t* resources);
+
+mca_mpool_vapi_component_t mca_mpool_vapi_component = {
+    {
+      /* First, the mca_base_component_t struct containing meta
+         information about the component itself */
+
+      {
+        /* Indicate that we are a mpool v1.0.0 component (which also
+           implies a specific MCA version) */
+
+        MCA_MPOOL_BASE_VERSION_1_0_0,
+
+        "vapi", /* MCA component name */
+        1,  /* MCA component major version */
+        0,  /* MCA component minor version */
+        0,  /* MCA component release version */
+        mca_mpool_vapi_open,  /* component open  */
+        NULL
+      },
+
+      /* Next the MCA v1.0.0 component meta data */
+      
+      {
+        /* Whether the component is checkpointable or not */
+        false
+      },
+
+      mca_mpool_vapi_init
+    }
+};
+
+
+
+static void mca_mpool_vapi_registration_constructor( mca_mpool_vapi_registration_t * registration ) 
+{ 
+    registration->is_leave_pinned = false; 
+}
+
+static void mca_mpool_vapi_registration_destructor( mca_mpool_vapi_registration_t * registration ) 
+{ 
+    mca_mpool_base_remove((void*) registration); 
+    registration->base_reg.mpool->mpool_deregister(
+                                                   registration->base_reg.mpool, 
+                                                   registration->base_reg.base, 
+                                                   0, 
+                                                   (mca_mpool_base_registration_t*) registration); 
+    
+    registration->base_reg.base = NULL; 
+    registration->base_reg.bound = NULL; 
+    registration->is_leave_pinned=false; 
+
+} 
+
+
+OBJ_CLASS_INSTANCE( 
+                   mca_mpool_vapi_registration_t, 
+                   mca_mpool_base_registration_t, 
+                   mca_mpool_vapi_registration_constructor, 
+                   mca_mpool_vapi_registration_destructor
+                   ); 
+
+
+
+
+static char* mca_mpool_vapi_param_register_string(
+    const char* param_name,
+    const char* default_value)
+{
+    char *param_value;
+    int id = mca_base_param_register_string("mpool","vapi",param_name,NULL,default_value);
+    mca_base_param_lookup_string(id, &param_value);
+    return param_value;
+}
+
+
+/**
+  * component open/close/init function
+  */
+static int mca_mpool_vapi_open(void)
+{
+    /* register VAPI component parameters */
+    
+    mca_mpool_vapi_component.vapi_allocator_name =
+        mca_mpool_vapi_param_register_string("allocator", "bucket");
+    /* get the page size for this architecture*/ 
+    mca_mpool_vapi_component.page_size = sysconf(_SC_PAGESIZE); 
+
+    return OMPI_SUCCESS;
+}
+
+/* Allocates a segment of memory and registers with IB, user_out returns the memory handle. */ 
+void* mca_common_vapi_segment_alloc(
+    struct mca_mpool_base_module_t* mpool,
+    size_t* size, 
+    mca_mpool_base_registration_t** registration)
+{
+    void* addr_malloc = (void*)malloc((*size) + mca_mpool_vapi_component.page_size); 
+    void* addr = (void*)  ALIGN_ADDR(addr_malloc, mca_mpool_vapi_component.page_size_log); 
+    if(OMPI_SUCCESS !=  mpool->mpool_register(mpool, addr, *size, registration)) { 
+        free(addr_malloc);
+        return NULL; 
+    } 
+    return addr; 
+}
+
+/* Allocates a segment of memory and registers with IB, user_out returns the memory handle. */ 
+static mca_mpool_base_module_t* mca_mpool_vapi_init(
+     struct mca_mpool_base_resources_t* resources)
+{
+    mca_mpool_vapi_module_t* mpool_module; 
+    mca_allocator_base_component_t* allocator_component;
+    long page_size = mca_mpool_vapi_component.page_size; 
+
+    mca_mpool_vapi_component.page_size_log = 0; 
+    while(page_size > 1){ 
+        page_size = page_size >> 1; 
+        mca_mpool_vapi_component.page_size_log++; 
+    }    
+    
+    /* if specified allocator cannout be loaded - look for an alternative */
+    allocator_component = mca_allocator_component_lookup(mca_mpool_vapi_component.vapi_allocator_name);
+    if(NULL == allocator_component) {
+        if(ompi_list_get_size(&mca_allocator_base_components) == 0) {
+            mca_base_component_list_item_t* item = (mca_base_component_list_item_t*)
+                ompi_list_get_first(&mca_allocator_base_components);
+            allocator_component = (mca_allocator_base_component_t*)item->cli_component;
+            ompi_output(0, "mca_mpool_vapi_init: unable to locate allocator: %s - using %s\n",
+                mca_mpool_vapi_component.vapi_allocator_name, allocator_component->allocator_version.mca_component_name);
+        } else {
+            ompi_output(0, "mca_mpool_vapi_init: unable to locate allocator: %s\n",
+                mca_mpool_vapi_component.vapi_allocator_name);
+            return NULL;
+        }
+    }
+    
+    mpool_module = (mca_mpool_vapi_module_t*)malloc(sizeof(mca_mpool_vapi_module_t)); 
+    mca_mpool_vapi_module_init(mpool_module); 
+    
+    /* setup allocator  TODO fix up */
+    mpool_module->hca_pd = *resources;
+    mpool_module->vapi_allocator = 
+        allocator_component->allocator_init(true, mca_common_vapi_segment_alloc, NULL, &mpool_module->super);
+    if(NULL == mpool_module->vapi_allocator) {
+      ompi_output(0, "mca_mpool_vapi_init: unable to initialize allocator");
+      return NULL;
+    }
+    return &mpool_module->super;
+}
+
+

src/mca/mpool/mvapi/mpool_vapi_module.c

@@ -0,0 +1,179 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include <string.h>
+#include "util/output.h"
+#include "mca/mpool/vapi/mpool_vapi.h"
+#include "mca/common/vapi/vapi_mem_reg.h"
+
+
+/* 
+ *  Initializes the mpool module.
+ */ 
+void mca_mpool_vapi_module_init(mca_mpool_vapi_module_t* mpool)
+{
+    mpool->super.mpool_component = &mca_mpool_vapi_component.super; 
+    mpool->super.mpool_base = NULL; /* no base .. */ 
+    mpool->super.mpool_alloc = mca_mpool_vapi_alloc; 
+    mpool->super.mpool_realloc = mca_mpool_vapi_realloc; 
+    mpool->super.mpool_free = mca_mpool_vapi_free; 
+    mpool->super.mpool_register = mca_mpool_vapi_register; 
+    mpool->super.mpool_deregister = mca_mpool_vapi_deregister; 
+    mpool->super.mpool_finalize = NULL; 
+}
+
+
+/**
+  * allocate function 
+  */
+void* mca_mpool_vapi_alloc(
+    mca_mpool_base_module_t* mpool, 
+    size_t size, 
+    size_t align, 
+    mca_mpool_base_registration_t** registration)
+{
+    mca_mpool_vapi_module_t* mpool_vapi = (mca_mpool_vapi_module_t*)mpool; 
+    return mpool_vapi->vapi_allocator->alc_alloc(mpool_vapi->vapi_allocator, size, align, registration);
+}
+
+
+/* 
+ * register memory 
+ */ 
+int mca_mpool_vapi_register(mca_mpool_base_module_t* mpool, 
+                            void *addr, 
+                            size_t size, 
+                            mca_mpool_base_registration_t** registration){
+    
+    
+    mca_mpool_vapi_module_t * mpool_module = (mca_mpool_vapi_module_t*) mpool; 
+    mca_mpool_vapi_registration_t * vapi_reg; 
+    VAPI_mrw_t mr_in, mr_out;
+    
+    VAPI_ret_t ret; 
+    
+    memset(&mr_in, 0, sizeof(VAPI_mrw_t)); 
+    memset(&mr_out, 0, sizeof(VAPI_mrw_t)); 
+    
+    *registration = (mca_mpool_base_registration_t*) OBJ_NEW(mca_mpool_vapi_registration_t);    /* (void*) malloc(sizeof(mca_mpool_base_registration_t));  */
+    vapi_reg = (mca_mpool_vapi_registration_t*) *registration; 
+    vapi_reg->base_reg.mpool = mpool;
+    
+    
+    vapi_reg->hndl = VAPI_INVAL_HNDL; 
+    
+    
+    mr_in.acl = VAPI_EN_LOCAL_WRITE | VAPI_EN_REMOTE_WRITE;
+    mr_in.l_key = 0;
+    mr_in.r_key = 0;
+    mr_in.pd_hndl = mpool_module->hca_pd.pd_tag;
+    mr_in.size = size;
+    mr_in.start = (VAPI_virt_addr_t) (MT_virt_addr_t) addr;
+    mr_in.type = VAPI_MR;
+    
+
+    ret = VAPI_register_mr(
+                           mpool_module->hca_pd.hca, 
+                           &mr_in, 
+                           &vapi_reg->hndl, 
+                           &mr_out
+                           ); 
+    
+    if(VAPI_OK != ret){ 
+        ompi_output(0, "error pinning vapi memory\n"); 
+        return OMPI_ERROR; 
+    }
+    
+    vapi_reg->l_key = mr_out.l_key; 
+    vapi_reg->r_key = mr_out.r_key; 
+    vapi_reg->base_reg.base = addr; 
+    vapi_reg->base_reg.bound = (void*) ((char*) addr + size - 1); 
+    
+    return OMPI_SUCCESS; 
+}
+
+
+/* 
+ * deregister memory 
+ */ 
+int mca_mpool_vapi_deregister(mca_mpool_base_module_t* mpool, void *addr, size_t size, 
+                              mca_mpool_base_registration_t* registration){
+    
+    VAPI_ret_t ret; 
+    mca_mpool_vapi_module_t * mpool_vapi = (mca_mpool_vapi_module_t*) mpool; 
+    mca_mpool_vapi_registration_t * vapi_reg; 
+    vapi_reg = (mca_mpool_vapi_registration_t*) registration; 
+    ret = VAPI_deregister_mr(
+                             mpool_vapi->hca_pd.hca, 
+                             vapi_reg->hndl 
+                             ); 
+    
+    if(VAPI_OK != ret){ 
+        ompi_output(0, "%s: error unpinning vapi memory\n", __func__); 
+        return OMPI_ERROR; 
+    }
+    free(registration); 
+    return OMPI_SUCCESS; 
+}
+
+/**
+  * realloc function 
+  */
+void* mca_mpool_vapi_realloc(
+    mca_mpool_base_module_t* mpool, 
+    void* addr, 
+    size_t size, 
+    mca_mpool_base_registration_t** registration)
+{
+    mca_mpool_vapi_module_t* mpool_vapi = (mca_mpool_vapi_module_t*)mpool; 
+    return mpool_vapi->vapi_allocator->alc_realloc( mpool_vapi->vapi_allocator, addr, size, registration);
+}
+
+/**
+  * free function 
+  */
+void mca_mpool_vapi_free(mca_mpool_base_module_t* mpool, void * addr,
+                         mca_mpool_base_registration_t* registration)
+{
+    
+    mca_mpool_vapi_module_t* mpool_vapi = (mca_mpool_vapi_module_t*)mpool; 
+    mpool_vapi->super.mpool_deregister(mpool, addr, 0, registration); 
+    mpool_vapi->vapi_allocator->alc_free(mpool_vapi->vapi_allocator, addr);
+    
+    
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+