gm memory pool

This commit was SVN r6228.

src/mca/mpool/gm/.ompi_unignore

@@ -0,0 +1,3 @@
+ompi
+gshipman
+twoodall

src/mca/mpool/gm/Makefile.am

@@ -0,0 +1,50 @@
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University.
+#                         All rights reserved.
+# Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+#                         All rights reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+
+# Use the top-level Makefile.options
+
+include $(top_ompi_srcdir)/config/Makefile.options
+
+sources = \
+	mpool_gm.h \
+	mpool_gm_module.c \
+	mpool_gm_component.c
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if OMPI_BUILD_mpool_gm_DSO
+component_noinst =
+component_install = mca_mpool_gm.la
+else
+component_noinst = libmca_mpool_gm.la
+component_install =
+endif
+
+# See src/mca/ptl/gm/Makefile.am for an explanation of
+# libmca_common_gm.la.
+
+mcacomponentdir = $(libdir)/openmpi
+mcacomponent_LTLIBRARIES = $(component_install)
+mca_mpool_gm_la_SOURCES = $(sources)
+mca_mpool_gm_la_LDFLAGS = -module -avoid-version
+
+noinst_LTLIBRARIES = $(component_noinst)
+libmca_mpool_gm_la_SOURCES = $(sources)
+libmca_mpool_gm_la_LDFLAGS = -module -avoid-version
+
+

src/mca/mpool/gm/configure.params

@@ -0,0 +1,32 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University.
+#                         All rights reserved.
+# Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+#                         All rights reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+
+# Specific to this module
+
+PARAM_INIT_FILE=mpool_gm_module.c
+PARAM_CONFIG_HEADER_FILE="gm_config.h"
+PARAM_CONFIG_FILES="Makefile"
+
+
+
+
+
+
+
+
+
+

src/mca/mpool/gm/configure.stub

@@ -0,0 +1,196 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University.
+#                         All rights reserved.
+# Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+#                         All rights reserved.
+# Copyright (c) 2004 The Ohio State University.
+#                    All rights reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+
+#
+# Main function.  This will be invoked in the middle of the templated
+# configure script.
+#
+AC_DEFUN([MCA_CONFIGURE_STUB],[
+
+    # Additional --with flags that can be specified
+
+    AC_ARG_WITH(btl-gm, 
+        AC_HELP_STRING([--with-btl-gm=DIR],
+                       [Specify the installation directory of GM]))
+    AC_ARG_WITH(btl-gm-libdir,
+        AC_HELP_STRING([--with-btl-gm-libdir=DIR],
+                       [directory where the GM library can be found, if it is not in \$GMDIR/lib or \$GMDIR/binary/lib]))
+    AC_ARG_ENABLE(gm-btl-cache,
+        AC_HELP_STRING([--enable-gm-btl-cache],[Enable/disable the internal registered memory cache (default=no)]),
+                 [enable_gm_cache=$(enable_gm_btl_cache)],
+                 [enable_gm_cache="no"])
+
+    if test "$enable_gm_cache" = "yes"; then
+        AC_DEFINE_UNQUOTED( OMPI_MCA_BMI_GM_CACHE_ENABLE, 1,
+                            [Enable GM BMI internal cache for registered memory])
+    else
+        AC_DEFINE_UNQUOTED( OMPI_MCA_BMI_GM_CACHE_ENABLE, 0,
+                            [Disable GM BMI internal cache for registered memory])
+    fi
+    AM_CONDITIONAL( OMPI_ENABLE_GM_CACHE, test "$enable_gm_cache" = "yes" )
+
+    # Add to CPPFLAGS if necessary
+
+    EXTRA_CPPFLAGS=
+    if test -n "$with_btl_gm"; then
+        if test -d "$with_btl_gm/include"; then
+            EXTRA_CPPFLAGS="-I$with_btl_gm/include"
+        else
+            AC_MSG_WARN([*** Warning: cannot find $with_btl_gm/include])
+            AC_MSG_WARN([*** Will still try to configure gm btl anyway...])
+        fi
+    fi
+
+    # See if we can find gm.h
+
+    CPPFLAGS="$CPPFLAGS $EXTRA_CPPFLAGS"
+    AC_CHECK_HEADERS(gm.h,,
+        AC_MSG_ERROR([*** Cannot find working gm.h.]))
+
+    # Add to LDFLAGS if necessary
+
+    EXTRA_LDFLAGS=
+    if test -n "$with_btl_gm_libdir"; then
+        if test -d "$with_btl_gm_libdir"; then
+            EXTRA_LDFLAGS="-L$with_btl_gm_libdir"
+        else
+            AC_MSG_WARN([*** Warning: cannot find $with_btl_gm_libdir])
+            AC_MSG_WARN([*** Will still try to configure gm btl anyway...])
+        fi
+    elif test -n "$with_btl_gm"; then
+        if test -d "$with_btl_gm/lib"; then
+            EXTRA_LDFLAGS="-L$with_btl_gm/lib"
+        elif test -d "$with_btl_gm/binary/lib"; then
+            EXTRA_LDFLAGS="-L$with_btl_gm/binary/lib"
+        else
+            AC_MSG_WARN([*** Warning: cannot find $with_btl_gm/lib])
+            AC_MSG_WARN([*** or $with_btl_gm/binary/lib])
+            AC_MSG_WARN([*** Will still try to configure gm btl anyway...])
+        fi
+    fi
+
+    # Try to find libgm
+
+    LDFLAGS="$LDFLAGS $EXTRA_LDFLAGS"
+    AC_CHECK_LIB([gm], [main], [],
+        AC_MSG_ERROR([*** Cannot find libgm]))
+
+    #
+    # See if we have GM_API_VERSION.  If we do, use it.  If not, find the
+    # highest one available.  It seems that GM_API_VERSION was introduced
+    # somewhere after 1.3 but before 1.6. :-\
+    #
+
+    AC_MSG_CHECKING(for GM_API_VERSION)
+    AC_TRY_COMPILE([#include<gm.h>],
+    [int foo = GM_API_VERSION;], 
+        have_gm_api_ver_msg=yes gm_api_ver=GM_API_VERSION,
+        have_gm_api_ver_msg=no  gm_api_ver="")
+    AC_MSG_RESULT([$have_gm_api_ver_msg])
+    if test "$gm_api_ver" = ""; then
+        found=0
+        for val in 5 4 3; do
+            if test "$found" = "0"; then
+                var="GM_API_VERSION_1_$val"
+                AC_MSG_CHECKING(for $var)
+                AC_TRY_COMPILE([#include<gm.h>],
+                    [int foo = $var;], 
+                    msg=yes found=1 gm_api_ver=$var,
+                    msg=no found=0 gm_api_ver="")
+                AC_MSG_RESULT($msg)
+            fi
+        done
+    fi
+    if test "$gm_api_ver" = ""; then
+        AC_MSG_WARN([*** Could not find a supported GM_API_VERSION])
+        AC_MSG_ERROR([*** Cannot continue])
+    fi
+    AC_DEFINE_UNQUOTED(OMPI_MCA_BMI_GM_API_VERSION, $gm_api_ver,
+        [Version of the GM API to use])
+    unset gm_api_ver have_gm_api_ver_msg found val msg
+
+    #
+    # Do we have gm_put()?
+    # gm_put() was introduced in gm 2.0, and is exactly identical to gm
+    # 1.6's gm_directed_send_with_callback().  The name was simply changed
+    # for consistency/symmtery with gm_get().
+    #
+
+    AC_MSG_CHECKING([for gm_put()])
+    AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include<gm.h>
+    ]], 
+    [[gm_put(0, 0, 0, 0, 0, 0, 0, 0, 0);]])], 
+        [HAVE_RDMA_PUT=1 MSG=yes], 
+        [HAVE_RDMA_PUT=0 MSG="no, use gm_directed_send_with_callback()"])
+    AC_DEFINE_UNQUOTED(OMPI_MCA_BMI_GM_HAVE_RDMA_PUT, $HAVE_RDMA_PUT,
+        [Whether we have gm_put() or gm_directed_send_with_callback()])
+    AC_MSG_RESULT([$MSG])
+
+    #
+    # Do we have gm_get()?
+    # gm_get() was introduced in gm 2.0.
+    #
+
+    AC_MSG_CHECKING([for gm_get()])
+    AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include<gm.h>
+    ]], 
+    [[gm_get(0, 0, 0, 0, 0, 0, 0, 0, 0);]])], 
+        [HAVE_RDMA_GET=1 MSG=yes], 
+        [HAVE_RDMA_GET=0 MSG=no])
+    AC_DEFINE_UNQUOTED(OMPI_MCA_BMI_GM_HAVE_RDMA_GET, $HAVE_RDMA_GET,
+        [Whether we have get_get() or not])
+    AC_MSG_RESULT([$MSG])
+
+    # Now test to see if the targetted GM is a broken one broken gm builds
+    AC_MSG_CHECKING([for broken GM 2.x RDMA gets build])
+    AC_TRY_COMPILE([
+        #include <gm.h>
+        ], [
+        #if GM_API_VERSION_2_0 &&                                 \
+           ((GM_API_VERSION_2_1_0 && GM_API_VERSION < 0x20102) || \
+             GM_API_VERSION < 0x2000c)
+          #error GM build is broken
+        #endif         ],
+       [ mca_btl_gm_broken=0 gm_build_broken=no ],
+       [ mca_btl_gm_broken=1 gm_build_broken=yes ]) 
+    AC_MSG_RESULT( [$gm_build_broken] )
+    AC_DEFINE_UNQUOTED( OMPI_MCA_BMI_GM_GET_BROKEN, $mca_btl_gm_broken,
+                        [The GM build has or not a broker gm_get function] )
+    unset gm_build_broken mca_btl_gm_broken
+
+    AC_DEFINE_UNQUOTED( OMPI_MCA_BMI_GM_SUPPORT_REGISTERING, 1,
+                        [The OS support or not the virtal page registration] )
+
+    #
+    # Save extra compiler/linker flags so that they can be added in
+    # the wrapper compilers, if necessary
+    #
+
+    WRAPPER_EXTRA_LDFLAGS="$EXTRA_LDFLAGS"
+    WRAPPER_EXTRA_LIBS="-lgm"
+])dnl
+
+#                                                                               
+# For dist                                                                      
+#                                                                               
+AC_DEFUN([MCA_CONFIGURE_DIST_STUB],[
+    AM_CONDITIONAL( OMPI_ENABLE_GM_CACHE, test "yes" = "yes" )
+])
+
+

src/mca/mpool/gm/mpool_gm.h

@@ -0,0 +1,138 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+/**
+ * @file
+ */
+#ifndef MCA_MPOOL_GM_H
+#define MCA_MPOOL_GM_H
+
+#include "ompi_config.h"
+#include <gm.h>
+#include "class/ompi_list.h"
+#include "class/ompi_free_list.h"
+#include "event/event.h"
+#include "mca/mpool/mpool.h"
+#include "mca/allocator/allocator.h"
+
+#if defined(c_plusplus) || defined(__cplusplus)
+extern "C" {
+#endif
+
+
+struct mca_mpool_gm_component_t {
+    mca_mpool_base_component_t super;
+    char*  gm_allocator_name;
+    long page_size; 
+    long page_size_log; 
+};
+
+typedef struct mca_mpool_gm_component_t mca_mpool_gm_component_t;
+
+OMPI_COMP_EXPORT extern mca_mpool_gm_component_t mca_mpool_gm_component;
+
+
+
+struct mca_mpool_base_resources_t {
+  struct gm_port *gm_port;   /* the gm NIC handle */
+}; 
+typedef struct mca_mpool_base_resources_t mca_mpool_base_resources_t;  
+
+struct mca_mpool_gm_module_t {
+    mca_mpool_base_module_t super;
+    mca_allocator_base_module_t *gm_allocator; 
+    struct gm_port* gm_port;
+}; 
+typedef struct mca_mpool_gm_module_t mca_mpool_gm_module_t; 
+
+
+
+/* 
+ *  Initializes the mpool module. 
+ */ 
+void mca_mpool_gm_module_init(mca_mpool_gm_module_t* mpool); 
+
+
+/*
+ *  Returns base address of shared memory mapping.
+ */
+void* mca_mpool_gm_base(mca_mpool_base_module_t*);
+
+/**
+  *  Allocate block of shared memory.
+  */
+void* mca_mpool_gm_alloc( 
+    mca_mpool_base_module_t* mpool, 
+    size_t size, 
+    size_t align, 
+    mca_mpool_base_registration_t** registration);
+
+/**
+  * realloc function typedef
+  */
+void* mca_mpool_gm_realloc(
+    mca_mpool_base_module_t* mpool, 
+    void* addr, 
+    size_t size, 
+    mca_mpool_base_registration_t** registration);
+
+/**
+  * register function typedef
+  */
+int mca_mpool_gm_register(
+    mca_mpool_base_module_t* mpool, 
+    void *addr, 
+    size_t size, 
+    mca_mpool_base_registration_t** registration);
+
+int mca_mpool_gm_deregister(
+    mca_mpool_base_module_t* mpool, 
+    void *addr, 
+    size_t size, 
+    mca_mpool_base_registration_t*);
+
+
+/**
+  * free function typedef
+  */
+void mca_mpool_gm_free(
+    mca_mpool_base_module_t* mpool, 
+    void * addr, 
+    mca_mpool_base_registration_t* registration);
+
+void* mca_common_gm_segment_alloc(
+    struct mca_mpool_base_module_t* module, 
+    size_t* size, 
+    mca_mpool_base_registration_t** registration);
+
+#if defined(c_plusplus) || defined(__cplusplus)
+}
+#endif
+#endif
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

src/mca/mpool/gm/mpool_gm_component.c

@@ -0,0 +1,150 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include "util/output.h"
+#include "mca/base/base.h"
+#include "mca/base/mca_base_param.h"
+#include "mca/allocator/base/base.h"
+#include "mpool_gm.h"
+#include "util/proc_info.h"
+#include "util/sys_info.h"
+#include <unistd.h> 
+
+/*
+ * Local functions
+ */
+static int mca_mpool_gm_open(void);
+static mca_mpool_base_module_t* mca_mpool_gm_init(
+    struct mca_mpool_base_resources_t* resources);
+
+mca_mpool_gm_component_t mca_mpool_gm_component = {
+    {
+      /* First, the mca_base_component_t struct containing meta
+         information about the component itself */
+
+      {
+        /* Indicate that we are a mpool v1.0.0 component (which also
+           implies a specific MCA version) */
+
+        MCA_MPOOL_BASE_VERSION_1_0_0,
+
+        "gm", /* MCA component name */
+        1,  /* MCA component major version */
+        0,  /* MCA component minor version */
+        0,  /* MCA component release version */
+        mca_mpool_gm_open,  /* component open  */
+        NULL
+      },
+
+      /* Next the MCA v1.0.0 component meta data */
+      
+      {
+        /* Whether the component is checkpointable or not */
+        false
+      },
+
+      mca_mpool_gm_init
+    }
+};
+
+
+
+static char* mca_mpool_gm_param_register_string(
+    const char* param_name,
+    const char* default_value)
+{
+    char *param_value;
+    int id = mca_base_param_register_string("mpool","gm",param_name,NULL,default_value);
+    mca_base_param_lookup_string(id, &param_value);
+    return param_value;
+}
+
+
+/**
+  * component open/close/init function
+  */
+static int mca_mpool_gm_open(void)
+{
+    /* register GM component parameters */
+    
+    mca_mpool_gm_component.gm_allocator_name =
+        mca_mpool_gm_param_register_string("allocator", "bucket");
+    return OMPI_SUCCESS;
+}
+
+/* Allocates a segment of memory and registers with GM, user_out returns the memory handle. */ 
+void* mca_common_gm_segment_alloc(
+    struct mca_mpool_base_module_t* mpool,
+    size_t* size, 
+    mca_mpool_base_registration_t** registration)
+{
+    mca_mpool_gm_module_t* gm_mpool; 
+    mca_mpool_base_registration_t* reg;
+    void* addr = gm_dma_malloc(gm_mpool->gm_port, *size);
+    if(NULL == addr)
+        return NULL;
+    if(NULL != registration) {
+        reg = OBJ_NEW(mca_mpool_base_registration_t);
+        reg->base = addr;
+        reg->bound = reg->base + *size - 1;
+        reg->mpool = mpool;
+        *registration = reg;
+    }
+    return addr; 
+}
+
+/* Allocates a segment of memory and registers with IB, user_out returns the memory handle. */ 
+static mca_mpool_base_module_t* mca_mpool_gm_init(
+     struct mca_mpool_base_resources_t* resources)
+{
+    mca_mpool_gm_module_t* gm_mpool; 
+    mca_allocator_base_component_t* allocator_component;
+
+    /* if specified allocator cannout be loaded - look for an alternative */
+    allocator_component = mca_allocator_component_lookup(mca_mpool_gm_component.gm_allocator_name);
+    if(NULL == allocator_component) {
+        if(ompi_list_get_size(&mca_allocator_base_components) == 0) {
+            mca_base_component_list_item_t* item = (mca_base_component_list_item_t*)
+                ompi_list_get_first(&mca_allocator_base_components);
+            allocator_component = (mca_allocator_base_component_t*)item->cli_component;
+            ompi_output(0, "[%d:%d] unable to locate allocator: %s - using %s\n",
+                __FILE__, __LINE__,
+                mca_mpool_gm_component.gm_allocator_name, 
+                allocator_component->allocator_version.mca_component_name);
+        } else {
+            ompi_output(0, "[%s:%d] unable to locate allocator: %s\n",
+                __FILE__, __LINE__,
+                mca_mpool_gm_component.gm_allocator_name);
+            return NULL;
+        }
+    }
+    
+    gm_mpool = (mca_mpool_gm_module_t*)malloc(sizeof(mca_mpool_gm_module_t)); 
+    mca_mpool_gm_module_init(gm_mpool); 
+    
+    /* setup allocator  TODO fix up */
+    gm_mpool->gm_port = resources->gm_port;
+    gm_mpool->gm_allocator = 
+      allocator_component->allocator_init(true, mca_common_gm_segment_alloc, NULL, &gm_mpool->super);
+    if(NULL == gm_mpool->gm_allocator) {
+        ompi_output(0, "[%s:%d] unable to initialize allocator", __FILE__, __LINE__);
+        return NULL;
+    }
+    return &gm_mpool->super;
+}
+
+

src/mca/mpool/gm/mpool_gm_module.c

@@ -0,0 +1,144 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include <string.h>
+#include "util/output.h"
+#include "mpool_gm.h"
+
+
+/* 
+ *  Initializes the mpool module.
+ */ 
+void mca_mpool_gm_module_init(mca_mpool_gm_module_t* mpool)
+{
+    mpool->super.mpool_component = &mca_mpool_gm_component.super; 
+    mpool->super.mpool_base = NULL; /* no base .. */ 
+    mpool->super.mpool_alloc = mca_mpool_gm_alloc; 
+    mpool->super.mpool_realloc = mca_mpool_gm_realloc; 
+    mpool->super.mpool_free = mca_mpool_gm_free; 
+    mpool->super.mpool_register = mca_mpool_gm_register; 
+    mpool->super.mpool_deregister = mca_mpool_gm_deregister; 
+    mpool->super.mpool_finalize = NULL; 
+}
+
+
+/**
+  * allocate function 
+  */
+void* mca_mpool_gm_alloc(
+    mca_mpool_base_module_t* mpool, 
+    size_t size, 
+    size_t align, 
+    mca_mpool_base_registration_t** registration)
+{
+    mca_mpool_gm_module_t* mpool_gm = (mca_mpool_gm_module_t*)mpool; 
+    return mpool_gm->gm_allocator->alc_alloc(mpool_gm->gm_allocator, size, align, registration);
+}
+
+
+/* 
+ * register memory 
+ */ 
+int mca_mpool_gm_register(
+    mca_mpool_base_module_t* mpool, 
+    void *addr, 
+    size_t size, 
+    mca_mpool_base_registration_t** registration)
+{
+    mca_mpool_gm_module_t * gm_mpool = (mca_mpool_gm_module_t*) mpool; 
+    mca_mpool_base_registration_t* reg = OBJ_NEW(mca_mpool_base_registration_t);
+    int rc;
+
+    if(NULL == reg) {
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+    if((rc = gm_register_memory(gm_mpool->gm_port, addr, size)) != GM_SUCCESS) {
+        ompi_output(0, "[%s:%d] error(%d) registering gm memory\n", __FILE__, __LINE__, rc);
+        OBJ_RELEASE(reg);
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+    reg->mpool = mpool;
+    reg->base = addr; 
+    reg->bound = reg->base + size - 1; 
+    *registration = reg;
+    return OMPI_SUCCESS; 
+}
+
+
+/* 
+ * deregister memory 
+ */ 
+int mca_mpool_gm_deregister(mca_mpool_base_module_t* mpool, void *addr, size_t size, 
+                              mca_mpool_base_registration_t* reg)
+{
+    mca_mpool_gm_module_t * mpool_gm = (mca_mpool_gm_module_t*) mpool; 
+    int rc = gm_deregister_memory(
+        mpool_gm->gm_port,
+        addr,
+        size);
+    if(GM_SUCCESS != rc) { 
+        ompi_output(0, "[%s:%d] error(%d) deregistering gm memory\n", __FILE__, __LINE__, rc); 
+        return OMPI_ERROR; 
+    }
+    return OMPI_SUCCESS; 
+}
+
+/**
+  * realloc function 
+  */
+void* mca_mpool_gm_realloc(
+    mca_mpool_base_module_t* mpool, 
+    void* addr, 
+    size_t size, 
+    mca_mpool_base_registration_t** registration)
+{
+    mca_mpool_gm_module_t* mpool_gm = (mca_mpool_gm_module_t*)mpool; 
+    return mpool_gm->gm_allocator->alc_realloc( mpool_gm->gm_allocator, addr, size, registration);
+}
+
+/**
+  * free function 
+  */
+void mca_mpool_gm_free(mca_mpool_base_module_t* mpool, void * addr,
+                         mca_mpool_base_registration_t* registration)
+{
+    
+    mca_mpool_gm_module_t* mpool_gm = (mca_mpool_gm_module_t*)mpool; 
+    mpool_gm->super.mpool_deregister(mpool, addr, 0, registration); 
+    mpool_gm->gm_allocator->alc_free(mpool_gm->gm_allocator, addr);
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+