Define a "fake" mpool to provide a memory release callback for the

memory hooks (munmap) and initialize the mallopt component, and 
nothing else.
Use this mpool in the MX common initialization, supporting both BTL 
and MTL. Automatically set the MX_RCACHE environment variable to 
enable registration cache in MX.

Tested with success for munmap() and large free().

This commit was SVN r20003.

ompi/mca/common/mx/common_mx.c

@@ -9,6 +9,8 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2006 The Regents of the University of California.
  *                         All rights reserved.
+ * Copyright (c) 2008      Myricom. All rights reserved.
+ * 
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -21,11 +23,29 @@
 #include "ompi/constants.h"
 #include "common_mx.h"
 
+#ifdef HAVE_MALLOC_H
+#include <malloc.h>
+#endif
+#include "opal/memoryhooks/memory.h"
+#include "opal/mca/base/mca_base_param.h"
+#include "ompi/runtime/params.h"
+#include "ompi/mca/mpool/mpool.h"
+#include "ompi/mca/mpool/base/base.h"
+#include "ompi/mca/mpool/fake/mpool_fake.h"
+
+
+int mx__regcache_clean(void *ptr, size_t size);
+
 static int ompi_common_mx_initialize_ref_cnt = 0;
+static mca_mpool_base_module_t *ompi_common_mx_fake_mpool = 0;
+
 int
 ompi_common_mx_initialize(void)
 {
     mx_return_t mx_return;
+    struct mca_mpool_base_resources_t mpool_resources;
+    int index, value;
+    
     ompi_common_mx_initialize_ref_cnt++;
     
     if(ompi_common_mx_initialize_ref_cnt == 1) { 
@@ -36,6 +56,36 @@ ompi_common_mx_initialize(void)
          */
         mx_set_error_handler(MX_ERRORS_RETURN);
 	
+	/* If we have a memory manager available, and
+	   mpi_leave_pinned == -1, then set mpi_leave_pinned to 1.
+
+	   We have a memory manager if:
+	   - we have both FREE and MUNMAP support
+	   - we have MUNMAP support and the linux mallopt */
+	value = opal_mem_hooks_support_level();
+	if (((value & (OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT))
+	     == (OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT))
+	    || ((value & OPAL_MEMORY_MUNMAP_SUPPORT) &&
+		OMPI_MPOOL_BASE_HAVE_LINUX_MALLOPT)) {
+	  index = mca_base_param_find("mpi", NULL, "leave_pinned");
+	  if (index >= 0)
+            if ((mca_base_param_lookup_int(index, &value) == OPAL_SUCCESS) 
+		&& (value == -1)) {
+	      
+	      ompi_mpi_leave_pinned = 1;
+	      setenv("MX_RCACHE", "2", 1);
+	      mpool_resources.regcache_clean = mx__regcache_clean;
+	      ompi_common_mx_fake_mpool = 
+		mca_mpool_base_module_create("fake", NULL, &mpool_resources);
+	      if (!ompi_common_mx_fake_mpool) {
+		ompi_mpi_leave_pinned = 0;
+		setenv("MX_RCACHE", "0", 1);
+		opal_output(0, "Error creating fake mpool (error %s)\n",
+			    strerror(errno));
+	      }
+	    }
+	}
+	
         /* initialize the mx library */
         mx_return = mx_init(); 
         
@@ -57,6 +107,10 @@ ompi_common_mx_finalize(void)
     mx_return_t mx_return;
     ompi_common_mx_initialize_ref_cnt--;
     if( 0 == ompi_common_mx_initialize_ref_cnt ) { 
+
+        if (ompi_common_mx_fake_mpool) 
+	  mca_mpool_base_module_destroy(ompi_common_mx_fake_mpool);
+        
         mx_return = mx_finalize(); 
         if(mx_return != MX_SUCCESS){ 
             opal_output(0, "Error in mx_finalize (error %s)\n", mx_strerror(mx_return));

ompi/mca/mpool/fake/Makefile.am

@@ -0,0 +1,56 @@
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2008      Myricom.  All rights reserved.
+#
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+AM_CPPFLAGS = $(mpool_fake_CPPFLAGS)
+
+sources = \
+	mpool_fake_module.c \
+	mpool_fake_component.c
+
+if WANT_INSTALL_HEADERS
+ompidir = $(includedir)/openmpi/ompi/mca/mpool/fake
+ompi_HEADERS = mpool_fake.h
+else
+ompidir = $(includedir)
+ompi_HEADERS =
+endif
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if OMPI_BUILD_mpool_fake_DSO
+component_noinst =
+component_install = mca_mpool_fake.la
+else
+component_noinst = libmca_mpool_fake.la
+component_install =
+endif
+
+mcacomponentdir = $(pkglibdir)
+mcacomponent_LTLIBRARIES = $(component_install)
+mca_mpool_fake_la_SOURCES = $(sources)
+mca_mpool_fake_la_LDFLAGS = -module -avoid-version
+mca_mpool_fake_la_LIBADD = $(mpool_fake_LIBS)
+
+noinst_LTLIBRARIES = $(component_noinst)
+libmca_mpool_fake_la_SOURCES = $(sources)
+libmca_mpool_fake_la_LDFLAGS = -module -avoid-version
+libmca_mpool_fake_la_LIBADD = $(mpool_fake_LIBS)

ompi/mca/mpool/fake/configure.params

@@ -0,0 +1,24 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2008      Myricom. All rights reserved.
+#
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# Specific to this module
+
+PARAM_CONFIG_FILES="Makefile"

ompi/mca/mpool/fake/mpool_fake.h

@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2006 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Myricom. All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+/**
+ * @file
+ */
+
+#ifndef MCA_MPOOL_FAKE_H
+#define MCA_MPOOL_FAKE_H
+
+#include "ompi/mca/mpool/mpool.h"
+
+#if defined(c_plusplus) || defined(__cplusplus)
+extern "C" {
+#endif
+
+typedef struct mca_mpool_fake_component_t {
+  mca_mpool_base_component_t super;
+} mca_mpool_fake_component_t;
+
+OMPI_DECLSPEC extern mca_mpool_fake_component_t mca_mpool_fake_component;
+
+typedef struct mca_mpool_base_resources_t {
+  int (*regcache_clean)(void *ptr, size_t size);
+} mca_mpool_base_resources_t;
+
+typedef struct mca_mpool_fake_module_t {
+  mca_mpool_base_module_t super;
+  mca_mpool_base_resources_t resources;
+} mca_mpool_fake_module_t;
+
+/*
+ *  Initializes the mpool module.
+ */
+void mca_mpool_fake_module_init(mca_mpool_fake_module_t *mpool);
+
+/**
+ * unregister all registration covering the block of memory
+ */
+int mca_mpool_fake_release_memory(mca_mpool_base_module_t* mpool, void *base,
+        size_t size);
+
+/**
+ * finalize mpool
+ */
+void mca_mpool_fake_finalize(struct mca_mpool_base_module_t *mpool);
+
+/**
+ * Fault Tolerance Event Notification Function
+ * @param state Checkpoint Stae
+ * @return OMPI_SUCCESS or failure status
+ */
+int mca_mpool_fake_ft_event(int state);
+
+#if defined(c_plusplus) || defined(__cplusplus)
+}
+#endif
+#endif

ompi/mca/mpool/fake/mpool_fake_component.c

@@ -0,0 +1,87 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Myricom. All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#define OMPI_DISABLE_ENABLE_MEM_DEBUG 1
+#include "ompi_config.h"
+#include "orte/util/show_help.h"
+#include "opal/mca/base/base.h"
+#include "opal/mca/base/mca_base_param.h"
+#include "mpool_fake.h"
+#include "orte/util/proc_info.h"
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#ifdef HAVE_MALLOC_H
+#include <malloc.h>
+#endif
+
+/*
+ * Local functions
+ */
+static int mca_mpool_fake_open(void);
+static mca_mpool_base_module_t* mca_mpool_fake_init(
+	struct mca_mpool_base_resources_t* resources);
+
+mca_mpool_fake_component_t mca_mpool_fake_component = {
+    {
+      /* First, the mca_base_component_t struct containing meta
+         information about the component itself */
+
+      {
+          MCA_MPOOL_BASE_VERSION_2_0_0,
+
+          "fake", /* MCA component name */
+          OMPI_MAJOR_VERSION,  /* MCA component major version */
+          OMPI_MINOR_VERSION,  /* MCA component minor version */
+          OMPI_RELEASE_VERSION,  /* MCA component release version */
+          mca_mpool_fake_open,  /* component open  */
+          NULL
+      },
+      {
+          /* The component is checkpoint ready */
+          MCA_BASE_METADATA_PARAM_CHECKPOINT
+      },
+
+      mca_mpool_fake_init
+    }
+};
+
+/**
+  * component open/close/init function
+  */
+static int mca_mpool_fake_open(void)
+{
+    return OMPI_SUCCESS;
+}
+
+static mca_mpool_base_module_t* mca_mpool_fake_init(
+     struct mca_mpool_base_resources_t* resources)
+{
+    mca_mpool_fake_module_t* mpool_module;
+
+    mpool_module = (mca_mpool_fake_module_t*)
+      malloc(sizeof(mca_mpool_fake_module_t));
+
+    mpool_module->resources = *resources;
+
+    mca_mpool_fake_module_init(mpool_module);
+
+    return &mpool_module->super;
+}

ompi/mca/mpool/fake/mpool_fake_module.c

@@ -0,0 +1,72 @@
+/* -*- Mode: C; c-basic-offset:4 ; -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2007 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Myricom.  All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#define OMPI_DISABLE_ENABLE_MEM_DEBUG 1
+#include "ompi_config.h"
+#include "opal/include/opal/align.h"
+#include "orte/util/proc_info.h"
+#include "orte/util/name_fns.h"
+#include "orte/runtime/orte_globals.h"
+#include "orte/util/show_help.h"
+#include "mpool_fake.h"
+#include <errno.h>
+#include <string.h>
+#include "ompi/mca/mpool/base/base.h"
+#include "ompi/runtime/params.h"
+
+/*
+ *  Initializes the mpool module.
+ */
+void mca_mpool_fake_module_init(mca_mpool_fake_module_t* mpool)
+{
+    mpool->super.mpool_component = &mca_mpool_fake_component.super;
+    mpool->super.mpool_base = NULL; /* no base .. */
+    mpool->super.mpool_alloc = NULL;
+    mpool->super.mpool_realloc = NULL;
+    mpool->super.mpool_free = NULL;
+    mpool->super.mpool_register = NULL;
+    mpool->super.mpool_find = NULL;
+    mpool->super.mpool_deregister = NULL;
+    mpool->super.mpool_release_memory = mca_mpool_fake_release_memory;
+    mpool->super.mpool_finalize = mca_mpool_fake_finalize;
+    mpool->super.mpool_ft_event = mca_mpool_fake_ft_event;
+}
+
+
+int mca_mpool_fake_release_memory(struct mca_mpool_base_module_t *mpool,
+        void *base, size_t size)
+{
+    mca_mpool_fake_module_t *mpool_module = (mca_mpool_fake_module_t*)mpool;
+
+    mpool_module->resources.regcache_clean(base, size);
+
+    return OMPI_SUCCESS;
+}
+
+void mca_mpool_fake_finalize(struct mca_mpool_base_module_t *mpool)
+{
+    ;
+}
+
+int mca_mpool_fake_ft_event(int state)
+{
+    return OMPI_SUCCESS;
+}