- add shared memory pool

- working on mpool/allocator framework

This commit was SVN r1313.

src/mca/allocator/allocator.h

@@ -78,5 +78,7 @@ typedef struct mca_allocator_base_module_1_0_0_t mca_allocator_base_module_t;
   /* ptl v1.0 */ \
   "allocator", 1, 0, 0
 
+extern int mca_allocator_base_output;
+
 #endif /* MCA_ALLOCATOR_H */
 

src/mca/allocator/base/allocator_base_close.c

@@ -20,24 +20,23 @@ int mca_allocator_base_close(void)
 
   /* Finalize all the allocator modules and free their list items */
 
-  for (item = ompi_list_remove_first(&mca_allocator_base_modules_initialized);
+  for (item = ompi_list_remove_first(&mca_allocator_base_modules);
        NULL != item; 
-       item = ompi_list_remove_first(&mca_allocator_base_modules_initialized)) {
+       item = ompi_list_remove_first(&mca_allocator_base_modules)) {
     sm = (mca_allocator_base_selected_module_t *) item;
 
     /* Blatently ignore the return code (what would we do to recover,
        anyway?  This module is going away, so errors don't matter
        anymore) */
 
-    sm->absm_actions->alc_finalize(sm->absm_actions);
+    sm->allocator_module->alc_finalize(sm->allocator_module);
-    free(sm);
+    OBJ_RELEASE(sm);
   }
 
   /* Close all remaining available modules (may be one if this is a
      OMPI RTE program, or [possibly] multiple if this is ompi_info) */
 
-  mca_base_modules_close(mca_allocator_base_output, 
+  mca_base_modules_close(mca_allocator_base_output, &mca_allocator_base_modules, NULL);
-                         &mca_allocator_base_modules_available, NULL);
 
   /* All done */
 

src/mca/allocator/base/allocator_base_open.c

@@ -24,10 +24,9 @@
 /*
  * Global variables
  */
+ompi_list_t mca_allocator_base_components;
+ompi_list_t mca_allocator_base_modules;
 int mca_allocator_base_output = -1;
-ompi_list_t mca_allocator_base_modules_available;
-ompi_list_t mca_allocator_base_modules_initialized;
-
 
 /**
  * Function for finding and opening either all MCA modules, or the one
@@ -39,18 +38,10 @@ int mca_allocator_base_open(void)
 
   if (OMPI_SUCCESS != 
       mca_base_modules_open("allocator", 0, mca_allocator_base_static_modules, 
-                            &mca_allocator_base_modules_available)) {
+                            &mca_allocator_base_components)) {
     return OMPI_ERROR;
   }
-
-  /* Initialize the list so that in mca_allocator_base_close(), we can
-     iterate over it (even if it's empty, as in the case of
-     ompi_info) */
-
-  OBJ_CONSTRUCT(&mca_allocator_base_modules_initialized, ompi_list_t);
-
   /* All done */
-
   return OMPI_SUCCESS;
 }
 

src/mca/allocator/base/base.h

@@ -16,11 +16,13 @@
 
 struct mca_allocator_base_selected_module_t {
   ompi_list_item_t super;
-  mca_allocator_base_module_t *apsm_module;
+  mca_allocator_base_module_t *allocator_component;
-  mca_allocator_t *absm_actions;
+  mca_allocator_t *allocator_module;
 };
 typedef struct mca_allocator_base_selected_module_t mca_allocator_base_selected_module_t;
                                                                                                      
+OBJ_CLASS_DECLARATION(mca_mpool_base_selected_module_t);
+                                                                                                     
 
 /*
  * Global functions for MCA: overall PTL open and close
@@ -32,7 +34,7 @@ extern "C" {
   int mca_allocator_base_open(void);
   int mca_allocator_base_init(bool *allow_multi_user_threads);
   int mca_allocator_base_close(void);
-  mca_allocator_t* mca_allocator_lookup(const char* name);
+  mca_allocator_base_module_t* mca_allocator_module_lookup(const char* name);
 #if defined(c_plusplus) || defined(__cplusplus)
 }
 #endif
@@ -43,8 +45,7 @@ extern "C" {
  */
 
 
-extern int mca_allocator_base_output;
+extern ompi_list_t mca_allocator_base_components;
-extern ompi_list_t mca_allocator_base_modules_available;
+extern ompi_list_t mca_allocator_base_modules;
-extern ompi_list_t mca_allocator_base_modules_initialized;
 
 #endif /* MCA_ALLOCATOR_BASE_H */

src/mca/mpool/base/Makefile.am

@@ -19,7 +19,7 @@ libmca_mpool_base_la_SOURCES = \
         $(headers) \
         mpool_base_open.c \
         mpool_base_close.c \
-        mpool_base_select.c 
+        mpool_base_init.c 
 
 # Conditionally install the header files
 

src/mca/mpool/base/base.h

@@ -16,11 +16,12 @@
 
 struct mca_mpool_base_selected_module_t {
   ompi_list_item_t super;
-  mca_mpool_base_module_t *mbsm_module;
+  mca_mpool_base_component_t *mpool_component;
-  mca_mpool_t *mbsm_actions;
+  mca_mpool_t *mpool_module;
 };
 typedef struct mca_mpool_base_selected_module_t mca_mpool_base_selected_module_t;
 
+OBJ_CLASS_DECLARATION(mca_mpool_base_selected_module_t);
 
 /*
  * Global functions for MCA: overall PTL open and close
@@ -30,8 +31,9 @@ typedef struct mca_mpool_base_selected_module_t mca_mpool_base_selected_module_t
 extern "C" {
 #endif
   int mca_mpool_base_open(void);
-  int mca_mpool_base_select(bool *allow_multi_user_threads);
+  int mca_mpool_base_init(bool *allow_multi_user_threads);
   int mca_mpool_base_close(void);
+  mca_mpool_t* mca_mpool_lookup(const char* name);
   void* mca_mpool_base_is_registered(void* addr, size_t size);
 #if defined(c_plusplus) || defined(__cplusplus)
 }

src/mca/mpool/base/mpool_base_close.c

@@ -29,7 +29,7 @@ int mca_mpool_base_close(void)
        anyway?  This module is going away, so errors don't matter
        anymore) */
 
-    sm->mbsm_actions->mpool_finalize(sm->mbsm_actions);
+    sm->mpool_module->mpool_finalize(sm->mpool_module);
     free(sm);
   }
 

src/mca/mpool/base/mpool_base_init.c

@@ -5,12 +5,16 @@
 #include "ompi_config.h"
 
 #include "runtime/runtime.h"
+#include "util/output.h"
 #include "mca/mca.h"
 #include "mca/base/base.h"
 #include "mca/mpool/mpool.h"
 #include "mca/mpool/base/base.h"
 
 
+OBJ_CLASS_INSTANCE(mca_mpool_base_selected_module_t, ompi_list_item_t, NULL, NULL);
+
+
 /**
  * Function for weeding out mpool modules that don't want to run.
  *
@@ -19,15 +23,13 @@
  * will be closed and unloaded.  The selected modules will be returned
  * to the caller in a ompi_list_t.
  */
-int mca_mpool_base_select(bool *allow_multi_user_threads)
+int mca_mpool_base_init(bool *allow_multi_user_threads)
 {
-#if 0
+  bool user_threads;
-  int i, num_mpools;
-  bool user_threads, hidden_threads;
   ompi_list_item_t *item;
   mca_base_module_list_item_t *mli;
-  mca_mpool_base_module_t *module;
+  mca_mpool_base_component_t *component;
-  mca_mpool_t **actions;
+  mca_mpool_t *module;
   mca_mpool_base_selected_module_t *sm;
 
   /* Traverse the list of available modules; call their init
@@ -37,64 +39,45 @@ int mca_mpool_base_select(bool *allow_multi_user_threads)
        ompi_list_get_end(&mca_mpool_base_modules_available) != item;
        item = ompi_list_get_next(item)) {
     mli = (mca_base_module_list_item_t *) item;
-    module = (mca_mpool_base_module_t *) mli->mli_module;
+    component = (mca_mpool_base_component_t *) mli->mli_module;
 
     ompi_output_verbose(10, mca_mpool_base_output, 
                        "select: initializing %s module %s",
-                       module->mpoolm_version.mca_type_name,
+                       component->mpool_version.mca_type_name,
-                       module->mpoolm_version.mca_module_name);
+                       component->mpool_version.mca_module_name);
-    if (NULL == module->mpoolm_init) {
+    if (NULL == component->mpool_init) {
       ompi_output_verbose(10, mca_mpool_base_output,
                          "select: no init function; ignoring module");
     } else {
-      actions = module->mpoolm_init(&num_mpools, &user_threads,
+       module = component->mpool_init(&user_threads);
-                                  &hidden_threads);
 
       /* If the module didn't initialize, unload it */
 
-      if (NULL == actions) {
+      if (NULL == module) {
         ompi_output_verbose(10, mca_mpool_base_output,
                            "select: init returned failure");
 
-        mca_base_module_repository_release((mca_base_module_t *) module);
+        mca_base_module_repository_release((mca_base_module_t *) component);
         ompi_output_verbose(10, mca_mpool_base_output,
-                           "select: module %s unloaded",
+                           "select: component %s unloaded",
-                           module->mpoolm_version.mca_module_name);
+                           component->mpool_version.mca_module_name);
       } 
 
       /* Otherwise, it initialized properly.  Save it. */
 
       else {
         *allow_multi_user_threads |= user_threads;
-        *have_hidden_threads |= hidden_threads;
 
         ompi_output_verbose(10, mca_mpool_base_output,
                            "select: init returned success");
 
-        for (i = 0; i < num_mpools; ++i) {
+          sm = OBJ_NEW(mca_mpool_base_selected_module_t);
-          sm = malloc(sizeof(mca_mpool_base_selected_module_t));
+          sm->mpool_component = component;
-          if (NULL == sm) {
+          sm->mpool_module = module;
-            return OMPI_ERR_OUT_OF_RESOURCE;
+          ompi_list_append(&mca_mpool_base_modules_initialized, (ompi_list_item_t*) sm);
-          }
-          OBJ_CONSTRUCT(sm, ompi_list_item_t);
-          sm->pbsm_module = module;
-          sm->pbsm_actions = actions[i];
-          ompi_list_append(&mca_mpool_base_modules_initialized,
-                          (ompi_list_item_t*) sm);
-        }
-        free(actions);
         }
       }
   }
-
-  /* Finished querying all modules.  Check for the bozo case. */
-
-  if (0 == ompi_list_get_size(&mca_mpool_base_modules_initialized)) {
-    /* JMS Replace with show_help */
-    ompi_abort(1, "No mpool module available.  This shouldn't happen.");
-  }
-
-  /* All done */
-#endif
   return OMPI_SUCCESS;
 }
+

src/mca/mpool/mpool.h

@@ -14,57 +14,74 @@ struct mca_mpool_t;
 /**
   * allocate function typedef
   */
-typedef void* (*mca_mpool_alloc_fn_t)(struct mca_mpool_t*, size_t size, size_t align);
+typedef void* (*mca_mpool_base_alloc_fn_t)(size_t size, size_t align);
 
 /**
   * realloc function typedef
   */
-typedef void* (*mca_mpool_realloc_fn_t)(struct mca_mpool_t*, void* addr, size_t size);
+typedef void* (*mca_mpool_base_realloc_fn_t)(void* addr, size_t size);
 
 /**
   * free function typedef
   */
-typedef void (*mca_mpool_free_fn_t)(struct mca_mpool_t*, void *);
+typedef void (*mca_mpool_base_free_fn_t)(void *);
 
 /**
   * register memory
   */
-typedef void (*mca_mpool_register_fn_t)(struct mca_mpool_t*, void * addr, size_t size, void* user);
+typedef void (*mca_mpool_base_register_fn_t)(void * addr, size_t size, void* user);
 
 /**
   * deregister memory
   */
-typedef void (*mca_mpool_deregister_fn_t)(struct mca_mpool_t*, void * addr);
+typedef void (*mca_mpool_base_deregister_fn_t)(void * addr, size_t size);
 
-typedef int (*mca_mpool_base_finalize_fn_t)(
+/**
-    struct mca_mpool_t* mpool 
+  * finalize
-);
+  */
+typedef void (*mca_mpool_base_finalize_fn_t)(struct mca_mpool_t*);
 
 
+/**
+ *  mpool component descriptor. Contains component version information
+ *  and open/close/init functions.
+ */
+                                                                                                                         
+typedef struct mca_mpool_t* (*mca_mpool_base_init_fn_t)(bool *allow_multi_user_threads);
+
+
+struct mca_mpool_base_component_1_0_0_t {
+  mca_base_module_t mpool_version;
+  mca_base_module_data_1_0_0_t mpool_data;
+  mca_mpool_base_init_fn_t mpool_init;
+};
+typedef struct mca_mpool_base_component_1_0_0_t mca_mpool_base_component_1_0_0_t;
+typedef struct mca_mpool_base_component_1_0_0_t mca_mpool_base_component_t;
+                                                                                                                         
+
+/**
+ *  mpool module descriptor. Contains functions exported
+ *  by the component.
+ */
 struct mca_mpool_t {
-    /* interface functions */
+    mca_mpool_base_component_t *mpool_component;
-    mca_mpool_alloc_fn_t mpool_alloc;
+    mca_mpool_base_alloc_fn_t mpool_alloc;
-    mca_mpool_alloc_fn_t mpool_realloc;
+    mca_mpool_base_realloc_fn_t mpool_realloc;
-    mca_mpool_free_fn_t mpool_free;
+    mca_mpool_base_free_fn_t mpool_free;
-    mca_mpool_register_fn_t mpool_register;
+    mca_mpool_base_register_fn_t mpool_register;
-    mca_mpool_deregister_fn_t mpool_deregister;
+    mca_mpool_base_deregister_fn_t mpool_deregister;
     mca_mpool_base_finalize_fn_t mpool_finalize;
 };
 typedef struct mca_mpool_t mca_mpool_t;
 
-/**
+/*
-  * module initialization function
+ * Macro for use in modules that are of type ptl v1.0.0
  */
-typedef struct mca_mpool_t* (*mca_mpool_base_module_init_fn_t)(
+#define MCA_MPOOL_BASE_VERSION_1_0_0 \
-    bool *allow_multi_user_threads
+  /* mpool v1.0 is chained to MCA v1.0 */ \
-);
+  MCA_BASE_VERSION_1_0_0, \
-
+  /* ptl v1.0 */ \
-struct mca_mpool_base_module_1_0_0_t {
+  "mpool", 1, 0, 0
-    mca_base_module_t mpool_version;
-    mca_base_module_data_1_0_0_t mpool_data;
-    mca_mpool_base_module_init_fn_t mpool_init;
-};
-typedef struct mca_mpool_base_module_1_0_0_t mca_mpool_base_module_t;
 
 #endif /* MCA_MPOOL_H */
 

src/mca/mpool/sm/Makefile.am

@@ -0,0 +1,19 @@
+# -*- makefile -*-
+#
+# $HEADER$
+#
+
+include $(top_ompi_srcdir)/config/Makefile.options
+
+AM_CPPFLAGS = \
+        -I$(top_ompi_builddir)/src/include \
+        -I$(top_ompi_srcdir)/src \
+        -I$(top_ompi_srcdir)/src/include
+
+noinst_LTLIBRARIES = libmca_mpool_sm.la
+libmca_mpool_sm_la_SOURCES = \
+	mpool_sm.c \
+	mpool_sm.h \
+	mpool_sm_mmap.c \
+	mpool_sm_mmap.h \
+	mpool_sm_component.c

src/mca/mpool/sm/configure.params

@@ -0,0 +1,10 @@
+# -*- shell-script -*-
+#
+# $HEADER$
+#
+
+# Specific to this module
+
+PARAM_INIT_FILE=mpool_sm.c
+PARAM_CONFIG_HEADER_FILE="sm_config.h"
+PARAM_CONFIG_FILES="Makefile"

src/mca/mpool/sm/mpool_sm.c

@@ -0,0 +1,46 @@
+/*
+ * $HEADER$
+ */
+
+#include <string.h>
+#include "util/output.h"
+#include "mca/mpool/sm/mpool_sm.h"
+
+
+mca_mpool_t mca_mpool_sm = {
+    &mca_mpool_sm_module.super,
+    mca_mpool_sm_alloc,
+    mca_mpool_sm_realloc,
+    mca_mpool_sm_free,
+    NULL,
+    NULL
+};
+
+
+/**
+  * allocate function 
+  */
+void* mca_mpool_sm_alloc(size_t size, size_t align)
+{
+    return mca_mpool_sm_module.sm_allocator->alc_alloc(
+        mca_mpool_sm_module.sm_allocator, size, align);
+}
+                                                                                                                         
+/**
+  * realloc function 
+  */
+void* mca_mpool_sm_realloc(void* addr, size_t size)
+{
+    return mca_mpool_sm_module.sm_allocator->alc_realloc(
+        mca_mpool_sm_module.sm_allocator, addr, size);
+}
+                                                                                                                         
+/**
+  * free function 
+  */
+void mca_mpool_sm_free(void * addr)
+{
+    mca_mpool_sm_module.sm_allocator->alc_free(
+        mca_mpool_sm_module.sm_allocator, addr);
+}
+                                                                                                                         

src/mca/mpool/sm/mpool_sm.h

@@ -0,0 +1,64 @@
+/*
+ * $HEADER$
+ */
+/**
+ * @file
+ */
+#ifndef MCA_MPOOL_SM_H
+#define MCA_MPOOL_SM_H
+
+#include "class/ompi_list.h"
+#include "class/ompi_free_list.h"
+#include "event/event.h"
+#include "mca/mpool/mpool.h"
+#include "mca/allocator/allocator.h"
+
+
+struct mca_mpool_sm_component_t {
+    mca_mpool_base_component_t super;
+    mca_allocator_t* sm_allocator;
+    char*  sm_allocator_name;
+    size_t sm_min_size;
+    size_t sm_max_size;
+    size_t sm_size;
+    size_t sm_segment;
+    ompi_list_t sm_mmap;
+};
+typedef struct mca_mpool_sm_component_t mca_mpool_sm_component_t;
+
+extern mca_mpool_sm_component_t mca_mpool_sm_module;
+
+/**
+  * allocate function typedef
+  */
+void* mca_mpool_sm_alloc(size_t size, size_t align);
+                                                                                                                         
+/**
+  * realloc function typedef
+  */
+void* mca_mpool_sm_realloc(void* addr, size_t size);
+                                                                                                                         
+/**
+  * free function typedef
+  */
+void mca_mpool_sm_free(void *);
+                                                                                                                         
+/**
+  * register memory
+  */
+void mca_mpool_sm_register(void * addr, size_t size, void* user);
+                                                                                                                         
+/**
+  * deregister memory
+  */
+void mca_mpool_sm_deregister(void * addr);
+
+/**
+  * component open/close/init function 
+  */
+int mca_mpool_sm_open(void);
+int mca_mpool_sm_close(void);
+mca_mpool_t* mca_mpool_sm_init(bool *allow_multi_user_threads);
+
+
+#endif

src/mca/mpool/sm/mpool_sm_component.c

@@ -0,0 +1,90 @@
+#include "mca/base/mca_base_param.h"
+#include "mca/mpool/sm/mpool_sm.h"
+
+
+mca_mpool_sm_component_t mca_mpool_sm_module = {
+    {
+    /* First, the mca_base_module_t struct containing meta information
+       about the module itself */
+                                                                                                                         
+    {
+    /* Indicate that we are a mpool v1.0.0 component (which also implies a
+       specific MCA version) */
+                                                                                                                         
+    MCA_MPOOL_BASE_VERSION_1_0_0,
+                                                                                                                         
+    "sm", /* MCA component name */
+    1,  /* MCA module major version */
+    0,  /* MCA module minor version */
+    0,  /* MCA module release version */
+    mca_mpool_sm_open,  /* component open  */
+    mca_mpool_sm_close  /* component close */
+    },
+                                                                                                                         
+                                                                                                                         
+    /* Next the MCA v1.0.0 module meta data */
+                                                                                                                         
+                                                                                                                         
+    {
+    /* Whether the module is checkpointable or not */
+                                                                                                                         
+                                                                                                                         
+    false
+    },
+                                                                                                                         
+    mca_mpool_sm_init
+    }
+};
+                                                                                                                         
+static inline char* mca_mpool_sm_param_register_string(
+    const char* param_name,
+    const char* default_value)
+{
+    char *param_value;
+    int id = mca_base_param_register_string("mpool","sm",param_name,NULL,default_value);
+    mca_base_param_lookup_string(id, &param_value);
+    return param_value;
+}
+                                                                                                                          
+static inline int mca_mpool_sm_param_register_int(
+    const char* param_name,
+    int default_value)
+{
+    int id = mca_base_param_register_int("mpool","sm",param_name,NULL,default_value);
+    int param_value = default_value;
+    mca_base_param_lookup_int(id,&param_value);
+    return param_value;
+}
+
+
+/**
+  * component open/close/init function
+  */
+int mca_mpool_sm_open(void)
+{
+    /* register SM module parameters */
+    mca_mpool_sm_module.sm_min_size =
+        mca_mpool_sm_param_register_int("min_size", 64*1024*1024);
+    mca_mpool_sm_module.sm_max_size =
+        mca_mpool_sm_param_register_int("max_size", 512*1024*1024);
+    mca_mpool_sm_module.sm_allocator_name =
+        mca_mpool_sm_param_register_string("allocator", "bucket");
+    mca_mpool_sm_module.sm_segment = 1;
+    mca_mpool_sm_module.sm_size = 0;
+    return OMPI_SUCCESS;
+}
+
+
+int mca_mpool_sm_close(void)
+{
+    return OMPI_SUCCESS;
+}
+
+
+mca_mpool_t* mca_mpool_sm_init(bool *allow_multi_user_threads)
+{
+    return NULL;
+}
+
+
+

src/mca/mpool/sm/mpool_sm_mmap.c

@@ -0,0 +1,136 @@
+#include <errno.h>
+#include <unistd.h>
+#include <string.h>
+#include <fcntl.h>
+#include <time.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/errno.h>
+#include <sys/mman.h>
+
+#include "constants.h"
+#include "util/output.h"
+#include "mca/pcm/pcm.h"
+#include "mpool_sm.h"
+#include "mpool_sm_mmap.h"
+
+
+OBJ_CLASS_INSTANCE(
+    mca_mpool_sm_mmap_t,
+    ompi_object_t,
+    NULL,
+    NULL
+);
+
+
+static mca_mpool_sm_mmap_t* mca_mpool_sm_mmap_open(char* path)
+{
+    mca_mpool_sm_mmap_t* map;
+    mca_mpool_sm_segment_t* seg;
+    int fd = -1;
+
+    while(fd < 0) {
+        struct timespec ts;
+        fd = open(path, O_CREAT|O_RDWR, 0000); 
+        if(fd < 0 && errno != EACCES) {
+            ompi_output(0, "mca_ptl_sm_mmap_open: open failed with errno=%d\n", errno);
+            return NULL;
+        }
+        ts.tv_sec = 0; 
+        ts.tv_nsec = 500000;
+        nanosleep(&ts,NULL);
+    }
+
+    /* map the file and initialize segment state */
+    seg = mmap(NULL, mca_mpool_sm_module.sm_min_size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
+    if(NULL == seg) {
+        ompi_output(0, "mca_mpool_sm_mmap_open: mmap failed with errno=%d\n", errno);
+        return NULL;
+    }
+    close(fd);
+                                                                                              
+    map = OBJ_NEW(mca_mpool_sm_mmap_t);
+    map->map_seg = seg;
+    map->map_addr = (unsigned char*)(seg + 1);
+    map->map_size = seg->seg_size - sizeof(mca_mpool_sm_segment_t);
+    close(fd);
+    return map;
+}
+
+
+mca_mpool_sm_mmap_t* mca_mpool_sm_mmap_init(size_t size)
+{
+    ompi_job_handle_t job_handle = mca_pcm.pcm_handle_get();
+    char hostname[64];
+    int fd;
+    mca_mpool_sm_segment_t* seg;
+    mca_mpool_sm_mmap_t* map;
+    char path[PATH_MAX];
+
+    gethostname(hostname, sizeof(hostname));
+    sprintf(path, "/tmp/%s.%s.%d", hostname, job_handle, 
+        ompi_list_get_size(&mca_mpool_sm_module.sm_mmap)+1);
+    fd = open(path, O_CREAT|O_RDWR, 0000); 
+    if(fd < 0) {
+        if(errno == EACCES)
+            return mca_mpool_sm_mmap_open(path);
+        ompi_output(0, "mca_mpool_sm_mmap_init: open failed with errno=%d\n", errno);
+        return NULL;
+    }
+
+    /* truncate the file to the requested size */
+    if(ftruncate(fd, mca_mpool_sm_module.sm_min_size) != 0) {
+        ompi_output(0, "mca_mpool_sm_mmap_init: ftruncate failed with errno=%d\n", errno);
+        return NULL;
+    }
+
+    /* map the file and initialize segment state */
+    seg = mmap(NULL, size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
+    if(NULL == seg) {
+        ompi_output(0, "mca_mpool_sm_mmap_init: mmap failed with errno=%d\n", errno);
+        return NULL;
+    }
+
+    spinunlock(&seg->seg_lock);
+    seg->seg_offset = 0;
+    seg->seg_size = size;
+
+    map = OBJ_NEW(mca_mpool_sm_mmap_t);
+    strncpy(map->map_path, path, PATH_MAX);
+    map->map_seg = seg;
+    map->map_addr = (unsigned char*)(seg + 1);
+    map->map_size = size - sizeof(mca_mpool_sm_segment_t);
+
+    /* enable access by other processes on this host */
+    if(fchmod(fd, 0600) != 0) {
+        ompi_output(0, "mca_mpool_sm_mmap_init: fchmod failed with errno=%d\n", errno);
+        OBJ_RELEASE(map);
+        close(fd);
+        return NULL;
+    }
+    close(fd);
+    return map;
+}
+
+
+void* mca_mpool_sm_mmap_alloc(size_t* size)
+{
+#if 0
+    mca_mpool_sm_mmap_t* map = mca_mpool_sm_module.sm_mmap;
+    mca_mpool_sm_segment_t* seg = map->map_seg;
+    void* addr;
+
+    spinlock(&seg->seg_lock);
+    if(seg->seg_offset + *size > seg->seg_size) {
+        addr = NULL;
+    } else {
+        addr = map->sm_addr + seg->seg_offset;
+        seg->seg_offset += *size;
+    }
+    spinunlock(&seg->seg_lock);
+    return addr;
+#else
+    return NULL;
+#endif
+}
+

src/mca/mpool/sm/mpool_sm_mmap.h

@@ -0,0 +1,34 @@
+#ifndef _MPOOL_SM_MMAP_H_
+#define _MPOOL_SM_MMAP_H_
+
+#include "class/ompi_object.h"
+#include "os/atomic.h"
+#include "class/ompi_list.h"
+
+struct mca_mpool_sm_segment_t {
+    ompi_lock_data_t seg_lock;
+    size_t seg_offset;
+    size_t seg_size;
+};
+typedef struct mca_mpool_sm_segment_t mca_mpool_sm_segment_t;
+
+
+struct mca_mpool_sm_mmap_t {
+    ompi_list_item_t map_item;
+    mca_mpool_sm_segment_t* map_seg;
+    unsigned char* map_addr;
+    size_t map_size;
+    char map_path[PATH_MAX];
+};
+typedef struct mca_mpool_sm_mmap_t mca_mpool_sm_mmap_t;
+
+OBJ_CLASS_DECLARATION(mca_mpool_sm_mmap_t);
+
+
+mca_mpool_sm_mmap_t* mca_mpool_sm_mmap_init(size_t size);
+void* mca_mpool_sm_mmap_alloc(size_t* size);
+void  mca_mpool_sm_mmap_free(void* addr);
+
+
+#endif
+

src/mca/ptl/sm/src/ptl_sm.h

@@ -22,19 +22,14 @@
  */
 struct mca_ptl_sm_module_1_0_0_t {
     mca_ptl_base_module_1_0_0_t super;   /**< base PTL module */
-    int sm_min_alloc;                    /**< min size of shared memory allocation */
-    int sm_max_alloc;                    /**< max size of shared memory allocation */
     int sm_free_list_num;                /**< initial size of free lists */
     int sm_free_list_max;                /**< maximum size of free lists */
     int sm_free_list_inc;                /**< number of elements to alloc when growing free lists */
     int sm_max_procs;                    /**< upper limit on the number of processes using the shared memory pool */
-    void* sm_base_addr;                  /**< base address of mmaped region */
     ompi_free_list_t sm_send_requests;    /**< free list of sm send requests -- sendreq + sendfrag */
     ompi_free_list_t sm_send_frags;       /**< free list of sm send fragments */
     ompi_free_list_t sm_recv_frags;       /**< free list of sm recv fragments */
     ompi_allocator_t sm_allocator;        /**< shared memory allocator */
-    char sm_mmap_file[PATH_MAX];         /**< full path to backing file */
-    mca_ptl_sm_mmap_t *sm_mmap;
     ompi_mutex_t sm_lock;
 };
 typedef struct mca_ptl_sm_module_1_0_0_t mca_ptl_sm_module_1_0_0_t;

src/mca/ptl/sm/src/ptl_sm_module.c

@@ -104,12 +104,6 @@ static inline int mca_ptl_sm_param_register_int(
 int mca_ptl_sm_module_open(void)
 {
     /* register SM module parameters */
-    mca_ptl_sm_module.sm_min_alloc =
-        mca_ptl_sm_param_register_int("min_alloc", 64*1024*1024);
-    mca_ptl_sm_module.sm_max_alloc =
-        mca_ptl_sm_param_register_int("max_alloc", 512*1024*1024);
-    mca_ptl_sm_module.sm_base_addr = (void*)
-        mca_ptl_sm_param_register_int("base_addr", 0x7fffffff);
     mca_ptl_sm_module.sm_free_list_num =
         mca_ptl_sm_param_register_int("free_list_num", 256);
     mca_ptl_sm_module.sm_free_list_max =