From c4fb4d7ef4d70a41d2d1de36d34c3a43176dd228 Mon Sep 17 00:00:00 2001 From: Tim Woodall Date: Wed, 16 Jun 2004 15:41:29 +0000 Subject: [PATCH] - add shared memory pool - working on mpool/allocator framework This commit was SVN r1313. --- src/mca/allocator/allocator.h | 2 + src/mca/allocator/base/allocator_base_close.c | 11 +- src/mca/allocator/base/allocator_base_open.c | 15 +- src/mca/allocator/base/base.h | 15 +- src/mca/mpool/base/Makefile.am | 2 +- src/mca/mpool/base/base.h | 8 +- src/mca/mpool/base/mpool_base_close.c | 2 +- ...{mpool_base_select.c => mpool_base_init.c} | 61 +++----- src/mca/mpool/mpool.h | 71 +++++---- src/mca/mpool/sm/Makefile.am | 19 +++ src/mca/mpool/sm/configure.params | 10 ++ src/mca/mpool/sm/mpool_sm.c | 46 ++++++ src/mca/mpool/sm/mpool_sm.h | 64 +++++++++ src/mca/mpool/sm/mpool_sm_component.c | 90 ++++++++++++ src/mca/mpool/sm/mpool_sm_mmap.c | 136 ++++++++++++++++++ src/mca/mpool/sm/mpool_sm_mmap.h | 34 +++++ src/mca/ptl/sm/src/ptl_sm.h | 5 - src/mca/ptl/sm/src/ptl_sm_module.c | 6 - 18 files changed, 490 insertions(+), 107 deletions(-) rename src/mca/mpool/base/{mpool_base_select.c => mpool_base_init.c} (54%) create mode 100644 src/mca/mpool/sm/Makefile.am create mode 100644 src/mca/mpool/sm/configure.params create mode 100644 src/mca/mpool/sm/mpool_sm.c create mode 100644 src/mca/mpool/sm/mpool_sm.h create mode 100644 src/mca/mpool/sm/mpool_sm_component.c create mode 100644 src/mca/mpool/sm/mpool_sm_mmap.c create mode 100644 src/mca/mpool/sm/mpool_sm_mmap.h diff --git a/src/mca/allocator/allocator.h b/src/mca/allocator/allocator.h index a618acdc33..21cf882d4e 100644 --- a/src/mca/allocator/allocator.h +++ b/src/mca/allocator/allocator.h @@ -78,5 +78,7 @@ typedef struct mca_allocator_base_module_1_0_0_t mca_allocator_base_module_t; /* ptl v1.0 */ \ "allocator", 1, 0, 0 +extern int mca_allocator_base_output; + #endif /* MCA_ALLOCATOR_H */ diff --git a/src/mca/allocator/base/allocator_base_close.c b/src/mca/allocator/base/allocator_base_close.c index b1d83e5249..38108e5f4f 100644 --- a/src/mca/allocator/base/allocator_base_close.c +++ b/src/mca/allocator/base/allocator_base_close.c @@ -20,24 +20,23 @@ int mca_allocator_base_close(void) /* Finalize all the allocator modules and free their list items */ - for (item = ompi_list_remove_first(&mca_allocator_base_modules_initialized); + for (item = ompi_list_remove_first(&mca_allocator_base_modules); NULL != item; - item = ompi_list_remove_first(&mca_allocator_base_modules_initialized)) { + item = ompi_list_remove_first(&mca_allocator_base_modules)) { sm = (mca_allocator_base_selected_module_t *) item; /* Blatently ignore the return code (what would we do to recover, anyway? This module is going away, so errors don't matter anymore) */ - sm->absm_actions->alc_finalize(sm->absm_actions); - free(sm); + sm->allocator_module->alc_finalize(sm->allocator_module); + OBJ_RELEASE(sm); } /* Close all remaining available modules (may be one if this is a OMPI RTE program, or [possibly] multiple if this is ompi_info) */ - mca_base_modules_close(mca_allocator_base_output, - &mca_allocator_base_modules_available, NULL); + mca_base_modules_close(mca_allocator_base_output, &mca_allocator_base_modules, NULL); /* All done */ diff --git a/src/mca/allocator/base/allocator_base_open.c b/src/mca/allocator/base/allocator_base_open.c index 36a514d7fb..321f632aef 100644 --- a/src/mca/allocator/base/allocator_base_open.c +++ b/src/mca/allocator/base/allocator_base_open.c @@ -24,10 +24,9 @@ /* * Global variables */ +ompi_list_t mca_allocator_base_components; +ompi_list_t mca_allocator_base_modules; int mca_allocator_base_output = -1; -ompi_list_t mca_allocator_base_modules_available; -ompi_list_t mca_allocator_base_modules_initialized; - /** * Function for finding and opening either all MCA modules, or the one @@ -39,18 +38,10 @@ int mca_allocator_base_open(void) if (OMPI_SUCCESS != mca_base_modules_open("allocator", 0, mca_allocator_base_static_modules, - &mca_allocator_base_modules_available)) { + &mca_allocator_base_components)) { return OMPI_ERROR; } - - /* Initialize the list so that in mca_allocator_base_close(), we can - iterate over it (even if it's empty, as in the case of - ompi_info) */ - - OBJ_CONSTRUCT(&mca_allocator_base_modules_initialized, ompi_list_t); - /* All done */ - return OMPI_SUCCESS; } diff --git a/src/mca/allocator/base/base.h b/src/mca/allocator/base/base.h index f2fcea5842..fc20be7f48 100644 --- a/src/mca/allocator/base/base.h +++ b/src/mca/allocator/base/base.h @@ -16,11 +16,13 @@ struct mca_allocator_base_selected_module_t { ompi_list_item_t super; - mca_allocator_base_module_t *apsm_module; - mca_allocator_t *absm_actions; + mca_allocator_base_module_t *allocator_component; + mca_allocator_t *allocator_module; }; typedef struct mca_allocator_base_selected_module_t mca_allocator_base_selected_module_t; - + +OBJ_CLASS_DECLARATION(mca_mpool_base_selected_module_t); + /* * Global functions for MCA: overall PTL open and close @@ -32,7 +34,7 @@ extern "C" { int mca_allocator_base_open(void); int mca_allocator_base_init(bool *allow_multi_user_threads); int mca_allocator_base_close(void); - mca_allocator_t* mca_allocator_lookup(const char* name); + mca_allocator_base_module_t* mca_allocator_module_lookup(const char* name); #if defined(c_plusplus) || defined(__cplusplus) } #endif @@ -43,8 +45,7 @@ extern "C" { */ -extern int mca_allocator_base_output; -extern ompi_list_t mca_allocator_base_modules_available; -extern ompi_list_t mca_allocator_base_modules_initialized; +extern ompi_list_t mca_allocator_base_components; +extern ompi_list_t mca_allocator_base_modules; #endif /* MCA_ALLOCATOR_BASE_H */ diff --git a/src/mca/mpool/base/Makefile.am b/src/mca/mpool/base/Makefile.am index 407ca75b73..2ea6df6add 100644 --- a/src/mca/mpool/base/Makefile.am +++ b/src/mca/mpool/base/Makefile.am @@ -19,7 +19,7 @@ libmca_mpool_base_la_SOURCES = \ $(headers) \ mpool_base_open.c \ mpool_base_close.c \ - mpool_base_select.c + mpool_base_init.c # Conditionally install the header files diff --git a/src/mca/mpool/base/base.h b/src/mca/mpool/base/base.h index 46d795dbcf..8546d49d30 100644 --- a/src/mca/mpool/base/base.h +++ b/src/mca/mpool/base/base.h @@ -16,11 +16,12 @@ struct mca_mpool_base_selected_module_t { ompi_list_item_t super; - mca_mpool_base_module_t *mbsm_module; - mca_mpool_t *mbsm_actions; + mca_mpool_base_component_t *mpool_component; + mca_mpool_t *mpool_module; }; typedef struct mca_mpool_base_selected_module_t mca_mpool_base_selected_module_t; +OBJ_CLASS_DECLARATION(mca_mpool_base_selected_module_t); /* * Global functions for MCA: overall PTL open and close @@ -30,8 +31,9 @@ typedef struct mca_mpool_base_selected_module_t mca_mpool_base_selected_module_t extern "C" { #endif int mca_mpool_base_open(void); - int mca_mpool_base_select(bool *allow_multi_user_threads); + int mca_mpool_base_init(bool *allow_multi_user_threads); int mca_mpool_base_close(void); + mca_mpool_t* mca_mpool_lookup(const char* name); void* mca_mpool_base_is_registered(void* addr, size_t size); #if defined(c_plusplus) || defined(__cplusplus) } diff --git a/src/mca/mpool/base/mpool_base_close.c b/src/mca/mpool/base/mpool_base_close.c index 634005e660..7c7680da2a 100644 --- a/src/mca/mpool/base/mpool_base_close.c +++ b/src/mca/mpool/base/mpool_base_close.c @@ -29,7 +29,7 @@ int mca_mpool_base_close(void) anyway? This module is going away, so errors don't matter anymore) */ - sm->mbsm_actions->mpool_finalize(sm->mbsm_actions); + sm->mpool_module->mpool_finalize(sm->mpool_module); free(sm); } diff --git a/src/mca/mpool/base/mpool_base_select.c b/src/mca/mpool/base/mpool_base_init.c similarity index 54% rename from src/mca/mpool/base/mpool_base_select.c rename to src/mca/mpool/base/mpool_base_init.c index b254c94002..699796356d 100644 --- a/src/mca/mpool/base/mpool_base_select.c +++ b/src/mca/mpool/base/mpool_base_init.c @@ -5,12 +5,16 @@ #include "ompi_config.h" #include "runtime/runtime.h" +#include "util/output.h" #include "mca/mca.h" #include "mca/base/base.h" #include "mca/mpool/mpool.h" #include "mca/mpool/base/base.h" +OBJ_CLASS_INSTANCE(mca_mpool_base_selected_module_t, ompi_list_item_t, NULL, NULL); + + /** * Function for weeding out mpool modules that don't want to run. * @@ -19,15 +23,13 @@ * will be closed and unloaded. The selected modules will be returned * to the caller in a ompi_list_t. */ -int mca_mpool_base_select(bool *allow_multi_user_threads) +int mca_mpool_base_init(bool *allow_multi_user_threads) { -#if 0 - int i, num_mpools; - bool user_threads, hidden_threads; + bool user_threads; ompi_list_item_t *item; mca_base_module_list_item_t *mli; - mca_mpool_base_module_t *module; - mca_mpool_t **actions; + mca_mpool_base_component_t *component; + mca_mpool_t *module; mca_mpool_base_selected_module_t *sm; /* Traverse the list of available modules; call their init @@ -37,64 +39,45 @@ int mca_mpool_base_select(bool *allow_multi_user_threads) ompi_list_get_end(&mca_mpool_base_modules_available) != item; item = ompi_list_get_next(item)) { mli = (mca_base_module_list_item_t *) item; - module = (mca_mpool_base_module_t *) mli->mli_module; + component = (mca_mpool_base_component_t *) mli->mli_module; ompi_output_verbose(10, mca_mpool_base_output, "select: initializing %s module %s", - module->mpoolm_version.mca_type_name, - module->mpoolm_version.mca_module_name); - if (NULL == module->mpoolm_init) { + component->mpool_version.mca_type_name, + component->mpool_version.mca_module_name); + if (NULL == component->mpool_init) { ompi_output_verbose(10, mca_mpool_base_output, "select: no init function; ignoring module"); } else { - actions = module->mpoolm_init(&num_mpools, &user_threads, - &hidden_threads); + module = component->mpool_init(&user_threads); /* If the module didn't initialize, unload it */ - if (NULL == actions) { + if (NULL == module) { ompi_output_verbose(10, mca_mpool_base_output, "select: init returned failure"); - mca_base_module_repository_release((mca_base_module_t *) module); + mca_base_module_repository_release((mca_base_module_t *) component); ompi_output_verbose(10, mca_mpool_base_output, - "select: module %s unloaded", - module->mpoolm_version.mca_module_name); + "select: component %s unloaded", + component->mpool_version.mca_module_name); } /* Otherwise, it initialized properly. Save it. */ else { *allow_multi_user_threads |= user_threads; - *have_hidden_threads |= hidden_threads; ompi_output_verbose(10, mca_mpool_base_output, "select: init returned success"); - for (i = 0; i < num_mpools; ++i) { - sm = malloc(sizeof(mca_mpool_base_selected_module_t)); - if (NULL == sm) { - return OMPI_ERR_OUT_OF_RESOURCE; - } - OBJ_CONSTRUCT(sm, ompi_list_item_t); - sm->pbsm_module = module; - sm->pbsm_actions = actions[i]; - ompi_list_append(&mca_mpool_base_modules_initialized, - (ompi_list_item_t*) sm); + sm = OBJ_NEW(mca_mpool_base_selected_module_t); + sm->mpool_component = component; + sm->mpool_module = module; + ompi_list_append(&mca_mpool_base_modules_initialized, (ompi_list_item_t*) sm); } - free(actions); } - } } - - /* Finished querying all modules. Check for the bozo case. */ - - if (0 == ompi_list_get_size(&mca_mpool_base_modules_initialized)) { - /* JMS Replace with show_help */ - ompi_abort(1, "No mpool module available. This shouldn't happen."); - } - - /* All done */ -#endif return OMPI_SUCCESS; } + diff --git a/src/mca/mpool/mpool.h b/src/mca/mpool/mpool.h index d19ddd166f..9eb9ac4af3 100644 --- a/src/mca/mpool/mpool.h +++ b/src/mca/mpool/mpool.h @@ -14,57 +14,74 @@ struct mca_mpool_t; /** * allocate function typedef */ -typedef void* (*mca_mpool_alloc_fn_t)(struct mca_mpool_t*, size_t size, size_t align); +typedef void* (*mca_mpool_base_alloc_fn_t)(size_t size, size_t align); /** * realloc function typedef */ -typedef void* (*mca_mpool_realloc_fn_t)(struct mca_mpool_t*, void* addr, size_t size); +typedef void* (*mca_mpool_base_realloc_fn_t)(void* addr, size_t size); /** * free function typedef */ -typedef void (*mca_mpool_free_fn_t)(struct mca_mpool_t*, void *); +typedef void (*mca_mpool_base_free_fn_t)(void *); /** * register memory */ -typedef void (*mca_mpool_register_fn_t)(struct mca_mpool_t*, void * addr, size_t size, void* user); +typedef void (*mca_mpool_base_register_fn_t)(void * addr, size_t size, void* user); /** * deregister memory */ -typedef void (*mca_mpool_deregister_fn_t)(struct mca_mpool_t*, void * addr); +typedef void (*mca_mpool_base_deregister_fn_t)(void * addr, size_t size); -typedef int (*mca_mpool_base_finalize_fn_t)( - struct mca_mpool_t* mpool -); +/** + * finalize + */ +typedef void (*mca_mpool_base_finalize_fn_t)(struct mca_mpool_t*); +/** + * mpool component descriptor. Contains component version information + * and open/close/init functions. + */ + +typedef struct mca_mpool_t* (*mca_mpool_base_init_fn_t)(bool *allow_multi_user_threads); + + +struct mca_mpool_base_component_1_0_0_t { + mca_base_module_t mpool_version; + mca_base_module_data_1_0_0_t mpool_data; + mca_mpool_base_init_fn_t mpool_init; +}; +typedef struct mca_mpool_base_component_1_0_0_t mca_mpool_base_component_1_0_0_t; +typedef struct mca_mpool_base_component_1_0_0_t mca_mpool_base_component_t; + + +/** + * mpool module descriptor. Contains functions exported + * by the component. + */ struct mca_mpool_t { - /* interface functions */ - mca_mpool_alloc_fn_t mpool_alloc; - mca_mpool_alloc_fn_t mpool_realloc; - mca_mpool_free_fn_t mpool_free; - mca_mpool_register_fn_t mpool_register; - mca_mpool_deregister_fn_t mpool_deregister; + mca_mpool_base_component_t *mpool_component; + mca_mpool_base_alloc_fn_t mpool_alloc; + mca_mpool_base_realloc_fn_t mpool_realloc; + mca_mpool_base_free_fn_t mpool_free; + mca_mpool_base_register_fn_t mpool_register; + mca_mpool_base_deregister_fn_t mpool_deregister; mca_mpool_base_finalize_fn_t mpool_finalize; }; typedef struct mca_mpool_t mca_mpool_t; -/** - * module initialization function - */ -typedef struct mca_mpool_t* (*mca_mpool_base_module_init_fn_t)( - bool *allow_multi_user_threads -); - -struct mca_mpool_base_module_1_0_0_t { - mca_base_module_t mpool_version; - mca_base_module_data_1_0_0_t mpool_data; - mca_mpool_base_module_init_fn_t mpool_init; -}; -typedef struct mca_mpool_base_module_1_0_0_t mca_mpool_base_module_t; +/* + * Macro for use in modules that are of type ptl v1.0.0 + */ +#define MCA_MPOOL_BASE_VERSION_1_0_0 \ + /* mpool v1.0 is chained to MCA v1.0 */ \ + MCA_BASE_VERSION_1_0_0, \ + /* ptl v1.0 */ \ + "mpool", 1, 0, 0 #endif /* MCA_MPOOL_H */ diff --git a/src/mca/mpool/sm/Makefile.am b/src/mca/mpool/sm/Makefile.am new file mode 100644 index 0000000000..0c845680d8 --- /dev/null +++ b/src/mca/mpool/sm/Makefile.am @@ -0,0 +1,19 @@ +# -*- makefile -*- +# +# $HEADER$ +# + +include $(top_ompi_srcdir)/config/Makefile.options + +AM_CPPFLAGS = \ + -I$(top_ompi_builddir)/src/include \ + -I$(top_ompi_srcdir)/src \ + -I$(top_ompi_srcdir)/src/include + +noinst_LTLIBRARIES = libmca_mpool_sm.la +libmca_mpool_sm_la_SOURCES = \ + mpool_sm.c \ + mpool_sm.h \ + mpool_sm_mmap.c \ + mpool_sm_mmap.h \ + mpool_sm_component.c diff --git a/src/mca/mpool/sm/configure.params b/src/mca/mpool/sm/configure.params new file mode 100644 index 0000000000..35c7aad5c8 --- /dev/null +++ b/src/mca/mpool/sm/configure.params @@ -0,0 +1,10 @@ +# -*- shell-script -*- +# +# $HEADER$ +# + +# Specific to this module + +PARAM_INIT_FILE=mpool_sm.c +PARAM_CONFIG_HEADER_FILE="sm_config.h" +PARAM_CONFIG_FILES="Makefile" diff --git a/src/mca/mpool/sm/mpool_sm.c b/src/mca/mpool/sm/mpool_sm.c new file mode 100644 index 0000000000..c921e7de71 --- /dev/null +++ b/src/mca/mpool/sm/mpool_sm.c @@ -0,0 +1,46 @@ +/* + * $HEADER$ + */ + +#include +#include "util/output.h" +#include "mca/mpool/sm/mpool_sm.h" + + +mca_mpool_t mca_mpool_sm = { + &mca_mpool_sm_module.super, + mca_mpool_sm_alloc, + mca_mpool_sm_realloc, + mca_mpool_sm_free, + NULL, + NULL +}; + + +/** + * allocate function + */ +void* mca_mpool_sm_alloc(size_t size, size_t align) +{ + return mca_mpool_sm_module.sm_allocator->alc_alloc( + mca_mpool_sm_module.sm_allocator, size, align); +} + +/** + * realloc function + */ +void* mca_mpool_sm_realloc(void* addr, size_t size) +{ + return mca_mpool_sm_module.sm_allocator->alc_realloc( + mca_mpool_sm_module.sm_allocator, addr, size); +} + +/** + * free function + */ +void mca_mpool_sm_free(void * addr) +{ + mca_mpool_sm_module.sm_allocator->alc_free( + mca_mpool_sm_module.sm_allocator, addr); +} + diff --git a/src/mca/mpool/sm/mpool_sm.h b/src/mca/mpool/sm/mpool_sm.h new file mode 100644 index 0000000000..8213ae58f4 --- /dev/null +++ b/src/mca/mpool/sm/mpool_sm.h @@ -0,0 +1,64 @@ +/* + * $HEADER$ + */ +/** + * @file + */ +#ifndef MCA_MPOOL_SM_H +#define MCA_MPOOL_SM_H + +#include "class/ompi_list.h" +#include "class/ompi_free_list.h" +#include "event/event.h" +#include "mca/mpool/mpool.h" +#include "mca/allocator/allocator.h" + + +struct mca_mpool_sm_component_t { + mca_mpool_base_component_t super; + mca_allocator_t* sm_allocator; + char* sm_allocator_name; + size_t sm_min_size; + size_t sm_max_size; + size_t sm_size; + size_t sm_segment; + ompi_list_t sm_mmap; +}; +typedef struct mca_mpool_sm_component_t mca_mpool_sm_component_t; + +extern mca_mpool_sm_component_t mca_mpool_sm_module; + +/** + * allocate function typedef + */ +void* mca_mpool_sm_alloc(size_t size, size_t align); + +/** + * realloc function typedef + */ +void* mca_mpool_sm_realloc(void* addr, size_t size); + +/** + * free function typedef + */ +void mca_mpool_sm_free(void *); + +/** + * register memory + */ +void mca_mpool_sm_register(void * addr, size_t size, void* user); + +/** + * deregister memory + */ +void mca_mpool_sm_deregister(void * addr); + +/** + * component open/close/init function + */ +int mca_mpool_sm_open(void); +int mca_mpool_sm_close(void); +mca_mpool_t* mca_mpool_sm_init(bool *allow_multi_user_threads); + + +#endif diff --git a/src/mca/mpool/sm/mpool_sm_component.c b/src/mca/mpool/sm/mpool_sm_component.c new file mode 100644 index 0000000000..140df38d03 --- /dev/null +++ b/src/mca/mpool/sm/mpool_sm_component.c @@ -0,0 +1,90 @@ +#include "mca/base/mca_base_param.h" +#include "mca/mpool/sm/mpool_sm.h" + + +mca_mpool_sm_component_t mca_mpool_sm_module = { + { + /* First, the mca_base_module_t struct containing meta information + about the module itself */ + + { + /* Indicate that we are a mpool v1.0.0 component (which also implies a + specific MCA version) */ + + MCA_MPOOL_BASE_VERSION_1_0_0, + + "sm", /* MCA component name */ + 1, /* MCA module major version */ + 0, /* MCA module minor version */ + 0, /* MCA module release version */ + mca_mpool_sm_open, /* component open */ + mca_mpool_sm_close /* component close */ + }, + + + /* Next the MCA v1.0.0 module meta data */ + + + { + /* Whether the module is checkpointable or not */ + + + false + }, + + mca_mpool_sm_init + } +}; + +static inline char* mca_mpool_sm_param_register_string( + const char* param_name, + const char* default_value) +{ + char *param_value; + int id = mca_base_param_register_string("mpool","sm",param_name,NULL,default_value); + mca_base_param_lookup_string(id, ¶m_value); + return param_value; +} + +static inline int mca_mpool_sm_param_register_int( + const char* param_name, + int default_value) +{ + int id = mca_base_param_register_int("mpool","sm",param_name,NULL,default_value); + int param_value = default_value; + mca_base_param_lookup_int(id,¶m_value); + return param_value; +} + + +/** + * component open/close/init function + */ +int mca_mpool_sm_open(void) +{ + /* register SM module parameters */ + mca_mpool_sm_module.sm_min_size = + mca_mpool_sm_param_register_int("min_size", 64*1024*1024); + mca_mpool_sm_module.sm_max_size = + mca_mpool_sm_param_register_int("max_size", 512*1024*1024); + mca_mpool_sm_module.sm_allocator_name = + mca_mpool_sm_param_register_string("allocator", "bucket"); + mca_mpool_sm_module.sm_segment = 1; + mca_mpool_sm_module.sm_size = 0; + return OMPI_SUCCESS; +} + + +int mca_mpool_sm_close(void) +{ + return OMPI_SUCCESS; +} + + +mca_mpool_t* mca_mpool_sm_init(bool *allow_multi_user_threads) +{ + return NULL; +} + + + diff --git a/src/mca/mpool/sm/mpool_sm_mmap.c b/src/mca/mpool/sm/mpool_sm_mmap.c new file mode 100644 index 0000000000..6b4e502fc9 --- /dev/null +++ b/src/mca/mpool/sm/mpool_sm_mmap.c @@ -0,0 +1,136 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "constants.h" +#include "util/output.h" +#include "mca/pcm/pcm.h" +#include "mpool_sm.h" +#include "mpool_sm_mmap.h" + + +OBJ_CLASS_INSTANCE( + mca_mpool_sm_mmap_t, + ompi_object_t, + NULL, + NULL +); + + +static mca_mpool_sm_mmap_t* mca_mpool_sm_mmap_open(char* path) +{ + mca_mpool_sm_mmap_t* map; + mca_mpool_sm_segment_t* seg; + int fd = -1; + + while(fd < 0) { + struct timespec ts; + fd = open(path, O_CREAT|O_RDWR, 0000); + if(fd < 0 && errno != EACCES) { + ompi_output(0, "mca_ptl_sm_mmap_open: open failed with errno=%d\n", errno); + return NULL; + } + ts.tv_sec = 0; + ts.tv_nsec = 500000; + nanosleep(&ts,NULL); + } + + /* map the file and initialize segment state */ + seg = mmap(NULL, mca_mpool_sm_module.sm_min_size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0); + if(NULL == seg) { + ompi_output(0, "mca_mpool_sm_mmap_open: mmap failed with errno=%d\n", errno); + return NULL; + } + close(fd); + + map = OBJ_NEW(mca_mpool_sm_mmap_t); + map->map_seg = seg; + map->map_addr = (unsigned char*)(seg + 1); + map->map_size = seg->seg_size - sizeof(mca_mpool_sm_segment_t); + close(fd); + return map; +} + + +mca_mpool_sm_mmap_t* mca_mpool_sm_mmap_init(size_t size) +{ + ompi_job_handle_t job_handle = mca_pcm.pcm_handle_get(); + char hostname[64]; + int fd; + mca_mpool_sm_segment_t* seg; + mca_mpool_sm_mmap_t* map; + char path[PATH_MAX]; + + gethostname(hostname, sizeof(hostname)); + sprintf(path, "/tmp/%s.%s.%d", hostname, job_handle, + ompi_list_get_size(&mca_mpool_sm_module.sm_mmap)+1); + fd = open(path, O_CREAT|O_RDWR, 0000); + if(fd < 0) { + if(errno == EACCES) + return mca_mpool_sm_mmap_open(path); + ompi_output(0, "mca_mpool_sm_mmap_init: open failed with errno=%d\n", errno); + return NULL; + } + + /* truncate the file to the requested size */ + if(ftruncate(fd, mca_mpool_sm_module.sm_min_size) != 0) { + ompi_output(0, "mca_mpool_sm_mmap_init: ftruncate failed with errno=%d\n", errno); + return NULL; + } + + /* map the file and initialize segment state */ + seg = mmap(NULL, size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0); + if(NULL == seg) { + ompi_output(0, "mca_mpool_sm_mmap_init: mmap failed with errno=%d\n", errno); + return NULL; + } + + spinunlock(&seg->seg_lock); + seg->seg_offset = 0; + seg->seg_size = size; + + map = OBJ_NEW(mca_mpool_sm_mmap_t); + strncpy(map->map_path, path, PATH_MAX); + map->map_seg = seg; + map->map_addr = (unsigned char*)(seg + 1); + map->map_size = size - sizeof(mca_mpool_sm_segment_t); + + /* enable access by other processes on this host */ + if(fchmod(fd, 0600) != 0) { + ompi_output(0, "mca_mpool_sm_mmap_init: fchmod failed with errno=%d\n", errno); + OBJ_RELEASE(map); + close(fd); + return NULL; + } + close(fd); + return map; +} + + +void* mca_mpool_sm_mmap_alloc(size_t* size) +{ +#if 0 + mca_mpool_sm_mmap_t* map = mca_mpool_sm_module.sm_mmap; + mca_mpool_sm_segment_t* seg = map->map_seg; + void* addr; + + spinlock(&seg->seg_lock); + if(seg->seg_offset + *size > seg->seg_size) { + addr = NULL; + } else { + addr = map->sm_addr + seg->seg_offset; + seg->seg_offset += *size; + } + spinunlock(&seg->seg_lock); + return addr; +#else + return NULL; +#endif +} + diff --git a/src/mca/mpool/sm/mpool_sm_mmap.h b/src/mca/mpool/sm/mpool_sm_mmap.h new file mode 100644 index 0000000000..a068fc8046 --- /dev/null +++ b/src/mca/mpool/sm/mpool_sm_mmap.h @@ -0,0 +1,34 @@ +#ifndef _MPOOL_SM_MMAP_H_ +#define _MPOOL_SM_MMAP_H_ + +#include "class/ompi_object.h" +#include "os/atomic.h" +#include "class/ompi_list.h" + +struct mca_mpool_sm_segment_t { + ompi_lock_data_t seg_lock; + size_t seg_offset; + size_t seg_size; +}; +typedef struct mca_mpool_sm_segment_t mca_mpool_sm_segment_t; + + +struct mca_mpool_sm_mmap_t { + ompi_list_item_t map_item; + mca_mpool_sm_segment_t* map_seg; + unsigned char* map_addr; + size_t map_size; + char map_path[PATH_MAX]; +}; +typedef struct mca_mpool_sm_mmap_t mca_mpool_sm_mmap_t; + +OBJ_CLASS_DECLARATION(mca_mpool_sm_mmap_t); + + +mca_mpool_sm_mmap_t* mca_mpool_sm_mmap_init(size_t size); +void* mca_mpool_sm_mmap_alloc(size_t* size); +void mca_mpool_sm_mmap_free(void* addr); + + +#endif + diff --git a/src/mca/ptl/sm/src/ptl_sm.h b/src/mca/ptl/sm/src/ptl_sm.h index dfab9cbb25..c2819ae51a 100644 --- a/src/mca/ptl/sm/src/ptl_sm.h +++ b/src/mca/ptl/sm/src/ptl_sm.h @@ -22,19 +22,14 @@ */ struct mca_ptl_sm_module_1_0_0_t { mca_ptl_base_module_1_0_0_t super; /**< base PTL module */ - int sm_min_alloc; /**< min size of shared memory allocation */ - int sm_max_alloc; /**< max size of shared memory allocation */ int sm_free_list_num; /**< initial size of free lists */ int sm_free_list_max; /**< maximum size of free lists */ int sm_free_list_inc; /**< number of elements to alloc when growing free lists */ int sm_max_procs; /**< upper limit on the number of processes using the shared memory pool */ - void* sm_base_addr; /**< base address of mmaped region */ ompi_free_list_t sm_send_requests; /**< free list of sm send requests -- sendreq + sendfrag */ ompi_free_list_t sm_send_frags; /**< free list of sm send fragments */ ompi_free_list_t sm_recv_frags; /**< free list of sm recv fragments */ ompi_allocator_t sm_allocator; /**< shared memory allocator */ - char sm_mmap_file[PATH_MAX]; /**< full path to backing file */ - mca_ptl_sm_mmap_t *sm_mmap; ompi_mutex_t sm_lock; }; typedef struct mca_ptl_sm_module_1_0_0_t mca_ptl_sm_module_1_0_0_t; diff --git a/src/mca/ptl/sm/src/ptl_sm_module.c b/src/mca/ptl/sm/src/ptl_sm_module.c index 5ce688aa4a..50941d7bd6 100644 --- a/src/mca/ptl/sm/src/ptl_sm_module.c +++ b/src/mca/ptl/sm/src/ptl_sm_module.c @@ -104,12 +104,6 @@ static inline int mca_ptl_sm_param_register_int( int mca_ptl_sm_module_open(void) { /* register SM module parameters */ - mca_ptl_sm_module.sm_min_alloc = - mca_ptl_sm_param_register_int("min_alloc", 64*1024*1024); - mca_ptl_sm_module.sm_max_alloc = - mca_ptl_sm_param_register_int("max_alloc", 512*1024*1024); - mca_ptl_sm_module.sm_base_addr = (void*) - mca_ptl_sm_param_register_int("base_addr", 0x7fffffff); mca_ptl_sm_module.sm_free_list_num = mca_ptl_sm_param_register_int("free_list_num", 256); mca_ptl_sm_module.sm_free_list_max =