diff --git a/ompi/mca/rcache/rb/rcache_rb.c b/ompi/mca/rcache/rb/rcache_rb.c index 133e165215..7a057c402e 100644 --- a/ompi/mca/rcache/rb/rcache_rb.c +++ b/ompi/mca/rcache/rb/rcache_rb.c @@ -96,14 +96,19 @@ int mca_rcache_rb_insert ( uint32_t flags ) { int rc = OMPI_SUCCESS; - OPAL_THREAD_LOCK(&rcache->lock); reg->flags = flags; - if(flags & MCA_MPOOL_FLAGS_CACHE) { rc = mca_rcache_rb_mru_insert( (mca_rcache_rb_module_t*) rcache, reg); if(OMPI_SUCCESS != rc) { - OPAL_THREAD_UNLOCK(&rcache->lock); + OPAL_THREAD_UNLOCK(&rcache->lock); + if(OMPI_ERR_TEMP_OUT_OF_RESOURCE == rc) { + /* if the registration is too big for the rcache, + don't cahce it and reset the flags so the upper level + handles things appropriatly */ + reg->flags = 0; + return OMPI_SUCCESS; + } return rc; } OPAL_THREAD_ADD32((int32_t*)®->ref_count, 1); diff --git a/ompi/mca/rcache/rb/rcache_rb.h b/ompi/mca/rcache/rb/rcache_rb.h index 64bd38e4f2..6db1003099 100644 --- a/ompi/mca/rcache/rb/rcache_rb.h +++ b/ompi/mca/rcache/rb/rcache_rb.h @@ -27,21 +27,21 @@ #include "ompi/class/ompi_rb_tree.h" #include "ompi/mca/rcache/rcache.h" - struct mca_rcache_rb_module_t { mca_rcache_base_module_t base; ompi_rb_tree_t rb_tree; ompi_free_list_t rb_tree_item_list; opal_list_t mru_list; size_t reg_mru_len; - + size_t reg_max_mru_size; + size_t reg_cur_mru_size; + }; typedef struct mca_rcache_rb_module_t mca_rcache_rb_module_t; struct mca_rcache_rb_component_t { mca_rcache_base_component_t super; - size_t reg_mru_len; }; typedef struct mca_rcache_rb_component_t mca_rcache_rb_component_t; OMPI_COMP_EXPORT extern mca_rcache_rb_component_t mca_rcache_rb_component; diff --git a/ompi/mca/rcache/rb/rcache_rb_component.c b/ompi/mca/rcache/rb/rcache_rb_component.c index bc416ebe62..f7c04441b1 100644 --- a/ompi/mca/rcache/rb/rcache_rb_component.c +++ b/ompi/mca/rcache/rb/rcache_rb_component.c @@ -57,16 +57,27 @@ static int mca_rcache_rb_component_open(void) mca_rcache_base_module_t* mca_rcache_rb_component_init(void) { mca_rcache_rb_module_t* rcache; + + rcache = (mca_rcache_rb_module_t*) malloc(sizeof(mca_rcache_rb_module_t)); + mca_rcache_rb_module_init(rcache); + mca_base_param_reg_int(&mca_rcache_rb_component.super.rcache_version, "mru_len", - "The maximum size of the MRU (most recently used) rcache list", + "The maximum size IN ENTRIES of the MRU (most recently used) rcache list", false, false, 256, - (int*)&(mca_rcache_rb_component.reg_mru_len)); + (int*)&(rcache->reg_mru_len)); + + mca_base_param_reg_int(&mca_rcache_rb_component.super.rcache_version, + "mru_size", + "The maximum size IN BYTES of the MRU (most recently used) rcache list", + false, + false, + 1*1024*1024*1024, /* default to 1GB? */ + (int*)&(rcache->reg_max_mru_size)); + - rcache = (mca_rcache_rb_module_t*) malloc(sizeof(mca_rcache_rb_module_t)); - mca_rcache_rb_module_init(rcache); return &rcache->base; } diff --git a/ompi/mca/rcache/rb/rcache_rb_mru.c b/ompi/mca/rcache/rb/rcache_rb_mru.c index 44aadb2373..ffd8a93522 100644 --- a/ompi/mca/rcache/rb/rcache_rb_mru.c +++ b/ompi/mca/rcache/rb/rcache_rb_mru.c @@ -29,7 +29,7 @@ */ int mca_rcache_rb_mru_init(mca_rcache_rb_module_t* rcache){ OBJ_CONSTRUCT(&rcache->mru_list, opal_list_t); - rcache->reg_mru_len = mca_rcache_rb_component.reg_mru_len; + rcache->reg_cur_mru_size = 0; return OMPI_SUCCESS; } @@ -42,6 +42,14 @@ int mca_rcache_rb_mru_insert( ) { mca_mpool_base_registration_t* old_reg; + size_t reg_size = reg->bound - reg->base + 1; + + if(reg_size > rcache->reg_max_mru_size) { + return OMPI_ERR_TEMP_OUT_OF_RESOURCE; + } + + rcache->reg_cur_mru_size += reg_size; + if(rcache->reg_mru_len <= rcache->mru_list.opal_list_length) { /* call deregister - which removes the registration from * the tree and mru list. memory will be deregistered when @@ -55,7 +63,20 @@ int mca_rcache_rb_mru_insert( old_reg->mpool->mpool_deregister(old_reg->mpool, old_reg); } + + while(rcache->reg_max_mru_size <= rcache->reg_cur_mru_size) { + old_reg = (mca_mpool_base_registration_t*) + opal_list_get_first(&rcache->mru_list); + /* we need to retain first, because we only want the registration + removed from the tree and the mru */ + old_reg->mpool->mpool_retain(old_reg->mpool, old_reg); + old_reg->mpool->mpool_deregister(old_reg->mpool, old_reg); + rcache->reg_cur_mru_size -= (old_reg->bound - old_reg->base + 1); + + } + opal_list_append(&rcache->mru_list,(opal_list_item_t*) reg); + return OMPI_SUCCESS; }