Allow registration cache to be limited both by number of entries as well as

total bytes registered. 
Currently this defaults to limiting the rcache to 256 entries or 1GB in total
registrations.

This commit was SVN r9110.

ompi/mca/rcache/rb/rcache_rb.c

@@ -96,14 +96,19 @@ int mca_rcache_rb_insert (
                           uint32_t flags
                           ) { 
     int rc = OMPI_SUCCESS;
-
     OPAL_THREAD_LOCK(&rcache->lock); 
     reg->flags = flags;
-
     if(flags & MCA_MPOOL_FLAGS_CACHE) { 
         rc = mca_rcache_rb_mru_insert( (mca_rcache_rb_module_t*) rcache, reg); 
         if(OMPI_SUCCESS != rc) { 
-            OPAL_THREAD_UNLOCK(&rcache->lock); 
+            OPAL_THREAD_UNLOCK(&rcache->lock);
+            if(OMPI_ERR_TEMP_OUT_OF_RESOURCE == rc) {
+                /* if the registration is too big for the rcache, 
+                   don't cahce it and reset the flags so the upper level 
+                   handles things appropriatly */ 
+                reg->flags = 0;
+                return OMPI_SUCCESS;
+            }
             return rc; 
         }
         OPAL_THREAD_ADD32((int32_t*)&reg->ref_count, 1); 

ompi/mca/rcache/rb/rcache_rb.h

@@ -27,21 +27,21 @@
 #include "ompi/class/ompi_rb_tree.h"
 #include "ompi/mca/rcache/rcache.h"
 
-
 struct mca_rcache_rb_module_t { 
     mca_rcache_base_module_t base; 
     ompi_rb_tree_t rb_tree;
     ompi_free_list_t rb_tree_item_list; 
     opal_list_t mru_list; 
     size_t reg_mru_len; 
-    
+    size_t reg_max_mru_size;
+    size_t reg_cur_mru_size;
+
 };
 typedef struct mca_rcache_rb_module_t mca_rcache_rb_module_t; 
 
 
 struct mca_rcache_rb_component_t { 
     mca_rcache_base_component_t super; 
-    size_t reg_mru_len; 
 }; typedef struct mca_rcache_rb_component_t mca_rcache_rb_component_t; 
 
 OMPI_COMP_EXPORT extern mca_rcache_rb_component_t mca_rcache_rb_component;

ompi/mca/rcache/rb/rcache_rb_component.c

@@ -57,16 +57,27 @@ static int mca_rcache_rb_component_open(void)
 
 mca_rcache_base_module_t* mca_rcache_rb_component_init(void) {
     mca_rcache_rb_module_t* rcache; 
+    
+    rcache = (mca_rcache_rb_module_t*) malloc(sizeof(mca_rcache_rb_module_t));
+    mca_rcache_rb_module_init(rcache);
+ 
     mca_base_param_reg_int(&mca_rcache_rb_component.super.rcache_version, 
                            "mru_len", 
-                           "The maximum size of the MRU (most recently used) rcache list", 
+                           "The maximum size IN ENTRIES of the MRU (most recently used) rcache list", 
                            false, 
                            false, 
                            256, 
-                           (int*)&(mca_rcache_rb_component.reg_mru_len)); 
+                           (int*)&(rcache->reg_mru_len)); 
+    
+    mca_base_param_reg_int(&mca_rcache_rb_component.super.rcache_version, 
+                           "mru_size", 
+                           "The maximum size IN BYTES of the MRU (most recently used) rcache list", 
+                           false, 
+                           false, 
+                           1*1024*1024*1024, /* default to 1GB? */  
+                           (int*)&(rcache->reg_max_mru_size)); 
+
     
-    rcache = (mca_rcache_rb_module_t*) malloc(sizeof(mca_rcache_rb_module_t));
-    mca_rcache_rb_module_init(rcache); 
     return &rcache->base; 
 }
 

ompi/mca/rcache/rb/rcache_rb_mru.c

@@ -29,7 +29,7 @@
  */
 int mca_rcache_rb_mru_init(mca_rcache_rb_module_t* rcache){
     OBJ_CONSTRUCT(&rcache->mru_list, opal_list_t); 
-    rcache->reg_mru_len = mca_rcache_rb_component.reg_mru_len;
+    rcache->reg_cur_mru_size = 0;
     return OMPI_SUCCESS; 
 }
 
@@ -42,6 +42,14 @@ int mca_rcache_rb_mru_insert(
         
                               ) {
     mca_mpool_base_registration_t* old_reg; 
+    size_t reg_size = reg->bound - reg->base + 1;
+    
+    if(reg_size > rcache->reg_max_mru_size) { 
+        return OMPI_ERR_TEMP_OUT_OF_RESOURCE; 
+    }
+    
+    rcache->reg_cur_mru_size += reg_size; 
+
     if(rcache->reg_mru_len <= rcache->mru_list.opal_list_length) {
         /* call deregister - which removes the registration from
          * the tree and mru list. memory will be deregistered when
@@ -55,7 +63,20 @@ int mca_rcache_rb_mru_insert(
         old_reg->mpool->mpool_deregister(old_reg->mpool, old_reg);
         
     }
+    
+    while(rcache->reg_max_mru_size <= rcache->reg_cur_mru_size) { 
+        old_reg = (mca_mpool_base_registration_t*)
+            opal_list_get_first(&rcache->mru_list);
+        /* we need to retain first, because we only want the registration 
+         removed from the tree and the mru */
+        old_reg->mpool->mpool_retain(old_reg->mpool, old_reg);
+        old_reg->mpool->mpool_deregister(old_reg->mpool, old_reg);
+        rcache->reg_cur_mru_size -= (old_reg->bound - old_reg->base + 1);
+                
+    }
+
     opal_list_append(&rcache->mru_list,(opal_list_item_t*) reg); 
+    
     return OMPI_SUCCESS; 
 }