maintain mru list of registrations for leave pinned option

This commit was SVN r6505.

ompi/mca/btl/gm/btl_gm.c

@@ -235,6 +235,7 @@ mca_btl_base_descriptor_t* mca_btl_gm_prepare_src(
     */
     if (NULL != registration && 0 == ompi_convertor_need_buffers(convertor)) {
 
+        bool is_leave_pinned = registration->is_leave_pinned;
         size_t reg_len;
         MCA_BTL_GM_FRAG_ALLOC_USER(gm_btl, frag, rc);
         if(NULL == frag){
@@ -257,6 +258,11 @@ mca_btl_base_descriptor_t* mca_btl_gm_prepare_src(
 
             /* remove old registration from tree and decrement reference count */
             mca_mpool_base_remove(base_addr);
+            if(is_leave_pinned) {
+                OPAL_THREAD_LOCK(&gm_btl->gm_lock);
+                opal_list_remove_item(&gm_btl->gm_mru_reg, (opal_list_item_t*)registration);
+                OPAL_THREAD_UNLOCK(&gm_btl->gm_lock);
+            }
             OBJ_RELEASE(registration);
 
             /* re-register at new size */
@@ -282,6 +288,17 @@ mca_btl_base_descriptor_t* mca_btl_gm_prepare_src(
                 OBJ_RELEASE(registration);
                 return NULL;
             }
+            if(is_leave_pinned) {
+                OPAL_THREAD_LOCK(&gm_btl->gm_lock);
+                registration->is_leave_pinned = true;
+                opal_list_append(&gm_btl->gm_mru_reg, (opal_list_item_t*)registration);
+                OPAL_THREAD_UNLOCK(&gm_btl->gm_lock);
+            }
+        }  else if (is_leave_pinned) {
+            OPAL_THREAD_LOCK(&gm_btl->gm_lock);
+            opal_list_remove_item(&gm_btl->gm_mru_reg, (opal_list_item_t*)registration);
+            opal_list_append(&gm_btl->gm_mru_reg, (opal_list_item_t*)registration);
+            OPAL_THREAD_UNLOCK(&gm_btl->gm_lock);
         }
 
         /* bump reference count as so that the registration
@@ -327,6 +344,8 @@ mca_btl_base_descriptor_t* mca_btl_gm_prepare_src(
              * insert the registration into the tree and bump the reference
              * count so that it doesn't go away on completion.
             */
+            OBJ_RETAIN(registration);
+            registration->is_leave_pinned = true;
             rc = mca_mpool_base_insert(
                 iov.iov_base,
                 iov.iov_len,
@@ -338,7 +357,22 @@ mca_btl_base_descriptor_t* mca_btl_gm_prepare_src(
                 OBJ_RELEASE(registration);
                 return NULL;
             }
-            OBJ_RETAIN(registration);
+
+            /*
+             * remove registrations that exceed the mru count
+             */
+            OPAL_THREAD_LOCK(&gm_btl->gm_lock);
+            while(opal_list_get_size(&gm_btl->gm_mru_reg) >= mca_btl_gm_component.gm_num_mru) {
+                mca_mpool_base_registration_t* lru = (mca_mpool_base_registration_t*)
+                    opal_list_remove_first(&gm_btl->gm_mru_reg);
+                int rc = mca_mpool_base_remove(lru->base);
+                if(OMPI_SUCCESS != rc) {
+                    opal_output(0, "[%s:%d] unable to remove registration\n", __FILE__,__LINE__);
+                }
+                OBJ_RELEASE(lru);
+            }
+            opal_list_append(&gm_btl->gm_mru_reg, (opal_list_item_t*)registration);
+            OPAL_THREAD_UNLOCK(&gm_btl->gm_lock);
         } 
         frag->registration = registration;
     } 

ompi/mca/btl/gm/btl_gm.h

@@ -58,6 +58,7 @@ struct mca_btl_gm_component_t {
     size_t gm_max_boards; /**< maximum number of boards */
     size_t gm_num_high_priority; /**< number of receive descriptors at high priority */
     size_t gm_num_repost;
+    size_t gm_num_mru;
     size_t gm_eager_frag_size;
     size_t gm_max_frag_size;
     char*  gm_port_name; 
@@ -103,6 +104,7 @@ struct mca_btl_gm_module_t {
     /* lock for accessing module state */
     opal_list_t gm_pending; /**< list of pending send descriptors */
     opal_list_t gm_repost; /**< list of pending fragments */
+    opal_list_t gm_mru_reg; /**< list of most recently used registrations */
     opal_mutex_t gm_lock;
     struct mca_mpool_base_module_t* gm_mpool;
 }; 

ompi/mca/btl/gm/btl_gm_component.c

@@ -132,6 +132,8 @@ int mca_btl_gm_component_open(void)
         mca_btl_gm_param_register_int("num_high_priority", 8); 
     mca_btl_gm_component.gm_num_repost = 
         mca_btl_gm_param_register_int("num_repost", 4); 
+    mca_btl_gm_component.gm_num_mru = 
+        mca_btl_gm_param_register_int("num_mru", 64); 
     mca_btl_gm_component.gm_port_name=
         mca_btl_gm_param_register_string("port_name", "OMPI"); 
 
@@ -206,6 +208,7 @@ mca_btl_gm_module_init (mca_btl_gm_module_t * btl)
     OBJ_CONSTRUCT(&btl->gm_frag_user, ompi_free_list_t);
     OBJ_CONSTRUCT(&btl->gm_pending, opal_list_t);
     OBJ_CONSTRUCT(&btl->gm_repost, opal_list_t);
+    OBJ_CONSTRUCT(&btl->gm_mru_reg, opal_list_t);
     OBJ_CONSTRUCT(&btl->gm_lock, opal_mutex_t);
                                                                                                   
     /* query nic tokens */