* track the actual allocations within the ompi and opal free_lists, so that

we can free all allocated memory when the free_list is destructed.  Fixes
  a whole bunch of memory leaks...

This commit was SVN r7173.

ompi/class/ompi_free_list.c

@@ -31,6 +31,15 @@ opal_class_t ompi_free_list_t_class = {
     (opal_destruct_t)ompi_free_list_destruct
 };
 
+struct ompi_free_list_memory_t {
+    opal_list_item_t super;
+    mca_mpool_base_registration_t *registration;
+};
+typedef struct ompi_free_list_memory_t ompi_free_list_memory_t;
+static OBJ_CLASS_INSTANCE(ompi_free_list_memory_t,
+                          opal_list_item_t,
+                          NULL, NULL);
+
 
 static void ompi_free_list_construct(ompi_free_list_t* fl)
 {
@@ -43,10 +52,25 @@ static void ompi_free_list_construct(ompi_free_list_t* fl)
     fl->fl_elem_size = 0;
     fl->fl_elem_class = 0;
     fl->fl_mpool = 0;
+    OBJ_CONSTRUCT(&(fl->fl_allocations), opal_list_t);
 }
 
 static void ompi_free_list_destruct(ompi_free_list_t* fl)
 {
+    opal_list_item_t *item;
+
+    while (NULL != (item = opal_list_remove_first(&(fl->fl_allocations)))) {
+        /* destruct the item (we constructed it), then free the memory chunk */
+        OBJ_DESTRUCT(item);
+        if (NULL != fl->fl_mpool) {
+            ompi_free_list_memory_t *fl_mem = (ompi_free_list_memory_t*) item;
+            fl->fl_mpool->mpool_free(fl->fl_mpool, item, fl_mem->registration);
+        } else {
+            free(item);
+        }
+    }
+
+    OBJ_DESTRUCT(&fl->fl_allocations);
     OBJ_DESTRUCT(&fl->fl_condition);
     OBJ_DESTRUCT(&fl->fl_lock);
 }
@@ -75,6 +99,7 @@ int ompi_free_list_init(
 int ompi_free_list_grow(ompi_free_list_t* flist, size_t num_elements)
 {
     unsigned char* ptr;
+    ompi_free_list_memory_t *alloc_ptr;
     size_t i;
     size_t mod;
     mca_mpool_base_registration_t* user_out = NULL; 
@@ -83,12 +108,25 @@ int ompi_free_list_grow(ompi_free_list_t* flist, size_t num_elements)
         return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
 
     if (NULL != flist->fl_mpool)
-        ptr = (unsigned char*)flist->fl_mpool->mpool_alloc(flist->fl_mpool, (num_elements * flist->fl_elem_size) + CACHE_LINE_SIZE, 0, &user_out);
+        alloc_ptr = flist->fl_mpool->mpool_alloc(flist->fl_mpool, 
+                                                 (num_elements * flist->fl_elem_size) + CACHE_LINE_SIZE + sizeof(ompi_free_list_memory_t), 
+                                                 0, &user_out);
     else
-        ptr = (unsigned char *)malloc((num_elements * flist->fl_elem_size) + CACHE_LINE_SIZE);
+        alloc_ptr = malloc((num_elements * flist->fl_elem_size) + CACHE_LINE_SIZE + sizeof(ompi_free_list_memory_t));
-    if(NULL == ptr)
+    if(NULL == alloc_ptr)
         return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
 
+    /* make the alloc_ptr a list item, save the chunk in the allocations list, and
+       have ptr point to memory right after the list item structure */
+    OBJ_CONSTRUCT(alloc_ptr, ompi_free_list_memory_t);
+    opal_list_append(&(flist->fl_allocations), (opal_list_item_t*) alloc_ptr);
+    if (NULL != flist->fl_mpool) {
+        alloc_ptr->registration = user_out;
+    } else {
+        alloc_ptr->registration = NULL;
+    }
+    ptr = (unsigned char*) alloc_ptr + sizeof(ompi_free_list_memory_t);
+
     mod = (unsigned long)ptr % CACHE_LINE_SIZE;
     if(mod != 0) {
         ptr += (CACHE_LINE_SIZE - mod);

ompi/class/ompi_free_list.h

@@ -43,7 +43,7 @@ struct ompi_free_list_t
     mca_mpool_base_module_t* fl_mpool;
     opal_mutex_t fl_lock;
     opal_condition_t fl_condition; 
-    
+    opal_list_t fl_allocations;
 };
 typedef struct ompi_free_list_t ompi_free_list_t;
 

opal/class/opal_free_list.c

@@ -42,10 +42,20 @@ static void opal_free_list_construct(opal_free_list_t* fl)
     fl->fl_num_waiting = 0;
     fl->fl_elem_size = 0;
     fl->fl_elem_class = 0;
+    OBJ_CONSTRUCT(&(fl->fl_allocations), opal_list_t);
 }
 
 static void opal_free_list_destruct(opal_free_list_t* fl)
 {
+    opal_list_item_t *item;
+
+    while (NULL != (item = opal_list_remove_first(&(fl->fl_allocations)))) {
+        /* destruct the item (we constructed it), then free the memory chunk */
+        OBJ_DESTRUCT(item);
+        free(item);
+    }
+
+    OBJ_DESTRUCT(&fl->fl_allocations);
     OBJ_DESTRUCT(&fl->fl_condition);
     OBJ_DESTRUCT(&fl->fl_lock);
 }
@@ -72,16 +82,25 @@ int opal_free_list_init(
 int opal_free_list_grow(opal_free_list_t* flist, size_t num_elements)
 {
     unsigned char* ptr;
+    unsigned char* alloc_ptr;
     size_t i;
     size_t mod;
 
     if (flist->fl_max_to_alloc > 0 && flist->fl_num_allocated + num_elements > flist->fl_max_to_alloc)
         return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
 
-    ptr = (unsigned char *)malloc((num_elements * flist->fl_elem_size) + CACHE_LINE_SIZE);
+    alloc_ptr = (unsigned char *)malloc((num_elements * flist->fl_elem_size) + 
-    if(NULL == ptr)
+                                        sizeof(opal_list_item_t) +
+                                        CACHE_LINE_SIZE);
+    if(NULL == alloc_ptr)
         return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
 
+    /* make the alloc_ptr a list item, save the chunk in the allocations list, and
+       have ptr point to memory right after the list item structure */
+    OBJ_CONSTRUCT(alloc_ptr, opal_list_item_t);
+    opal_list_append(&(flist->fl_allocations), (opal_list_item_t*) alloc_ptr);
+    ptr = alloc_ptr + sizeof(opal_list_item_t);
+
     mod = (unsigned long)ptr % CACHE_LINE_SIZE;
     if(mod != 0) {
         ptr += (CACHE_LINE_SIZE - mod);

opal/class/opal_free_list.h

@@ -41,6 +41,7 @@ struct opal_free_list_t
     opal_class_t* fl_elem_class;
     opal_mutex_t fl_lock;
     opal_condition_t fl_condition; 
+    opal_list_t fl_allocations;
 };
 typedef struct opal_free_list_t opal_free_list_t;