potential fix for ticket #81

Added a tree to track memory allocation from MPI_Alloc_mem, this allows us to
free the registrations in a sane fashion.. also should be faster.. 

This commit was SVN r10248.

ompi/mca/mpool/base/Makefile.am

@@ -18,12 +18,13 @@
 
 headers += \
         base/base.h \
-	base/mpool_base_mem_cb.h
-
+	base/mpool_base_mem_cb.h \
+	base/mpool_base_tree.h
 libmca_mpool_la_SOURCES += \
         base/mpool_base_open.c \
         base/mpool_base_close.c \
         base/mpool_base_init.c \
         base/mpool_base_lookup.c \
         base/mpool_base_alloc.c \
-        base/mpool_base_mem_cb.c
+        base/mpool_base_mem_cb.c \
+	base/mpool_base_tree.c

ompi/mca/mpool/base/mpool_base_alloc.c

@@ -24,7 +24,8 @@
 #include <string.h>
 #endif  /* HAVE_STRING_H */
 #include "ompi/mca/mpool/mpool.h"
-#include "ompi/mca/mpool/base/base.h"
+#include "base.h"
+#include "mpool_base_tree.h"
 #include "opal/threads/mutex.h" 
 
 /**
@@ -75,20 +76,26 @@ void * mca_mpool_base_alloc(size_t size, ompi_info_t * info)
     opal_list_item_t * item;
     int num_modules = opal_list_get_size(&mca_mpool_base_modules);
     int reg_module_num = 0;
-    int i, num_keys;
+    int i, j, num_keys;
     mca_mpool_base_selected_module_t * current;
     mca_mpool_base_selected_module_t * no_reg_function = NULL;
     mca_mpool_base_selected_module_t ** has_reg_function = NULL;
     mca_mpool_base_registration_t * registration;
+    mca_mpool_base_tree_item_t* mpool_tree_item;
+    
     void * mem = NULL;
     char * key;
     bool match_found;
-
+    
     if (num_modules > 0) {
         has_reg_function = (mca_mpool_base_selected_module_t **)
                            malloc(num_modules * sizeof(mca_mpool_base_module_t *));
     }
 
+    mpool_tree_item = mca_mpool_base_tree_item_get();
+    if(NULL == mpool_tree_item){ 
+        return NULL;
+    }
     if(&ompi_mpi_info_null == info)
     {
         for(item = opal_list_get_first(&mca_mpool_base_modules);
@@ -178,13 +185,16 @@ void * mca_mpool_base_alloc(size_t size, ompi_info_t * info)
     }
     
     
-    i = 0;
+    i = j = 0;
     num_modules = 0;
     if(NULL != no_reg_function)
     {
         mca_mpool_base_module_t* mpool = no_reg_function->mpool_module;
         mem = mpool->mpool_alloc(mpool, size, 0, MCA_MPOOL_FLAGS_PERSIST, &registration);
         num_modules++;
+        mpool_tree_item->key = mem;
+        mpool_tree_item->mpools[j] = mpool;
+        mpool_tree_item->regs[j++] = registration;
     }
     else
     {
@@ -192,6 +202,9 @@ void * mca_mpool_base_alloc(size_t size, ompi_info_t * info)
         mem = mpool->mpool_alloc(mpool, size, 0, MCA_MPOOL_FLAGS_PERSIST, &registration);
         i++;
         num_modules++;
+        mpool_tree_item->key = mem;
+        mpool_tree_item->mpools[j] = mpool;
+        mpool_tree_item->regs[j++] = registration;
     }
     
     while(i < reg_module_num)
@@ -203,14 +216,25 @@ void * mca_mpool_base_alloc(size_t size, ompi_info_t * info)
                 free(has_reg_function);
             }
             return NULL;
+        } else { 
+            mpool_tree_item->key = mem;
+            mpool_tree_item->mpools[j] = mpool;
+            mpool_tree_item->regs[j++] = registration;
+            num_modules++;
         }
-        num_modules++;
         i++;
     }
 
     if (NULL != has_reg_function) {
         free(has_reg_function);
     }
+    
+    /* null terminated array */
+    mpool_tree_item->mpools[j] = NULL;
+    mpool_tree_item->regs[j] = NULL;
+
+    mca_mpool_base_tree_insert(mpool_tree_item);
+    
     return mem;
 }
 
@@ -224,71 +248,31 @@ void * mca_mpool_base_alloc(size_t size, ompi_info_t * info)
  */
 int mca_mpool_base_free(void * base)
 {
-    int rc = OMPI_SUCCESS;
-    opal_list_item_t * item;
-    mca_mpool_base_selected_module_t * current = NULL;
-    mca_mpool_base_selected_module_t * free_function = NULL;
-    uint32_t i, cnt;
-    ompi_pointer_array_t regs;
-    OBJ_CONSTRUCT(&regs, ompi_pointer_array_t);
-                                                                                                       
-    for(item = opal_list_get_first(&mca_mpool_base_modules);
-        item != opal_list_get_end(&mca_mpool_base_modules);
-        item = opal_list_get_next(item)) {
-        current = ((mca_mpool_base_selected_module_t *) item);
-        /* 
-         * Check if a mpool has been used for allocating the memory. This 
-         * approach only works for the case that the user specified MPI_INFO_NULL
-         * in MPI_Alloc_mem.
-         * Maybe all possible mpools should be asked if they can free the 
-         * memory until the right returns OK ?
-         */
-        if(current->mpool_module->flags & MCA_MPOOL_FLAGS_MPI_ALLOC_MEM) {
-            if(NULL == current->mpool_module->mpool_register){
-                free_function = current;
-            } else {
-                if ( NULL == free_function ) {
-                    free_function = current;
-                }
-            }
-        }
-        if(NULL != current->mpool_module->mpool_find)  {
-            rc = current->mpool_module->mpool_find(
-                current->mpool_module,
-                base,
-                1,
-                &regs,
-                &cnt
-                );
-            if(OMPI_SUCCESS != rc) {
-                continue;
-            }
-            for(i = 0; i < cnt; i++) {
-                mca_mpool_base_registration_t* reg = (mca_mpool_base_registration_t*)
-                    ompi_pointer_array_get_item(&regs, i);
-                                                                                                       
-                rc = current->mpool_module->mpool_deregister(current->mpool_module, reg);
-                if(OMPI_SUCCESS != rc) {
-                    goto cleanup;
-                }
-            }
-            ompi_pointer_array_remove_all(&regs);
+    int i = 0, rc = OMPI_SUCCESS;
+    mca_mpool_base_tree_item_t* mpool_tree_item = 
+        mca_mpool_base_tree_find(base); 
+    mca_mpool_base_module_t* mpool;
+    mca_mpool_base_registration_t* reg;
+    
+    if(!mpool_tree_item) { 
+        return OMPI_ERROR;
+    }
+    mpool = mpool_tree_item->mpools[0];
+    reg =  mpool_tree_item->regs[0];
+    mpool->mpool_free(mpool, base, reg);
+    
+    for(i = 1; i < MCA_MPOOL_BASE_TREE_MAX; i++) {
+        mpool = mpool_tree_item->mpools[i];
+        reg = mpool_tree_item->regs[i];
+        if(mpool) { 
+            mpool->mpool_deregister(mpool, reg); 
+        } else { 
+            break;
         }
     }
-
-    /* free the memory */
-    if ( NULL == free_function ) {
-        /* If there is no mpool the memory has been allocated with malloc */
-        free(base);
-    } else {
-        /* free the memory with the mpool that was responsible for the allocation.
-         * The registration is NULL because the buffer has been unregistered above.
-         */
-        free_function->mpool_module->mpool_free(free_function->mpool_module, base, NULL);
-    }
-
-cleanup:
-    OBJ_DESTRUCT(&regs);
+    
+    rc = mca_mpool_base_tree_delete(mpool_tree_item);
+    
     return rc;
 }
 

ompi/mca/mpool/base/mpool_base_open.c

@@ -36,6 +36,7 @@
  */
 #include "ompi/mca/mpool/base/static-components.h"
 
+#include "mpool_base_tree.h"
 /*
  * Global variables
  */
@@ -145,6 +146,10 @@ int mca_mpool_base_open(void)
     /* get the page size for this architecture*/ 
     mca_mpool_base_page_size = sysconf(_SC_PAGESIZE); 
     mca_mpool_base_page_size_log = my_log2(mca_mpool_base_page_size); 
+
+    /* setup tree for tracking MPI_Alloc_mem */ 
+    mca_mpool_base_tree_init();
+    
     return OMPI_SUCCESS;
 }
 

ompi/mca/mpool/base/mpool_base_tree.c

@@ -0,0 +1,110 @@
+/**
+  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+  *                         University Research and Technology
+  *                         Corporation.  All rights reserved.
+  * Copyright (c) 2004-2005 The University of Tennessee and The University
+  *                         of Tennessee Research Foundation.  All rights
+  *                         reserved.
+  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+  *                         University of Stuttgart.  All rights reserved.
+  * Copyright (c) 2004-2005 The Regents of the University of California.
+  *                         All rights reserved.5A
+  * $COPYRIGHT$
+  * 
+  * Additional copyrights may follow
+  * 
+  * $HEADER$
+  */
+/**
+  * @file
+  * Description of the Registration Cache framework
+  */
+
+#include "opal/mca/mca.h"
+#include "mpool_base_tree.h"
+
+
+OBJ_CLASS_INSTANCE(mca_mpool_base_tree_item_t, opal_list_item_t, NULL, NULL); 
+
+/* 
+ * use globals for the tree and the tree_item free list.. 
+ */
+ompi_rb_tree_t mca_mpool_base_tree; 
+ompi_free_list_t mca_mpool_base_tree_item_free_list;
+
+/*
+ *  simple minded compare function... 
+ */
+int mca_mpool_base_tree_node_compare(void * key1, void * key2)
+{
+    if(key1 < key2)
+    {
+        return -1;
+    }
+    else if(key1 > key2)
+    {
+        return 1;
+    }
+    else
+    {
+        return 0;
+    }
+}
+
+/*
+ * initialize the rb tree
+ */
+int mca_mpool_base_tree_init(void) { 
+    int rc;
+    OBJ_CONSTRUCT(&mca_mpool_base_tree, ompi_rb_tree_t); 
+    OBJ_CONSTRUCT(&mca_mpool_base_tree_item_free_list, ompi_free_list_t); 
+    rc = ompi_free_list_init(&mca_mpool_base_tree_item_free_list, sizeof(mca_mpool_base_tree_item_t), 
+                             OBJ_CLASS(mca_mpool_base_tree_item_t), 0, -1 , 4, NULL);
+    if(OMPI_SUCCESS == rc) { 
+        rc = ompi_rb_tree_init(&mca_mpool_base_tree, mca_mpool_base_tree_node_compare);
+    }
+    return rc;
+}
+
+/* 
+ * insert an item in the rb tree 
+ */ 
+int mca_mpool_base_tree_insert(mca_mpool_base_tree_item_t* item) { 
+    return ompi_rb_tree_insert(&mca_mpool_base_tree, item->key, item); 
+}
+
+/* 
+ * remove an item from the rb tree 
+ */
+int mca_mpool_base_tree_delete(mca_mpool_base_tree_item_t* item) { 
+    int rc; 
+    rc = ompi_rb_tree_delete(&mca_mpool_base_tree, item->key); 
+    if(OMPI_SUCCESS == rc) { 
+        OMPI_FREE_LIST_RETURN(&mca_mpool_base_tree_item_free_list, 
+                              item->key); 
+    }
+    return rc;
+}
+
+/**
+ *  find the item in the rb tree  
+ */
+mca_mpool_base_tree_item_t* mca_mpool_base_tree_find(void* base) { 
+    return(ompi_rb_tree_find(&mca_mpool_base_tree, base));
+}
+    
+/* 
+ * get a tree item from the free list 
+ */
+mca_mpool_base_tree_item_t* mca_mpool_base_tree_item_get(void) { 
+    opal_list_item_t* item = NULL;
+    int rc;
+    OMPI_FREE_LIST_GET(&mca_mpool_base_tree_item_free_list, 
+                       item, 
+                       rc); 
+    if(OMPI_SUCCESS == rc) { 
+        return (mca_mpool_base_tree_item_t*) item; 
+    } else { 
+        return NULL;
+    }
+}

ompi/mca/mpool/base/mpool_base_tree.h

@@ -0,0 +1,74 @@
+/**
+  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+  *                         University Research and Technology
+  *                         Corporation.  All rights reserved.
+  * Copyright (c) 2004-2005 The University of Tennessee and The University
+  *                         of Tennessee Research Foundation.  All rights
+  *                         reserved.
+  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+  *                         University of Stuttgart.  All rights reserved.
+  * Copyright (c) 2004-2006 The Regents of the University of California.
+  *                         All rights reserved.
+  * $COPYRIGHT$
+  * 
+  * Additional copyrights may follow
+  * 
+  * $HEADER$
+  */
+
+#ifndef MCA_MPOOL_BASE_TREE_H
+#define MCA_MPOOL_BASE_TREE_H
+#define MCA_MPOOL_BASE_TREE_MAX 8
+#include "opal/mca/mca.h"
+#include "ompi/info/info.h"
+#include "opal/class/opal_list.h" 
+#include "ompi/class/ompi_rb_tree.h"
+#include "ompi/mca/mpool/mpool.h"
+
+/*
+ * Data structures for the tree of allocated memory
+ * used for MPI_Alloc_mem and MPI_Free_mem 
+ */
+
+/**
+ * The item in the tree itself
+ */
+struct mca_mpool_base_tree_item_t
+{
+    opal_list_item_t super;   /**< the parent class */
+    void* key; /* the address this was alloc'd on */
+    mca_mpool_base_module_t* mpools[MCA_MPOOL_BASE_TREE_MAX]; /**< the mpools */
+    mca_mpool_base_registration_t* regs[MCA_MPOOL_BASE_TREE_MAX]; /**< the registrations */  
+};
+typedef struct mca_mpool_base_tree_item_t mca_mpool_base_tree_item_t;
+
+OBJ_CLASS_DECLARATION(mca_mpool_base_tree_item_t);
+
+
+/*
+ * initialize the rb tree
+ */
+int mca_mpool_base_tree_init(void); 
+
+/* 
+ * insert an item in the rb tree 
+ */ 
+int mca_mpool_base_tree_insert(mca_mpool_base_tree_item_t* item); 
+
+/* 
+ * remove an item from the rb tree 
+ */
+int mca_mpool_base_tree_delete(mca_mpool_base_tree_item_t* item); 
+
+
+/**
+ *  find the item in the rb tree  
+ */
+mca_mpool_base_tree_item_t* mca_mpool_base_tree_find(void* base);
+
+/* 
+ * get a tree item from the free list 
+ */
+mca_mpool_base_tree_item_t* mca_mpool_base_tree_item_get(void); 
+
+#endif /* MCA_MPOOL_BASE_TREE_H */

ompi/mca/mpool/openib/mpool_openib_module.c

@@ -180,10 +180,9 @@ void* mca_mpool_openib_realloc(
 void mca_mpool_openib_free(mca_mpool_base_module_t* mpool, void * addr,
                          mca_mpool_base_registration_t* registration)
 {
-    if(registration){
-        mpool->mpool_deregister(mpool, registration); 
-    }
-   	free(addr); 
+    
+    mpool->mpool_deregister(mpool, registration); 
+    free(registration->alloc_base); 
 }