Compromise:

- move mpool and allocator frameworks back to ompi (from opal)
- specialize the ompi_free_list class to use an mpool instance
- un-specialize opal_free_list to *not* use mpool; just use malloc/free

This commit was SVN r6292.

ompi/class/Makefile.am

@@ -24,12 +24,14 @@ noinst_LTLIBRARIES = libclass.la
 headers = \
         ompi_circular_buffer_fifo.h \
         ompi_fifo.h \
+        ompi_free_list.h \
         ompi_bitmap.h \
         ompi_pointer_array.h
 
 libclass_la_SOURCES = \
 	$(headers) \
         ompi_bitmap.c \
+        ompi_free_list.c \
         ompi_pointer_array.c
 
 # Conditionally install the header files

ompi/mca/Makefile.am

@@ -19,10 +19,12 @@ include $(top_srcdir)/config/Makefile.options
 EXTRA_DIST = win_makefile
 
 SUBDIRS = \
+        allocator \
         btl \
         coll \
         common \
         io \
+        mpool \
         pml \
         ptl \
         topo

opal/class/Makefile.am

@@ -22,7 +22,7 @@ noinst_LTLIBRARIES = libclass.la
 # Source code files
 
 headers = \
-        ompi_free_list.h \
+        opal_free_list.h \
         ompi_hash_table.h \
         ompi_list.h \
         ompi_object.h \
@@ -31,7 +31,7 @@ headers = \
 
 libclass_la_SOURCES = \
 	$(headers) \
-        ompi_free_list.c \
+        opal_free_list.c \
         ompi_hash_table.c \
         ompi_list.c \
         ompi_object.c \

opal/class/opal_free_list.c

@@ -0,0 +1,109 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "class/ompi_free_list.h"
+#include "include/sys/cache.h"
+
+
+static void ompi_free_list_construct(ompi_free_list_t* fl);
+static void ompi_free_list_destruct(ompi_free_list_t* fl);
+
+
+ompi_class_t ompi_free_list_t_class = {
+    "ompi_free_list_t", 
+    OBJ_CLASS(ompi_list_t),
+    (ompi_construct_t)ompi_free_list_construct, 
+    (ompi_destruct_t)ompi_free_list_destruct
+};
+
+
+static void ompi_free_list_construct(ompi_free_list_t* fl)
+{
+    OBJ_CONSTRUCT(&fl->fl_lock, ompi_mutex_t);
+    fl->fl_max_to_alloc = 0;
+    fl->fl_num_allocated = 0;
+    fl->fl_num_per_alloc = 0;
+    fl->fl_num_waiting = 0;
+    fl->fl_elem_size = 0;
+    fl->fl_elem_class = 0;
+    fl->fl_mpool = 0;
+}
+
+static void ompi_free_list_destruct(ompi_free_list_t* fl)
+{
+    OBJ_DESTRUCT(&fl->fl_lock);
+}
+
+int ompi_free_list_init(
+    ompi_free_list_t *flist,
+    size_t elem_size,
+    ompi_class_t* elem_class,
+    int num_elements_to_alloc,
+    int max_elements_to_alloc,
+    int num_elements_per_alloc,
+    mca_mpool_base_module_t* mpool)
+{
+    flist->fl_elem_size = elem_size;
+    flist->fl_elem_class = elem_class;
+    flist->fl_max_to_alloc = max_elements_to_alloc;
+    flist->fl_num_allocated = 0;
+    flist->fl_num_per_alloc = num_elements_per_alloc;
+    flist->fl_mpool = mpool;
+    if(num_elements_to_alloc)
+        return ompi_free_list_grow(flist, num_elements_to_alloc);
+    return OMPI_SUCCESS;
+}
+
+
+int ompi_free_list_grow(ompi_free_list_t* flist, size_t num_elements)
+{
+    unsigned char* ptr;
+    size_t i;
+    size_t mod;
+    mca_mpool_base_registration_t* user_out; 
+
+    if (flist->fl_max_to_alloc > 0 && flist->fl_num_allocated + num_elements > flist->fl_max_to_alloc)
+        return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
+
+    if (NULL != flist->fl_mpool)
+        ptr = (unsigned char*)flist->fl_mpool->mpool_alloc(flist->fl_mpool, (num_elements * flist->fl_elem_size) + CACHE_LINE_SIZE, 0, &user_out);
+    else
+        ptr = (unsigned char *)malloc((num_elements * flist->fl_elem_size) + CACHE_LINE_SIZE);
+    if(NULL == ptr)
+        return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
+
+    mod = (unsigned long)ptr % CACHE_LINE_SIZE;
+    if(mod != 0) {
+        ptr += (CACHE_LINE_SIZE - mod);
+    }
+
+    for(i=0; i<num_elements; i++) {
+        ompi_free_list_item_t* item = (ompi_free_list_item_t*)ptr;
+        item->user_data = user_out; 
+        if (NULL != flist->fl_elem_class) {
+            OBJ_CONSTRUCT_INTERNAL(item, flist->fl_elem_class);
+        }
+        ompi_list_append(&(flist->super), &(item->super));
+        ptr += flist->fl_elem_size;
+    }
+    flist->fl_num_allocated += num_elements;
+    return OMPI_SUCCESS;
+}
+
+
+

opal/class/opal_free_list.h

@@ -0,0 +1,167 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#ifndef OMPI_FREE_LIST_H
+#define OMPI_FREE_LIST_H
+
+#include "ompi_config.h"
+#include "class/ompi_list.h"
+#include "threads/thread.h"
+#include "threads/condition.h"
+#include "include/constants.h"
+#include "mca/mpool/mpool.h"
+
+#if defined(c_plusplus) || defined(__cplusplus)
+extern "C" {
+#endif
+OMPI_DECLSPEC extern ompi_class_t ompi_free_list_t_class;
+struct mca_mem_pool_t;
+
+
+struct ompi_free_list_t
+{
+    ompi_list_t super;
+    size_t fl_max_to_alloc;
+    size_t fl_num_allocated;
+    size_t fl_num_per_alloc;
+    size_t fl_num_waiting;
+    size_t fl_elem_size;
+    ompi_class_t* fl_elem_class;
+    mca_mpool_base_module_t* fl_mpool;
+    ompi_mutex_t fl_lock;
+    ompi_condition_t fl_condition; 
+    
+};
+typedef struct ompi_free_list_t ompi_free_list_t;
+
+struct ompi_free_list_item_t
+{ 
+    ompi_list_item_t super; 
+    void* user_data; 
+}; 
+typedef struct ompi_free_list_item_t ompi_free_list_item_t; 
+
+/**
+ * Initialize a free list.
+ *
+ * @param free_list (IN)           Free list.
+ * @param element_size (IN)        Size of each element.
+ * @param element_class (IN)       ompi_class_t of element - used to initialize allocated elements.
+ * @param num_elements_to_alloc    Initial number of elements to allocate.
+ * @param max_elements_to_alloc    Maximum number of elements to allocate.
+ * @param num_elements_per_alloc   Number of elements to grow by per allocation.
+ * @param mpool                    Optional memory pool for allocation.s
+ */
+ 
+OMPI_DECLSPEC int ompi_free_list_init(
+    ompi_free_list_t *free_list, 
+    size_t element_size,
+    ompi_class_t* element_class,
+    int num_elements_to_alloc,
+    int max_elements_to_alloc,
+    int num_elements_per_alloc,
+    mca_mpool_base_module_t*);
+
+OMPI_DECLSPEC int ompi_free_list_grow(ompi_free_list_t* flist, size_t num_elements);
+    
+/**
+ * Attemp to obtain an item from a free list. 
+ *
+ * @param fl (IN)        Free list.
+ * @param item (OUT)     Allocated item.
+ * @param rc (OUT)       OMPI_SUCCESS or error status on failure.
+ *
+ * If the requested item is not available the free list is grown to 
+ * accomodate the request - unless the max number of allocations has 
+ * been reached.  If this is the case - an out of resource error is 
+ * returned to the caller.
+ */
+ 
+#define OMPI_FREE_LIST_GET(fl, item, rc) \
+{ \
+    if(ompi_using_threads()) { \
+        ompi_mutex_lock(&((fl)->fl_lock)); \
+        item = ompi_list_remove_first(&((fl)->super)); \
+        if(NULL == item) { \
+            ompi_free_list_grow((fl), (fl)->fl_num_per_alloc); \
+            item = ompi_list_remove_first(&((fl)->super)); \
+        } \
+        ompi_mutex_unlock(&((fl)->fl_lock)); \
+    } else { \
+        item = ompi_list_remove_first(&((fl)->super)); \
+        if(NULL == item) { \
+            ompi_free_list_grow((fl), (fl)->fl_num_per_alloc); \
+            item = ompi_list_remove_first(&((fl)->super)); \
+        } \
+    }  \
+    rc = (NULL == item) ?  OMPI_ERR_TEMP_OUT_OF_RESOURCE : OMPI_SUCCESS; \
+} 
+
+/**
+ * Blocking call to obtain an item from a free list.
+ *
+ * @param fl (IN)        Free list.
+ * @param item (OUT)     Allocated item.
+ * @param rc (OUT)       OMPI_SUCCESS or error status on failure.
+ *
+ * If the requested item is not available the free list is grown to 
+ * accomodate the request - unless the max number of allocations has 
+ * been reached. In this case the caller is blocked until an item
+ * is returned to the list.
+ */
+ 
+
+#define OMPI_FREE_LIST_WAIT(fl, item, rc)                                  \
+{                                                                          \
+    OMPI_THREAD_LOCK(&((fl)->fl_lock));                                    \
+    item = ompi_list_remove_first(&((fl)->super));                         \
+    while(NULL == item) {                                                  \
+        if((fl)->fl_max_to_alloc <= (fl)->fl_num_allocated) {              \
+            (fl)->fl_num_waiting++;                                        \
+            ompi_condition_wait(&((fl)->fl_condition), &((fl)->fl_lock));  \
+            (fl)->fl_num_waiting--;                                        \
+        } else {                                                           \
+            ompi_free_list_grow((fl), (fl)->fl_num_per_alloc);             \
+        }                                                                  \
+        item = ompi_list_remove_first(&((fl)->super));                     \
+    }                                                                      \
+    OMPI_THREAD_UNLOCK(&((fl)->fl_lock));                                  \
+    rc = (NULL == item) ?  OMPI_ERR_OUT_OF_RESOURCE : OMPI_SUCCESS;        \
+} 
+
+
+/**
+ * Return an item to a free list. 
+ *
+ * @param fl (IN)        Free list.
+ * @param item (OUT)     Allocated item.
+ *
+ */
+ 
+#define OMPI_FREE_LIST_RETURN(fl, item)                                    \
+{                                                                          \
+    OMPI_THREAD_LOCK(&(fl)->fl_lock);                                      \
+    ompi_list_prepend(&((fl)->super), (item));                            \
+    if((fl)->fl_num_waiting > 0) {                                         \
+        ompi_condition_signal(&((fl)->fl_condition));                      \
+    }                                                                      \
+    OMPI_THREAD_UNLOCK(&(fl)->fl_lock);                                    \
+}
+#if defined(c_plusplus) || defined(__cplusplus)
+}
+#endif
+#endif 
+

opal/mca/Makefile.am

@@ -16,10 +16,6 @@
 
 include $(top_srcdir)/config/Makefile.options
 
-SUBDIRS = \
-        allocator \
-	mpool
-
 # Source code files
 
 headers = mca.h