ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

Combine opal_free_list_t and ompi_free_list_t

Просмотр исходного кода 
Historically these two lists were different due to ompi_free_list_t
dependencies in ompi (mpool). Those dependencies have since been moved
to opal so it is safe to (finally) combine them. The combined free
list comes in three flavors:

 - Single-threaded. Only to be used when it is guaranteed that no
   concurrent access will be made to the free list. Single-threaded
   functions are suffixed with _st.

 - Mutli-threaded. To be used when the free list may be accessed by
   multiple threads despite the setting of opal_using_threads.
   Multi-threaded functins are suffixed with _mt.

 - Conditionally multi-threaded. Common use case. These functions are
   thread-safe if opal_using_threads is set to true.

Compatibility functions for the ompi_free_list_t and the old accessor
functions (OPAL_FREE_LIST_*) are available while the code base is
transitioned to the new class/functions.

Signed-off-by: Nathan Hjelm <hjelmn@lanl.gov>
Этот коммит содержится в:
Nathan Hjelm 2015-01-14 13:53:30 -07:00
родитель b70fa3e2cb
Коммит 88251a6b94
5 изменённых файлов: 581 добавлений и 684 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

1
opal/class/Makefile.am
Просмотреть файл

@ -57,4 +57,3 @@ lib@OPAL_LIB_PREFIX@open_pal_la_SOURCES += \
class/opal_ring_buffer.c \
class/opal_rb_tree.c \
class/ompi_free_list.c

292
opal/class/ompi_free_list.c
Просмотреть файл

@ -1,19 +1,6 @@
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2009 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006-2007 Mellanox Technologies. All rights reserved.
* Copyright (c) 2010-2013 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2011 NVIDIA Corporation. All rights reserved.
* Copyright (c) 2012-2014 Los Alamos National Security, LLC. All rights
* Copyright (c) 2014 Los Alamos National Security, LLC. All rights
* reserved.
* $COPYRIGHT$
*
@ -22,281 +9,6 @@
* $HEADER$
*/
#include "opal_config.h"
#include "opal/class/ompi_free_list.h"
#include "opal/align.h"
#include "opal/util/output.h"
#include "opal/mca/mpool/mpool.h"
static void ompi_free_list_construct(ompi_free_list_t* fl);
static void ompi_free_list_destruct(ompi_free_list_t* fl);
OBJ_CLASS_INSTANCE(ompi_free_list_t, opal_lifo_t,
ompi_free_list_construct, ompi_free_list_destruct);
typedef struct ompi_free_list_item_t ompi_free_list_memory_t;
OBJ_CLASS_INSTANCE(ompi_free_list_item_t,
opal_list_item_t,
NULL, NULL);
static void ompi_free_list_construct(ompi_free_list_t* fl)
{
OBJ_CONSTRUCT(&fl->fl_lock, opal_mutex_t);
OBJ_CONSTRUCT(&fl->fl_condition, opal_condition_t);
fl->fl_max_to_alloc = 0;
fl->fl_num_allocated = 0;
fl->fl_num_per_alloc = 0;
fl->fl_num_waiting = 0;
fl->fl_frag_size = sizeof(ompi_free_list_item_t);
fl->fl_frag_alignment = 0;
fl->fl_payload_buffer_size=0;
fl->fl_payload_buffer_alignment=0;
fl->fl_frag_class = OBJ_CLASS(ompi_free_list_item_t);
fl->fl_mpool = 0;
fl->ctx = NULL;
OBJ_CONSTRUCT(&(fl->fl_allocations), opal_list_t);
}
static void ompi_free_list_destruct(ompi_free_list_t* fl)
{
opal_list_item_t *item;
ompi_free_list_item_t *fl_item;
ompi_free_list_memory_t *fl_mem;
#if 0 && OPAL_ENABLE_DEBUG
if(opal_list_get_size(&fl->super) != fl->fl_num_allocated) {
opal_output(0, "ompi_free_list: %d allocated %d returned: %s:%d\n",
fl->fl_num_allocated, opal_list_get_size(&fl->super),
fl->super.super.cls_init_file_name, fl->super.super.cls_init_lineno);
}
#endif
while(NULL != (item = opal_lifo_pop(&(fl->super)))) {
fl_item = (ompi_free_list_item_t*)item;
/* destruct the item (we constructed it), the underlying memory will be
* reclaimed when we free the slab (ompi_free_list_memory_t ptr)
* containing it */
OBJ_DESTRUCT(fl_item);
}
while(NULL != (item = opal_list_remove_first(&(fl->fl_allocations)))) {
fl_mem = (ompi_free_list_memory_t*)item;
if( NULL != fl->fl_mpool ) {
fl->fl_mpool->mpool_free(fl->fl_mpool, fl_mem->ptr,
fl_mem->registration);
} else if (fl_mem->ptr) {
free (fl_mem->ptr);
}
/* destruct the item (we constructed it), then free the memory chunk */
OBJ_DESTRUCT(item);
free(item);
}
OBJ_DESTRUCT(&fl->fl_allocations);
OBJ_DESTRUCT(&fl->fl_condition);
OBJ_DESTRUCT(&fl->fl_lock);
}
int ompi_free_list_init_ex(
ompi_free_list_t *flist,
size_t elem_size,
size_t alignment,
opal_class_t* elem_class,
int num_elements_to_alloc,
int max_elements_to_alloc,
int num_elements_per_alloc,
mca_mpool_base_module_t* mpool,
ompi_free_list_item_init_fn_t item_init,
void* ctx)
{
/* alignment must be more than zero and power of two */
if(alignment <= 1 || (alignment & (alignment - 1)))
return OPAL_ERROR;
if(elem_size > flist->fl_frag_size)
flist->fl_frag_size = elem_size;
flist->fl_frag_alignment = alignment;
if(elem_class)
flist->fl_frag_class = elem_class;
flist->fl_payload_buffer_size=flist->fl_frag_size-
flist->fl_frag_class->cls_sizeof;
flist->fl_payload_buffer_alignment=alignment;
flist->fl_max_to_alloc = max_elements_to_alloc;
flist->fl_num_allocated = 0;
flist->fl_num_per_alloc = num_elements_per_alloc;
flist->fl_mpool = mpool;
flist->item_init = item_init;
flist->ctx = ctx;
if(num_elements_to_alloc)
return ompi_free_list_grow(flist, num_elements_to_alloc);
return OPAL_SUCCESS;
}
/* this will replace ompi_free_list_init_ex */
int ompi_free_list_init_ex_new(
ompi_free_list_t *flist,
size_t frag_size,
size_t frag_alignment,
opal_class_t* frag_class,
size_t payload_buffer_size,
size_t payload_buffer_alignment,
int num_elements_to_alloc,
int max_elements_to_alloc,
int num_elements_per_alloc,
mca_mpool_base_module_t* mpool,
ompi_free_list_item_init_fn_t item_init,
void* ctx)
{
/* alignment must be more than zero and power of two */
if (frag_alignment <= 1 || (frag_alignment & (frag_alignment - 1)))
return OPAL_ERROR;
if (0 < payload_buffer_size) {
if (payload_buffer_alignment <= 1 || (payload_buffer_alignment & (payload_buffer_alignment - 1)))
return OPAL_ERROR;
}
if (frag_size > flist->fl_frag_size)
flist->fl_frag_size = frag_size;
if (frag_class)
flist->fl_frag_class = frag_class;
flist->fl_payload_buffer_size=payload_buffer_size;
flist->fl_max_to_alloc = max_elements_to_alloc;
flist->fl_num_allocated = 0;
flist->fl_num_per_alloc = num_elements_per_alloc;
flist->fl_mpool = mpool;
flist->fl_frag_alignment = frag_alignment;
flist->fl_payload_buffer_alignment = payload_buffer_alignment;
flist->item_init = item_init;
flist->ctx = ctx;
if (num_elements_to_alloc)
return ompi_free_list_grow(flist, num_elements_to_alloc);
return OPAL_SUCCESS;
}
int ompi_free_list_grow(ompi_free_list_t* flist, size_t num_elements)
{
unsigned char *ptr, *mpool_alloc_ptr = NULL, *payload_ptr = NULL;
ompi_free_list_memory_t *alloc_ptr;
size_t i, alloc_size, head_size, elem_size = 0;
mca_mpool_base_registration_t *reg = NULL;
if(flist->fl_max_to_alloc > 0)
if(flist->fl_num_allocated + num_elements > flist->fl_max_to_alloc)
num_elements = flist->fl_max_to_alloc - flist->fl_num_allocated;
if(num_elements == 0)
return OPAL_ERR_TEMP_OUT_OF_RESOURCE;
head_size = (NULL == flist->fl_mpool) ? flist->fl_frag_size:
flist->fl_frag_class->cls_sizeof;
head_size = OPAL_ALIGN(head_size, flist->fl_frag_alignment, size_t);
/* calculate head allocation size */
alloc_size = num_elements * head_size + sizeof(ompi_free_list_memory_t) +
flist->fl_frag_alignment;
alloc_ptr = (ompi_free_list_memory_t*)malloc(alloc_size);
if(NULL == alloc_ptr)
return OPAL_ERR_TEMP_OUT_OF_RESOURCE;
if (0 != flist->fl_payload_buffer_size) {
elem_size = OPAL_ALIGN(flist->fl_payload_buffer_size,
flist->fl_payload_buffer_alignment, size_t);
/* elem_size should not be 0 here */
assert (elem_size > 0);
/* allocate the rest from the mpool (or use memalign/malloc) */
if(flist->fl_mpool != NULL) {
payload_ptr = mpool_alloc_ptr =
(unsigned char *) flist->fl_mpool->mpool_alloc(flist->fl_mpool,
num_elements * elem_size,
flist->fl_payload_buffer_alignment,
MCA_MPOOL_FLAGS_CACHE_BYPASS |
MCA_MPOOL_FLAGS_CUDA_REGISTER_MEM, &reg);
} else {
#ifdef HAVE_POSIX_MEMALIGN
posix_memalign ((void **) &mpool_alloc_ptr, flist->fl_payload_buffer_alignment,
num_elements * elem_size);
payload_ptr = mpool_alloc_ptr;
#else
mpool_alloc_ptr = (unsigned char *) malloc (num_elements * elem_size +
flist->fl_payload_buffer_alignment);
payload_ptr = (unsigned char *) OPAL_ALIGN((uintptr_t)mpool_alloc_ptr,
flist->fl_payload_buffer_alignment,
uintptr_t);
#endif
}
if(NULL == mpool_alloc_ptr) {
free(alloc_ptr);
return OPAL_ERR_TEMP_OUT_OF_RESOURCE;
}
}
/* make the alloc_ptr a list item, save the chunk in the allocations list,
* and have ptr point to memory right after the list item structure */
OBJ_CONSTRUCT(alloc_ptr, ompi_free_list_item_t);
opal_list_append(&(flist->fl_allocations), (opal_list_item_t*)alloc_ptr);
alloc_ptr->registration = reg;
alloc_ptr->ptr = mpool_alloc_ptr;
ptr = (unsigned char*)alloc_ptr + sizeof(ompi_free_list_memory_t);
ptr = OPAL_ALIGN_PTR(ptr, flist->fl_frag_alignment, unsigned char*);
for(i=0; i<num_elements; i++) {
ompi_free_list_item_t* item = (ompi_free_list_item_t*)ptr;
item->registration = reg;
item->ptr = payload_ptr;
OBJ_CONSTRUCT_INTERNAL(item, flist->fl_frag_class);
item->super.item_free = 0;
/* run the initialize function if present */
if(flist->item_init) {
flist->item_init(item, flist->ctx);
}
opal_lifo_push(&(flist->super), &(item->super));
ptr += head_size;
payload_ptr += elem_size;
}
flist->fl_num_allocated += num_elements;
return OPAL_SUCCESS;
}
/**
* This function resize the free_list to contain at least the specified
* number of elements. We do not create all of them in the same memory
* segment. Instead we will several time the fl_num_per_alloc elements
* until we reach the required number of the maximum allowed by the
* initialization.
*/
int
ompi_free_list_resize_mt(ompi_free_list_t* flist, size_t size)
{
ssize_t inc_num;
int ret = OPAL_SUCCESS;
if (flist->fl_num_allocated > size) {
return OPAL_SUCCESS;
}
OPAL_THREAD_LOCK(&((flist)->fl_lock));
inc_num = (ssize_t)size - (ssize_t)flist->fl_num_allocated;
while( inc_num > 0 ) {
ret = ompi_free_list_grow(flist, flist->fl_num_per_alloc);
if( OPAL_SUCCESS != ret ) break;
inc_num = (ssize_t)size - (ssize_t)flist->fl_num_allocated;
}
OPAL_THREAD_UNLOCK(&((flist)->fl_lock));
return ret;
}
OBJ_CLASS_INSTANCE(ompi_free_list_t, opal_free_list_t, NULL, NULL);

322
opal/class/ompi_free_list.h
Просмотреть файл

@ -24,271 +24,105 @@
#ifndef OMPI_FREE_LIST_H
#define OMPI_FREE_LIST_H
#include "opal_config.h"
#include "opal/class/opal_lifo.h"
#include "opal/prefetch.h"
#include "opal/threads/condition.h"
#include "opal/constants.h"
#include "opal/runtime/opal.h"
#include "opal_free_list.h"
BEGIN_C_DECLS
#define ompi_free_list_item_t opal_free_list_item_t
struct mca_mem_pool_t;
struct ompi_free_list_item_t;
typedef void (*ompi_free_list_item_init_fn_t) (
struct ompi_free_list_item_t*, void* ctx);
typedef void (*ompi_free_list_item_init_fn_t) (ompi_free_list_item_t*, void* ctx);
struct ompi_free_list_t
{
opal_lifo_t super;
size_t fl_max_to_alloc;
size_t fl_num_allocated;
size_t fl_num_per_alloc;
size_t fl_num_waiting;
size_t fl_frag_size; /* size of the fragment descriptor */
size_t fl_frag_alignment; /* fragment descriptor alignment */
size_t fl_payload_buffer_size; /* size of payload buffer */
size_t fl_payload_buffer_alignment; /* payload buffer alignment */
opal_class_t* fl_frag_class;
struct mca_mpool_base_module_t* fl_mpool;
opal_mutex_t fl_lock;
opal_condition_t fl_condition;
opal_list_t fl_allocations;
struct ompi_free_list_t {
opal_free_list_t super;
ompi_free_list_item_init_fn_t item_init;
void* ctx;
void *ctx;
};
typedef struct ompi_free_list_t ompi_free_list_t;
OPAL_DECLSPEC OBJ_CLASS_DECLARATION(ompi_free_list_t);
struct mca_mpool_base_registration_t;
struct ompi_free_list_item_t
{
opal_list_item_t super;
struct mca_mpool_base_registration_t *registration;
void *ptr;
};
typedef struct ompi_free_list_item_t ompi_free_list_item_t;
OPAL_DECLSPEC OBJ_CLASS_DECLARATION(ompi_free_list_item_t);
#define OMPI_FREE_LIST_GET_MT(fl, item) \
opal_free_list_get (&(fl)->super, &(item))
/**
* Initialize a free list.
*
* @param free_list (IN) Free list.
* @param element_size (IN) Size of each element.
* @param element_class (IN) opal_class_t of element - used to initialize allocated elements.
* @param num_elements_to_alloc Initial number of elements to allocate.
* @param max_elements_to_alloc Maximum number of elements to allocate.
* @param num_elements_per_alloc Number of elements to grow by per allocation.
* @param mpool Optional memory pool for allocation.s
*/
OPAL_DECLSPEC int ompi_free_list_init_ex(
ompi_free_list_t *free_list,
size_t element_size,
size_t alignment,
opal_class_t* element_class,
int num_elements_to_alloc,
int max_elements_to_alloc,
int num_elements_per_alloc,
struct mca_mpool_base_module_t*,
ompi_free_list_item_init_fn_t item_init,
void *ctx
);
#define OMPI_FREE_LIST_WAIT_MT(fl, item) \
opal_free_list_wait (&(fl)->super, &(item))
static inline int ompi_free_list_init(
ompi_free_list_t *free_list,
size_t element_size,
opal_class_t* element_class,
int num_elements_to_alloc,
int max_elements_to_alloc,
int num_elements_per_alloc,
struct mca_mpool_base_module_t* mpool)
#define OMPI_FREE_LIST_RETURN_MT(fl, item) \
opal_free_list_return (&(fl)->super, item)
static inline int ompi_free_list_item_init_compat (ompi_free_list_item_t *item,
void *ctx)
{
return ompi_free_list_init_ex(free_list, element_size, opal_cache_line_size,
element_class, num_elements_to_alloc, max_elements_to_alloc,
num_elements_per_alloc, mpool, NULL, NULL);
ompi_free_list_t *free_list = (ompi_free_list_t *) ctx;
free_list->item_init (item, free_list->ctx);
return OPAL_SUCCESS;
}
/**
* Initialize a free list. - this will replace ompi_free_list_init_ex
*
* @param free_list (IN) Free list.
* @param frag_size (IN) Size of each element - allocated by malloc.
* @param frag_alignment (IN) Fragment alignment.
* @param frag_class (IN) opal_class_t of element - used to initialize allocated elements.
* @param payload_buffer_size (IN) Size of payload buffer - allocated from mpool.
* @param payload_buffer_alignment (IN) Payload buffer alignment.
* @param num_elements_to_alloc (IN) Initial number of elements to allocate.
* @param max_elements_to_alloc (IN) Maximum number of elements to allocate.
* @param num_elements_per_alloc (IN) Number of elements to grow by per allocation.
* @param mpool (IN) Optional memory pool for allocation.s
* @param item_init (IN)
* @param ctx (IN)
*/
OPAL_DECLSPEC int ompi_free_list_init_ex_new(
ompi_free_list_t *free_list,
size_t frag_size,
size_t frag_alignment,
opal_class_t* frag_class,
size_t payload_buffer_size,
size_t payload_buffer_alignment,
int num_elements_to_alloc,
int max_elements_to_alloc,
int num_elements_per_alloc,
struct mca_mpool_base_module_t*,
ompi_free_list_item_init_fn_t item_init,
void *ctx
);
/**
* Initialize a free list. - this will replace ompi_free_list_init
*
* @param free_list (IN) Free list.
* @param frag_size (IN) Size of each element - allocated by malloc.
* @param frag_alignment (IN) Fragment alignment.
* @param frag_class (IN) opal_class_t of element - used to initialize allocated elements.
* @param payload_buffer_size (IN) Size of payload buffer - allocated from mpool.
* @param payload_buffer_alignment (IN) Payload buffer alignment.
* @param num_elements_to_alloc (IN) Initial number of elements to allocate.
* @param max_elements_to_alloc (IN) Maximum number of elements to allocate.
* @param num_elements_per_alloc (IN) Number of elements to grow by per allocation.
* @param mpool (IN) Optional memory pool for allocation.s
*/
static inline int ompi_free_list_init_new(
ompi_free_list_t *free_list,
size_t frag_size,
size_t frag_alignment,
opal_class_t* frag_class,
size_t payload_buffer_size,
size_t payload_buffer_alignment,
int num_elements_to_alloc,
int max_elements_to_alloc,
int num_elements_per_alloc,
struct mca_mpool_base_module_t* mpool)
static inline int __opal_attribute_deprecated__
ompi_free_list_init_ex_new (ompi_free_list_t *free_list, size_t frag_size,
size_t frag_alignment, opal_class_t* frag_class,
size_t payload_buffer_size, size_t payload_buffer_alignment,
int num_elements_to_alloc, int max_elements_to_alloc,
int num_elements_per_alloc, struct mca_mpool_base_module_t *mpool,
ompi_free_list_item_init_fn_t item_init, void *ctx)
{
return ompi_free_list_init_ex_new(free_list,
frag_size, frag_alignment, frag_class,
payload_buffer_size, payload_buffer_alignment,
num_elements_to_alloc, max_elements_to_alloc,
num_elements_per_alloc, mpool, NULL, NULL);
free_list->item_init = item_init;
free_list->ctx = ctx;
return opal_free_list_init (&free_list->super, frag_size, frag_alignment,
frag_class, payload_buffer_size, payload_buffer_alignment,
num_elements_to_alloc, max_elements_to_alloc,
num_elements_per_alloc, mpool, 0, NULL,
ompi_free_list_item_init_compat,
(void *) free_list);
}
OPAL_DECLSPEC int ompi_free_list_grow(ompi_free_list_t* flist, size_t num_elements);
static inline int __opal_attribute_deprecated__
ompi_free_list_init_new (ompi_free_list_t *free_list, size_t frag_size,
size_t frag_alignment, opal_class_t* frag_class,
size_t payload_buffer_size, size_t payload_buffer_alignment,
int num_elements_to_alloc,int max_elements_to_alloc,
int num_elements_per_alloc, struct mca_mpool_base_module_t* mpool)
{
return opal_free_list_init (&free_list->super, frag_size, frag_alignment,
frag_class, payload_buffer_size,
payload_buffer_alignment, num_elements_to_alloc,
max_elements_to_alloc, num_elements_per_alloc,
mpool, 0, NULL, NULL, NULL);
}
static inline int __opal_attribute_deprecated__
ompi_free_list_init_ex (ompi_free_list_t *free_list, size_t element_size,
size_t alignment, opal_class_t* element_class,
int num_elements_to_alloc, int max_elements_to_alloc,
int num_elements_per_alloc, struct mca_mpool_base_module_t *mpool,
ompi_free_list_item_init_fn_t item_init, void *ctx)
{
return ompi_free_list_init_ex_new (free_list, element_size, alignment,
element_class, 0, 0, num_elements_to_alloc,
max_elements_to_alloc, num_elements_per_alloc,
mpool, item_init, ctx);
}
static inline int __opal_attribute_deprecated__
ompi_free_list_init (ompi_free_list_t *free_list, size_t element_size,
opal_class_t* element_class, int num_elements_to_alloc,
int max_elements_to_alloc, int num_elements_per_alloc,
struct mca_mpool_base_module_t* mpool)
{
return opal_free_list_init (&free_list->super, element_size, opal_cache_line_size,
element_class, 0, 0, num_elements_to_alloc,
max_elements_to_alloc, num_elements_per_alloc,
mpool, 0, NULL, NULL, NULL);
}
/* Grow the free list to be *at least* size elements. This function
will not shrink the list if it is already larger than size and may
grow it past size if necessary (it will grow in
num_elements_per_alloc chunks) */
OPAL_DECLSPEC int ompi_free_list_resize_mt(ompi_free_list_t *flist, size_t size);
/**
* Attemp to obtain an item from a free list.
*
* @param fl (IN) Free list.
* @param item (OUT) Allocated item.
*
* If the requested item is not available the free list is grown to
* accomodate the request - unless the max number of allocations has
* been reached. If this is the case - a NULL pointer is returned
* to the caller.
*/
#define OMPI_FREE_LIST_GET_MT(fl, item) \
{ \
item = (ompi_free_list_item_t*) opal_lifo_pop (&((fl)->super)); \
if( OPAL_UNLIKELY(NULL == item) ) { \
if(opal_using_threads()) { \
opal_mutex_lock(&((fl)->fl_lock)); \
ompi_free_list_grow((fl), (fl)->fl_num_per_alloc); \
opal_mutex_unlock(&((fl)->fl_lock)); \
} else { \
ompi_free_list_grow((fl), (fl)->fl_num_per_alloc); \
} \
item = (ompi_free_list_item_t*) opal_lifo_pop(&((fl)->super)); \
} \
}
/**
* Blocking call to obtain an item from a free list.
*
* @param fl (IN) Free list.
* @param item (OUT) Allocated item.
*
* If the requested item is not available the free list is grown to
* accomodate the request - unless the max number of allocations has
* been reached. In this case the caller is blocked until an item
* is returned to the list.
*/
#define OMPI_FREE_LIST_WAIT_MT(fl, item) \
__ompi_free_list_wait_mt( (fl), &(item) )
static inline void __ompi_free_list_wait_mt( ompi_free_list_t* fl,
ompi_free_list_item_t** item )
static inline int __opal_attribute_deprecated__
ompi_free_list_resize_mt(ompi_free_list_t *flist, size_t size)
{
*item = (ompi_free_list_item_t*)opal_lifo_pop(&((fl)->super));
while( NULL == *item ) {
if( !OPAL_THREAD_TRYLOCK(&((fl)->fl_lock)) ) {
if((fl)->fl_max_to_alloc <= (fl)->fl_num_allocated) {
(fl)->fl_num_waiting++;
opal_condition_wait(&((fl)->fl_condition), &((fl)->fl_lock));
(fl)->fl_num_waiting--;
} else {
if(ompi_free_list_grow((fl), (fl)->fl_num_per_alloc)
== OPAL_SUCCESS) {
if( 0 < (fl)->fl_num_waiting ) {
if( 1 == (fl)->fl_num_waiting ) {
opal_condition_signal(&((fl)->fl_condition));
} else {
opal_condition_broadcast(&((fl)->fl_condition));
}
}
} else {
(fl)->fl_num_waiting++;
opal_condition_wait(&((fl)->fl_condition), &((fl)->fl_lock));
(fl)->fl_num_waiting--;
}
}
} else {
/* If I wasn't able to get the lock in the begining when I finaly grab it
* the one holding the lock in the begining already grow the list. I will
* release the lock and try to get a new element until I succeed.
*/
OPAL_THREAD_LOCK(&((fl)->fl_lock));
}
OPAL_THREAD_UNLOCK(&((fl)->fl_lock));
*item = (ompi_free_list_item_t*)opal_lifo_pop(&((fl)->super));
}
}
/**
* Return an item to a free list.
*
* @param fl (IN) Free list.
* @param item (OUT) Allocated item.
*
*/
#define OMPI_FREE_LIST_RETURN_MT(fl, item) \
do { \
opal_list_item_t* original; \
\
original = opal_lifo_push( &(fl)->super, &(item)->super); \
if( &(fl)->super.opal_lifo_ghost == original ) { \
OPAL_THREAD_LOCK(&(fl)->fl_lock); \
if((fl)->fl_num_waiting > 0) { \
if( 1 == (fl)->fl_num_waiting ) { \
opal_condition_signal(&((fl)->fl_condition)); \
} else { \
opal_condition_broadcast(&((fl)->fl_condition)); \
} \
} \
OPAL_THREAD_UNLOCK(&(fl)->fl_lock); \
} \
} while(0)
END_C_DECLS
#endif
return opal_free_list_resize_mt (&flist->super, size);
}
#endif

292
opal/class/opal_free_list.c
Просмотреть файл

@ -3,15 +3,17 @@
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2008 The University of Tennessee and The University
* Copyright (c) 2004-2009 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2014 Los Alamos National Security, LLC. All rights
* Copyright (c) 2006-2007 Mellanox Technologies. All rights reserved.
* Copyright (c) 2010-2013 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2011 NVIDIA Corporation. All rights reserved.
* Copyright (c) 2012-2015 Los Alamos National Security, LLC. All rights
* reserved.
* $COPYRIGHT$
*
@ -23,15 +25,12 @@
#include "opal_config.h"
#include "opal/class/opal_free_list.h"
#include "opal/runtime/opal.h"
#include "opal/align.h"
#include "opal/util/output.h"
#include "opal/mca/mpool/mpool.h"
static void opal_free_list_construct(opal_free_list_t* fl);
static void opal_free_list_destruct(opal_free_list_t* fl);
typedef struct opal_free_list_item_t opal_free_list_memory_t;
OBJ_CLASS_INSTANCE(opal_free_list_t,
opal_list_t,
opal_free_list_construct,
opal_free_list_destruct);
OBJ_CLASS_INSTANCE(opal_free_list_item_t,
opal_list_item_t,
NULL, NULL);
@ -44,27 +43,56 @@ static void opal_free_list_construct(opal_free_list_t* fl)
fl->fl_num_allocated = 0;
fl->fl_num_per_alloc = 0;
fl->fl_num_waiting = 0;
fl->fl_elem_size = 0;
fl->fl_elem_class = 0;
fl->fl_frag_size = sizeof(opal_free_list_item_t);
fl->fl_frag_alignment = 0;
fl->fl_payload_buffer_size = 0;
fl->fl_payload_buffer_alignment = 0;
fl->fl_frag_class = OBJ_CLASS(opal_free_list_item_t);
fl->fl_mpool = NULL;
/* default flags */
fl->fl_mpool_reg_flags = MCA_MPOOL_FLAGS_CACHE_BYPASS |
MCA_MPOOL_FLAGS_CUDA_REGISTER_MEM;
fl->ctx = NULL;
OBJ_CONSTRUCT(&(fl->fl_allocations), opal_list_t);
}
static void opal_free_list_destruct(opal_free_list_t* fl)
static void opal_free_list_allocation_release (opal_free_list_t *fl, opal_free_list_memory_t *fl_mem)
{
opal_list_item_t *item;
if (fl->fl_elem_class) {
while (NULL != (item = opal_list_remove_first (&fl->super))) {
/* destruct the item (we constructed it), the underlying memory will be
* reclaimed when we free the slab below */
OBJ_DESTRUCT(item);
}
if (NULL != fl->fl_mpool) {
fl->fl_mpool->mpool_free (fl->fl_mpool, fl_mem->ptr, fl_mem->registration);
} else if (fl_mem->ptr) {
free (fl_mem->ptr);
}
while (NULL != (item = opal_list_remove_first(&(fl->fl_allocations)))) {
/* destruct the item (we constructed it), then free the memory chunk */
OBJ_DESTRUCT(item);
free(item);
/* destruct the item (we constructed it), then free the memory chunk */
OBJ_DESTRUCT(fl_mem);
free(fl_mem);
}
static void opal_free_list_destruct(opal_free_list_t *fl)
{
opal_list_item_t *item;
opal_free_list_item_t *fl_item;
#if 0 && OPAL_ENABLE_DEBUG
if(opal_list_get_size(&fl->super) != fl->fl_num_allocated) {
opal_output(0, "opal_free_list: %d allocated %d returned: %s:%d\n",
fl->fl_num_allocated, opal_list_get_size(&fl->super),
fl->super.super.cls_init_file_name, fl->super.super.cls_init_lineno);
}
#endif
while(NULL != (item = opal_lifo_pop(&(fl->super)))) {
fl_item = (opal_free_list_item_t*)item;
/* destruct the item (we constructed it), the underlying memory will be
* reclaimed when we free the slab (opal_free_list_memory_t ptr)
* containing it */
OBJ_DESTRUCT(fl_item);
}
while(NULL != (item = opal_list_remove_first(&fl->fl_allocations))) {
opal_free_list_allocation_release (fl, (opal_free_list_memory_t *) item);
}
OBJ_DESTRUCT(&fl->fl_allocations);
@ -72,69 +100,191 @@ static void opal_free_list_destruct(opal_free_list_t* fl)
OBJ_DESTRUCT(&fl->fl_lock);
}
int opal_free_list_init(
opal_free_list_t *flist,
size_t elem_size,
opal_class_t* elem_class,
int num_elements_to_alloc,
int max_elements_to_alloc,
int num_elements_per_alloc)
OBJ_CLASS_INSTANCE(opal_free_list_t, opal_lifo_t, opal_free_list_construct,
opal_free_list_destruct);
int opal_free_list_init (opal_free_list_t *flist, size_t frag_size, size_t frag_alignment,
opal_class_t *frag_class, size_t payload_buffer_size,
size_t payload_buffer_alignment, int num_elements_to_alloc,
int max_elements_to_alloc, int num_elements_per_alloc,
mca_mpool_base_module_t* mpool, int mpool_reg_flags,
void *unused0, opal_free_list_item_init_fn_t item_init, void *ctx)
{
flist->fl_elem_size = elem_size;
flist->fl_elem_class = elem_class;
/* alignment must be more than zero and power of two */
if (frag_alignment <= 1 || (frag_alignment & (frag_alignment - 1))) {
return OPAL_ERROR;
}
if (0 < payload_buffer_size) {
if (payload_buffer_alignment <= 1 || (payload_buffer_alignment & (payload_buffer_alignment - 1)))
return OPAL_ERROR;
}
if (frag_size > flist->fl_frag_size) {
flist->fl_frag_size = frag_size;
}
if (frag_class) {
flist->fl_frag_class = frag_class;
}
flist->fl_payload_buffer_size = payload_buffer_size;
flist->fl_max_to_alloc = max_elements_to_alloc;
flist->fl_num_allocated = 0;
flist->fl_num_per_alloc = num_elements_per_alloc;
if(num_elements_to_alloc)
return opal_free_list_grow(flist, num_elements_to_alloc);
flist->fl_mpool = mpool;
flist->fl_frag_alignment = frag_alignment;
flist->fl_payload_buffer_alignment = payload_buffer_alignment;
flist->item_init = item_init;
flist->fl_mpool_reg_flags |= mpool_reg_flags;
flist->ctx = ctx;
if (num_elements_to_alloc) {
return opal_free_list_grow_st (flist, num_elements_to_alloc);
}
return OPAL_SUCCESS;
}
int opal_free_list_grow(opal_free_list_t* flist, size_t num_elements)
int opal_free_list_grow_st (opal_free_list_t* flist, size_t num_elements)
{
unsigned char* ptr;
unsigned char* alloc_ptr;
size_t i;
size_t mod;
unsigned char *ptr, *mpool_alloc_ptr = NULL, *payload_ptr = NULL;
opal_free_list_memory_t *alloc_ptr;
size_t alloc_size, head_size, elem_size = 0;
mca_mpool_base_registration_t *reg = NULL;
int rc = OPAL_SUCCESS;
if (flist->fl_max_to_alloc > 0 && flist->fl_num_allocated + num_elements > flist->fl_max_to_alloc)
return OPAL_ERR_TEMP_OUT_OF_RESOURCE;
alloc_ptr = (unsigned char *)malloc((num_elements * flist->fl_elem_size) +
sizeof(opal_list_item_t) +
opal_cache_line_size);
if(NULL == alloc_ptr)
return OPAL_ERR_TEMP_OUT_OF_RESOURCE;
/* make the alloc_ptr a list item, save the chunk in the allocations list, and
have ptr point to memory right after the list item structure */
OBJ_CONSTRUCT(alloc_ptr, opal_list_item_t);
opal_list_append(&(flist->fl_allocations), (opal_list_item_t*) alloc_ptr);
ptr = alloc_ptr + sizeof(opal_list_item_t);
mod = (uintptr_t)ptr % opal_cache_line_size;
if(mod != 0) {
ptr += (opal_cache_line_size - mod);
if (flist->fl_max_to_alloc && (flist->fl_num_allocated + num_elements) >
flist->fl_max_to_alloc) {
num_elements = flist->fl_max_to_alloc - flist->fl_num_allocated;
}
if (NULL != flist->fl_elem_class) {
for(i=0; i<num_elements; i++) {
opal_free_list_item_t* item = (opal_free_list_item_t*)ptr;
OBJ_CONSTRUCT_INTERNAL(item, flist->fl_elem_class);
opal_list_append(&(flist->super), &(item->super));
ptr += flist->fl_elem_size;
if (num_elements == 0) {
return OPAL_ERR_TEMP_OUT_OF_RESOURCE;
}
head_size = (NULL == flist->fl_mpool) ? flist->fl_frag_size:
flist->fl_frag_class->cls_sizeof;
head_size = OPAL_ALIGN(head_size, flist->fl_frag_alignment, size_t);
/* calculate head allocation size */
alloc_size = num_elements * head_size + sizeof(opal_free_list_memory_t) +
flist->fl_frag_alignment;
alloc_ptr = (opal_free_list_memory_t *) malloc(alloc_size);
if (OPAL_UNLIKELY(NULL == alloc_ptr)) {
return OPAL_ERR_TEMP_OUT_OF_RESOURCE;
}
if (0 != flist->fl_payload_buffer_size) {
elem_size = OPAL_ALIGN(flist->fl_payload_buffer_size,
flist->fl_payload_buffer_alignment, size_t);
/* elem_size should not be 0 here */
assert (elem_size > 0);
/* allocate the rest from the mpool (or use memalign/malloc) */
if(flist->fl_mpool != NULL) {
payload_ptr = mpool_alloc_ptr =
(unsigned char *) flist->fl_mpool->mpool_alloc(flist->fl_mpool,
num_elements * elem_size,
flist->fl_payload_buffer_alignment,
flist->fl_mpool_reg_flags, &reg);
} else {
#ifdef HAVE_POSIX_MEMALIGN
posix_memalign ((void **) &mpool_alloc_ptr, flist->fl_payload_buffer_alignment,
num_elements * elem_size);
payload_ptr = mpool_alloc_ptr;
#else
mpool_alloc_ptr = (unsigned char *) malloc (num_elements * elem_size +
flist->fl_payload_buffer_alignment);
payload_ptr = (unsigned char *) OPAL_ALIGN((uintptr_t)mpool_alloc_ptr,
flist->fl_payload_buffer_alignment,
uintptr_t);
#endif
}
} else {
for(i=0; i<num_elements; i++) {
opal_free_list_item_t* item = (opal_free_list_item_t*)ptr;
opal_list_append(&(flist->super), &(item->super));
ptr += flist->fl_elem_size;
if(NULL == mpool_alloc_ptr) {
free(alloc_ptr);
return OPAL_ERR_TEMP_OUT_OF_RESOURCE;
}
}
/* make the alloc_ptr a list item, save the chunk in the allocations list,
* and have ptr point to memory right after the list item structure */
OBJ_CONSTRUCT(alloc_ptr, opal_free_list_item_t);
opal_list_append(&(flist->fl_allocations), (opal_list_item_t*)alloc_ptr);
alloc_ptr->registration = reg;
alloc_ptr->ptr = mpool_alloc_ptr;
ptr = (unsigned char*)alloc_ptr + sizeof(opal_free_list_memory_t);
ptr = OPAL_ALIGN_PTR(ptr, flist->fl_frag_alignment, unsigned char*);
for(size_t i = 0; i < num_elements ; ++i) {
opal_free_list_item_t* item = (opal_free_list_item_t*)ptr;
item->registration = reg;
item->ptr = payload_ptr;
OBJ_CONSTRUCT_INTERNAL(item, flist->fl_frag_class);
item->super.item_free = 0;
/* run the initialize function if present */
if (flist->item_init) {
if (OPAL_SUCCESS != (rc = flist->item_init(item, flist->ctx))) {
num_elements = i;
OBJ_DESTRUCT (item);
break;
}
}
/* NTH: in case the free list may be accessed from multiple threads
* use the atomic lifo push. The overhead is small compared to the
* overall overhead of opal_free_list_grow(). */
opal_lifo_push_atomic (&flist->super, &item->super);
ptr += head_size;
payload_ptr += elem_size;
}
if (OPAL_SUCCESS != rc && 0 == num_elements) {
/* couldn't initialize any items */
opal_list_remove_item (&flist->fl_allocations, (opal_list_item_t *) alloc_ptr);
opal_free_list_allocation_release (flist, alloc_ptr);
return OPAL_ERR_OUT_OF_RESOURCE;
}
flist->fl_num_allocated += num_elements;
return OPAL_SUCCESS;
}
/**
* This function resize the free_list to contain at least the specified
* number of elements. We do not create all of them in the same memory
* segment. Instead we will several time the fl_num_per_alloc elements
* until we reach the required number of the maximum allowed by the
* initialization.
*/
int opal_free_list_resize_mt(opal_free_list_t *flist, size_t size)
{
ssize_t inc_num;
int ret = OPAL_SUCCESS;
if (flist->fl_num_allocated > size) {
return OPAL_SUCCESS;
}
opal_mutex_lock (&flist->fl_lock);
do {
ret = opal_free_list_grow_st (flist, flist->fl_num_per_alloc);
if (OPAL_SUCCESS != ret) {
break;
}
inc_num = (ssize_t)size - (ssize_t)flist->fl_num_allocated;
} while (inc_num > 0);
opal_mutex_unlock (&flist->fl_lock);
return ret;
}

358
opal/class/opal_free_list.h
Просмотреть файл

@ -1,14 +1,19 @@
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2006 The University of Tennessee and The University
* Copyright (c) 2004-2013 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2010 IBM Corporation. All rights reserved.
* Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2014-2015 Los Alamos National Security, LLC. All rights
* reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -20,101 +25,213 @@
#define OPAL_FREE_LIST_H
#include "opal_config.h"
#include "opal/class/opal_list.h"
#include "opal/class/opal_lifo.h"
#include "opal/prefetch.h"
#include "opal/threads/condition.h"
#include "opal/constants.h"
#include "opal/prefetch.h"
#include "opal/runtime/opal.h"
BEGIN_C_DECLS
struct opal_free_list_t
{
opal_list_t super;
struct mca_mem_pool_t;
struct opal_free_list_item_t;
/**
* Free list item initializtion function.
*
* @param item (IN) Free list item to initialize
* @param ctx (IN) Free list initialization context
*
* @returns OPAL_SUCCESS on success
* @returns opal error code on failure
*
* This function attempts to initialize the free list item
* specified in item. If the item can be initialized the
* function should return OPAL_SUCCESS. On any error
* opal_free_list_grow will stop initializing new items.
*/
typedef int (*opal_free_list_item_init_fn_t) (
struct opal_free_list_item_t *item, void *ctx);
struct opal_free_list_t {
/** Items in a free list are stored last-in first-out */
opal_lifo_t super;
/** Maximum number of items to allocate in the free list */
size_t fl_max_to_alloc;
/** Current number of items allocated */
size_t fl_num_allocated;
/** Number of items to allocate when growing the free list */
size_t fl_num_per_alloc;
/** Number of threads waiting on free list item availability */
size_t fl_num_waiting;
size_t fl_elem_size;
opal_class_t* fl_elem_class;
/** Size of each free list item */
size_t fl_frag_size;
/** Free list item alignment */
size_t fl_frag_alignment;
/** Free list item buffer size */
size_t fl_payload_buffer_size;
/** Free list item buffer alignment */
size_t fl_payload_buffer_alignment;
/** Class of free list items */
opal_class_t *fl_frag_class;
/** mpool to use for free list buffer allocation (posix_memalign/malloc
* are used if this is NULL) */
struct mca_mpool_base_module_t *fl_mpool;
/** Multi-threaded lock. Used when the free list is empty. */
opal_mutex_t fl_lock;
opal_condition_t fl_condition;
/** Multi-threaded condition. Used when threads are waiting on free
* list item availability. */
opal_condition_t fl_condition;
/** List of free list allocation */
opal_list_t fl_allocations;
/** Flags to pass to the mpool register function */
int fl_mpool_reg_flags;
/** Free list item initialization function */
opal_free_list_item_init_fn_t item_init;
/** Initialization function context */
void *ctx;
};
typedef struct opal_free_list_t opal_free_list_t;
OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_free_list_t);
struct mca_mpool_base_registration_t;
struct opal_free_list_item_t
{
opal_list_item_t super;
struct mca_mpool_base_registration_t *registration;
void *ptr;
};
typedef struct opal_free_list_item_t opal_free_list_item_t;
OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_free_list_item_t);
/**
* Initialize a free list.
*
* @param free_list (IN) Free list.
* @param element_size (IN) Size of each element.
* @param element_class (IN) opal_class_t of element - used to initialize allocated elements.
* @param num_elements_to_alloc Initial number of elements to allocate.
* @param max_elements_to_alloc Maximum number of elements to allocate.
* @param num_elements_per_alloc Number of elements to grow by per allocation.
* @param free_list (IN) Free list.
* @param frag_size (IN) Size of each element - allocated by malloc.
* @param frag_alignment (IN) Fragment alignment.
* @param frag_class (IN) opal_class_t of element - used to initialize allocated elements.
* @param payload_buffer_size (IN) Size of payload buffer - allocated from mpool.
* @param payload_buffer_alignment (IN) Payload buffer alignment.
* @param num_elements_to_alloc (IN) Initial number of elements to allocate.
* @param max_elements_to_alloc (IN) Maximum number of elements to allocate.
* @param num_elements_per_alloc (IN) Number of elements to grow by per allocation.
* @param mpool (IN) Optional memory pool for allocation.s
* @param mpool_reg_flags (IN) Flags to pass to mpool registration function
* @param unused0 (IN) Future. Must be NULL.
* @param item_init (IN) Optional item initialization function
* @param ctx (IN) Initialization function context.
*/
OPAL_DECLSPEC int opal_free_list_init(
opal_free_list_t *free_list,
size_t element_size,
opal_class_t* element_class,
int num_elements_to_alloc,
int max_elements_to_alloc,
int num_elements_per_alloc);
OPAL_DECLSPEC int opal_free_list_init (opal_free_list_t *free_list,
size_t frag_size,
size_t frag_alignment,
opal_class_t* frag_class,
size_t payload_buffer_size,
size_t payload_buffer_alignment,
int num_elements_to_alloc,
int max_elements_to_alloc,
int num_elements_per_alloc,
struct mca_mpool_base_module_t *mpool,
int mpool_reg_flags,
void *unused0,
opal_free_list_item_init_fn_t item_init,
void *ctx);
/**
* Grow the free list by at most num_elements elements.
*
* @param flist (IN) Free list to grow
* @param num_elements (IN) Number of elements to add
*
* @returns OPAL_SUCCESS if any elements were added
* @returns OPAL_ERR_OUT_OF_RESOURCE if no elements could be added
*
* This function will attempt to grow the free list by num_elements items. The
* caller must hold the free list lock if calling this function on a free list
* that may be accessed by multiple threads simultaneously. Note: this is an
* internal function that will be used when needed by opal_free_list_get* and
* opal_free_list_wait*.
*/
OPAL_DECLSPEC int opal_free_list_grow_st (opal_free_list_t *flist, size_t num_elements);
/**
* Grow the free list to be at least size elements.
*
* @param flist (IN) Free list to resize.
* @param size (IN) New size
*
* @returns OPAL_SUCCESS if the free list was resized
* @returns OPAL_ERR_OUT_OF_RESOURCE if resources could not be allocated
*
* This function will not shrink the list if it is already larger than size
* and may grow it past size if necessary (it will grow in num_elements_per_alloc
* chunks). This function is thread-safe and will obtain the free list lock before
* growing the free list.
*/
OPAL_DECLSPEC int opal_free_list_resize_mt (opal_free_list_t *flist, size_t size);
OPAL_DECLSPEC int opal_free_list_grow(opal_free_list_t* flist, size_t num_elements) __opal_attribute_nonnull__(1);
/**
* Attemp to obtain an item from a free list.
*
* @param fl (IN) Free list.
* @param item (OUT) Allocated item.
* @param rc (OUT) OPAL_SUCCESS or error status on failure.
*
* If the requested item is not available the free list is grown to
* accomodate the request - unless the max number of allocations has
* been reached. If this is the case - an out of resource error is
* returned to the caller.
* been reached. If this is the case - a NULL pointer is returned
* to the caller. This function comes in three flavor: thread safe
* (opal_free_list_get_mt), single threaded (opal_free_list_get_st),
* and opal_using_threads conditioned (opal_free_list_get).
*/
#define OPAL_FREE_LIST_GET(fl, item, rc) \
{ \
if(opal_using_threads()) { \
opal_mutex_lock(&((fl)->fl_lock)); \
item = (opal_free_list_item_t*) \
opal_list_remove_first(&((fl)->super)); \
if( OPAL_UNLIKELY(NULL == item) ) { \
opal_free_list_grow((fl), (fl)->fl_num_per_alloc); \
item = (opal_free_list_item_t*) \
opal_list_remove_first(&((fl)->super)); \
} \
opal_mutex_unlock(&((fl)->fl_lock)); \
} else { \
item = (opal_free_list_item_t*) \
opal_list_remove_first(&((fl)->super)); \
if( OPAL_UNLIKELY(NULL == item) ) { \
opal_free_list_grow((fl), (fl)->fl_num_per_alloc); \
item = (opal_free_list_item_t*) \
opal_list_remove_first(&((fl)->super)); \
} \
} \
rc = (NULL == item) ? OPAL_ERR_TEMP_OUT_OF_RESOURCE : OPAL_SUCCESS; \
static inline opal_free_list_item_t *opal_free_list_get_mt (opal_free_list_t *flist)
{
opal_free_list_item_t *item =
(opal_free_list_item_t*) opal_lifo_pop_atomic (&flist->super);
if (OPAL_UNLIKELY(NULL == item)) {
opal_mutex_lock (&flist->fl_lock);
opal_free_list_grow_st (flist, flist->fl_num_per_alloc);
opal_mutex_unlock (&flist->fl_lock);
item = (opal_free_list_item_t *) opal_lifo_pop_atomic (&flist->super);
}
return item;
}
static inline opal_free_list_item_t *opal_free_list_get_st (opal_free_list_t *flist)
{
opal_free_list_item_t *item =
(opal_free_list_item_t*) opal_lifo_pop_st (&flist->super);
if (OPAL_UNLIKELY(NULL == item)) {
opal_free_list_grow_st (flist, flist->fl_num_per_alloc);
item = (opal_free_list_item_t *) opal_lifo_pop_atomic (&flist->super);
}
return item;
}
static inline opal_free_list_item_t *opal_free_list_get (opal_free_list_t *flist)
{
if (opal_using_threads ()) {
return opal_free_list_get_mt (flist);
}
return opal_free_list_get_st (flist);
}
/** compatibility macro */
#define OPAL_FREE_LIST_GET(fl, item) \
(item) = opal_free_list_get (fl)
/**
* Blocking call to obtain an item from a free list.
*
* @param fl (IN) Free list.
* @param item (OUT) Allocated item.
* @param rc (OUT) OPAL_SUCCESS or error status on failure.
*
* If the requested item is not available the free list is grown to
* accomodate the request - unless the max number of allocations has
@ -122,22 +239,71 @@ OPAL_DECLSPEC int opal_free_list_grow(opal_free_list_t* flist, size_t num_elemen
* is returned to the list.
*/
#define OPAL_FREE_LIST_WAIT(fl, item, rc) \
do { \
OPAL_THREAD_LOCK(&((fl)->fl_lock)); \
while( NULL == (item = (opal_free_list_item_t*) opal_list_remove_first(&((fl)->super))) ) { \
if( OPAL_LIKELY((fl)->fl_max_to_alloc <= (fl)->fl_num_allocated) ) { \
(fl)->fl_num_waiting++; \
opal_condition_wait(&((fl)->fl_condition), &((fl)->fl_lock)); \
(fl)->fl_num_waiting--; \
} else { \
opal_free_list_grow((fl), (fl)->fl_num_per_alloc); \
} \
} \
OPAL_THREAD_UNLOCK(&((fl)->fl_lock)); \
rc = OPAL_SUCCESS; \
} while(0)
/** compatibility macro */
#define OPAL_FREE_LIST_WAIT(fl, item) \
(item) = opal_free_list_wait (fl)
static inline opal_free_list_item_t *opal_free_list_wait_mt (opal_free_list_t *fl)
{
opal_free_list_item_t *item =
(opal_free_list_item_t *) opal_lifo_pop (&fl->super);
while (NULL == item) {
if (!opal_mutex_trylock (&fl->fl_lock)) {
if (fl->fl_max_to_alloc <= fl->fl_num_allocated ||
OPAL_SUCCESS != opal_free_list_grow_st (fl, fl->fl_num_per_alloc)) {
fl->fl_num_waiting++;
opal_condition_wait (&fl->fl_condition, &fl->fl_lock);
fl->fl_num_waiting--;
} else {
if (0 < fl->fl_num_waiting) {
if (1 == fl->fl_num_waiting) {
opal_condition_signal (&fl->fl_condition);
} else {
opal_condition_broadcast (&fl->fl_condition);
}
}
}
} else {
/* If I wasn't able to get the lock in the begining when I finaly grab it
* the one holding the lock in the begining already grow the list. I will
* release the lock and try to get a new element until I succeed.
*/
opal_mutex_lock (&fl->fl_lock);
}
opal_mutex_unlock (&fl->fl_lock);
item = (opal_free_list_item_t *) opal_lifo_pop (&fl->super);
}
return item;
}
static inline opal_free_list_item_t *opal_free_list_wait_st (opal_free_list_t *fl)
{
opal_free_list_item_t *item =
(opal_free_list_item_t *) opal_lifo_pop (&fl->super);
while (NULL == item) {
if (fl->fl_max_to_alloc <= fl->fl_num_allocated ||
OPAL_SUCCESS != opal_free_list_grow_st (fl, fl->fl_num_per_alloc)) {
/* try to make progress */
opal_progress ();
}
item = (opal_free_list_item_t *) opal_lifo_pop (&fl->super);
}
return item;
}
static inline opal_free_list_item_t *opal_free_list_wait (opal_free_list_t *fl)
{
if (opal_using_threads ()) {
return opal_free_list_wait_mt (fl);
} else {
return opal_free_list_wait_st (fl);
}
}
/**
* Return an item to a free list.
@ -146,18 +312,54 @@ OPAL_DECLSPEC int opal_free_list_grow(opal_free_list_t* flist, size_t num_elemen
* @param item (OUT) Allocated item.
*
*/
#define OPAL_FREE_LIST_RETURN(fl, item) \
do { \
OPAL_THREAD_LOCK(&(fl)->fl_lock); \
opal_list_prepend(&((fl)->super), ((opal_list_item_t*) item)); \
if( OPAL_UNLIKELY((fl)->fl_num_waiting > 0) ) { \
opal_condition_signal(&((fl)->fl_condition)); \
} \
OPAL_THREAD_UNLOCK(&(fl)->fl_lock); \
} while(0)
static inline void opal_free_list_return_mt (opal_free_list_t *flist,
opal_free_list_item_t *item)
{
opal_list_item_t* original;
original = opal_lifo_push_atomic (&flist->super, &item->super);
if (&flist->super.opal_lifo_ghost == original) {
if (flist->fl_num_waiting > 0) {
/* one one item is being returned so it doesn't make sense to wake
* more than a single waiting thread. additionally, posix thread
* semantics do not require that the lock be held to signal the
* condition variable. */
opal_condition_signal (&flist->fl_condition);
}
}
}
static inline void opal_free_list_return_st (opal_free_list_t *flist,
opal_free_list_item_t *item)
{
opal_list_item_t* original;
original = opal_lifo_push_st (&flist->super, &item->super);
if (&flist->super.opal_lifo_ghost == original) {
if (flist->fl_num_waiting > 0) {
/* one one item is being returned so it doesn't make sense to wake
* more than a single waiting thread. additionally, posix thread
* semantics do not require that the lock be held to signal the
* condition variable. */
opal_condition_signal (&flist->fl_condition);
}
}
}
static inline void opal_free_list_return (opal_free_list_t *flist,
opal_free_list_item_t *item)
{
if (opal_using_threads ()) {
opal_free_list_return_mt (flist, item);
} else {
opal_free_list_return_st (flist, item);
}
}
/** compatibility macro */
#define OPAL_FREE_LIST_RETURN(fl, item) \
opal_free_list_return (fl, item)
END_C_DECLS
#endif