366 строки
13 KiB
C
366 строки
13 KiB
C
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
|
|
/*
|
|
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
|
* University Research and Technology
|
|
* Corporation. All rights reserved.
|
|
* Copyright (c) 2004-2013 The University of Tennessee and The University
|
|
* of Tennessee Research Foundation. All rights
|
|
* reserved.
|
|
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
|
* University of Stuttgart. All rights reserved.
|
|
* Copyright (c) 2004-2005 The Regents of the University of California.
|
|
* All rights reserved.
|
|
* Copyright (c) 2010 IBM Corporation. All rights reserved.
|
|
* Copyright (c) 2010-2015 Cisco Systems, Inc. All rights reserved.
|
|
* Copyright (c) 2014-2015 Los Alamos National Security, LLC. All rights
|
|
* reserved.
|
|
* $COPYRIGHT$
|
|
*
|
|
* Additional copyrights may follow
|
|
*
|
|
* $HEADER$
|
|
*/
|
|
|
|
#ifndef OPAL_FREE_LIST_H
|
|
#define OPAL_FREE_LIST_H
|
|
|
|
#include "opal_config.h"
|
|
#include "opal/class/opal_lifo.h"
|
|
#include "opal/prefetch.h"
|
|
#include "opal/threads/condition.h"
|
|
#include "opal/constants.h"
|
|
#include "opal/runtime/opal.h"
|
|
|
|
BEGIN_C_DECLS
|
|
|
|
struct mca_mem_pool_t;
|
|
struct opal_free_list_item_t;
|
|
|
|
/**
|
|
* Free list item initializtion function.
|
|
*
|
|
* @param item (IN) Free list item to initialize
|
|
* @param ctx (IN) Free list initialization context
|
|
*
|
|
* @returns OPAL_SUCCESS on success
|
|
* @returns opal error code on failure
|
|
*
|
|
* This function attempts to initialize the free list item
|
|
* specified in item. If the item can be initialized the
|
|
* function should return OPAL_SUCCESS. On any error
|
|
* opal_free_list_grow will stop initializing new items.
|
|
*/
|
|
typedef int (*opal_free_list_item_init_fn_t) (
|
|
struct opal_free_list_item_t *item, void *ctx);
|
|
|
|
struct opal_free_list_t {
|
|
/** Items in a free list are stored last-in first-out */
|
|
opal_lifo_t super;
|
|
/** Maximum number of items to allocate in the free list */
|
|
size_t fl_max_to_alloc;
|
|
/** Current number of items allocated */
|
|
size_t fl_num_allocated;
|
|
/** Number of items to allocate when growing the free list */
|
|
size_t fl_num_per_alloc;
|
|
/** Number of threads waiting on free list item availability */
|
|
size_t fl_num_waiting;
|
|
/** Size of each free list item */
|
|
size_t fl_frag_size;
|
|
/** Free list item alignment */
|
|
size_t fl_frag_alignment;
|
|
/** Free list item buffer size */
|
|
size_t fl_payload_buffer_size;
|
|
/** Free list item buffer alignment */
|
|
size_t fl_payload_buffer_alignment;
|
|
/** Class of free list items */
|
|
opal_class_t *fl_frag_class;
|
|
/** mpool to use for free list buffer allocation (posix_memalign/malloc
|
|
* are used if this is NULL) */
|
|
struct mca_mpool_base_module_t *fl_mpool;
|
|
/** Multi-threaded lock. Used when the free list is empty. */
|
|
opal_mutex_t fl_lock;
|
|
/** Multi-threaded condition. Used when threads are waiting on free
|
|
* list item availability. */
|
|
opal_condition_t fl_condition;
|
|
/** List of free list allocation */
|
|
opal_list_t fl_allocations;
|
|
/** Flags to pass to the mpool register function */
|
|
int fl_mpool_reg_flags;
|
|
/** Free list item initialization function */
|
|
opal_free_list_item_init_fn_t item_init;
|
|
/** Initialization function context */
|
|
void *ctx;
|
|
};
|
|
typedef struct opal_free_list_t opal_free_list_t;
|
|
OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_free_list_t);
|
|
|
|
struct mca_mpool_base_registration_t;
|
|
struct opal_free_list_item_t
|
|
{
|
|
opal_list_item_t super;
|
|
struct mca_mpool_base_registration_t *registration;
|
|
void *ptr;
|
|
};
|
|
typedef struct opal_free_list_item_t opal_free_list_item_t;
|
|
OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_free_list_item_t);
|
|
|
|
|
|
/**
|
|
* Initialize a free list.
|
|
*
|
|
* @param free_list (IN) Free list.
|
|
* @param frag_size (IN) Size of each element - allocated by malloc.
|
|
* @param frag_alignment (IN) Fragment alignment.
|
|
* @param frag_class (IN) opal_class_t of element - used to initialize allocated elements.
|
|
* @param payload_buffer_size (IN) Size of payload buffer - allocated from mpool.
|
|
* @param payload_buffer_alignment (IN) Payload buffer alignment.
|
|
* @param num_elements_to_alloc (IN) Initial number of elements to allocate.
|
|
* @param max_elements_to_alloc (IN) Maximum number of elements to allocate.
|
|
* @param num_elements_per_alloc (IN) Number of elements to grow by per allocation.
|
|
* @param mpool (IN) Optional memory pool for allocations.
|
|
* @param mpool_reg_flags (IN) Flags to pass to mpool registration function.
|
|
* @param unused0 (IN) Future. Must be NULL.
|
|
* @param item_init (IN) Optional item initialization function
|
|
* @param ctx (IN) Initialization function context.
|
|
*/
|
|
|
|
OPAL_DECLSPEC int opal_free_list_init (opal_free_list_t *free_list,
|
|
size_t frag_size,
|
|
size_t frag_alignment,
|
|
opal_class_t* frag_class,
|
|
size_t payload_buffer_size,
|
|
size_t payload_buffer_alignment,
|
|
int num_elements_to_alloc,
|
|
int max_elements_to_alloc,
|
|
int num_elements_per_alloc,
|
|
struct mca_mpool_base_module_t *mpool,
|
|
int mpool_reg_flags,
|
|
void *unused0,
|
|
opal_free_list_item_init_fn_t item_init,
|
|
void *ctx);
|
|
|
|
/**
|
|
* Grow the free list by at most num_elements elements.
|
|
*
|
|
* @param flist (IN) Free list to grow
|
|
* @param num_elements (IN) Number of elements to add
|
|
*
|
|
* @returns OPAL_SUCCESS if any elements were added
|
|
* @returns OPAL_ERR_OUT_OF_RESOURCE if no elements could be added
|
|
*
|
|
* This function will attempt to grow the free list by num_elements items. The
|
|
* caller must hold the free list lock if calling this function on a free list
|
|
* that may be accessed by multiple threads simultaneously. Note: this is an
|
|
* internal function that will be used when needed by opal_free_list_get* and
|
|
* opal_free_list_wait*.
|
|
*/
|
|
OPAL_DECLSPEC int opal_free_list_grow_st (opal_free_list_t *flist, size_t num_elements);
|
|
|
|
/**
|
|
* Grow the free list to be at least size elements.
|
|
*
|
|
* @param flist (IN) Free list to resize.
|
|
* @param size (IN) New size
|
|
*
|
|
* @returns OPAL_SUCCESS if the free list was resized
|
|
* @returns OPAL_ERR_OUT_OF_RESOURCE if resources could not be allocated
|
|
*
|
|
* This function will not shrink the list if it is already larger than size
|
|
* and may grow it past size if necessary (it will grow in num_elements_per_alloc
|
|
* chunks). This function is thread-safe and will obtain the free list lock before
|
|
* growing the free list.
|
|
*/
|
|
OPAL_DECLSPEC int opal_free_list_resize_mt (opal_free_list_t *flist, size_t size);
|
|
|
|
|
|
/**
|
|
* Attemp to obtain an item from a free list.
|
|
*
|
|
* @param fl (IN) Free list.
|
|
* @param item (OUT) Allocated item.
|
|
*
|
|
* If the requested item is not available the free list is grown to
|
|
* accomodate the request - unless the max number of allocations has
|
|
* been reached. If this is the case - a NULL pointer is returned
|
|
* to the caller. This function comes in three flavor: thread safe
|
|
* (opal_free_list_get_mt), single threaded (opal_free_list_get_st),
|
|
* and opal_using_threads conditioned (opal_free_list_get).
|
|
*/
|
|
static inline opal_free_list_item_t *opal_free_list_get_mt (opal_free_list_t *flist)
|
|
{
|
|
opal_free_list_item_t *item =
|
|
(opal_free_list_item_t*) opal_lifo_pop_atomic (&flist->super);
|
|
|
|
if (OPAL_UNLIKELY(NULL == item)) {
|
|
opal_mutex_lock (&flist->fl_lock);
|
|
opal_free_list_grow_st (flist, flist->fl_num_per_alloc);
|
|
opal_mutex_unlock (&flist->fl_lock);
|
|
item = (opal_free_list_item_t *) opal_lifo_pop_atomic (&flist->super);
|
|
}
|
|
|
|
return item;
|
|
}
|
|
|
|
static inline opal_free_list_item_t *opal_free_list_get_st (opal_free_list_t *flist)
|
|
{
|
|
opal_free_list_item_t *item =
|
|
(opal_free_list_item_t*) opal_lifo_pop_st (&flist->super);
|
|
|
|
if (OPAL_UNLIKELY(NULL == item)) {
|
|
opal_free_list_grow_st (flist, flist->fl_num_per_alloc);
|
|
item = (opal_free_list_item_t *) opal_lifo_pop_atomic (&flist->super);
|
|
}
|
|
|
|
return item;
|
|
}
|
|
|
|
static inline opal_free_list_item_t *opal_free_list_get (opal_free_list_t *flist)
|
|
{
|
|
if (opal_using_threads ()) {
|
|
return opal_free_list_get_mt (flist);
|
|
}
|
|
|
|
return opal_free_list_get_st (flist);
|
|
}
|
|
|
|
/** compatibility macro */
|
|
#define OPAL_FREE_LIST_GET(fl, item) \
|
|
(item) = opal_free_list_get (fl)
|
|
|
|
/**
|
|
* Blocking call to obtain an item from a free list.
|
|
*
|
|
* @param fl (IN) Free list.
|
|
* @param item (OUT) Allocated item.
|
|
*
|
|
* If the requested item is not available the free list is grown to
|
|
* accomodate the request - unless the max number of allocations has
|
|
* been reached. In this case the caller is blocked until an item
|
|
* is returned to the list.
|
|
*/
|
|
|
|
/** compatibility macro */
|
|
#define OPAL_FREE_LIST_WAIT(fl, item) \
|
|
(item) = opal_free_list_wait (fl)
|
|
|
|
static inline opal_free_list_item_t *opal_free_list_wait_mt (opal_free_list_t *fl)
|
|
{
|
|
opal_free_list_item_t *item =
|
|
(opal_free_list_item_t *) opal_lifo_pop (&fl->super);
|
|
|
|
while (NULL == item) {
|
|
if (!opal_mutex_trylock (&fl->fl_lock)) {
|
|
if (fl->fl_max_to_alloc <= fl->fl_num_allocated ||
|
|
OPAL_SUCCESS != opal_free_list_grow_st (fl, fl->fl_num_per_alloc)) {
|
|
fl->fl_num_waiting++;
|
|
opal_condition_wait (&fl->fl_condition, &fl->fl_lock);
|
|
fl->fl_num_waiting--;
|
|
} else {
|
|
if (0 < fl->fl_num_waiting) {
|
|
if (1 == fl->fl_num_waiting) {
|
|
opal_condition_signal (&fl->fl_condition);
|
|
} else {
|
|
opal_condition_broadcast (&fl->fl_condition);
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
/* If I wasn't able to get the lock in the begining when I finaly grab it
|
|
* the one holding the lock in the begining already grow the list. I will
|
|
* release the lock and try to get a new element until I succeed.
|
|
*/
|
|
opal_mutex_lock (&fl->fl_lock);
|
|
}
|
|
opal_mutex_unlock (&fl->fl_lock);
|
|
item = (opal_free_list_item_t *) opal_lifo_pop (&fl->super);
|
|
}
|
|
|
|
return item;
|
|
}
|
|
|
|
static inline opal_free_list_item_t *opal_free_list_wait_st (opal_free_list_t *fl)
|
|
{
|
|
opal_free_list_item_t *item =
|
|
(opal_free_list_item_t *) opal_lifo_pop (&fl->super);
|
|
|
|
while (NULL == item) {
|
|
if (fl->fl_max_to_alloc <= fl->fl_num_allocated ||
|
|
OPAL_SUCCESS != opal_free_list_grow_st (fl, fl->fl_num_per_alloc)) {
|
|
/* try to make progress */
|
|
opal_progress ();
|
|
}
|
|
|
|
item = (opal_free_list_item_t *) opal_lifo_pop (&fl->super);
|
|
}
|
|
|
|
return item;
|
|
}
|
|
|
|
static inline opal_free_list_item_t *opal_free_list_wait (opal_free_list_t *fl)
|
|
{
|
|
if (opal_using_threads ()) {
|
|
return opal_free_list_wait_mt (fl);
|
|
} else {
|
|
return opal_free_list_wait_st (fl);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Return an item to a free list.
|
|
*
|
|
* @param fl (IN) Free list.
|
|
* @param item (OUT) Allocated item.
|
|
*
|
|
*/
|
|
static inline void opal_free_list_return_mt (opal_free_list_t *flist,
|
|
opal_free_list_item_t *item)
|
|
{
|
|
opal_list_item_t* original;
|
|
|
|
original = opal_lifo_push_atomic (&flist->super, &item->super);
|
|
if (&flist->super.opal_lifo_ghost == original) {
|
|
if (flist->fl_num_waiting > 0) {
|
|
/* one one item is being returned so it doesn't make sense to wake
|
|
* more than a single waiting thread. additionally, posix thread
|
|
* semantics do not require that the lock be held to signal the
|
|
* condition variable. */
|
|
opal_condition_signal (&flist->fl_condition);
|
|
}
|
|
}
|
|
}
|
|
|
|
static inline void opal_free_list_return_st (opal_free_list_t *flist,
|
|
opal_free_list_item_t *item)
|
|
{
|
|
opal_list_item_t* original;
|
|
|
|
original = opal_lifo_push_st (&flist->super, &item->super);
|
|
if (&flist->super.opal_lifo_ghost == original) {
|
|
if (flist->fl_num_waiting > 0) {
|
|
/* one one item is being returned so it doesn't make sense to wake
|
|
* more than a single waiting thread. additionally, posix thread
|
|
* semantics do not require that the lock be held to signal the
|
|
* condition variable. */
|
|
opal_condition_signal (&flist->fl_condition);
|
|
}
|
|
}
|
|
}
|
|
|
|
static inline void opal_free_list_return (opal_free_list_t *flist,
|
|
opal_free_list_item_t *item)
|
|
{
|
|
if (opal_using_threads ()) {
|
|
opal_free_list_return_mt (flist, item);
|
|
} else {
|
|
opal_free_list_return_st (flist, item);
|
|
}
|
|
}
|
|
|
|
/** compatibility macro */
|
|
#define OPAL_FREE_LIST_RETURN(fl, item) \
|
|
opal_free_list_return (fl, item)
|
|
|
|
END_C_DECLS
|
|
#endif
|
|
|