2004-08-13 03:31:15 +04:00
|
|
|
/*
|
2004-11-22 04:38:40 +03:00
|
|
|
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
|
|
|
* All rights reserved.
|
|
|
|
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
|
|
|
* All rights reserved.
|
2004-11-28 23:09:25 +03:00
|
|
|
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
|
|
|
* University of Stuttgart. All rights reserved.
|
2005-03-24 15:43:37 +03:00
|
|
|
* Copyright (c) 2004-2005 The Regents of the University of California.
|
|
|
|
* All rights reserved.
|
2004-11-22 04:38:40 +03:00
|
|
|
* $COPYRIGHT$
|
|
|
|
*
|
|
|
|
* Additional copyrights may follow
|
|
|
|
*
|
2004-08-13 03:31:15 +04:00
|
|
|
* $HEADER$
|
|
|
|
*/
|
|
|
|
|
|
|
|
#ifndef _OMPI_CIRCULAR_BUFFER_FIFO
|
|
|
|
#define _OMPI_CIRCULAR_BUFFER_FIFO
|
|
|
|
|
2005-08-13 01:42:07 +04:00
|
|
|
#include "ompi/include/constants.h"
|
2004-08-13 03:31:15 +04:00
|
|
|
#include "include/sys/cache.h"
|
2004-08-19 03:24:27 +04:00
|
|
|
#include "include/sys/atomic.h"
|
2004-08-13 23:49:13 +04:00
|
|
|
#include "mca/mpool/mpool.h"
|
2005-07-04 05:59:52 +04:00
|
|
|
#include "opal/util/pow2.h"
|
2004-08-13 03:31:15 +04:00
|
|
|
|
|
|
|
|
|
|
|
/** @file
|
|
|
|
*
|
|
|
|
* This defines a set of functions to create, and manipulate a FIFO
|
|
|
|
* set up in a circular buffer. FIFO elements are assumed to be
|
|
|
|
* pointers. Pointers are written to the head, and read from the
|
2004-08-18 01:49:01 +04:00
|
|
|
* tail. For thread safety, a spin lock is provided in the
|
|
|
|
* ompi_cb_fifo_ctl_t structure, but it's use must be managed by
|
|
|
|
* the calling routines - this is not by these set of routines.
|
2004-08-30 05:25:34 +04:00
|
|
|
* Queues are addressed relative to an offset from the base of
|
|
|
|
* a memory pool, in this way, different processes with different
|
|
|
|
* base addresses can access these queue at the same time.
|
2004-08-13 03:31:15 +04:00
|
|
|
*/
|
|
|
|
|
|
|
|
/* error code */
|
|
|
|
#define OMPI_CB_ERROR -1
|
2004-08-25 02:25:59 +04:00
|
|
|
#define OMPI_CB_FREE (void *)-2
|
|
|
|
#define OMPI_CB_RESERVED (void *)-3
|
2004-08-30 05:25:34 +04:00
|
|
|
#define OMPI_CB_NULL (void *)-4
|
2004-08-13 03:31:15 +04:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Structure used to keep track of the fifo status
|
|
|
|
*/
|
|
|
|
struct ompi_cb_fifo_ctl_t {
|
|
|
|
/* spin-lock for access control */
|
2005-07-04 01:38:51 +04:00
|
|
|
opal_atomic_lock_t lock;
|
2004-08-13 03:31:15 +04:00
|
|
|
|
|
|
|
/* current queue index */
|
2004-10-19 03:24:53 +04:00
|
|
|
volatile int fifo_index;
|
2004-08-13 03:31:15 +04:00
|
|
|
|
|
|
|
/* number of entries that have been used, but not invalidated. used
|
|
|
|
* for lazy resource reclamation */
|
2004-10-19 03:24:53 +04:00
|
|
|
volatile int num_to_clear;
|
2004-08-13 03:31:15 +04:00
|
|
|
|
|
|
|
};
|
|
|
|
typedef struct ompi_cb_fifo_ctl_t ompi_cb_fifo_ctl_t;
|
|
|
|
|
|
|
|
/* data structure used to describe the fifo */
|
|
|
|
struct ompi_cb_fifo_t {
|
|
|
|
|
|
|
|
/* size of fifo */
|
|
|
|
int size;
|
|
|
|
|
2004-08-13 23:49:13 +04:00
|
|
|
/* frequency of lazy free */
|
|
|
|
int lazy_free_frequency;
|
|
|
|
|
2004-08-25 00:17:44 +04:00
|
|
|
/* fifo memory locality index */
|
|
|
|
int fifo_memory_locality_index;
|
|
|
|
|
|
|
|
/* head memory locality index */
|
|
|
|
int head_memory_locality_index;
|
|
|
|
|
|
|
|
/* tail memory locality index */
|
|
|
|
int tail_memory_locality_index;
|
|
|
|
|
2004-08-13 03:31:15 +04:00
|
|
|
/* head of queue - where next entry will be written */
|
|
|
|
ompi_cb_fifo_ctl_t *head;
|
|
|
|
|
|
|
|
/* tail of queue - next element to read */
|
|
|
|
ompi_cb_fifo_ctl_t *tail;
|
|
|
|
|
|
|
|
/* mask - to handle wrap around */
|
|
|
|
unsigned int mask;
|
|
|
|
|
|
|
|
/* circular buffer array */
|
2004-08-25 00:17:44 +04:00
|
|
|
volatile void **queue;
|
2004-08-13 03:31:15 +04:00
|
|
|
|
2004-08-13 23:49:13 +04:00
|
|
|
};
|
|
|
|
|
|
|
|
typedef struct ompi_cb_fifo_t ompi_cb_fifo_t;
|
2004-08-13 03:31:15 +04:00
|
|
|
|
|
|
|
/**
|
|
|
|
* Initialize a fifo
|
|
|
|
*
|
|
|
|
* @param size_of_fifo Length of fifo array (IN)
|
|
|
|
*
|
|
|
|
* @param fifo_memory_locality_index Locality index to apply to
|
|
|
|
* the fifo array. Not currently
|
|
|
|
* in use (IN)
|
|
|
|
*
|
|
|
|
* @param tail_memory_locality_index Locality index to apply to the
|
|
|
|
* head control structure. Not
|
|
|
|
* currently in use (IN)
|
|
|
|
*
|
|
|
|
* @param tail_memory_locality_index Locality index to apply to the
|
|
|
|
* tail control structure. Not
|
|
|
|
* currently in use (IN)
|
|
|
|
*
|
|
|
|
* @param fifo Pointer to data structure defining this fifo (IN)
|
|
|
|
*
|
|
|
|
* @param memory_allocator Pointer to the memory allocator to use
|
|
|
|
* to allocate memory for this fifo. (IN)
|
|
|
|
*
|
|
|
|
* @returncode Error code
|
|
|
|
*
|
|
|
|
*/
|
2004-08-13 23:49:13 +04:00
|
|
|
static inline int ompi_cb_fifo_init(int size_of_fifo, int lazy_free_freq,
|
|
|
|
int fifo_memory_locality_index, int head_memory_locality_index,
|
|
|
|
int tail_memory_locality_index, ompi_cb_fifo_t *fifo,
|
|
|
|
mca_mpool_base_module_t *memory_allocator)
|
2004-08-13 03:31:15 +04:00
|
|
|
{
|
|
|
|
|
2004-08-17 03:06:33 +04:00
|
|
|
int errorCode = OMPI_SUCCESS,i;
|
|
|
|
size_t len_to_allocate;
|
2004-08-13 03:31:15 +04:00
|
|
|
|
|
|
|
/* verify that size is power of 2, and greatter that 0 - if not,
|
|
|
|
* round up */
|
|
|
|
if ( 0 >= size_of_fifo) {
|
|
|
|
return OMPI_ERROR;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* set fifo size */
|
2005-07-04 05:59:52 +04:00
|
|
|
fifo->size = opal_round_up_to_nearest_pow2(size_of_fifo);
|
2004-08-13 23:49:13 +04:00
|
|
|
|
|
|
|
/* set lazy free frequence */
|
|
|
|
if( ( 0 >= lazy_free_freq ) ||
|
|
|
|
( lazy_free_freq > fifo->size) ) {
|
|
|
|
return OMPI_ERROR;
|
|
|
|
}
|
|
|
|
fifo->lazy_free_frequency=lazy_free_freq;
|
2004-08-13 03:31:15 +04:00
|
|
|
|
|
|
|
/* this will be used to mask off the higher order bits,
|
|
|
|
* and use the & operator for the wrap-around */
|
2004-08-17 03:37:07 +04:00
|
|
|
fifo->mask = (fifo->size - 1);
|
2004-08-13 03:31:15 +04:00
|
|
|
|
|
|
|
/* allocate fifo array */
|
2004-08-17 03:37:07 +04:00
|
|
|
len_to_allocate = sizeof(void *) * fifo->size;
|
2005-05-31 21:06:55 +04:00
|
|
|
fifo->queue=memory_allocator->mpool_alloc(memory_allocator, len_to_allocate,CACHE_LINE_SIZE, NULL);
|
2004-08-13 03:31:15 +04:00
|
|
|
if ( NULL == fifo->queue) {
|
2004-08-13 23:49:13 +04:00
|
|
|
return OMPI_ERR_OUT_OF_RESOURCE;
|
2004-08-13 03:31:15 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
/* initialize the queue entries */
|
2004-08-17 03:37:07 +04:00
|
|
|
for (i = 0; i < fifo->size; i++) {
|
2004-08-13 23:49:13 +04:00
|
|
|
fifo->queue[i] = OMPI_CB_FREE;
|
2004-08-13 03:31:15 +04:00
|
|
|
}
|
|
|
|
|
2004-08-30 05:25:34 +04:00
|
|
|
/* change address be relative to the base of the memory segment */
|
|
|
|
fifo->queue=(volatile void **)( (char *)(fifo->queue) -
|
2005-05-31 21:06:55 +04:00
|
|
|
(size_t)(memory_allocator->mpool_base(memory_allocator)));
|
2004-08-30 05:25:34 +04:00
|
|
|
|
2004-08-13 03:31:15 +04:00
|
|
|
/* allocate head control structure */
|
|
|
|
len_to_allocate = sizeof(ompi_cb_fifo_ctl_t);
|
2005-05-31 21:06:55 +04:00
|
|
|
fifo->head=memory_allocator->mpool_alloc(memory_allocator, len_to_allocate,CACHE_LINE_SIZE, NULL);
|
2004-08-13 03:31:15 +04:00
|
|
|
if ( NULL == fifo->head) {
|
2004-08-13 23:49:13 +04:00
|
|
|
return OMPI_ERR_OUT_OF_RESOURCE;
|
2004-08-13 03:31:15 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
/* initialize the head structure */
|
2005-07-04 01:38:51 +04:00
|
|
|
opal_atomic_unlock(&(fifo->head->lock));
|
2004-08-13 23:49:13 +04:00
|
|
|
fifo->head->fifo_index=0;
|
|
|
|
fifo->head->num_to_clear=0;
|
2004-08-13 03:31:15 +04:00
|
|
|
|
|
|
|
/* allocate tail control structure */
|
|
|
|
len_to_allocate = sizeof(ompi_cb_fifo_ctl_t);
|
2005-05-31 21:06:55 +04:00
|
|
|
fifo->tail=memory_allocator->mpool_alloc(memory_allocator, len_to_allocate,CACHE_LINE_SIZE, NULL);
|
2004-08-13 03:31:15 +04:00
|
|
|
if ( NULL == fifo->tail) {
|
2004-08-13 23:49:13 +04:00
|
|
|
return OMPI_ERR_OUT_OF_RESOURCE;
|
2004-08-13 03:31:15 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
/* initialize the head structure */
|
2005-07-04 01:38:51 +04:00
|
|
|
opal_atomic_unlock(&(fifo->tail->lock));
|
2004-08-13 23:49:13 +04:00
|
|
|
fifo->tail->fifo_index=0;
|
|
|
|
fifo->tail->num_to_clear=0;
|
2004-08-13 03:31:15 +04:00
|
|
|
|
2004-08-25 00:17:44 +04:00
|
|
|
/* set memory locality indecies */
|
|
|
|
fifo->fifo_memory_locality_index=fifo_memory_locality_index;
|
|
|
|
fifo->head_memory_locality_index=head_memory_locality_index;
|
|
|
|
fifo->tail_memory_locality_index=tail_memory_locality_index;
|
|
|
|
|
2004-08-30 05:25:34 +04:00
|
|
|
/* change addresses be relative to the base of the memory segment */
|
|
|
|
fifo->head=(ompi_cb_fifo_ctl_t *)( (char *)(fifo->head) -
|
2005-05-31 21:06:55 +04:00
|
|
|
(size_t)(memory_allocator->mpool_base(memory_allocator)));
|
2004-08-30 05:25:34 +04:00
|
|
|
fifo->tail=(ompi_cb_fifo_ctl_t *)( (char *)(fifo->tail) -
|
2005-05-31 21:06:55 +04:00
|
|
|
(size_t)(memory_allocator->mpool_base(memory_allocator)));
|
2004-08-30 05:25:34 +04:00
|
|
|
|
2004-08-13 03:31:15 +04:00
|
|
|
/* return */
|
|
|
|
return errorCode;
|
|
|
|
}
|
|
|
|
|
2004-08-17 03:06:33 +04:00
|
|
|
/**
|
|
|
|
* function to cleanup the fifo
|
|
|
|
*
|
|
|
|
* @param fifo Pointer to data structure defining this fifo (IN)
|
|
|
|
*
|
|
|
|
* @param memory_allocator Pointer to the memory allocator to use
|
|
|
|
* to allocate memory for this fifo. (IN)
|
|
|
|
*
|
|
|
|
*/
|
|
|
|
static inline int ompi_cb_fifo_free( ompi_cb_fifo_t *fifo,
|
|
|
|
mca_mpool_base_module_t *memory_allocator)
|
|
|
|
{
|
|
|
|
|
|
|
|
int errorCode = OMPI_SUCCESS;
|
2004-08-30 05:25:34 +04:00
|
|
|
char *ptr;
|
2004-08-17 03:06:33 +04:00
|
|
|
|
|
|
|
/* make sure null fifo is not passed in */
|
|
|
|
if ( NULL == fifo) {
|
|
|
|
return OMPI_ERROR;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* free fifo array */
|
2005-03-23 20:42:27 +03:00
|
|
|
if( OMPI_CB_NULL != fifo->queue ){
|
2005-05-31 21:06:55 +04:00
|
|
|
ptr=(char *)(fifo->queue)+(size_t)(memory_allocator->mpool_base(memory_allocator));
|
2005-06-22 01:20:23 +04:00
|
|
|
memory_allocator->mpool_free(memory_allocator, ptr, NULL);
|
2004-08-30 05:25:34 +04:00
|
|
|
fifo->queue=OMPI_CB_NULL;
|
2004-08-17 03:06:33 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
/* free head control structure */
|
2004-08-30 05:25:34 +04:00
|
|
|
if( OMPI_CB_NULL != fifo->head) {
|
2005-05-31 21:06:55 +04:00
|
|
|
ptr=(char *)(fifo->head)+(size_t)(memory_allocator->mpool_base(memory_allocator));
|
2005-06-22 01:20:23 +04:00
|
|
|
memory_allocator->mpool_free(memory_allocator, ptr, NULL);
|
2004-08-30 05:25:34 +04:00
|
|
|
fifo->head=OMPI_CB_NULL;
|
2004-08-25 00:17:44 +04:00
|
|
|
|
2004-08-17 03:06:33 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
/* free tail control structure */
|
2004-08-30 05:25:34 +04:00
|
|
|
if( OMPI_CB_NULL != fifo->tail) {
|
2005-05-31 21:06:55 +04:00
|
|
|
ptr=(char *)(fifo->tail)+(size_t)(memory_allocator->mpool_base(memory_allocator));
|
2005-06-22 01:20:23 +04:00
|
|
|
memory_allocator->mpool_free(memory_allocator, ptr, NULL);
|
2004-08-30 05:25:34 +04:00
|
|
|
fifo->tail=OMPI_CB_NULL;
|
2004-08-17 03:06:33 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
/* return */
|
|
|
|
return errorCode;
|
|
|
|
}
|
|
|
|
|
2004-08-13 03:31:15 +04:00
|
|
|
|
|
|
|
/**
|
|
|
|
* Write pointer to the specified slot
|
|
|
|
*
|
|
|
|
* @param slot Slot index (IN)
|
|
|
|
*
|
|
|
|
* @param data Pointer value to write in specified slot (IN)
|
|
|
|
*
|
|
|
|
* @param fifo Pointer to data structure defining this fifo (IN)
|
|
|
|
*
|
|
|
|
* @returncode Slot index to which data is written
|
|
|
|
*
|
|
|
|
*/
|
2004-08-30 05:25:34 +04:00
|
|
|
static inline int ompi_cb_fifo_write_to_slot(int slot, void* data,
|
|
|
|
ompi_cb_fifo_t *fifo, size_t offset)
|
|
|
|
{
|
|
|
|
void **ptr;
|
2004-08-13 03:31:15 +04:00
|
|
|
int wrote_to_slot = OMPI_CB_ERROR;
|
|
|
|
/* make sure that this slot is already reserved */
|
2004-08-30 05:25:34 +04:00
|
|
|
ptr=(void **)( (char *)(fifo->queue) + (size_t)offset);
|
|
|
|
if (ptr[slot] == OMPI_CB_RESERVED ) {
|
|
|
|
ptr[slot] = data;
|
2004-08-13 03:31:15 +04:00
|
|
|
return slot;
|
|
|
|
} else {
|
|
|
|
return wrote_to_slot;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Try to write pointer to the head of the queue
|
|
|
|
*
|
|
|
|
* @param data Pointer value to write in specified slot (IN)
|
|
|
|
*
|
|
|
|
* @param fifo Pointer to data structure defining this fifo (IN)
|
|
|
|
*
|
|
|
|
* @returncode Slot index to which data is written
|
|
|
|
*
|
|
|
|
*/
|
2004-08-30 05:25:34 +04:00
|
|
|
static inline int ompi_cb_fifo_write_to_head(void *data, ompi_cb_fifo_t
|
|
|
|
*fifo, size_t offset)
|
|
|
|
{
|
|
|
|
void **ptr;
|
|
|
|
ompi_cb_fifo_ctl_t *h_ptr;
|
2004-08-13 03:31:15 +04:00
|
|
|
int slot = OMPI_CB_ERROR, index;
|
2004-08-30 05:25:34 +04:00
|
|
|
|
|
|
|
h_ptr=(ompi_cb_fifo_ctl_t *) ((char *)(fifo->head) +
|
|
|
|
(size_t)offset);
|
|
|
|
index = h_ptr->fifo_index;
|
2004-08-13 03:31:15 +04:00
|
|
|
/* make sure the head is pointing at a free element - avoid wrap
|
|
|
|
* around */
|
2004-08-30 05:25:34 +04:00
|
|
|
ptr=(void **)( (char *)(fifo->queue) + (size_t)offset);
|
|
|
|
if (ptr[index] == OMPI_CB_FREE) {
|
2004-08-13 03:31:15 +04:00
|
|
|
slot = index;
|
2004-08-30 05:25:34 +04:00
|
|
|
ptr[slot] = data;
|
|
|
|
(h_ptr->fifo_index)++;
|
|
|
|
(h_ptr->fifo_index) &= fifo->mask;
|
2004-08-13 03:31:15 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
/* return */
|
|
|
|
return slot;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Reserve slot in the fifo array
|
|
|
|
*
|
|
|
|
* @param fifo Pointer to data structure defining this fifo (IN)
|
|
|
|
*
|
|
|
|
* @returncode Slot index to which data is written
|
|
|
|
*
|
|
|
|
* @returncode OMPI_CB_ERROR failed to allocate index
|
|
|
|
*
|
|
|
|
*/
|
2004-08-30 05:25:34 +04:00
|
|
|
static inline int ompi_cb_fifo_get_slot(ompi_cb_fifo_t *fifo,
|
|
|
|
size_t offset)
|
|
|
|
{
|
|
|
|
void **ptr;
|
|
|
|
ompi_cb_fifo_ctl_t *h_ptr;
|
2004-08-13 03:31:15 +04:00
|
|
|
int return_value = OMPI_CB_ERROR,index;
|
|
|
|
|
2004-08-30 05:25:34 +04:00
|
|
|
h_ptr=(ompi_cb_fifo_ctl_t *)( (char *)(fifo->head) + (size_t)offset);
|
|
|
|
ptr=(void **)( (char *)(fifo->queue) + (size_t)offset);
|
|
|
|
index = h_ptr->fifo_index;
|
2004-08-13 03:31:15 +04:00
|
|
|
/* try and reserve slot */
|
2004-08-30 05:25:34 +04:00
|
|
|
if ( OMPI_CB_FREE == ptr[index] ) {
|
|
|
|
ptr[index] = OMPI_CB_RESERVED;
|
2004-08-13 03:31:15 +04:00
|
|
|
return_value = index;
|
2004-08-30 05:25:34 +04:00
|
|
|
(h_ptr->fifo_index)++;
|
|
|
|
(h_ptr->fifo_index) &= fifo->mask;
|
2004-08-13 03:31:15 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
/* return */
|
|
|
|
return return_value;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Try to read pointer from the tail of the queue
|
|
|
|
*
|
2004-08-25 00:17:44 +04:00
|
|
|
* @param data Pointer to where data was be written (OUT)
|
2004-08-13 03:31:15 +04:00
|
|
|
*
|
|
|
|
* @param fifo Pointer to data structure defining this fifo (IN)
|
|
|
|
*
|
2004-08-19 02:53:04 +04:00
|
|
|
* @param flush_entries_read force the lazy free to happen (IN)
|
|
|
|
*
|
2004-08-25 00:17:44 +04:00
|
|
|
* @param queue_empty checks to see if the fifo is empty, but only if
|
|
|
|
* flush_entries_read is set (OUT)
|
|
|
|
*
|
2004-08-13 03:31:15 +04:00
|
|
|
* @returncode Slot index to which data is written
|
|
|
|
*
|
|
|
|
*/
|
2004-08-19 02:53:04 +04:00
|
|
|
static inline void *ompi_cb_fifo_read_from_tail(ompi_cb_fifo_t *fifo,
|
2004-08-30 05:25:34 +04:00
|
|
|
bool flush_entries_read, bool *queue_empty, size_t offset)
|
2004-08-13 03:31:15 +04:00
|
|
|
{
|
2004-08-18 01:49:01 +04:00
|
|
|
int index = 0,clearIndex, i;
|
2004-08-30 05:25:34 +04:00
|
|
|
void **q_ptr;
|
|
|
|
ompi_cb_fifo_ctl_t *h_ptr, *t_ptr;
|
2004-08-18 01:49:01 +04:00
|
|
|
void *read_from_tail = (void *)OMPI_CB_ERROR;
|
2004-08-30 05:25:34 +04:00
|
|
|
|
2004-08-25 00:17:44 +04:00
|
|
|
*queue_empty=false;
|
2004-08-13 03:31:15 +04:00
|
|
|
|
2004-08-30 05:25:34 +04:00
|
|
|
h_ptr=(ompi_cb_fifo_ctl_t *)( (char *)(fifo->head) + (size_t)offset);
|
|
|
|
t_ptr=(ompi_cb_fifo_ctl_t *)( (char *)(fifo->tail) + (size_t)offset);
|
|
|
|
q_ptr=(void **)( (char *)(fifo->queue) + (size_t)offset);
|
|
|
|
|
2004-08-13 03:31:15 +04:00
|
|
|
/* check to see that the data is valid */
|
2004-08-30 05:25:34 +04:00
|
|
|
if ((q_ptr[t_ptr->fifo_index] == OMPI_CB_FREE) ||
|
|
|
|
(q_ptr[t_ptr->fifo_index] == OMPI_CB_RESERVED))
|
2004-08-13 03:31:15 +04:00
|
|
|
{
|
2004-10-12 02:56:46 +04:00
|
|
|
return (void *)OMPI_CB_FREE;
|
2004-08-13 03:31:15 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
/* set return data */
|
2004-08-30 05:25:34 +04:00
|
|
|
index = t_ptr->fifo_index;
|
|
|
|
read_from_tail = (void *)q_ptr[index];
|
|
|
|
t_ptr->num_to_clear++;
|
2004-08-13 03:31:15 +04:00
|
|
|
|
2004-08-27 20:32:09 +04:00
|
|
|
/* increment counter for later lazy free */
|
2004-08-30 05:25:34 +04:00
|
|
|
(t_ptr->fifo_index)++;
|
|
|
|
(t_ptr->fifo_index) &= fifo->mask;
|
2004-08-27 20:32:09 +04:00
|
|
|
|
2004-08-13 03:31:15 +04:00
|
|
|
/* check to see if time to do a lazy free of queue slots */
|
2004-08-30 05:25:34 +04:00
|
|
|
if ( (t_ptr->num_to_clear == fifo->lazy_free_frequency) ||
|
2004-08-19 02:53:04 +04:00
|
|
|
flush_entries_read ) {
|
2004-08-30 05:25:34 +04:00
|
|
|
clearIndex = index - t_ptr->num_to_clear + 1;
|
2004-08-13 03:31:15 +04:00
|
|
|
clearIndex &= fifo->mask;
|
|
|
|
|
2004-08-30 05:25:34 +04:00
|
|
|
for (i = 0; i < t_ptr->num_to_clear; i++) {
|
|
|
|
q_ptr[clearIndex] = OMPI_CB_FREE;
|
2004-08-13 03:31:15 +04:00
|
|
|
clearIndex++;
|
2004-08-13 23:49:13 +04:00
|
|
|
clearIndex &= fifo->mask;
|
2004-08-13 03:31:15 +04:00
|
|
|
}
|
2004-08-30 05:25:34 +04:00
|
|
|
t_ptr->num_to_clear = 0;
|
2004-08-25 00:17:44 +04:00
|
|
|
|
|
|
|
/* check to see if queue is empty */
|
|
|
|
if( flush_entries_read &&
|
2004-08-30 05:25:34 +04:00
|
|
|
(t_ptr->fifo_index == h_ptr->fifo_index) ) {
|
2004-08-25 00:17:44 +04:00
|
|
|
*queue_empty=true;
|
|
|
|
}
|
2004-08-13 03:31:15 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
/* return */
|
|
|
|
return read_from_tail;
|
|
|
|
}
|
2004-08-17 03:37:07 +04:00
|
|
|
|
|
|
|
/**
|
|
|
|
* Return the fifo size
|
|
|
|
*
|
|
|
|
* @param fifo Pointer to data structure defining this fifo (IN)
|
|
|
|
*
|
|
|
|
* @returncode fifo size
|
|
|
|
*
|
|
|
|
*/
|
|
|
|
static inline int ompi_cb_fifo_size(ompi_cb_fifo_t *fifo) {
|
|
|
|
|
|
|
|
return fifo->size;
|
|
|
|
}
|
|
|
|
|
2004-11-07 01:00:24 +03:00
|
|
|
/**
|
|
|
|
* Initialize a fifo
|
|
|
|
*
|
|
|
|
* @param size_of_fifo Length of fifo array (IN)
|
|
|
|
*
|
|
|
|
* @param fifo_memory_locality_index Locality index to apply to
|
|
|
|
* the fifo array. Not currently
|
|
|
|
* in use (IN)
|
|
|
|
*
|
|
|
|
* @param tail_memory_locality_index Locality index to apply to the
|
|
|
|
* head control structure. Not
|
|
|
|
* currently in use (IN)
|
|
|
|
*
|
|
|
|
* @param tail_memory_locality_index Locality index to apply to the
|
|
|
|
* tail control structure. Not
|
|
|
|
* currently in use (IN)
|
|
|
|
*
|
|
|
|
* @param fifo Pointer to data structure defining this fifo (IN)
|
|
|
|
*
|
|
|
|
* @param memory_allocator Pointer to the memory allocator to use
|
|
|
|
* to allocate memory for this fifo. (IN)
|
|
|
|
*
|
|
|
|
* @returncode Error code
|
|
|
|
*
|
|
|
|
*/
|
|
|
|
static inline int ompi_cb_fifo_init_same_base_addr(int size_of_fifo,
|
|
|
|
int lazy_free_freq, int fifo_memory_locality_index,
|
|
|
|
int head_memory_locality_index, int tail_memory_locality_index,
|
|
|
|
ompi_cb_fifo_t *fifo, mca_mpool_base_module_t *memory_allocator)
|
|
|
|
{
|
|
|
|
|
|
|
|
int errorCode = OMPI_SUCCESS,i;
|
|
|
|
size_t len_to_allocate;
|
|
|
|
|
|
|
|
/* verify that size is power of 2, and greatter that 0 - if not,
|
|
|
|
* round up */
|
|
|
|
if ( 0 >= size_of_fifo) {
|
|
|
|
return OMPI_ERROR;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* set fifo size */
|
2005-07-04 05:59:52 +04:00
|
|
|
fifo->size = opal_round_up_to_nearest_pow2(size_of_fifo);
|
2004-11-07 01:00:24 +03:00
|
|
|
|
|
|
|
/* set lazy free frequence */
|
|
|
|
if( ( 0 >= lazy_free_freq ) ||
|
|
|
|
( lazy_free_freq > fifo->size) ) {
|
|
|
|
return OMPI_ERROR;
|
|
|
|
}
|
|
|
|
fifo->lazy_free_frequency=lazy_free_freq;
|
|
|
|
|
|
|
|
/* this will be used to mask off the higher order bits,
|
|
|
|
* and use the & operator for the wrap-around */
|
|
|
|
fifo->mask = (fifo->size - 1);
|
|
|
|
|
|
|
|
/* allocate fifo array */
|
|
|
|
len_to_allocate = sizeof(void *) * fifo->size;
|
2005-05-31 21:06:55 +04:00
|
|
|
fifo->queue=memory_allocator->mpool_alloc(memory_allocator, len_to_allocate,CACHE_LINE_SIZE, NULL);
|
2004-11-07 01:00:24 +03:00
|
|
|
if ( NULL == fifo->queue) {
|
|
|
|
return OMPI_ERR_OUT_OF_RESOURCE;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* initialize the queue entries */
|
|
|
|
for (i = 0; i < fifo->size; i++) {
|
|
|
|
fifo->queue[i] = OMPI_CB_FREE;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* allocate head control structure */
|
|
|
|
len_to_allocate = sizeof(ompi_cb_fifo_ctl_t);
|
2005-05-31 21:06:55 +04:00
|
|
|
fifo->head=memory_allocator->mpool_alloc(memory_allocator, len_to_allocate,CACHE_LINE_SIZE, NULL);
|
2004-11-07 01:00:24 +03:00
|
|
|
if ( NULL == fifo->head) {
|
|
|
|
return OMPI_ERR_OUT_OF_RESOURCE;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* initialize the head structure */
|
2005-07-04 01:38:51 +04:00
|
|
|
opal_atomic_unlock(&(fifo->head->lock));
|
2004-11-07 01:00:24 +03:00
|
|
|
fifo->head->fifo_index=0;
|
|
|
|
fifo->head->num_to_clear=0;
|
|
|
|
|
|
|
|
/* allocate tail control structure */
|
|
|
|
len_to_allocate = sizeof(ompi_cb_fifo_ctl_t);
|
2005-05-31 21:06:55 +04:00
|
|
|
fifo->tail=memory_allocator->mpool_alloc(memory_allocator, len_to_allocate,CACHE_LINE_SIZE, NULL);
|
2004-11-07 01:00:24 +03:00
|
|
|
if ( NULL == fifo->tail) {
|
|
|
|
return OMPI_ERR_OUT_OF_RESOURCE;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* initialize the head structure */
|
2005-07-04 01:38:51 +04:00
|
|
|
opal_atomic_unlock(&(fifo->tail->lock));
|
2004-11-07 01:00:24 +03:00
|
|
|
fifo->tail->fifo_index=0;
|
|
|
|
fifo->tail->num_to_clear=0;
|
|
|
|
|
|
|
|
/* set memory locality indecies */
|
|
|
|
fifo->fifo_memory_locality_index=fifo_memory_locality_index;
|
|
|
|
fifo->head_memory_locality_index=head_memory_locality_index;
|
|
|
|
fifo->tail_memory_locality_index=tail_memory_locality_index;
|
|
|
|
|
|
|
|
/* return */
|
|
|
|
return errorCode;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* function to cleanup the fifo
|
|
|
|
*
|
|
|
|
* @param fifo Pointer to data structure defining this fifo (IN)
|
|
|
|
*
|
|
|
|
* @param memory_allocator Pointer to the memory allocator to use
|
|
|
|
* to allocate memory for this fifo. (IN)
|
|
|
|
*
|
|
|
|
*/
|
|
|
|
static inline int ompi_cb_fifo_free_same_base_addr( ompi_cb_fifo_t *fifo,
|
|
|
|
mca_mpool_base_module_t *memory_allocator)
|
|
|
|
{
|
|
|
|
|
|
|
|
int errorCode = OMPI_SUCCESS;
|
|
|
|
char *ptr;
|
|
|
|
|
|
|
|
/* make sure null fifo is not passed in */
|
|
|
|
if ( NULL == fifo) {
|
|
|
|
return OMPI_ERROR;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* free fifo array */
|
2005-03-23 21:45:26 +03:00
|
|
|
if( OMPI_CB_NULL != fifo->head ){
|
2004-11-07 01:00:24 +03:00
|
|
|
ptr=(char *)(fifo->queue);
|
2005-06-22 01:20:23 +04:00
|
|
|
memory_allocator->mpool_free(memory_allocator, ptr, NULL);
|
2004-11-07 01:00:24 +03:00
|
|
|
fifo->queue=OMPI_CB_NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* free head control structure */
|
|
|
|
if( OMPI_CB_NULL != fifo->head) {
|
|
|
|
ptr=(char *)(fifo->head);
|
2005-06-22 01:20:23 +04:00
|
|
|
memory_allocator->mpool_free(memory_allocator, ptr, NULL);
|
2004-11-07 01:00:24 +03:00
|
|
|
fifo->head=OMPI_CB_NULL;
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
/* free tail control structure */
|
|
|
|
if( OMPI_CB_NULL != fifo->tail) {
|
|
|
|
ptr=(char *)(fifo->tail);
|
2005-06-22 01:20:23 +04:00
|
|
|
memory_allocator->mpool_free(memory_allocator, ptr, NULL);
|
2004-11-07 01:00:24 +03:00
|
|
|
fifo->tail=OMPI_CB_NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* return */
|
|
|
|
return errorCode;
|
|
|
|
}
|
|
|
|
/**
|
|
|
|
* Write pointer to the specified slot
|
|
|
|
*
|
|
|
|
* @param slot Slot index (IN)
|
|
|
|
*
|
|
|
|
* @param data Pointer value to write in specified slot (IN)
|
|
|
|
*
|
|
|
|
* @param fifo Pointer to data structure defining this fifo (IN)
|
|
|
|
*
|
|
|
|
* @returncode Slot index to which data is written
|
|
|
|
*
|
|
|
|
*/
|
|
|
|
static inline int ompi_cb_fifo_write_to_slot_same_base_addr(int slot, void* data,
|
|
|
|
ompi_cb_fifo_t *fifo)
|
|
|
|
{
|
|
|
|
volatile void **ptr;
|
|
|
|
int wrote_to_slot = OMPI_CB_ERROR;
|
|
|
|
/* make sure that this slot is already reserved */
|
|
|
|
ptr=fifo->queue;
|
|
|
|
if (ptr[slot] == OMPI_CB_RESERVED ) {
|
|
|
|
ptr[slot] = data;
|
|
|
|
return slot;
|
|
|
|
} else {
|
|
|
|
return wrote_to_slot;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Try to write pointer to the head of the queue
|
|
|
|
*
|
|
|
|
* @param data Pointer value to write in specified slot (IN)
|
|
|
|
*
|
|
|
|
* @param fifo Pointer to data structure defining this fifo (IN)
|
|
|
|
*
|
|
|
|
* @returncode Slot index to which data is written
|
|
|
|
*
|
|
|
|
*/
|
|
|
|
static inline int ompi_cb_fifo_write_to_head_same_base_addr(void *data, ompi_cb_fifo_t *fifo)
|
|
|
|
{
|
|
|
|
volatile void **ptr;
|
|
|
|
ompi_cb_fifo_ctl_t *h_ptr;
|
|
|
|
int slot = OMPI_CB_ERROR, index;
|
|
|
|
|
|
|
|
h_ptr=fifo->head;
|
|
|
|
index = h_ptr->fifo_index;
|
|
|
|
/* make sure the head is pointing at a free element */
|
|
|
|
ptr=fifo->queue;
|
|
|
|
if (ptr[index] == OMPI_CB_FREE) {
|
|
|
|
slot = index;
|
|
|
|
ptr[slot] = data;
|
|
|
|
(h_ptr->fifo_index)++;
|
|
|
|
/* wrap around */
|
|
|
|
(h_ptr->fifo_index) &= fifo->mask;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* return */
|
|
|
|
return slot;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Reserve slot in the fifo array
|
|
|
|
*
|
|
|
|
* @param fifo Pointer to data structure defining this fifo (IN)
|
|
|
|
*
|
|
|
|
* @returncode Slot index to which data is written
|
|
|
|
*
|
|
|
|
* @returncode OMPI_CB_ERROR failed to allocate index
|
|
|
|
*
|
|
|
|
*/
|
|
|
|
static inline int ompi_cb_fifo_get_slot_same_base_addr(ompi_cb_fifo_t *fifo)
|
|
|
|
{
|
|
|
|
volatile void **ptr;
|
|
|
|
ompi_cb_fifo_ctl_t *h_ptr;
|
|
|
|
int return_value = OMPI_CB_ERROR,index;
|
|
|
|
|
|
|
|
h_ptr=fifo->head;
|
|
|
|
ptr=fifo->queue;
|
|
|
|
index = h_ptr->fifo_index;
|
|
|
|
/* try and reserve slot */
|
|
|
|
if ( OMPI_CB_FREE == ptr[index] ) {
|
|
|
|
ptr[index] = OMPI_CB_RESERVED;
|
|
|
|
return_value = index;
|
|
|
|
(h_ptr->fifo_index)++;
|
|
|
|
(h_ptr->fifo_index) &= fifo->mask;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* return */
|
|
|
|
return return_value;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Try to read pointer from the tail of the queue
|
|
|
|
*
|
|
|
|
* @param data Pointer to where data was be written (OUT)
|
|
|
|
*
|
|
|
|
* @param fifo Pointer to data structure defining this fifo (IN)
|
|
|
|
*
|
|
|
|
* @param flush_entries_read force the lazy free to happen (IN)
|
|
|
|
*
|
|
|
|
* @param queue_empty checks to see if the fifo is empty, but only if
|
|
|
|
* flush_entries_read is set (OUT)
|
|
|
|
*
|
|
|
|
* @returncode Slot index to which data is written
|
|
|
|
*
|
|
|
|
*/
|
|
|
|
static inline void *ompi_cb_fifo_read_from_tail_same_base_addr(
|
|
|
|
ompi_cb_fifo_t *fifo,
|
|
|
|
bool flush_entries_read, bool *queue_empty)
|
2005-01-18 00:26:17 +03:00
|
|
|
{
|
|
|
|
int index = 0,clearIndex, i;
|
|
|
|
volatile void **q_ptr;
|
|
|
|
ompi_cb_fifo_ctl_t *h_ptr, *t_ptr;
|
|
|
|
void *read_from_tail = (void *)OMPI_CB_ERROR;
|
|
|
|
|
|
|
|
*queue_empty=false;
|
|
|
|
|
|
|
|
h_ptr=fifo->head;
|
|
|
|
t_ptr=fifo->tail;
|
|
|
|
q_ptr=fifo->queue;
|
|
|
|
|
|
|
|
/* check to see that the data is valid */
|
|
|
|
if ((q_ptr[t_ptr->fifo_index] == OMPI_CB_FREE) ||
|
|
|
|
(q_ptr[t_ptr->fifo_index] == OMPI_CB_RESERVED)) {
|
|
|
|
read_from_tail=(void *)OMPI_CB_FREE;
|
|
|
|
goto CLEANUP;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* set return data */
|
|
|
|
index = t_ptr->fifo_index;
|
|
|
|
read_from_tail = (void *)q_ptr[index];
|
|
|
|
t_ptr->num_to_clear++;
|
|
|
|
|
|
|
|
/* increment counter for later lazy free */
|
|
|
|
(t_ptr->fifo_index)++;
|
|
|
|
(t_ptr->fifo_index) &= fifo->mask;
|
|
|
|
|
|
|
|
/* check to see if time to do a lazy free of queue slots */
|
|
|
|
if ( (t_ptr->num_to_clear == fifo->lazy_free_frequency) ||
|
|
|
|
flush_entries_read ) {
|
|
|
|
clearIndex = index - t_ptr->num_to_clear + 1;
|
|
|
|
clearIndex &= fifo->mask;
|
|
|
|
|
|
|
|
for (i = 0; i < t_ptr->num_to_clear; i++) {
|
|
|
|
q_ptr[clearIndex] = OMPI_CB_FREE;
|
|
|
|
clearIndex++;
|
|
|
|
clearIndex &= fifo->mask;
|
|
|
|
}
|
|
|
|
t_ptr->num_to_clear = 0;
|
|
|
|
|
|
|
|
/* check to see if queue is empty */
|
|
|
|
if( flush_entries_read &&
|
|
|
|
(t_ptr->fifo_index == h_ptr->fifo_index) ) {
|
|
|
|
*queue_empty=true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
CLEANUP:
|
|
|
|
return read_from_tail;
|
|
|
|
}
|
2004-11-07 01:00:24 +03:00
|
|
|
|
2004-08-13 03:31:15 +04:00
|
|
|
#endif /* !_OMPI_CIRCULAR_BUFFER_FIFO */
|