ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

Merge pull request #300 from hjelmn/topic/atomic_lifo_fifo

Просмотр исходного кода 
Add opal_fifo_t class and rename opal_atomic_lifo_t to opal_lifo_t
Этот коммит содержится в:
Nathan Hjelm 2014-12-09 10:54:50 -06:00
родитель c40fd09d2a 3aefd78842
Коммит b2b7ecc7c4
12 изменённых файлов: 1101 добавлений и 134 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

8
opal/class/Makefile.am
Просмотреть файл

@ -11,6 +11,8 @@
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2014 Cisco Systems, Inc. All rights reserved.
# Copyright (c) 2014 Los Alamos National Security, LLC. All rights
# reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
@ -31,7 +33,8 @@ headers += \
class/opal_list.h \
class/opal_object.h \
class/opal_graph.h\
class/opal_atomic_lifo.h \
class/opal_lifo.h \
class/opal_fifo.h \
class/opal_pointer_array.h \
class/opal_value_array.h \
class/opal_ring_buffer.h \
@ -47,7 +50,8 @@ lib@OPAL_LIB_PREFIX@open_pal_la_SOURCES += \
class/opal_list.c \
class/opal_object.c \
class/opal_graph.c\
class/opal_atomic_lifo.c \
class/opal_lifo.c \
class/opal_fifo.c \
class/opal_pointer_array.c \
class/opal_value_array.c \
class/opal_ring_buffer.c \

9
opal/class/ompi_free_list.c
Просмотреть файл

@ -13,7 +13,7 @@
* Copyright (c) 2006-2007 Mellanox Technologies. All rights reserved.
* Copyright (c) 2010-2013 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2011 NVIDIA Corporation. All rights reserved.
* Copyright (c) 2012 Los Alamos National Security, LLC. All rights
* Copyright (c) 2012-2014 Los Alamos National Security, LLC. All rights
* reserved.
* $COPYRIGHT$
*
@ -32,7 +32,7 @@
static void ompi_free_list_construct(ompi_free_list_t* fl);
static void ompi_free_list_destruct(ompi_free_list_t* fl);
OBJ_CLASS_INSTANCE(ompi_free_list_t, opal_atomic_lifo_t,
OBJ_CLASS_INSTANCE(ompi_free_list_t, opal_lifo_t,
ompi_free_list_construct, ompi_free_list_destruct);
typedef struct ompi_free_list_item_t ompi_free_list_memory_t;
@ -73,7 +73,7 @@ static void ompi_free_list_destruct(ompi_free_list_t* fl)
}
#endif
while(NULL != (item = opal_atomic_lifo_pop(&(fl->super)))) {
while(NULL != (item = opal_lifo_pop(&(fl->super)))) {
fl_item = (ompi_free_list_item_t*)item;
/* destruct the item (we constructed it), the underlying memory will be
@ -257,13 +257,14 @@ int ompi_free_list_grow(ompi_free_list_t* flist, size_t num_elements)
item->ptr = payload_ptr;
OBJ_CONSTRUCT_INTERNAL(item, flist->fl_frag_class);
item->super.item_free = 0;
/* run the initialize function if present */
if(flist->item_init) {
flist->item_init(item, flist->ctx);
}
opal_atomic_lifo_push(&(flist->super), &(item->super));
opal_lifo_push(&(flist->super), &(item->super));
ptr += head_size;
payload_ptr += elem_size;

18
opal/class/ompi_free_list.h
Просмотреть файл

@ -1,3 +1,4 @@
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
@ -11,6 +12,8 @@
* All rights reserved.
* Copyright (c) 2010 IBM Corporation. All rights reserved.
* Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2014 Los Alamos National Security, LLC. All rights
* reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -22,7 +25,7 @@
#define OMPI_FREE_LIST_H
#include "opal_config.h"
#include "opal/class/opal_atomic_lifo.h"
#include "opal/class/opal_lifo.h"
#include "opal/prefetch.h"
#include "opal/threads/condition.h"
#include "opal/constants.h"
@ -38,7 +41,7 @@ typedef void (*ompi_free_list_item_init_fn_t) (
struct ompi_free_list_t
{
opal_atomic_lifo_t super;
opal_lifo_t super;
size_t fl_max_to_alloc;
size_t fl_num_allocated;
size_t fl_num_per_alloc;
@ -194,7 +197,7 @@ OPAL_DECLSPEC int ompi_free_list_resize_mt(ompi_free_list_t *flist, size_t size)
#define OMPI_FREE_LIST_GET_MT(fl, item) \
{ \
item = (ompi_free_list_item_t*) opal_atomic_lifo_pop(&((fl)->super)); \
item = (ompi_free_list_item_t*) opal_lifo_pop (&((fl)->super)); \
if( OPAL_UNLIKELY(NULL == item) ) { \
if(opal_using_threads()) { \
opal_mutex_lock(&((fl)->fl_lock)); \
@ -203,7 +206,7 @@ OPAL_DECLSPEC int ompi_free_list_resize_mt(ompi_free_list_t *flist, size_t size)
} else { \
ompi_free_list_grow((fl), (fl)->fl_num_per_alloc); \
} \
item = (ompi_free_list_item_t*) opal_atomic_lifo_pop(&((fl)->super)); \
item = (ompi_free_list_item_t*) opal_lifo_pop(&((fl)->super)); \
} \
}
@ -225,7 +228,7 @@ OPAL_DECLSPEC int ompi_free_list_resize_mt(ompi_free_list_t *flist, size_t size)
static inline void __ompi_free_list_wait_mt( ompi_free_list_t* fl,
ompi_free_list_item_t** item )
{
*item = (ompi_free_list_item_t*)opal_atomic_lifo_pop(&((fl)->super));
*item = (ompi_free_list_item_t*)opal_lifo_pop(&((fl)->super));
while( NULL == *item ) {
if( !OPAL_THREAD_TRYLOCK(&((fl)->fl_lock)) ) {
if((fl)->fl_max_to_alloc <= (fl)->fl_num_allocated) {
@ -256,7 +259,7 @@ static inline void __ompi_free_list_wait_mt( ompi_free_list_t* fl,
OPAL_THREAD_LOCK(&((fl)->fl_lock));
}
OPAL_THREAD_UNLOCK(&((fl)->fl_lock));
*item = (ompi_free_list_item_t*)opal_atomic_lifo_pop(&((fl)->super));
*item = (ompi_free_list_item_t*)opal_lifo_pop(&((fl)->super));
}
}
@ -272,8 +275,7 @@ static inline void __ompi_free_list_wait_mt( ompi_free_list_t* fl,
do { \
opal_list_item_t* original; \
\
original = opal_atomic_lifo_push( &(fl)->super, \
&(item)->super); \
original = opal_lifo_push( &(fl)->super, &(item)->super); \
if( &(fl)->super.opal_lifo_ghost == original ) { \
OPAL_THREAD_LOCK(&(fl)->fl_lock); \
if((fl)->fl_num_waiting > 0) { \

108
opal/class/opal_atomic_lifo.h
Просмотреть файл

@ -1,108 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2007 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2007 Voltaire All rights reserved.
* Copyright (c) 2010 IBM Corporation. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef OPAL_ATOMIC_LIFO_H_HAS_BEEN_INCLUDED
#define OPAL_ATOMIC_LIFO_H_HAS_BEEN_INCLUDED
#include "opal_config.h"
#include "opal/class/opal_list.h"
#include "opal/sys/atomic.h"
BEGIN_C_DECLS
/* Atomic Last In First Out lists. If we are in a multi-threaded environment then the
* atomicity is insured via the compare-and-swap operation, if not we simply do a read
* and/or a write.
*
* There is a trick. The ghost element at the end of the list. This ghost element has
* the next pointer pointing to itself, therefore we cannot go past the end of the list.
* With this approach we will never have a NULL element in the list, so we never have
* to test for the NULL.
*/
struct opal_atomic_lifo_t
{
opal_object_t super;
opal_list_item_t* opal_lifo_head;
opal_list_item_t opal_lifo_ghost;
};
typedef struct opal_atomic_lifo_t opal_atomic_lifo_t;
OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_atomic_lifo_t);
/* The ghost pointer will never change. The head will change via an atomic
* compare-and-swap. On most architectures the reading of a pointer is an
* atomic operation so we don't have to protect it.
*/
static inline bool opal_atomic_lifo_is_empty( opal_atomic_lifo_t* lifo )
{
return (lifo->opal_lifo_head == &(lifo->opal_lifo_ghost) ? true : false);
}
/* Add one element to the LIFO. We will return the last head of the list
* to allow the upper level to detect if this element is the first one in the
* list (if the list was empty before this operation).
*/
static inline opal_list_item_t* opal_atomic_lifo_push( opal_atomic_lifo_t* lifo,
opal_list_item_t* item )
{
do {
item->opal_list_next = lifo->opal_lifo_head;
opal_atomic_wmb();
if( opal_atomic_cmpset_ptr( &(lifo->opal_lifo_head),
(void*)item->opal_list_next,
item ) ) {
opal_atomic_cmpset_32((volatile int32_t*)&item->item_free, 1, 0);
return (opal_list_item_t*)item->opal_list_next;
}
/* DO some kind of pause to release the bus */
} while( 1 );
}
/* Retrieve one element from the LIFO. If we reach the ghost element then the LIFO
* is empty so we return NULL.
*/
static inline opal_list_item_t* opal_atomic_lifo_pop( opal_atomic_lifo_t* lifo )
{
opal_list_item_t* item;
while((item = lifo->opal_lifo_head) != &(lifo->opal_lifo_ghost))
{
opal_atomic_rmb();
if(!opal_atomic_cmpset_32((volatile int32_t*)&item->item_free, 0, 1))
continue;
if( opal_atomic_cmpset_ptr( &(lifo->opal_lifo_head),
item,
(void*)item->opal_list_next ) )
break;
opal_atomic_cmpset_32((volatile int32_t*)&item->item_free, 1, 0);
/* Do some kind of pause to release the bus */
}
if( item == &(lifo->opal_lifo_ghost) ) return NULL;
item->opal_list_next = NULL;
return item;
}
END_C_DECLS
#endif /* OPAL_ATOMIC_LIFO_H_HAS_BEEN_INCLUDED */

41
opal/class/opal_fifo.c Обычный файл
Просмотреть файл

@ -0,0 +1,41 @@
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2014 Los Alamos National Security, LLC. All rights
* reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "opal_config.h"
#include "opal/class/opal_fifo.h"
static void opal_fifo_construct (opal_fifo_t *fifo)
{
OBJ_CONSTRUCT(&fifo->opal_fifo_ghost, opal_list_item_t);
fifo->opal_fifo_ghost.opal_list_next = &fifo->opal_fifo_ghost;
/** used to protect against ABA problems when not using a 128-bit compare-and-set */
fifo->opal_fifo_ghost.item_free = 0;
fifo->opal_fifo_head.data.counter = 0;
fifo->opal_fifo_head.data.item = &fifo->opal_fifo_ghost;
fifo->opal_fifo_tail.data.counter = 0;
fifo->opal_fifo_tail.data.item = &fifo->opal_fifo_ghost;
}
OBJ_CLASS_INSTANCE(opal_fifo_t, opal_object_t, opal_fifo_construct, NULL);

310
opal/class/opal_fifo.h Обычный файл
Просмотреть файл

@ -0,0 +1,310 @@
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2007 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2007 Voltaire All rights reserved.
* Copyright (c) 2010 IBM Corporation. All rights reserved.
* Copyright (c) 2014 Los Alamos National Security, LLC. All rights
* reseved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef OPAL_FIFO_H_HAS_BEEN_INCLUDED
#define OPAL_FIFO_H_HAS_BEEN_INCLUDED
#include "opal_config.h"
#include "opal/class/opal_lifo.h"
#include "opal/sys/atomic.h"
#include "opal/threads/mutex.h"
BEGIN_C_DECLS
/* Atomic First In First Out lists. If we are in a multi-threaded environment then the
* atomicity is insured via the compare-and-swap operation, if not we simply do a read
* and/or a write.
*
* There is a trick. The ghost element at the end of the list. This ghost element has
* the next pointer pointing to itself, therefore we cannot go past the end of the list.
* With this approach we will never have a NULL element in the list, so we never have
* to test for the NULL.
*/
struct opal_fifo_t {
opal_object_t super;
/** first element on the fifo */
volatile opal_counted_pointer_t opal_fifo_head;
/** last element on the fifo */
volatile opal_counted_pointer_t opal_fifo_tail;
/** list sentinel (always points to self) */
opal_list_item_t opal_fifo_ghost;
};
typedef struct opal_fifo_t opal_fifo_t;
OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_fifo_t);
static inline opal_list_item_t *opal_fifo_head (opal_fifo_t* fifo)
{
return (opal_list_item_t *) fifo->opal_fifo_head.data.item;
}
static inline opal_list_item_t *opal_fifo_tail (opal_fifo_t* fifo)
{
return (opal_list_item_t *) fifo->opal_fifo_tail.data.item;
}
/* The ghost pointer will never change. The head will change via an atomic
* compare-and-swap. On most architectures the reading of a pointer is an
* atomic operation so we don't have to protect it.
*/
static inline bool opal_fifo_is_empty( opal_fifo_t* fifo )
{
return opal_fifo_head (fifo) == &fifo->opal_fifo_ghost;
}
#if OPAL_HAVE_ATOMIC_CMPSET_128
/* Add one element to the FIFO. We will return the last head of the list
* to allow the upper level to detect if this element is the first one in the
* list (if the list was empty before this operation).
*/
static inline opal_list_item_t *opal_fifo_push_atomic (opal_fifo_t *fifo,
opal_list_item_t *item)
{
opal_counted_pointer_t tail;
item->opal_list_next = &fifo->opal_fifo_ghost;
do {
tail.value = fifo->opal_fifo_tail.value;
if (opal_update_counted_pointer (&fifo->opal_fifo_tail, tail, item)) {
break;
}
} while (1);
opal_atomic_wmb ();
if (&fifo->opal_fifo_ghost == tail.data.item) {
/* update the head */
fifo->opal_fifo_head.data.item = item;
} else {
/* update previous item */
tail.data.item->opal_list_next = item;
}
return (opal_list_item_t *) tail.data.item;
}
/* Retrieve one element from the FIFO. If we reach the ghost element then the FIFO
* is empty so we return NULL.
*/
static inline opal_list_item_t *opal_fifo_pop_atomic (opal_fifo_t *fifo)
{
opal_list_item_t *item, *next;
opal_counted_pointer_t head, tail;
do {
head.value = fifo->opal_fifo_head.value;
tail.value = fifo->opal_fifo_tail.value;
opal_atomic_rmb ();
item = (opal_list_item_t *) head.data.item;
next = (opal_list_item_t *) item->opal_list_next;
if (&fifo->opal_fifo_ghost == tail.data.item && &fifo->opal_fifo_ghost == item) {
return NULL;
}
/* the head or next pointer are in an inconsistent state. keep looping. */
if (tail.data.item != item && &fifo->opal_fifo_ghost != tail.data.item &&
&fifo->opal_fifo_ghost == next) {
continue;
}
/* try popping the head */
if (opal_update_counted_pointer (&fifo->opal_fifo_head, head, next)) {
break;
}
} while (1);
opal_atomic_wmb ();
/* check for tail and head consistency */
if (&fifo->opal_fifo_ghost == next) {
/* the head was just set to &fifo->opal_fifo_ghost. try to update the tail as well */
if (!opal_update_counted_pointer (&fifo->opal_fifo_tail, tail, &fifo->opal_fifo_ghost)) {
/* tail was changed by a push operation. wait for the item's next pointer to be se then
* update the head */
/* wait for next pointer to be updated by push */
while (&fifo->opal_fifo_ghost == item->opal_list_next) {
opal_atomic_rmb ();
}
/* update the head with the real next value. note that no other thread
* will be attempting to update the head until after it has been updated
* with the next pointer. push will not see an empty list and other pop
* operations will loop until the head is consistent. */
head.value = fifo->opal_fifo_head.value;
next = (opal_list_item_t *) item->opal_list_next;
assert (&fifo->opal_fifo_ghost == head.data.item);
fifo->opal_fifo_head.data.item = next;
}
}
item->opal_list_next = NULL;
return item;
}
#else
/* When compare-and-set 128 is not available we avoid the ABA problem by
* using a spin-lock on the head (using the head counter). Otherwise
* the algorithm is identical to the compare-and-set 128 version. */
static inline opal_list_item_t *opal_fifo_push_atomic (opal_fifo_t *fifo,
opal_list_item_t *item)
{
opal_list_item_t *tail_item;
item->opal_list_next = &fifo->opal_fifo_ghost;
/* try to get the tail */
tail_item = opal_atomic_swap_ptr (&fifo->opal_fifo_tail.data.item, item);
opal_atomic_wmb ();
if (&fifo->opal_fifo_ghost == tail_item) {
/* update the head */
fifo->opal_fifo_head.data.item = item;
} else {
/* update previous item */
tail_item->opal_list_next = item;
}
opal_atomic_wmb ();
return (opal_list_item_t *) tail_item;
}
/* Retrieve one element from the FIFO. If we reach the ghost element then the FIFO
* is empty so we return NULL.
*/
static inline opal_list_item_t *opal_fifo_pop_atomic (opal_fifo_t *fifo)
{
opal_list_item_t *item, *next;
do {
if (opal_atomic_cmpset_32 ((int32_t *) &fifo->opal_fifo_head.data.counter, 0, 1)) {
break;
}
opal_atomic_wmb ();
} while (1);
item = opal_fifo_head (fifo);
if (&fifo->opal_fifo_ghost == item) {
fifo->opal_fifo_head.data.counter = 0;
return NULL;
}
next = (opal_list_item_t *) item->opal_list_next;
fifo->opal_fifo_head.data.item = next;
if (&fifo->opal_fifo_ghost == next) {
if (!opal_atomic_cmpset_ptr (&fifo->opal_fifo_tail.data.item, item, &fifo->opal_fifo_ghost)) {
while (&fifo->opal_fifo_ghost == item->opal_list_next) {
opal_atomic_rmb ();
}
fifo->opal_fifo_head.data.item = (opal_list_item_t *) item->opal_list_next;
}
}
opal_atomic_wmb ();
/* unlock the head */
fifo->opal_fifo_head.data.counter = 0;
item->opal_list_next = NULL;
return item;
}
#endif
/* single threaded versions of push/pop */
static inline opal_list_item_t *opal_fifo_push_st (opal_fifo_t *fifo,
opal_list_item_t *item)
{
opal_list_item_t *prev = opal_fifo_tail (fifo);
item->opal_list_next = &fifo->opal_fifo_ghost;
fifo->opal_fifo_tail.data.item = item;
if (&fifo->opal_fifo_ghost == opal_fifo_head (fifo)) {
fifo->opal_fifo_head.data.item = item;
} else {
prev->opal_list_next = item;
}
return (opal_list_item_t *) item->opal_list_next;
}
static inline opal_list_item_t *opal_fifo_pop_st (opal_fifo_t *fifo)
{
opal_list_item_t *item = opal_fifo_head (fifo);
if (item == &fifo->opal_fifo_ghost) {
return NULL;
}
fifo->opal_fifo_head.data.item = (opal_list_item_t *) item->opal_list_next;
if (&fifo->opal_fifo_ghost == opal_fifo_head (fifo)) {
fifo->opal_fifo_tail.data.item = &fifo->opal_fifo_ghost;
}
item->opal_list_next = NULL;
return item;
}
/* push/pop versions conditioned off opal_using_threads() */
static inline opal_list_item_t *opal_fifo_push (opal_fifo_t *fifo,
opal_list_item_t *item)
{
if (opal_using_threads ()) {
return opal_fifo_push_atomic (fifo, item);
}
return opal_fifo_push_st (fifo, item);
}
static inline opal_list_item_t *opal_fifo_pop (opal_fifo_t *fifo)
{
if (opal_using_threads ()) {
return opal_fifo_pop_atomic (fifo);
}
return opal_fifo_pop_st (fifo);
}
END_C_DECLS
#endif /* OPAL_FIFO_H_HAS_BEEN_INCLUDED */

19
opal/class/opal_atomic_lifo.c → opal/class/opal_lifo.c
Просмотреть файл

@ -1,3 +1,4 @@
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
@ -9,6 +10,8 @@
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2014 Los Alamos National Security, LLC. All rights
* reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -17,16 +20,14 @@
*/
#include "opal_config.h"
#include "opal/class/opal_atomic_lifo.h"
#include "opal/class/opal_lifo.h"
static void opal_atomic_lifo_construct( opal_atomic_lifo_t* lifo )
static void opal_lifo_construct (opal_lifo_t *lifo)
{
OBJ_CONSTRUCT( &(lifo->opal_lifo_ghost), opal_list_item_t );
lifo->opal_lifo_ghost.opal_list_next = &(lifo->opal_lifo_ghost);
lifo->opal_lifo_head = &(lifo->opal_lifo_ghost);
OBJ_CONSTRUCT(&lifo->opal_lifo_ghost, opal_list_item_t);
lifo->opal_lifo_ghost.opal_list_next = &lifo->opal_lifo_ghost;
lifo->opal_lifo_head.data.item = &lifo->opal_lifo_ghost;
lifo->opal_lifo_head.data.counter = 0;
}
OBJ_CLASS_INSTANCE( opal_atomic_lifo_t,
opal_object_t,
opal_atomic_lifo_construct,
NULL );
OBJ_CLASS_INSTANCE(opal_lifo_t, opal_object_t, opal_lifo_construct, NULL);

261
opal/class/opal_lifo.h Обычный файл
Просмотреть файл

@ -0,0 +1,261 @@
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2007 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2007 Voltaire All rights reserved.
* Copyright (c) 2010 IBM Corporation. All rights reserved.
* Copyright (c) 2014 Los Alamos National Security, LLC. All rights
* reseved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef OPAL_LIFO_H_HAS_BEEN_INCLUDED
#define OPAL_LIFO_H_HAS_BEEN_INCLUDED
#include "opal_config.h"
#include "opal/class/opal_list.h"
#include "opal/sys/atomic.h"
#include "opal/threads/mutex.h"
BEGIN_C_DECLS
/* NTH: temporarily suppress warnings about this not being defined */
#if !defined(OPAL_HAVE_ATOMIC_CMPSET_128)
#define OPAL_HAVE_ATOMIC_CMPSET_128 0
#endif
/**
* Counted pointer to avoid the ABA problem.
*/
union opal_counted_pointer_t {
struct {
/** update counter used when cmpset_128 is available */
uint64_t counter;
/** list item pointer */
opal_list_item_t *item;
} data;
#if OPAL_HAVE_ATOMIC_CMPSET_128 && HAVE_OPAL_INT128_T
/** used for atomics when there is a cmpset that can operate on
* two 64-bit values */
opal_int128_t value;
#endif
};
typedef union opal_counted_pointer_t opal_counted_pointer_t;
#if OPAL_HAVE_ATOMIC_CMPSET_128
/* Add one element to the FIFO. We will return the last head of the list
* to allow the upper level to detect if this element is the first one in the
* list (if the list was empty before this operation).
*/
static inline bool opal_update_counted_pointer (volatile opal_counted_pointer_t *addr, opal_counted_pointer_t old,
opal_list_item_t *item)
{
opal_counted_pointer_t new = {.data = {.item = item, .counter = old.data.counter + 1}};
return opal_atomic_cmpset_128 (&addr->value, old.value, new.value);
}
#endif
/* Atomic Last In First Out lists. If we are in a multi-threaded environment then the
* atomicity is insured via the compare-and-swap operation, if not we simply do a read
* and/or a write.
*
* There is a trick. The ghost element at the end of the list. This ghost element has
* the next pointer pointing to itself, therefore we cannot go past the end of the list.
* With this approach we will never have a NULL element in the list, so we never have
* to test for the NULL.
*/
struct opal_lifo_t {
opal_object_t super;
/** head element of the lifo. points to opal_lifo_ghost if the lifo is empty */
volatile opal_counted_pointer_t opal_lifo_head;
/** list sentinel (always points to self) */
opal_list_item_t opal_lifo_ghost;
};
typedef struct opal_lifo_t opal_lifo_t;
OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_lifo_t);
/* The ghost pointer will never change. The head will change via an atomic
* compare-and-swap. On most architectures the reading of a pointer is an
* atomic operation so we don't have to protect it.
*/
static inline bool opal_lifo_is_empty( opal_lifo_t* lifo )
{
return (opal_list_item_t *) lifo->opal_lifo_head.data.item == &lifo->opal_lifo_ghost;
}
#if OPAL_HAVE_ATOMIC_CMPSET_128
/* Add one element to the LIFO. We will return the last head of the list
* to allow the upper level to detect if this element is the first one in the
* list (if the list was empty before this operation).
*/
static inline opal_list_item_t *opal_lifo_push_atomic (opal_lifo_t *lifo,
opal_list_item_t *item)
{
do {
opal_list_item_t *next = (opal_list_item_t *) lifo->opal_lifo_head.data.item;
item->opal_list_next = next;
opal_atomic_wmb ();
/* to protect against ABA issues it is sufficient to only update the counter in pop */
if (opal_atomic_cmpset_ptr (&lifo->opal_lifo_head.data.item, next, item)) {
return next;
}
/* DO some kind of pause to release the bus */
} while (1);
}
/* Retrieve one element from the LIFO. If we reach the ghost element then the LIFO
* is empty so we return NULL.
*/
static inline opal_list_item_t *opal_lifo_pop_atomic (opal_lifo_t* lifo)
{
opal_list_item_t *item;
do {
opal_counted_pointer_t old_head;
old_head.value = lifo->opal_lifo_head.value;
item = (opal_list_item_t *) lifo->opal_lifo_head.data.item;
if (item == &lifo->opal_lifo_ghost) {
return NULL;
}
if (opal_update_counted_pointer (&lifo->opal_lifo_head, old_head,
(opal_list_item_t *) item->opal_list_next)) {
item->opal_list_next = NULL;
return item;
}
} while (1);
}
#else
/* Add one element to the LIFO. We will return the last head of the list
* to allow the upper level to detect if this element is the first one in the
* list (if the list was empty before this operation).
*/
static inline opal_list_item_t *opal_lifo_push_atomic (opal_lifo_t *lifo,
opal_list_item_t *item)
{
/* item free acts as a mini lock to avoid ABA problems */
item->item_free = 1;
do {
opal_list_item_t *next = (opal_list_item_t *) lifo->opal_lifo_head.data.item;
item->opal_list_next = next;
opal_atomic_wmb();
if (opal_atomic_cmpset_ptr (&lifo->opal_lifo_head.data.item, next, item)) {
opal_atomic_wmb ();
/* now safe to pop this item */
item->item_free = 0;
return next;
}
/* DO some kind of pause to release the bus */
} while (1);
}
/* Retrieve one element from the LIFO. If we reach the ghost element then the LIFO
* is empty so we return NULL.
*/
static inline opal_list_item_t *opal_lifo_pop_atomic (opal_lifo_t* lifo)
{
opal_list_item_t *item;
while ((item = (opal_list_item_t *) lifo->opal_lifo_head.data.item) != &lifo->opal_lifo_ghost) {
opal_atomic_rmb();
/* ensure it is safe to pop the head */
if (opal_atomic_swap_32((volatile int32_t *) &item->item_free, 1)) {
continue;
}
/* try to swap out the head pointer */
if (opal_atomic_cmpset_ptr (&lifo->opal_lifo_head.data.item, item,
(void *) item->opal_list_next)) {
break;
}
/* NTH: don't need another atomic here */
item->item_free = 0;
/* Do some kind of pause to release the bus */
}
if (item == &lifo->opal_lifo_ghost) {
return NULL;
}
item->opal_list_next = NULL;
return item;
}
#endif
/* single-threaded versions of the lifo functions */
static inline opal_list_item_t *opal_lifo_push_st (opal_lifo_t *lifo,
opal_list_item_t *item)
{
item->opal_list_next = (opal_list_item_t *) lifo->opal_lifo_head.data.item;
item->item_free = 0;
lifo->opal_lifo_head.data.item = item;
return (opal_list_item_t *) item->opal_list_next;
}
static inline opal_list_item_t *opal_lifo_pop_st (opal_lifo_t *lifo)
{
opal_list_item_t *item;
item = (opal_list_item_t *) lifo->opal_lifo_head.data.item;
lifo->opal_lifo_head.data.item = (opal_list_item_t *) item->opal_list_next;
if (item == &lifo->opal_lifo_ghost) {
return NULL;
}
item->opal_list_next = NULL;
item->item_free = 1;
return item;
}
/* conditional versions of lifo functions. use atomics if opal_using_threads is set */
static inline opal_list_item_t *opal_lifo_push (opal_lifo_t *lifo,
opal_list_item_t *item)
{
if (opal_using_threads ()) {
return opal_lifo_push_atomic (lifo, item);
}
return opal_lifo_push_st (lifo, item);
}
static inline opal_list_item_t *opal_lifo_pop (opal_lifo_t *lifo)
{
if (opal_using_threads ()) {
return opal_lifo_pop_atomic (lifo);
}
return opal_lifo_pop_st (lifo);
}
END_C_DECLS
#endif /* OPAL_LIFO_H_HAS_BEEN_INCLUDED */

2
opal/mca/mpool/gpusm/mpool_gpusm_module.c
Просмотреть файл

@ -183,7 +183,7 @@ void mca_mpool_gpusm_finalize(struct mca_mpool_base_module_t *mpool)
/* Need to run the destructor on each item in the free list explicitly.
* The destruction of the free list only runs the destructor on the
* main free list, not each item. */
while (NULL != (item = (ompi_free_list_item_t *)opal_atomic_lifo_pop(&(mpool_gpusm->reg_list.super)))) {
while (NULL != (item = (ompi_free_list_item_t *)opal_lifo_pop(&(mpool_gpusm->reg_list.super)))) {
OBJ_DESTRUCT(item);
}

18
test/class/Makefile.am
Просмотреть файл

@ -33,11 +33,13 @@ check_PROGRAMS = \
opal_tree \
opal_list \
opal_value_array \
opal_pointer_array
opal_pointer_array \
opal_lifo \
opal_fifo
# The tests are built, but since they're not actively maintained, they
# are not run during "make check".
TESTS =
TESTS = opal_lifo opal_fifo
opal_bitmap_SOURCES = opal_bitmap.c
opal_bitmap_LDADD = \
@ -87,5 +89,17 @@ ompi_rb_tree_LDADD = \
$(top_builddir)/test/support/libsupport.a
ompi_rb_tree_DEPENDENCIES = $(ompi_rb_tree_LDADD)
opal_lifo_SOURCES = opal_lifo.c
opal_lifo_LDADD = \
$(top_builddir)/opal/libopen-pal.la \
$(top_builddir)/test/support/libsupport.a
opal_lifo_DEPENDENCIES = $(opal_lifo_LDADD)
opal_fifo_SOURCES = opal_fifo.c
opal_fifo_LDADD = \
$(top_builddir)/opal/libopen-pal.la \
$(top_builddir)/test/support/libsupport.a
opal_fifo_DEPENDENCIES = $(opal_fifo_LDADD)
clean-local:
rm -f opal_bitmap_test_out.txt opal_hash_table_test_out.txt opal_proc_table_test_out.txt

252
test/class/opal_fifo.c Обычный файл
Просмотреть файл

@ -0,0 +1,252 @@
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/*
* Copyright (c) 2014 Los Alamos National Security, LLC. All rights
* reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "opal_config.h"
#include <assert.h>
#include "support.h"
#include "opal/class/opal_fifo.h"
#include "opal/runtime/opal.h"
#include "opal/constants.h"
#include <stdlib.h>
#include <stdio.h>
#include <stddef.h>
#include <sys/time.h>
#define OPAL_FIFO_TEST_THREAD_COUNT 8
#define ITERATIONS 1000000
#define ITEM_COUNT 100
#define ITEMS_PER_LOOP 30
static void *thread_test (void *arg) {
opal_fifo_t *fifo = (opal_fifo_t *) arg;
opal_list_item_t *item;
struct timeval start, stop, total;
double timing;
gettimeofday (&start, NULL);
for (int i = 0 ; i < ITERATIONS ; ++i) {
item = opal_fifo_pop_atomic (fifo);
if (NULL != item) {
(void) opal_fifo_push_atomic (fifo, item);
}
}
gettimeofday (&stop, NULL);
timersub(&stop, &start, &total);
timing = ((double) total.tv_sec + (double) total.tv_usec * 1e-6) / (double) ITERATIONS;
printf ("Atomics thread finished. Time: %d s %d us %d nsec/poppush\n", (int) total.tv_sec,
total.tv_usec, (int)(timing / 1e-9));
return NULL;
}
static void *thread_test_exhaust (void *arg) {
opal_fifo_t *fifo = (opal_fifo_t *) arg;
opal_list_item_t *items[ITEMS_PER_LOOP];
struct timeval start, stop, total;
int item_count = 0;
double timing;
gettimeofday (&start, NULL);
for (int i = 0 ; i < ITERATIONS ; i += ITEMS_PER_LOOP) {
for (int j = 0 ; j < ITEMS_PER_LOOP ; ++j) {
items[j] = opal_fifo_pop_atomic (fifo);
if (items[j]) {
++item_count;
}
}
for (int j = 0 ; j < ITEMS_PER_LOOP ; ++j) {
if (items[j]) {
(void) opal_fifo_push_atomic (fifo, items[j]);
}
}
}
gettimeofday (&stop, NULL);
timersub(&stop, &start, &total);
timing = ((double) total.tv_sec + (double) total.tv_usec * 1e-6) / (double) item_count;
fprintf (stderr, "Exhaustive atomics thread finished. Popped %d items. Time: %d s %d us %d nsec/poppush\n", item_count,
(int) total.tv_sec, total.tv_usec, (int)(timing / 1e-9));
return NULL;
}
static bool check_fifo_consistency (opal_fifo_t *fifo, int expected_count)
{
opal_list_item_t *item;
int count;
for (count = 0, item = fifo->opal_fifo_head.data.item ; item != &fifo->opal_fifo_ghost ;
item = opal_list_get_next(item), count++);
return count == expected_count;
}
int main (int argc, char *argv[]) {
pthread_t threads[OPAL_FIFO_TEST_THREAD_COUNT];
opal_list_item_t *item, *prev, *item2;
struct timeval start, stop, total;
opal_fifo_t fifo;
bool success;
double timing;
int rc;
rc = opal_init (&argc, &argv);
test_verify_int(OPAL_SUCCESS, rc);
if (OPAL_SUCCESS != rc) {
test_finalize();
exit (1);
}
test_init("opal_fifo_t");
OBJ_CONSTRUCT(&fifo, opal_fifo_t);
item = OBJ_NEW(opal_list_item_t);
prev = opal_fifo_push_st (&fifo, item);
if (&fifo.opal_fifo_ghost == prev) {
test_success ();
} else {
test_failure (" opal_fifo_push_st on empty fifo");
}
item2 = opal_fifo_pop_st (&fifo);
if (item == item2) {
test_success ();
} else {
test_failure (" opal_fifo_pop_st");
}
OBJ_RELEASE(item);
for (int i = 0 ; i < ITEM_COUNT ; ++i) {
item = OBJ_NEW(opal_list_item_t);
item->item_free = 0;
opal_fifo_push_st (&fifo, item);
}
if (check_fifo_consistency (&fifo, ITEM_COUNT)) {
test_success ();
} else {
test_failure (" opal_fifo_push_st(multiple items)");
}
gettimeofday (&start, NULL);
for (int i = 0 ; i < ITERATIONS ; ++i) {
item = opal_fifo_pop_st (&fifo);
(void) opal_fifo_push_st (&fifo, item);
}
gettimeofday (&stop, NULL);
timersub(&stop, &start, &total);
timing = ((double) total.tv_sec + (double) total.tv_usec * 1e-6) / (double) ITERATIONS;
if (check_fifo_consistency (&fifo, ITEM_COUNT)) {
test_success ();
} else {
test_failure (" fifo push/pop");
}
printf ("Single thread test. Time: %d s %d us %d nsec/poppush\n", (int) total.tv_sec,
total.tv_usec, (int)(timing / 1e-9));
thread_test (&fifo);
if (check_fifo_consistency (&fifo, ITEM_COUNT)) {
test_success ();
} else {
test_failure (" fifo push/pop single-threaded with atomics");
}
gettimeofday (&start, NULL);
for (int i = 0 ; i < OPAL_FIFO_TEST_THREAD_COUNT ; ++i) {
pthread_create (threads + i, NULL, thread_test, &fifo);
}
for (int i = 0 ; i < OPAL_FIFO_TEST_THREAD_COUNT ; ++i) {
void *ret;
pthread_join (threads[i], &ret);
}
gettimeofday (&stop, NULL);
timersub(&stop, &start, &total);
timing = ((double) total.tv_sec + (double) total.tv_usec * 1e-6) / (double) (ITERATIONS * OPAL_FIFO_TEST_THREAD_COUNT);
if (check_fifo_consistency (&fifo, ITEM_COUNT)) {
test_success ();
} else {
test_failure (" fifo push/pop multi-threaded with atomics");
}
printf ("All threads finished. Thread count: %d Time: %d s %d us %d nsec/poppush\n",
OPAL_FIFO_TEST_THREAD_COUNT, (int) total.tv_sec, total.tv_usec, (int)(timing / 1e-9));
gettimeofday (&start, NULL);
for (int i = 0 ; i < OPAL_FIFO_TEST_THREAD_COUNT ; ++i) {
pthread_create (threads + i, NULL, thread_test_exhaust, &fifo);
}
for (int i = 0 ; i < OPAL_FIFO_TEST_THREAD_COUNT ; ++i) {
void *ret;
pthread_join (threads[i], &ret);
}
gettimeofday (&stop, NULL);
timersub(&stop, &start, &total);
timing = ((double) total.tv_sec + (double) total.tv_usec * 1e-6) / (double) (ITERATIONS * OPAL_FIFO_TEST_THREAD_COUNT);
if (check_fifo_consistency (&fifo, ITEM_COUNT)) {
test_success ();
} else {
test_failure (" fifo push/pop multi-threaded with atomics when there are insufficient items");
}
printf ("All threads finished. Thread count: %d Time: %d s %d us %d nsec/poppush\n",
OPAL_FIFO_TEST_THREAD_COUNT, (int) total.tv_sec, total.tv_usec, (int)(timing / 1e-9));
success = true;
for (int i = 0 ; i < ITEM_COUNT ; ++i) {
item = opal_fifo_pop_st (&fifo);
if (NULL == item) {
success = false;
break;
}
OBJ_RELEASE(item);
}
if (success) {
test_success ();
} else {
test_failure (" fifo pop all items");
}
OBJ_DESTRUCT(&fifo);
opal_finalize ();
return test_finalize ();
}

189
test/class/opal_lifo.c Обычный файл
Просмотреть файл

@ -0,0 +1,189 @@
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/*
* Copyright (c) 2014 Los Alamos National Security, LLC. All rights
* reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "opal_config.h"
#include <assert.h>
#include "support.h"
#include "opal/class/opal_lifo.h"
#include "opal/runtime/opal.h"
#include "opal/constants.h"
#include <stdlib.h>
#include <stdio.h>
#include <stddef.h>
#include <sys/time.h>
#define OPAL_LIFO_TEST_THREAD_COUNT 8
#define ITERATIONS 1000000
#define ITEM_COUNT 100
static void *thread_test (void *arg) {
opal_lifo_t *lifo = (opal_lifo_t *) arg;
opal_list_item_t *item;
struct timeval start, stop, total;
double timing;
gettimeofday (&start, NULL);
for (int i = 0 ; i < ITERATIONS ; ++i) {
item = opal_lifo_pop_atomic (lifo);
if (NULL != item) {
(void) opal_lifo_push_atomic (lifo, item);
}
}
gettimeofday (&stop, NULL);
timersub(&stop, &start, &total);
timing = ((double) total.tv_sec + (double) total.tv_usec * 1e-6) / (double) ITERATIONS;
printf ("Atomics thread finished. Time: %d s %d us %d nsec/poppush\n", (int) total.tv_sec,
total.tv_usec, (int)(timing / 1e-9));
return NULL;
}
static bool check_lifo_consistency (opal_lifo_t *lifo, int expected_count)
{
opal_list_item_t *item;
int count;
for (count = 0, item = lifo->opal_lifo_head.data.item ; item != &lifo->opal_lifo_ghost ;
item = opal_list_get_next(item), count++);
return count == expected_count;
}
int main (int argc, char *argv[]) {
pthread_t threads[OPAL_LIFO_TEST_THREAD_COUNT];
opal_list_item_t *item, *prev, *item2;
struct timeval start, stop, total;
opal_lifo_t lifo;
bool success;
double timing;
int rc;
rc = opal_init (&argc, &argv);
test_verify_int(OPAL_SUCCESS, rc);
if (OPAL_SUCCESS != rc) {
test_finalize();
exit (1);
}
test_init("opal_lifo_t");
OBJ_CONSTRUCT(&lifo, opal_lifo_t);
item = OBJ_NEW(opal_list_item_t);
prev = opal_lifo_push_st (&lifo, item);
if (&lifo.opal_lifo_ghost == prev) {
test_success ();
} else {
test_failure (" opal_lifo_push_st on empty lifo");
}
item2 = opal_lifo_pop_st (&lifo);
if (item == item2) {
test_success ();
} else {
test_failure (" opal_lifo_pop_st");
}
OBJ_RELEASE(item);
for (int i = 0 ; i < ITEM_COUNT ; ++i) {
item = OBJ_NEW(opal_list_item_t);
item->item_free = 0;
opal_lifo_push_st (&lifo, item);
}
if (check_lifo_consistency (&lifo, ITEM_COUNT)) {
test_success ();
} else {
test_failure (" opal_lifo_push_st(multiple items)");
}
gettimeofday (&start, NULL);
for (int i = 0 ; i < ITERATIONS ; ++i) {
item = opal_lifo_pop_st (&lifo);
(void) opal_lifo_push_st (&lifo, item);
}
gettimeofday (&stop, NULL);
timersub(&stop, &start, &total);
timing = ((double) total.tv_sec + (double) total.tv_usec * 1e-6) / (double) ITERATIONS;
if (check_lifo_consistency (&lifo, ITEM_COUNT)) {
test_success ();
} else {
test_failure (" lifo push/pop");
}
printf ("Single thread test. Time: %d s %d us %d nsec/poppush\n", (int) total.tv_sec,
total.tv_usec, (int)(timing / 1e-9));
thread_test (&lifo);
if (check_lifo_consistency (&lifo, ITEM_COUNT)) {
test_success ();
} else {
test_failure (" lifo push/pop single-threaded with atomics");
}
gettimeofday (&start, NULL);
for (int i = 0 ; i < OPAL_LIFO_TEST_THREAD_COUNT ; ++i) {
pthread_create (threads + i, NULL, thread_test, &lifo);
}
for (int i = 0 ; i < OPAL_LIFO_TEST_THREAD_COUNT ; ++i) {
void *ret;
pthread_join (threads[i], &ret);
}
gettimeofday (&stop, NULL);
timersub(&stop, &start, &total);
timing = ((double) total.tv_sec + (double) total.tv_usec * 1e-6) / (double) (ITERATIONS * OPAL_LIFO_TEST_THREAD_COUNT);
if (check_lifo_consistency (&lifo, ITEM_COUNT)) {
test_success ();
} else {
test_failure (" lifo push/pop multi-threaded with atomics");
}
printf ("All threads finished. Thread count: %d Time: %d s %d us %d nsec/poppush\n",
OPAL_LIFO_TEST_THREAD_COUNT, (int) total.tv_sec, total.tv_usec, (int)(timing / 1e-9));
success = true;
for (int i = 0 ; i < ITEM_COUNT ; ++i) {
item = opal_lifo_pop_st (&lifo);
if (NULL == item) {
success = false;
break;
}
OBJ_RELEASE(item);
}
if (success) {
test_success ();
} else {
test_failure (" list pop all items");
}
OBJ_DESTRUCT(&lifo);
opal_finalize ();
return test_finalize ();
}