ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

opal: add types for atomic variables

Просмотр исходного кода 
This commit updates the entire codebase to use specific opal types for
all atomic variables. This is a change from the prior atomic support
which required the use of the volatile keyword. This is the first step
towards implementing support for C11 atomics as that interface
requires the use of types declared with the _Atomic keyword.

Signed-off-by: Nathan Hjelm <hjelmn@lanl.gov>
Этот коммит содержится в:
Nathan Hjelm 2018-07-11 13:34:03 -06:00
родитель 9a514c6794
Коммит 000f9eed4d
108 изменённых файлов: 616 добавлений и 557 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

4
ompi/communicator/comm_request.c
Просмотреть файл

@ -1,6 +1,6 @@
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/* /*
* Copyright (c) 2013-2016 Los Alamos National Security, LLC. All rights * Copyright (c) 2013-2018 Los Alamos National Security, LLC. All rights
* reseved. * reseved.
* Copyright (c) 2015 Research Organization for Information Science * Copyright (c) 2015 Research Organization for Information Science
* and Technology (RIST). All rights reserved. * and Technology (RIST). All rights reserved.
@ -99,7 +99,7 @@ int ompi_comm_request_schedule_append (ompi_comm_request_t *request, ompi_comm_r
static int ompi_comm_request_progress (void) static int ompi_comm_request_progress (void)
{ {
ompi_comm_request_t *request, *next; ompi_comm_request_t *request, *next;
static int32_t progressing = 0; static opal_atomic_int32_t progressing = 0;
/* don't allow re-entry */ /* don't allow re-entry */
if (opal_atomic_swap_32 (&progressing, 1)) { if (opal_atomic_swap_32 (&progressing, 1)) {

2
ompi/datatype/ompi_datatype.h
Просмотреть файл

@ -75,7 +75,7 @@ struct ompi_datatype_t {
struct opal_hash_table_t *d_keyhash; /**< Attribute fields */ struct opal_hash_table_t *d_keyhash; /**< Attribute fields */
void* args; /**< Data description for the user */ void* args; /**< Data description for the user */
void* packed_description; /**< Packed description of the datatype */ opal_atomic_intptr_t packed_description; /**< Packed description of the datatype */
uint64_t pml_data; /**< PML-specific information */ uint64_t pml_data; /**< PML-specific information */
/* --- cacheline 6 boundary (384 bytes) --- */ /* --- cacheline 6 boundary (384 bytes) --- */
char name[MPI_MAX_OBJECT_NAME];/**< Externally visible name */ char name[MPI_MAX_OBJECT_NAME];/**< Externally visible name */

18
ompi/datatype/ompi_datatype_args.c
Просмотреть файл

@ -45,7 +45,7 @@ __ompi_datatype_create_from_args( int32_t* i, ptrdiff_t * a,
ompi_datatype_t** d, int32_t type ); ompi_datatype_t** d, int32_t type );
typedef struct __dt_args { typedef struct __dt_args {
int32_t ref_count; opal_atomic_int32_t ref_count;
int32_t create_type; int32_t create_type;
size_t total_pack_size; size_t total_pack_size;
int32_t ci; int32_t ci;
@ -104,7 +104,7 @@ typedef struct __dt_args {
pArgs->total_pack_size = (4 + (IC) + (DC)) * sizeof(int) + \ pArgs->total_pack_size = (4 + (IC) + (DC)) * sizeof(int) + \
(AC) * sizeof(ptrdiff_t); \ (AC) * sizeof(ptrdiff_t); \
(PDATA)->args = (void*)pArgs; \ (PDATA)->args = (void*)pArgs; \
(PDATA)->packed_description = NULL; \ (PDATA)->packed_description = 0; \
} while(0) } while(0)
@ -483,12 +483,12 @@ int ompi_datatype_get_pack_description( ompi_datatype_t* datatype,
{ {
ompi_datatype_args_t* args = (ompi_datatype_args_t*)datatype->args; ompi_datatype_args_t* args = (ompi_datatype_args_t*)datatype->args;
int next_index = OMPI_DATATYPE_MAX_PREDEFINED; int next_index = OMPI_DATATYPE_MAX_PREDEFINED;
void *packed_description = datatype->packed_description; void *packed_description = (void *) datatype->packed_description;
void* recursive_buffer; void* recursive_buffer;
if (NULL == packed_description) { if (NULL == packed_description) {
void *_tmp_ptr = NULL; void *_tmp_ptr = NULL;
if (opal_atomic_compare_exchange_strong_ptr (&datatype->packed_description, (void *) &_tmp_ptr, (void *) 1)) { if (opal_atomic_compare_exchange_strong_ptr (&datatype->packed_description, (intptr_t *) &_tmp_ptr, 1)) {
if( ompi_datatype_is_predefined(datatype) ) { if( ompi_datatype_is_predefined(datatype) ) {
packed_description = malloc(2 * sizeof(int)); packed_description = malloc(2 * sizeof(int));
} else if( NULL == args ) { } else if( NULL == args ) {
@ -510,10 +510,10 @@ int ompi_datatype_get_pack_description( ompi_datatype_t* datatype,
} }
opal_atomic_wmb (); opal_atomic_wmb ();
datatype->packed_description = packed_description; datatype->packed_description = (intptr_t) packed_description;
} else { } else {
/* another thread beat us to it */ /* another thread beat us to it */
packed_description = datatype->packed_description; packed_description = (void *) datatype->packed_description;
} }
} }
@ -521,11 +521,11 @@ int ompi_datatype_get_pack_description( ompi_datatype_t* datatype,
struct timespec interval = {.tv_sec = 0, .tv_nsec = 1000}; struct timespec interval = {.tv_sec = 0, .tv_nsec = 1000};
/* wait until the packed description is updated */ /* wait until the packed description is updated */
while ((void *) 1 == datatype->packed_description) { while (1 == datatype->packed_description) {
nanosleep (&interval, NULL); nanosleep (&interval, NULL);
} }
packed_description = datatype->packed_description; packed_description = (void *) datatype->packed_description;
} }
*packed_buffer = (const void *) packed_description; *packed_buffer = (const void *) packed_description;
@ -534,7 +534,7 @@ int ompi_datatype_get_pack_description( ompi_datatype_t* datatype,
size_t ompi_datatype_pack_description_length( ompi_datatype_t* datatype ) size_t ompi_datatype_pack_description_length( ompi_datatype_t* datatype )
{ {
void *packed_description = datatype->packed_description; void *packed_description = (void *) datatype->packed_description;
if( ompi_datatype_is_predefined(datatype) ) { if( ompi_datatype_is_predefined(datatype) ) {
return 2 * sizeof(int); return 2 * sizeof(int);

10
ompi/datatype/ompi_datatype_create.c
Просмотреть файл

@ -36,7 +36,7 @@ static void __ompi_datatype_allocate( ompi_datatype_t* datatype )
datatype->id = -1; datatype->id = -1;
datatype->d_keyhash = NULL; datatype->d_keyhash = NULL;
datatype->name[0] = '\0'; datatype->name[0] = '\0';
datatype->packed_description = NULL; datatype->packed_description = 0;
datatype->pml_data = 0; datatype->pml_data = 0;
} }
@ -46,10 +46,10 @@ static void __ompi_datatype_release(ompi_datatype_t * datatype)
ompi_datatype_release_args( datatype ); ompi_datatype_release_args( datatype );
datatype->args = NULL; datatype->args = NULL;
} }
if( NULL != datatype->packed_description ) {
free( datatype->packed_description ); free ((void *) datatype->packed_description );
datatype->packed_description = NULL; datatype->packed_description = 0;
}
if( datatype->d_f_to_c_index >= 0 ) { if( datatype->d_f_to_c_index >= 0 ) {
opal_pointer_array_set_item( &ompi_datatype_f_to_c_table, datatype->d_f_to_c_index, NULL ); opal_pointer_array_set_item( &ompi_datatype_f_to_c_table, datatype->d_f_to_c_index, NULL );
datatype->d_f_to_c_index = -1; datatype->d_f_to_c_index = -1;

2
ompi/datatype/ompi_datatype_internal.h
Просмотреть файл

@ -406,7 +406,7 @@ extern const ompi_datatype_t* ompi_datatype_basicDatatypes[OMPI_DATATYPE_MPI_MAX
.d_f_to_c_index = -1, \ .d_f_to_c_index = -1, \
.d_keyhash = NULL, \ .d_keyhash = NULL, \
.args = NULL, \ .args = NULL, \
.packed_description = NULL, \ .packed_description = 0, \
.name = "MPI_" # NAME .name = "MPI_" # NAME
#define OMPI_DATATYPE_INITIALIZER_UNAVAILABLE(FLAGS) \ #define OMPI_DATATYPE_INITIALIZER_UNAVAILABLE(FLAGS) \

2
ompi/datatype/ompi_datatype_module.c
Просмотреть файл

@ -383,7 +383,7 @@ opal_pointer_array_t ompi_datatype_f_to_c_table = {{0}};
(PDST)->super.desc = (PSRC)->super.desc; \ (PDST)->super.desc = (PSRC)->super.desc; \
(PDST)->super.opt_desc = (PSRC)->super.opt_desc; \ (PDST)->super.opt_desc = (PSRC)->super.opt_desc; \
(PDST)->packed_description = (PSRC)->packed_description; \ (PDST)->packed_description = (PSRC)->packed_description; \
(PSRC)->packed_description = NULL; \ (PSRC)->packed_description = 0; \
/* transfer the ptypes */ \ /* transfer the ptypes */ \
(PDST)->super.ptypes = (PSRC)->super.ptypes; \ (PDST)->super.ptypes = (PSRC)->super.ptypes; \
(PSRC)->super.ptypes = NULL; \ (PSRC)->super.ptypes = NULL; \

3
ompi/group/group.h
Просмотреть файл

@ -356,7 +356,8 @@ static inline struct ompi_proc_t *ompi_group_dense_lookup (ompi_group_t *group,
ompi_proc_t *real_proc = ompi_proc_t *real_proc =
(ompi_proc_t *) ompi_proc_for_name (ompi_proc_sentinel_to_name ((uintptr_t) proc)); (ompi_proc_t *) ompi_proc_for_name (ompi_proc_sentinel_to_name ((uintptr_t) proc));
if (opal_atomic_compare_exchange_strong_ptr (group->grp_proc_pointers + peer_id, &proc, real_proc)) { if (opal_atomic_compare_exchange_strong_ptr ((opal_atomic_intptr_t *)(group->grp_proc_pointers + peer_id),
(intptr_t *) &proc, (intptr_t) real_proc)) {
OBJ_RETAIN(real_proc); OBJ_RETAIN(real_proc);
} }

2
ompi/mca/coll/libnbc/coll_libnbc.h
Просмотреть файл

@ -75,7 +75,7 @@ struct ompi_coll_libnbc_component_t {
mca_coll_base_component_2_0_0_t super; mca_coll_base_component_2_0_0_t super;
opal_free_list_t requests; opal_free_list_t requests;
opal_list_t active_requests; opal_list_t active_requests;
int32_t active_comms; opal_atomic_int32_t active_comms;
opal_mutex_t lock; /* protect access to the active_requests list */ opal_mutex_t lock; /* protect access to the active_requests list */
}; };
typedef struct ompi_coll_libnbc_component_t ompi_coll_libnbc_component_t; typedef struct ompi_coll_libnbc_component_t ompi_coll_libnbc_component_t;

2
ompi/mca/coll/monitoring/coll_monitoring.h
Просмотреть файл

@ -36,7 +36,7 @@ struct mca_coll_monitoring_module_t {
mca_coll_base_module_t super; mca_coll_base_module_t super;
mca_coll_base_comm_coll_t real; mca_coll_base_comm_coll_t real;
mca_monitoring_coll_data_t*data; mca_monitoring_coll_data_t*data;
int32_t is_initialized; opal_atomic_int32_t is_initialized;
}; };
typedef struct mca_coll_monitoring_module_t mca_coll_monitoring_module_t; typedef struct mca_coll_monitoring_module_t mca_coll_monitoring_module_t;
OMPI_DECLSPEC OBJ_CLASS_DECLARATION(mca_coll_monitoring_module_t); OMPI_DECLSPEC OBJ_CLASS_DECLARATION(mca_coll_monitoring_module_t);

5
ompi/mca/coll/portals4/coll_portals4.h
Просмотреть файл

@ -1,6 +1,7 @@
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/* /*
* Copyright (c) 2013-2015 Sandia National Laboratories. All rights reserved. * Copyright (c) 2013-2015 Sandia National Laboratories. All rights reserved.
* Copyright (c) 2015 Los Alamos National Security, LLC. All rights * Copyright (c) 2015-2018 Los Alamos National Security, LLC. All rights
* reserved. * reserved.
* Copyright (c) 2015 Bull SAS. All rights reserved. * Copyright (c) 2015 Bull SAS. All rights reserved.
* Copyright (c) 2015 Research Organization for Information Science * Copyright (c) 2015 Research Organization for Information Science
@ -91,7 +92,7 @@ typedef struct ompi_coll_portals4_tree_t {
struct mca_coll_portals4_module_t { struct mca_coll_portals4_module_t {
mca_coll_base_module_t super; mca_coll_base_module_t super;
size_t coll_count; opal_atomic_size_t coll_count;
/* record handlers dedicated to fallback if offloaded operations are not supported */ /* record handlers dedicated to fallback if offloaded operations are not supported */
mca_coll_base_module_reduce_fn_t previous_reduce; mca_coll_base_module_reduce_fn_t previous_reduce;

4
ompi/mca/coll/sm/coll_sm.h
Просмотреть файл

@ -114,7 +114,7 @@ BEGIN_C_DECLS
typedef struct mca_coll_sm_in_use_flag_t { typedef struct mca_coll_sm_in_use_flag_t {
/** Number of processes currently using this set of /** Number of processes currently using this set of
segments */ segments */
volatile uint32_t mcsiuf_num_procs_using; opal_atomic_uint32_t mcsiuf_num_procs_using;
/** Must match data->mcb_count */ /** Must match data->mcb_count */
volatile uint32_t mcsiuf_operation_count; volatile uint32_t mcsiuf_operation_count;
} mca_coll_sm_in_use_flag_t; } mca_coll_sm_in_use_flag_t;
@ -152,7 +152,7 @@ BEGIN_C_DECLS
/** Pointer to my parent's barrier control pages (will be NULL /** Pointer to my parent's barrier control pages (will be NULL
for communicator rank 0; odd index pages are "in", even for communicator rank 0; odd index pages are "in", even
index pages are "out") */ index pages are "out") */
uint32_t *mcb_barrier_control_parent; opal_atomic_uint32_t *mcb_barrier_control_parent;
/** Pointers to my childrens' barrier control pages (they're /** Pointers to my childrens' barrier control pages (they're
contiguous in memory, so we only point to the base -- the contiguous in memory, so we only point to the base -- the

3
ompi/mca/coll/sm/coll_sm_barrier.c
Просмотреть файл

@ -56,7 +56,8 @@ int mca_coll_sm_barrier_intra(struct ompi_communicator_t *comm,
int rank, buffer_set; int rank, buffer_set;
mca_coll_sm_comm_t *data; mca_coll_sm_comm_t *data;
uint32_t i, num_children; uint32_t i, num_children;
volatile uint32_t *me_in, *me_out, *parent, *children = NULL; volatile uint32_t *me_in, *me_out, *children = NULL;
opal_atomic_uint32_t *parent;
int uint_control_size; int uint_control_size;
mca_coll_sm_module_t *sm_module = (mca_coll_sm_module_t*) module; mca_coll_sm_module_t *sm_module = (mca_coll_sm_module_t*) module;

2
ompi/mca/coll/sm/coll_sm_module.c
Просмотреть файл

@ -372,7 +372,7 @@ int ompi_coll_sm_lazy_enable(mca_coll_base_module_t *module,
data->mcb_barrier_control_me = (uint32_t*) data->mcb_barrier_control_me = (uint32_t*)
(base + (rank * control_size * num_barrier_buffers * 2)); (base + (rank * control_size * num_barrier_buffers * 2));
if (data->mcb_tree[rank].mcstn_parent) { if (data->mcb_tree[rank].mcstn_parent) {
data->mcb_barrier_control_parent = (uint32_t*) data->mcb_barrier_control_parent = (opal_atomic_uint32_t*)
(base + (base +
(data->mcb_tree[rank].mcstn_parent->mcstn_id * control_size * (data->mcb_tree[rank].mcstn_parent->mcstn_id * control_size *
num_barrier_buffers * 2)); num_barrier_buffers * 2));

32
ompi/mca/common/monitoring/common_monitoring.c
Просмотреть файл

@ -7,7 +7,7 @@
* Copyright (c) 2015 Bull SAS. All rights reserved. * Copyright (c) 2015 Bull SAS. All rights reserved.
* Copyright (c) 2016-2017 Research Organization for Information Science * Copyright (c) 2016-2017 Research Organization for Information Science
* and Technology (RIST). All rights reserved. * and Technology (RIST). All rights reserved.
* Copyright (c) 2017 Los Alamos National Security, LLC. All rights * Copyright (c) 2017-2018 Los Alamos National Security, LLC. All rights
* reserved. * reserved.
* $COPYRIGHT$ * $COPYRIGHT$
* *
@ -34,7 +34,7 @@
/*** Monitoring specific variables ***/ /*** Monitoring specific variables ***/
/* Keep tracks of how many components are currently using the common part */ /* Keep tracks of how many components are currently using the common part */
static int32_t mca_common_monitoring_hold = 0; static opal_atomic_int32_t mca_common_monitoring_hold = 0;
/* Output parameters */ /* Output parameters */
int mca_common_monitoring_output_stream_id = -1; int mca_common_monitoring_output_stream_id = -1;
static opal_output_stream_t mca_common_monitoring_output_stream_obj = { static opal_output_stream_t mca_common_monitoring_output_stream_obj = {
@ -61,18 +61,18 @@ static char* mca_common_monitoring_initial_filename = "";
static char* mca_common_monitoring_current_filename = NULL; static char* mca_common_monitoring_current_filename = NULL;
/* array for stroring monitoring data*/ /* array for stroring monitoring data*/
static size_t* pml_data = NULL; static opal_atomic_size_t* pml_data = NULL;
static size_t* pml_count = NULL; static opal_atomic_size_t* pml_count = NULL;
static size_t* filtered_pml_data = NULL; static opal_atomic_size_t* filtered_pml_data = NULL;
static size_t* filtered_pml_count = NULL; static opal_atomic_size_t* filtered_pml_count = NULL;
static size_t* osc_data_s = NULL; static opal_atomic_size_t* osc_data_s = NULL;
static size_t* osc_count_s = NULL; static opal_atomic_size_t* osc_count_s = NULL;
static size_t* osc_data_r = NULL; static opal_atomic_size_t* osc_data_r = NULL;
static size_t* osc_count_r = NULL; static opal_atomic_size_t* osc_count_r = NULL;
static size_t* coll_data = NULL; static opal_atomic_size_t* coll_data = NULL;
static size_t* coll_count = NULL; static opal_atomic_size_t* coll_count = NULL;
static size_t* size_histogram = NULL; static opal_atomic_size_t* size_histogram = NULL;
static const int max_size_histogram = 66; static const int max_size_histogram = 66;
static double log10_2 = 0.; static double log10_2 = 0.;
@ -241,7 +241,7 @@ void mca_common_monitoring_finalize( void )
opal_output_close(mca_common_monitoring_output_stream_id); opal_output_close(mca_common_monitoring_output_stream_id);
free(mca_common_monitoring_output_stream_obj.lds_prefix); free(mca_common_monitoring_output_stream_obj.lds_prefix);
/* Free internal data structure */ /* Free internal data structure */
free(pml_data); /* a single allocation */ free((void *) pml_data); /* a single allocation */
opal_hash_table_remove_all( common_monitoring_translation_ht ); opal_hash_table_remove_all( common_monitoring_translation_ht );
OBJ_RELEASE(common_monitoring_translation_ht); OBJ_RELEASE(common_monitoring_translation_ht);
mca_common_monitoring_coll_finalize(); mca_common_monitoring_coll_finalize();
@ -446,7 +446,7 @@ int mca_common_monitoring_add_procs(struct ompi_proc_t **procs,
if( NULL == pml_data ) { if( NULL == pml_data ) {
int array_size = (10 + max_size_histogram) * nprocs_world; int array_size = (10 + max_size_histogram) * nprocs_world;
pml_data = (size_t*)calloc(array_size, sizeof(size_t)); pml_data = (opal_atomic_size_t*)calloc(array_size, sizeof(size_t));
pml_count = pml_data + nprocs_world; pml_count = pml_data + nprocs_world;
filtered_pml_data = pml_count + nprocs_world; filtered_pml_data = pml_count + nprocs_world;
filtered_pml_count = filtered_pml_data + nprocs_world; filtered_pml_count = filtered_pml_data + nprocs_world;
@ -493,7 +493,7 @@ int mca_common_monitoring_add_procs(struct ompi_proc_t **procs,
static void mca_common_monitoring_reset( void ) static void mca_common_monitoring_reset( void )
{ {
int array_size = (10 + max_size_histogram) * nprocs_world; int array_size = (10 + max_size_histogram) * nprocs_world;
memset(pml_data, 0, array_size * sizeof(size_t)); memset((void *) pml_data, 0, array_size * sizeof(size_t));
mca_common_monitoring_coll_reset(); mca_common_monitoring_coll_reset();
} }

12
ompi/mca/common/monitoring/common_monitoring_coll.c
Просмотреть файл

@ -30,12 +30,12 @@ struct mca_monitoring_coll_data_t {
int world_rank; int world_rank;
int is_released; int is_released;
ompi_communicator_t*p_comm; ompi_communicator_t*p_comm;
size_t o2a_count; opal_atomic_size_t o2a_count;
size_t o2a_size; opal_atomic_size_t o2a_size;
size_t a2o_count; opal_atomic_size_t a2o_count;
size_t a2o_size; opal_atomic_size_t a2o_size;
size_t a2a_count; opal_atomic_size_t a2a_count;
size_t a2a_size; opal_atomic_size_t a2a_size;
}; };
/* Collectives operation monitoring */ /* Collectives operation monitoring */

2
ompi/mca/common/ompio/common_ompio_cuda.c
Просмотреть файл

@ -34,7 +34,7 @@ static opal_mutex_t mca_common_ompio_cuda_mutex; /* lock for thread saf
static mca_allocator_base_component_t* mca_common_ompio_allocator_component=NULL; static mca_allocator_base_component_t* mca_common_ompio_allocator_component=NULL;
static mca_allocator_base_module_t* mca_common_ompio_allocator=NULL; static mca_allocator_base_module_t* mca_common_ompio_allocator=NULL;
static int32_t mca_common_ompio_cuda_init = 0; static opal_atomic_int32_t mca_common_ompio_cuda_init = 0;
static int32_t mca_common_ompio_pagesize=4096; static int32_t mca_common_ompio_pagesize=4096;
static void* mca_common_ompio_cuda_alloc_seg ( void *ctx, size_t *size ); static void* mca_common_ompio_cuda_alloc_seg ( void *ctx, size_t *size );
static void mca_common_ompio_cuda_free_seg ( void *ctx, void *buf ); static void mca_common_ompio_cuda_free_seg ( void *ctx, void *buf );

4
ompi/mca/mtl/portals4/mtl_portals4.h
Просмотреть файл

@ -115,12 +115,12 @@ struct mca_mtl_portals4_module_t {
opal_mutex_t short_block_mutex; opal_mutex_t short_block_mutex;
/** number of send-side operations started */ /** number of send-side operations started */
uint64_t opcount; opal_atomic_uint64_t opcount;
#if OPAL_ENABLE_DEBUG #if OPAL_ENABLE_DEBUG
/** number of receive-side operations started. Used only for /** number of receive-side operations started. Used only for
debugging */ debugging */
uint64_t recv_opcount; opal_atomic_uint64_t recv_opcount;
#endif #endif
#if OMPI_MTL_PORTALS4_FLOW_CONTROL #if OMPI_MTL_PORTALS4_FLOW_CONTROL

2
ompi/mca/mtl/portals4/mtl_portals4_flowctl.c
Просмотреть файл

@ -1,7 +1,7 @@
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/* /*
* Copyright (c) 2012 Sandia National Laboratories. All rights reserved. * Copyright (c) 2012 Sandia National Laboratories. All rights reserved.
* Copyright (c) 2015-2017 Los Alamos National Security, LLC. All rights * Copyright (c) 2015-2018 Los Alamos National Security, LLC. All rights
* reserved. * reserved.
* $COPYRIGHT$ * $COPYRIGHT$
* *

4
ompi/mca/mtl/portals4/mtl_portals4_flowctl.h
Просмотреть файл

@ -36,7 +36,7 @@ OBJ_CLASS_DECLARATION(ompi_mtl_portals4_pending_request_t);
struct ompi_mtl_portals4_flowctl_t { struct ompi_mtl_portals4_flowctl_t {
int32_t flowctl_active; int32_t flowctl_active;
int32_t send_slots; opal_atomic_int32_t send_slots;
int32_t max_send_slots; int32_t max_send_slots;
opal_list_t pending_sends; opal_list_t pending_sends;
opal_free_list_t pending_fl; opal_free_list_t pending_fl;
@ -46,7 +46,7 @@ struct ompi_mtl_portals4_flowctl_t {
/** Flow control epoch counter. Triggered events should be /** Flow control epoch counter. Triggered events should be
based on epoch counter. */ based on epoch counter. */
int64_t epoch_counter; opal_atomic_int64_t epoch_counter;
/** Flow control trigger CT. Only has meaning at root. */ /** Flow control trigger CT. Only has meaning at root. */
ptl_handle_ct_t trigger_ct_h; ptl_handle_ct_t trigger_ct_h;

6
ompi/mca/mtl/portals4/mtl_portals4_request.h
Просмотреть файл

@ -54,8 +54,8 @@ struct ompi_mtl_portals4_isend_request_t {
struct ompi_mtl_portals4_pending_request_t *pending; struct ompi_mtl_portals4_pending_request_t *pending;
#endif #endif
ptl_size_t length; ptl_size_t length;
int32_t pending_get; opal_atomic_int32_t pending_get;
uint32_t event_count; opal_atomic_uint32_t event_count;
}; };
typedef struct ompi_mtl_portals4_isend_request_t ompi_mtl_portals4_isend_request_t; typedef struct ompi_mtl_portals4_isend_request_t ompi_mtl_portals4_isend_request_t;
@ -76,7 +76,7 @@ struct ompi_mtl_portals4_recv_request_t {
void *delivery_ptr; void *delivery_ptr;
size_t delivery_len; size_t delivery_len;
volatile bool req_started; volatile bool req_started;
int32_t pending_reply; opal_atomic_int32_t pending_reply;
#if OPAL_ENABLE_DEBUG #if OPAL_ENABLE_DEBUG
uint64_t opcount; uint64_t opcount;
ptl_hdr_data_t hdr_data; ptl_hdr_data_t hdr_data;

2
ompi/mca/osc/monitoring/osc_monitoring_module.h
Просмотреть файл

@ -50,7 +50,7 @@
OSC_MONITORING_SET_TEMPLATE_FCT_NAME(template) (ompi_osc_base_module_t*module) \ OSC_MONITORING_SET_TEMPLATE_FCT_NAME(template) (ompi_osc_base_module_t*module) \
{ \ { \
/* Define the ompi_osc_monitoring_module_## template ##_init_done variable */ \ /* Define the ompi_osc_monitoring_module_## template ##_init_done variable */ \
static int32_t init_done = 0; \ opal_atomic_int32_t init_done = 0; \
/* Define and set the ompi_osc_monitoring_## template \ /* Define and set the ompi_osc_monitoring_## template \
* ##_template variable. The functions recorded here are \ * ##_template variable. The functions recorded here are \
* linked to the original functions of the original \ * linked to the original functions of the original \

2
ompi/mca/osc/portals4/osc_portals4.h
Просмотреть файл

@ -95,7 +95,7 @@ struct ompi_osc_portals4_module_t {
ptl_handle_md_t req_md_h; /* memory descriptor with event completion used by this window */ ptl_handle_md_t req_md_h; /* memory descriptor with event completion used by this window */
ptl_handle_me_t data_me_h; /* data match list entry (MB are CID | OSC_PORTALS4_MB_DATA) */ ptl_handle_me_t data_me_h; /* data match list entry (MB are CID | OSC_PORTALS4_MB_DATA) */
ptl_handle_me_t control_me_h; /* match list entry for control data (node_state_t). Match bits are (CID | OSC_PORTALS4_MB_CONTROL). */ ptl_handle_me_t control_me_h; /* match list entry for control data (node_state_t). Match bits are (CID | OSC_PORTALS4_MB_CONTROL). */
int64_t opcount; opal_atomic_int64_t opcount;
ptl_match_bits_t match_bits; /* match bits for module. Same as cid for comm in most cases. */ ptl_match_bits_t match_bits; /* match bits for module. Same as cid for comm in most cases. */
ptl_iovec_t *origin_iovec_list; /* list of memory segments that compose the noncontiguous region */ ptl_iovec_t *origin_iovec_list; /* list of memory segments that compose the noncontiguous region */

14
ompi/mca/osc/portals4/osc_portals4_comm.c
Просмотреть файл

@ -189,7 +189,7 @@ number_of_fragments(ptl_size_t length, ptl_size_t maxlength)
/* put in segments no larger than segment_length */ /* put in segments no larger than segment_length */
static int static int
segmentedPut(int64_t *opcount, segmentedPut(opal_atomic_int64_t *opcount,
ptl_handle_md_t md_h, ptl_handle_md_t md_h,
ptl_size_t origin_offset, ptl_size_t origin_offset,
ptl_size_t put_length, ptl_size_t put_length,
@ -236,7 +236,7 @@ segmentedPut(int64_t *opcount,
/* get in segments no larger than segment_length */ /* get in segments no larger than segment_length */
static int static int
segmentedGet(int64_t *opcount, segmentedGet(opal_atomic_int64_t *opcount,
ptl_handle_md_t md_h, ptl_handle_md_t md_h,
ptl_size_t origin_offset, ptl_size_t origin_offset,
ptl_size_t get_length, ptl_size_t get_length,
@ -280,7 +280,7 @@ segmentedGet(int64_t *opcount,
/* atomic op in segments no larger than segment_length */ /* atomic op in segments no larger than segment_length */
static int static int
segmentedAtomic(int64_t *opcount, segmentedAtomic(opal_atomic_int64_t *opcount,
ptl_handle_md_t md_h, ptl_handle_md_t md_h,
ptl_size_t origin_offset, ptl_size_t origin_offset,
ptl_size_t length, ptl_size_t length,
@ -329,7 +329,7 @@ segmentedAtomic(int64_t *opcount,
/* atomic op in segments no larger than segment_length */ /* atomic op in segments no larger than segment_length */
static int static int
segmentedFetchAtomic(int64_t *opcount, segmentedFetchAtomic(opal_atomic_int64_t *opcount,
ptl_handle_md_t result_md_h, ptl_handle_md_t result_md_h,
ptl_size_t result_offset, ptl_size_t result_offset,
ptl_handle_md_t origin_md_h, ptl_handle_md_t origin_md_h,
@ -381,7 +381,7 @@ segmentedFetchAtomic(int64_t *opcount,
/* swap in segments no larger than segment_length */ /* swap in segments no larger than segment_length */
static int static int
segmentedSwap(int64_t *opcount, segmentedSwap(opal_atomic_int64_t *opcount,
ptl_handle_md_t result_md_h, ptl_handle_md_t result_md_h,
ptl_size_t result_offset, ptl_size_t result_offset,
ptl_handle_md_t origin_md_h, ptl_handle_md_t origin_md_h,
@ -1187,7 +1187,7 @@ fetch_atomic_to_iovec(ompi_osc_portals4_module_t *module,
/* put in the largest chunks possible given the noncontiguous restriction */ /* put in the largest chunks possible given the noncontiguous restriction */
static int static int
put_to_noncontig(int64_t *opcount, put_to_noncontig(opal_atomic_int64_t *opcount,
ptl_handle_md_t md_h, ptl_handle_md_t md_h,
const void *origin_address, const void *origin_address,
int origin_count, int origin_count,
@ -1521,7 +1521,7 @@ atomic_to_noncontig(ompi_osc_portals4_module_t *module,
/* get from a noncontiguous remote to an (non)contiguous local */ /* get from a noncontiguous remote to an (non)contiguous local */
static int static int
get_from_noncontig(int64_t *opcount, get_from_noncontig(opal_atomic_int64_t *opcount,
ptl_handle_md_t md_h, ptl_handle_md_t md_h,
const void *origin_address, const void *origin_address,
int origin_count, int origin_count,

4
ompi/mca/osc/portals4/osc_portals4_request.h
Просмотреть файл

@ -1,7 +1,7 @@
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/* /*
* Copyright (c) 2011-2013 Sandia National Laboratories. All rights reserved. * Copyright (c) 2011-2013 Sandia National Laboratories. All rights reserved.
* Copyright (c) 2015 Los Alamos National Security, LLC. All rights * Copyright (c) 2015-2018 Los Alamos National Security, LLC. All rights
* reserved. * reserved.
* $COPYRIGHT$ * $COPYRIGHT$
* *
@ -18,7 +18,7 @@
struct ompi_osc_portals4_request_t { struct ompi_osc_portals4_request_t {
ompi_request_t super; ompi_request_t super;
int32_t ops_expected; int32_t ops_expected;
volatile int32_t ops_committed; opal_atomic_int32_t ops_committed;
}; };
typedef struct ompi_osc_portals4_request_t ompi_osc_portals4_request_t; typedef struct ompi_osc_portals4_request_t ompi_osc_portals4_request_t;

30
ompi/mca/osc/pt2pt/osc_pt2pt.h
Просмотреть файл

@ -8,7 +8,7 @@
* University of Stuttgart. All rights reserved. * University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California. * Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved. * All rights reserved.
* Copyright (c) 2007-2017 Los Alamos National Security, LLC. All rights * Copyright (c) 2007-2018 Los Alamos National Security, LLC. All rights
* reserved. * reserved.
* Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2012-2013 Sandia National Laboratories. All rights reserved. * Copyright (c) 2012-2013 Sandia National Laboratories. All rights reserved.
@ -110,7 +110,7 @@ struct ompi_osc_pt2pt_peer_t {
int rank; int rank;
/** pointer to the current send fragment for each outgoing target */ /** pointer to the current send fragment for each outgoing target */
struct ompi_osc_pt2pt_frag_t *active_frag; opal_atomic_intptr_t active_frag;
/** lock for this peer */ /** lock for this peer */
opal_mutex_t lock; opal_mutex_t lock;
@ -119,10 +119,10 @@ struct ompi_osc_pt2pt_peer_t {
opal_list_t queued_frags; opal_list_t queued_frags;
/** number of fragments incomming (negative - expected, positive - unsynchronized) */ /** number of fragments incomming (negative - expected, positive - unsynchronized) */
volatile int32_t passive_incoming_frag_count; opal_atomic_int32_t passive_incoming_frag_count;
/** peer flags */ /** peer flags */
volatile int32_t flags; opal_atomic_int32_t flags;
}; };
typedef struct ompi_osc_pt2pt_peer_t ompi_osc_pt2pt_peer_t; typedef struct ompi_osc_pt2pt_peer_t ompi_osc_pt2pt_peer_t;
@ -208,16 +208,16 @@ struct ompi_osc_pt2pt_module_t {
/** Nmber of communication fragments started for this epoch, by /** Nmber of communication fragments started for this epoch, by
peer. Not in peer data to make fence more manageable. */ peer. Not in peer data to make fence more manageable. */
uint32_t *epoch_outgoing_frag_count; opal_atomic_uint32_t *epoch_outgoing_frag_count;
/** cyclic counter for a unique tage for long messages. */ /** cyclic counter for a unique tage for long messages. */
volatile uint32_t tag_counter; opal_atomic_uint32_t tag_counter;
/** number of outgoing fragments still to be completed */ /** number of outgoing fragments still to be completed */
volatile int32_t outgoing_frag_count; opal_atomic_int32_t outgoing_frag_count;
/** number of incoming fragments */ /** number of incoming fragments */
volatile int32_t active_incoming_frag_count; opal_atomic_int32_t active_incoming_frag_count;
/** Number of targets locked/being locked */ /** Number of targets locked/being locked */
unsigned int passive_target_access_epoch; unsigned int passive_target_access_epoch;
@ -230,13 +230,13 @@ struct ompi_osc_pt2pt_module_t {
/** Number of "count" messages from the remote complete group /** Number of "count" messages from the remote complete group
we've received */ we've received */
volatile int32_t num_complete_msgs; opal_atomic_int32_t num_complete_msgs;
/* ********************* LOCK data ************************ */ /* ********************* LOCK data ************************ */
/** Status of the local window lock. One of 0 (unlocked), /** Status of the local window lock. One of 0 (unlocked),
MPI_LOCK_EXCLUSIVE, or MPI_LOCK_SHARED. */ MPI_LOCK_EXCLUSIVE, or MPI_LOCK_SHARED. */
int32_t lock_status; opal_atomic_int32_t lock_status;
/** lock for locks_pending list */ /** lock for locks_pending list */
opal_mutex_t locks_pending_lock; opal_mutex_t locks_pending_lock;
@ -526,7 +526,7 @@ static inline void mark_incoming_completion (ompi_osc_pt2pt_module_t *module, in
OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output,
"mark_incoming_completion marking passive incoming complete. module %p, source = %d, count = %d", "mark_incoming_completion marking passive incoming complete. module %p, source = %d, count = %d",
(void *) module, source, (int) peer->passive_incoming_frag_count + 1)); (void *) module, source, (int) peer->passive_incoming_frag_count + 1));
new_value = OPAL_THREAD_ADD_FETCH32((int32_t *) &peer->passive_incoming_frag_count, 1); new_value = OPAL_THREAD_ADD_FETCH32((opal_atomic_int32_t *) &peer->passive_incoming_frag_count, 1);
if (0 == new_value) { if (0 == new_value) {
OPAL_THREAD_LOCK(&module->lock); OPAL_THREAD_LOCK(&module->lock);
opal_condition_broadcast(&module->cond); opal_condition_broadcast(&module->cond);
@ -550,7 +550,7 @@ static inline void mark_incoming_completion (ompi_osc_pt2pt_module_t *module, in
*/ */
static inline void mark_outgoing_completion (ompi_osc_pt2pt_module_t *module) static inline void mark_outgoing_completion (ompi_osc_pt2pt_module_t *module)
{ {
int32_t new_value = OPAL_THREAD_ADD_FETCH32((int32_t *) &module->outgoing_frag_count, 1); int32_t new_value = OPAL_THREAD_ADD_FETCH32((opal_atomic_int32_t *) &module->outgoing_frag_count, 1);
OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output,
"mark_outgoing_completion: outgoing_frag_count = %d", new_value)); "mark_outgoing_completion: outgoing_frag_count = %d", new_value));
if (new_value >= 0) { if (new_value >= 0) {
@ -574,12 +574,12 @@ static inline void mark_outgoing_completion (ompi_osc_pt2pt_module_t *module)
*/ */
static inline void ompi_osc_signal_outgoing (ompi_osc_pt2pt_module_t *module, int target, int count) static inline void ompi_osc_signal_outgoing (ompi_osc_pt2pt_module_t *module, int target, int count)
{ {
OPAL_THREAD_ADD_FETCH32((int32_t *) &module->outgoing_frag_count, -count); OPAL_THREAD_ADD_FETCH32((opal_atomic_int32_t *) &module->outgoing_frag_count, -count);
if (MPI_PROC_NULL != target) { if (MPI_PROC_NULL != target) {
OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output,
"ompi_osc_signal_outgoing_passive: target = %d, count = %d, total = %d", target, "ompi_osc_signal_outgoing_passive: target = %d, count = %d, total = %d", target,
count, module->epoch_outgoing_frag_count[target] + count)); count, module->epoch_outgoing_frag_count[target] + count));
OPAL_THREAD_ADD_FETCH32((int32_t *) (module->epoch_outgoing_frag_count + target), count); OPAL_THREAD_ADD_FETCH32((opal_atomic_int32_t *) (module->epoch_outgoing_frag_count + target), count);
} }
} }
@ -717,7 +717,7 @@ static inline int get_tag(ompi_osc_pt2pt_module_t *module)
/* the LSB of the tag is used be the receiver to determine if the /* the LSB of the tag is used be the receiver to determine if the
message is a passive or active target (ie, where to mark message is a passive or active target (ie, where to mark
completion). */ completion). */
int32_t tmp = OPAL_THREAD_ADD_FETCH32((volatile int32_t *) &module->tag_counter, 4); int32_t tmp = OPAL_THREAD_ADD_FETCH32((opal_atomic_int32_t *) &module->tag_counter, 4);
return (tmp & OSC_PT2PT_FRAG_MASK) | !!(module->passive_target_access_epoch); return (tmp & OSC_PT2PT_FRAG_MASK) | !!(module->passive_target_access_epoch);
} }

20
ompi/mca/osc/pt2pt/osc_pt2pt_active_target.c
Просмотреть файл

@ -8,7 +8,7 @@
* University of Stuttgart. All rights reserved. * University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California. * Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved. * All rights reserved.
* Copyright (c) 2007-2016 Los Alamos National Security, LLC. All rights * Copyright (c) 2007-2018 Los Alamos National Security, LLC. All rights
* reserved. * reserved.
* Copyright (c) 2010-2016 IBM Corporation. All rights reserved. * Copyright (c) 2010-2016 IBM Corporation. All rights reserved.
* Copyright (c) 2012-2013 Sandia National Laboratories. All rights reserved. * Copyright (c) 2012-2013 Sandia National Laboratories. All rights reserved.
@ -166,17 +166,16 @@ int ompi_osc_pt2pt_fence(int assert, ompi_win_t *win)
"osc pt2pt: fence done sending")); "osc pt2pt: fence done sending"));
/* find out how much data everyone is going to send us. */ /* find out how much data everyone is going to send us. */
ret = module->comm->c_coll->coll_reduce_scatter_block (module->epoch_outgoing_frag_count, ret = module->comm->c_coll->coll_reduce_scatter_block ((void *) module->epoch_outgoing_frag_count,
&incoming_reqs, 1, MPI_UINT32_T, &incoming_reqs, 1, MPI_UINT32_T,
MPI_SUM, module->comm, MPI_SUM, module->comm,
module->comm->c_coll->coll_reduce_scatter_block_module); module->comm->c_coll->coll_reduce_scatter_block_module);
if (OMPI_SUCCESS != ret) { if (OMPI_SUCCESS != ret) {
return ret; return ret;
} }
OPAL_THREAD_LOCK(&module->lock); OPAL_THREAD_LOCK(&module->lock);
bzero(module->epoch_outgoing_frag_count, bzero ((void *) module->epoch_outgoing_frag_count, sizeof(uint32_t) * ompi_comm_size(module->comm));
sizeof(uint32_t) * ompi_comm_size(module->comm));
OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output,
"osc pt2pt: fence expects %d requests", "osc pt2pt: fence expects %d requests",
@ -366,8 +365,11 @@ int ompi_osc_pt2pt_complete (ompi_win_t *win)
/* XXX -- TODO -- since fragment are always delivered in order we do not need to count anything but long /* XXX -- TODO -- since fragment are always delivered in order we do not need to count anything but long
* requests. once that is done this can be removed. */ * requests. once that is done this can be removed. */
if (peer->active_frag && (peer->active_frag->remain_len < sizeof (complete_req))) { if (peer->active_frag) {
++complete_req.frag_count; ompi_osc_pt2pt_frag_t *active_frag = (ompi_osc_pt2pt_frag_t *) peer->active_frag;
if (active_frag->remain_len < sizeof (complete_req)) {
++complete_req.frag_count;
}
} }
OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output,

2
ompi/mca/osc/pt2pt/osc_pt2pt_component.c
Просмотреть файл

@ -501,7 +501,7 @@ static void ompi_osc_pt2pt_peer_construct (ompi_osc_pt2pt_peer_t *peer)
{ {
OBJ_CONSTRUCT(&peer->queued_frags, opal_list_t); OBJ_CONSTRUCT(&peer->queued_frags, opal_list_t);
OBJ_CONSTRUCT(&peer->lock, opal_mutex_t); OBJ_CONSTRUCT(&peer->lock, opal_mutex_t);
peer->active_frag = NULL; peer->active_frag = 0;
peer->passive_incoming_frag_count = 0; peer->passive_incoming_frag_count = 0;
peer->flags = 0; peer->flags = 0;
} }

4
ompi/mca/osc/pt2pt/osc_pt2pt_data_move.c
Просмотреть файл

@ -8,7 +8,7 @@
* University of Stuttgart. All rights reserved. * University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California. * Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved. * All rights reserved.
* Copyright (c) 2007-2017 Los Alamos National Security, LLC. All rights * Copyright (c) 2007-2018 Los Alamos National Security, LLC. All rights
* reserved. * reserved.
* Copyright (c) 2009-2011 Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2009-2011 Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2012-2013 Sandia National Laboratories. All rights reserved. * Copyright (c) 2012-2013 Sandia National Laboratories. All rights reserved.
@ -56,7 +56,7 @@ struct osc_pt2pt_accumulate_data_t {
int peer; int peer;
ompi_datatype_t *datatype; ompi_datatype_t *datatype;
ompi_op_t *op; ompi_op_t *op;
int request_count; opal_atomic_int32_t request_count;
}; };
typedef struct osc_pt2pt_accumulate_data_t osc_pt2pt_accumulate_data_t; typedef struct osc_pt2pt_accumulate_data_t osc_pt2pt_accumulate_data_t;

8
ompi/mca/osc/pt2pt/osc_pt2pt_frag.c
Просмотреть файл

@ -1,7 +1,7 @@
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/* /*
* Copyright (c) 2012-2013 Sandia National Laboratories. All rights reserved. * Copyright (c) 2012-2013 Sandia National Laboratories. All rights reserved.
* Copyright (c) 2014-2017 Los Alamos National Security, LLC. All rights * Copyright (c) 2014-2018 Los Alamos National Security, LLC. All rights
* reserved. * reserved.
* Copyright (c) 2015 Research Organization for Information Science * Copyright (c) 2015 Research Organization for Information Science
* and Technology (RIST). All rights reserved. * and Technology (RIST). All rights reserved.
@ -65,7 +65,7 @@ int ompi_osc_pt2pt_frag_start (ompi_osc_pt2pt_module_t *module,
ompi_osc_pt2pt_peer_t *peer = ompi_osc_pt2pt_peer_lookup (module, frag->target); ompi_osc_pt2pt_peer_t *peer = ompi_osc_pt2pt_peer_lookup (module, frag->target);
int ret; int ret;
assert(0 == frag->pending && peer->active_frag != frag); assert(0 == frag->pending && peer->active_frag != (intptr_t) frag);
/* we need to signal now that a frag is outgoing to ensure the count sent /* we need to signal now that a frag is outgoing to ensure the count sent
* with the unlock message is correct */ * with the unlock message is correct */
@ -93,7 +93,7 @@ int ompi_osc_pt2pt_frag_start (ompi_osc_pt2pt_module_t *module,
static int ompi_osc_pt2pt_flush_active_frag (ompi_osc_pt2pt_module_t *module, ompi_osc_pt2pt_peer_t *peer) static int ompi_osc_pt2pt_flush_active_frag (ompi_osc_pt2pt_module_t *module, ompi_osc_pt2pt_peer_t *peer)
{ {
ompi_osc_pt2pt_frag_t *active_frag = peer->active_frag; ompi_osc_pt2pt_frag_t *active_frag = (ompi_osc_pt2pt_frag_t *) peer->active_frag;
int ret = OMPI_SUCCESS; int ret = OMPI_SUCCESS;
if (NULL == active_frag) { if (NULL == active_frag) {
@ -105,7 +105,7 @@ static int ompi_osc_pt2pt_flush_active_frag (ompi_osc_pt2pt_module_t *module, om
"osc pt2pt: flushing active fragment to target %d. pending: %d", "osc pt2pt: flushing active fragment to target %d. pending: %d",
active_frag->target, active_frag->pending)); active_frag->target, active_frag->pending));
if (opal_atomic_compare_exchange_strong_ptr (&peer->active_frag, &active_frag, NULL)) { if (opal_atomic_compare_exchange_strong_ptr (&peer->active_frag, (intptr_t *) &active_frag, 0)) {
if (0 != OPAL_THREAD_ADD_FETCH32(&active_frag->pending, -1)) { if (0 != OPAL_THREAD_ADD_FETCH32(&active_frag->pending, -1)) {
/* communication going on while synchronizing; this is an rma usage bug */ /* communication going on while synchronizing; this is an rma usage bug */
return OMPI_ERR_RMA_SYNC; return OMPI_ERR_RMA_SYNC;

10
ompi/mca/osc/pt2pt/osc_pt2pt_frag.h
Просмотреть файл

@ -33,7 +33,7 @@ struct ompi_osc_pt2pt_frag_t {
char *top; char *top;
/* Number of operations which have started writing into the frag, but not yet completed doing so */ /* Number of operations which have started writing into the frag, but not yet completed doing so */
volatile int32_t pending; opal_atomic_int32_t pending;
int32_t pending_long_sends; int32_t pending_long_sends;
ompi_osc_pt2pt_frag_header_t *header; ompi_osc_pt2pt_frag_header_t *header;
ompi_osc_pt2pt_module_t *module; ompi_osc_pt2pt_module_t *module;
@ -66,8 +66,8 @@ static inline ompi_osc_pt2pt_frag_t *ompi_osc_pt2pt_frag_alloc_non_buffered (omp
ompi_osc_pt2pt_frag_t *curr; ompi_osc_pt2pt_frag_t *curr;
/* to ensure ordering flush the buffer on the peer */ /* to ensure ordering flush the buffer on the peer */
curr = peer->active_frag; curr = (ompi_osc_pt2pt_frag_t *) peer->active_frag;
if (NULL != curr && opal_atomic_compare_exchange_strong_ptr (&peer->active_frag, &curr, NULL)) { if (NULL != curr && opal_atomic_compare_exchange_strong_ptr (&peer->active_frag, (intptr_t *) &curr, 0)) {
/* If there's something pending, the pending finish will /* If there's something pending, the pending finish will
start the buffer. Otherwise, we need to start it now. */ start the buffer. Otherwise, we need to start it now. */
int ret = ompi_osc_pt2pt_frag_finish (module, curr); int ret = ompi_osc_pt2pt_frag_finish (module, curr);
@ -131,7 +131,7 @@ static inline int _ompi_osc_pt2pt_frag_alloc (ompi_osc_pt2pt_module_t *module, i
OPAL_THREAD_LOCK(&module->lock); OPAL_THREAD_LOCK(&module->lock);
if (buffered) { if (buffered) {
curr = peer->active_frag; curr = (ompi_osc_pt2pt_frag_t *) peer->active_frag;
if (NULL == curr || curr->remain_len < request_len || (long_send && curr->pending_long_sends == 32)) { if (NULL == curr || curr->remain_len < request_len || (long_send && curr->pending_long_sends == 32)) {
curr = ompi_osc_pt2pt_frag_alloc_non_buffered (module, peer, request_len); curr = ompi_osc_pt2pt_frag_alloc_non_buffered (module, peer, request_len);
if (OPAL_UNLIKELY(NULL == curr)) { if (OPAL_UNLIKELY(NULL == curr)) {
@ -140,7 +140,7 @@ static inline int _ompi_osc_pt2pt_frag_alloc (ompi_osc_pt2pt_module_t *module, i
} }
curr->pending_long_sends = long_send; curr->pending_long_sends = long_send;
peer->active_frag = curr; peer->active_frag = (uintptr_t) curr;
} else { } else {
OPAL_THREAD_ADD_FETCH32(&curr->header->num_ops, 1); OPAL_THREAD_ADD_FETCH32(&curr->header->num_ops, 1);
curr->pending_long_sends += long_send; curr->pending_long_sends += long_send;

4
ompi/mca/osc/pt2pt/osc_pt2pt_header.h
Просмотреть файл

@ -8,7 +8,7 @@
* University of Stuttgart. All rights reserved. * University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California. * Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved. * All rights reserved.
* Copyright (c) 2007-2015 Los Alamos National Security, LLC. All rights * Copyright (c) 2007-2018 Los Alamos National Security, LLC. All rights
* reserved. * reserved.
* Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved.
@ -180,7 +180,7 @@ typedef struct ompi_osc_pt2pt_header_flush_ack_t ompi_osc_pt2pt_header_flush_ack
struct ompi_osc_pt2pt_frag_header_t { struct ompi_osc_pt2pt_frag_header_t {
ompi_osc_pt2pt_header_base_t base; ompi_osc_pt2pt_header_base_t base;
uint32_t source; /* rank in window of source process */ uint32_t source; /* rank in window of source process */
int32_t num_ops; /* number of operations in this buffer */ opal_atomic_int32_t num_ops; /* number of operations in this buffer */
uint32_t pad; /* ensure the fragment header is a multiple of 8 bytes */ uint32_t pad; /* ensure the fragment header is a multiple of 8 bytes */
}; };
typedef struct ompi_osc_pt2pt_frag_header_t ompi_osc_pt2pt_frag_header_t; typedef struct ompi_osc_pt2pt_frag_header_t ompi_osc_pt2pt_frag_header_t;

6
ompi/mca/osc/pt2pt/osc_pt2pt_module.c
Просмотреть файл

@ -8,7 +8,7 @@
* University of Stuttgart. All rights reserved. * University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California. * Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved. * All rights reserved.
* Copyright (c) 2007-2016 Los Alamos National Security, LLC. All rights * Copyright (c) 2007-2018 Los Alamos National Security, LLC. All rights
* reserved. * reserved.
* Copyright (c) 2012-2013 Sandia National Laboratories. All rights reserved. * Copyright (c) 2012-2013 Sandia National Laboratories. All rights reserved.
* Copyright (c) 2015 Research Organization for Information Science * Copyright (c) 2015 Research Organization for Information Science
@ -104,13 +104,13 @@ int ompi_osc_pt2pt_free(ompi_win_t *win)
free (module->recv_frags); free (module->recv_frags);
} }
if (NULL != module->epoch_outgoing_frag_count) free(module->epoch_outgoing_frag_count); free ((void *) module->epoch_outgoing_frag_count);
if (NULL != module->comm) { if (NULL != module->comm) {
ompi_comm_free(&module->comm); ompi_comm_free(&module->comm);
} }
if (NULL != module->free_after) free(module->free_after); free ((void *) module->free_after);
free (module); free (module);

28
ompi/mca/osc/pt2pt/osc_pt2pt_passive_target.c
Просмотреть файл

@ -8,7 +8,7 @@
* University of Stuttgart. All rights reserved. * University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California. * Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved. * All rights reserved.
* Copyright (c) 2007-2017 Los Alamos National Security, LLC. All rights * Copyright (c) 2007-2018 Los Alamos National Security, LLC. All rights
* reserved. * reserved.
* Copyright (c) 2010-2016 IBM Corporation. All rights reserved. * Copyright (c) 2010-2016 IBM Corporation. All rights reserved.
* Copyright (c) 2012-2013 Sandia National Laboratories. All rights reserved. * Copyright (c) 2012-2013 Sandia National Laboratories. All rights reserved.
@ -157,7 +157,7 @@ int ompi_osc_pt2pt_lock_remote (ompi_osc_pt2pt_module_t *module, int target, omp
static inline int ompi_osc_pt2pt_unlock_remote (ompi_osc_pt2pt_module_t *module, int target, ompi_osc_pt2pt_sync_t *lock) static inline int ompi_osc_pt2pt_unlock_remote (ompi_osc_pt2pt_module_t *module, int target, ompi_osc_pt2pt_sync_t *lock)
{ {
int32_t frag_count = opal_atomic_swap_32 ((int32_t *) module->epoch_outgoing_frag_count + target, -1); int32_t frag_count = opal_atomic_swap_32 ((opal_atomic_int32_t *) module->epoch_outgoing_frag_count + target, -1);
ompi_osc_pt2pt_peer_t *peer = ompi_osc_pt2pt_peer_lookup (module, target); ompi_osc_pt2pt_peer_t *peer = ompi_osc_pt2pt_peer_lookup (module, target);
int lock_type = lock->sync.lock.type; int lock_type = lock->sync.lock.type;
ompi_osc_pt2pt_header_unlock_t unlock_req; ompi_osc_pt2pt_header_unlock_t unlock_req;
@ -178,10 +178,13 @@ static inline int ompi_osc_pt2pt_unlock_remote (ompi_osc_pt2pt_module_t *module,
unlock_req.lock_ptr = (uint64_t) (uintptr_t) lock; unlock_req.lock_ptr = (uint64_t) (uintptr_t) lock;
OSC_PT2PT_HTON(&unlock_req, module, target); OSC_PT2PT_HTON(&unlock_req, module, target);
if (peer->active_frag && peer->active_frag->remain_len < sizeof (unlock_req)) { if (peer->active_frag) {
/* the peer should expect one more packet */ ompi_osc_pt2pt_frag_t *active_frag = (ompi_osc_pt2pt_frag_t *) peer->active_frag;
++unlock_req.frag_count; if (active_frag->remain_len < sizeof (unlock_req)) {
--module->epoch_outgoing_frag_count[target]; /* the peer should expect one more packet */
++unlock_req.frag_count;
--module->epoch_outgoing_frag_count[target];
}
} }
OPAL_OUTPUT_VERBOSE((25, ompi_osc_base_framework.framework_output, OPAL_OUTPUT_VERBOSE((25, ompi_osc_base_framework.framework_output,
@ -204,7 +207,7 @@ static inline int ompi_osc_pt2pt_flush_remote (ompi_osc_pt2pt_module_t *module,
{ {
ompi_osc_pt2pt_peer_t *peer = ompi_osc_pt2pt_peer_lookup (module, target); ompi_osc_pt2pt_peer_t *peer = ompi_osc_pt2pt_peer_lookup (module, target);
ompi_osc_pt2pt_header_flush_t flush_req; ompi_osc_pt2pt_header_flush_t flush_req;
int32_t frag_count = opal_atomic_swap_32 ((int32_t *) module->epoch_outgoing_frag_count + target, -1); int32_t frag_count = opal_atomic_swap_32 ((opal_atomic_int32_t *) module->epoch_outgoing_frag_count + target, -1);
int ret; int ret;
(void) OPAL_THREAD_ADD_FETCH32(&lock->sync_expected, 1); (void) OPAL_THREAD_ADD_FETCH32(&lock->sync_expected, 1);
@ -218,10 +221,13 @@ static inline int ompi_osc_pt2pt_flush_remote (ompi_osc_pt2pt_module_t *module,
/* XXX -- TODO -- since fragment are always delivered in order we do not need to count anything but long /* XXX -- TODO -- since fragment are always delivered in order we do not need to count anything but long
* requests. once that is done this can be removed. */ * requests. once that is done this can be removed. */
if (peer->active_frag && (peer->active_frag->remain_len < sizeof (flush_req))) { if (peer->active_frag) {
/* the peer should expect one more packet */ ompi_osc_pt2pt_frag_t *active_frag = (ompi_osc_pt2pt_frag_t *) peer->active_frag;
++flush_req.frag_count; if (active_frag->remain_len < sizeof (flush_req)) {
--module->epoch_outgoing_frag_count[target]; /* the peer should expect one more packet */
++flush_req.frag_count;
--module->epoch_outgoing_frag_count[target];
}
} }
OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, "flushing to target %d, frag_count: %d", OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, "flushing to target %d, frag_count: %d",

2
ompi/mca/osc/pt2pt/osc_pt2pt_request.h
Просмотреть файл

@ -28,7 +28,7 @@ struct ompi_osc_pt2pt_request_t {
int origin_count; int origin_count;
struct ompi_datatype_t *origin_dt; struct ompi_datatype_t *origin_dt;
ompi_osc_pt2pt_module_t* module; ompi_osc_pt2pt_module_t* module;
int32_t outstanding_requests; opal_atomic_int32_t outstanding_requests;
bool internal; bool internal;
}; };
typedef struct ompi_osc_pt2pt_request_t ompi_osc_pt2pt_request_t; typedef struct ompi_osc_pt2pt_request_t ompi_osc_pt2pt_request_t;

4
ompi/mca/osc/pt2pt/osc_pt2pt_sync.h
Просмотреть файл

@ -1,6 +1,6 @@
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/* /*
* Copyright (c) 2015-2016 Los Alamos National Security, LLC. All rights * Copyright (c) 2015-2018 Los Alamos National Security, LLC. All rights
* reserved. * reserved.
* $COPYRIGHT$ * $COPYRIGHT$
* *
@ -74,7 +74,7 @@ struct ompi_osc_pt2pt_sync_t {
int num_peers; int num_peers;
/** number of synchronization messages expected */ /** number of synchronization messages expected */
volatile int32_t sync_expected; opal_atomic_int32_t sync_expected;
/** eager sends are active to all peers in this access epoch */ /** eager sends are active to all peers in this access epoch */
volatile bool eager_send_active; volatile bool eager_send_active;

2
ompi/mca/osc/rdma/osc_rdma.h
Просмотреть файл

@ -265,7 +265,7 @@ struct ompi_osc_rdma_module_t {
unsigned long get_retry_count; unsigned long get_retry_count;
/** outstanding atomic operations */ /** outstanding atomic operations */
volatile int32_t pending_ops; opal_atomic_int32_t pending_ops;
}; };
typedef struct ompi_osc_rdma_module_t ompi_osc_rdma_module_t; typedef struct ompi_osc_rdma_module_t ompi_osc_rdma_module_t;
OMPI_MODULE_DECLSPEC extern ompi_osc_rdma_component_t mca_osc_rdma_component; OMPI_MODULE_DECLSPEC extern ompi_osc_rdma_component_t mca_osc_rdma_component;

6
ompi/mca/osc/rdma/osc_rdma_active_target.c
Просмотреть файл

@ -259,7 +259,7 @@ static int ompi_osc_rdma_post_peer (ompi_osc_rdma_module_t *module, ompi_osc_rdm
return ret; return ret;
} }
} else { } else {
post_index = ompi_osc_rdma_counter_add ((osc_rdma_counter_t *) (intptr_t) target, 1) - 1; post_index = ompi_osc_rdma_counter_add ((osc_rdma_atomic_counter_t *) (intptr_t) target, 1) - 1;
} }
post_index &= OMPI_OSC_RDMA_POST_PEER_MAX - 1; post_index &= OMPI_OSC_RDMA_POST_PEER_MAX - 1;
@ -279,7 +279,7 @@ static int ompi_osc_rdma_post_peer (ompi_osc_rdma_module_t *module, ompi_osc_rdm
return ret; return ret;
} }
} else { } else {
result = !ompi_osc_rdma_lock_compare_exchange ((osc_rdma_counter_t *) target, &_tmp_value, result = !ompi_osc_rdma_lock_compare_exchange ((osc_rdma_atomic_counter_t *) target, &_tmp_value,
1 + (osc_rdma_counter_t) my_rank); 1 + (osc_rdma_counter_t) my_rank);
} }
@ -491,7 +491,7 @@ int ompi_osc_rdma_complete_atomic (ompi_win_t *win)
ret = ompi_osc_rdma_lock_btl_op (module, peer, target, MCA_BTL_ATOMIC_ADD, 1, true); ret = ompi_osc_rdma_lock_btl_op (module, peer, target, MCA_BTL_ATOMIC_ADD, 1, true);
assert (OMPI_SUCCESS == ret); assert (OMPI_SUCCESS == ret);
} else { } else {
(void) ompi_osc_rdma_counter_add ((osc_rdma_counter_t *) target, 1); (void) ompi_osc_rdma_counter_add ((osc_rdma_atomic_counter_t *) target, 1);
} }
} }

2
ompi/mca/osc/rdma/osc_rdma_frag.h
Просмотреть файл

@ -72,7 +72,7 @@ static inline int ompi_osc_rdma_frag_alloc (ompi_osc_rdma_module_t *module, size
} }
} }
if (!opal_atomic_compare_exchange_strong_ptr (&module->rdma_frag, &(void *){NULL}, curr)) { if (!opal_atomic_compare_exchange_strong_ptr ((opal_atomic_intptr_t *) &module->rdma_frag, &(intptr_t){0}, (intptr_t) curr)) {
ompi_osc_rdma_deregister (module, curr->handle); ompi_osc_rdma_deregister (module, curr->handle);
curr->handle = NULL; curr->handle = NULL;

16
ompi/mca/osc/rdma/osc_rdma_lock.h
Просмотреть файл

@ -1,6 +1,6 @@
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/* /*
* Copyright (c) 2014-2017 Los Alamos National Security, LLC. All rights * Copyright (c) 2014-2018 Los Alamos National Security, LLC. All rights
* reserved. * reserved.
* $COPYRIGHT$ * $COPYRIGHT$
* *
@ -15,13 +15,13 @@
#include "osc_rdma_types.h" #include "osc_rdma_types.h"
#include "osc_rdma_frag.h" #include "osc_rdma_frag.h"
static inline int ompi_osc_rdma_trylock_local (volatile ompi_osc_rdma_lock_t *lock) static inline int ompi_osc_rdma_trylock_local (ompi_osc_rdma_atomic_lock_t *lock)
{ {
ompi_osc_rdma_lock_t _tmp_value = 0; ompi_osc_rdma_lock_t _tmp_value = 0;
return !ompi_osc_rdma_lock_compare_exchange (lock, &_tmp_value, OMPI_OSC_RDMA_LOCK_EXCLUSIVE); return !ompi_osc_rdma_lock_compare_exchange (lock, &_tmp_value, OMPI_OSC_RDMA_LOCK_EXCLUSIVE);
} }
static inline void ompi_osc_rdma_unlock_local (volatile ompi_osc_rdma_lock_t *lock) static inline void ompi_osc_rdma_unlock_local (ompi_osc_rdma_atomic_lock_t *lock)
{ {
(void) ompi_osc_rdma_lock_add (lock, -OMPI_OSC_RDMA_LOCK_EXCLUSIVE); (void) ompi_osc_rdma_lock_add (lock, -OMPI_OSC_RDMA_LOCK_EXCLUSIVE);
} }
@ -266,7 +266,7 @@ static inline int ompi_osc_rdma_lock_release_shared (ompi_osc_rdma_module_t *mod
return ompi_osc_rdma_lock_btl_op (module, peer, lock, MCA_BTL_ATOMIC_ADD, value, false); return ompi_osc_rdma_lock_btl_op (module, peer, lock, MCA_BTL_ATOMIC_ADD, value, false);
} }
(void) ompi_osc_rdma_lock_add ((volatile ompi_osc_rdma_lock_t *) lock, value); (void) ompi_osc_rdma_lock_add ((ompi_osc_rdma_atomic_lock_t *) lock, value);
return OMPI_SUCCESS; return OMPI_SUCCESS;
} }
@ -320,14 +320,14 @@ static inline int ompi_osc_rdma_lock_acquire_shared (ompi_osc_rdma_module_t *mod
} while (1); } while (1);
} else { } else {
do { do {
lock_state = ompi_osc_rdma_lock_add ((volatile ompi_osc_rdma_lock_t *) lock, value); lock_state = ompi_osc_rdma_lock_add ((ompi_osc_rdma_atomic_lock_t *) lock, value);
OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_DEBUG, "local shared lock incremented. old value 0x%lx", OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_DEBUG, "local shared lock incremented. old value 0x%lx",
(unsigned long) lock_state); (unsigned long) lock_state);
if (!(lock_state & check)) { if (!(lock_state & check)) {
break; break;
} }
(void) ompi_osc_rdma_lock_add ((volatile ompi_osc_rdma_lock_t *) lock, -value); (void) ompi_osc_rdma_lock_add ((ompi_osc_rdma_atomic_lock_t *) lock, -value);
ompi_osc_rdma_progress (module); ompi_osc_rdma_progress (module);
} while (1); } while (1);
} }
@ -378,7 +378,7 @@ static inline int ompi_osc_rdma_lock_try_acquire_exclusive (ompi_osc_rdma_module
return lock_state != 0; return lock_state != 0;
} }
return ompi_osc_rdma_trylock_local ((int64_t *)(intptr_t) lock); return ompi_osc_rdma_trylock_local ((ompi_osc_rdma_atomic_lock_t *)(intptr_t) lock);
} }
/** /**
@ -431,7 +431,7 @@ static inline int ompi_osc_rdma_lock_release_exclusive (ompi_osc_rdma_module_t *
abort (); abort ();
} }
} else { } else {
ompi_osc_rdma_unlock_local ((volatile ompi_osc_rdma_lock_t *)(intptr_t) lock); ompi_osc_rdma_unlock_local ((ompi_osc_rdma_atomic_lock_t *)(intptr_t) lock);
} }
OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_DEBUG, "exclusive lock released"); OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_DEBUG, "exclusive lock released");

2
ompi/mca/osc/rdma/osc_rdma_peer.h
Просмотреть файл

@ -43,7 +43,7 @@ struct ompi_osc_rdma_peer_t {
int rank; int rank;
/** peer flags */ /** peer flags */
volatile int32_t flags; opal_atomic_int32_t flags;
}; };
typedef struct ompi_osc_rdma_peer_t ompi_osc_rdma_peer_t; typedef struct ompi_osc_rdma_peer_t ompi_osc_rdma_peer_t;

2
ompi/mca/osc/rdma/osc_rdma_request.h
Просмотреть файл

@ -40,7 +40,7 @@ struct ompi_osc_rdma_request_t {
void *origin_addr; void *origin_addr;
ompi_osc_rdma_module_t *module; ompi_osc_rdma_module_t *module;
volatile int32_t outstanding_requests; opal_atomic_int32_t outstanding_requests;
bool internal; bool internal;
ptrdiff_t offset; ptrdiff_t offset;

2
ompi/mca/osc/rdma/osc_rdma_sync.h
Просмотреть файл

@ -34,7 +34,7 @@ typedef enum ompi_osc_rdma_sync_type_t ompi_osc_rdma_sync_type_t;
struct ompi_osc_rdma_module_t; struct ompi_osc_rdma_module_t;
struct ompi_osc_rdma_sync_aligned_counter_t { struct ompi_osc_rdma_sync_aligned_counter_t {
volatile osc_rdma_counter_t counter; osc_rdma_atomic_counter_t counter;
/* pad out to next cache line */ /* pad out to next cache line */
uint64_t padding[7]; uint64_t padding[7];
}; };

20
ompi/mca/osc/rdma/osc_rdma_types.h
Просмотреть файл

@ -24,6 +24,7 @@ struct ompi_osc_rdma_peer_t;
typedef int64_t osc_rdma_base_t; typedef int64_t osc_rdma_base_t;
typedef int64_t osc_rdma_size_t; typedef int64_t osc_rdma_size_t;
typedef int64_t osc_rdma_counter_t; typedef int64_t osc_rdma_counter_t;
typedef opal_atomic_int64_t osc_rdma_atomic_counter_t;
#define ompi_osc_rdma_counter_add opal_atomic_add_fetch_64 #define ompi_osc_rdma_counter_add opal_atomic_add_fetch_64
@ -32,6 +33,7 @@ typedef int64_t osc_rdma_counter_t;
typedef int32_t osc_rdma_base_t; typedef int32_t osc_rdma_base_t;
typedef int32_t osc_rdma_size_t; typedef int32_t osc_rdma_size_t;
typedef int32_t osc_rdma_counter_t; typedef int32_t osc_rdma_counter_t;
typedef opal_atomic_int32_t osc_rdma_atomic_counter_t;
#define ompi_osc_rdma_counter_add opal_atomic_add_fetch_32 #define ompi_osc_rdma_counter_add opal_atomic_add_fetch_32
@ -42,8 +44,9 @@ typedef int32_t osc_rdma_counter_t;
#define OMPI_OSC_RDMA_LOCK_EXCLUSIVE 0x8000000000000000l #define OMPI_OSC_RDMA_LOCK_EXCLUSIVE 0x8000000000000000l
typedef int64_t ompi_osc_rdma_lock_t; typedef int64_t ompi_osc_rdma_lock_t;
typedef opal_atomic_int64_t ompi_osc_rdma_atomic_lock_t;
static inline int64_t ompi_osc_rdma_lock_add (volatile int64_t *p, int64_t value) static inline int64_t ompi_osc_rdma_lock_add (opal_atomic_int64_t *p, int64_t value)
{ {
int64_t new; int64_t new;
@ -54,7 +57,7 @@ static inline int64_t ompi_osc_rdma_lock_add (volatile int64_t *p, int64_t value
return new; return new;
} }
static inline int ompi_osc_rdma_lock_compare_exchange (volatile int64_t *p, int64_t *comp, int64_t value) static inline int ompi_osc_rdma_lock_compare_exchange (opal_atomic_int64_t *p, int64_t *comp, int64_t value)
{ {
int ret; int ret;
@ -70,8 +73,9 @@ static inline int ompi_osc_rdma_lock_compare_exchange (volatile int64_t *p, int6
#define OMPI_OSC_RDMA_LOCK_EXCLUSIVE 0x80000000l #define OMPI_OSC_RDMA_LOCK_EXCLUSIVE 0x80000000l
typedef int32_t ompi_osc_rdma_lock_t; typedef int32_t ompi_osc_rdma_lock_t;
typedef opal_atomic_int32_t ompi_osc_rdma_atomic_lock_t;
static inline int32_t ompi_osc_rdma_lock_add (volatile int32_t *p, int32_t value) static inline int32_t ompi_osc_rdma_lock_add (opal_atomic_int32_t *p, int32_t value)
{ {
int32_t new; int32_t new;
@ -83,7 +87,7 @@ static inline int32_t ompi_osc_rdma_lock_add (volatile int32_t *p, int32_t value
return new; return new;
} }
static inline int ompi_osc_rdma_lock_compare_exchange (volatile int32_t *p, int32_t *comp, int32_t value) static inline int ompi_osc_rdma_lock_compare_exchange (opal_atomic_int32_t *p, int32_t *comp, int32_t value)
{ {
int ret; int ret;
@ -157,7 +161,7 @@ struct ompi_osc_rdma_state_t {
/** post buffers */ /** post buffers */
osc_rdma_counter_t post_peers[OMPI_OSC_RDMA_POST_PEER_MAX]; osc_rdma_counter_t post_peers[OMPI_OSC_RDMA_POST_PEER_MAX];
/** counter for number of post messages received */ /** counter for number of post messages received */
osc_rdma_counter_t num_post_msgs; osc_rdma_atomic_counter_t num_post_msgs;
/** counter for number of complete messages received */ /** counter for number of complete messages received */
osc_rdma_counter_t num_complete_msgs; osc_rdma_counter_t num_complete_msgs;
/** lock for the region state to ensure consistency */ /** lock for the region state to ensure consistency */
@ -195,11 +199,11 @@ struct ompi_osc_rdma_frag_t {
opal_free_list_item_t super; opal_free_list_item_t super;
/* Number of operations which have started writing into the frag, but not yet completed doing so */ /* Number of operations which have started writing into the frag, but not yet completed doing so */
volatile int32_t pending; opal_atomic_int32_t pending;
#if OPAL_HAVE_ATOMIC_MATH_64 #if OPAL_HAVE_ATOMIC_MATH_64
volatile int64_t curr_index; opal_atomic_int64_t curr_index;
#else #else
volatile int32_t curr_index; opal_atomic_int32_t curr_index;
#endif #endif
struct ompi_osc_rdma_module_t *module; struct ompi_osc_rdma_module_t *module;

6
ompi/mca/osc/sm/osc_sm.h
Просмотреть файл

@ -21,12 +21,14 @@
#if OPAL_HAVE_ATOMIC_MATH_64 #if OPAL_HAVE_ATOMIC_MATH_64
typedef uint64_t osc_sm_post_type_t; typedef uint64_t osc_sm_post_type_t;
typedef opal_atomic_uint64_t osc_sm_post_atomic_type_t;
#define OSC_SM_POST_BITS 6 #define OSC_SM_POST_BITS 6
#define OSC_SM_POST_MASK 0x3f #define OSC_SM_POST_MASK 0x3f
#else #else
typedef uint32_t osc_sm_post_type_t; typedef uint32_t osc_sm_post_type_t;
typedef opal_atomic_uint32_t osc_sm_post_atomic_type_t;
#define OSC_SM_POST_BITS 5 #define OSC_SM_POST_BITS 5
#define OSC_SM_POST_MASK 0x1f #define OSC_SM_POST_MASK 0x1f
@ -53,7 +55,7 @@ struct ompi_osc_sm_lock_t {
typedef struct ompi_osc_sm_lock_t ompi_osc_sm_lock_t; typedef struct ompi_osc_sm_lock_t ompi_osc_sm_lock_t;
struct ompi_osc_sm_node_state_t { struct ompi_osc_sm_node_state_t {
int32_t complete_count; opal_atomic_int32_t complete_count;
ompi_osc_sm_lock_t lock; ompi_osc_sm_lock_t lock;
opal_atomic_lock_t accumulate_lock; opal_atomic_lock_t accumulate_lock;
}; };
@ -98,7 +100,7 @@ struct ompi_osc_sm_module_t {
ompi_osc_sm_node_state_t *my_node_state; ompi_osc_sm_node_state_t *my_node_state;
ompi_osc_sm_node_state_t *node_states; ompi_osc_sm_node_state_t *node_states;
osc_sm_post_type_t ** volatile posts; osc_sm_post_atomic_type_t **posts;
opal_mutex_t lock; opal_mutex_t lock;
}; };

12
ompi/mca/osc/sm/osc_sm_active_target.c
Просмотреть файл

@ -162,9 +162,9 @@ ompi_osc_sm_start(struct ompi_group_t *group,
opal_atomic_rmb (); opal_atomic_rmb ();
#if OPAL_HAVE_ATOMIC_MATH_64 #if OPAL_HAVE_ATOMIC_MATH_64
(void) opal_atomic_fetch_xor_64 ((volatile int64_t *) module->posts[my_rank] + rank_byte, rank_bit); (void) opal_atomic_fetch_xor_64 ((opal_atomic_int64_t *) module->posts[my_rank] + rank_byte, rank_bit);
#else #else
(void) opal_atomic_fetch_xor_32 ((volatile int32_t *) module->posts[my_rank] + rank_byte, rank_bit); (void) opal_atomic_fetch_xor_32 ((opal_atomic_int32_t *) module->posts[my_rank] + rank_byte, rank_bit);
#endif #endif
} }
@ -188,7 +188,7 @@ ompi_osc_sm_complete(struct ompi_win_t *win)
opal_atomic_mb(); opal_atomic_mb();
group = module->start_group; group = module->start_group;
if (NULL == group || !OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_PTR(&module->start_group, &group, NULL)) { if (NULL == group || !OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_PTR((opal_atomic_intptr_t *) &module->start_group, (opal_atomic_intptr_t *) &group, 0)) {
return OMPI_ERR_RMA_SYNC; return OMPI_ERR_RMA_SYNC;
} }
@ -247,7 +247,11 @@ ompi_osc_sm_post(struct ompi_group_t *group,
gsize = ompi_group_size(module->post_group); gsize = ompi_group_size(module->post_group);
for (int i = 0 ; i < gsize ; ++i) { for (int i = 0 ; i < gsize ; ++i) {
opal_atomic_add ((volatile osc_sm_post_type_t *) module->posts[ranks[i]] + my_byte, my_bit); #if OPAL_HAVE_ATOMIC_MATH_64
(void) opal_atomic_fetch_add_64 ((opal_atomic_int64_t *) module->posts[ranks[i]] + my_byte, my_bit);
#else
(void) opal_atomic_fetch_add_32 ((opal_atomic_int32_t *) module->posts[ranks[i]] + my_byte, my_bit);
#endif
} }
opal_atomic_wmb (); opal_atomic_wmb ();

4
ompi/mca/osc/sm/osc_sm_component.c
Просмотреть файл

@ -236,7 +236,7 @@ component_select(struct ompi_win_t *win, void **base, size_t size, int disp_unit
if (NULL == module->node_states) return OMPI_ERR_TEMP_OUT_OF_RESOURCE; if (NULL == module->node_states) return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
module->posts = calloc (1, sizeof(module->posts[0]) + sizeof (module->posts[0][0])); module->posts = calloc (1, sizeof(module->posts[0]) + sizeof (module->posts[0][0]));
if (NULL == module->posts) return OMPI_ERR_TEMP_OUT_OF_RESOURCE; if (NULL == module->posts) return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
module->posts[0] = (osc_sm_post_type_t *) (module->posts + 1); module->posts[0] = (osc_sm_post_atomic_type_t *) (module->posts + 1);
} else { } else {
unsigned long total, *rbuf; unsigned long total, *rbuf;
int i, flag; int i, flag;
@ -328,7 +328,7 @@ component_select(struct ompi_win_t *win, void **base, size_t size, int disp_unit
if (NULL == module->posts) return OMPI_ERR_TEMP_OUT_OF_RESOURCE; if (NULL == module->posts) return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
/* set module->posts[0] first to ensure 64-bit alignment */ /* set module->posts[0] first to ensure 64-bit alignment */
module->posts[0] = (osc_sm_post_type_t *) (module->segment_base); module->posts[0] = (osc_sm_post_atomic_type_t *) (module->segment_base);
module->global_state = (ompi_osc_sm_global_state_t *) (module->posts[0] + comm_size * post_size); module->global_state = (ompi_osc_sm_global_state_t *) (module->posts[0] + comm_size * post_size);
module->node_states = (ompi_osc_sm_node_state_t *) (module->global_state + 1); module->node_states = (ompi_osc_sm_node_state_t *) (module->global_state + 1);

4
ompi/mca/osc/sm/osc_sm_passive_target.c
Просмотреть файл

@ -28,7 +28,7 @@ lk_fetch_add32(ompi_osc_sm_module_t *module,
{ {
/* opal_atomic_add_fetch_32 is an add then fetch so delta needs to be subtracted out to get the /* opal_atomic_add_fetch_32 is an add then fetch so delta needs to be subtracted out to get the
* old value */ * old value */
return opal_atomic_add_fetch_32((int32_t*) ((char*) &module->node_states[target].lock + offset), return opal_atomic_add_fetch_32((opal_atomic_int32_t *) ((char*) &module->node_states[target].lock + offset),
delta) - delta; delta) - delta;
} }
@ -39,7 +39,7 @@ lk_add32(ompi_osc_sm_module_t *module,
size_t offset, size_t offset,
uint32_t delta) uint32_t delta)
{ {
opal_atomic_add_fetch_32((int32_t*) ((char*) &module->node_states[target].lock + offset), opal_atomic_add_fetch_32((opal_atomic_int32_t *) ((char*) &module->node_states[target].lock + offset),
delta); delta);
} }

2
ompi/mca/pml/base/pml_base_bsend.c
Просмотреть файл

@ -51,7 +51,7 @@ static size_t mca_pml_bsend_size; /* adjusted size of user buffe
static size_t mca_pml_bsend_count; /* number of outstanding requests */ static size_t mca_pml_bsend_count; /* number of outstanding requests */
static size_t mca_pml_bsend_pagesz; /* mmap page size */ static size_t mca_pml_bsend_pagesz; /* mmap page size */
static int mca_pml_bsend_pagebits; /* number of bits in pagesz */ static int mca_pml_bsend_pagebits; /* number of bits in pagesz */
static int32_t mca_pml_bsend_init = 0; static opal_atomic_int32_t mca_pml_bsend_init = 0;
/* defined in pml_base_open.c */ /* defined in pml_base_open.c */
extern char *ompi_pml_base_bsend_allocator_name; extern char *ompi_pml_base_bsend_allocator_name;

2
ompi/mca/pml/ob1/pml_ob1_comm.h
Просмотреть файл

@ -44,7 +44,7 @@ struct mca_pml_ob1_comm_proc_t {
opal_object_t super; opal_object_t super;
struct ompi_proc_t* ompi_proc; struct ompi_proc_t* ompi_proc;
uint16_t expected_sequence; /**< send message sequence number - receiver side */ uint16_t expected_sequence; /**< send message sequence number - receiver side */
volatile int32_t send_sequence; /**< send side sequence number */ opal_atomic_int32_t send_sequence; /**< send side sequence number */
struct mca_pml_ob1_recv_frag_t* frags_cant_match; /**< out-of-order fragment queues */ struct mca_pml_ob1_recv_frag_t* frags_cant_match; /**< out-of-order fragment queues */
#if !MCA_PML_OB1_CUSTOM_MATCH #if !MCA_PML_OB1_CUSTOM_MATCH
opal_list_t specific_receives; /**< queues of unmatched specific receives */ opal_list_t specific_receives; /**< queues of unmatched specific receives */

2
ompi/mca/pml/ob1/pml_ob1_progress.c
Просмотреть файл

@ -53,7 +53,7 @@ static inline int mca_pml_ob1_process_pending_cuda_async_copies(void)
} }
#endif /* OPAL_CUDA_SUPPORT */ #endif /* OPAL_CUDA_SUPPORT */
static int mca_pml_ob1_progress_needed = 0; static opal_atomic_int32_t mca_pml_ob1_progress_needed = 0;
int mca_pml_ob1_enable_progress(int32_t count) int mca_pml_ob1_enable_progress(int32_t count)
{ {
int32_t progress_count = OPAL_ATOMIC_ADD_FETCH32(&mca_pml_ob1_progress_needed, count); int32_t progress_count = OPAL_ATOMIC_ADD_FETCH32(&mca_pml_ob1_progress_needed, count);

6
ompi/mca/pml/ob1/pml_ob1_recvreq.h
Просмотреть файл

@ -41,9 +41,9 @@ BEGIN_C_DECLS
struct mca_pml_ob1_recv_request_t { struct mca_pml_ob1_recv_request_t {
mca_pml_base_recv_request_t req_recv; mca_pml_base_recv_request_t req_recv;
opal_ptr_t remote_req_send; opal_ptr_t remote_req_send;
int32_t req_lock; opal_atomic_int32_t req_lock;
int32_t req_pipeline_depth; opal_atomic_int32_t req_pipeline_depth;
size_t req_bytes_received; /**< amount of data transferred into the user buffer */ opal_atomic_size_t req_bytes_received; /**< amount of data transferred into the user buffer */
size_t req_bytes_expected; /**< local size of the data as suggested by the user */ size_t req_bytes_expected; /**< local size of the data as suggested by the user */
size_t req_rdma_offset; size_t req_rdma_offset;
size_t req_send_offset; size_t req_send_offset;

8
ompi/mca/pml/ob1/pml_ob1_sendreq.h
Просмотреть файл

@ -46,11 +46,11 @@ struct mca_pml_ob1_send_request_t {
mca_pml_base_send_request_t req_send; mca_pml_base_send_request_t req_send;
mca_bml_base_endpoint_t* req_endpoint; mca_bml_base_endpoint_t* req_endpoint;
opal_ptr_t req_recv; opal_ptr_t req_recv;
int32_t req_state; opal_atomic_int32_t req_state;
int32_t req_lock; opal_atomic_int32_t req_lock;
bool req_throttle_sends; bool req_throttle_sends;
int32_t req_pipeline_depth; opal_atomic_int32_t req_pipeline_depth;
size_t req_bytes_delivered; opal_atomic_size_t req_bytes_delivered;
uint32_t req_rdma_cnt; uint32_t req_rdma_cnt;
mca_pml_ob1_send_pending_t req_pending; mca_pml_ob1_send_pending_t req_pending;
opal_mutex_t req_send_range_lock; opal_mutex_t req_send_range_lock;

2
ompi/request/grequestx.c
Просмотреть файл

@ -28,7 +28,7 @@
static bool requests_initialized = false; static bool requests_initialized = false;
static opal_list_t requests; static opal_list_t requests;
static int32_t active_requests = 0; static opal_atomic_int32_t active_requests = 0;
static bool in_progress = false; static bool in_progress = false;
static opal_mutex_t lock; static opal_mutex_t lock;

2
ompi/runtime/mpiruntime.h
Просмотреть файл

@ -51,7 +51,7 @@ struct ompi_predefined_datatype_t;
/** Mutex to protect all the _init and _finalize variables */ /** Mutex to protect all the _init and _finalize variables */
OMPI_DECLSPEC extern opal_mutex_t ompi_mpi_bootstrap_mutex; OMPI_DECLSPEC extern opal_mutex_t ompi_mpi_bootstrap_mutex;
/** Did MPI start to initialize? */ /** Did MPI start to initialize? */
OMPI_DECLSPEC extern volatile int32_t ompi_mpi_state; OMPI_DECLSPEC extern opal_atomic_int32_t ompi_mpi_state;
/** Has the RTE been initialized? */ /** Has the RTE been initialized? */
OMPI_DECLSPEC extern volatile bool ompi_rte_initialized; OMPI_DECLSPEC extern volatile bool ompi_rte_initialized;

2
ompi/runtime/ompi_mpi_init.c
Просмотреть файл

@ -130,7 +130,7 @@ const char ompi_version_string[] = OMPI_IDENT_STRING;
* Global variables and symbols for the MPI layer * Global variables and symbols for the MPI layer
*/ */
volatile int32_t ompi_mpi_state = OMPI_MPI_STATE_NOT_INITIALIZED; opal_atomic_int32_t ompi_mpi_state = OMPI_MPI_STATE_NOT_INITIALIZED;
volatile bool ompi_rte_initialized = false; volatile bool ompi_rte_initialized = false;
bool ompi_mpi_thread_multiple = false; bool ompi_mpi_thread_multiple = false;

2
ompi/runtime/ompi_spc.c
Просмотреть файл

@ -456,7 +456,7 @@ void ompi_spc_timer_stop(unsigned int event_id, opal_timer_t *cycles)
/* This is denoted unlikely because the counters will often be turned off. */ /* This is denoted unlikely because the counters will often be turned off. */
if( OPAL_UNLIKELY(IS_SPC_BIT_SET(ompi_spc_attached_event, event_id)) ) { if( OPAL_UNLIKELY(IS_SPC_BIT_SET(ompi_spc_attached_event, event_id)) ) {
*cycles = opal_timer_base_get_cycles() - *cycles; *cycles = opal_timer_base_get_cycles() - *cycles;
OPAL_THREAD_ADD_FETCH_SIZE_T(&ompi_spc_events[event_id].value, (ompi_spc_value_t)*cycles); OPAL_THREAD_ADD_FETCH_SIZE_T(&ompi_spc_events[event_id].value, (size_t) *cycles);
} }
} }

2
ompi/runtime/ompi_spc.h
Просмотреть файл

@ -166,7 +166,7 @@ typedef enum ompi_spc_counters {
/* There is currently no support for atomics on long long values so we will default to /* There is currently no support for atomics on long long values so we will default to
* size_t for now until support for such atomics is implemented. * size_t for now until support for such atomics is implemented.
*/ */
typedef size_t ompi_spc_value_t; typedef opal_atomic_size_t ompi_spc_value_t;
/* A structure for storing the event data */ /* A structure for storing the event data */
typedef struct ompi_spc_s{ typedef struct ompi_spc_s{

6
opal/class/opal_fifo.c
Просмотреть файл

@ -10,7 +10,7 @@
* University of Stuttgart. All rights reserved. * University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California. * Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved. * All rights reserved.
* Copyright (c) 2014 Los Alamos National Security, LLC. All rights * Copyright (c) 2014-2018 Los Alamos National Security, LLC. All rights
* reserved. * reserved.
* $COPYRIGHT$ * $COPYRIGHT$
* *
@ -32,10 +32,10 @@ static void opal_fifo_construct (opal_fifo_t *fifo)
fifo->opal_fifo_ghost.item_free = 0; fifo->opal_fifo_ghost.item_free = 0;
fifo->opal_fifo_head.data.counter = 0; fifo->opal_fifo_head.data.counter = 0;
fifo->opal_fifo_head.data.item = &fifo->opal_fifo_ghost; fifo->opal_fifo_head.data.item = (intptr_t) &fifo->opal_fifo_ghost;
fifo->opal_fifo_tail.data.counter = 0; fifo->opal_fifo_tail.data.counter = 0;
fifo->opal_fifo_tail.data.item = &fifo->opal_fifo_ghost; fifo->opal_fifo_tail.data.item = (intptr_t) &fifo->opal_fifo_ghost;
} }
OBJ_CLASS_INSTANCE(opal_fifo_t, opal_object_t, opal_fifo_construct, NULL); OBJ_CLASS_INSTANCE(opal_fifo_t, opal_object_t, opal_fifo_construct, NULL);

30
opal/class/opal_fifo.h
Просмотреть файл

@ -100,13 +100,13 @@ static inline opal_list_item_t *opal_fifo_push_atomic (opal_fifo_t *fifo,
opal_atomic_wmb (); opal_atomic_wmb ();
if (ghost == tail.data.item) { if ((intptr_t) ghost == tail.data.item) {
/* update the head */ /* update the head */
opal_counted_pointer_t head = {.value = fifo->opal_fifo_head.value}; opal_counted_pointer_t head = {.value = fifo->opal_fifo_head.value};
opal_update_counted_pointer (&fifo->opal_fifo_head, &head, item); opal_update_counted_pointer (&fifo->opal_fifo_head, &head, item);
} else { } else {
/* update previous item */ /* update previous item */
tail.data.item->opal_list_next = item; ((opal_list_item_t *) tail.data.item)->opal_list_next = item;
} }
return (opal_list_item_t *) tail.data.item; return (opal_list_item_t *) tail.data.item;
@ -129,12 +129,12 @@ static inline opal_list_item_t *opal_fifo_pop_atomic (opal_fifo_t *fifo)
item = (opal_list_item_t *) head.data.item; item = (opal_list_item_t *) head.data.item;
next = (opal_list_item_t *) item->opal_list_next; next = (opal_list_item_t *) item->opal_list_next;
if (ghost == tail.data.item && ghost == item) { if ((intptr_t) ghost == tail.data.item && ghost == item) {
return NULL; return NULL;
} }
/* the head or next pointer are in an inconsistent state. keep looping. */ /* the head or next pointer are in an inconsistent state. keep looping. */
if (tail.data.item != item && ghost != tail.data.item && ghost == next) { if (tail.data.item != (intptr_t) item && (intptr_t) ghost != tail.data.item && ghost == next) {
opal_read_counted_pointer (&fifo->opal_fifo_head, &head); opal_read_counted_pointer (&fifo->opal_fifo_head, &head);
continue; continue;
} }
@ -165,7 +165,7 @@ static inline opal_list_item_t *opal_fifo_pop_atomic (opal_fifo_t *fifo)
* will be attempting to update the head until after it has been updated * will be attempting to update the head until after it has been updated
* with the next pointer. push will not see an empty list and other pop * with the next pointer. push will not see an empty list and other pop
* operations will loop until the head is consistent. */ * operations will loop until the head is consistent. */
fifo->opal_fifo_head.data.item = (opal_list_item_t *) item->opal_list_next; fifo->opal_fifo_head.data.item = (intptr_t) item->opal_list_next;
opal_atomic_wmb (); opal_atomic_wmb ();
} }
} }
@ -191,13 +191,13 @@ static inline opal_list_item_t *opal_fifo_push_atomic (opal_fifo_t *fifo,
opal_atomic_wmb (); opal_atomic_wmb ();
/* try to get the tail */ /* try to get the tail */
tail_item = opal_atomic_swap_ptr (&fifo->opal_fifo_tail.data.item, item); tail_item = (opal_list_item_t *) opal_atomic_swap_ptr (&fifo->opal_fifo_tail.data.item, (intptr_t) item);
opal_atomic_wmb (); opal_atomic_wmb ();
if (ghost == tail_item) { if (ghost == tail_item) {
/* update the head */ /* update the head */
fifo->opal_fifo_head.data.item = item; fifo->opal_fifo_head.data.item = (intptr_t) item;
} else { } else {
/* update previous item */ /* update previous item */
tail_item->opal_list_next = item; tail_item->opal_list_next = item;
@ -230,7 +230,7 @@ static inline opal_list_item_t *opal_fifo_pop_atomic (opal_fifo_t *fifo)
opal_atomic_ll_ptr(&fifo->opal_fifo_head.data.item, item); opal_atomic_ll_ptr(&fifo->opal_fifo_head.data.item, item);
if (ghost == item) { if (ghost == item) {
if (ghost == fifo->opal_fifo_tail.data.item) { if ((intptr_t) ghost == fifo->opal_fifo_tail.data.item) {
return NULL; return NULL;
} }
@ -248,7 +248,7 @@ static inline opal_list_item_t *opal_fifo_pop_atomic (opal_fifo_t *fifo)
/* protect against ABA issues by "locking" the head */ /* protect against ABA issues by "locking" the head */
do { do {
if (!opal_atomic_swap_32 ((volatile int32_t *) &fifo->opal_fifo_head.data.counter, 1)) { if (!opal_atomic_swap_32 ((opal_atomic_int32_t *) &fifo->opal_fifo_head.data.counter, 1)) {
break; break;
} }
@ -270,12 +270,12 @@ static inline opal_list_item_t *opal_fifo_pop_atomic (opal_fifo_t *fifo)
if (ghost == next) { if (ghost == next) {
void *tmp = item; void *tmp = item;
if (!opal_atomic_compare_exchange_strong_ptr (&fifo->opal_fifo_tail.data.item, &tmp, (void *) ghost)) { if (!opal_atomic_compare_exchange_strong_ptr (&fifo->opal_fifo_tail.data.item, (intptr_t *) &tmp, (intptr_t) ghost)) {
do { do {
opal_atomic_rmb (); opal_atomic_rmb ();
} while (ghost == item->opal_list_next); } while (ghost == item->opal_list_next);
fifo->opal_fifo_head.data.item = (opal_list_item_t *) item->opal_list_next; fifo->opal_fifo_head.data.item = (intptr_t) item->opal_list_next;
} }
} }
@ -299,9 +299,9 @@ static inline opal_list_item_t *opal_fifo_push_st (opal_fifo_t *fifo,
item->opal_list_next = &fifo->opal_fifo_ghost; item->opal_list_next = &fifo->opal_fifo_ghost;
fifo->opal_fifo_tail.data.item = item; fifo->opal_fifo_tail.data.item = (intptr_t) item;
if (&fifo->opal_fifo_ghost == opal_fifo_head (fifo)) { if (&fifo->opal_fifo_ghost == opal_fifo_head (fifo)) {
fifo->opal_fifo_head.data.item = item; fifo->opal_fifo_head.data.item = (intptr_t) item;
} else { } else {
prev->opal_list_next = item; prev->opal_list_next = item;
} }
@ -317,9 +317,9 @@ static inline opal_list_item_t *opal_fifo_pop_st (opal_fifo_t *fifo)
return NULL; return NULL;
} }
fifo->opal_fifo_head.data.item = (opal_list_item_t *) item->opal_list_next; fifo->opal_fifo_head.data.item = (intptr_t) item->opal_list_next;
if (&fifo->opal_fifo_ghost == opal_fifo_head (fifo)) { if (&fifo->opal_fifo_ghost == opal_fifo_head (fifo)) {
fifo->opal_fifo_tail.data.item = &fifo->opal_fifo_ghost; fifo->opal_fifo_tail.data.item = (intptr_t) &fifo->opal_fifo_ghost;
} }
item->opal_list_next = NULL; item->opal_list_next = NULL;

2
opal/class/opal_interval_tree.c
Просмотреть файл

@ -130,7 +130,7 @@ static opal_interval_tree_token_t opal_interval_tree_reader_get_token (opal_inte
} }
} }
while (!OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_32((volatile int32_t *) &tree->reader_epochs[token], while (!OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_32((opal_atomic_int32_t *) &tree->reader_epochs[token],
&(int32_t) {UINT_MAX}, tree->epoch)); &(int32_t) {UINT_MAX}, tree->epoch));
return token; return token;

8
opal/class/opal_interval_tree.h
Просмотреть файл

@ -83,11 +83,11 @@ struct opal_interval_tree_t {
opal_free_list_t free_list; /**< the free list to get the memory from */ opal_free_list_t free_list; /**< the free list to get the memory from */
opal_list_t gc_list; /**< list of nodes that need to be released */ opal_list_t gc_list; /**< list of nodes that need to be released */
uint32_t epoch; /**< current update epoch */ uint32_t epoch; /**< current update epoch */
volatile size_t tree_size; /**< the current size of the tree */ opal_atomic_size_t tree_size; /**< the current size of the tree */
volatile int32_t lock; /**< update lock */ opal_atomic_int32_t lock; /**< update lock */
volatile int32_t reader_count; /**< current highest reader slot to check */ opal_atomic_int32_t reader_count; /**< current highest reader slot to check */
volatile uint32_t reader_id; /**< next reader slot to check */ volatile uint32_t reader_id; /**< next reader slot to check */
volatile uint32_t reader_epochs[OPAL_INTERVAL_TREE_MAX_READERS]; opal_atomic_uint32_t reader_epochs[OPAL_INTERVAL_TREE_MAX_READERS];
}; };
typedef struct opal_interval_tree_t opal_interval_tree_t; typedef struct opal_interval_tree_t opal_interval_tree_t;

2
opal/class/opal_lifo.c
Просмотреть файл

@ -26,7 +26,7 @@ static void opal_lifo_construct (opal_lifo_t *lifo)
{ {
OBJ_CONSTRUCT(&lifo->opal_lifo_ghost, opal_list_item_t); OBJ_CONSTRUCT(&lifo->opal_lifo_ghost, opal_list_item_t);
lifo->opal_lifo_ghost.opal_list_next = &lifo->opal_lifo_ghost; lifo->opal_lifo_ghost.opal_list_next = &lifo->opal_lifo_ghost;
lifo->opal_lifo_head.data.item = &lifo->opal_lifo_ghost; lifo->opal_lifo_head.data.item = (intptr_t) &lifo->opal_lifo_ghost;
lifo->opal_lifo_head.data.counter = 0; lifo->opal_lifo_head.data.counter = 0;
} }

25
opal/class/opal_lifo.h
Просмотреть файл

@ -48,11 +48,12 @@ union opal_counted_pointer_t {
/** update counter used when cmpset_128 is available */ /** update counter used when cmpset_128 is available */
uint64_t counter; uint64_t counter;
/** list item pointer */ /** list item pointer */
volatile opal_list_item_t * volatile item; volatile opal_atomic_intptr_t item;
} data; } data;
#if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 && HAVE_OPAL_INT128_T #if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 && HAVE_OPAL_INT128_T
/** used for atomics when there is a cmpset that can operate on /** used for atomics when there is a cmpset that can operate on
* two 64-bit values */ * two 64-bit values */
opal_atomic_int128_t atomic_value;
opal_int128_t value; opal_int128_t value;
#endif #endif
}; };
@ -65,17 +66,17 @@ typedef union opal_counted_pointer_t opal_counted_pointer_t;
* to allow the upper level to detect if this element is the first one in the * to allow the upper level to detect if this element is the first one in the
* list (if the list was empty before this operation). * list (if the list was empty before this operation).
*/ */
static inline bool opal_update_counted_pointer (volatile opal_counted_pointer_t *addr, opal_counted_pointer_t *old, static inline bool opal_update_counted_pointer (volatile opal_counted_pointer_t * volatile addr, opal_counted_pointer_t *old,
opal_list_item_t *item) opal_list_item_t *item)
{ {
opal_counted_pointer_t new_p; opal_counted_pointer_t new_p;
new_p.data.item = item; new_p.data.item = (intptr_t) item;
new_p.data.counter = old->data.counter + 1; new_p.data.counter = old->data.counter + 1;
return opal_atomic_compare_exchange_strong_128 (&addr->value, &old->value, new_p.value); return opal_atomic_compare_exchange_strong_128 (&addr->atomic_value, &old->value, new_p.value);
} }
__opal_attribute_always_inline__ __opal_attribute_always_inline__
static inline void opal_read_counted_pointer (volatile opal_counted_pointer_t *addr, opal_counted_pointer_t *value) static inline void opal_read_counted_pointer (volatile opal_counted_pointer_t * volatile addr, opal_counted_pointer_t *value)
{ {
/* most platforms do not read the value atomically so make sure we read the counted pointer in a specific order */ /* most platforms do not read the value atomically so make sure we read the counted pointer in a specific order */
value->data.counter = addr->data.counter; value->data.counter = addr->data.counter;
@ -151,7 +152,7 @@ static inline opal_list_item_t *opal_lifo_push_atomic (opal_lifo_t *lifo,
opal_atomic_wmb (); opal_atomic_wmb ();
/* to protect against ABA issues it is sufficient to only update the counter in pop */ /* to protect against ABA issues it is sufficient to only update the counter in pop */
if (opal_atomic_compare_exchange_strong_ptr (&lifo->opal_lifo_head.data.item, &next, item)) { if (opal_atomic_compare_exchange_strong_ptr (&lifo->opal_lifo_head.data.item, (intptr_t *) &next, (intptr_t) item)) {
return next; return next;
} }
/* DO some kind of pause to release the bus */ /* DO some kind of pause to release the bus */
@ -200,7 +201,7 @@ static inline opal_list_item_t *opal_lifo_push_atomic (opal_lifo_t *lifo,
do { do {
item->opal_list_next = next; item->opal_list_next = next;
opal_atomic_wmb(); opal_atomic_wmb();
if (opal_atomic_compare_exchange_strong_ptr (&lifo->opal_lifo_head.data.item, &next, item)) { if (opal_atomic_compare_exchange_strong_ptr (&lifo->opal_lifo_head.data.item, (intptr_t *) &next, (intptr_t) item)) {
opal_atomic_wmb (); opal_atomic_wmb ();
/* now safe to pop this item */ /* now safe to pop this item */
item->item_free = 0; item->item_free = 0;
@ -254,7 +255,7 @@ static inline opal_list_item_t *opal_lifo_pop_atomic (opal_lifo_t* lifo)
while ((item=(opal_list_item_t *)lifo->opal_lifo_head.data.item) != ghost) { while ((item=(opal_list_item_t *)lifo->opal_lifo_head.data.item) != ghost) {
/* ensure it is safe to pop the head */ /* ensure it is safe to pop the head */
if (opal_atomic_swap_32((volatile int32_t *) &item->item_free, 1)) { if (opal_atomic_swap_32((opal_atomic_int32_t *) &item->item_free, 1)) {
continue; continue;
} }
@ -262,8 +263,8 @@ static inline opal_list_item_t *opal_lifo_pop_atomic (opal_lifo_t* lifo)
head = item; head = item;
/* try to swap out the head pointer */ /* try to swap out the head pointer */
if (opal_atomic_compare_exchange_strong_ptr (&lifo->opal_lifo_head.data.item, &head, if (opal_atomic_compare_exchange_strong_ptr (&lifo->opal_lifo_head.data.item, (intptr_t *) &head,
(void *) item->opal_list_next)) { (intptr_t) item->opal_list_next)) {
break; break;
} }
@ -294,7 +295,7 @@ static inline opal_list_item_t *opal_lifo_push_st (opal_lifo_t *lifo,
{ {
item->opal_list_next = (opal_list_item_t *) lifo->opal_lifo_head.data.item; item->opal_list_next = (opal_list_item_t *) lifo->opal_lifo_head.data.item;
item->item_free = 0; item->item_free = 0;
lifo->opal_lifo_head.data.item = item; lifo->opal_lifo_head.data.item = (intptr_t) item;
return (opal_list_item_t *) item->opal_list_next; return (opal_list_item_t *) item->opal_list_next;
} }
@ -302,7 +303,7 @@ static inline opal_list_item_t *opal_lifo_pop_st (opal_lifo_t *lifo)
{ {
opal_list_item_t *item; opal_list_item_t *item;
item = (opal_list_item_t *) lifo->opal_lifo_head.data.item; item = (opal_list_item_t *) lifo->opal_lifo_head.data.item;
lifo->opal_lifo_head.data.item = (opal_list_item_t *) item->opal_list_next; lifo->opal_lifo_head.data.item = (intptr_t) item->opal_list_next;
if (item == &lifo->opal_lifo_ghost) { if (item == &lifo->opal_lifo_ghost) {
return NULL; return NULL;
} }

18
opal/class/opal_list.h
Просмотреть файл

@ -111,7 +111,7 @@ struct opal_list_item_t
#if OPAL_ENABLE_DEBUG #if OPAL_ENABLE_DEBUG
/** Atomic reference count for debugging */ /** Atomic reference count for debugging */
volatile int32_t opal_list_item_refcount; opal_atomic_int32_t opal_list_item_refcount;
/** The list this item belong to */ /** The list this item belong to */
volatile struct opal_list_t* opal_list_item_belong_to; volatile struct opal_list_t* opal_list_item_belong_to;
#endif #endif
@ -654,7 +654,7 @@ static inline opal_list_item_t *opal_list_remove_first(opal_list_t *list)
Caller now owns the item and should release the item Caller now owns the item and should release the item
when caller is done with it. when caller is done with it.
*/ */
volatile opal_list_item_t *item; opal_list_item_t *item;
if ( 0 == list->opal_list_length ) { if ( 0 == list->opal_list_length ) {
return (opal_list_item_t *)NULL; return (opal_list_item_t *)NULL;
} }
@ -669,7 +669,7 @@ static inline opal_list_item_t *opal_list_remove_first(opal_list_t *list)
list->opal_list_length--; list->opal_list_length--;
/* get pointer to first element on the list */ /* get pointer to first element on the list */
item = list->opal_list_sentinel.opal_list_next; item = (opal_list_item_t *) list->opal_list_sentinel.opal_list_next;
/* reset previous pointer of next item on the list */ /* reset previous pointer of next item on the list */
item->opal_list_next->opal_list_prev = item->opal_list_prev; item->opal_list_next->opal_list_prev = item->opal_list_prev;
@ -686,11 +686,11 @@ static inline opal_list_item_t *opal_list_remove_first(opal_list_t *list)
/* Spot check: ensure that the item we're returning is now on no /* Spot check: ensure that the item we're returning is now on no
lists */ lists */
OPAL_THREAD_ADD_FETCH32( &(item->opal_list_item_refcount), -1 ); OPAL_THREAD_ADD_FETCH32( &item->opal_list_item_refcount, -1 );
assert(0 == item->opal_list_item_refcount); assert(0 == item->opal_list_item_refcount);
#endif #endif
return (opal_list_item_t *) item; return item;
} }
@ -716,7 +716,7 @@ static inline opal_list_item_t *opal_list_remove_last(opal_list_t *list)
Caller now owns the item and should release the item Caller now owns the item and should release the item
when caller is done with it. when caller is done with it.
*/ */
volatile opal_list_item_t *item; opal_list_item_t *item;
if ( 0 == list->opal_list_length ) { if ( 0 == list->opal_list_length ) {
return (opal_list_item_t *)NULL; return (opal_list_item_t *)NULL;
} }
@ -731,7 +731,7 @@ static inline opal_list_item_t *opal_list_remove_last(opal_list_t *list)
list->opal_list_length--; list->opal_list_length--;
/* get item */ /* get item */
item = list->opal_list_sentinel.opal_list_prev; item = (opal_list_item_t *) list->opal_list_sentinel.opal_list_prev;
/* reset previous pointer on next to last pointer */ /* reset previous pointer on next to last pointer */
item->opal_list_prev->opal_list_next = item->opal_list_next; item->opal_list_prev->opal_list_next = item->opal_list_next;
@ -746,12 +746,12 @@ static inline opal_list_item_t *opal_list_remove_last(opal_list_t *list)
/* Spot check: ensure that the item we're returning is now on no /* Spot check: ensure that the item we're returning is now on no
lists */ lists */
OPAL_THREAD_ADD_FETCH32( &(item->opal_list_item_refcount), -1 ); OPAL_THREAD_ADD_FETCH32(&item->opal_list_item_refcount, -1 );
assert(0 == item->opal_list_item_refcount); assert(0 == item->opal_list_item_refcount);
item->opal_list_item_belong_to = NULL; item->opal_list_item_belong_to = NULL;
#endif #endif
return (opal_list_item_t *) item; return item;
} }
/** /**

2
opal/class/opal_object.h
Просмотреть файл

@ -198,7 +198,7 @@ struct opal_object_t {
uint64_t obj_magic_id; uint64_t obj_magic_id;
#endif #endif
opal_class_t *obj_class; /**< class descriptor */ opal_class_t *obj_class; /**< class descriptor */
volatile int32_t obj_reference_count; /**< reference count */ opal_atomic_int32_t obj_reference_count; /**< reference count */
#if OPAL_ENABLE_DEBUG #if OPAL_ENABLE_DEBUG
const char* cls_init_file_name; /**< In debug mode store the file where the object get contructed */ const char* cls_init_file_name; /**< In debug mode store the file where the object get contructed */
int cls_init_lineno; /**< In debug mode store the line number where the object get contructed */ int cls_init_lineno; /**< In debug mode store the line number where the object get contructed */

2
opal/class/opal_tree.h
Просмотреть файл

@ -120,7 +120,7 @@ typedef struct opal_tree_item_t
#if OPAL_ENABLE_DEBUG #if OPAL_ENABLE_DEBUG
/** Atomic reference count for debugging */ /** Atomic reference count for debugging */
int32_t opal_tree_item_refcount; opal_atomic_int32_t opal_tree_item_refcount;
/** The tree this item belong to */ /** The tree this item belong to */
struct opal_tree_t* opal_tree_item_belong_to; struct opal_tree_t* opal_tree_item_belong_to;
#endif #endif

5
opal/include/Makefile.am
Просмотреть файл

@ -10,6 +10,8 @@
# Copyright (c) 2004-2005 The Regents of the University of California. # Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved. # All rights reserved.
# Copyright (c) 2010-2011 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2010-2011 Cisco Systems, Inc. All rights reserved.
# Copyright (c) 2018 Los Alamos National Security, LLC. All rights
# reserved.
# $COPYRIGHT$ # $COPYRIGHT$
# #
# Additional copyrights may follow # Additional copyrights may follow
@ -20,7 +22,8 @@
headers = \ headers = \
opal_config_top.h \ opal_config_top.h \
opal_config_bottom.h \ opal_config_bottom.h \
opal_stdint.h opal_stdint.h \
opal_stdatomic.h
nodist_headers = opal_config.h nodist_headers = opal_config.h

18
opal/include/opal/sys/arm/atomic.h
Просмотреть файл

@ -12,7 +12,7 @@
* All rights reserved. * All rights reserved.
* Copyright (c) 2010 IBM Corporation. All rights reserved. * Copyright (c) 2010 IBM Corporation. All rights reserved.
* Copyright (c) 2010 ARM ltd. All rights reserved. * Copyright (c) 2010 ARM ltd. All rights reserved.
* Copyright (c) 2017 Los Alamos National Security, LLC. All rights * Copyright (c) 2017-2018 Los Alamos National Security, LLC. All rights
* reserved. * reserved.
* $COPYRIGHT$ * $COPYRIGHT$
* *
@ -109,7 +109,7 @@ void opal_atomic_isync(void)
#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1
#define OPAL_HAVE_ATOMIC_MATH_32 1 #define OPAL_HAVE_ATOMIC_MATH_32 1
static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) static inline bool opal_atomic_compare_exchange_strong_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
{ {
int32_t prev, tmp; int32_t prev, tmp;
bool ret; bool ret;
@ -137,7 +137,7 @@ static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *add
atomic_?mb can be inlined). Instead, we "inline" them by hand in atomic_?mb can be inlined). Instead, we "inline" them by hand in
the assembly, meaning there is one function call overhead instead the assembly, meaning there is one function call overhead instead
of two */ of two */
static inline bool opal_atomic_compare_exchange_strong_acq_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) static inline bool opal_atomic_compare_exchange_strong_acq_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
{ {
bool rc; bool rc;
@ -148,7 +148,7 @@ static inline bool opal_atomic_compare_exchange_strong_acq_32 (volatile int32_t
} }
static inline bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) static inline bool opal_atomic_compare_exchange_strong_rel_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
{ {
opal_atomic_wmb(); opal_atomic_wmb();
return opal_atomic_compare_exchange_strong_32 (addr, oldval, newval); return opal_atomic_compare_exchange_strong_32 (addr, oldval, newval);
@ -157,7 +157,7 @@ static inline bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t
#if (OPAL_ASM_SUPPORT_64BIT == 1) #if (OPAL_ASM_SUPPORT_64BIT == 1)
#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1
static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) static inline bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
{ {
int64_t prev; int64_t prev;
int tmp; int tmp;
@ -188,7 +188,7 @@ static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *add
atomic_?mb can be inlined). Instead, we "inline" them by hand in atomic_?mb can be inlined). Instead, we "inline" them by hand in
the assembly, meaning there is one function call overhead instead the assembly, meaning there is one function call overhead instead
of two */ of two */
static inline bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) static inline bool opal_atomic_compare_exchange_strong_acq_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
{ {
bool rc; bool rc;
@ -199,7 +199,7 @@ static inline bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t
} }
static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) static inline bool opal_atomic_compare_exchange_strong_rel_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
{ {
opal_atomic_wmb(); opal_atomic_wmb();
return opal_atomic_compare_exchange_strong_64 (addr, oldval, newval); return opal_atomic_compare_exchange_strong_64 (addr, oldval, newval);
@ -209,7 +209,7 @@ static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t
#define OPAL_HAVE_ATOMIC_ADD_32 1 #define OPAL_HAVE_ATOMIC_ADD_32 1
static inline int32_t opal_atomic_fetch_add_32(volatile int32_t* v, int inc) static inline int32_t opal_atomic_fetch_add_32(opal_atomic_int32_t* v, int inc)
{ {
int32_t t, old; int32_t t, old;
int tmp; int tmp;
@ -230,7 +230,7 @@ static inline int32_t opal_atomic_fetch_add_32(volatile int32_t* v, int inc)
} }
#define OPAL_HAVE_ATOMIC_SUB_32 1 #define OPAL_HAVE_ATOMIC_SUB_32 1
static inline int32_t opal_atomic_fetch_sub_32(volatile int32_t* v, int dec) static inline int32_t opal_atomic_fetch_sub_32(opal_atomic_int32_t* v, int dec)
{ {
int32_t t, old; int32_t t, old;
int tmp; int tmp;

28
opal/include/opal/sys/arm64/atomic.h
Просмотреть файл

@ -12,7 +12,7 @@
* All rights reserved. * All rights reserved.
* Copyright (c) 2010 IBM Corporation. All rights reserved. * Copyright (c) 2010 IBM Corporation. All rights reserved.
* Copyright (c) 2010 ARM ltd. All rights reserved. * Copyright (c) 2010 ARM ltd. All rights reserved.
* Copyright (c) 2016-2017 Los Alamos National Security, LLC. All rights * Copyright (c) 2016-2018 Los Alamos National Security, LLC. All rights
* reserved. * reserved.
* $COPYRIGHT$ * $COPYRIGHT$
* *
@ -82,7 +82,7 @@ static inline void opal_atomic_isync (void)
* *
*********************************************************************/ *********************************************************************/
static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) static inline bool opal_atomic_compare_exchange_strong_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
{ {
int32_t prev, tmp; int32_t prev, tmp;
bool ret; bool ret;
@ -102,7 +102,7 @@ static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *add
return ret; return ret;
} }
static inline int32_t opal_atomic_swap_32(volatile int32_t *addr, int32_t newval) static inline int32_t opal_atomic_swap_32(opal_atomic_int32_t *addr, int32_t newval)
{ {
int32_t ret, tmp; int32_t ret, tmp;
@ -121,7 +121,7 @@ static inline int32_t opal_atomic_swap_32(volatile int32_t *addr, int32_t newval
atomic_?mb can be inlined). Instead, we "inline" them by hand in atomic_?mb can be inlined). Instead, we "inline" them by hand in
the assembly, meaning there is one function call overhead instead the assembly, meaning there is one function call overhead instead
of two */ of two */
static inline bool opal_atomic_compare_exchange_strong_acq_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) static inline bool opal_atomic_compare_exchange_strong_acq_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
{ {
int32_t prev, tmp; int32_t prev, tmp;
bool ret; bool ret;
@ -142,7 +142,7 @@ static inline bool opal_atomic_compare_exchange_strong_acq_32 (volatile int32_t
} }
static inline bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) static inline bool opal_atomic_compare_exchange_strong_rel_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
{ {
int32_t prev, tmp; int32_t prev, tmp;
bool ret; bool ret;
@ -164,7 +164,7 @@ static inline bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t
#define opal_atomic_ll_32(addr, ret) \ #define opal_atomic_ll_32(addr, ret) \
do { \ do { \
volatile int32_t *_addr = (addr); \ opal_atomic_int32_t *_addr = (addr); \
int32_t _ret; \ int32_t _ret; \
\ \
__asm__ __volatile__ ("ldaxr %w0, [%1] \n" \ __asm__ __volatile__ ("ldaxr %w0, [%1] \n" \
@ -176,7 +176,7 @@ static inline bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t
#define opal_atomic_sc_32(addr, newval, ret) \ #define opal_atomic_sc_32(addr, newval, ret) \
do { \ do { \
volatile int32_t *_addr = (addr); \ opal_atomic_int32_t *_addr = (addr); \
int32_t _newval = (int32_t) newval; \ int32_t _newval = (int32_t) newval; \
int _ret; \ int _ret; \
\ \
@ -188,7 +188,7 @@ static inline bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t
ret = (_ret == 0); \ ret = (_ret == 0); \
} while (0) } while (0)
static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) static inline bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
{ {
int64_t prev; int64_t prev;
int tmp; int tmp;
@ -209,7 +209,7 @@ static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *add
return ret; return ret;
} }
static inline int64_t opal_atomic_swap_64 (volatile int64_t *addr, int64_t newval) static inline int64_t opal_atomic_swap_64 (opal_atomic_int64_t *addr, int64_t newval)
{ {
int64_t ret; int64_t ret;
int tmp; int tmp;
@ -229,7 +229,7 @@ static inline int64_t opal_atomic_swap_64 (volatile int64_t *addr, int64_t newva
atomic_?mb can be inlined). Instead, we "inline" them by hand in atomic_?mb can be inlined). Instead, we "inline" them by hand in
the assembly, meaning there is one function call overhead instead the assembly, meaning there is one function call overhead instead
of two */ of two */
static inline bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) static inline bool opal_atomic_compare_exchange_strong_acq_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
{ {
int64_t prev; int64_t prev;
int tmp; int tmp;
@ -251,7 +251,7 @@ static inline bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t
} }
static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) static inline bool opal_atomic_compare_exchange_strong_rel_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
{ {
int64_t prev; int64_t prev;
int tmp; int tmp;
@ -274,7 +274,7 @@ static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t
#define opal_atomic_ll_64(addr, ret) \ #define opal_atomic_ll_64(addr, ret) \
do { \ do { \
volatile int64_t *_addr = (addr); \ opal_atomic_int64_t *_addr = (addr); \
int64_t _ret; \ int64_t _ret; \
\ \
__asm__ __volatile__ ("ldaxr %0, [%1] \n" \ __asm__ __volatile__ ("ldaxr %0, [%1] \n" \
@ -286,7 +286,7 @@ static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t
#define opal_atomic_sc_64(addr, newval, ret) \ #define opal_atomic_sc_64(addr, newval, ret) \
do { \ do { \
volatile int64_t *_addr = (addr); \ opal_atomic_int64_t *_addr = (addr); \
int64_t _newval = (int64_t) newval; \ int64_t _newval = (int64_t) newval; \
int _ret; \ int _ret; \
\ \
@ -299,7 +299,7 @@ static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t
} while (0) } while (0)
#define OPAL_ASM_MAKE_ATOMIC(type, bits, name, inst, reg) \ #define OPAL_ASM_MAKE_ATOMIC(type, bits, name, inst, reg) \
static inline type opal_atomic_fetch_ ## name ## _ ## bits (volatile type *addr, type value) \ static inline type opal_atomic_fetch_ ## name ## _ ## bits (opal_atomic_ ## type *addr, type value) \
{ \ { \
type newval, old; \ type newval, old; \
int32_t tmp; \ int32_t tmp; \

134
opal/include/opal/sys/atomic.h
Просмотреть файл

@ -56,7 +56,7 @@
#include <stdbool.h> #include <stdbool.h>
#include "opal/sys/architecture.h" #include "opal/sys/architecture.h"
#include "opal_stdint.h" #include "opal_stdatomic.h"
/* do some quick #define cleanup in cases where we are doing /* do some quick #define cleanup in cases where we are doing
testing... */ testing... */
@ -92,7 +92,7 @@ BEGIN_C_DECLS
*/ */
struct opal_atomic_lock_t { struct opal_atomic_lock_t {
union { union {
volatile int32_t lock; /**< The lock address (an integer) */ opal_atomic_int32_t lock; /**< The lock address (an integer) */
volatile unsigned char sparc_lock; /**< The lock address on sparc */ volatile unsigned char sparc_lock; /**< The lock address on sparc */
char padding[sizeof(int)]; /**< Array for optional padding */ char padding[sizeof(int)]; /**< Array for optional padding */
} u; } u;
@ -350,19 +350,19 @@ void opal_atomic_unlock(opal_atomic_lock_t *lock);
#if OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_32 #if OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_32
static inline static inline
#endif #endif
bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, bool opal_atomic_compare_exchange_strong_32 (opal_atomic_int32_t *addr, int32_t *oldval,
int32_t newval); int32_t newval);
#if OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_32 #if OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_32
static inline static inline
#endif #endif
bool opal_atomic_compare_exchange_strong_acq_32 (volatile int32_t *addr, int32_t *oldval, bool opal_atomic_compare_exchange_strong_acq_32 (opal_atomic_int32_t *addr, int32_t *oldval,
int32_t newval); int32_t newval);
#if OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_32 #if OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_32
static inline static inline
#endif #endif
bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t *oldval, bool opal_atomic_compare_exchange_strong_rel_32 (opal_atomic_int32_t *addr, int32_t *oldval,
int32_t newval); int32_t newval);
#endif #endif
@ -375,19 +375,19 @@ bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t
#if OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_64 #if OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_64
static inline static inline
#endif #endif
bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval,
int64_t newval); int64_t newval);
#if OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_64 #if OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_64
static inline static inline
#endif #endif
bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval, bool opal_atomic_compare_exchange_strong_acq_64 (opal_atomic_int64_t *addr, int64_t *oldval,
int64_t newval); int64_t newval);
#if OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_64 #if OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_64
static inline static inline
#endif #endif
bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval, bool opal_atomic_compare_exchange_strong_rel_64 (opal_atomic_int64_t *addr, int64_t *oldval,
int64_t newval); int64_t newval);
#endif #endif
@ -399,20 +399,20 @@ bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t
#if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_MATH_32 || OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 #if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_MATH_32 || OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32
static inline int32_t opal_atomic_add_fetch_32(volatile int32_t *addr, int delta); static inline int32_t opal_atomic_add_fetch_32(opal_atomic_int32_t *addr, int delta);
static inline int32_t opal_atomic_fetch_add_32(volatile int32_t *addr, int delta); static inline int32_t opal_atomic_fetch_add_32(opal_atomic_int32_t *addr, int delta);
static inline int32_t opal_atomic_and_fetch_32(volatile int32_t *addr, int32_t value); static inline int32_t opal_atomic_and_fetch_32(opal_atomic_int32_t *addr, int32_t value);
static inline int32_t opal_atomic_fetch_and_32(volatile int32_t *addr, int32_t value); static inline int32_t opal_atomic_fetch_and_32(opal_atomic_int32_t *addr, int32_t value);
static inline int32_t opal_atomic_or_fetch_32(volatile int32_t *addr, int32_t value); static inline int32_t opal_atomic_or_fetch_32(opal_atomic_int32_t *addr, int32_t value);
static inline int32_t opal_atomic_fetch_or_32(volatile int32_t *addr, int32_t value); static inline int32_t opal_atomic_fetch_or_32(opal_atomic_int32_t *addr, int32_t value);
static inline int32_t opal_atomic_xor_fetch_32(volatile int32_t *addr, int32_t value); static inline int32_t opal_atomic_xor_fetch_32(opal_atomic_int32_t *addr, int32_t value);
static inline int32_t opal_atomic_fetch_xor_32(volatile int32_t *addr, int32_t value); static inline int32_t opal_atomic_fetch_xor_32(opal_atomic_int32_t *addr, int32_t value);
static inline int32_t opal_atomic_sub_fetch_32(volatile int32_t *addr, int delta); static inline int32_t opal_atomic_sub_fetch_32(opal_atomic_int32_t *addr, int delta);
static inline int32_t opal_atomic_fetch_sub_32(volatile int32_t *addr, int delta); static inline int32_t opal_atomic_fetch_sub_32(opal_atomic_int32_t *addr, int delta);
static inline int32_t opal_atomic_min_fetch_32 (volatile int32_t *addr, int32_t value); static inline int32_t opal_atomic_min_fetch_32 (opal_atomic_int32_t *addr, int32_t value);
static inline int32_t opal_atomic_fetch_min_32 (volatile int32_t *addr, int32_t value); static inline int32_t opal_atomic_fetch_min_32 (opal_atomic_int32_t *addr, int32_t value);
static inline int32_t opal_atomic_max_fetch_32 (volatile int32_t *addr, int32_t value); static inline int32_t opal_atomic_max_fetch_32 (opal_atomic_int32_t *addr, int32_t value);
static inline int32_t opal_atomic_fetch_max_32 (volatile int32_t *addr, int32_t value); static inline int32_t opal_atomic_fetch_max_32 (opal_atomic_int32_t *addr, int32_t value);
#endif /* OPAL_HAVE_ATOMIC_MATH_32 */ #endif /* OPAL_HAVE_ATOMIC_MATH_32 */
@ -429,19 +429,19 @@ static inline int32_t opal_atomic_fetch_max_32 (volatile int32_t *addr, int32_t
#if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_MATH_64 || OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 #if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_MATH_64 || OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64
static inline int64_t opal_atomic_add_fetch_64(volatile int64_t *addr, int64_t delta); static inline int64_t opal_atomic_add_fetch_64(opal_atomic_int64_t *addr, int64_t delta);
static inline int64_t opal_atomic_fetch_add_64(volatile int64_t *addr, int64_t delta); static inline int64_t opal_atomic_fetch_add_64(opal_atomic_int64_t *addr, int64_t delta);
static inline int64_t opal_atomic_and_fetch_64(volatile int64_t *addr, int64_t value); static inline int64_t opal_atomic_and_fetch_64(opal_atomic_int64_t *addr, int64_t value);
static inline int64_t opal_atomic_fetch_and_64(volatile int64_t *addr, int64_t value); static inline int64_t opal_atomic_fetch_and_64(opal_atomic_int64_t *addr, int64_t value);
static inline int64_t opal_atomic_or_fetch_64(volatile int64_t *addr, int64_t value); static inline int64_t opal_atomic_or_fetch_64(opal_atomic_int64_t *addr, int64_t value);
static inline int64_t opal_atomic_fetch_or_64(volatile int64_t *addr, int64_t value); static inline int64_t opal_atomic_fetch_or_64(opal_atomic_int64_t *addr, int64_t value);
static inline int64_t opal_atomic_fetch_xor_64(volatile int64_t *addr, int64_t value); static inline int64_t opal_atomic_fetch_xor_64(opal_atomic_int64_t *addr, int64_t value);
static inline int64_t opal_atomic_sub_fetch_64(volatile int64_t *addr, int64_t delta); static inline int64_t opal_atomic_sub_fetch_64(opal_atomic_int64_t *addr, int64_t delta);
static inline int64_t opal_atomic_fetch_sub_64(volatile int64_t *addr, int64_t delta); static inline int64_t opal_atomic_fetch_sub_64(opal_atomic_int64_t *addr, int64_t delta);
static inline int64_t opal_atomic_min_fetch_64 (volatile int64_t *addr, int64_t value); static inline int64_t opal_atomic_min_fetch_64 (opal_atomic_int64_t *addr, int64_t value);
static inline int64_t opal_atomic_fetch_min_64 (volatile int64_t *addr, int64_t value); static inline int64_t opal_atomic_fetch_min_64 (opal_atomic_int64_t *addr, int64_t value);
static inline int64_t opal_atomic_max_fetch_64 (volatile int64_t *addr, int64_t value); static inline int64_t opal_atomic_max_fetch_64 (opal_atomic_int64_t *addr, int64_t value);
static inline int64_t opal_atomic_fetch_max_64 (volatile int64_t *addr, int64_t value); static inline int64_t opal_atomic_fetch_max_64 (opal_atomic_int64_t *addr, int64_t value);
#endif /* OPAL_HAVE_ATOMIC_MATH_64 */ #endif /* OPAL_HAVE_ATOMIC_MATH_64 */
@ -458,7 +458,7 @@ static inline int64_t opal_atomic_fetch_max_64 (volatile int64_t *addr, int64_t
*/ */
#if defined(DOXYGEN) || OPAL_ENABLE_DEBUG #if defined(DOXYGEN) || OPAL_ENABLE_DEBUG
static inline size_t static inline size_t
opal_atomic_add_fetch_size_t(volatile size_t *addr, size_t delta) opal_atomic_add_fetch_size_t(opal_atomic_size_t *addr, size_t delta)
{ {
#if SIZEOF_SIZE_T == 4 #if SIZEOF_SIZE_T == 4
return (size_t) opal_atomic_add_fetch_32((int32_t*) addr, delta); return (size_t) opal_atomic_add_fetch_32((int32_t*) addr, delta);
@ -470,7 +470,7 @@ opal_atomic_add_fetch_size_t(volatile size_t *addr, size_t delta)
} }
static inline size_t static inline size_t
opal_atomic_fetch_add_size_t(volatile size_t *addr, size_t delta) opal_atomic_fetch_add_size_t(opal_atomic_size_t *addr, size_t delta)
{ {
#if SIZEOF_SIZE_T == 4 #if SIZEOF_SIZE_T == 4
return (size_t) opal_atomic_fetch_add_32((int32_t*) addr, delta); return (size_t) opal_atomic_fetch_add_32((int32_t*) addr, delta);
@ -482,7 +482,7 @@ opal_atomic_fetch_add_size_t(volatile size_t *addr, size_t delta)
} }
static inline size_t static inline size_t
opal_atomic_sub_fetch_size_t(volatile size_t *addr, size_t delta) opal_atomic_sub_fetch_size_t(opal_atomic_size_t *addr, size_t delta)
{ {
#if SIZEOF_SIZE_T == 4 #if SIZEOF_SIZE_T == 4
return (size_t) opal_atomic_sub_fetch_32((int32_t*) addr, delta); return (size_t) opal_atomic_sub_fetch_32((int32_t*) addr, delta);
@ -494,7 +494,7 @@ opal_atomic_sub_fetch_size_t(volatile size_t *addr, size_t delta)
} }
static inline size_t static inline size_t
opal_atomic_fetch_sub_size_t(volatile size_t *addr, size_t delta) opal_atomic_fetch_sub_size_t(opal_atomic_size_t *addr, size_t delta)
{ {
#if SIZEOF_SIZE_T == 4 #if SIZEOF_SIZE_T == 4
return (size_t) opal_atomic_fetch_sub_32((int32_t*) addr, delta); return (size_t) opal_atomic_fetch_sub_32((int32_t*) addr, delta);
@ -507,15 +507,15 @@ opal_atomic_fetch_sub_size_t(volatile size_t *addr, size_t delta)
#else #else
#if SIZEOF_SIZE_T == 4 #if SIZEOF_SIZE_T == 4
#define opal_atomic_add_fetch_size_t(addr, delta) ((size_t) opal_atomic_add_fetch_32((volatile int32_t *) addr, delta)) #define opal_atomic_add_fetch_size_t(addr, delta) ((size_t) opal_atomic_add_fetch_32((opal_atomic_int32_t *) addr, delta))
#define opal_atomic_fetch_add_size_t(addr, delta) ((size_t) opal_atomic_fetch_add_32((volatile int32_t *) addr, delta)) #define opal_atomic_fetch_add_size_t(addr, delta) ((size_t) opal_atomic_fetch_add_32((opal_atomic_int32_t *) addr, delta))
#define opal_atomic_sub_fetch_size_t(addr, delta) ((size_t) opal_atomic_sub_fetch_32((volatile int32_t *) addr, delta)) #define opal_atomic_sub_fetch_size_t(addr, delta) ((size_t) opal_atomic_sub_fetch_32((opal_atomic_int32_t *) addr, delta))
#define opal_atomic_fetch_sub_size_t(addr, delta) ((size_t) opal_atomic_fetch_sub_32((volatile int32_t *) addr, delta)) #define opal_atomic_fetch_sub_size_t(addr, delta) ((size_t) opal_atomic_fetch_sub_32((opal_atomic_int32_t *) addr, delta))
#elif SIZEOF_SIZE_T == 8 #elif SIZEOF_SIZE_T == 8
#define opal_atomic_add_fetch_size_t(addr, delta) ((size_t) opal_atomic_add_fetch_64((volatile int64_t *) addr, delta)) #define opal_atomic_add_fetch_size_t(addr, delta) ((size_t) opal_atomic_add_fetch_64((opal_atomic_int64_t *) addr, delta))
#define opal_atomic_fetch_add_size_t(addr, delta) ((size_t) opal_atomic_fetch_add_64((volatile int64_t *) addr, delta)) #define opal_atomic_fetch_add_size_t(addr, delta) ((size_t) opal_atomic_fetch_add_64((opal_atomic_int64_t *) addr, delta))
#define opal_atomic_sub_fetch_size_t(addr, delta) ((size_t) opal_atomic_sub_fetch_64((volatile int64_t *) addr, delta)) #define opal_atomic_sub_fetch_size_t(addr, delta) ((size_t) opal_atomic_sub_fetch_64((opal_atomic_int64_t *) addr, delta))
#define opal_atomic_fetch_sub_size_t(addr, delta) ((size_t) opal_atomic_fetch_sub_64((volatile int64_t *) addr, delta)) #define opal_atomic_fetch_sub_size_t(addr, delta) ((size_t) opal_atomic_fetch_sub_64((opal_atomic_int64_t *) addr, delta))
#else #else
#error "Unknown size_t size" #error "Unknown size_t size"
#endif #endif
@ -525,20 +525,20 @@ opal_atomic_fetch_sub_size_t(volatile size_t *addr, size_t delta)
/* these are always done with inline functions, so always mark as /* these are always done with inline functions, so always mark as
static inline */ static inline */
static inline bool opal_atomic_compare_exchange_strong_xx (volatile void *addr, void *oldval, static inline bool opal_atomic_compare_exchange_strong_xx (opal_atomic_intptr_t *addr, intptr_t *oldval,
int64_t newval, size_t length); int64_t newval, size_t length);
static inline bool opal_atomic_compare_exchange_strong_acq_xx (volatile void *addr, void *oldval, static inline bool opal_atomic_compare_exchange_strong_acq_xx (opal_atomic_intptr_t *addr, intptr_t *oldval,
int64_t newval, size_t length); int64_t newval, size_t length);
static inline bool opal_atomic_compare_exchange_strong_rel_xx (volatile void *addr, void *oldval, static inline bool opal_atomic_compare_exchange_strong_rel_xx (opal_atomic_intptr_t *addr, intptr_t *oldval,
int64_t newval, size_t length); int64_t newval, size_t length);
static inline bool opal_atomic_compare_exchange_strong_ptr (volatile void* addr, void *oldval, static inline bool opal_atomic_compare_exchange_strong_ptr (opal_atomic_intptr_t* addr, intptr_t *oldval,
void *newval); intptr_t newval);
static inline bool opal_atomic_compare_exchange_strong_acq_ptr (volatile void* addr, void *oldval, static inline bool opal_atomic_compare_exchange_strong_acq_ptr (opal_atomic_intptr_t* addr, intptr_t *oldval,
void *newval); intptr_t newval);
static inline bool opal_atomic_compare_exchange_strong_rel_ptr (volatile void* addr, void *oldval, static inline bool opal_atomic_compare_exchange_strong_rel_ptr (opal_atomic_intptr_t* addr, intptr_t *oldval,
void *newval); intptr_t newval);
/** /**
* Atomic compare and set of generic type with relaxed semantics. This * Atomic compare and set of generic type with relaxed semantics. This
@ -554,7 +554,7 @@ static inline bool opal_atomic_compare_exchange_strong_rel_ptr (volatile void* a
* See opal_atomic_compare_exchange_* for pseudo-code. * See opal_atomic_compare_exchange_* for pseudo-code.
*/ */
#define opal_atomic_compare_exchange_strong( ADDR, OLDVAL, NEWVAL ) \ #define opal_atomic_compare_exchange_strong( ADDR, OLDVAL, NEWVAL ) \
opal_atomic_compare_exchange_strong_xx( (volatile void*)(ADDR), (void *)(OLDVAL), \ opal_atomic_compare_exchange_strong_xx( (opal_atomic_intptr_t*)(ADDR), (intptr_t *)(OLDVAL), \
(intptr_t)(NEWVAL), sizeof(*(ADDR)) ) (intptr_t)(NEWVAL), sizeof(*(ADDR)) )
/** /**
@ -571,7 +571,7 @@ static inline bool opal_atomic_compare_exchange_strong_rel_ptr (volatile void* a
* See opal_atomic_compare_exchange_acq_* for pseudo-code. * See opal_atomic_compare_exchange_acq_* for pseudo-code.
*/ */
#define opal_atomic_compare_exchange_strong_acq( ADDR, OLDVAL, NEWVAL ) \ #define opal_atomic_compare_exchange_strong_acq( ADDR, OLDVAL, NEWVAL ) \
opal_atomic_compare_exchange_strong_acq_xx( (volatile void*)(ADDR), (void *)(OLDVAL), \ opal_atomic_compare_exchange_strong_acq_xx( (opal_atomic_intptr_t*)(ADDR), (intptr_t *)(OLDVAL), \
(intptr_t)(NEWVAL), sizeof(*(ADDR)) ) (intptr_t)(NEWVAL), sizeof(*(ADDR)) )
/** /**
@ -588,7 +588,7 @@ static inline bool opal_atomic_compare_exchange_strong_rel_ptr (volatile void* a
* See opal_atomic_compare_exchange_rel_* for pseudo-code. * See opal_atomic_compare_exchange_rel_* for pseudo-code.
*/ */
#define opal_atomic_compare_exchange_strong_rel( ADDR, OLDVAL, NEWVAL ) \ #define opal_atomic_compare_exchange_strong_rel( ADDR, OLDVAL, NEWVAL ) \
opal_atomic_compare_exchange_strong_rel_xx( (volatile void*)(ADDR), (void *)(OLDVAL), \ opal_atomic_compare_exchange_strong_rel_xx( (opal_atomic_intptr_t*)(ADDR), (intptr_t *)(OLDVAL), \
(intptr_t)(NEWVAL), sizeof(*(ADDR)) ) (intptr_t)(NEWVAL), sizeof(*(ADDR)) )
@ -596,15 +596,15 @@ static inline bool opal_atomic_compare_exchange_strong_rel_ptr (volatile void* a
#if defined(DOXYGEN) || (OPAL_HAVE_ATOMIC_MATH_32 || OPAL_HAVE_ATOMIC_MATH_64) #if defined(DOXYGEN) || (OPAL_HAVE_ATOMIC_MATH_32 || OPAL_HAVE_ATOMIC_MATH_64)
static inline void opal_atomic_add_xx(volatile void* addr, static inline void opal_atomic_add_xx(opal_atomic_intptr_t* addr,
int32_t value, size_t length); int32_t value, size_t length);
static inline void opal_atomic_sub_xx(volatile void* addr, static inline void opal_atomic_sub_xx(opal_atomic_intptr_t* addr,
int32_t value, size_t length); int32_t value, size_t length);
static inline intptr_t opal_atomic_add_fetch_ptr( volatile void* addr, void* delta ); static inline intptr_t opal_atomic_add_fetch_ptr( opal_atomic_intptr_t* addr, void* delta );
static inline intptr_t opal_atomic_fetch_add_ptr( volatile void* addr, void* delta ); static inline intptr_t opal_atomic_fetch_add_ptr( opal_atomic_intptr_t* addr, void* delta );
static inline intptr_t opal_atomic_sub_fetch_ptr( volatile void* addr, void* delta ); static inline intptr_t opal_atomic_sub_fetch_ptr( opal_atomic_intptr_t* addr, void* delta );
static inline intptr_t opal_atomic_fetch_sub_ptr( volatile void* addr, void* delta ); static inline intptr_t opal_atomic_fetch_sub_ptr( opal_atomic_intptr_t* addr, void* delta );
/** /**
* Atomically increment the content depending on the type. This * Atomically increment the content depending on the type. This
@ -617,7 +617,7 @@ static inline intptr_t opal_atomic_fetch_sub_ptr( volatile void* addr, void* del
* @param delta Value to add (converted to <TYPE>). * @param delta Value to add (converted to <TYPE>).
*/ */
#define opal_atomic_add( ADDR, VALUE ) \ #define opal_atomic_add( ADDR, VALUE ) \
opal_atomic_add_xx( (volatile void*)(ADDR), (int32_t)(VALUE), \ opal_atomic_add_xx( (opal_atomic_intptr_t*)(ADDR), (int32_t)(VALUE), \
sizeof(*(ADDR)) ) sizeof(*(ADDR)) )
/** /**
@ -631,7 +631,7 @@ static inline intptr_t opal_atomic_fetch_sub_ptr( volatile void* addr, void* del
* @param delta Value to substract (converted to <TYPE>). * @param delta Value to substract (converted to <TYPE>).
*/ */
#define opal_atomic_sub( ADDR, VALUE ) \ #define opal_atomic_sub( ADDR, VALUE ) \
opal_atomic_sub_xx( (volatile void*)(ADDR), (int32_t)(VALUE), \ opal_atomic_sub_xx( (opal_atomic_intptr_t*)(ADDR), (int32_t)(VALUE), \
sizeof(*(ADDR)) ) sizeof(*(ADDR)) )
#endif /* OPAL_HAVE_ATOMIC_MATH_32 || OPAL_HAVE_ATOMIC_MATH_64 */ #endif /* OPAL_HAVE_ATOMIC_MATH_32 || OPAL_HAVE_ATOMIC_MATH_64 */

92
opal/include/opal/sys/atomic_impl.h
Просмотреть файл

@ -11,7 +11,7 @@
* Copyright (c) 2004-2005 The Regents of the University of California. * Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved. * All rights reserved.
* Copyright (c) 2010-2014 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2010-2014 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2012-2017 Los Alamos National Security, LLC. All rights * Copyright (c) 2012-2018 Los Alamos National Security, LLC. All rights
* reserved. * reserved.
* $COPYRIGHT$ * $COPYRIGHT$
* *
@ -40,7 +40,7 @@
#if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 #if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32
#if !defined(OPAL_HAVE_ATOMIC_MIN_32) #if !defined(OPAL_HAVE_ATOMIC_MIN_32)
static inline int32_t opal_atomic_fetch_min_32 (volatile int32_t *addr, int32_t value) static inline int32_t opal_atomic_fetch_min_32 (opal_atomic_int32_t *addr, int32_t value)
{ {
int32_t old = *addr; int32_t old = *addr;
do { do {
@ -57,7 +57,7 @@ static inline int32_t opal_atomic_fetch_min_32 (volatile int32_t *addr, int32_t
#endif /* OPAL_HAVE_ATOMIC_MIN_32 */ #endif /* OPAL_HAVE_ATOMIC_MIN_32 */
#if !defined(OPAL_HAVE_ATOMIC_MAX_32) #if !defined(OPAL_HAVE_ATOMIC_MAX_32)
static inline int32_t opal_atomic_fetch_max_32 (volatile int32_t *addr, int32_t value) static inline int32_t opal_atomic_fetch_max_32 (opal_atomic_int32_t *addr, int32_t value)
{ {
int32_t old = *addr; int32_t old = *addr;
do { do {
@ -73,7 +73,7 @@ static inline int32_t opal_atomic_fetch_max_32 (volatile int32_t *addr, int32_t
#endif /* OPAL_HAVE_ATOMIC_MAX_32 */ #endif /* OPAL_HAVE_ATOMIC_MAX_32 */
#define OPAL_ATOMIC_DEFINE_CMPXCG_OP(type, bits, operation, name) \ #define OPAL_ATOMIC_DEFINE_CMPXCG_OP(type, bits, operation, name) \
static inline type opal_atomic_fetch_ ## name ## _ ## bits (volatile type *addr, type value) \ static inline type opal_atomic_fetch_ ## name ## _ ## bits (opal_atomic_ ## type *addr, type value) \
{ \ { \
type oldval; \ type oldval; \
do { \ do { \
@ -85,7 +85,7 @@ static inline int32_t opal_atomic_fetch_max_32 (volatile int32_t *addr, int32_t
#if !defined(OPAL_HAVE_ATOMIC_SWAP_32) #if !defined(OPAL_HAVE_ATOMIC_SWAP_32)
#define OPAL_HAVE_ATOMIC_SWAP_32 1 #define OPAL_HAVE_ATOMIC_SWAP_32 1
static inline int32_t opal_atomic_swap_32(volatile int32_t *addr, static inline int32_t opal_atomic_swap_32(opal_atomic_int32_t *addr,
int32_t newval) int32_t newval)
{ {
int32_t old = *addr; int32_t old = *addr;
@ -138,7 +138,7 @@ OPAL_ATOMIC_DEFINE_CMPXCG_OP(int32_t, 32, -, sub)
#if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 #if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64
#if !defined(OPAL_HAVE_ATOMIC_MIN_64) #if !defined(OPAL_HAVE_ATOMIC_MIN_64)
static inline int64_t opal_atomic_fetch_min_64 (volatile int64_t *addr, int64_t value) static inline int64_t opal_atomic_fetch_min_64 (opal_atomic_int64_t *addr, int64_t value)
{ {
int64_t old = *addr; int64_t old = *addr;
do { do {
@ -155,7 +155,7 @@ static inline int64_t opal_atomic_fetch_min_64 (volatile int64_t *addr, int64_t
#endif /* OPAL_HAVE_ATOMIC_MIN_64 */ #endif /* OPAL_HAVE_ATOMIC_MIN_64 */
#if !defined(OPAL_HAVE_ATOMIC_MAX_64) #if !defined(OPAL_HAVE_ATOMIC_MAX_64)
static inline int64_t opal_atomic_fetch_max_64 (volatile int64_t *addr, int64_t value) static inline int64_t opal_atomic_fetch_max_64 (opal_atomic_int64_t *addr, int64_t value)
{ {
int64_t old = *addr; int64_t old = *addr;
do { do {
@ -172,7 +172,7 @@ static inline int64_t opal_atomic_fetch_max_64 (volatile int64_t *addr, int64_t
#if !defined(OPAL_HAVE_ATOMIC_SWAP_64) #if !defined(OPAL_HAVE_ATOMIC_SWAP_64)
#define OPAL_HAVE_ATOMIC_SWAP_64 1 #define OPAL_HAVE_ATOMIC_SWAP_64 1
static inline int64_t opal_atomic_swap_64(volatile int64_t *addr, static inline int64_t opal_atomic_swap_64(opal_atomic_int64_t *addr,
int64_t newval) int64_t newval)
{ {
int64_t old = *addr; int64_t old = *addr;
@ -235,15 +235,15 @@ OPAL_ATOMIC_DEFINE_CMPXCG_OP(int64_t, 64, -, sub)
#if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 #if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64
#define OPAL_ATOMIC_DEFINE_CMPXCG_XX(semantics) \ #define OPAL_ATOMIC_DEFINE_CMPXCG_XX(semantics) \
static inline bool \ static inline bool \
opal_atomic_compare_exchange_strong ## semantics ## xx (volatile void* addr, void *oldval, \ opal_atomic_compare_exchange_strong ## semantics ## xx (opal_atomic_intptr_t* addr, intptr_t *oldval, \
int64_t newval, const size_t length) \ int64_t newval, const size_t length) \
{ \ { \
switch (length) { \ switch (length) { \
case 4: \ case 4: \
return opal_atomic_compare_exchange_strong_32 ((volatile int32_t *) addr, \ return opal_atomic_compare_exchange_strong_32 ((opal_atomic_int32_t *) addr, \
(int32_t *) oldval, (int32_t) newval); \ (int32_t *) oldval, (int32_t) newval); \
case 8: \ case 8: \
return opal_atomic_compare_exchange_strong_64 ((volatile int64_t *) addr, \ return opal_atomic_compare_exchange_strong_64 ((opal_atomic_int64_t *) addr, \
(int64_t *) oldval, (int64_t) newval); \ (int64_t *) oldval, (int64_t) newval); \
} \ } \
abort(); \ abort(); \
@ -251,12 +251,12 @@ OPAL_ATOMIC_DEFINE_CMPXCG_OP(int64_t, 64, -, sub)
#elif OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 #elif OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32
#define OPAL_ATOMIC_DEFINE_CMPXCG_XX(semantics) \ #define OPAL_ATOMIC_DEFINE_CMPXCG_XX(semantics) \
static inline bool \ static inline bool \
opal_atomic_compare_exchange_strong ## semantics ## xx (volatile void* addr, void *oldval, \ opal_atomic_compare_exchange_strong ## semantics ## xx (opal_atomic_intptr_t* addr, intptr_t *oldval, \
int64_t newval, const size_t length) \ int64_t newval, const size_t length) \
{ \ { \
switch (length) { \ switch (length) { \
case 4: \ case 4: \
return opal_atomic_compare_exchange_strong_32 ((volatile int32_t *) addr, \ return opal_atomic_compare_exchange_strong_32 ((opal_atomic_int32_t *) addr, \
(int32_t *) oldval, (int32_t) newval); \ (int32_t *) oldval, (int32_t) newval); \
} \ } \
abort(); \ abort(); \
@ -272,16 +272,16 @@ OPAL_ATOMIC_DEFINE_CMPXCG_XX(_rel_)
#if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 #if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32
#define OPAL_ATOMIC_DEFINE_CMPXCG_PTR_XX(semantics) \ #define OPAL_ATOMIC_DEFINE_CMPXCG_PTR_XX(semantics) \
static inline bool \ static inline bool \
opal_atomic_compare_exchange_strong ## semantics ## ptr (volatile void* addr, void *oldval, void *newval) \ opal_atomic_compare_exchange_strong ## semantics ## ptr (opal_atomic_intptr_t* addr, intptr_t *oldval, intptr_t newval) \
{ \ { \
return opal_atomic_compare_exchange_strong_32 ((volatile int32_t *) addr, (int32_t *) oldval, (int32_t) newval); \ return opal_atomic_compare_exchange_strong_32 ((opal_atomic_int32_t *) addr, (int32_t *) oldval, (int32_t) newval); \
} }
#elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 #elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64
#define OPAL_ATOMIC_DEFINE_CMPXCG_PTR_XX(semantics) \ #define OPAL_ATOMIC_DEFINE_CMPXCG_PTR_XX(semantics) \
static inline bool \ static inline bool \
opal_atomic_compare_exchange_strong ## semantics ## ptr (volatile void* addr, void *oldval, void *newval) \ opal_atomic_compare_exchange_strong ## semantics ## ptr (opal_atomic_intptr_t* addr, intptr_t *oldval, intptr_t newval) \
{ \ { \
return opal_atomic_compare_exchange_strong_64 ((volatile int64_t *) addr, (int64_t *) oldval, (int64_t) newval); \ return opal_atomic_compare_exchange_strong_64 ((opal_atomic_int64_t *) addr, (int64_t *) oldval, (int64_t) newval); \
} }
#else #else
#error "Can not define opal_atomic_compare_exchange_strong_ptr with existing atomics" #error "Can not define opal_atomic_compare_exchange_strong_ptr with existing atomics"
@ -297,9 +297,9 @@ OPAL_ATOMIC_DEFINE_CMPXCG_PTR_XX(_rel_)
#if (OPAL_HAVE_ATOMIC_SWAP_32 || OPAL_HAVE_ATOMIC_SWAP_64) #if (OPAL_HAVE_ATOMIC_SWAP_32 || OPAL_HAVE_ATOMIC_SWAP_64)
#if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_SWAP_32 #if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_SWAP_32
#define opal_atomic_swap_ptr(addr, value) (void *) opal_atomic_swap_32((int32_t *) addr, (int32_t) value) #define opal_atomic_swap_ptr(addr, value) (intptr_t) opal_atomic_swap_32((opal_atomic_int32_t *) addr, (int32_t) value)
#elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_SWAP_64 #elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_SWAP_64
#define opal_atomic_swap_ptr(addr, value) (void *) opal_atomic_swap_64((int64_t *) addr, (int64_t) value) #define opal_atomic_swap_ptr(addr, value) (intptr_t) opal_atomic_swap_64((opal_atomic_int64_t *) addr, (int64_t) value)
#endif #endif
#endif /* (OPAL_HAVE_ATOMIC_SWAP_32 || OPAL_HAVE_ATOMIC_SWAP_64) */ #endif /* (OPAL_HAVE_ATOMIC_SWAP_32 || OPAL_HAVE_ATOMIC_SWAP_64) */
@ -308,15 +308,15 @@ OPAL_ATOMIC_DEFINE_CMPXCG_PTR_XX(_rel_)
#if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_LLSC_32 #if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_LLSC_32
#define opal_atomic_ll_ptr(addr, ret) opal_atomic_ll_32((volatile int32_t *) (addr), ret) #define opal_atomic_ll_ptr(addr, ret) opal_atomic_ll_32((opal_atomic_int32_t *) (addr), ret)
#define opal_atomic_sc_ptr(addr, value, ret) opal_atomic_sc_32((volatile int32_t *) (addr), (intptr_t) (value), ret) #define opal_atomic_sc_ptr(addr, value, ret) opal_atomic_sc_32((opal_atomic_int32_t *) (addr), (intptr_t) (value), ret)
#define OPAL_HAVE_ATOMIC_LLSC_PTR 1 #define OPAL_HAVE_ATOMIC_LLSC_PTR 1
#elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_LLSC_64 #elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_LLSC_64
#define opal_atomic_ll_ptr(addr, ret) opal_atomic_ll_64((volatile int64_t *) (addr), ret) #define opal_atomic_ll_ptr(addr, ret) opal_atomic_ll_64((opal_atomic_int64_t *) (addr), ret)
#define opal_atomic_sc_ptr(addr, value, ret) opal_atomic_sc_64((volatile int64_t *) (addr), (intptr_t) (value), ret) #define opal_atomic_sc_ptr(addr, value, ret) opal_atomic_sc_64((opal_atomic_int64_t *) (addr), (intptr_t) (value), ret)
#define OPAL_HAVE_ATOMIC_LLSC_PTR 1 #define OPAL_HAVE_ATOMIC_LLSC_PTR 1
@ -331,18 +331,18 @@ OPAL_ATOMIC_DEFINE_CMPXCG_PTR_XX(_rel_)
#if OPAL_HAVE_ATOMIC_MATH_32 || OPAL_HAVE_ATOMIC_MATH_64 #if OPAL_HAVE_ATOMIC_MATH_32 || OPAL_HAVE_ATOMIC_MATH_64
static inline void static inline void
opal_atomic_add_xx(volatile void* addr, int32_t value, size_t length) opal_atomic_add_xx(opal_atomic_intptr_t* addr, int32_t value, size_t length)
{ {
switch( length ) { switch( length ) {
#if OPAL_HAVE_ATOMIC_ADD_32 #if OPAL_HAVE_ATOMIC_ADD_32
case 4: case 4:
(void) opal_atomic_fetch_add_32( (volatile int32_t*)addr, (int32_t)value ); (void) opal_atomic_fetch_add_32( (opal_atomic_int32_t*)addr, (int32_t)value );
break; break;
#endif /* OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 */ #endif /* OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 */
#if OPAL_HAVE_ATOMIC_ADD_64 #if OPAL_HAVE_ATOMIC_ADD_64
case 8: case 8:
(void) opal_atomic_fetch_add_64( (volatile int64_t*)addr, (int64_t)value ); (void) opal_atomic_fetch_add_64( (opal_atomic_int64_t*)addr, (int64_t)value );
break; break;
#endif /* OPAL_HAVE_ATOMIC_ADD_64 */ #endif /* OPAL_HAVE_ATOMIC_ADD_64 */
default: default:
@ -354,18 +354,18 @@ static inline void
static inline void static inline void
opal_atomic_sub_xx(volatile void* addr, int32_t value, size_t length) opal_atomic_sub_xx(opal_atomic_intptr_t* addr, int32_t value, size_t length)
{ {
switch( length ) { switch( length ) {
#if OPAL_HAVE_ATOMIC_SUB_32 #if OPAL_HAVE_ATOMIC_SUB_32
case 4: case 4:
(void) opal_atomic_fetch_sub_32( (volatile int32_t*)addr, (int32_t)value ); (void) opal_atomic_fetch_sub_32( (opal_atomic_int32_t*)addr, (int32_t)value );
break; break;
#endif /* OPAL_HAVE_ATOMIC_SUB_32 */ #endif /* OPAL_HAVE_ATOMIC_SUB_32 */
#if OPAL_HAVE_ATOMIC_SUB_64 #if OPAL_HAVE_ATOMIC_SUB_64
case 8: case 8:
(void) opal_atomic_fetch_sub_64( (volatile int64_t*)addr, (int64_t)value ); (void) opal_atomic_fetch_sub_64( (opal_atomic_int64_t*)addr, (int64_t)value );
break; break;
#endif /* OPAL_HAVE_ATOMIC_SUB_64 */ #endif /* OPAL_HAVE_ATOMIC_SUB_64 */
default: default:
@ -376,7 +376,7 @@ opal_atomic_sub_xx(volatile void* addr, int32_t value, size_t length)
} }
#define OPAL_ATOMIC_DEFINE_OP_FETCH(op, operation, type, ptr_type, suffix) \ #define OPAL_ATOMIC_DEFINE_OP_FETCH(op, operation, type, ptr_type, suffix) \
static inline type opal_atomic_ ## op ## _fetch_ ## suffix (volatile ptr_type *addr, type value) \ static inline type opal_atomic_ ## op ## _fetch_ ## suffix (opal_atomic_ ## ptr_type *addr, type value) \
{ \ { \
return opal_atomic_fetch_ ## op ## _ ## suffix (addr, value) operation value; \ return opal_atomic_fetch_ ## op ## _ ## suffix (addr, value) operation value; \
} }
@ -387,13 +387,13 @@ OPAL_ATOMIC_DEFINE_OP_FETCH(or, |, int32_t, int32_t, 32)
OPAL_ATOMIC_DEFINE_OP_FETCH(xor, ^, int32_t, int32_t, 32) OPAL_ATOMIC_DEFINE_OP_FETCH(xor, ^, int32_t, int32_t, 32)
OPAL_ATOMIC_DEFINE_OP_FETCH(sub, -, int32_t, int32_t, 32) OPAL_ATOMIC_DEFINE_OP_FETCH(sub, -, int32_t, int32_t, 32)
static inline int32_t opal_atomic_min_fetch_32 (volatile int32_t *addr, int32_t value) static inline int32_t opal_atomic_min_fetch_32 (opal_atomic_int32_t *addr, int32_t value)
{ {
int32_t old = opal_atomic_fetch_min_32 (addr, value); int32_t old = opal_atomic_fetch_min_32 (addr, value);
return old <= value ? old : value; return old <= value ? old : value;
} }
static inline int32_t opal_atomic_max_fetch_32 (volatile int32_t *addr, int32_t value) static inline int32_t opal_atomic_max_fetch_32 (opal_atomic_int32_t *addr, int32_t value)
{ {
int32_t old = opal_atomic_fetch_max_32 (addr, value); int32_t old = opal_atomic_fetch_max_32 (addr, value);
return old >= value ? old : value; return old >= value ? old : value;
@ -406,13 +406,13 @@ OPAL_ATOMIC_DEFINE_OP_FETCH(or, |, int64_t, int64_t, 64)
OPAL_ATOMIC_DEFINE_OP_FETCH(xor, ^, int64_t, int64_t, 64) OPAL_ATOMIC_DEFINE_OP_FETCH(xor, ^, int64_t, int64_t, 64)
OPAL_ATOMIC_DEFINE_OP_FETCH(sub, -, int64_t, int64_t, 64) OPAL_ATOMIC_DEFINE_OP_FETCH(sub, -, int64_t, int64_t, 64)
static inline int64_t opal_atomic_min_fetch_64 (volatile int64_t *addr, int64_t value) static inline int64_t opal_atomic_min_fetch_64 (opal_atomic_int64_t *addr, int64_t value)
{ {
int64_t old = opal_atomic_fetch_min_64 (addr, value); int64_t old = opal_atomic_fetch_min_64 (addr, value);
return old <= value ? old : value; return old <= value ? old : value;
} }
static inline int64_t opal_atomic_max_fetch_64 (volatile int64_t *addr, int64_t value) static inline int64_t opal_atomic_max_fetch_64 (opal_atomic_int64_t *addr, int64_t value)
{ {
int64_t old = opal_atomic_fetch_max_64 (addr, value); int64_t old = opal_atomic_fetch_max_64 (addr, value);
return old >= value ? old : value; return old >= value ? old : value;
@ -420,52 +420,52 @@ static inline int64_t opal_atomic_max_fetch_64 (volatile int64_t *addr, int64_t
#endif #endif
static inline intptr_t opal_atomic_fetch_add_ptr( volatile void* addr, static inline intptr_t opal_atomic_fetch_add_ptr( opal_atomic_intptr_t* addr,
void* delta ) void* delta )
{ {
#if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_ADD_32 #if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_ADD_32
return opal_atomic_fetch_add_32((int32_t*) addr, (unsigned long) delta); return opal_atomic_fetch_add_32((opal_atomic_int32_t*) addr, (unsigned long) delta);
#elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_ADD_64 #elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_ADD_64
return opal_atomic_fetch_add_64((int64_t*) addr, (unsigned long) delta); return opal_atomic_fetch_add_64((opal_atomic_int64_t*) addr, (unsigned long) delta);
#else #else
abort (); abort ();
return 0; return 0;
#endif #endif
} }
static inline intptr_t opal_atomic_add_fetch_ptr( volatile void* addr, static inline intptr_t opal_atomic_add_fetch_ptr( opal_atomic_intptr_t* addr,
void* delta ) void* delta )
{ {
#if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_ADD_32 #if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_ADD_32
return opal_atomic_add_fetch_32((int32_t*) addr, (unsigned long) delta); return opal_atomic_add_fetch_32((opal_atomic_int32_t*) addr, (unsigned long) delta);
#elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_ADD_64 #elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_ADD_64
return opal_atomic_add_fetch_64((int64_t*) addr, (unsigned long) delta); return opal_atomic_add_fetch_64((opal_atomic_int64_t*) addr, (unsigned long) delta);
#else #else
abort (); abort ();
return 0; return 0;
#endif #endif
} }
static inline intptr_t opal_atomic_fetch_sub_ptr( volatile void* addr, static inline intptr_t opal_atomic_fetch_sub_ptr( opal_atomic_intptr_t* addr,
void* delta ) void* delta )
{ {
#if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_SUB_32 #if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_SUB_32
return opal_atomic_fetch_sub_32((int32_t*) addr, (unsigned long) delta); return opal_atomic_fetch_sub_32((opal_atomic_int32_t*) addr, (unsigned long) delta);
#elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_SUB_32 #elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_SUB_32
return opal_atomic_fetch_sub_64((int64_t*) addr, (unsigned long) delta); return opal_atomic_fetch_sub_64((opal_atomic_int64_t*) addr, (unsigned long) delta);
#else #else
abort(); abort();
return 0; return 0;
#endif #endif
} }
static inline intptr_t opal_atomic_sub_fetch_ptr( volatile void* addr, static inline intptr_t opal_atomic_sub_fetch_ptr( opal_atomic_intptr_t* addr,
void* delta ) void* delta )
{ {
#if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_SUB_32 #if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_SUB_32
return opal_atomic_sub_fetch_32((int32_t*) addr, (unsigned long) delta); return opal_atomic_sub_fetch_32((opal_atomic_int32_t*) addr, (unsigned long) delta);
#elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_SUB_32 #elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_SUB_32
return opal_atomic_sub_fetch_64((int64_t*) addr, (unsigned long) delta); return opal_atomic_sub_fetch_64((opal_atomic_int64_t*) addr, (unsigned long) delta);
#else #else
abort(); abort();
return 0; return 0;

42
opal/include/opal/sys/gcc_builtin/atomic.h
Просмотреть файл

@ -11,7 +11,7 @@
* Copyright (c) 2004-2005 The Regents of the University of California. * Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved. * All rights reserved.
* Copyright (c) 2011 Sandia National Laboratories. All rights reserved. * Copyright (c) 2011 Sandia National Laboratories. All rights reserved.
* Copyright (c) 2014-2017 Los Alamos National Security, LLC. All rights * Copyright (c) 2014-2018 Los Alamos National Security, LLC. All rights
* reserved. * reserved.
* Copyright (c) 2016-2017 Research Organization for Information Science * Copyright (c) 2016-2017 Research Organization for Information Science
* and Technology (RIST). All rights reserved. * and Technology (RIST). All rights reserved.
@ -81,98 +81,98 @@ static inline void opal_atomic_wmb(void)
#pragma error_messages(off, E_ARG_INCOMPATIBLE_WITH_ARG_L) #pragma error_messages(off, E_ARG_INCOMPATIBLE_WITH_ARG_L)
#endif #endif
static inline bool opal_atomic_compare_exchange_strong_acq_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) static inline bool opal_atomic_compare_exchange_strong_acq_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
{ {
return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
} }
static inline bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) static inline bool opal_atomic_compare_exchange_strong_rel_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
{ {
return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_RELEASE, __ATOMIC_RELAXED); return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
} }
static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) static inline bool opal_atomic_compare_exchange_strong_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
{ {
return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
} }
static inline int32_t opal_atomic_swap_32 (volatile int32_t *addr, int32_t newval) static inline int32_t opal_atomic_swap_32 (opal_atomic_int32_t *addr, int32_t newval)
{ {
int32_t oldval; int32_t oldval;
__atomic_exchange (addr, &newval, &oldval, __ATOMIC_RELAXED); __atomic_exchange (addr, &newval, &oldval, __ATOMIC_RELAXED);
return oldval; return oldval;
} }
static inline int32_t opal_atomic_fetch_add_32(volatile int32_t *addr, int32_t delta) static inline int32_t opal_atomic_fetch_add_32(opal_atomic_int32_t *addr, int32_t delta)
{ {
return __atomic_fetch_add (addr, delta, __ATOMIC_RELAXED); return __atomic_fetch_add (addr, delta, __ATOMIC_RELAXED);
} }
static inline int32_t opal_atomic_fetch_and_32(volatile int32_t *addr, int32_t value) static inline int32_t opal_atomic_fetch_and_32(opal_atomic_int32_t *addr, int32_t value)
{ {
return __atomic_fetch_and (addr, value, __ATOMIC_RELAXED); return __atomic_fetch_and (addr, value, __ATOMIC_RELAXED);
} }
static inline int32_t opal_atomic_fetch_or_32(volatile int32_t *addr, int32_t value) static inline int32_t opal_atomic_fetch_or_32(opal_atomic_int32_t *addr, int32_t value)
{ {
return __atomic_fetch_or (addr, value, __ATOMIC_RELAXED); return __atomic_fetch_or (addr, value, __ATOMIC_RELAXED);
} }
static inline int32_t opal_atomic_fetch_xor_32(volatile int32_t *addr, int32_t value) static inline int32_t opal_atomic_fetch_xor_32(opal_atomic_int32_t *addr, int32_t value)
{ {
return __atomic_fetch_xor (addr, value, __ATOMIC_RELAXED); return __atomic_fetch_xor (addr, value, __ATOMIC_RELAXED);
} }
static inline int32_t opal_atomic_fetch_sub_32(volatile int32_t *addr, int32_t delta) static inline int32_t opal_atomic_fetch_sub_32(opal_atomic_int32_t *addr, int32_t delta)
{ {
return __atomic_fetch_sub (addr, delta, __ATOMIC_RELAXED); return __atomic_fetch_sub (addr, delta, __ATOMIC_RELAXED);
} }
static inline bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) static inline bool opal_atomic_compare_exchange_strong_acq_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
{ {
return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
} }
static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) static inline bool opal_atomic_compare_exchange_strong_rel_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
{ {
return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_RELEASE, __ATOMIC_RELAXED); return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
} }
static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) static inline bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
{ {
return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
} }
static inline int64_t opal_atomic_swap_64 (volatile int64_t *addr, int64_t newval) static inline int64_t opal_atomic_swap_64 (opal_atomic_int64_t *addr, int64_t newval)
{ {
int64_t oldval; int64_t oldval;
__atomic_exchange (addr, &newval, &oldval, __ATOMIC_RELAXED); __atomic_exchange (addr, &newval, &oldval, __ATOMIC_RELAXED);
return oldval; return oldval;
} }
static inline int64_t opal_atomic_fetch_add_64(volatile int64_t *addr, int64_t delta) static inline int64_t opal_atomic_fetch_add_64(opal_atomic_int64_t *addr, int64_t delta)
{ {
return __atomic_fetch_add (addr, delta, __ATOMIC_RELAXED); return __atomic_fetch_add (addr, delta, __ATOMIC_RELAXED);
} }
static inline int64_t opal_atomic_fetch_and_64(volatile int64_t *addr, int64_t value) static inline int64_t opal_atomic_fetch_and_64(opal_atomic_int64_t *addr, int64_t value)
{ {
return __atomic_fetch_and (addr, value, __ATOMIC_RELAXED); return __atomic_fetch_and (addr, value, __ATOMIC_RELAXED);
} }
static inline int64_t opal_atomic_fetch_or_64(volatile int64_t *addr, int64_t value) static inline int64_t opal_atomic_fetch_or_64(opal_atomic_int64_t *addr, int64_t value)
{ {
return __atomic_fetch_or (addr, value, __ATOMIC_RELAXED); return __atomic_fetch_or (addr, value, __ATOMIC_RELAXED);
} }
static inline int64_t opal_atomic_fetch_xor_64(volatile int64_t *addr, int64_t value) static inline int64_t opal_atomic_fetch_xor_64(opal_atomic_int64_t *addr, int64_t value)
{ {
return __atomic_fetch_xor (addr, value, __ATOMIC_RELAXED); return __atomic_fetch_xor (addr, value, __ATOMIC_RELAXED);
} }
static inline int64_t opal_atomic_fetch_sub_64(volatile int64_t *addr, int64_t delta) static inline int64_t opal_atomic_fetch_sub_64(opal_atomic_int64_t *addr, int64_t delta)
{ {
return __atomic_fetch_sub (addr, delta, __ATOMIC_RELAXED); return __atomic_fetch_sub (addr, delta, __ATOMIC_RELAXED);
} }
@ -181,7 +181,7 @@ static inline int64_t opal_atomic_fetch_sub_64(volatile int64_t *addr, int64_t d
#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1
static inline bool opal_atomic_compare_exchange_strong_128 (volatile opal_int128_t *addr, static inline bool opal_atomic_compare_exchange_strong_128 (opal_atomic_int128_t *addr,
opal_int128_t *oldval, opal_int128_t newval) opal_int128_t *oldval, opal_int128_t newval)
{ {
return __atomic_compare_exchange_n (addr, oldval, newval, false, return __atomic_compare_exchange_n (addr, oldval, newval, false,
@ -194,7 +194,7 @@ static inline bool opal_atomic_compare_exchange_strong_128 (volatile opal_int128
/* __atomic version is not lock-free so use legacy __sync version */ /* __atomic version is not lock-free so use legacy __sync version */
static inline bool opal_atomic_compare_exchange_strong_128 (volatile opal_int128_t *addr, static inline bool opal_atomic_compare_exchange_strong_128 (opal_atomic_opal_int128_t *addr,
opal_int128_t *oldval, opal_int128_t newval) opal_int128_t *oldval, opal_int128_t newval)
{ {
opal_int128_t prev = __sync_val_compare_and_swap (addr, *oldval, newval); opal_int128_t prev = __sync_val_compare_and_swap (addr, *oldval, newval);

10
opal/include/opal/sys/ia32/atomic.h
Просмотреть файл

@ -13,7 +13,7 @@
* Copyright (c) 2007-2010 Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2007-2010 Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2015 Research Organization for Information Science * Copyright (c) 2015 Research Organization for Information Science
* and Technology (RIST). All rights reserved. * and Technology (RIST). All rights reserved.
* Copyright (c) 2015-2017 Los Alamos National Security, LLC. All rights * Copyright (c) 2015-2018 Los Alamos National Security, LLC. All rights
* reserved. * reserved.
* $COPYRIGHT$ * $COPYRIGHT$
* *
@ -84,7 +84,7 @@ static inline void opal_atomic_isync(void)
*********************************************************************/ *********************************************************************/
#if OPAL_GCC_INLINE_ASSEMBLY #if OPAL_GCC_INLINE_ASSEMBLY
static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) static inline bool opal_atomic_compare_exchange_strong_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
{ {
unsigned char ret; unsigned char ret;
__asm__ __volatile__ ( __asm__ __volatile__ (
@ -106,7 +106,7 @@ static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *add
#define OPAL_HAVE_ATOMIC_SWAP_32 1 #define OPAL_HAVE_ATOMIC_SWAP_32 1
static inline int32_t opal_atomic_swap_32( volatile int32_t *addr, static inline int32_t opal_atomic_swap_32( opal_atomic_int32_t *addr,
int32_t newval) int32_t newval)
{ {
int32_t oldval; int32_t oldval;
@ -130,7 +130,7 @@ static inline int32_t opal_atomic_swap_32( volatile int32_t *addr,
* *
* Atomically adds @i to @v. * Atomically adds @i to @v.
*/ */
static inline int32_t opal_atomic_fetch_add_32(volatile int32_t* v, int i) static inline int32_t opal_atomic_fetch_add_32(opal_atomic_int32_t* v, int i)
{ {
int ret = i; int ret = i;
__asm__ __volatile__( __asm__ __volatile__(
@ -150,7 +150,7 @@ static inline int32_t opal_atomic_fetch_add_32(volatile int32_t* v, int i)
* *
* Atomically subtracts @i from @v. * Atomically subtracts @i from @v.
*/ */
static inline int32_t opal_atomic_fetch_sub_32(volatile int32_t* v, int i) static inline int32_t opal_atomic_fetch_sub_32(opal_atomic_int32_t* v, int i)
{ {
int ret = -i; int ret = -i;
__asm__ __volatile__( __asm__ __volatile__(

30
opal/include/opal/sys/powerpc/atomic.h
Просмотреть файл

@ -144,7 +144,7 @@ void opal_atomic_isync(void)
#define OPAL_ASM_VALUE64(x) x #define OPAL_ASM_VALUE64(x) x
#endif #endif
static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) static inline bool opal_atomic_compare_exchange_strong_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
{ {
int32_t prev; int32_t prev;
bool ret; bool ret;
@ -170,7 +170,7 @@ static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *add
* load the arguments to/from the stack. This sequence may cause the ll reservation to be cancelled. */ * load the arguments to/from the stack. This sequence may cause the ll reservation to be cancelled. */
#define opal_atomic_ll_32(addr, ret) \ #define opal_atomic_ll_32(addr, ret) \
do { \ do { \
volatile int32_t *_addr = (addr); \ opal_atomic_int32_t *_addr = (addr); \
int32_t _ret; \ int32_t _ret; \
__asm__ __volatile__ ("lwarx %0, 0, %1 \n\t" \ __asm__ __volatile__ ("lwarx %0, 0, %1 \n\t" \
: "=&r" (_ret) \ : "=&r" (_ret) \
@ -181,7 +181,7 @@ static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *add
#define opal_atomic_sc_32(addr, value, ret) \ #define opal_atomic_sc_32(addr, value, ret) \
do { \ do { \
volatile int32_t *_addr = (addr); \ opal_atomic_int32_t *_addr = (addr); \
int32_t _ret, _foo, _newval = (int32_t) value; \ int32_t _ret, _foo, _newval = (int32_t) value; \
\ \
__asm__ __volatile__ (" stwcx. %4, 0, %3 \n\t" \ __asm__ __volatile__ (" stwcx. %4, 0, %3 \n\t" \
@ -200,7 +200,7 @@ static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *add
atomic_?mb can be inlined). Instead, we "inline" them by hand in atomic_?mb can be inlined). Instead, we "inline" them by hand in
the assembly, meaning there is one function call overhead instead the assembly, meaning there is one function call overhead instead
of two */ of two */
static inline bool opal_atomic_compare_exchange_strong_acq_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) static inline bool opal_atomic_compare_exchange_strong_acq_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
{ {
bool rc; bool rc;
@ -211,13 +211,13 @@ static inline bool opal_atomic_compare_exchange_strong_acq_32 (volatile int32_t
} }
static inline bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) static inline bool opal_atomic_compare_exchange_strong_rel_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
{ {
opal_atomic_wmb(); opal_atomic_wmb();
return opal_atomic_compare_exchange_strong_32 (addr, oldval, newval); return opal_atomic_compare_exchange_strong_32 (addr, oldval, newval);
} }
static inline int32_t opal_atomic_swap_32(volatile int32_t *addr, int32_t newval) static inline int32_t opal_atomic_swap_32(opal_atomic_int32_t *addr, int32_t newval)
{ {
int32_t ret; int32_t ret;
@ -239,7 +239,7 @@ static inline int32_t opal_atomic_swap_32(volatile int32_t *addr, int32_t newval
#if OPAL_GCC_INLINE_ASSEMBLY #if OPAL_GCC_INLINE_ASSEMBLY
#define OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(type, instr) \ #define OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(type, instr) \
static inline int64_t opal_atomic_fetch_ ## type ## _64(volatile int64_t* v, int64_t val) \ static inline int64_t opal_atomic_fetch_ ## type ## _64(opal_atomic_int64_t* v, int64_t val) \
{ \ { \
int64_t t, old; \ int64_t t, old; \
\ \
@ -261,7 +261,7 @@ OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(or, or)
OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(xor, xor) OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(xor, xor)
OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(sub, subf) OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(sub, subf)
static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) static inline bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
{ {
int64_t prev; int64_t prev;
bool ret; bool ret;
@ -284,7 +284,7 @@ static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *add
#define opal_atomic_ll_64(addr, ret) \ #define opal_atomic_ll_64(addr, ret) \
do { \ do { \
volatile int64_t *_addr = (addr); \ opal_atomic_int64_t *_addr = (addr); \
int64_t _ret; \ int64_t _ret; \
__asm__ __volatile__ ("ldarx %0, 0, %1 \n\t" \ __asm__ __volatile__ ("ldarx %0, 0, %1 \n\t" \
: "=&r" (_ret) \ : "=&r" (_ret) \
@ -295,7 +295,7 @@ static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *add
#define opal_atomic_sc_64(addr, value, ret) \ #define opal_atomic_sc_64(addr, value, ret) \
do { \ do { \
volatile int64_t *_addr = (addr); \ opal_atomic_int64_t *_addr = (addr); \
int64_t _foo, _newval = (int64_t) value; \ int64_t _foo, _newval = (int64_t) value; \
int32_t _ret; \ int32_t _ret; \
\ \
@ -310,7 +310,7 @@ static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *add
ret = _ret; \ ret = _ret; \
} while (0) } while (0)
static inline int64_t opal_atomic_swap_64(volatile int64_t *addr, int64_t newval) static inline int64_t opal_atomic_swap_64(opal_atomic_int64_t *addr, int64_t newval)
{ {
int64_t ret; int64_t ret;
@ -335,7 +335,7 @@ static inline int64_t opal_atomic_swap_64(volatile int64_t *addr, int64_t newval
#if OPAL_GCC_INLINE_ASSEMBLY #if OPAL_GCC_INLINE_ASSEMBLY
static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) static inline bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
{ {
int64_t prev; int64_t prev;
int ret; int ret;
@ -382,7 +382,7 @@ static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *add
atomic_?mb can be inlined). Instead, we "inline" them by hand in atomic_?mb can be inlined). Instead, we "inline" them by hand in
the assembly, meaning there is one function call overhead instead the assembly, meaning there is one function call overhead instead
of two */ of two */
static inline bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) static inline bool opal_atomic_compare_exchange_strong_acq_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
{ {
bool rc; bool rc;
@ -393,7 +393,7 @@ static inline bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t
} }
static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) static inline bool opal_atomic_compare_exchange_strong_rel_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
{ {
opal_atomic_wmb(); opal_atomic_wmb();
return opal_atomic_compare_exchange_strong_64 (addr, oldval, newval); return opal_atomic_compare_exchange_strong_64 (addr, oldval, newval);
@ -401,7 +401,7 @@ static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t
#define OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_32(type, instr) \ #define OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_32(type, instr) \
static inline int32_t opal_atomic_fetch_ ## type ## _32(volatile int32_t* v, int val) \ static inline int32_t opal_atomic_fetch_ ## type ## _32(opal_atomic_int32_t* v, int val) \
{ \ { \
int32_t t, old; \ int32_t t, old; \
\ \

16
opal/include/opal/sys/sparcv9/atomic.h
Просмотреть файл

@ -13,7 +13,7 @@
* Copyright (c) 2007 Sun Microsystems, Inc. All rights reserverd. * Copyright (c) 2007 Sun Microsystems, Inc. All rights reserverd.
* Copyright (c) 2016 Research Organization for Information Science * Copyright (c) 2016 Research Organization for Information Science
* and Technology (RIST). All rights reserved. * and Technology (RIST). All rights reserved.
* Copyright (c) 2017 Los Alamos National Security, LLC. All rights * Copyright (c) 2017-2018 Los Alamos National Security, LLC. All rights
* reserved. * reserved.
* $COPYRIGHT$ * $COPYRIGHT$
* *
@ -85,7 +85,7 @@ static inline void opal_atomic_isync(void)
*********************************************************************/ *********************************************************************/
#if OPAL_GCC_INLINE_ASSEMBLY #if OPAL_GCC_INLINE_ASSEMBLY
static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) static inline bool opal_atomic_compare_exchange_strong_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
{ {
/* casa [reg(rs1)] %asi, reg(rs2), reg(rd) /* casa [reg(rs1)] %asi, reg(rs2), reg(rd)
* *
@ -107,7 +107,7 @@ static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *add
} }
static inline bool opal_atomic_compare_exchange_strong_acq_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) static inline bool opal_atomic_compare_exchange_strong_acq_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
{ {
bool rc; bool rc;
@ -118,7 +118,7 @@ static inline bool opal_atomic_compare_exchange_strong_acq_32 (volatile int32_t
} }
static inline bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) static inline bool opal_atomic_compare_exchange_strong_rel_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
{ {
opal_atomic_wmb(); opal_atomic_wmb();
return opal_atomic_compare_exchange_strong_32 (addr, oldval, newval); return opal_atomic_compare_exchange_strong_32 (addr, oldval, newval);
@ -127,7 +127,7 @@ static inline bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t
#if OPAL_ASSEMBLY_ARCH == OPAL_SPARCV9_64 #if OPAL_ASSEMBLY_ARCH == OPAL_SPARCV9_64
static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) static inline bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
{ {
/* casa [reg(rs1)] %asi, reg(rs2), reg(rd) /* casa [reg(rs1)] %asi, reg(rs2), reg(rd)
* *
@ -149,7 +149,7 @@ static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *add
#else /* OPAL_ASSEMBLY_ARCH == OPAL_SPARCV9_64 */ #else /* OPAL_ASSEMBLY_ARCH == OPAL_SPARCV9_64 */
static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) static inline bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
{ {
/* casa [reg(rs1)] %asi, reg(rs2), reg(rd) /* casa [reg(rs1)] %asi, reg(rs2), reg(rd)
* *
@ -179,7 +179,7 @@ static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *add
#endif /* OPAL_ASSEMBLY_ARCH == OPAL_SPARCV9_64 */ #endif /* OPAL_ASSEMBLY_ARCH == OPAL_SPARCV9_64 */
static inline bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) static inline bool opal_atomic_compare_exchange_strong_acq_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
{ {
bool rc; bool rc;
@ -190,7 +190,7 @@ static inline bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t
} }
static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) static inline bool opal_atomic_compare_exchange_strong_rel_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
{ {
opal_atomic_wmb(); opal_atomic_wmb();
return opal_atomic_compare_exchange_strong_64 (addr, oldval, newval); return opal_atomic_compare_exchange_strong_64 (addr, oldval, newval);

28
opal/include/opal/sys/sync_builtin/atomic.h
Просмотреть файл

@ -11,7 +11,7 @@
* Copyright (c) 2004-2005 The Regents of the University of California. * Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved. * All rights reserved.
* Copyright (c) 2011 Sandia National Laboratories. All rights reserved. * Copyright (c) 2011 Sandia National Laboratories. All rights reserved.
* Copyright (c) 2014-2017 Los Alamos National Security, LLC. All rights * Copyright (c) 2014-2018 Los Alamos National Security, LLC. All rights
* reserved. * reserved.
* Copyright (c) 2017 Research Organization for Information Science * Copyright (c) 2017 Research Organization for Information Science
* and Technology (RIST). All rights reserved. * and Technology (RIST). All rights reserved.
@ -57,7 +57,7 @@ static inline void opal_atomic_wmb(void)
#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1
static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) static inline bool opal_atomic_compare_exchange_strong_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
{ {
int32_t prev = __sync_val_compare_and_swap (addr, *oldval, newval); int32_t prev = __sync_val_compare_and_swap (addr, *oldval, newval);
bool ret = prev == *oldval; bool ret = prev == *oldval;
@ -71,31 +71,31 @@ static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *add
#define OPAL_HAVE_ATOMIC_MATH_32 1 #define OPAL_HAVE_ATOMIC_MATH_32 1
#define OPAL_HAVE_ATOMIC_ADD_32 1 #define OPAL_HAVE_ATOMIC_ADD_32 1
static inline int32_t opal_atomic_fetch_add_32(volatile int32_t *addr, int32_t delta) static inline int32_t opal_atomic_fetch_add_32(opal_atomic_int32_t *addr, int32_t delta)
{ {
return __sync_fetch_and_add(addr, delta); return __sync_fetch_and_add(addr, delta);
} }
#define OPAL_HAVE_ATOMIC_AND_32 1 #define OPAL_HAVE_ATOMIC_AND_32 1
static inline int32_t opal_atomic_fetch_and_32(volatile int32_t *addr, int32_t value) static inline int32_t opal_atomic_fetch_and_32(opal_atomic_int32_t *addr, int32_t value)
{ {
return __sync_fetch_and_and(addr, value); return __sync_fetch_and_and(addr, value);
} }
#define OPAL_HAVE_ATOMIC_OR_32 1 #define OPAL_HAVE_ATOMIC_OR_32 1
static inline int32_t opal_atomic_fetch_or_32(volatile int32_t *addr, int32_t value) static inline int32_t opal_atomic_fetch_or_32(opal_atomic_int32_t *addr, int32_t value)
{ {
return __sync_fetch_and_or(addr, value); return __sync_fetch_and_or(addr, value);
} }
#define OPAL_HAVE_ATOMIC_XOR_32 1 #define OPAL_HAVE_ATOMIC_XOR_32 1
static inline int32_t opal_atomic_fetch_xor_32(volatile int32_t *addr, int32_t value) static inline int32_t opal_atomic_fetch_xor_32(opal_atomic_int32_t *addr, int32_t value)
{ {
return __sync_fetch_and_xor(addr, value); return __sync_fetch_and_xor(addr, value);
} }
#define OPAL_HAVE_ATOMIC_SUB_32 1 #define OPAL_HAVE_ATOMIC_SUB_32 1
static inline int32_t opal_atomic_fetch_sub_32(volatile int32_t *addr, int32_t delta) static inline int32_t opal_atomic_fetch_sub_32(opal_atomic_int32_t *addr, int32_t delta)
{ {
return __sync_fetch_and_sub(addr, delta); return __sync_fetch_and_sub(addr, delta);
} }
@ -104,7 +104,7 @@ static inline int32_t opal_atomic_fetch_sub_32(volatile int32_t *addr, int32_t d
#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1
static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) static inline bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
{ {
int64_t prev = __sync_val_compare_and_swap (addr, *oldval, newval); int64_t prev = __sync_val_compare_and_swap (addr, *oldval, newval);
bool ret = prev == *oldval; bool ret = prev == *oldval;
@ -117,31 +117,31 @@ static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *add
#define OPAL_HAVE_ATOMIC_MATH_64 1 #define OPAL_HAVE_ATOMIC_MATH_64 1
#define OPAL_HAVE_ATOMIC_ADD_64 1 #define OPAL_HAVE_ATOMIC_ADD_64 1
static inline int64_t opal_atomic_fetch_add_64(volatile int64_t *addr, int64_t delta) static inline int64_t opal_atomic_fetch_add_64(opal_atomic_int64_t *addr, int64_t delta)
{ {
return __sync_fetch_and_add(addr, delta); return __sync_fetch_and_add(addr, delta);
} }
#define OPAL_HAVE_ATOMIC_AND_64 1 #define OPAL_HAVE_ATOMIC_AND_64 1
static inline int64_t opal_atomic_fetch_and_64(volatile int64_t *addr, int64_t value) static inline int64_t opal_atomic_fetch_and_64(opal_atomic_int64_t *addr, int64_t value)
{ {
return __sync_fetch_and_and(addr, value); return __sync_fetch_and_and(addr, value);
} }
#define OPAL_HAVE_ATOMIC_OR_64 1 #define OPAL_HAVE_ATOMIC_OR_64 1
static inline int64_t opal_atomic_fetch_or_64(volatile int64_t *addr, int64_t value) static inline int64_t opal_atomic_fetch_or_64(opal_atomic_int64_t *addr, int64_t value)
{ {
return __sync_fetch_and_or(addr, value); return __sync_fetch_and_or(addr, value);
} }
#define OPAL_HAVE_ATOMIC_XOR_64 1 #define OPAL_HAVE_ATOMIC_XOR_64 1
static inline int64_t opal_atomic_fetch_xor_64(volatile int64_t *addr, int64_t value) static inline int64_t opal_atomic_fetch_xor_64(opal_atomic_int64_t *addr, int64_t value)
{ {
return __sync_fetch_and_xor(addr, value); return __sync_fetch_and_xor(addr, value);
} }
#define OPAL_HAVE_ATOMIC_SUB_64 1 #define OPAL_HAVE_ATOMIC_SUB_64 1
static inline int64_t opal_atomic_fetch_sub_64(volatile int64_t *addr, int64_t delta) static inline int64_t opal_atomic_fetch_sub_64(opal_atomic_int64_t *addr, int64_t delta)
{ {
return __sync_fetch_and_sub(addr, delta); return __sync_fetch_and_sub(addr, delta);
} }
@ -149,7 +149,7 @@ static inline int64_t opal_atomic_fetch_sub_64(volatile int64_t *addr, int64_t d
#endif #endif
#if OPAL_HAVE_SYNC_BUILTIN_CSWAP_INT128 #if OPAL_HAVE_SYNC_BUILTIN_CSWAP_INT128
static inline bool opal_atomic_compare_exchange_strong_128 (volatile opal_int128_t *addr, static inline bool opal_atomic_compare_exchange_strong_128 (opal_atomic_opal_int128_t *addr,
opal_int128_t *oldval, opal_int128_t newval) opal_int128_t *oldval, opal_int128_t newval)
{ {
opal_int128_t prev = __sync_val_compare_and_swap (addr, *oldval, newval); opal_int128_t prev = __sync_val_compare_and_swap (addr, *oldval, newval);

22
opal/include/opal/sys/x86_64/atomic.h
Просмотреть файл

@ -11,7 +11,7 @@
* Copyright (c) 2004-2005 The Regents of the University of California. * Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved. * All rights reserved.
* Copyright (c) 2007 Sun Microsystems, Inc. All rights reserverd. * Copyright (c) 2007 Sun Microsystems, Inc. All rights reserverd.
* Copyright (c) 2012-2017 Los Alamos National Security, LLC. All rights * Copyright (c) 2012-2018 Los Alamos National Security, LLC. All rights
* reserved. * reserved.
* Copyright (c) 2016-2017 Research Organization for Information Science * Copyright (c) 2016-2017 Research Organization for Information Science
* and Technology (RIST). All rights reserved. * and Technology (RIST). All rights reserved.
@ -82,7 +82,7 @@ static inline void opal_atomic_isync(void)
*********************************************************************/ *********************************************************************/
#if OPAL_GCC_INLINE_ASSEMBLY #if OPAL_GCC_INLINE_ASSEMBLY
static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) static inline bool opal_atomic_compare_exchange_strong_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
{ {
unsigned char ret; unsigned char ret;
__asm__ __volatile__ ( __asm__ __volatile__ (
@ -102,13 +102,13 @@ static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *add
#if OPAL_GCC_INLINE_ASSEMBLY #if OPAL_GCC_INLINE_ASSEMBLY
static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) static inline bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
{ {
unsigned char ret; unsigned char ret;
__asm__ __volatile__ ( __asm__ __volatile__ (
SMPLOCK "cmpxchgq %3,%2 \n\t" SMPLOCK "cmpxchgq %3,%2 \n\t"
"sete %0 \n\t" "sete %0 \n\t"
: "=qm" (ret), "+a" (*oldval), "+m" (*((volatile long*)addr)) : "=qm" (ret), "+a" (*oldval), "+m" (*((opal_atomic_long_t *)addr))
: "q"(newval) : "q"(newval)
: "memory", "cc" : "memory", "cc"
); );
@ -123,7 +123,7 @@ static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *add
#if OPAL_GCC_INLINE_ASSEMBLY && OPAL_HAVE_CMPXCHG16B && HAVE_OPAL_INT128_T #if OPAL_GCC_INLINE_ASSEMBLY && OPAL_HAVE_CMPXCHG16B && HAVE_OPAL_INT128_T
static inline bool opal_atomic_compare_exchange_strong_128 (volatile opal_int128_t *addr, opal_int128_t *oldval, opal_int128_t newval) static inline bool opal_atomic_compare_exchange_strong_128 (opal_atomic_int128_t *addr, opal_int128_t *oldval, opal_int128_t newval)
{ {
unsigned char ret; unsigned char ret;
@ -150,7 +150,7 @@ static inline bool opal_atomic_compare_exchange_strong_128 (volatile opal_int128
#define OPAL_HAVE_ATOMIC_SWAP_64 1 #define OPAL_HAVE_ATOMIC_SWAP_64 1
static inline int32_t opal_atomic_swap_32( volatile int32_t *addr, static inline int32_t opal_atomic_swap_32( opal_atomic_int32_t *addr,
int32_t newval) int32_t newval)
{ {
int32_t oldval; int32_t oldval;
@ -166,7 +166,7 @@ static inline int32_t opal_atomic_swap_32( volatile int32_t *addr,
#if OPAL_GCC_INLINE_ASSEMBLY #if OPAL_GCC_INLINE_ASSEMBLY
static inline int64_t opal_atomic_swap_64( volatile int64_t *addr, static inline int64_t opal_atomic_swap_64( opal_atomic_int64_t *addr,
int64_t newval) int64_t newval)
{ {
int64_t oldval; int64_t oldval;
@ -196,7 +196,7 @@ static inline int64_t opal_atomic_swap_64( volatile int64_t *addr,
* *
* Atomically adds @i to @v. * Atomically adds @i to @v.
*/ */
static inline int32_t opal_atomic_fetch_add_32(volatile int32_t* v, int i) static inline int32_t opal_atomic_fetch_add_32(opal_atomic_int32_t* v, int i)
{ {
int ret = i; int ret = i;
__asm__ __volatile__( __asm__ __volatile__(
@ -217,7 +217,7 @@ static inline int32_t opal_atomic_fetch_add_32(volatile int32_t* v, int i)
* *
* Atomically adds @i to @v. * Atomically adds @i to @v.
*/ */
static inline int64_t opal_atomic_fetch_add_64(volatile int64_t* v, int64_t i) static inline int64_t opal_atomic_fetch_add_64(opal_atomic_int64_t* v, int64_t i)
{ {
int64_t ret = i; int64_t ret = i;
__asm__ __volatile__( __asm__ __volatile__(
@ -238,7 +238,7 @@ static inline int64_t opal_atomic_fetch_add_64(volatile int64_t* v, int64_t i)
* *
* Atomically subtracts @i from @v. * Atomically subtracts @i from @v.
*/ */
static inline int32_t opal_atomic_fetch_sub_32(volatile int32_t* v, int i) static inline int32_t opal_atomic_fetch_sub_32(opal_atomic_int32_t* v, int i)
{ {
int ret = -i; int ret = -i;
__asm__ __volatile__( __asm__ __volatile__(
@ -259,7 +259,7 @@ static inline int32_t opal_atomic_fetch_sub_32(volatile int32_t* v, int i)
* *
* Atomically subtracts @i from @v. * Atomically subtracts @i from @v.
*/ */
static inline int64_t opal_atomic_fetch_sub_64(volatile int64_t* v, int64_t i) static inline int64_t opal_atomic_fetch_sub_64(opal_atomic_int64_t* v, int64_t i)
{ {
int64_t ret = -i; int64_t ret = -i;
__asm__ __volatile__( __asm__ __volatile__(

35
opal/include/opal_stdatomic.h Обычный файл
Просмотреть файл

@ -0,0 +1,35 @@
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/*
* Copyright (c) 2018 Los Alamos National Security, LLC. All rights
* reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#if !defined(OPAL_STDATOMIC_H)
#define OPAL_STDATOMIC_H
#include "opal_stdint.h"
typedef volatile long opal_atomic_long_t;
typedef volatile int32_t opal_atomic_int32_t;
typedef volatile uint32_t opal_atomic_uint32_t;
typedef volatile int64_t opal_atomic_int64_t;
typedef volatile uint64_t opal_atomic_uint64_t;
typedef volatile size_t opal_atomic_size_t;
typedef volatile intptr_t opal_atomic_intptr_t;
typedef volatile uintptr_t opal_atomic_uintptr_t;
#if HAVE_OPAL_INT128_T
typedef volatile opal_int128_t opal_atomic_int128_t;
#endif
#endif /* !defined(OPAL_STDATOMIC_H) */

4
opal/mca/btl/ofi/btl_ofi.h
Просмотреть файл

@ -123,8 +123,8 @@ struct mca_btl_ofi_module_t {
bool use_virt_addr; bool use_virt_addr;
bool is_scalable_ep; bool is_scalable_ep;
int64_t outstanding_rdma; opal_atomic_int64_t outstanding_rdma;
int64_t outstanding_send; opal_atomic_int64_t outstanding_send;
/** linked list of BTL endpoints. this list is never searched so /** linked list of BTL endpoints. this list is never searched so
* there is no need for a complicated structure here at this time*/ * there is no need for a complicated structure here at this time*/

14
opal/mca/btl/openib/btl_openib.h
Просмотреть файл

@ -237,7 +237,7 @@ struct mca_btl_openib_component_t {
int apm_lmc; int apm_lmc;
int apm_ports; int apm_ports;
unsigned int buffer_alignment; /**< Preferred communication buffer alignment in Bytes (must be power of two) */ unsigned int buffer_alignment; /**< Preferred communication buffer alignment in Bytes (must be power of two) */
int32_t error_counter; /**< Counts number on error events that we got on all devices */ opal_atomic_int32_t error_counter; /**< Counts number on error events that we got on all devices */
opal_event_base_t *async_evbase; /**< Async event base */ opal_event_base_t *async_evbase; /**< Async event base */
bool use_async_event_thread; /**< Use the async event handler */ bool use_async_event_thread; /**< Use the async event handler */
mca_btl_openib_srq_manager_t srq_manager; /**< Hash table for all BTL SRQs */ mca_btl_openib_srq_manager_t srq_manager; /**< Hash table for all BTL SRQs */
@ -404,8 +404,8 @@ typedef struct mca_btl_openib_device_t {
#endif #endif
int xrc_fd; int xrc_fd;
#endif #endif
int32_t non_eager_rdma_endpoints; opal_atomic_int32_t non_eager_rdma_endpoints;
int32_t eager_rdma_buffers_count; opal_atomic_int32_t eager_rdma_buffers_count;
struct mca_btl_base_endpoint_t **eager_rdma_buffers; struct mca_btl_base_endpoint_t **eager_rdma_buffers;
/**< frags for control massages */ /**< frags for control massages */
opal_free_list_t send_free_control; opal_free_list_t send_free_control;
@ -428,8 +428,8 @@ struct mca_btl_openib_module_pp_qp_t {
struct mca_btl_openib_module_srq_qp_t { struct mca_btl_openib_module_srq_qp_t {
struct ibv_srq *srq; struct ibv_srq *srq;
int32_t rd_posted; opal_atomic_int32_t rd_posted;
int32_t sd_credits; /* the max number of outstanding sends on a QP when using SRQ */ opal_atomic_int32_t sd_credits; /* the max number of outstanding sends on a QP when using SRQ */
/* i.e. the number of frags that can be outstanding (down counter) */ /* i.e. the number of frags that can be outstanding (down counter) */
opal_list_t pending_frags[2]; /**< list of high/low prio frags */ opal_list_t pending_frags[2]; /**< list of high/low prio frags */
/** The number of receive buffers that can be post in the current time. /** The number of receive buffers that can be post in the current time.
@ -487,12 +487,12 @@ struct mca_btl_openib_module_t {
int apm_port; /**< Alternative port that may be used for APM */ int apm_port; /**< Alternative port that may be used for APM */
uint8_t src_path_bits; /**< offset from base lid (for LMC) */ uint8_t src_path_bits; /**< offset from base lid (for LMC) */
int32_t num_peers; opal_atomic_int32_t num_peers;
opal_mutex_t ib_lock; /**< module level lock */ opal_mutex_t ib_lock; /**< module level lock */
size_t eager_rdma_frag_size; /**< length of eager frag */ size_t eager_rdma_frag_size; /**< length of eager frag */
volatile int32_t eager_rdma_channels; /**< number of open RDMA channels */ opal_atomic_int32_t eager_rdma_channels; /**< number of open RDMA channels */
mca_btl_base_module_error_cb_fn_t error_cb; /**< error handler */ mca_btl_base_module_error_cb_fn_t error_cb; /**< error handler */

4
opal/mca/btl/openib/btl_openib_async.c
Просмотреть файл

@ -4,7 +4,7 @@
* Copyright (c) 2007-2013 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2007-2013 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2006-2007 Voltaire All rights reserved. * Copyright (c) 2006-2007 Voltaire All rights reserved.
* Copyright (c) 2009-2010 Oracle and/or its affiliates. All rights reserved * Copyright (c) 2009-2010 Oracle and/or its affiliates. All rights reserved
* Copyright (c) 2013-2015 Los Alamos National Security, LLC. All rights * Copyright (c) 2013-2018 Los Alamos National Security, LLC. All rights
* reserved. * reserved.
* Copyright (c) 2014 Intel, Inc. All rights reserved. * Copyright (c) 2014 Intel, Inc. All rights reserved.
* Copyright (c) 2014 Bull SAS. All rights reserved. * Copyright (c) 2014 Bull SAS. All rights reserved.
@ -38,7 +38,7 @@
static opal_list_t ignore_qp_err_list; static opal_list_t ignore_qp_err_list;
static opal_mutex_t ignore_qp_err_list_lock; static opal_mutex_t ignore_qp_err_list_lock;
static int32_t btl_openib_async_device_count = 0; static opal_atomic_int32_t btl_openib_async_device_count = 0;
struct mca_btl_openib_async_poll { struct mca_btl_openib_async_poll {
int active_poll_size; int active_poll_size;

10
opal/mca/btl/openib/btl_openib_eager_rdma.h
Просмотреть файл

@ -1,7 +1,7 @@
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/* /*
* Copyright (c) 2006-2007 Voltaire All rights reserved. * Copyright (c) 2006-2007 Voltaire All rights reserved.
* Copyright (c) 2015 Los Alamos National Security, LLC. All rights * Copyright (c) 2015-2018 Los Alamos National Security, LLC. All rights
* reserved. * reserved.
* $COPYRIGHT$ * $COPYRIGHT$
* *
@ -25,7 +25,7 @@ struct mca_btl_openib_eager_rdma_local_t {
mca_btl_openib_reg_t *reg; mca_btl_openib_reg_t *reg;
uint16_t head; /**< RDMA buffer to poll */ uint16_t head; /**< RDMA buffer to poll */
uint16_t tail; /**< Needed for credit managment */ uint16_t tail; /**< Needed for credit managment */
int32_t credits; /**< number of RDMA credits */ opal_atomic_int32_t credits; /**< number of RDMA credits */
int32_t rd_win; int32_t rd_win;
#if OPAL_ENABLE_DEBUG #if OPAL_ENABLE_DEBUG
uint32_t seq; uint32_t seq;
@ -38,8 +38,8 @@ typedef struct mca_btl_openib_eager_rdma_local_t mca_btl_openib_eager_rdma_local
struct mca_btl_openib_eager_rdma_remote_t { struct mca_btl_openib_eager_rdma_remote_t {
opal_ptr_t base; /**< address of remote buffer */ opal_ptr_t base; /**< address of remote buffer */
uint32_t rkey; /**< RKey for accessing remote buffer */ uint32_t rkey; /**< RKey for accessing remote buffer */
int32_t head; /**< RDMA buffer to post to */ opal_atomic_int32_t head; /**< RDMA buffer to post to */
int32_t tokens; /**< number of rdam tokens */ opal_atomic_int32_t tokens; /**< number of rdma tokens */
#if OPAL_ENABLE_DEBUG #if OPAL_ENABLE_DEBUG
uint32_t seq; uint32_t seq;
#endif #endif
@ -108,7 +108,7 @@ typedef struct mca_btl_openib_eager_rdma_remote_t mca_btl_openib_eager_rdma_remo
#define MCA_BTL_OPENIB_RDMA_MOVE_INDEX(HEAD, OLD_HEAD) \ #define MCA_BTL_OPENIB_RDMA_MOVE_INDEX(HEAD, OLD_HEAD) \
do { \ do { \
(OLD_HEAD) = (OPAL_THREAD_ADD_FETCH32(&(HEAD), 1) - 1) % mca_btl_openib_component.eager_rdma_num; \ (OLD_HEAD) = (OPAL_THREAD_ADD_FETCH32((opal_atomic_int32_t *) &(HEAD), 1) - 1) % mca_btl_openib_component.eager_rdma_num; \
} while(0) } while(0)
#endif #endif

9
opal/mca/btl/openib/btl_openib_endpoint.c
Просмотреть файл

@ -11,7 +11,7 @@
* Copyright (c) 2004-2005 The Regents of the University of California. * Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved. * All rights reserved.
* Copyright (c) 2006-2013 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2006-2013 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2006-2017 Los Alamos National Security, LLC. All rights * Copyright (c) 2006-2018 Los Alamos National Security, LLC. All rights
* reserved. * reserved.
* Copyright (c) 2006-2007 Voltaire All rights reserved. * Copyright (c) 2006-2007 Voltaire All rights reserved.
* Copyright (c) 2006-2009 Mellanox Technologies, Inc. All rights reserved. * Copyright (c) 2006-2009 Mellanox Technologies, Inc. All rights reserved.
@ -378,7 +378,7 @@ static void mca_btl_openib_endpoint_destruct(mca_btl_base_endpoint_t* endpoint)
* was not in "connect" or "bad" flow (failed to allocate memory) * was not in "connect" or "bad" flow (failed to allocate memory)
* and changed the pointer back to NULL * and changed the pointer back to NULL
*/ */
if(!opal_atomic_compare_exchange_strong_ptr(&endpoint->eager_rdma_local.base.pval, (void *) &_tmp_ptr, (void *) 1)) { if(!opal_atomic_compare_exchange_strong_ptr((opal_atomic_intptr_t *) &endpoint->eager_rdma_local.base.pval, (intptr_t *) &_tmp_ptr, 1)) {
if (NULL != endpoint->eager_rdma_local.reg) { if (NULL != endpoint->eager_rdma_local.reg) {
endpoint->endpoint_btl->device->rcache->rcache_deregister (endpoint->endpoint_btl->device->rcache, endpoint->endpoint_btl->device->rcache->rcache_deregister (endpoint->endpoint_btl->device->rcache,
&endpoint->eager_rdma_local.reg->base); &endpoint->eager_rdma_local.reg->base);
@ -899,8 +899,7 @@ void mca_btl_openib_endpoint_connect_eager_rdma(
/* Set local rdma pointer to 1 temporarily so other threads will not try /* Set local rdma pointer to 1 temporarily so other threads will not try
* to enter the function */ * to enter the function */
if(!opal_atomic_compare_exchange_strong_ptr (&endpoint->eager_rdma_local.base.pval, (void *) &_tmp_ptr, if(!opal_atomic_compare_exchange_strong_ptr ((opal_atomic_intptr_t *) &endpoint->eager_rdma_local.base.pval, (intptr_t *) &_tmp_ptr, 1)) {
(void *) 1)) {
return; return;
} }
@ -990,7 +989,7 @@ void mca_btl_openib_endpoint_connect_eager_rdma(
do { do {
_tmp_ptr = NULL; _tmp_ptr = NULL;
p = &device->eager_rdma_buffers[device->eager_rdma_buffers_count]; p = &device->eager_rdma_buffers[device->eager_rdma_buffers_count];
} while(!opal_atomic_compare_exchange_strong_ptr (p, (void *) &_tmp_ptr, endpoint)); } while(!opal_atomic_compare_exchange_strong_ptr ((opal_atomic_intptr_t *) p, (intptr_t *) &_tmp_ptr, (intptr_t) endpoint));
OPAL_THREAD_ADD_FETCH32(&openib_btl->eager_rdma_channels, 1); OPAL_THREAD_ADD_FETCH32(&openib_btl->eager_rdma_channels, 1);
/* from this point progress function starts to poll new buffer */ /* from this point progress function starts to poll new buffer */

24
opal/mca/btl/openib/btl_openib_endpoint.h
Просмотреть файл

@ -11,7 +11,7 @@
* Copyright (c) 2004-2005 The Regents of the University of California. * Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved. * All rights reserved.
* Copyright (c) 2007-2009 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2007-2009 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2006-2017 Los Alamos National Security, LLC. All rights * Copyright (c) 2006-2018 Los Alamos National Security, LLC. All rights
* reserved. * reserved.
* Copyright (c) 2006-2007 Voltaire All rights reserved. * Copyright (c) 2006-2007 Voltaire All rights reserved.
* Copyright (c) 2007-2009 Mellanox Technologies. All rights reserved. * Copyright (c) 2007-2009 Mellanox Technologies. All rights reserved.
@ -117,17 +117,17 @@ typedef struct mca_btl_openib_rem_info_t {
* Agggregates all per peer qp info for an endpoint * Agggregates all per peer qp info for an endpoint
*/ */
typedef struct mca_btl_openib_endpoint_pp_qp_t { typedef struct mca_btl_openib_endpoint_pp_qp_t {
int32_t sd_credits; /**< this rank's view of the credits opal_atomic_int32_t sd_credits; /**< this rank's view of the credits
* available for sending: * available for sending:
* this is the credits granted by the * this is the credits granted by the
* remote peer which has some relation to the * remote peer which has some relation to the
* number of receive buffers posted remotely * number of receive buffers posted remotely
*/ */
int32_t rd_posted; /**< number of descriptors posted to the nic*/ opal_atomic_int32_t rd_posted; /**< number of descriptors posted to the nic*/
int32_t rd_credits; /**< number of credits to return to peer */ opal_atomic_int32_t rd_credits; /**< number of credits to return to peer */
int32_t cm_received; /**< Credit messages received */ opal_atomic_int32_t cm_received; /**< Credit messages received */
int32_t cm_return; /**< how may credits to return */ opal_atomic_int32_t cm_return; /**< how may credits to return */
int32_t cm_sent; /**< Outstanding number of credit messages */ opal_atomic_int32_t cm_sent; /**< Outstanding number of credit messages */
} mca_btl_openib_endpoint_pp_qp_t; } mca_btl_openib_endpoint_pp_qp_t;
@ -141,8 +141,8 @@ typedef struct mca_btl_openib_endpoint_srq_qp_t {
typedef struct mca_btl_openib_qp_t { typedef struct mca_btl_openib_qp_t {
struct ibv_qp *lcl_qp; struct ibv_qp *lcl_qp;
uint32_t lcl_psn; uint32_t lcl_psn;
volatile int32_t sd_wqe; /**< number of available send wqe entries */ opal_atomic_int32_t sd_wqe; /**< number of available send wqe entries */
int32_t sd_wqe_inflight; opal_atomic_int32_t sd_wqe_inflight;
int wqe_count; int wqe_count;
int users; int users;
opal_mutex_t lock; opal_mutex_t lock;
@ -154,7 +154,7 @@ typedef struct mca_btl_openib_endpoint_qp_t {
available */ available */
opal_list_t no_wqe_pending_frags[2]; /**< put fragments here if there is no wqe opal_list_t no_wqe_pending_frags[2]; /**< put fragments here if there is no wqe
available */ available */
int32_t rd_credit_send_lock; /**< Lock credit send fragment */ opal_atomic_int32_t rd_credit_send_lock; /**< Lock credit send fragment */
mca_btl_openib_send_control_frag_t *credit_frag; mca_btl_openib_send_control_frag_t *credit_frag;
size_t ib_inline_max; /**< max size of inline send*/ size_t ib_inline_max; /**< max size of inline send*/
union { union {
@ -226,7 +226,7 @@ struct mca_btl_base_endpoint_t {
opal_list_t pending_put_frags; opal_list_t pending_put_frags;
/** number of available get tokens */ /** number of available get tokens */
int32_t get_tokens; opal_atomic_int32_t get_tokens;
/** subnet id of this endpoint*/ /** subnet id of this endpoint*/
uint64_t subnet_id; uint64_t subnet_id;
@ -235,7 +235,7 @@ struct mca_btl_base_endpoint_t {
struct ib_address_t *ib_addr; struct ib_address_t *ib_addr;
/** number of eager received */ /** number of eager received */
int32_t eager_recv_count; opal_atomic_int32_t eager_recv_count;
/** info about remote RDMA buffer */ /** info about remote RDMA buffer */
mca_btl_openib_eager_rdma_remote_t eager_rdma_remote; mca_btl_openib_eager_rdma_remote_t eager_rdma_remote;
/** info about local RDMA buffer */ /** info about local RDMA buffer */

6
opal/mca/btl/portals4/btl_portals4.h
Просмотреть файл

@ -92,7 +92,7 @@ struct mca_btl_portals4_module_t {
/* number of processes we're actively connected to. Needed to /* number of processes we're actively connected to. Needed to
know when to do activation / shutdown */ know when to do activation / shutdown */
int32_t portals_num_procs; opal_atomic_int32_t portals_num_procs;
/* number of the interface (btl) */ /* number of the interface (btl) */
uint32_t interface_num; uint32_t interface_num;
@ -111,11 +111,11 @@ struct mca_btl_portals4_module_t {
ptl_handle_eq_t recv_eq_h; ptl_handle_eq_t recv_eq_h;
/* number outstanding sends and local rdma */ /* number outstanding sends and local rdma */
volatile int32_t portals_outstanding_ops; opal_atomic_int32_t portals_outstanding_ops;
int32_t portals_max_outstanding_ops; int32_t portals_max_outstanding_ops;
/* key to use for next rdma operation */ /* key to use for next rdma operation */
volatile int64_t portals_rdma_key; opal_atomic_int64_t portals_rdma_key;
/* our portals network interface */ /* our portals network interface */
ptl_handle_ni_t portals_ni_h; ptl_handle_ni_t portals_ni_h;

4
opal/mca/btl/smcuda/btl_smcuda.h
Просмотреть файл

@ -159,8 +159,8 @@ struct mca_btl_smcuda_component_t {
struct mca_btl_base_endpoint_t **sm_peers; struct mca_btl_base_endpoint_t **sm_peers;
opal_free_list_t pending_send_fl; opal_free_list_t pending_send_fl;
int num_outstanding_frags; /**< number of fragments sent but not yet returned to free list */ opal_atomic_int32_t num_outstanding_frags; /**< number of fragments sent but not yet returned to free list */
int num_pending_sends; /**< total number on all of my pending-send queues */ opal_atomic_int32_t num_pending_sends; /**< total number on all of my pending-send queues */
int mem_node; int mem_node;
int num_mem_nodes; int num_mem_nodes;

6
opal/mca/btl/uct/btl_uct_device_context.h
Просмотреть файл

@ -59,7 +59,7 @@ static inline void mca_btl_uct_context_unlock (mca_btl_uct_device_context_t *con
static inline int mca_btl_uct_get_context_index (void) static inline int mca_btl_uct_get_context_index (void)
{ {
static volatile uint32_t next_uct_index = 0; static opal_atomic_uint32_t next_uct_index = 0;
int context_id; int context_id;
#if OPAL_C_HAVE__THREAD_LOCAL #if OPAL_C_HAVE__THREAD_LOCAL
@ -68,7 +68,7 @@ static inline int mca_btl_uct_get_context_index (void)
context_id = uct_index; context_id = uct_index;
if (OPAL_UNLIKELY(-1 == context_id)) { if (OPAL_UNLIKELY(-1 == context_id)) {
context_id = uct_index = opal_atomic_fetch_add_32 ((volatile int32_t *) &next_uct_index, 1) % context_id = uct_index = opal_atomic_fetch_add_32 ((opal_atomic_int32_t *) &next_uct_index, 1) %
mca_btl_uct_component.num_contexts_per_module; mca_btl_uct_component.num_contexts_per_module;
} }
} else { } else {
@ -92,7 +92,7 @@ mca_btl_uct_module_get_tl_context_specific (mca_btl_uct_module_t *module, mca_bt
mca_btl_uct_device_context_t *new_context; mca_btl_uct_device_context_t *new_context;
new_context = mca_btl_uct_context_create (module, tl, context_id); new_context = mca_btl_uct_context_create (module, tl, context_id);
if (!opal_atomic_compare_exchange_strong_ptr (&tl->uct_dev_contexts[context_id], &context, new_context)) { if (!opal_atomic_compare_exchange_strong_ptr ((opal_atomic_intptr_t *) &tl->uct_dev_contexts[context_id], &context, new_context)) {
mca_btl_uct_context_destroy (new_context); mca_btl_uct_context_destroy (new_context);
} else { } else {
context = new_context; context = new_context;

2
opal/mca/btl/uct/btl_uct_types.h
Просмотреть файл

@ -94,7 +94,7 @@ typedef struct mca_btl_uct_conn_req_t mca_btl_uct_conn_req_t;
*/ */
struct mca_btl_uct_tl_endpoint_t { struct mca_btl_uct_tl_endpoint_t {
/** current flags (connected, requested, etc) */ /** current flags (connected, requested, etc) */
volatile int32_t flags; opal_atomic_int32_t flags;
/** UCT endpoint handle */ /** UCT endpoint handle */
uct_ep_h uct_ep; uct_ep_h uct_ep;

8
opal/mca/btl/ugni/btl_ugni.h
Просмотреть файл

@ -114,7 +114,7 @@ struct mca_btl_ugni_device_t {
int dev_index; int dev_index;
/** number of SMSG connections */ /** number of SMSG connections */
volatile int32_t smsg_connections; opal_atomic_int32_t smsg_connections;
/** boolean indicating that the device was recently flushed */ /** boolean indicating that the device was recently flushed */
volatile bool flushed; volatile bool flushed;
@ -168,7 +168,7 @@ typedef struct mca_btl_ugni_module_t {
gni_ep_handle_t wildcard_ep; gni_ep_handle_t wildcard_ep;
struct mca_btl_base_endpoint_t *local_ep; struct mca_btl_base_endpoint_t *local_ep;
volatile int32_t active_datagrams; opal_atomic_int32_t active_datagrams;
opal_event_t connection_event; opal_event_t connection_event;
struct mca_btl_ugni_endpoint_attr_t wc_remote_attr, wc_local_attr; struct mca_btl_ugni_endpoint_attr_t wc_remote_attr, wc_local_attr;
@ -188,13 +188,13 @@ typedef struct mca_btl_ugni_module_t {
opal_pointer_array_t pending_smsg_frags_bb; opal_pointer_array_t pending_smsg_frags_bb;
int32_t reg_max; int32_t reg_max;
volatile int32_t reg_count; opal_atomic_int32_t reg_count;
/* used to calculate the fraction of registered memory resources /* used to calculate the fraction of registered memory resources
* this rank should be limited too */ * this rank should be limited too */
int nlocal_procs; int nlocal_procs;
volatile int32_t active_rdma_count; opal_atomic_int32_t active_rdma_count;
mca_rcache_base_module_t *rcache; mca_rcache_base_module_t *rcache;
} mca_btl_ugni_module_t; } mca_btl_ugni_module_t;

2
opal/mca/btl/ugni/btl_ugni_endpoint.h
Просмотреть файл

@ -70,7 +70,7 @@ typedef struct mca_btl_base_endpoint_t {
bool dg_posted; bool dg_posted;
/** protect against re-entry to SMSG */ /** protect against re-entry to SMSG */
int32_t smsg_progressing; opal_atomic_int32_t smsg_progressing;
int index; int index;
} mca_btl_base_endpoint_t; } mca_btl_base_endpoint_t;

2
opal/mca/btl/ugni/btl_ugni_frag.h
Просмотреть файл

@ -68,7 +68,7 @@ struct mca_btl_ugni_base_frag_t;
typedef struct mca_btl_ugni_base_frag_t { typedef struct mca_btl_ugni_base_frag_t {
mca_btl_base_descriptor_t base; mca_btl_base_descriptor_t base;
volatile int32_t ref_cnt; opal_atomic_int32_t ref_cnt;
uint32_t msg_id; uint32_t msg_id;
uint16_t hdr_size; uint16_t hdr_size;
uint16_t flags; uint16_t flags;

2
opal/mca/btl/vader/btl_vader_component.c
Просмотреть файл

@ -715,7 +715,7 @@ static void mca_btl_vader_progress_endpoints (void)
static int mca_btl_vader_component_progress (void) static int mca_btl_vader_component_progress (void)
{ {
static int32_t lock = 0; static opal_atomic_int32_t lock = 0;
int count = 0; int count = 0;
if (opal_using_threads()) { if (opal_using_threads()) {

2
opal/mca/btl/vader/btl_vader_endpoint.h
Просмотреть файл

@ -62,7 +62,7 @@ typedef struct mca_btl_base_endpoint_t {
int32_t peer_smp_rank; /**< my peer's SMP process rank. Used for accessing int32_t peer_smp_rank; /**< my peer's SMP process rank. Used for accessing
* SMP specfic data structures. */ * SMP specfic data structures. */
volatile size_t send_count; /**< number of fragments sent to this peer */ opal_atomic_size_t send_count; /**< number of fragments sent to this peer */
char *segment_base; /**< start of the peer's segment (in the address space char *segment_base; /**< start of the peer's segment (in the address space
* of this process) */ * of this process) */

19
opal/mca/btl/vader/btl_vader_fifo.h
Просмотреть файл

@ -30,26 +30,25 @@
#include "btl_vader_endpoint.h" #include "btl_vader_endpoint.h"
#include "btl_vader_frag.h" #include "btl_vader_frag.h"
#define vader_item_compare_exchange(x, y, z) opal_atomic_compare_exchange_strong_ptr ((volatile void **) (x), (void **) (y), (void *) (z)) #define vader_item_compare_exchange(x, y, z) opal_atomic_compare_exchange_strong_ptr ((opal_atomic_intptr_t *) (x), (intptr_t *) (y), (intptr_t) (z))
#if SIZEOF_VOID_P == 8 #if SIZEOF_VOID_P == 8
#define vader_item_swap(x, y) opal_atomic_swap_64((volatile int64_t *)(x), (int64_t)(y)) #define vader_item_swap(x, y) opal_atomic_swap_64((opal_atomic_int64_t *)(x), (int64_t)(y))
#define MCA_BTL_VADER_OFFSET_MASK 0xffffffffll #define MCA_BTL_VADER_OFFSET_MASK 0xffffffffll
#define MCA_BTL_VADER_OFFSET_BITS 32 #define MCA_BTL_VADER_OFFSET_BITS 32
#define MCA_BTL_VADER_BITNESS 64 #define MCA_BTL_VADER_BITNESS 64
typedef int64_t fifo_value_t;
#else #else
#define vader_item_swap(x, y) opal_atomic_swap_32((volatile int32_t *)(x), (int32_t)(y)) #define vader_item_swap(x, y) opal_atomic_swap_32((opal_atomic_int32_t *)(x), (int32_t)(y))
#define MCA_BTL_VADER_OFFSET_MASK 0x00ffffffl #define MCA_BTL_VADER_OFFSET_MASK 0x00ffffffl
#define MCA_BTL_VADER_OFFSET_BITS 24 #define MCA_BTL_VADER_OFFSET_BITS 24
#define MCA_BTL_VADER_BITNESS 32 #define MCA_BTL_VADER_BITNESS 32
typedef int32_t fifo_value_t;
#endif #endif
typedef opal_atomic_intptr_t atomic_fifo_value_t;
typedef intptr_t fifo_value_t;
#define VADER_FIFO_FREE ((fifo_value_t)-2) #define VADER_FIFO_FREE ((fifo_value_t)-2)
/* /*
@ -68,9 +67,9 @@
/* lock free fifo */ /* lock free fifo */
typedef struct vader_fifo_t { typedef struct vader_fifo_t {
volatile fifo_value_t fifo_head; atomic_fifo_value_t fifo_head;
volatile fifo_value_t fifo_tail; atomic_fifo_value_t fifo_tail;
volatile int32_t fbox_available; opal_atomic_int32_t fbox_available;
} vader_fifo_t; } vader_fifo_t;
/* large enough to ensure the fifo is on its own cache line */ /* large enough to ensure the fifo is on its own cache line */

8
opal/mca/btl/vader/btl_vader_sc_emu.c
Просмотреть файл

@ -13,7 +13,7 @@
#include "btl_vader_frag.h" #include "btl_vader_frag.h"
#if OPAL_HAVE_ATOMIC_MATH_64 #if OPAL_HAVE_ATOMIC_MATH_64
static void mca_btl_vader_sc_emu_atomic_64 (int64_t *operand, volatile int64_t *addr, mca_btl_base_atomic_op_t op) static void mca_btl_vader_sc_emu_atomic_64 (int64_t *operand, opal_atomic_int64_t *addr, mca_btl_base_atomic_op_t op)
{ {
int64_t result = 0; int64_t result = 0;
@ -52,7 +52,7 @@ static void mca_btl_vader_sc_emu_atomic_64 (int64_t *operand, volatile int64_t *
#endif #endif
#if OPAL_HAVE_ATOMIC_MATH_32 #if OPAL_HAVE_ATOMIC_MATH_32
static void mca_btl_vader_sc_emu_atomic_32 (int32_t *operand, volatile int32_t *addr, mca_btl_base_atomic_op_t op) static void mca_btl_vader_sc_emu_atomic_32 (int32_t *operand, opal_atomic_int32_t *addr, mca_btl_base_atomic_op_t op)
{ {
int32_t result = 0; int32_t result = 0;
@ -123,10 +123,10 @@ static void mca_btl_vader_sc_emu_rdma (mca_btl_base_module_t *btl, mca_btl_base_
#if OPAL_HAVE_ATOMIC_MATH_64 #if OPAL_HAVE_ATOMIC_MATH_64
case MCA_BTL_VADER_OP_CSWAP: case MCA_BTL_VADER_OP_CSWAP:
if (!(hdr->flags & MCA_BTL_ATOMIC_FLAG_32BIT)) { if (!(hdr->flags & MCA_BTL_ATOMIC_FLAG_32BIT)) {
opal_atomic_compare_exchange_strong_64 ((volatile int64_t *) hdr->addr, &hdr->operand[0], hdr->operand[1]); opal_atomic_compare_exchange_strong_64 ((opal_atomic_int64_t *) hdr->addr, &hdr->operand[0], hdr->operand[1]);
#if OPAL_HAVE_ATOMIC_MATH_32 #if OPAL_HAVE_ATOMIC_MATH_32
} else { } else {
opal_atomic_compare_exchange_strong_32 ((volatile int32_t *) hdr->addr, (int32_t *) &hdr->operand[0], opal_atomic_compare_exchange_strong_32 ((opal_atomic_int32_t *) hdr->addr, (int32_t *) &hdr->operand[0],
(int32_t) hdr->operand[1]); (int32_t) hdr->operand[1]);
#else #else
} else { } else {

7
opal/mca/common/sm/common_sm.h
Просмотреть файл

@ -1,3 +1,4 @@
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/* /*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology * University Research and Technology
@ -10,7 +11,7 @@
* Copyright (c) 2004-2005 The Regents of the University of California. * Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved. * All rights reserved.
* Copyright (c) 2009-2010 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2009-2010 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2010-2012 Los Alamos National Security, LLC. * Copyright (c) 2010-2018 Los Alamos National Security, LLC.
* All rights reserved. * All rights reserved.
* $COPYRIGHT$ * $COPYRIGHT$
* *
@ -42,13 +43,13 @@ typedef struct mca_common_sm_seg_header_t {
/* lock to control atomic access */ /* lock to control atomic access */
opal_atomic_lock_t seg_lock; opal_atomic_lock_t seg_lock;
/* indicates whether or not the segment is ready for use */ /* indicates whether or not the segment is ready for use */
volatile int32_t seg_inited; opal_atomic_int32_t seg_inited;
/* number of local processes that are attached to the shared memory segment. /* number of local processes that are attached to the shared memory segment.
* this is primarily used as a way of determining whether or not it is safe * this is primarily used as a way of determining whether or not it is safe
* to unlink the shared memory backing store. for example, once seg_att * to unlink the shared memory backing store. for example, once seg_att
* is equal to the number of local processes, then we can safely unlink. * is equal to the number of local processes, then we can safely unlink.
*/ */
volatile size_t seg_num_procs_inited; opal_atomic_size_t seg_num_procs_inited;
/* offset to next available memory location available for allocation */ /* offset to next available memory location available for allocation */
size_t seg_offset; size_t seg_offset;
/* total size of the segment */ /* total size of the segment */

4
opal/mca/mpool/hugepage/mpool_hugepage.h
Просмотреть файл

@ -46,7 +46,7 @@ struct mca_mpool_hugepage_component_t {
opal_list_t huge_pages; opal_list_t huge_pages;
mca_mpool_hugepage_module_t *modules; mca_mpool_hugepage_module_t *modules;
int module_count; int module_count;
unsigned long bytes_allocated; opal_atomic_size_t bytes_allocated;
}; };
typedef struct mca_mpool_hugepage_component_t mca_mpool_hugepage_component_t; typedef struct mca_mpool_hugepage_component_t mca_mpool_hugepage_component_t;
@ -62,7 +62,7 @@ struct mca_mpool_hugepage_hugepage_t {
/** path for mmapped files */ /** path for mmapped files */
char *path; char *path;
/** counter to help ensure unique file names for mmaped files */ /** counter to help ensure unique file names for mmaped files */
volatile int32_t count; opal_atomic_int32_t count;
/** some platforms allow allocation of hugepages through mmap flags */ /** some platforms allow allocation of hugepages through mmap flags */
int mmap_flags; int mmap_flags;
}; };

2
opal/mca/mpool/hugepage/mpool_hugepage_component.c
Просмотреть файл

@ -135,7 +135,7 @@ static int mca_mpool_hugepage_register(void)
"hugepage component", OPAL_INFO_LVL_3, MCA_BASE_PVAR_CLASS_SIZE, "hugepage component", OPAL_INFO_LVL_3, MCA_BASE_PVAR_CLASS_SIZE,
MCA_BASE_VAR_TYPE_UNSIGNED_LONG, NULL, MCA_BASE_VAR_BIND_NO_OBJECT, MCA_BASE_VAR_TYPE_UNSIGNED_LONG, NULL, MCA_BASE_VAR_BIND_NO_OBJECT,
MCA_BASE_PVAR_FLAG_READONLY | MCA_BASE_PVAR_FLAG_CONTINUOUS, MCA_BASE_PVAR_FLAG_READONLY | MCA_BASE_PVAR_FLAG_CONTINUOUS,
NULL, NULL, NULL, &mca_mpool_hugepage_component.bytes_allocated); NULL, NULL, NULL, (void *) &mca_mpool_hugepage_component.bytes_allocated);
return OPAL_SUCCESS; return OPAL_SUCCESS;
} }

4
opal/mca/mpool/hugepage/mpool_hugepage_module.c
Просмотреть файл

@ -183,7 +183,7 @@ void *mca_mpool_hugepage_seg_alloc (void *ctx, size_t *sizep)
opal_mutex_lock (&hugepage_module->lock); opal_mutex_lock (&hugepage_module->lock);
opal_rb_tree_insert (&hugepage_module->allocation_tree, base, (void *) (intptr_t) size); opal_rb_tree_insert (&hugepage_module->allocation_tree, base, (void *) (intptr_t) size);
opal_atomic_add (&mca_mpool_hugepage_component.bytes_allocated, (int64_t) size); (void) opal_atomic_fetch_add_size_t (&mca_mpool_hugepage_component.bytes_allocated, size);
opal_mutex_unlock (&hugepage_module->lock); opal_mutex_unlock (&hugepage_module->lock);
OPAL_OUTPUT_VERBOSE((MCA_BASE_VERBOSE_TRACE, opal_mpool_base_framework.framework_verbose, OPAL_OUTPUT_VERBOSE((MCA_BASE_VERBOSE_TRACE, opal_mpool_base_framework.framework_verbose,
@ -207,7 +207,7 @@ void mca_mpool_hugepage_seg_free (void *ctx, void *addr)
OPAL_OUTPUT_VERBOSE((MCA_BASE_VERBOSE_TRACE, opal_mpool_base_framework.framework_verbose, OPAL_OUTPUT_VERBOSE((MCA_BASE_VERBOSE_TRACE, opal_mpool_base_framework.framework_verbose,
"freeing segment %p of size %lu bytes", addr, size)); "freeing segment %p of size %lu bytes", addr, size));
munmap (addr, size); munmap (addr, size);
opal_atomic_add (&mca_mpool_hugepage_component.bytes_allocated, -(int64_t) size); (void) opal_atomic_fetch_add_size_t (&mca_mpool_hugepage_component.bytes_allocated, -size);
} }
opal_mutex_unlock (&hugepage_module->lock); opal_mutex_unlock (&hugepage_module->lock);

8
opal/mca/rcache/grdma/rcache_grdma_module.c
Просмотреть файл

@ -215,7 +215,7 @@ static inline void mca_rcache_grdma_add_to_lru (mca_rcache_grdma_module_t *rcach
opal_atomic_wmb (); opal_atomic_wmb ();
/* mark this registration as being in the LRU */ /* mark this registration as being in the LRU */
opal_atomic_fetch_or_32 ((volatile int32_t *) &grdma_reg->flags, MCA_RCACHE_GRDMA_REG_FLAG_IN_LRU); opal_atomic_fetch_or_32 ((opal_atomic_int32_t *) &grdma_reg->flags, MCA_RCACHE_GRDMA_REG_FLAG_IN_LRU);
opal_mutex_unlock (&rcache_grdma->cache->vma_module->vma_lock); opal_mutex_unlock (&rcache_grdma->cache->vma_module->vma_lock);
} }
@ -263,7 +263,7 @@ static int mca_rcache_grdma_check_cached (mca_rcache_base_registration_t *grdma_
} }
/* This segment fits fully within an existing segment. */ /* This segment fits fully within an existing segment. */
(void) opal_atomic_fetch_add_32 ((volatile int32_t *) &rcache_grdma->stat_cache_hit, 1); (void) opal_atomic_fetch_add_32 ((opal_atomic_int32_t *) &rcache_grdma->stat_cache_hit, 1);
OPAL_OUTPUT_VERBOSE((MCA_BASE_VERBOSE_TRACE, opal_rcache_base_framework.framework_output, OPAL_OUTPUT_VERBOSE((MCA_BASE_VERBOSE_TRACE, opal_rcache_base_framework.framework_output,
"returning existing registration %p. references %d", (void *) grdma_reg, ref_cnt)); "returning existing registration %p. references %d", (void *) grdma_reg, ref_cnt));
return 1; return 1;
@ -321,7 +321,7 @@ static int mca_rcache_grdma_register (mca_rcache_base_module_t *rcache, void *ad
/* get updated access flags */ /* get updated access flags */
access_flags = find_args.access_flags; access_flags = find_args.access_flags;
OPAL_THREAD_ADD_FETCH32((volatile int32_t *) &rcache_grdma->stat_cache_miss, 1); OPAL_THREAD_ADD_FETCH32((opal_atomic_int32_t *) &rcache_grdma->stat_cache_miss, 1);
} }
item = opal_free_list_get_mt (&rcache_grdma->reg_list); item = opal_free_list_get_mt (&rcache_grdma->reg_list);
@ -447,7 +447,7 @@ typedef struct gc_add_args_t gc_add_args_t;
static int mca_rcache_grdma_add_to_gc (mca_rcache_base_registration_t *grdma_reg) static int mca_rcache_grdma_add_to_gc (mca_rcache_base_registration_t *grdma_reg)
{ {
mca_rcache_grdma_module_t *rcache_grdma = (mca_rcache_grdma_module_t *) grdma_reg->rcache; mca_rcache_grdma_module_t *rcache_grdma = (mca_rcache_grdma_module_t *) grdma_reg->rcache;
uint32_t flags = opal_atomic_fetch_or_32 ((volatile int32_t *) &grdma_reg->flags, MCA_RCACHE_FLAGS_INVALID); uint32_t flags = opal_atomic_fetch_or_32 ((opal_atomic_int32_t *) &grdma_reg->flags, MCA_RCACHE_FLAGS_INVALID);
if ((flags & MCA_RCACHE_FLAGS_INVALID) || 0 != grdma_reg->ref_count) { if ((flags & MCA_RCACHE_FLAGS_INVALID) || 0 != grdma_reg->ref_count) {
/* nothing to do */ /* nothing to do */

Некоторые файлы не были показаны из-за слишком большого количества измененных файлов Показать больше