/*
* Copyright (c) 2009-2013 Oak Ridge National Laboratory. All rights reserved.
* Copyright (c) 2009-2012 Mellanox Technologies. All rights reserved.
* Copyright (c) 2013 Los Alamos National Security, LLC. All rights
* reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/* Recursive doubling blocking barrier */
#include "ompi_config.h"
#include "ompi/constants.h"
#include "ompi/communicator/communicator.h"
#include "ompi/mca/bcol/bcol.h"
#include "ompi/patterns/net/netpatterns.h"
#include "opal/sys/atomic.h"
#include "bcol_basesmuma.h"
#if 0
int bcol_basesmuma_recursive_double_barrier(bcol_function_args_t *input_args,
coll_ml_function_t *c_input_args)
{
/* local variables */
int ret=OMPI_SUCCESS, idx, leading_dim, loop_cnt, exchange, flag_to_set;
int pair_rank, flag_offset;
mca_bcol_basesmuma_ctl_struct_t **ctl_structs;
netpatterns_pair_exchange_node_t *my_exchange_node;
int extra_rank, my_rank, pow_2;
volatile mca_bcol_basesmuma_ctl_struct_t *partner_ctl;
volatile mca_bcol_basesmuma_ctl_struct_t *my_ctl;
int64_t sequence_number;
bool found;
int buff_index, first_instance=0;
mca_bcol_basesmuma_module_t* bcol_module =
(mca_bcol_basesmuma_module_t *)c_input_args->bcol_module;
#if 0
fprintf(stderr,"Entering the sm rd barrier\n");
fflush(stderr);
#endif
/* get the pointer to the segment of control structures */
my_exchange_node=&(bcol_module->recursive_doubling_tree);
my_rank=bcol_module->super.sbgp_partner_module->my_index;
pow_2=bcol_module->super.sbgp_partner_module->pow_2;
/* figure out what instance of the basesmuma bcol I am */
leading_dim=bcol_module->colls_no_user_data.size_of_group;
sequence_number=input_args->sequence_num - c_input_args->bcol_module->squence_number_offset;
buff_index=sequence_number & (bcol_module->colls_no_user_data.mask);
idx=SM_ARRAY_INDEX(leading_dim,buff_index,0);
ctl_structs=(mca_bcol_basesmuma_ctl_struct_t **)
bcol_module->colls_no_user_data.ctl_buffs+idx;
my_ctl=ctl_structs[my_rank];
if( my_ctl->sequence_number < sequence_number ) {
first_instance=1;
}
/* get the pool index */
if( first_instance ) {
idx = -1;
while( idx == -1 ) {
idx=bcol_basesmuma_get_buff_index(
&(bcol_module->colls_no_user_data),sequence_number);
}
if( -1 == idx ){
return ORTE_ERR_TEMP_OUT_OF_RESOURCE;
}
my_ctl->index=1;
/* this does not need to use any flag values , so only need to
* set the value for subsequent values that may need this */
my_ctl->starting_flag_value=0;
flag_offset=0;
} else {
/* only one thread at a time will be making progress on this
* collective, so no need to make this atomic */
my_ctl->index++;
flag_offset=my_ctl->starting_flag_value;
}
/* signal that I have arrived */
my_ctl->flag = -1;
/* don't need to set this flag anymore */
my_ctl->sequence_number = sequence_number;
/* MB();*/
if(0 < my_exchange_node->n_extra_sources) {
if (EXCHANGE_NODE == my_exchange_node->node_type) {
volatile int64_t *partner_sn;
int cnt=0;
/* I will participate in the exchange - wait for signal from extra
** process */
extra_rank = my_exchange_node->rank_extra_source;
partner_ctl=(volatile mca_bcol_basesmuma_ctl_struct_t *)ctl_structs[extra_rank];
/*partner_ctl=ctl_structs[extra_rank];*/
partner_sn=(volatile int64_t *)&(partner_ctl->sequence_number);
/* spin n iterations until partner registers */
loop_cnt=0;
found=false;
while( !found )
{
if( *partner_sn >= sequence_number ) {
found=true;
}
cnt++;
if( cnt == 1000 ) {
opal_progress();
cnt=0;
}
}
} else {
/* Nothing to do, already registared that I am here */
}
}
for(exchange = 0; exchange < my_exchange_node->n_exchanges; exchange++) {
volatile int64_t *partner_sn;
volatile int *partner_flag;
int cnt=0;
/* rank of exchange partner */
pair_rank = my_rank ^ ( 1 SHIFT_UP exchange );
partner_ctl=ctl_structs[pair_rank];
partner_sn=(volatile int64_t *)&(partner_ctl->sequence_number);
partner_flag=(volatile int *)&(partner_ctl->flag);
/* signal that I am at iteration exchange of the algorithm */
flag_to_set=flag_offset+exchange;
my_ctl->flag = flag_to_set;
/* check to see if the partner has arrived */
/* spin n iterations until partner registers */
found=false;
while( !found )
{
if( (*partner_sn > sequence_number) ||
( *partner_sn == sequence_number &&
*partner_flag >= flag_to_set ) ) {
found=true;
} else {
cnt++;
if( cnt == 1000 ) {
opal_progress();
cnt=0;
}
}
}
}
if(0 < my_exchange_node->n_extra_sources) {
if ( EXTRA_NODE == my_exchange_node->node_type ) {
int cnt=0;
/* I will not participate in the exchange -
* wait for signal from extra partner */
extra_rank = my_exchange_node->rank_extra_source;
partner_ctl=ctl_structs[extra_rank];
flag_to_set=flag_offset+my_exchange_node->log_2;
/* spin n iterations until partner registers */
found=false;
while( !found )
{
if (IS_PEER_READY(partner_ctl, flag_to_set, sequence_number)){
found=true;
} else {
cnt++;
if( cnt == 1000 ) {
opal_progress();
cnt=0;
}
}
}
} else {
/* signal the extra rank that I am done with the recursive
* doubling phase.
*/
flag_to_set=flag_offset+my_exchange_node->log_2;
my_ctl->flag = flag_to_set;
}
}
/* if I am the last instance of a basesmuma function in this collectie,
* release the resrouces */
if (IS_LAST_BCOL_FUNC(c_input_args)){
idx=bcol_basesmuma_free_buff(
&(bcol_module->colls_no_user_data),
sequence_number);
} else {
/* increment flag value - so next sm collective in the hierarchy
* will not collide with the current one, as they share the
* control structure */
my_ctl->starting_flag_value+=(my_exchange_node->log_2+1);
}
/* return */
return ret;
}
#endif