/* * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** * @file * * Most of the description of the data layout is in the * coll_sm_module.c file. */ #include "ompi_config.h" #include #include #include #include #include "ompi/constants.h" #include "coll_sm2.h" #include "ompi/mca/coll/base/base.h" /* setup an multi-nomial tree - for each node in the tree * this returns it's parent, and it's children */ int setup_multinomial_tree(int tree_order, int num_nodes, tree_node_t *tree_nodes) { /* local variables */ int i,result; int cnt, parent_cnt,n_nodes_in_this_level,node_index; int n_cum_nodes,current_level,node,n_nodes_prev_level,rank,parent_rank; int n_nodes_in_last_level,n_full_stripes,n_in_partial_stipe,n_children; int n_lvls_in_tree; /* sanity check */ if( 1 >= tree_order ) { goto Error; } /* figure out number of levels in the tree */ n_lvls_in_tree=0; result=num_nodes; /* cnt - number of ranks in given level */ cnt=1; /* parent_cnt - cummulative count of ranks */ parent_cnt=0; while( 0 < result ) { result-=cnt; cnt*=tree_order; n_lvls_in_tree++; }; /* loop over tree levels */ n_nodes_in_this_level=1; node_index=-1; n_cum_nodes=0; for( current_level = 0 ; current_level < n_lvls_in_tree ; current_level++) { /* loop over nodes in current level */ for ( node=0 ; node < n_nodes_in_this_level ; node++ ) { /* get node index */ node_index++; /* break if reach group size */ if( node_index == num_nodes) { break; } tree_nodes[node_index].my_rank=node_index; tree_nodes[node_index].children_ranks=NULL; /* * Parents */ if( 0 == current_level ) { tree_nodes[node_index].n_parents=0; /* get parent index */ tree_nodes[node_index].parent_rank=-1; } else { tree_nodes[node_index].n_parents=1; /* get parent index */ n_nodes_prev_level=n_nodes_in_this_level/tree_order; if( current_level == n_lvls_in_tree -1 ) { /* load balance the lowest level */ parent_rank=node- (node/n_nodes_prev_level)*n_nodes_prev_level; parent_rank=n_cum_nodes-n_nodes_prev_level+ parent_rank; tree_nodes[node_index].parent_rank=parent_rank; } else { tree_nodes[node_index].parent_rank= (n_cum_nodes-n_nodes_prev_level)+node/tree_order; } } /* * Children */ /* get number of children */ if( (n_lvls_in_tree-1) == current_level ) { /* leaves have no nodes */ tree_nodes[node_index].n_children=0; tree_nodes[node_index].children_ranks=NULL; } else { /* take into account last level being incomplete */ if( (n_lvls_in_tree-2) == current_level ) { /* last level is load balanced */ n_nodes_in_last_level=num_nodes- (n_cum_nodes+n_nodes_in_this_level); n_full_stripes=n_nodes_in_last_level/n_nodes_in_this_level; n_in_partial_stipe=n_nodes_in_last_level- n_full_stripes*n_nodes_in_this_level; n_children=n_full_stripes; if( n_full_stripes < tree_order ) { if( node <= n_in_partial_stipe-1 ) { n_children++; } } tree_nodes[node_index].n_children=n_children; if( 0 < n_children ) { tree_nodes[node_index].children_ranks=(int *) malloc(sizeof(int)*n_children); if( NULL == tree_nodes[node_index].children_ranks) { goto Error; } } else { tree_nodes[node_index].children_ranks=NULL; } /* fill in list */ for( rank=0 ; rank < n_children ; rank++ ) { tree_nodes[node_index].children_ranks[rank]= node+rank*n_nodes_in_this_level; tree_nodes[node_index].children_ranks[rank]+= (n_cum_nodes+n_nodes_in_this_level); } } else { n_children=tree_order; tree_nodes[node_index].n_children=tree_order; tree_nodes[node_index].children_ranks=(int *) malloc(sizeof(int)*n_children); if( NULL == tree_nodes[node_index].children_ranks) { goto Error; } for( rank=0 ; rank < n_children ; rank++ ) { tree_nodes[node_index].children_ranks[rank]= rank+tree_order*node; tree_nodes[node_index].children_ranks[rank]+= (n_cum_nodes+n_nodes_in_this_level); } } } } /* end node loop */ /* update helper counters */ n_cum_nodes+=n_nodes_in_this_level; n_nodes_in_this_level*=tree_order; } /* set node type */ for(i=0 ; i < num_nodes ; i++ ) { if( 0 == tree_nodes[i].n_parents ) { tree_nodes[i].my_node_type=ROOT_NODE; } else if ( 0 == tree_nodes[i].n_children ) { tree_nodes[i].my_node_type=LEAF_NODE; } else { tree_nodes[i].my_node_type=INTERIOR_NODE; } } /* successful return */ return OMPI_SUCCESS; Error: /* free allocated memory */ for( i=0 ; i < num_nodes ; i++ ) { if( NULL != tree_nodes[i].children_ranks ) { free(tree_nodes[i].children_ranks); } } /* error return */ return OMPI_ERROR; } /* setup recursive doubleing tree node */ int setup_recursive_doubling_tree_node(int num_nodes, int node_rank, pair_exchange_node_t *exchange_node) { /* local variables */ int i,tmp,cnt,result,tree_order,n_extra_nodes; int n_exchanges; /* figure out number of levels in the tree */ n_exchanges=0; result=num_nodes; tree_order=2; /* cnt - number of ranks in given level */ cnt=1; while( num_nodes > cnt ) { cnt*=tree_order; n_exchanges++; }; /* figure out the largest power of 2 that is less than or equal to * num_nodes */ if( cnt > num_nodes) { cnt/=tree_order; n_exchanges--; } exchange_node->log_2=n_exchanges; tmp=1; for(i=0 ; i < n_exchanges ; i++ ) { tmp*=2; } exchange_node->n_largest_pow_2=tmp; /* set node characteristics - node that is not within the largest * power of 2 will just send it's data to node that will participate * in the recursive doubling, and get the result back at the end. */ if( node_rank+1 > cnt ) { exchange_node->node_type=EXTRA_NODE; } else { exchange_node->node_type=EXCHANGE_NODE; } /* set the initial and final data exchanges - those that are not * part of the recursive doubling. */ n_extra_nodes=num_nodes-cnt; if ( EXCHANGE_NODE == exchange_node->node_type ) { if( node_rank < n_extra_nodes ) { exchange_node->n_extra_sources=1; exchange_node->rank_extra_source=cnt+node_rank; } else { exchange_node->n_extra_sources=0; exchange_node->rank_extra_source=-1; } } else { exchange_node->n_extra_sources=1; exchange_node->rank_extra_source=node_rank-cnt; } /* set the exchange pattern */ if( EXCHANGE_NODE == exchange_node->node_type ) { exchange_node->n_exchanges=n_exchanges; exchange_node->rank_exchanges=(int *) malloc (n_exchanges*sizeof(int)); if( NULL == exchange_node->rank_exchanges ) { goto Error; } /* fill in exchange partners */ result=1; tmp=node_rank; for( i=0 ; i < n_exchanges ; i++ ) { if(tmp & 1 ) { exchange_node->rank_exchanges[i]= node_rank-result; } else { exchange_node->rank_exchanges[i]= node_rank+result; } result*=2; tmp/=2; } } else { exchange_node->n_exchanges=0; exchange_node->rank_exchanges=NULL; } /* set the number of tags needed per stripe - this must be the * same across all procs in the communicator. */ exchange_node->n_tags=2*n_exchanges+1; /* successful return */ return OMPI_SUCCESS; Error: /* error return */ return OMPI_ERROR; }