ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

convert the hiearch component to the new structure. More testing required before we remove the .ompi_ignore flag again.

Просмотр исходного кода 
This commit was SVN r15954.
Этот коммит содержится в:
Edgar Gabriel 2007-08-23 20:41:29 +00:00
родитель daaf5a9bf1
Коммит a2f5cada1a
7 изменённых файлов: 265 добавлений и 291 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

307
ompi/mca/coll/hierarch/coll_hierarch.c
Просмотреть файл

@ -9,6 +9,7 @@
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2007 University of Houston. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -48,70 +49,7 @@ static void mca_coll_hierarch_checkfor_component (struct ompi_communicator_t *co
int component_level,
char *component_name,
int *key, int *ncount);
static void mca_coll_hierarch_dump_struct ( struct mca_coll_base_comm_t *c);
/*
* Linear set of collective algorithms
*/
static const mca_coll_base_module_1_0_0_t intra = {
/* Initialization / finalization functions */
mca_coll_hierarch_module_init,
mca_coll_hierarch_module_finalize,
/* Collective function pointers */
/* function pointers marked with NULL are not yet implemented
and will use the functions provided in the basic module */
NULL, /* allgather */
NULL, /* allgatherv */
mca_coll_hierarch_allreduce_intra,
NULL, /* alltoall */
NULL, /* alltoallv */
NULL, /* alltoallw */
mca_coll_hierarch_barrier_intra,
mca_coll_hierarch_bcast_intra,
NULL, /* exscan */
NULL, /* gather */
NULL, /* gatherv */
mca_coll_hierarch_reduce_intra,
NULL, /* reduce_scatter */
NULL, /* scan */
NULL, /* scatter */
NULL, /* scatterv */
mca_coll_hierarch_ft_event
};
static const mca_coll_base_module_1_0_0_t null_intra = {
/* Initialization / finalization functions */
mca_coll_hierarch_module_init,
mca_coll_hierarch_module_finalize,
/* Collective function pointers */
/* function pointers marked with NULL are not yet implemented
and will use the functions provided in the basic module */
NULL,
NULL,
NULL,
NULL,
NULL,
NULL,
NULL,
NULL,
NULL,
NULL,
NULL,
NULL,
NULL,
NULL,
NULL,
NULL,
NULL
};
static void mca_coll_hierarch_dump_struct ( mca_coll_hierarch_module_t *c);
/*
@ -132,17 +70,42 @@ int mca_coll_hierarch_init_query(bool allow_hierarch_user_threads,
* Look at the communicator and decide which set of functions and
* priority we want to return.
*/
const mca_coll_base_module_1_0_0_t *
mca_coll_hierarch_comm_query(struct ompi_communicator_t *comm, int *priority,
struct mca_coll_base_comm_t **data)
mca_coll_base_module_1_1_0_t *
mca_coll_hierarch_comm_query(struct ompi_communicator_t *comm, int *priority )
{
int size, rank;
int color, ncount, maxncount;
struct mca_coll_base_comm_t *tdata=NULL;
int level;
int ret=OMPI_SUCCESS;
int ignore_sm=0;
int symmetric=0;
mca_coll_hierarch_module_t *hierarch_module;
hierarch_module = OBJ_NEW ( mca_coll_hierarch_module_t);
if ( NULL == hierarch_module ) {
return NULL;
}
hierarch_module->super.coll_module_enable = mca_coll_hierarch_module_enable;
hierarch_module->super.ft_event = mca_coll_hierarch_ft_event;
hierarch_module->super.coll_allgather = NULL;
hierarch_module->super.coll_allgatherv = NULL;
hierarch_module->super.coll_allreduce = mca_coll_hierarch_allreduce_intra;
hierarch_module->super.coll_alltoall = NULL;
hierarch_module->super.coll_alltoallv = NULL;
hierarch_module->super.coll_alltoallw = NULL;
hierarch_module->super.coll_barrier = mca_coll_hierarch_barrier_intra;
hierarch_module->super.coll_bcast = mca_coll_hierarch_bcast_intra;
hierarch_module->super.coll_exscan = NULL;
hierarch_module->super.coll_gather = NULL;
hierarch_module->super.coll_gatherv = NULL;
hierarch_module->super.coll_reduce = mca_coll_hierarch_reduce_intra;
hierarch_module->super.coll_reduce_scatter = NULL;
hierarch_module->super.coll_scan = NULL;
hierarch_module->super.coll_scatter = NULL;
hierarch_module->super.coll_scatterv = NULL;
/* This module only works for intra-communicators at the moment */
if ( OMPI_COMM_IS_INTER(comm) ) {
@ -170,7 +133,6 @@ mca_coll_hierarch_comm_query(struct ompi_communicator_t *comm, int *priority,
symmetric = mca_coll_hierarch_symmetric_param;
size = ompi_comm_size(comm);
if ( size < 3 ) {
/* No need for hierarchical collectives for 1 or 2 procs. */
return NULL;
@ -178,16 +140,9 @@ mca_coll_hierarch_comm_query(struct ompi_communicator_t *comm, int *priority,
rank = ompi_comm_rank(comm);
/* allocate the data structure holding all information */
tdata = (struct mca_coll_base_comm_t*)calloc ( 1, sizeof(struct mca_coll_base_comm_t));
if ( NULL == tdata ) {
*priority = 0;
return NULL;
}
tdata->hier_num_colorarr = size;
tdata->hier_colorarr = (int *) malloc ( sizeof(int) * size);
if ( NULL == tdata->hier_colorarr ) {
hierarch_module->hier_num_colorarr = size;
hierarch_module->hier_colorarr = (int *) malloc ( sizeof(int) * size);
if ( NULL == hierarch_module->hier_colorarr ) {
*priority = 0;
return NULL;
}
@ -250,27 +205,18 @@ mca_coll_hierarch_comm_query(struct ompi_communicator_t *comm, int *priority,
}
ret = mca_coll_hierarch_allgather_tmp (&color, 1, MPI_INT,
tdata->hier_colorarr, 1,
hierarch_module->hier_colorarr, 1,
MPI_INT, comm );
if ( OMPI_SUCCESS != ret ) {
return NULL;
}
tdata->hier_level = level;
*data = tdata;
return &null_intra;
hierarch_module->hier_level = level;
return &(hierarch_module->super);
}
}
exit:
if ( NULL != tdata->hier_colorarr ) {
free ( tdata->hier_colorarr ) ;
}
if ( NULL != tdata ) {
free ( tdata );
}
*priority = 0;
return NULL;
}
@ -279,8 +225,8 @@ mca_coll_hierarch_comm_query(struct ompi_communicator_t *comm, int *priority,
/*
* Init module on the communicator
*/
const struct mca_coll_base_module_1_0_0_t *
mca_coll_hierarch_module_init(struct ompi_communicator_t *comm)
int mca_coll_hierarch_module_enable (mca_coll_base_module_1_1_0_t *module,
struct ompi_communicator_t *comm)
{
int color;
int size, rank, ret=OMPI_SUCCESS;
@ -288,14 +234,13 @@ mca_coll_hierarch_module_init(struct ompi_communicator_t *comm)
struct ompi_communicator_t *lcomm=NULL;
struct ompi_communicator_t *llcomm=NULL;
struct mca_coll_base_comm_t *data=NULL;
struct mca_coll_hierarch_llead_t *llead=NULL;
mca_coll_hierarch_module_t *hierarch_module = (mca_coll_hierarch_module_t *) module;
rank = ompi_comm_rank(comm);
size = ompi_comm_size(comm);
data = comm->c_coll_selected_data;
color = data->hier_colorarr[rank];
color = hierarch_module->hier_colorarr[rank];
/* Generate the subcommunicator based on the color returned by
the previous function. */
@ -304,11 +249,11 @@ mca_coll_hierarch_module_init(struct ompi_communicator_t *comm)
goto exit;
}
data->hier_comm = comm;
data->hier_lcomm = lcomm;
data->hier_num_reqs = 2 * size;
data->hier_reqs = (ompi_request_t **) malloc (sizeof(ompi_request_t)*size*2);
if ( NULL == data->hier_reqs ) {
hierarch_module->hier_comm = comm;
hierarch_module->hier_lcomm = lcomm;
hierarch_module->hier_num_reqs = 2 * size;
hierarch_module->hier_reqs = (ompi_request_t **) malloc (sizeof(ompi_request_t)*size*2);
if ( NULL == hierarch_module->hier_reqs ) {
goto exit;
}
@ -327,8 +272,8 @@ mca_coll_hierarch_module_init(struct ompi_communicator_t *comm)
* depending on the offset, and has to be called therefore every time we need
* a new llcomm
*/
mca_coll_hierarch_get_llr ( data );
mca_coll_hierarch_get_all_lleaders ( rank, data, llead, 1 );
mca_coll_hierarch_get_llr ( hierarch_module );
mca_coll_hierarch_get_all_lleaders ( rank, hierarch_module, llead, 1 );
/* Generate the lleader communicator assuming that all lleaders are the first
process in the list of processes with the same color. A function generating
@ -340,11 +285,11 @@ mca_coll_hierarch_module_init(struct ompi_communicator_t *comm)
llead->llcomm = llcomm;
/* Store it now on the data structure */
OBJ_CONSTRUCT(&(data->hier_llead), ompi_pointer_array_t);
ompi_pointer_array_add ( &(data->hier_llead), llead);
OBJ_CONSTRUCT(&(hierarch_module->hier_llead), ompi_pointer_array_t);
ompi_pointer_array_add ( &(hierarch_module->hier_llead), llead);
if ( mca_coll_hierarch_verbose_param ) {
mca_coll_hierarch_dump_struct (data);
mca_coll_hierarch_dump_struct (hierarch_module);
}
exit:
@ -353,75 +298,15 @@ mca_coll_hierarch_module_init(struct ompi_communicator_t *comm)
}
if ( OMPI_SUCCESS != ret ) {
ompi_comm_free ( &lcomm );
if ( NULL != data ) {
if ( NULL != data->hier_reqs ) {
free ( data->hier_reqs);
}
if ( NULL != data->hier_colorarr ) {
free ( data->hier_colorarr ) ;
}
if ( NULL != llead->lleaders ) {
free ( llead->lleaders);
}
if ( NULL != data->hier_llr ) {
free ( data->hier_llr );
}
free ( data );
}
return NULL;
return OMPI_ERROR;
}
return &intra;
}
/*
* Finalize module on the communicator
*/
int mca_coll_hierarch_module_finalize(struct ompi_communicator_t *comm)
{
int i;
int size;
struct ompi_communicator_t *lcomm=NULL;
struct mca_coll_base_comm_t *data=NULL;
struct mca_coll_hierarch_llead_t *current=NULL;
data = comm->c_coll_selected_data;
lcomm = data->hier_lcomm;
ompi_comm_free (&lcomm);
free ( data->hier_reqs );
size = ompi_pointer_array_get_size ( &(data->hier_llead));
for ( i=0; i<size; i++) {
current = (struct mca_coll_hierarch_llead_t *)ompi_pointer_array_get_item (
&(data->hier_llead), i ) ;
if ( current->lleaders != NULL ) {
ompi_comm_free ( &(current->llcomm));
free ( current->lleaders );
}
free ( current );
}
ompi_pointer_array_remove_all ( &(data->hier_llead));
OBJ_DESTRUCT (&(data->hier_llead));
free ( data->hier_colorarr );
free ( data->hier_llr);
free ( data );
comm->c_coll_selected_data = NULL;
return OMPI_SUCCESS;
}
int mca_coll_hierarch_comm_unquery ( struct ompi_communicator_t *comm,
struct mca_coll_base_comm_t *data )
{
free ( data->hier_colorarr );
free ( data );
return OMPI_SUCCESS;
}
int mca_coll_hierarch_get_all_lleaders ( int rank, struct mca_coll_base_comm_t *data,
int mca_coll_hierarch_get_all_lleaders ( int rank, mca_coll_hierarch_module_t *hierarch_module,
struct mca_coll_hierarch_llead_t * llead,
int offset )
{
@ -429,34 +314,34 @@ int mca_coll_hierarch_get_all_lleaders ( int rank, struct mca_coll_base_comm_t *
int *cntarr=NULL;
int mycolor;
cntarr = (int *)calloc (1, sizeof (int)* data->hier_num_lleaders );
cntarr = (int *)calloc (1, sizeof (int)* hierarch_module->hier_num_lleaders );
if ( NULL == cntarr ) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
llead->lleaders = (int *) malloc (sizeof(int) * data->hier_num_lleaders);
llead->lleaders = (int *) malloc (sizeof(int) * hierarch_module->hier_num_lleaders);
if ( NULL == llead->lleaders ) {
ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
llead->offset = offset;
for ( i=0; i < data->hier_num_lleaders; i++ ) {
if ( data->hier_llr[i] == MPI_UNDEFINED ) {
for ( i=0; i < hierarch_module->hier_num_lleaders; i++ ) {
if ( MPI_UNDEFINED == hierarch_module->hier_llr[i] ) {
cntarr[i] = 1;
llead->lleaders[i] = MPI_UNDEFINED;
}
}
for ( i=0; i<data->hier_num_colorarr; i++) {
if ( data->hier_colorarr[i] == MPI_UNDEFINED ) {
for ( i=0; i<hierarch_module->hier_num_colorarr; i++) {
if ( MPI_UNDEFINED == hierarch_module->hier_colorarr[i] ) {
continue;
}
for ( j=0; j<data->hier_num_lleaders; j++) {
for ( j=0; j<hierarch_module->hier_num_lleaders; j++) {
if ( cntarr[j] >= offset ) {
continue;
}
if ( data->hier_colorarr[i] == data->hier_llr[j]) {
if ( hierarch_module->hier_colorarr[i] == hierarch_module->hier_llr[j]) {
cntarr[j]++;
llead->lleaders[j] = i;
break;
@ -464,15 +349,15 @@ int mca_coll_hierarch_get_all_lleaders ( int rank, struct mca_coll_base_comm_t *
}
}
mycolor = data->hier_colorarr[rank];
if ( mycolor == MPI_UNDEFINED ) {
mycolor = hierarch_module->hier_colorarr[rank];
if ( MPI_UNDEFINED == mycolor ) {
llead->am_lleader = 1;
llead->my_lleader = MPI_UNDEFINED;
}
else {
llead->am_lleader = 0;
for ( i=0; i< data->hier_num_lleaders; i++ ) {
if ( data->hier_llr[i] == mycolor ) {
for ( i=0; i< hierarch_module->hier_num_lleaders; i++ ) {
if ( hierarch_module->hier_llr[i] == mycolor ) {
llead->my_lleader = cntarr[i]-1;
if ( llead->lleaders[i] == rank ) {
llead->am_lleader = 1;
@ -489,39 +374,39 @@ int mca_coll_hierarch_get_all_lleaders ( int rank, struct mca_coll_base_comm_t *
return ret;
}
int mca_coll_hierarch_get_llr ( struct mca_coll_base_comm_t *data )
int mca_coll_hierarch_get_llr ( mca_coll_hierarch_module_t *hierarch_module )
{
int i, j, cnt, found;
int ncount;
ncount = mca_coll_hierarch_count_lleaders ( data->hier_num_colorarr,
data->hier_colorarr);
data->hier_num_lleaders = ncount;
data->hier_llr = (int *) malloc ( data->hier_num_lleaders * sizeof(int));
data->hier_max_offset = (int *) calloc ( 1, data->hier_num_lleaders * sizeof(int));
if ( ( NULL == data->hier_llr) || ( NULL == data->hier_max_offset )) {
ncount = mca_coll_hierarch_count_lleaders ( hierarch_module->hier_num_colorarr,
hierarch_module->hier_colorarr);
hierarch_module->hier_num_lleaders = ncount;
hierarch_module->hier_llr = (int *) malloc ( hierarch_module->hier_num_lleaders * sizeof(int));
hierarch_module->hier_max_offset = (int *) calloc ( 1, hierarch_module->hier_num_lleaders * sizeof(int));
if ( ( NULL == hierarch_module->hier_llr) || ( NULL == hierarch_module->hier_max_offset )) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
data->hier_llr[0] = data->hier_colorarr[0];
data->hier_max_offset[0]=1;
for ( cnt=1, i=1; i<data->hier_num_colorarr; i++ ) {
if ( data->hier_colorarr[i] == MPI_UNDEFINED ) {
data->hier_llr[cnt] = data->hier_colorarr[i];
data->hier_max_offset[cnt] = 1;
hierarch_module->hier_llr[0] = hierarch_module->hier_colorarr[0];
hierarch_module->hier_max_offset[0]=1;
for ( cnt=1, i=1; i<hierarch_module->hier_num_colorarr; i++ ) {
if ( MPI_UNDEFINED == hierarch_module->hier_colorarr[i] ) {
hierarch_module->hier_llr[cnt] = hierarch_module->hier_colorarr[i];
hierarch_module->hier_max_offset[cnt] = 1;
cnt++;
continue;
}
for ( found=0, j=0; j<cnt; j++ ) {
if ( data->hier_llr[j] == data->hier_colorarr[i]) {
data->hier_max_offset[j]++;
if ( hierarch_module->hier_llr[j] == hierarch_module->hier_colorarr[i]) {
hierarch_module->hier_max_offset[j]++;
found = 1;
break;
}
}
if ( !found ) {
data->hier_llr[cnt] = data->hier_colorarr[i];
data->hier_max_offset[cnt]++;
hierarch_module->hier_llr[cnt] = hierarch_module->hier_colorarr[i];
hierarch_module->hier_max_offset[cnt]++;
cnt++;
}
}
@ -531,7 +416,7 @@ int mca_coll_hierarch_get_llr ( struct mca_coll_base_comm_t *data )
struct ompi_communicator_t* mca_coll_hierarch_get_llcomm (int root,
struct mca_coll_base_comm_t *data,
mca_coll_hierarch_module_t *hierarch_module,
int* llroot,
int* lroot)
{
@ -540,18 +425,18 @@ struct ompi_communicator_t* mca_coll_hierarch_get_llcomm (int root,
struct ompi_group_t *group=NULL;
struct mca_coll_hierarch_llead_t *llead=NULL;
int found, i, rc, num_llead, offset;
int rank = ompi_comm_rank (data->hier_comm);
int rank = ompi_comm_rank (hierarch_module->hier_comm);
/* determine what our offset of root is in the colorarr */
offset = mca_coll_hierarch_get_offset ( root,
data->hier_num_colorarr,
data->hier_colorarr );
hierarch_module->hier_num_colorarr,
hierarch_module->hier_colorarr );
num_llead = ompi_pointer_array_get_size ( &(data->hier_llead) );
num_llead = ompi_pointer_array_get_size ( &(hierarch_module->hier_llead) );
for ( found=0, i=0; i < num_llead; i++ ) {
llead = (struct mca_coll_hierarch_llead_t *) ompi_pointer_array_get_item (
&(data->hier_llead), i );
if ( llead == NULL ) {
&(hierarch_module->hier_llead), i );
if ( NULL == llead ) {
continue;
}
@ -578,17 +463,17 @@ struct ompi_communicator_t* mca_coll_hierarch_get_llcomm (int root,
}
/* generate the list of lleaders with this offset */
mca_coll_hierarch_get_all_lleaders ( rank, data, llead, offset );
mca_coll_hierarch_get_all_lleaders ( rank, hierarch_module, llead, offset );
/* create new lleader subcommunicator */
rc = ompi_comm_split ( data->hier_comm, llead->am_lleader, root, &llcomm, 0);
rc = ompi_comm_split ( hierarch_module->hier_comm, llead->am_lleader, root, &llcomm, 0);
if ( OMPI_SUCCESS != rc ) {
return NULL;
}
llead->llcomm = llcomm;
/* Store the new element on the data struct */
ompi_pointer_array_add ( &(data->hier_llead), llead);
/* Store the new element on the hierarch_module struct */
ompi_pointer_array_add ( &(hierarch_module->hier_llead), llead);
}
llcomm = llead->llcomm;
@ -596,7 +481,7 @@ struct ompi_communicator_t* mca_coll_hierarch_get_llcomm (int root,
*llroot = MPI_UNDEFINED;
if ( MPI_COMM_NULL != llcomm ) {
rc = ompi_comm_group ( data->hier_comm, &group);
rc = ompi_comm_group ( hierarch_module->hier_comm, &group);
if ( OMPI_SUCCESS != rc ) {
return NULL;
}
@ -741,7 +626,7 @@ mca_coll_hierarch_checkfor_component ( struct ompi_communicator_t *comm,
/********************************************************************************/
/********************************************************************************/
static void mca_coll_hierarch_dump_struct ( struct mca_coll_base_comm_t *c)
static void mca_coll_hierarch_dump_struct ( mca_coll_hierarch_module_t *c)
{
int i, j;
int rank;

93
ompi/mca/coll/hierarch/coll_hierarch.h
Просмотреть файл

@ -9,6 +9,7 @@
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2007 University of Houston. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -26,15 +27,14 @@
#include "ompi/mca/coll/coll.h"
#include "ompi/request/request.h"
#if defined(c_plusplus) || defined(__cplusplus)
extern "C" {
#endif
BEGIN_C_DECLS
/*
* Globally exported variable
*/
OMPI_MODULE_DECLSPEC extern const mca_coll_base_component_1_0_0_t mca_coll_hierarch_component;
OMPI_MODULE_DECLSPEC extern const mca_coll_base_component_1_1_0_t mca_coll_hierarch_component;
extern int mca_coll_hierarch_priority_param;
extern int mca_coll_hierarch_verbose_param;
extern int mca_coll_hierarch_use_rdma_param;
@ -56,7 +56,9 @@ extern int mca_coll_hierarch_symmetric_param;
* llcomm: local leader communicator, grouping all local leaders of a comm.
*/
struct mca_coll_base_comm_t {
struct mca_coll_hierarch_module_t {
struct mca_coll_base_module_1_1_0_t super;
struct ompi_communicator_t *hier_comm; /* link back to the attached comm */
struct ompi_communicator_t *hier_lcomm; /* low level communicator */
ompi_pointer_array_t hier_llead; /* local leader communicator structure */
@ -71,6 +73,9 @@ extern int mca_coll_hierarch_symmetric_param;
Array of size hier_num_lleaders */
int *hier_colorarr; /* array containing the color of all procs */
};
typedef struct mca_coll_hierarch_module_t mca_coll_hierarch_module_t;
OBJ_CLASS_DECLARATION(mca_coll_hierarch_module_t);
struct mca_coll_hierarch_llead_t {
struct ompi_communicator_t *llcomm; /* local leader communicator */
@ -145,7 +150,7 @@ static inline int mca_coll_hierarch_get_offset ( int rank, int size, int *carr)
* collective operations. It is called from the collective operations themselves.
*
* @param root (input): rank of the root process in comm
* @param data (input): coll_base_comm structure of the hierarchical module. Contains
* @param hierarch_module (input): module structure. Contains
* all relevant, precomputed data for this set of collectives.
*
* @param llroot (output): rank of the root process in llcomm, MPI_UNDEFINED for all
@ -157,33 +162,33 @@ static inline int mca_coll_hierarch_get_offset ( int rank, int size, int *carr)
*/
struct ompi_communicator_t* mca_coll_hierarch_get_llcomm (int rroot,
struct mca_coll_base_comm_t *data,
mca_coll_hierarch_module_t *hierarch_module,
int* llroot,
int* lleader);
/* This function is supposed to set up all elements of the mca_coll_base_comm_t
* structure, including:
* data->hier_num_lleaders: determine number of local leaders in the comms
* data->hier_llr: array of size hier_num_lleaders containing the colors
* data->hier_max_offset: array containing the counter for each color how often
* hierarch_module->hier_num_lleaders: determine number of local leaders in the comms
* hierarch_module->hier_llr: array of size hier_num_lleaders containing the colors
* hierarch_module->hier_max_offset: array containing the counter for each color how often
* it appears in the colorarr array.
*/
int mca_coll_hierarch_get_llr ( struct mca_coll_base_comm_t *data );
int mca_coll_hierarch_get_llr ( mca_coll_hierarch_module_t *hierarch_module );
/* This function is supposed to set all elements of the llead structure based on the
* offset and the rank of the process.
*
* @param rank(input): rank of the calling process in comm
* @param data(input): coll_base_comm structure of the hierarchical module. Contains
* @param hierarch_module(input): structure of the hierarchical module. Contains
* all relevant, precomputed data for this set of collectives.
* @param llead(output): ptr to the mca_coll_hierarch_llead_t element which should
* be set
* @param offset(input): offset which shall be used.
*/
int mca_coll_hierarch_get_all_lleaders ( int rank, struct mca_coll_base_comm_t *data,
int mca_coll_hierarch_get_all_lleaders ( int rank, mca_coll_hierarch_module_t *hierarch_module,
struct mca_coll_hierarch_llead_t *llead,
int offset );
@ -194,14 +199,12 @@ int mca_coll_hierarch_get_all_lleaders ( int rank, struct mca_coll_base_comm_t *
*/
int mca_coll_hierarch_init_query(bool allow_hierarch_user_threads,
bool have_hidden_threads);
const struct mca_coll_base_module_1_0_0_t *
mca_coll_hierarch_comm_query(struct ompi_communicator_t *comm,
int *priority, struct mca_coll_base_comm_t **data);
int mca_coll_hierarch_comm_unquery(struct ompi_communicator_t *comm,
struct mca_coll_base_comm_t *data);
struct mca_coll_base_module_1_1_0_t *
mca_coll_hierarch_comm_query(struct ompi_communicator_t *comm, int *priority );
const struct mca_coll_base_module_1_0_0_t *
mca_coll_hierarch_module_init(struct ompi_communicator_t *comm);
int mca_coll_hierarch_module_enable( struct mca_coll_base_module_1_1_0_t *module,
struct ompi_communicator_t *comm);
int mca_coll_hierarch_module_finalize(struct ompi_communicator_t *comm);
@ -209,41 +212,49 @@ int mca_coll_hierarch_allgather_intra(void *sbuf, int scount,
struct ompi_datatype_t *sdtype,
void *rbuf, int rcount,
struct ompi_datatype_t *rdtype,
struct ompi_communicator_t *comm);
struct ompi_communicator_t *comm,
struct mca_coll_base_module_1_1_0_t *module );
int mca_coll_hierarch_allgatherv_intra(void *sbuf, int scount,
struct ompi_datatype_t *sdtype,
void * rbuf, int *rcounts,
int *disps,
struct ompi_datatype_t *rdtype,
struct ompi_communicator_t *comm);
struct ompi_communicator_t *comm,
struct mca_coll_base_module_1_1_0_t *module);
int mca_coll_hierarch_allreduce_intra(void *sbuf, void *rbuf, int count,
struct ompi_datatype_t *dtype,
struct ompi_op_t *op,
struct ompi_communicator_t *comm);
struct ompi_communicator_t *comm,
struct mca_coll_base_module_1_1_0_t *module);
int mca_coll_hierarch_alltoall_intra(void *sbuf, int scount,
struct ompi_datatype_t *sdtype,
void* rbuf, int rcount,
struct ompi_datatype_t *rdtype,
struct ompi_communicator_t *comm);
struct ompi_communicator_t *comm,
struct mca_coll_base_module_1_1_0_t *module);
int mca_coll_hierarch_alltoallv_intra(void *sbuf, int *scounts,
int *sdisps,
struct ompi_datatype_t *sdtype,
void *rbuf, int *rcounts,
int *rdisps,
struct ompi_datatype_t *rdtype,
struct ompi_communicator_t *comm);
struct ompi_communicator_t *comm,
struct mca_coll_base_module_1_1_0_t *module);
int mca_coll_hierarch_alltoallw_intra(void *sbuf, int *scounts,
int *sdisps,
struct ompi_datatype_t **sdtypes,
void *rbuf, int *rcounts,
int *rdisps,
struct ompi_datatype_t **rdtypes,
struct ompi_communicator_t *comm);
int mca_coll_hierarch_barrier_intra(struct ompi_communicator_t *comm);
struct ompi_communicator_t *comm,
struct mca_coll_base_module_1_1_0_t *module);
int mca_coll_hierarch_barrier_intra(struct ompi_communicator_t *comm,
struct mca_coll_base_module_1_1_0_t *module);
int mca_coll_hierarch_bcast_intra(void *buff, int count,
struct ompi_datatype_t *datatype,
int root,
struct ompi_communicator_t *comm);
struct ompi_communicator_t *comm,
struct mca_coll_base_module_1_1_0_t *module);
int mca_coll_hierarch_exscan_intra(void *sbuf, void *rbuf, int count,
struct ompi_datatype_t *dtype,
struct ompi_op_t *op,
@ -253,36 +264,43 @@ int mca_coll_hierarch_gather_intra(void *sbuf, int scount,
void *rbuf, int rcount,
struct ompi_datatype_t *rdtype,
int root,
struct ompi_communicator_t *comm);
struct ompi_communicator_t *comm,
struct mca_coll_base_module_1_1_0_t *module);
int mca_coll_hierarch_gatherv_intra(void *sbuf, int scount,
struct ompi_datatype_t *sdtype,
void *rbuf, int *rcounts, int *disps,
struct ompi_datatype_t *rdtype,
int root,
struct ompi_communicator_t *comm);
struct ompi_communicator_t *comm,
struct mca_coll_base_module_1_1_0_t *module);
int mca_coll_hierarch_reduce_intra(void *sbuf, void* rbuf, int count,
struct ompi_datatype_t *dtype,
struct ompi_op_t *op,
int root,
struct ompi_communicator_t *comm);
struct ompi_communicator_t *comm,
struct mca_coll_base_module_1_1_0_t *module);
int mca_coll_hierarch_reduce_scatter_intra(void *sbuf, void *rbuf,
int *rcounts,
struct ompi_datatype_t *dtype,
struct ompi_op_t *op,
struct ompi_communicator_t *comm);
struct ompi_communicator_t *comm,
struct mca_coll_base_module_1_1_0_t *module);
int mca_coll_hierarch_scan_intra(void *sbuf, void *rbuf, int count,
struct ompi_datatype_t *dtype,
struct ompi_op_t *op,
struct ompi_communicator_t *comm);
struct ompi_communicator_t *comm,
struct mca_coll_base_module_1_1_0_t *module);
int mca_coll_hierarch_scatter_intra(void *sbuf, int scount,
struct ompi_datatype_t *sdtype, void *rbuf,
int rcount, struct ompi_datatype_t *rdtype,
int root, struct ompi_communicator_t *comm);
int root, struct ompi_communicator_t *comm,
struct mca_coll_base_module_1_1_0_t *module);
int mca_coll_hierarch_scatterv_intra(void *sbuf, int *scounts, int *disps,
struct ompi_datatype_t *sdtype,
void* rbuf, int rcount,
struct ompi_datatype_t *rdtype, int root,
struct ompi_communicator_t *comm);
struct ompi_communicator_t *comm,
struct mca_coll_base_module_1_1_0_t *module);
/*
* These are trivial implementations of these routines used during comm_query/init,
@ -312,7 +330,6 @@ int mca_coll_hierarch_reduce_tmp(void *sbuf, void *rbuf, int count,
int mca_coll_hierarch_ft_event(int status);
#if defined(c_plusplus) || defined(__cplusplus)
}
#endif
END_C_DECLS
#endif /* MCA_COLL_HIERARCH_EXPORT_H */

26
ompi/mca/coll/hierarch/coll_hierarch_allreduce.c
Просмотреть файл

@ -9,6 +9,7 @@
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2007 University of Houston. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -39,11 +40,12 @@
int mca_coll_hierarch_allreduce_intra(void *sbuf, void *rbuf, int count,
struct ompi_datatype_t *dtype,
struct ompi_op_t *op,
struct ompi_communicator_t *comm)
struct ompi_communicator_t *comm,
struct mca_coll_base_module_1_1_0_t *module)
{
struct mca_coll_base_comm_t *data=NULL;
struct ompi_communicator_t *llcomm=NULL;
struct ompi_communicator_t *lcomm=NULL;
mca_coll_hierarch_module_t *hierarch_module = (mca_coll_hierarch_module_t *) module;
int rank;
int lroot, llroot;
ptrdiff_t extent, true_extent, lb, true_lb;
@ -52,15 +54,14 @@ int mca_coll_hierarch_allreduce_intra(void *sbuf, void *rbuf, int count,
int root=0;
rank = ompi_comm_rank ( comm );
data = comm->c_coll_selected_data;
lcomm = data->hier_lcomm;
lcomm = hierarch_module->hier_lcomm;
if ( mca_coll_hierarch_verbose_param ) {
printf("%s:%d: executing hierarchical allreduce with cnt=%d \n",
comm->c_name, rank, count );
}
llcomm = mca_coll_hierarch_get_llcomm ( root, data, &llroot, &lroot);
llcomm = mca_coll_hierarch_get_llcomm ( root, hierarch_module, &llroot, &lroot);
if ( MPI_COMM_NULL != lcomm ) {
ompi_ddt_get_extent(dtype, &lb, &extent);
@ -74,11 +75,13 @@ int mca_coll_hierarch_allreduce_intra(void *sbuf, void *rbuf, int count,
if ( MPI_IN_PLACE != sbuf ) {
ret = lcomm->c_coll.coll_reduce (sbuf, tmpbuf, count, dtype,
op, lroot, lcomm);
op, lroot, lcomm,
lcomm->c_coll.coll_reduce_module);
}
else {
ret = lcomm->c_coll.coll_reduce (rbuf, tmpbuf, count, dtype,
op, lroot, lcomm);
op, lroot, lcomm,
lcomm->c_coll.coll_reduce_module);
}
if ( OMPI_SUCCESS != ret ) {
goto exit;
@ -88,16 +91,19 @@ int mca_coll_hierarch_allreduce_intra(void *sbuf, void *rbuf, int count,
if ( MPI_UNDEFINED != llroot ) {
if ( MPI_COMM_NULL != lcomm ) {
ret = llcomm->c_coll.coll_allreduce (tmpbuf, rbuf, count, dtype,
op, llcomm);
op, llcomm,
llcomm->c_coll.coll_allreduce_module);
}
else {
ret = llcomm->c_coll.coll_allreduce (sbuf, rbuf, count, dtype,
op, llcomm);
op, llcomm,
llcomm->c_coll.coll_allreduce_module);
}
}
if ( MPI_COMM_NULL != lcomm ) {
ret = lcomm->c_coll.coll_bcast(rbuf, count, dtype, lroot, lcomm );
ret = lcomm->c_coll.coll_bcast(rbuf, count, dtype, lroot, lcomm,
lcomm->c_coll.coll_bcast_module );
}

17
ompi/mca/coll/hierarch/coll_hierarch_barrier.c
Просмотреть файл

@ -9,6 +9,7 @@
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2007 University of Houston. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -34,24 +35,24 @@
* Accepts: - same as MPI_Barrier()
* Returns: - MPI_SUCCESS or error code
*/
int mca_coll_hierarch_barrier_intra(struct ompi_communicator_t *comm)
int mca_coll_hierarch_barrier_intra(struct ompi_communicator_t *comm,
struct mca_coll_base_module_1_1_0_t *module)
{
struct mca_coll_base_comm_t *data=NULL;
struct ompi_communicator_t *llcomm=NULL;
struct ompi_communicator_t *lcomm=NULL;
mca_coll_hierarch_module_t *hierarch_module = (mca_coll_hierarch_module_t *) module;
int root=0;
int lroot, llroot;
int rank, ret=OMPI_SUCCESS;
rank = ompi_comm_rank ( comm );
data = comm->c_coll_selected_data;
lcomm = data->hier_lcomm;
lcomm = hierarch_module->hier_lcomm;
if ( mca_coll_hierarch_verbose_param ) {
printf("%s:%d: executing hierarchical barrier\n", comm->c_name, rank );
}
llcomm = mca_coll_hierarch_get_llcomm ( root, data, &llroot, &lroot);
llcomm = mca_coll_hierarch_get_llcomm ( root, hierarch_module, &llroot, &lroot);
/*
* Barrier consists of three steps:
@ -61,18 +62,18 @@ int mca_coll_hierarch_barrier_intra(struct ompi_communicator_t *comm)
* necessary to avoid that any non local leaders exit too early.
*/
if ( MPI_COMM_NULL != lcomm ) {
ret = lcomm->c_coll.coll_barrier ( lcomm );
ret = lcomm->c_coll.coll_barrier ( lcomm, lcomm->c_coll.coll_barrier_module );
if ( OMPI_SUCCESS != ret ) {
return ret;
}
}
if ( MPI_UNDEFINED != llroot ) {
ret = llcomm->c_coll.coll_barrier ( llcomm );
ret = llcomm->c_coll.coll_barrier ( llcomm, llcomm->c_coll.coll_barrier_module );
}
if ( MPI_COMM_NULL != lcomm ) {
ret = lcomm->c_coll.coll_barrier ( lcomm );
ret = lcomm->c_coll.coll_barrier ( lcomm, lcomm->c_coll.coll_barrier_module );
}
return ret;

17
ompi/mca/coll/hierarch/coll_hierarch_bcast.c
Просмотреть файл

@ -9,6 +9,7 @@
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2007 University of Houston. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -38,18 +39,18 @@ int mca_coll_hierarch_bcast_intra(void *buff,
int count,
struct ompi_datatype_t *datatype,
int root,
struct ompi_communicator_t *comm)
struct ompi_communicator_t *comm,
struct mca_coll_base_module_1_1_0_t *module)
{
struct mca_coll_base_comm_t *data=NULL;
struct ompi_communicator_t *llcomm=NULL;
struct ompi_communicator_t *lcomm=NULL;
mca_coll_hierarch_module_t *hierarch_module = (mca_coll_hierarch_module_t *) module;
int lroot, llroot;
int rank, ret=OMPI_SUCCESS;
rank = ompi_comm_rank ( comm );
data = comm->c_coll_selected_data;
lcomm = data->hier_lcomm;
lcomm = hierarch_module->hier_lcomm;
if ( mca_coll_hierarch_verbose_param ) {
printf("%s:%d: executing hierarchical bcast with cnt=%d and root=%d\n",
@ -63,11 +64,12 @@ int mca_coll_hierarch_bcast_intra(void *buff,
* also the reason, that *every* process in comm has to call
* this function
*/
llcomm = mca_coll_hierarch_get_llcomm ( root, data, &llroot, &lroot);
llcomm = mca_coll_hierarch_get_llcomm ( root, hierarch_module, &llroot, &lroot);
/* Bcast on the upper level among the local leaders */
if ( MPI_UNDEFINED != llroot ) {
ret = llcomm->c_coll.coll_bcast(buff, count, datatype, llroot, llcomm);
ret = llcomm->c_coll.coll_bcast(buff, count, datatype, llroot, llcomm,
llcomm->c_coll.coll_bcast_module);
if ( OMPI_SUCCESS != ret ) {
return ret;
}
@ -77,7 +79,8 @@ int mca_coll_hierarch_bcast_intra(void *buff,
* it to the processes in their local, low-leve communicator.
*/
if ( MPI_COMM_NULL != lcomm ) {
ret = lcomm->c_coll.coll_bcast(buff, count, datatype, lroot, lcomm );
ret = lcomm->c_coll.coll_bcast(buff, count, datatype, lroot, lcomm,
lcomm->c_coll.coll_bcast_module );
}
return ret;

65
ompi/mca/coll/hierarch/coll_hierarch_component.c
Просмотреть файл

@ -9,6 +9,7 @@
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2007 University of Houston. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -26,6 +27,7 @@
#include "coll_hierarch.h"
#include "mpi.h"
#include "ompi/communicator/communicator.h"
#include "ompi/mca/coll/coll.h"
/*
@ -54,16 +56,16 @@ static int hierarch_open(void);
* and pointers to our public functions in it
*/
const mca_coll_base_component_1_0_0_t mca_coll_hierarch_component = {
const mca_coll_base_component_1_1_0_t mca_coll_hierarch_component = {
/* First, the mca_component_t struct containing meta information
about the component itself */
{
/* Indicate that we are a coll v1.0.0 component (which also implies a
/* Indicate that we are a coll v1.1.0 component (which also implies a
specific MCA version) */
MCA_COLL_BASE_VERSION_1_0_0,
MCA_COLL_BASE_VERSION_1_1_0,
/* Component name and version */
@ -88,7 +90,6 @@ const mca_coll_base_component_1_0_0_t mca_coll_hierarch_component = {
/* Initialization / querying functions */
mca_coll_hierarch_init_query,
mca_coll_hierarch_comm_query,
mca_coll_hierarch_comm_unquery
};
@ -133,3 +134,59 @@ static int hierarch_open(void)
return OMPI_SUCCESS;
}
static void
mca_coll_hierarch_module_construct(mca_coll_hierarch_module_t *module)
{
module->hier_lcomm = MPI_COMM_NULL;
module->hier_reqs = NULL;
/* module->hier_llead = (ompi_pointer_array_t ) NULL; */
module->hier_colorarr = NULL;
module->hier_llr = NULL;
return;
}
static void
mca_coll_hierarch_module_destruct(mca_coll_hierarch_module_t *hierarch_module)
{
int i, size;
struct mca_coll_hierarch_llead_t *current=NULL;
if ( MPI_COMM_NULL != hierarch_module->hier_lcomm ) {
ompi_comm_free (&(hierarch_module->hier_lcomm) );
}
if ( NULL != hierarch_module->hier_reqs ) {
free ( hierarch_module->hier_reqs );
}
/* if ( NULL != hierarch_module->hier_llead ) { */
size = ompi_pointer_array_get_size ( &(hierarch_module->hier_llead));
for ( i=0; i<size; i++) {
current = (struct mca_coll_hierarch_llead_t *)ompi_pointer_array_get_item (
&(hierarch_module->hier_llead), i ) ;
if ( current->lleaders != NULL ) {
ompi_comm_free ( &(current->llcomm));
free ( current->lleaders );
}
free ( current );
}
ompi_pointer_array_remove_all ( &(hierarch_module->hier_llead));
OBJ_DESTRUCT (&(hierarch_module->hier_llead));
/* } */
if ( NULL != hierarch_module->hier_colorarr ) {
free ( hierarch_module->hier_colorarr );
}
if ( NULL != hierarch_module->hier_llr ) {
free ( hierarch_module->hier_llr);
}
return;
}
OBJ_CLASS_INSTANCE(mca_coll_hierarch_module_t,
mca_coll_base_module_1_1_0_t,
mca_coll_hierarch_module_construct,
mca_coll_hierarch_module_destruct);

23
ompi/mca/coll/hierarch/coll_hierarch_reduce.c
Просмотреть файл

@ -9,6 +9,7 @@
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2007 University of Houston. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -39,11 +40,12 @@
int mca_coll_hierarch_reduce_intra(void *sbuf, void *rbuf, int count,
struct ompi_datatype_t *dtype,
struct ompi_op_t *op,
int root, struct ompi_communicator_t *comm)
int root, struct ompi_communicator_t *comm,
struct mca_coll_base_module_1_1_0_t *module)
{
struct mca_coll_base_comm_t *data=NULL;
struct ompi_communicator_t *llcomm=NULL;
struct ompi_communicator_t *lcomm=NULL;
mca_coll_hierarch_module_t *hierarch_module = (mca_coll_hierarch_module_t *) module;
int rank;
int lroot, llroot;
ptrdiff_t extent, true_extent, lb, true_lb;
@ -51,15 +53,14 @@ int mca_coll_hierarch_reduce_intra(void *sbuf, void *rbuf, int count,
int ret=OMPI_SUCCESS;
rank = ompi_comm_rank ( comm );
data = comm->c_coll_selected_data;
lcomm = data->hier_lcomm;
lcomm = hierarch_module->hier_lcomm;
if ( mca_coll_hierarch_verbose_param ) {
printf("%s:%d: executing hierarchical reduce with cnt=%d and root=%d\n",
comm->c_name, rank, count, root );
}
llcomm = mca_coll_hierarch_get_llcomm ( root, data, &llroot, &lroot);
llcomm = mca_coll_hierarch_get_llcomm ( root, hierarch_module, &llroot, &lroot);
if ( MPI_COMM_NULL != lcomm ) {
ompi_ddt_get_extent(dtype, &lb, &extent);
@ -74,11 +75,13 @@ int mca_coll_hierarch_reduce_intra(void *sbuf, void *rbuf, int count,
if ( MPI_IN_PLACE != sbuf ) {
ret = lcomm->c_coll.coll_reduce (sbuf, tmpbuf, count, dtype,
op, lroot, lcomm);
op, lroot, lcomm,
lcomm->c_coll.coll_reduce_module);
}
else {
ret = lcomm->c_coll.coll_reduce (rbuf, tmpbuf, count, dtype,
op, lroot, lcomm);
op, lroot, lcomm,
lcomm->c_coll.coll_reduce_module);
}
if ( OMPI_SUCCESS != ret ) {
goto exit;
@ -88,11 +91,13 @@ int mca_coll_hierarch_reduce_intra(void *sbuf, void *rbuf, int count,
if ( MPI_UNDEFINED != llroot ) {
if ( MPI_COMM_NULL != lcomm ) {
ret = llcomm->c_coll.coll_reduce (tmpbuf, rbuf, count, dtype,
op, llroot, llcomm);
op, llroot, llcomm,
lcomm->c_coll.coll_reduce_module);
}
else {
ret = llcomm->c_coll.coll_reduce (sbuf, rbuf, count, dtype,
op, llroot, llcomm);
op, llroot, llcomm,
lcomm->c_coll.coll_reduce_module);
}
}