ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

convert the hiearch component to the new structure. More testing required before we remove the .ompi_ignore flag again.

Просмотр исходного кода 
This commit was SVN r15954.
Этот коммит содержится в:
Edgar Gabriel 2007-08-23 20:41:29 +00:00
родитель daaf5a9bf1
Коммит a2f5cada1a
7 изменённых файлов: 265 добавлений и 291 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

313
ompi/mca/coll/hierarch/coll_hierarch.c
Просмотреть файл

@ -9,6 +9,7 @@
* University of Stuttgart. All rights reserved. * University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California. * Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved. * All rights reserved.
* Copyright (c) 2007 University of Houston. All rights reserved.
* $COPYRIGHT$ * $COPYRIGHT$
* *
* Additional copyrights may follow * Additional copyrights may follow
@ -48,70 +49,7 @@ static void mca_coll_hierarch_checkfor_component (struct ompi_communicator_t *co
int component_level, int component_level,
char *component_name, char *component_name,
int *key, int *ncount); int *key, int *ncount);
static void mca_coll_hierarch_dump_struct ( struct mca_coll_base_comm_t *c); static void mca_coll_hierarch_dump_struct ( mca_coll_hierarch_module_t *c);
/*
* Linear set of collective algorithms
*/
static const mca_coll_base_module_1_0_0_t intra = {
/* Initialization / finalization functions */
mca_coll_hierarch_module_init,
mca_coll_hierarch_module_finalize,
/* Collective function pointers */
/* function pointers marked with NULL are not yet implemented
and will use the functions provided in the basic module */
NULL, /* allgather */
NULL, /* allgatherv */
mca_coll_hierarch_allreduce_intra,
NULL, /* alltoall */
NULL, /* alltoallv */
NULL, /* alltoallw */
mca_coll_hierarch_barrier_intra,
mca_coll_hierarch_bcast_intra,
NULL, /* exscan */
NULL, /* gather */
NULL, /* gatherv */
mca_coll_hierarch_reduce_intra,
NULL, /* reduce_scatter */
NULL, /* scan */
NULL, /* scatter */
NULL, /* scatterv */
mca_coll_hierarch_ft_event
};
static const mca_coll_base_module_1_0_0_t null_intra = {
/* Initialization / finalization functions */
mca_coll_hierarch_module_init,
mca_coll_hierarch_module_finalize,
/* Collective function pointers */
/* function pointers marked with NULL are not yet implemented
and will use the functions provided in the basic module */
NULL,
NULL,
NULL,
NULL,
NULL,
NULL,
NULL,
NULL,
NULL,
NULL,
NULL,
NULL,
NULL,
NULL,
NULL,
NULL,
NULL
};
/* /*
@ -132,17 +70,42 @@ int mca_coll_hierarch_init_query(bool allow_hierarch_user_threads,
* Look at the communicator and decide which set of functions and * Look at the communicator and decide which set of functions and
* priority we want to return. * priority we want to return.
*/ */
const mca_coll_base_module_1_0_0_t * mca_coll_base_module_1_1_0_t *
mca_coll_hierarch_comm_query(struct ompi_communicator_t *comm, int *priority, mca_coll_hierarch_comm_query(struct ompi_communicator_t *comm, int *priority )
struct mca_coll_base_comm_t **data)
{ {
int size, rank; int size, rank;
int color, ncount, maxncount; int color, ncount, maxncount;
struct mca_coll_base_comm_t *tdata=NULL;
int level; int level;
int ret=OMPI_SUCCESS; int ret=OMPI_SUCCESS;
int ignore_sm=0; int ignore_sm=0;
int symmetric=0; int symmetric=0;
mca_coll_hierarch_module_t *hierarch_module;
hierarch_module = OBJ_NEW ( mca_coll_hierarch_module_t);
if ( NULL == hierarch_module ) {
return NULL;
}
hierarch_module->super.coll_module_enable = mca_coll_hierarch_module_enable;
hierarch_module->super.ft_event = mca_coll_hierarch_ft_event;
hierarch_module->super.coll_allgather = NULL;
hierarch_module->super.coll_allgatherv = NULL;
hierarch_module->super.coll_allreduce = mca_coll_hierarch_allreduce_intra;
hierarch_module->super.coll_alltoall = NULL;
hierarch_module->super.coll_alltoallv = NULL;
hierarch_module->super.coll_alltoallw = NULL;
hierarch_module->super.coll_barrier = mca_coll_hierarch_barrier_intra;
hierarch_module->super.coll_bcast = mca_coll_hierarch_bcast_intra;
hierarch_module->super.coll_exscan = NULL;
hierarch_module->super.coll_gather = NULL;
hierarch_module->super.coll_gatherv = NULL;
hierarch_module->super.coll_reduce = mca_coll_hierarch_reduce_intra;
hierarch_module->super.coll_reduce_scatter = NULL;
hierarch_module->super.coll_scan = NULL;
hierarch_module->super.coll_scatter = NULL;
hierarch_module->super.coll_scatterv = NULL;
/* This module only works for intra-communicators at the moment */ /* This module only works for intra-communicators at the moment */
if ( OMPI_COMM_IS_INTER(comm) ) { if ( OMPI_COMM_IS_INTER(comm) ) {
@ -170,7 +133,6 @@ mca_coll_hierarch_comm_query(struct ompi_communicator_t *comm, int *priority,
symmetric = mca_coll_hierarch_symmetric_param; symmetric = mca_coll_hierarch_symmetric_param;
size = ompi_comm_size(comm); size = ompi_comm_size(comm);
if ( size < 3 ) { if ( size < 3 ) {
/* No need for hierarchical collectives for 1 or 2 procs. */ /* No need for hierarchical collectives for 1 or 2 procs. */
return NULL; return NULL;
@ -178,16 +140,9 @@ mca_coll_hierarch_comm_query(struct ompi_communicator_t *comm, int *priority,
rank = ompi_comm_rank(comm); rank = ompi_comm_rank(comm);
/* allocate the data structure holding all information */ hierarch_module->hier_num_colorarr = size;
tdata = (struct mca_coll_base_comm_t*)calloc ( 1, sizeof(struct mca_coll_base_comm_t)); hierarch_module->hier_colorarr = (int *) malloc ( sizeof(int) * size);
if ( NULL == tdata ) { if ( NULL == hierarch_module->hier_colorarr ) {
*priority = 0;
return NULL;
}
tdata->hier_num_colorarr = size;
tdata->hier_colorarr = (int *) malloc ( sizeof(int) * size);
if ( NULL == tdata->hier_colorarr ) {
*priority = 0; *priority = 0;
return NULL; return NULL;
} }
@ -250,27 +205,18 @@ mca_coll_hierarch_comm_query(struct ompi_communicator_t *comm, int *priority,
} }
ret = mca_coll_hierarch_allgather_tmp (&color, 1, MPI_INT, ret = mca_coll_hierarch_allgather_tmp (&color, 1, MPI_INT,
tdata->hier_colorarr, 1, hierarch_module->hier_colorarr, 1,
MPI_INT, comm ); MPI_INT, comm );
if ( OMPI_SUCCESS != ret ) { if ( OMPI_SUCCESS != ret ) {
return NULL; return NULL;
} }
tdata->hier_level = level; hierarch_module->hier_level = level;
*data = tdata; return &(hierarch_module->super);
return &null_intra;
} }
} }
exit: exit:
if ( NULL != tdata->hier_colorarr ) {
free ( tdata->hier_colorarr ) ;
}
if ( NULL != tdata ) {
free ( tdata );
}
*priority = 0; *priority = 0;
return NULL; return NULL;
} }
@ -279,8 +225,8 @@ mca_coll_hierarch_comm_query(struct ompi_communicator_t *comm, int *priority,
/* /*
* Init module on the communicator * Init module on the communicator
*/ */
const struct mca_coll_base_module_1_0_0_t * int mca_coll_hierarch_module_enable (mca_coll_base_module_1_1_0_t *module,
mca_coll_hierarch_module_init(struct ompi_communicator_t *comm) struct ompi_communicator_t *comm)
{ {
int color; int color;
int size, rank, ret=OMPI_SUCCESS; int size, rank, ret=OMPI_SUCCESS;
@ -288,14 +234,13 @@ mca_coll_hierarch_module_init(struct ompi_communicator_t *comm)
struct ompi_communicator_t *lcomm=NULL; struct ompi_communicator_t *lcomm=NULL;
struct ompi_communicator_t *llcomm=NULL; struct ompi_communicator_t *llcomm=NULL;
struct mca_coll_base_comm_t *data=NULL;
struct mca_coll_hierarch_llead_t *llead=NULL; struct mca_coll_hierarch_llead_t *llead=NULL;
mca_coll_hierarch_module_t *hierarch_module = (mca_coll_hierarch_module_t *) module;
rank = ompi_comm_rank(comm); rank = ompi_comm_rank(comm);
size = ompi_comm_size(comm); size = ompi_comm_size(comm);
data = comm->c_coll_selected_data; color = hierarch_module->hier_colorarr[rank];
color = data->hier_colorarr[rank];
/* Generate the subcommunicator based on the color returned by /* Generate the subcommunicator based on the color returned by
the previous function. */ the previous function. */
@ -304,11 +249,11 @@ mca_coll_hierarch_module_init(struct ompi_communicator_t *comm)
goto exit; goto exit;
} }
data->hier_comm = comm; hierarch_module->hier_comm = comm;
data->hier_lcomm = lcomm; hierarch_module->hier_lcomm = lcomm;
data->hier_num_reqs = 2 * size; hierarch_module->hier_num_reqs = 2 * size;
data->hier_reqs = (ompi_request_t **) malloc (sizeof(ompi_request_t)*size*2); hierarch_module->hier_reqs = (ompi_request_t **) malloc (sizeof(ompi_request_t)*size*2);
if ( NULL == data->hier_reqs ) { if ( NULL == hierarch_module->hier_reqs ) {
goto exit; goto exit;
} }
@ -327,8 +272,8 @@ mca_coll_hierarch_module_init(struct ompi_communicator_t *comm)
* depending on the offset, and has to be called therefore every time we need * depending on the offset, and has to be called therefore every time we need
* a new llcomm * a new llcomm
*/ */
mca_coll_hierarch_get_llr ( data ); mca_coll_hierarch_get_llr ( hierarch_module );
mca_coll_hierarch_get_all_lleaders ( rank, data, llead, 1 ); mca_coll_hierarch_get_all_lleaders ( rank, hierarch_module, llead, 1 );
/* Generate the lleader communicator assuming that all lleaders are the first /* Generate the lleader communicator assuming that all lleaders are the first
process in the list of processes with the same color. A function generating process in the list of processes with the same color. A function generating
@ -340,11 +285,11 @@ mca_coll_hierarch_module_init(struct ompi_communicator_t *comm)
llead->llcomm = llcomm; llead->llcomm = llcomm;
/* Store it now on the data structure */ /* Store it now on the data structure */
OBJ_CONSTRUCT(&(data->hier_llead), ompi_pointer_array_t); OBJ_CONSTRUCT(&(hierarch_module->hier_llead), ompi_pointer_array_t);
ompi_pointer_array_add ( &(data->hier_llead), llead); ompi_pointer_array_add ( &(hierarch_module->hier_llead), llead);
if ( mca_coll_hierarch_verbose_param ) { if ( mca_coll_hierarch_verbose_param ) {
mca_coll_hierarch_dump_struct (data); mca_coll_hierarch_dump_struct (hierarch_module);
} }
exit: exit:
@ -353,110 +298,50 @@ mca_coll_hierarch_module_init(struct ompi_communicator_t *comm)
} }
if ( OMPI_SUCCESS != ret ) { if ( OMPI_SUCCESS != ret ) {
ompi_comm_free ( &lcomm ); ompi_comm_free ( &lcomm );
if ( NULL != data ) { return OMPI_ERROR;
if ( NULL != data->hier_reqs ) {
free ( data->hier_reqs);
}
if ( NULL != data->hier_colorarr ) {
free ( data->hier_colorarr ) ;
}
if ( NULL != llead->lleaders ) {
free ( llead->lleaders);
}
if ( NULL != data->hier_llr ) {
free ( data->hier_llr );
}
free ( data );
}
return NULL;
} }
return &intra;
}
/*
* Finalize module on the communicator
*/
int mca_coll_hierarch_module_finalize(struct ompi_communicator_t *comm)
{
int i;
int size;
struct ompi_communicator_t *lcomm=NULL;
struct mca_coll_base_comm_t *data=NULL;
struct mca_coll_hierarch_llead_t *current=NULL;
data = comm->c_coll_selected_data;
lcomm = data->hier_lcomm;
ompi_comm_free (&lcomm);
free ( data->hier_reqs );
size = ompi_pointer_array_get_size ( &(data->hier_llead));
for ( i=0; i<size; i++) {
current = (struct mca_coll_hierarch_llead_t *)ompi_pointer_array_get_item (
&(data->hier_llead), i ) ;
if ( current->lleaders != NULL ) {
ompi_comm_free ( &(current->llcomm));
free ( current->lleaders );
}
free ( current );
}
ompi_pointer_array_remove_all ( &(data->hier_llead));
OBJ_DESTRUCT (&(data->hier_llead));
free ( data->hier_colorarr );
free ( data->hier_llr);
free ( data );
comm->c_coll_selected_data = NULL;
return OMPI_SUCCESS;
}
int mca_coll_hierarch_comm_unquery ( struct ompi_communicator_t *comm,
struct mca_coll_base_comm_t *data )
{
free ( data->hier_colorarr );
free ( data );
return OMPI_SUCCESS; return OMPI_SUCCESS;
} }
int mca_coll_hierarch_get_all_lleaders ( int rank, struct mca_coll_base_comm_t *data,
struct mca_coll_hierarch_llead_t * llead, int mca_coll_hierarch_get_all_lleaders ( int rank, mca_coll_hierarch_module_t *hierarch_module,
int offset ) struct mca_coll_hierarch_llead_t * llead,
int offset )
{ {
int i, j, ret=OMPI_SUCCESS; int i, j, ret=OMPI_SUCCESS;
int *cntarr=NULL; int *cntarr=NULL;
int mycolor; int mycolor;
cntarr = (int *)calloc (1, sizeof (int)* data->hier_num_lleaders ); cntarr = (int *)calloc (1, sizeof (int)* hierarch_module->hier_num_lleaders );
if ( NULL == cntarr ) { if ( NULL == cntarr ) {
return OMPI_ERR_OUT_OF_RESOURCE; return OMPI_ERR_OUT_OF_RESOURCE;
} }
llead->lleaders = (int *) malloc (sizeof(int) * data->hier_num_lleaders); llead->lleaders = (int *) malloc (sizeof(int) * hierarch_module->hier_num_lleaders);
if ( NULL == llead->lleaders ) { if ( NULL == llead->lleaders ) {
ret = OMPI_ERR_OUT_OF_RESOURCE; ret = OMPI_ERR_OUT_OF_RESOURCE;
goto exit; goto exit;
} }
llead->offset = offset; llead->offset = offset;
for ( i=0; i < data->hier_num_lleaders; i++ ) { for ( i=0; i < hierarch_module->hier_num_lleaders; i++ ) {
if ( data->hier_llr[i] == MPI_UNDEFINED ) { if ( MPI_UNDEFINED == hierarch_module->hier_llr[i] ) {
cntarr[i] = 1; cntarr[i] = 1;
llead->lleaders[i] = MPI_UNDEFINED; llead->lleaders[i] = MPI_UNDEFINED;
} }
} }
for ( i=0; i<data->hier_num_colorarr; i++) { for ( i=0; i<hierarch_module->hier_num_colorarr; i++) {
if ( data->hier_colorarr[i] == MPI_UNDEFINED ) { if ( MPI_UNDEFINED == hierarch_module->hier_colorarr[i] ) {
continue; continue;
} }
for ( j=0; j<data->hier_num_lleaders; j++) { for ( j=0; j<hierarch_module->hier_num_lleaders; j++) {
if ( cntarr[j] >= offset ) { if ( cntarr[j] >= offset ) {
continue; continue;
} }
if ( data->hier_colorarr[i] == data->hier_llr[j]) { if ( hierarch_module->hier_colorarr[i] == hierarch_module->hier_llr[j]) {
cntarr[j]++; cntarr[j]++;
llead->lleaders[j] = i; llead->lleaders[j] = i;
break; break;
@ -464,15 +349,15 @@ int mca_coll_hierarch_get_all_lleaders ( int rank, struct mca_coll_base_comm_t *
} }
} }
mycolor = data->hier_colorarr[rank]; mycolor = hierarch_module->hier_colorarr[rank];
if ( mycolor == MPI_UNDEFINED ) { if ( MPI_UNDEFINED == mycolor ) {
llead->am_lleader = 1; llead->am_lleader = 1;
llead->my_lleader = MPI_UNDEFINED; llead->my_lleader = MPI_UNDEFINED;
} }
else { else {
llead->am_lleader = 0; llead->am_lleader = 0;
for ( i=0; i< data->hier_num_lleaders; i++ ) { for ( i=0; i< hierarch_module->hier_num_lleaders; i++ ) {
if ( data->hier_llr[i] == mycolor ) { if ( hierarch_module->hier_llr[i] == mycolor ) {
llead->my_lleader = cntarr[i]-1; llead->my_lleader = cntarr[i]-1;
if ( llead->lleaders[i] == rank ) { if ( llead->lleaders[i] == rank ) {
llead->am_lleader = 1; llead->am_lleader = 1;
@ -489,39 +374,39 @@ int mca_coll_hierarch_get_all_lleaders ( int rank, struct mca_coll_base_comm_t *
return ret; return ret;
} }
int mca_coll_hierarch_get_llr ( struct mca_coll_base_comm_t *data ) int mca_coll_hierarch_get_llr ( mca_coll_hierarch_module_t *hierarch_module )
{ {
int i, j, cnt, found; int i, j, cnt, found;
int ncount; int ncount;
ncount = mca_coll_hierarch_count_lleaders ( data->hier_num_colorarr, ncount = mca_coll_hierarch_count_lleaders ( hierarch_module->hier_num_colorarr,
data->hier_colorarr); hierarch_module->hier_colorarr);
data->hier_num_lleaders = ncount; hierarch_module->hier_num_lleaders = ncount;
data->hier_llr = (int *) malloc ( data->hier_num_lleaders * sizeof(int)); hierarch_module->hier_llr = (int *) malloc ( hierarch_module->hier_num_lleaders * sizeof(int));
data->hier_max_offset = (int *) calloc ( 1, data->hier_num_lleaders * sizeof(int)); hierarch_module->hier_max_offset = (int *) calloc ( 1, hierarch_module->hier_num_lleaders * sizeof(int));
if ( ( NULL == data->hier_llr) || ( NULL == data->hier_max_offset )) { if ( ( NULL == hierarch_module->hier_llr) || ( NULL == hierarch_module->hier_max_offset )) {
return OMPI_ERR_OUT_OF_RESOURCE; return OMPI_ERR_OUT_OF_RESOURCE;
} }
data->hier_llr[0] = data->hier_colorarr[0]; hierarch_module->hier_llr[0] = hierarch_module->hier_colorarr[0];
data->hier_max_offset[0]=1; hierarch_module->hier_max_offset[0]=1;
for ( cnt=1, i=1; i<data->hier_num_colorarr; i++ ) { for ( cnt=1, i=1; i<hierarch_module->hier_num_colorarr; i++ ) {
if ( data->hier_colorarr[i] == MPI_UNDEFINED ) { if ( MPI_UNDEFINED == hierarch_module->hier_colorarr[i] ) {
data->hier_llr[cnt] = data->hier_colorarr[i]; hierarch_module->hier_llr[cnt] = hierarch_module->hier_colorarr[i];
data->hier_max_offset[cnt] = 1; hierarch_module->hier_max_offset[cnt] = 1;
cnt++; cnt++;
continue; continue;
} }
for ( found=0, j=0; j<cnt; j++ ) { for ( found=0, j=0; j<cnt; j++ ) {
if ( data->hier_llr[j] == data->hier_colorarr[i]) { if ( hierarch_module->hier_llr[j] == hierarch_module->hier_colorarr[i]) {
data->hier_max_offset[j]++; hierarch_module->hier_max_offset[j]++;
found = 1; found = 1;
break; break;
} }
} }
if ( !found ) { if ( !found ) {
data->hier_llr[cnt] = data->hier_colorarr[i]; hierarch_module->hier_llr[cnt] = hierarch_module->hier_colorarr[i];
data->hier_max_offset[cnt]++; hierarch_module->hier_max_offset[cnt]++;
cnt++; cnt++;
} }
} }
@ -531,7 +416,7 @@ int mca_coll_hierarch_get_llr ( struct mca_coll_base_comm_t *data )
struct ompi_communicator_t* mca_coll_hierarch_get_llcomm (int root, struct ompi_communicator_t* mca_coll_hierarch_get_llcomm (int root,
struct mca_coll_base_comm_t *data, mca_coll_hierarch_module_t *hierarch_module,
int* llroot, int* llroot,
int* lroot) int* lroot)
{ {
@ -540,18 +425,18 @@ struct ompi_communicator_t* mca_coll_hierarch_get_llcomm (int root,
struct ompi_group_t *group=NULL; struct ompi_group_t *group=NULL;
struct mca_coll_hierarch_llead_t *llead=NULL; struct mca_coll_hierarch_llead_t *llead=NULL;
int found, i, rc, num_llead, offset; int found, i, rc, num_llead, offset;
int rank = ompi_comm_rank (data->hier_comm); int rank = ompi_comm_rank (hierarch_module->hier_comm);
/* determine what our offset of root is in the colorarr */ /* determine what our offset of root is in the colorarr */
offset = mca_coll_hierarch_get_offset ( root, offset = mca_coll_hierarch_get_offset ( root,
data->hier_num_colorarr, hierarch_module->hier_num_colorarr,
data->hier_colorarr ); hierarch_module->hier_colorarr );
num_llead = ompi_pointer_array_get_size ( &(data->hier_llead) ); num_llead = ompi_pointer_array_get_size ( &(hierarch_module->hier_llead) );
for ( found=0, i=0; i < num_llead; i++ ) { for ( found=0, i=0; i < num_llead; i++ ) {
llead = (struct mca_coll_hierarch_llead_t *) ompi_pointer_array_get_item ( llead = (struct mca_coll_hierarch_llead_t *) ompi_pointer_array_get_item (
&(data->hier_llead), i ); &(hierarch_module->hier_llead), i );
if ( llead == NULL ) { if ( NULL == llead ) {
continue; continue;
} }
@ -578,17 +463,17 @@ struct ompi_communicator_t* mca_coll_hierarch_get_llcomm (int root,
} }
/* generate the list of lleaders with this offset */ /* generate the list of lleaders with this offset */
mca_coll_hierarch_get_all_lleaders ( rank, data, llead, offset ); mca_coll_hierarch_get_all_lleaders ( rank, hierarch_module, llead, offset );
/* create new lleader subcommunicator */ /* create new lleader subcommunicator */
rc = ompi_comm_split ( data->hier_comm, llead->am_lleader, root, &llcomm, 0); rc = ompi_comm_split ( hierarch_module->hier_comm, llead->am_lleader, root, &llcomm, 0);
if ( OMPI_SUCCESS != rc ) { if ( OMPI_SUCCESS != rc ) {
return NULL; return NULL;
} }
llead->llcomm = llcomm; llead->llcomm = llcomm;
/* Store the new element on the data struct */ /* Store the new element on the hierarch_module struct */
ompi_pointer_array_add ( &(data->hier_llead), llead); ompi_pointer_array_add ( &(hierarch_module->hier_llead), llead);
} }
llcomm = llead->llcomm; llcomm = llead->llcomm;
@ -596,7 +481,7 @@ struct ompi_communicator_t* mca_coll_hierarch_get_llcomm (int root,
*llroot = MPI_UNDEFINED; *llroot = MPI_UNDEFINED;
if ( MPI_COMM_NULL != llcomm ) { if ( MPI_COMM_NULL != llcomm ) {
rc = ompi_comm_group ( data->hier_comm, &group); rc = ompi_comm_group ( hierarch_module->hier_comm, &group);
if ( OMPI_SUCCESS != rc ) { if ( OMPI_SUCCESS != rc ) {
return NULL; return NULL;
} }
@ -741,7 +626,7 @@ mca_coll_hierarch_checkfor_component ( struct ompi_communicator_t *comm,
/********************************************************************************/ /********************************************************************************/
/********************************************************************************/ /********************************************************************************/
static void mca_coll_hierarch_dump_struct ( struct mca_coll_base_comm_t *c) static void mca_coll_hierarch_dump_struct ( mca_coll_hierarch_module_t *c)
{ {
int i, j; int i, j;
int rank; int rank;

95
ompi/mca/coll/hierarch/coll_hierarch.h
Просмотреть файл

@ -9,6 +9,7 @@
* University of Stuttgart. All rights reserved. * University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California. * Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved. * All rights reserved.
* Copyright (c) 2007 University of Houston. All rights reserved.
* $COPYRIGHT$ * $COPYRIGHT$
* *
* Additional copyrights may follow * Additional copyrights may follow
@ -26,15 +27,14 @@
#include "ompi/mca/coll/coll.h" #include "ompi/mca/coll/coll.h"
#include "ompi/request/request.h" #include "ompi/request/request.h"
#if defined(c_plusplus) || defined(__cplusplus) BEGIN_C_DECLS
extern "C" {
#endif
/* /*
* Globally exported variable * Globally exported variable
*/ */
OMPI_MODULE_DECLSPEC extern const mca_coll_base_component_1_0_0_t mca_coll_hierarch_component; OMPI_MODULE_DECLSPEC extern const mca_coll_base_component_1_1_0_t mca_coll_hierarch_component;
extern int mca_coll_hierarch_priority_param; extern int mca_coll_hierarch_priority_param;
extern int mca_coll_hierarch_verbose_param; extern int mca_coll_hierarch_verbose_param;
extern int mca_coll_hierarch_use_rdma_param; extern int mca_coll_hierarch_use_rdma_param;
@ -56,7 +56,9 @@ extern int mca_coll_hierarch_symmetric_param;
* llcomm: local leader communicator, grouping all local leaders of a comm. * llcomm: local leader communicator, grouping all local leaders of a comm.
*/ */
struct mca_coll_base_comm_t { struct mca_coll_hierarch_module_t {
struct mca_coll_base_module_1_1_0_t super;
struct ompi_communicator_t *hier_comm; /* link back to the attached comm */ struct ompi_communicator_t *hier_comm; /* link back to the attached comm */
struct ompi_communicator_t *hier_lcomm; /* low level communicator */ struct ompi_communicator_t *hier_lcomm; /* low level communicator */
ompi_pointer_array_t hier_llead; /* local leader communicator structure */ ompi_pointer_array_t hier_llead; /* local leader communicator structure */
@ -70,7 +72,10 @@ extern int mca_coll_hierarch_symmetric_param;
int *hier_max_offset; /* Number of processes for each color. int *hier_max_offset; /* Number of processes for each color.
Array of size hier_num_lleaders */ Array of size hier_num_lleaders */
int *hier_colorarr; /* array containing the color of all procs */ int *hier_colorarr; /* array containing the color of all procs */
}; };
typedef struct mca_coll_hierarch_module_t mca_coll_hierarch_module_t;
OBJ_CLASS_DECLARATION(mca_coll_hierarch_module_t);
struct mca_coll_hierarch_llead_t { struct mca_coll_hierarch_llead_t {
struct ompi_communicator_t *llcomm; /* local leader communicator */ struct ompi_communicator_t *llcomm; /* local leader communicator */
@ -145,7 +150,7 @@ static inline int mca_coll_hierarch_get_offset ( int rank, int size, int *carr)
* collective operations. It is called from the collective operations themselves. * collective operations. It is called from the collective operations themselves.
* *
* @param root (input): rank of the root process in comm * @param root (input): rank of the root process in comm
* @param data (input): coll_base_comm structure of the hierarchical module. Contains * @param hierarch_module (input): module structure. Contains
* all relevant, precomputed data for this set of collectives. * all relevant, precomputed data for this set of collectives.
* *
* @param llroot (output): rank of the root process in llcomm, MPI_UNDEFINED for all * @param llroot (output): rank of the root process in llcomm, MPI_UNDEFINED for all
@ -157,33 +162,33 @@ static inline int mca_coll_hierarch_get_offset ( int rank, int size, int *carr)
*/ */
struct ompi_communicator_t* mca_coll_hierarch_get_llcomm (int rroot, struct ompi_communicator_t* mca_coll_hierarch_get_llcomm (int rroot,
struct mca_coll_base_comm_t *data, mca_coll_hierarch_module_t *hierarch_module,
int* llroot, int* llroot,
int* lleader); int* lleader);
/* This function is supposed to set up all elements of the mca_coll_base_comm_t /* This function is supposed to set up all elements of the mca_coll_base_comm_t
* structure, including: * structure, including:
* data->hier_num_lleaders: determine number of local leaders in the comms * hierarch_module->hier_num_lleaders: determine number of local leaders in the comms
* data->hier_llr: array of size hier_num_lleaders containing the colors * hierarch_module->hier_llr: array of size hier_num_lleaders containing the colors
* data->hier_max_offset: array containing the counter for each color how often * hierarch_module->hier_max_offset: array containing the counter for each color how often
* it appears in the colorarr array. * it appears in the colorarr array.
*/ */
int mca_coll_hierarch_get_llr ( struct mca_coll_base_comm_t *data ); int mca_coll_hierarch_get_llr ( mca_coll_hierarch_module_t *hierarch_module );
/* This function is supposed to set all elements of the llead structure based on the /* This function is supposed to set all elements of the llead structure based on the
* offset and the rank of the process. * offset and the rank of the process.
* *
* @param rank(input): rank of the calling process in comm * @param rank(input): rank of the calling process in comm
* @param data(input): coll_base_comm structure of the hierarchical module. Contains * @param hierarch_module(input): structure of the hierarchical module. Contains
* all relevant, precomputed data for this set of collectives. * all relevant, precomputed data for this set of collectives.
* @param llead(output): ptr to the mca_coll_hierarch_llead_t element which should * @param llead(output): ptr to the mca_coll_hierarch_llead_t element which should
* be set * be set
* @param offset(input): offset which shall be used. * @param offset(input): offset which shall be used.
*/ */
int mca_coll_hierarch_get_all_lleaders ( int rank, struct mca_coll_base_comm_t *data, int mca_coll_hierarch_get_all_lleaders ( int rank, mca_coll_hierarch_module_t *hierarch_module,
struct mca_coll_hierarch_llead_t *llead, struct mca_coll_hierarch_llead_t *llead,
int offset ); int offset );
@ -194,14 +199,12 @@ int mca_coll_hierarch_get_all_lleaders ( int rank, struct mca_coll_base_comm_t *
*/ */
int mca_coll_hierarch_init_query(bool allow_hierarch_user_threads, int mca_coll_hierarch_init_query(bool allow_hierarch_user_threads,
bool have_hidden_threads); bool have_hidden_threads);
const struct mca_coll_base_module_1_0_0_t * struct mca_coll_base_module_1_1_0_t *
mca_coll_hierarch_comm_query(struct ompi_communicator_t *comm, mca_coll_hierarch_comm_query(struct ompi_communicator_t *comm, int *priority );
int *priority, struct mca_coll_base_comm_t **data);
int mca_coll_hierarch_comm_unquery(struct ompi_communicator_t *comm,
struct mca_coll_base_comm_t *data);
const struct mca_coll_base_module_1_0_0_t *
mca_coll_hierarch_module_init(struct ompi_communicator_t *comm); int mca_coll_hierarch_module_enable( struct mca_coll_base_module_1_1_0_t *module,
struct ompi_communicator_t *comm);
int mca_coll_hierarch_module_finalize(struct ompi_communicator_t *comm); int mca_coll_hierarch_module_finalize(struct ompi_communicator_t *comm);
@ -209,41 +212,49 @@ int mca_coll_hierarch_allgather_intra(void *sbuf, int scount,
struct ompi_datatype_t *sdtype, struct ompi_datatype_t *sdtype,
void *rbuf, int rcount, void *rbuf, int rcount,
struct ompi_datatype_t *rdtype, struct ompi_datatype_t *rdtype,
struct ompi_communicator_t *comm); struct ompi_communicator_t *comm,
struct mca_coll_base_module_1_1_0_t *module );
int mca_coll_hierarch_allgatherv_intra(void *sbuf, int scount, int mca_coll_hierarch_allgatherv_intra(void *sbuf, int scount,
struct ompi_datatype_t *sdtype, struct ompi_datatype_t *sdtype,
void * rbuf, int *rcounts, void * rbuf, int *rcounts,
int *disps, int *disps,
struct ompi_datatype_t *rdtype, struct ompi_datatype_t *rdtype,
struct ompi_communicator_t *comm); struct ompi_communicator_t *comm,
struct mca_coll_base_module_1_1_0_t *module);
int mca_coll_hierarch_allreduce_intra(void *sbuf, void *rbuf, int count, int mca_coll_hierarch_allreduce_intra(void *sbuf, void *rbuf, int count,
struct ompi_datatype_t *dtype, struct ompi_datatype_t *dtype,
struct ompi_op_t *op, struct ompi_op_t *op,
struct ompi_communicator_t *comm); struct ompi_communicator_t *comm,
struct mca_coll_base_module_1_1_0_t *module);
int mca_coll_hierarch_alltoall_intra(void *sbuf, int scount, int mca_coll_hierarch_alltoall_intra(void *sbuf, int scount,
struct ompi_datatype_t *sdtype, struct ompi_datatype_t *sdtype,
void* rbuf, int rcount, void* rbuf, int rcount,
struct ompi_datatype_t *rdtype, struct ompi_datatype_t *rdtype,
struct ompi_communicator_t *comm); struct ompi_communicator_t *comm,
struct mca_coll_base_module_1_1_0_t *module);
int mca_coll_hierarch_alltoallv_intra(void *sbuf, int *scounts, int mca_coll_hierarch_alltoallv_intra(void *sbuf, int *scounts,
int *sdisps, int *sdisps,
struct ompi_datatype_t *sdtype, struct ompi_datatype_t *sdtype,
void *rbuf, int *rcounts, void *rbuf, int *rcounts,
int *rdisps, int *rdisps,
struct ompi_datatype_t *rdtype, struct ompi_datatype_t *rdtype,
struct ompi_communicator_t *comm); struct ompi_communicator_t *comm,
struct mca_coll_base_module_1_1_0_t *module);
int mca_coll_hierarch_alltoallw_intra(void *sbuf, int *scounts, int mca_coll_hierarch_alltoallw_intra(void *sbuf, int *scounts,
int *sdisps, int *sdisps,
struct ompi_datatype_t **sdtypes, struct ompi_datatype_t **sdtypes,
void *rbuf, int *rcounts, void *rbuf, int *rcounts,
int *rdisps, int *rdisps,
struct ompi_datatype_t **rdtypes, struct ompi_datatype_t **rdtypes,
struct ompi_communicator_t *comm); struct ompi_communicator_t *comm,
int mca_coll_hierarch_barrier_intra(struct ompi_communicator_t *comm); struct mca_coll_base_module_1_1_0_t *module);
int mca_coll_hierarch_barrier_intra(struct ompi_communicator_t *comm,
struct mca_coll_base_module_1_1_0_t *module);
int mca_coll_hierarch_bcast_intra(void *buff, int count, int mca_coll_hierarch_bcast_intra(void *buff, int count,
struct ompi_datatype_t *datatype, struct ompi_datatype_t *datatype,
int root, int root,
struct ompi_communicator_t *comm); struct ompi_communicator_t *comm,
struct mca_coll_base_module_1_1_0_t *module);
int mca_coll_hierarch_exscan_intra(void *sbuf, void *rbuf, int count, int mca_coll_hierarch_exscan_intra(void *sbuf, void *rbuf, int count,
struct ompi_datatype_t *dtype, struct ompi_datatype_t *dtype,
struct ompi_op_t *op, struct ompi_op_t *op,
@ -253,36 +264,43 @@ int mca_coll_hierarch_gather_intra(void *sbuf, int scount,
void *rbuf, int rcount, void *rbuf, int rcount,
struct ompi_datatype_t *rdtype, struct ompi_datatype_t *rdtype,
int root, int root,
struct ompi_communicator_t *comm); struct ompi_communicator_t *comm,
struct mca_coll_base_module_1_1_0_t *module);
int mca_coll_hierarch_gatherv_intra(void *sbuf, int scount, int mca_coll_hierarch_gatherv_intra(void *sbuf, int scount,
struct ompi_datatype_t *sdtype, struct ompi_datatype_t *sdtype,
void *rbuf, int *rcounts, int *disps, void *rbuf, int *rcounts, int *disps,
struct ompi_datatype_t *rdtype, struct ompi_datatype_t *rdtype,
int root, int root,
struct ompi_communicator_t *comm); struct ompi_communicator_t *comm,
struct mca_coll_base_module_1_1_0_t *module);
int mca_coll_hierarch_reduce_intra(void *sbuf, void* rbuf, int count, int mca_coll_hierarch_reduce_intra(void *sbuf, void* rbuf, int count,
struct ompi_datatype_t *dtype, struct ompi_datatype_t *dtype,
struct ompi_op_t *op, struct ompi_op_t *op,
int root, int root,
struct ompi_communicator_t *comm); struct ompi_communicator_t *comm,
struct mca_coll_base_module_1_1_0_t *module);
int mca_coll_hierarch_reduce_scatter_intra(void *sbuf, void *rbuf, int mca_coll_hierarch_reduce_scatter_intra(void *sbuf, void *rbuf,
int *rcounts, int *rcounts,
struct ompi_datatype_t *dtype, struct ompi_datatype_t *dtype,
struct ompi_op_t *op, struct ompi_op_t *op,
struct ompi_communicator_t *comm); struct ompi_communicator_t *comm,
struct mca_coll_base_module_1_1_0_t *module);
int mca_coll_hierarch_scan_intra(void *sbuf, void *rbuf, int count, int mca_coll_hierarch_scan_intra(void *sbuf, void *rbuf, int count,
struct ompi_datatype_t *dtype, struct ompi_datatype_t *dtype,
struct ompi_op_t *op, struct ompi_op_t *op,
struct ompi_communicator_t *comm); struct ompi_communicator_t *comm,
struct mca_coll_base_module_1_1_0_t *module);
int mca_coll_hierarch_scatter_intra(void *sbuf, int scount, int mca_coll_hierarch_scatter_intra(void *sbuf, int scount,
struct ompi_datatype_t *sdtype, void *rbuf, struct ompi_datatype_t *sdtype, void *rbuf,
int rcount, struct ompi_datatype_t *rdtype, int rcount, struct ompi_datatype_t *rdtype,
int root, struct ompi_communicator_t *comm); int root, struct ompi_communicator_t *comm,
struct mca_coll_base_module_1_1_0_t *module);
int mca_coll_hierarch_scatterv_intra(void *sbuf, int *scounts, int *disps, int mca_coll_hierarch_scatterv_intra(void *sbuf, int *scounts, int *disps,
struct ompi_datatype_t *sdtype, struct ompi_datatype_t *sdtype,
void* rbuf, int rcount, void* rbuf, int rcount,
struct ompi_datatype_t *rdtype, int root, struct ompi_datatype_t *rdtype, int root,
struct ompi_communicator_t *comm); struct ompi_communicator_t *comm,
struct mca_coll_base_module_1_1_0_t *module);
/* /*
* These are trivial implementations of these routines used during comm_query/init, * These are trivial implementations of these routines used during comm_query/init,
@ -312,7 +330,6 @@ int mca_coll_hierarch_reduce_tmp(void *sbuf, void *rbuf, int count,
int mca_coll_hierarch_ft_event(int status); int mca_coll_hierarch_ft_event(int status);
#if defined(c_plusplus) || defined(__cplusplus) END_C_DECLS
}
#endif
#endif /* MCA_COLL_HIERARCH_EXPORT_H */ #endif /* MCA_COLL_HIERARCH_EXPORT_H */

26
ompi/mca/coll/hierarch/coll_hierarch_allreduce.c
Просмотреть файл

@ -9,6 +9,7 @@
* University of Stuttgart. All rights reserved. * University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California. * Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved. * All rights reserved.
* Copyright (c) 2007 University of Houston. All rights reserved.
* $COPYRIGHT$ * $COPYRIGHT$
* *
* Additional copyrights may follow * Additional copyrights may follow
@ -39,11 +40,12 @@
int mca_coll_hierarch_allreduce_intra(void *sbuf, void *rbuf, int count, int mca_coll_hierarch_allreduce_intra(void *sbuf, void *rbuf, int count,
struct ompi_datatype_t *dtype, struct ompi_datatype_t *dtype,
struct ompi_op_t *op, struct ompi_op_t *op,
struct ompi_communicator_t *comm) struct ompi_communicator_t *comm,
struct mca_coll_base_module_1_1_0_t *module)
{ {
struct mca_coll_base_comm_t *data=NULL;
struct ompi_communicator_t *llcomm=NULL; struct ompi_communicator_t *llcomm=NULL;
struct ompi_communicator_t *lcomm=NULL; struct ompi_communicator_t *lcomm=NULL;
mca_coll_hierarch_module_t *hierarch_module = (mca_coll_hierarch_module_t *) module;
int rank; int rank;
int lroot, llroot; int lroot, llroot;
ptrdiff_t extent, true_extent, lb, true_lb; ptrdiff_t extent, true_extent, lb, true_lb;
@ -52,15 +54,14 @@ int mca_coll_hierarch_allreduce_intra(void *sbuf, void *rbuf, int count,
int root=0; int root=0;
rank = ompi_comm_rank ( comm ); rank = ompi_comm_rank ( comm );
data = comm->c_coll_selected_data; lcomm = hierarch_module->hier_lcomm;
lcomm = data->hier_lcomm;
if ( mca_coll_hierarch_verbose_param ) { if ( mca_coll_hierarch_verbose_param ) {
printf("%s:%d: executing hierarchical allreduce with cnt=%d \n", printf("%s:%d: executing hierarchical allreduce with cnt=%d \n",
comm->c_name, rank, count ); comm->c_name, rank, count );
} }
llcomm = mca_coll_hierarch_get_llcomm ( root, data, &llroot, &lroot); llcomm = mca_coll_hierarch_get_llcomm ( root, hierarch_module, &llroot, &lroot);
if ( MPI_COMM_NULL != lcomm ) { if ( MPI_COMM_NULL != lcomm ) {
ompi_ddt_get_extent(dtype, &lb, &extent); ompi_ddt_get_extent(dtype, &lb, &extent);
@ -74,11 +75,13 @@ int mca_coll_hierarch_allreduce_intra(void *sbuf, void *rbuf, int count,
if ( MPI_IN_PLACE != sbuf ) { if ( MPI_IN_PLACE != sbuf ) {
ret = lcomm->c_coll.coll_reduce (sbuf, tmpbuf, count, dtype, ret = lcomm->c_coll.coll_reduce (sbuf, tmpbuf, count, dtype,
op, lroot, lcomm); op, lroot, lcomm,
lcomm->c_coll.coll_reduce_module);
} }
else { else {
ret = lcomm->c_coll.coll_reduce (rbuf, tmpbuf, count, dtype, ret = lcomm->c_coll.coll_reduce (rbuf, tmpbuf, count, dtype,
op, lroot, lcomm); op, lroot, lcomm,
lcomm->c_coll.coll_reduce_module);
} }
if ( OMPI_SUCCESS != ret ) { if ( OMPI_SUCCESS != ret ) {
goto exit; goto exit;
@ -88,16 +91,19 @@ int mca_coll_hierarch_allreduce_intra(void *sbuf, void *rbuf, int count,
if ( MPI_UNDEFINED != llroot ) { if ( MPI_UNDEFINED != llroot ) {
if ( MPI_COMM_NULL != lcomm ) { if ( MPI_COMM_NULL != lcomm ) {
ret = llcomm->c_coll.coll_allreduce (tmpbuf, rbuf, count, dtype, ret = llcomm->c_coll.coll_allreduce (tmpbuf, rbuf, count, dtype,
op, llcomm); op, llcomm,
llcomm->c_coll.coll_allreduce_module);
} }
else { else {
ret = llcomm->c_coll.coll_allreduce (sbuf, rbuf, count, dtype, ret = llcomm->c_coll.coll_allreduce (sbuf, rbuf, count, dtype,
op, llcomm); op, llcomm,
llcomm->c_coll.coll_allreduce_module);
} }
} }
if ( MPI_COMM_NULL != lcomm ) { if ( MPI_COMM_NULL != lcomm ) {
ret = lcomm->c_coll.coll_bcast(rbuf, count, dtype, lroot, lcomm ); ret = lcomm->c_coll.coll_bcast(rbuf, count, dtype, lroot, lcomm,
lcomm->c_coll.coll_bcast_module );
} }

17
ompi/mca/coll/hierarch/coll_hierarch_barrier.c
Просмотреть файл

@ -9,6 +9,7 @@
* University of Stuttgart. All rights reserved. * University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California. * Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved. * All rights reserved.
* Copyright (c) 2007 University of Houston. All rights reserved.
* $COPYRIGHT$ * $COPYRIGHT$
* *
* Additional copyrights may follow * Additional copyrights may follow
@ -34,24 +35,24 @@
* Accepts: - same as MPI_Barrier() * Accepts: - same as MPI_Barrier()
* Returns: - MPI_SUCCESS or error code * Returns: - MPI_SUCCESS or error code
*/ */
int mca_coll_hierarch_barrier_intra(struct ompi_communicator_t *comm) int mca_coll_hierarch_barrier_intra(struct ompi_communicator_t *comm,
struct mca_coll_base_module_1_1_0_t *module)
{ {
struct mca_coll_base_comm_t *data=NULL;
struct ompi_communicator_t *llcomm=NULL; struct ompi_communicator_t *llcomm=NULL;
struct ompi_communicator_t *lcomm=NULL; struct ompi_communicator_t *lcomm=NULL;
mca_coll_hierarch_module_t *hierarch_module = (mca_coll_hierarch_module_t *) module;
int root=0; int root=0;
int lroot, llroot; int lroot, llroot;
int rank, ret=OMPI_SUCCESS; int rank, ret=OMPI_SUCCESS;
rank = ompi_comm_rank ( comm ); rank = ompi_comm_rank ( comm );
data = comm->c_coll_selected_data; lcomm = hierarch_module->hier_lcomm;
lcomm = data->hier_lcomm;
if ( mca_coll_hierarch_verbose_param ) { if ( mca_coll_hierarch_verbose_param ) {
printf("%s:%d: executing hierarchical barrier\n", comm->c_name, rank ); printf("%s:%d: executing hierarchical barrier\n", comm->c_name, rank );
} }
llcomm = mca_coll_hierarch_get_llcomm ( root, data, &llroot, &lroot); llcomm = mca_coll_hierarch_get_llcomm ( root, hierarch_module, &llroot, &lroot);
/* /*
* Barrier consists of three steps: * Barrier consists of three steps:
@ -61,18 +62,18 @@ int mca_coll_hierarch_barrier_intra(struct ompi_communicator_t *comm)
* necessary to avoid that any non local leaders exit too early. * necessary to avoid that any non local leaders exit too early.
*/ */
if ( MPI_COMM_NULL != lcomm ) { if ( MPI_COMM_NULL != lcomm ) {
ret = lcomm->c_coll.coll_barrier ( lcomm ); ret = lcomm->c_coll.coll_barrier ( lcomm, lcomm->c_coll.coll_barrier_module );
if ( OMPI_SUCCESS != ret ) { if ( OMPI_SUCCESS != ret ) {
return ret; return ret;
} }
} }
if ( MPI_UNDEFINED != llroot ) { if ( MPI_UNDEFINED != llroot ) {
ret = llcomm->c_coll.coll_barrier ( llcomm ); ret = llcomm->c_coll.coll_barrier ( llcomm, llcomm->c_coll.coll_barrier_module );
} }
if ( MPI_COMM_NULL != lcomm ) { if ( MPI_COMM_NULL != lcomm ) {
ret = lcomm->c_coll.coll_barrier ( lcomm ); ret = lcomm->c_coll.coll_barrier ( lcomm, lcomm->c_coll.coll_barrier_module );
} }
return ret; return ret;

17
ompi/mca/coll/hierarch/coll_hierarch_bcast.c
Просмотреть файл

@ -9,6 +9,7 @@
* University of Stuttgart. All rights reserved. * University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California. * Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved. * All rights reserved.
* Copyright (c) 2007 University of Houston. All rights reserved.
* $COPYRIGHT$ * $COPYRIGHT$
* *
* Additional copyrights may follow * Additional copyrights may follow
@ -38,18 +39,18 @@ int mca_coll_hierarch_bcast_intra(void *buff,
int count, int count,
struct ompi_datatype_t *datatype, struct ompi_datatype_t *datatype,
int root, int root,
struct ompi_communicator_t *comm) struct ompi_communicator_t *comm,
struct mca_coll_base_module_1_1_0_t *module)
{ {
struct mca_coll_base_comm_t *data=NULL;
struct ompi_communicator_t *llcomm=NULL; struct ompi_communicator_t *llcomm=NULL;
struct ompi_communicator_t *lcomm=NULL; struct ompi_communicator_t *lcomm=NULL;
mca_coll_hierarch_module_t *hierarch_module = (mca_coll_hierarch_module_t *) module;
int lroot, llroot; int lroot, llroot;
int rank, ret=OMPI_SUCCESS; int rank, ret=OMPI_SUCCESS;
rank = ompi_comm_rank ( comm ); rank = ompi_comm_rank ( comm );
data = comm->c_coll_selected_data; lcomm = hierarch_module->hier_lcomm;
lcomm = data->hier_lcomm;
if ( mca_coll_hierarch_verbose_param ) { if ( mca_coll_hierarch_verbose_param ) {
printf("%s:%d: executing hierarchical bcast with cnt=%d and root=%d\n", printf("%s:%d: executing hierarchical bcast with cnt=%d and root=%d\n",
@ -63,11 +64,12 @@ int mca_coll_hierarch_bcast_intra(void *buff,
* also the reason, that *every* process in comm has to call * also the reason, that *every* process in comm has to call
* this function * this function
*/ */
llcomm = mca_coll_hierarch_get_llcomm ( root, data, &llroot, &lroot); llcomm = mca_coll_hierarch_get_llcomm ( root, hierarch_module, &llroot, &lroot);
/* Bcast on the upper level among the local leaders */ /* Bcast on the upper level among the local leaders */
if ( MPI_UNDEFINED != llroot ) { if ( MPI_UNDEFINED != llroot ) {
ret = llcomm->c_coll.coll_bcast(buff, count, datatype, llroot, llcomm); ret = llcomm->c_coll.coll_bcast(buff, count, datatype, llroot, llcomm,
llcomm->c_coll.coll_bcast_module);
if ( OMPI_SUCCESS != ret ) { if ( OMPI_SUCCESS != ret ) {
return ret; return ret;
} }
@ -77,7 +79,8 @@ int mca_coll_hierarch_bcast_intra(void *buff,
* it to the processes in their local, low-leve communicator. * it to the processes in their local, low-leve communicator.
*/ */
if ( MPI_COMM_NULL != lcomm ) { if ( MPI_COMM_NULL != lcomm ) {
ret = lcomm->c_coll.coll_bcast(buff, count, datatype, lroot, lcomm ); ret = lcomm->c_coll.coll_bcast(buff, count, datatype, lroot, lcomm,
lcomm->c_coll.coll_bcast_module );
} }
return ret; return ret;

65
ompi/mca/coll/hierarch/coll_hierarch_component.c
Просмотреть файл

@ -9,6 +9,7 @@
* University of Stuttgart. All rights reserved. * University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California. * Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved. * All rights reserved.
* Copyright (c) 2007 University of Houston. All rights reserved.
* $COPYRIGHT$ * $COPYRIGHT$
* *
* Additional copyrights may follow * Additional copyrights may follow
@ -26,6 +27,7 @@
#include "coll_hierarch.h" #include "coll_hierarch.h"
#include "mpi.h" #include "mpi.h"
#include "ompi/communicator/communicator.h"
#include "ompi/mca/coll/coll.h" #include "ompi/mca/coll/coll.h"
/* /*
@ -54,16 +56,16 @@ static int hierarch_open(void);
* and pointers to our public functions in it * and pointers to our public functions in it
*/ */
const mca_coll_base_component_1_0_0_t mca_coll_hierarch_component = { const mca_coll_base_component_1_1_0_t mca_coll_hierarch_component = {
/* First, the mca_component_t struct containing meta information /* First, the mca_component_t struct containing meta information
about the component itself */ about the component itself */
{ {
/* Indicate that we are a coll v1.0.0 component (which also implies a /* Indicate that we are a coll v1.1.0 component (which also implies a
specific MCA version) */ specific MCA version) */
MCA_COLL_BASE_VERSION_1_0_0, MCA_COLL_BASE_VERSION_1_1_0,
/* Component name and version */ /* Component name and version */
@ -88,7 +90,6 @@ const mca_coll_base_component_1_0_0_t mca_coll_hierarch_component = {
/* Initialization / querying functions */ /* Initialization / querying functions */
mca_coll_hierarch_init_query, mca_coll_hierarch_init_query,
mca_coll_hierarch_comm_query, mca_coll_hierarch_comm_query,
mca_coll_hierarch_comm_unquery
}; };
@ -133,3 +134,59 @@ static int hierarch_open(void)
return OMPI_SUCCESS; return OMPI_SUCCESS;
} }
static void
mca_coll_hierarch_module_construct(mca_coll_hierarch_module_t *module)
{
module->hier_lcomm = MPI_COMM_NULL;
module->hier_reqs = NULL;
/* module->hier_llead = (ompi_pointer_array_t ) NULL; */
module->hier_colorarr = NULL;
module->hier_llr = NULL;
return;
}
static void
mca_coll_hierarch_module_destruct(mca_coll_hierarch_module_t *hierarch_module)
{
int i, size;
struct mca_coll_hierarch_llead_t *current=NULL;
if ( MPI_COMM_NULL != hierarch_module->hier_lcomm ) {
ompi_comm_free (&(hierarch_module->hier_lcomm) );
}
if ( NULL != hierarch_module->hier_reqs ) {
free ( hierarch_module->hier_reqs );
}
/* if ( NULL != hierarch_module->hier_llead ) { */
size = ompi_pointer_array_get_size ( &(hierarch_module->hier_llead));
for ( i=0; i<size; i++) {
current = (struct mca_coll_hierarch_llead_t *)ompi_pointer_array_get_item (
&(hierarch_module->hier_llead), i ) ;
if ( current->lleaders != NULL ) {
ompi_comm_free ( &(current->llcomm));
free ( current->lleaders );
}
free ( current );
}
ompi_pointer_array_remove_all ( &(hierarch_module->hier_llead));
OBJ_DESTRUCT (&(hierarch_module->hier_llead));
/* } */
if ( NULL != hierarch_module->hier_colorarr ) {
free ( hierarch_module->hier_colorarr );
}
if ( NULL != hierarch_module->hier_llr ) {
free ( hierarch_module->hier_llr);
}
return;
}
OBJ_CLASS_INSTANCE(mca_coll_hierarch_module_t,
mca_coll_base_module_1_1_0_t,
mca_coll_hierarch_module_construct,
mca_coll_hierarch_module_destruct);

23
ompi/mca/coll/hierarch/coll_hierarch_reduce.c
Просмотреть файл

@ -9,6 +9,7 @@
* University of Stuttgart. All rights reserved. * University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California. * Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved. * All rights reserved.
* Copyright (c) 2007 University of Houston. All rights reserved.
* $COPYRIGHT$ * $COPYRIGHT$
* *
* Additional copyrights may follow * Additional copyrights may follow
@ -39,11 +40,12 @@
int mca_coll_hierarch_reduce_intra(void *sbuf, void *rbuf, int count, int mca_coll_hierarch_reduce_intra(void *sbuf, void *rbuf, int count,
struct ompi_datatype_t *dtype, struct ompi_datatype_t *dtype,
struct ompi_op_t *op, struct ompi_op_t *op,
int root, struct ompi_communicator_t *comm) int root, struct ompi_communicator_t *comm,
struct mca_coll_base_module_1_1_0_t *module)
{ {
struct mca_coll_base_comm_t *data=NULL;
struct ompi_communicator_t *llcomm=NULL; struct ompi_communicator_t *llcomm=NULL;
struct ompi_communicator_t *lcomm=NULL; struct ompi_communicator_t *lcomm=NULL;
mca_coll_hierarch_module_t *hierarch_module = (mca_coll_hierarch_module_t *) module;
int rank; int rank;
int lroot, llroot; int lroot, llroot;
ptrdiff_t extent, true_extent, lb, true_lb; ptrdiff_t extent, true_extent, lb, true_lb;
@ -51,15 +53,14 @@ int mca_coll_hierarch_reduce_intra(void *sbuf, void *rbuf, int count,
int ret=OMPI_SUCCESS; int ret=OMPI_SUCCESS;
rank = ompi_comm_rank ( comm ); rank = ompi_comm_rank ( comm );
data = comm->c_coll_selected_data; lcomm = hierarch_module->hier_lcomm;
lcomm = data->hier_lcomm;
if ( mca_coll_hierarch_verbose_param ) { if ( mca_coll_hierarch_verbose_param ) {
printf("%s:%d: executing hierarchical reduce with cnt=%d and root=%d\n", printf("%s:%d: executing hierarchical reduce with cnt=%d and root=%d\n",
comm->c_name, rank, count, root ); comm->c_name, rank, count, root );
} }
llcomm = mca_coll_hierarch_get_llcomm ( root, data, &llroot, &lroot); llcomm = mca_coll_hierarch_get_llcomm ( root, hierarch_module, &llroot, &lroot);
if ( MPI_COMM_NULL != lcomm ) { if ( MPI_COMM_NULL != lcomm ) {
ompi_ddt_get_extent(dtype, &lb, &extent); ompi_ddt_get_extent(dtype, &lb, &extent);
@ -74,11 +75,13 @@ int mca_coll_hierarch_reduce_intra(void *sbuf, void *rbuf, int count,
if ( MPI_IN_PLACE != sbuf ) { if ( MPI_IN_PLACE != sbuf ) {
ret = lcomm->c_coll.coll_reduce (sbuf, tmpbuf, count, dtype, ret = lcomm->c_coll.coll_reduce (sbuf, tmpbuf, count, dtype,
op, lroot, lcomm); op, lroot, lcomm,
lcomm->c_coll.coll_reduce_module);
} }
else { else {
ret = lcomm->c_coll.coll_reduce (rbuf, tmpbuf, count, dtype, ret = lcomm->c_coll.coll_reduce (rbuf, tmpbuf, count, dtype,
op, lroot, lcomm); op, lroot, lcomm,
lcomm->c_coll.coll_reduce_module);
} }
if ( OMPI_SUCCESS != ret ) { if ( OMPI_SUCCESS != ret ) {
goto exit; goto exit;
@ -88,11 +91,13 @@ int mca_coll_hierarch_reduce_intra(void *sbuf, void *rbuf, int count,
if ( MPI_UNDEFINED != llroot ) { if ( MPI_UNDEFINED != llroot ) {
if ( MPI_COMM_NULL != lcomm ) { if ( MPI_COMM_NULL != lcomm ) {
ret = llcomm->c_coll.coll_reduce (tmpbuf, rbuf, count, dtype, ret = llcomm->c_coll.coll_reduce (tmpbuf, rbuf, count, dtype,
op, llroot, llcomm); op, llroot, llcomm,
lcomm->c_coll.coll_reduce_module);
} }
else { else {
ret = llcomm->c_coll.coll_reduce (sbuf, rbuf, count, dtype, ret = llcomm->c_coll.coll_reduce (sbuf, rbuf, count, dtype,
op, llroot, llcomm); op, llroot, llcomm,
lcomm->c_coll.coll_reduce_module);
} }
} }