From 7e45f64065783df34d40c0a6eeac5bdca8c88024 Mon Sep 17 00:00:00 2001 From: Edgar Gabriel Date: Sat, 15 Oct 2005 22:24:44 +0000 Subject: [PATCH] reduce has now been tested quite extensively for all (predefined) operations and for all root nodes and passed all tests. First cut on barrier (which from my perspective does not make sense from the performance point of view) and on allreduce (which might make sense), This commit was SVN r7774. --- ompi/mca/coll/hierarch/Makefile.am | 1 + ompi/mca/coll/hierarch/coll_hierarch.c | 26 ++--- ompi/mca/coll/hierarch/coll_hierarch.h | 1 + .../coll/hierarch/coll_hierarch_allreduce.c | 105 ++++++++++++++++++ .../mca/coll/hierarch/coll_hierarch_barrier.c | 48 +++++++- ompi/mca/coll/hierarch/coll_hierarch_bcast.c | 2 - ompi/mca/coll/hierarch/coll_hierarch_reduce.c | 1 - 7 files changed, 163 insertions(+), 21 deletions(-) create mode 100644 ompi/mca/coll/hierarch/coll_hierarch_allreduce.c diff --git a/ompi/mca/coll/hierarch/Makefile.am b/ompi/mca/coll/hierarch/Makefile.am index 99e5627694..1165ee7e17 100644 --- a/ompi/mca/coll/hierarch/Makefile.am +++ b/ompi/mca/coll/hierarch/Makefile.am @@ -45,6 +45,7 @@ libmca_coll_hierarch_la_LDFLAGS = -module -avoid-version sources = \ coll_hierarch.h \ coll_hierarch.c \ + coll_hierarch_allreduce.c \ coll_hierarch_barrier.c \ coll_hierarch_bcast.c \ coll_hierarch_component.c \ diff --git a/ompi/mca/coll/hierarch/coll_hierarch.c b/ompi/mca/coll/hierarch/coll_hierarch.c index 64219272ed..6ba5b03a7c 100644 --- a/ompi/mca/coll/hierarch/coll_hierarch.c +++ b/ompi/mca/coll/hierarch/coll_hierarch.c @@ -69,22 +69,22 @@ static const mca_coll_base_module_1_0_0_t intra = { /* Collective function pointers */ /* function pointers marked with NULL are not yet implemented and will use the functions provided in the basic module */ - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, + NULL, /* allgather */ + NULL, /* allgatherv */ + mca_coll_hierarch_allreduce_intra, + NULL, /* alltoall */ + NULL, /* alltoallv */ + NULL, /* alltoallw */ mca_coll_hierarch_barrier_intra, mca_coll_hierarch_bcast_intra, - NULL, - NULL, - NULL, + NULL, /* exscan */ + NULL, /* gather */ + NULL, /* gatherv */ mca_coll_hierarch_reduce_intra, - NULL, - NULL, - NULL, - NULL + NULL, /* reduce_scatter */ + NULL, /* scan */ + NULL, /* scatter */ + NULL /* scatterv */ }; diff --git a/ompi/mca/coll/hierarch/coll_hierarch.h b/ompi/mca/coll/hierarch/coll_hierarch.h index aaf00129e7..5bec28bc79 100644 --- a/ompi/mca/coll/hierarch/coll_hierarch.h +++ b/ompi/mca/coll/hierarch/coll_hierarch.h @@ -37,6 +37,7 @@ extern int mca_coll_hierarch_priority_param; extern int mca_coll_hierarch_verbose_param; extern int mca_coll_hierarch_use_rdma_param; extern int mca_coll_hierarch_ignore_sm_param; +extern int mca_coll_hier_verbose; #define HIER_DEFAULT_NUM_LLEAD 5 diff --git a/ompi/mca/coll/hierarch/coll_hierarch_allreduce.c b/ompi/mca/coll/hierarch/coll_hierarch_allreduce.c new file mode 100644 index 0000000000..4a6d3af236 --- /dev/null +++ b/ompi/mca/coll/hierarch/coll_hierarch_allreduce.c @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University. + * All rights reserved. + * Copyright (c) 2004-2005 The Trustees of the University of Tennessee. + * All rights reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "ompi_config.h" +#include "coll_hierarch.h" + +#include + +#include "mpi.h" +#include "ompi/include/constants.h" +#include "opal/util/output.h" +#include "communicator/communicator.h" +#include "datatype/datatype.h" +#include "mca/coll/coll.h" + + +/* + * reduce_intra + * + * Function: - reduction using two level hierarchy algorithm + * Accepts: - same as MPI_Reduce() + * Returns: - MPI_SUCCESS or error code + */ +int mca_coll_hierarch_allreduce_intra(void *sbuf, void *rbuf, int count, + struct ompi_datatype_t *dtype, + struct ompi_op_t *op, + struct ompi_communicator_t *comm) +{ + struct mca_coll_base_comm_t *data=NULL; + struct ompi_communicator_t *llcomm=NULL; + struct ompi_communicator_t *lcomm=NULL; + int rank, lrank; + int lroot, llroot; + long extent, true_extent, lb, true_lb; + char *tmpbuf=NULL, *tbuf=NULL; + int ret; + int root=0; + + rank = ompi_comm_rank ( comm ); + data = comm->c_coll_selected_data; + lcomm = data->hier_lcomm; + + if ( mca_coll_hier_verbose ) { + printf("%s:%d: executing hierarchical allreduce with cnt=%d \n", + comm->c_name, rank, count ); + } + + llcomm = mca_coll_hierarch_get_llcomm ( root, data, &llroot, &lroot); + + if ( MPI_COMM_NULL != lcomm ) { + lrank = ompi_comm_rank (lcomm); + if ( lrank == lroot ) { + ompi_ddt_get_extent(dtype, &lb, &extent); + ompi_ddt_get_true_extent(dtype, &true_lb, &true_extent); + + tbuf = malloc(true_extent + (count - 1) * extent); + if (NULL == tbuf) { + return OMPI_ERR_OUT_OF_RESOURCE; + } + tmpbuf = tbuf - lb; + } + + if ( MPI_IN_PLACE != sbuf ) { + ret = lcomm->c_coll.coll_reduce (sbuf, tmpbuf, count, dtype, + op, lroot, lcomm); + } + else { + ret = lcomm->c_coll.coll_reduce (rbuf, tmpbuf, count, dtype, + op, lroot, lcomm); + } + if ( OMPI_SUCCESS != ret ) { + goto exit; + } + } + + if ( MPI_UNDEFINED != llroot ) { + ret = llcomm->c_coll.coll_allreduce (tmpbuf, rbuf, count, dtype, + op, llcomm); + } + + if ( MPI_COMM_NULL != lcomm ) { + ret = lcomm->c_coll.coll_bcast(rbuf, count, dtype, lroot, lcomm ); + } + + + exit: + if ( NULL != tmpbuf ) { + free ( tmpbuf ); + } + + return ret; +} diff --git a/ompi/mca/coll/hierarch/coll_hierarch_barrier.c b/ompi/mca/coll/hierarch/coll_hierarch_barrier.c index 7bde9d8a72..1b8312726a 100644 --- a/ompi/mca/coll/hierarch/coll_hierarch_barrier.c +++ b/ompi/mca/coll/hierarch/coll_hierarch_barrier.c @@ -14,28 +14,66 @@ * $HEADER$ */ + #include "ompi_config.h" #include "coll_hierarch.h" #include "mpi.h" #include "ompi/include/constants.h" #include "opal/util/output.h" +#include "communicator/communicator.h" #include "mca/coll/coll.h" -#include "mca/coll/base/base.h" -#include "coll_hierarch.h" /* * barrier_intra * - * Function: - barrier using O(N) algorithm + * Function: - barrier using hierarchical algorithm * Accepts: - same as MPI_Barrier() * Returns: - MPI_SUCCESS or error code */ int mca_coll_hierarch_barrier_intra(struct ompi_communicator_t *comm) { - opal_output_verbose(10, mca_coll_base_output, "In hierarch barrier_intra"); - return MPI_SUCCESS; + struct mca_coll_base_comm_t *data=NULL; + struct ompi_communicator_t *llcomm=NULL; + struct ompi_communicator_t *lcomm=NULL; + int root=0; + int lroot, llroot; + int rank, ret; + + rank = ompi_comm_rank ( comm ); + data = comm->c_coll_selected_data; + lcomm = data->hier_lcomm; + + if ( mca_coll_hier_verbose ) { + printf("%s:%d: executing hierarchical barrier\n", comm->c_name, rank ); + } + + llcomm = mca_coll_hierarch_get_llcomm ( root, data, &llroot, &lroot); + + /* + * Barrier consists of three steps: + * - barrier on the low-level communicators + * - barrier among the local leaders + * - barrier on the low-level communicators. This step is + * necessary to avoid that any non local leaders exit too early. + */ + if ( MPI_COMM_NULL != lcomm ) { + ret = lcomm->c_coll.coll_barrier ( lcomm ); + if ( OMPI_SUCCESS != ret ) { + return ret; + } + } + + if ( MPI_UNDEFINED != llroot ) { + ret = llcomm->c_coll.coll_barrier ( llcomm ); + } + + if ( MPI_COMM_NULL != lcomm ) { + ret = lcomm->c_coll.coll_barrier ( lcomm ); + } + + return ret; } diff --git a/ompi/mca/coll/hierarch/coll_hierarch_bcast.c b/ompi/mca/coll/hierarch/coll_hierarch_bcast.c index 23a7d29511..863ffce855 100644 --- a/ompi/mca/coll/hierarch/coll_hierarch_bcast.c +++ b/ompi/mca/coll/hierarch/coll_hierarch_bcast.c @@ -31,8 +31,6 @@ * Returns: - MPI_SUCCESS or error code */ -extern int mca_coll_hier_verbose; - int mca_coll_hierarch_bcast_intra(void *buff, int count, diff --git a/ompi/mca/coll/hierarch/coll_hierarch_reduce.c b/ompi/mca/coll/hierarch/coll_hierarch_reduce.c index ad0f958c41..9a2124a8df 100644 --- a/ompi/mca/coll/hierarch/coll_hierarch_reduce.c +++ b/ompi/mca/coll/hierarch/coll_hierarch_reduce.c @@ -26,7 +26,6 @@ #include "datatype/datatype.h" #include "mca/coll/coll.h" -extern int mca_coll_hier_verbose; /* * reduce_intra