diff --git a/ompi/mca/coll/tuned/Makefile.am b/ompi/mca/coll/tuned/Makefile.am index 05e955603b..67d0f8a7c2 100644 --- a/ompi/mca/coll/tuned/Makefile.am +++ b/ompi/mca/coll/tuned/Makefile.am @@ -27,6 +27,9 @@ sources = \ coll_tuned_alltoall_decision_fixed.c \ coll_tuned_alltoall_decision_dynamic.c \ coll_tuned_alltoall.c \ + coll_tuned_barrier_decision_fixed.c \ + coll_tuned_barrier_decision_dynamic.c \ + coll_tuned_barrier.c \ coll_tuned_bcast_decision_fixed.c \ coll_tuned_bcast_decision_dynamic.c \ coll_tuned_bcast.c \ diff --git a/ompi/mca/coll/tuned/coll_tuned.h b/ompi/mca/coll/tuned/coll_tuned.h index 20dc755261..0bd0f80797 100644 --- a/ompi/mca/coll/tuned/coll_tuned.h +++ b/ompi/mca/coll/tuned/coll_tuned.h @@ -230,6 +230,13 @@ OMPI_COMP_EXPORT extern int mca_coll_tuned_priority_param; int mca_coll_tuned_barrier_inter_dec_dynamic( struct ompi_communicator_t *comm); + + int mca_coll_tuned_barrier_intra_doublering(struct ompi_communicator_t *comm); + int mca_coll_tuned_barrier_intra_recursivedoubling(struct ompi_communicator_t *comm); + int mca_coll_tuned_barrier_intra_bruck(struct ompi_communicator_t *comm); + int mca_coll_tuned_barrier_intra_two_procs(struct ompi_communicator_t *comm); + int mca_coll_tuned_barrier_intra_linear(struct ompi_communicator_t *comm); + int mca_coll_tuned_bcast_intra_dec_fixed(void *buff, int count, struct ompi_datatype_t *datatype, diff --git a/ompi/mca/coll/tuned/coll_tuned_alltoall_decision_fixed.c b/ompi/mca/coll/tuned/coll_tuned_alltoall_decision_fixed.c index cbe0d9c2d5..f64376b914 100644 --- a/ompi/mca/coll/tuned/coll_tuned_alltoall_decision_fixed.c +++ b/ompi/mca/coll/tuned/coll_tuned_alltoall_decision_fixed.c @@ -58,8 +58,8 @@ int mca_coll_tuned_alltoall_intra_dec_fixed(void *sbuf, int scount, return mca_coll_tuned_alltoall_intra_two_procs (sbuf, scount, sdtype, rbuf, rcount, rdtype, comm); } else { - return mca_coll_tuned_alltoall_intra_pairwise (sbuf, scount, sdtype, rbuf, rcount, rdtype, comm); -/* return mca_coll_tuned_alltoall_intra_bruck (sbuf, scount, sdtype, rbuf, rcount, rdtype, comm); */ +/* return mca_coll_tuned_alltoall_intra_pairwise (sbuf, scount, sdtype, rbuf, rcount, rdtype, comm); */ + return mca_coll_tuned_alltoall_intra_bruck (sbuf, scount, sdtype, rbuf, rcount, rdtype, comm); } /* return OMPI_ERR_NOT_IMPLEMENTED; */ diff --git a/ompi/mca/coll/tuned/coll_tuned_barrier.c b/ompi/mca/coll/tuned/coll_tuned_barrier.c new file mode 100644 index 0000000000..1a429d437c --- /dev/null +++ b/ompi/mca/coll/tuned/coll_tuned_barrier.c @@ -0,0 +1,229 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University. + * All rights reserved. + * Copyright (c) 2004-2005 The Trustees of the University of Tennessee. + * All rights reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "ompi_config.h" + +#include "mpi.h" +#include "ompi/include/constants.h" +#include "datatype/datatype.h" +#include "communicator/communicator.h" +#include "mca/coll/coll.h" +#include "mca/coll/base/coll_tags.h" +#include "mca/pml/pml.h" +#include "op/op.h" +#include "coll_tuned.h" +#include "coll_tuned_topo.h" +#include "coll_tuned_util.h" + +#include +#include + + +int mca_coll_tuned_barrier_intra_doublering(struct ompi_communicator_t *comm) +{ + int rank, size; + int err=0, line=0; + int left, right; + + + rank = ompi_comm_rank(comm); + size = ompi_comm_size(comm); + + printf("mca_coll_tuned_bcast_intra_doublering rank %d\n", rank); + + left = ((rank-1)%size); + right = ((rank+1)%size); + + if (rank > 0) { /* receive message from the left */ + err = MCA_PML_CALL(recv((void*)NULL, 0, MPI_BYTE, left, + MCA_COLL_BASE_TAG_BARRIER, comm, MPI_STATUS_IGNORE)); + if (err != MPI_SUCCESS) { line = __LINE__; goto err_hndl; } + } + + /* Send message to the right */ + err = MCA_PML_CALL(send((void*)NULL, 0, MPI_BYTE, right, MCA_COLL_BASE_TAG_BARRIER, + MCA_PML_BASE_SEND_STANDARD, comm)); + if (err != MPI_SUCCESS) { line = __LINE__; goto err_hndl; } + + /* root needs to receive from the last node */ + if (rank == 0) { + err = MCA_PML_CALL(recv((void*)NULL, 0, MPI_BYTE, left, + MCA_COLL_BASE_TAG_BARRIER, comm, MPI_STATUS_IGNORE)); + if (err != MPI_SUCCESS) { line = __LINE__; goto err_hndl; } + } + + /* Allow nodes to exit */ + if (rank > 0) { /* post Receive from left */ + err = MCA_PML_CALL(recv((void*)NULL, 0, MPI_BYTE, left, + MCA_COLL_BASE_TAG_BARRIER, comm, MPI_STATUS_IGNORE)); + if (err != MPI_SUCCESS) { line = __LINE__; goto err_hndl; } + } + + /* send message to the right one */ + err = MCA_PML_CALL(send((void*)NULL, 0, MPI_BYTE, right, MCA_COLL_BASE_TAG_BARRIER, + MCA_PML_BASE_SEND_STANDARD, comm)); + if (err != MPI_SUCCESS) { line = __LINE__; goto err_hndl; } + + /* rank 0 post receive from the last node */ + if (rank == 0) { + err = MCA_PML_CALL(recv((void*)NULL, 0, MPI_BYTE, left, + MCA_COLL_BASE_TAG_BARRIER, comm, MPI_STATUS_IGNORE)); + if (err != MPI_SUCCESS) { line = __LINE__; goto err_hndl; } + } + + return MPI_SUCCESS; + + err_hndl: + fprintf(stderr,"%s:%4d\tError occurred %d\n",__FILE__,line,err); + return err; +} + + +int mca_coll_tuned_barrier_intra_recursivedoubling(struct ompi_communicator_t *comm) +{ + int rank, size, adjsize; + int i, err, line; + int mask, remote; + + rank = ompi_comm_rank(comm); + size = ompi_comm_size(comm); + printf("mca_coll_tuned_bcast_intra_recursivedoubling rank %d\n", rank); + + /* do nearest power of 2 less than size calc */ + adjsize = 1; + for(i=0;adjsize*2= adjsize) { + /* send message to lower ranked node */ + err = MCA_PML_CALL(send((void*)NULL, 0, MPI_BYTE, rank-adjsize, + MCA_COLL_BASE_TAG_BARRIER, MCA_PML_BASE_SEND_STANDARD, comm)); + + if (err != MPI_SUCCESS) { line = __LINE__; goto err_hndl;} + + /* post receive from lower ranked node */ + err = MCA_PML_CALL(recv((void*)NULL, 0, MPI_BYTE, rank-adjsize, + MCA_COLL_BASE_TAG_BARRIER, comm, MPI_STATUS_IGNORE)); + + if (err != MPI_SUCCESS) { line = __LINE__; goto err_hndl;} + + } else if (rank < (size - adjsize)) { + + /* receive message from high level rank */ + err = MCA_PML_CALL(recv((void*)NULL, 0, MPI_BYTE, rank+adjsize, + MCA_COLL_BASE_TAG_BARRIER, comm, MPI_STATUS_IGNORE)); + + if (err != MPI_SUCCESS) { line = __LINE__; goto err_hndl;} + } + } + + /* exchange messages */ + if ( rank < adjsize ) { + mask = 0x1; + while ( mask < adjsize ) { + remote = rank ^ mask; + mask <<= 1; + if (remote >= adjsize) continue; + + err = coll_tuned_sendrecv (NULL, 0, MPI_BYTE, remote, MCA_COLL_BASE_TAG_BARRIER, + NULL, 0, MPI_BYTE, remote, MCA_COLL_BASE_TAG_BARRIER, + comm, MPI_STATUS_IGNORE, rank); + + if (err != MPI_SUCCESS) { line = __LINE__; goto err_hndl;} + } + } + + /* non-power of 2 case */ + if (adjsize != size) { + if (rank < (size - adjsize)) { + + /* send enter message to higher ranked node */ + err = MCA_PML_CALL(send((void*)NULL, 0, MPI_BYTE, rank+adjsize, + MCA_COLL_BASE_TAG_BARRIER, MCA_PML_BASE_SEND_STANDARD, comm)); + + if (err != MPI_SUCCESS) { line = __LINE__; goto err_hndl;} + } + } + + return MPI_SUCCESS; + + err_hndl: + fprintf(stderr,"%s:%4d\tError occurred %d\n",__FILE__,line,err); + return err; +} + + +int mca_coll_tuned_barrier_intra_bruck(struct ompi_communicator_t *comm) +{ + int rank, size; + int distance, to, from; + int err, line = 0; + + rank = ompi_comm_rank(comm); + size = ompi_comm_size(comm); + printf("mca_coll_tuned_bcast_intra_bruck rank %d\n", rank); + + /* exchange data with rank-2^k and rank+2^k */ + for (distance = 1; distance < size; distance <<= 1) { + from = (rank + size - distance)%size; + to = (rank + distance)%size; + err = coll_tuned_sendrecv (NULL, 0, MPI_BYTE, to, MCA_COLL_BASE_TAG_BARRIER, + NULL, 0, MPI_BYTE, from, MCA_COLL_BASE_TAG_BARRIER, + comm, MPI_STATUS_IGNORE, rank); + if (err != MPI_SUCCESS) { line = __LINE__; goto err_hndl;} + } + + return MPI_SUCCESS; + + err_hndl: + fprintf(stderr,"%s:%4d\tError occurred %d\n",__FILE__,line,err); + return err; +} + + +/* special case for two processes */ +int mca_coll_tuned_barrier_intra_two_procs(struct ompi_communicator_t *comm) +{ + int rank; + int err=0; + + rank = ompi_comm_rank(comm); + printf("mca_coll_tuned_bcast_intra_two_procs rank %d\n", rank); + + if (0==rank) { + err = coll_tuned_sendrecv (NULL, 0, MPI_BYTE, 1, MCA_COLL_BASE_TAG_BARRIER, + NULL, 0, MPI_BYTE, 1, MCA_COLL_BASE_TAG_BARRIER, + comm, MPI_STATUS_IGNORE, rank); + } + else { + err = coll_tuned_sendrecv (NULL, 0, MPI_BYTE, 0, MCA_COLL_BASE_TAG_BARRIER, + NULL, 0, MPI_BYTE, 0, MCA_COLL_BASE_TAG_BARRIER, + comm, MPI_STATUS_IGNORE, rank); + } + + return (err); +} + + +int mca_coll_tuned_barrier_intra_linear(struct ompi_communicator_t *comm) +{ +return OMPI_ERR_NOT_IMPLEMENTED; +} + + + + diff --git a/ompi/mca/coll/tuned/coll_tuned_barrier_decision_dynamic.c b/ompi/mca/coll/tuned/coll_tuned_barrier_decision_dynamic.c new file mode 100644 index 0000000000..8c5326b57e --- /dev/null +++ b/ompi/mca/coll/tuned/coll_tuned_barrier_decision_dynamic.c @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University. + * All rights reserved. + * Copyright (c) 2004-2005 The Trustees of the University of Tennessee. + * All rights reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "ompi_config.h" + +#include "mpi.h" +#include "include/constants.h" +#include "datatype/datatype.h" +#include "communicator/communicator.h" +#include "mca/coll/coll.h" +#include "mca/coll/base/coll_tags.h" +#include "coll_tuned.h" +#include "mca/pml/pml.h" +#include "opal/util/bit_ops.h" + +#include "coll_tuned.h" + +/* + * barrier_intra_dec + * + * Function: - seletects barrier algorithm to use + * Accepts: - same arguments as MPI_Barrier() + * Returns: - MPI_SUCCESS or error code (passed from the barrier implementation) + */ +int mca_coll_tuned_barrier_intra_dec_dynamic(struct ompi_communicator_t *comm) +{ + int i; + int size; + int rank; + int err; + int contig; + int dsize; + + printf("mca_coll_tuned_barrier_intra_dec_dynamic\n"); + + size = ompi_comm_size(comm); + rank = ompi_comm_rank(comm); + + +return OMPI_ERR_NOT_IMPLEMENTED; +} + + diff --git a/ompi/mca/coll/tuned/coll_tuned_barrier_decision_fixed.c b/ompi/mca/coll/tuned/coll_tuned_barrier_decision_fixed.c new file mode 100644 index 0000000000..67c3348a99 --- /dev/null +++ b/ompi/mca/coll/tuned/coll_tuned_barrier_decision_fixed.c @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University. + * All rights reserved. + * Copyright (c) 2004-2005 The Trustees of the University of Tennessee. + * All rights reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "ompi_config.h" + +#include "mpi.h" +#include "include/constants.h" +#include "datatype/datatype.h" +#include "communicator/communicator.h" +#include "mca/coll/coll.h" +#include "mca/coll/base/coll_tags.h" +#include "coll_tuned.h" +#include "mca/pml/pml.h" +#include "opal/util/bit_ops.h" + +#include "coll_tuned.h" + +/* + * barrier_intra_dec + * + * Function: - seletects barrier algorithm to use + * Accepts: - same arguments as MPI_Barrier() + * Returns: - MPI_SUCCESS or error code (passed from the barrier implementation) + */ +int mca_coll_tuned_barrier_intra_dec_fixed(struct ompi_communicator_t *comm) +{ + int i; + int size; + int rank; + int err; + int contig; + int dsize; + + printf("mca_coll_tuned_barrier_intra_dec_fixed\n"); + + size = ompi_comm_size(comm); + rank = ompi_comm_rank(comm); + + if (2==size) + return mca_coll_tuned_barrier_intra_two_procs(comm); + else +/* return mca_coll_tuned_barrier_intra_doublering(comm); */ + return mca_coll_tuned_barrier_intra_recursivedoubling(comm); +/* return mca_coll_tuned_barrier_intra_bruck(comm); */ +/* return mca_coll_tuned_barrier_intra_linear(comm); */ + +} + + diff --git a/ompi/mca/coll/tuned/coll_tuned_module.c b/ompi/mca/coll/tuned/coll_tuned_module.c index 34f575957e..ee69ea9850 100644 --- a/ompi/mca/coll/tuned/coll_tuned_module.c +++ b/ompi/mca/coll/tuned/coll_tuned_module.c @@ -67,8 +67,8 @@ static const mca_coll_base_module_1_0_0_t intra_fixed = { NULL, /* mca_coll_tuned_alltoallw_intra_dec_fixed, */ NULL, -/* mca_coll_tuned_barrier_intra_dec_fixed, */ - NULL, + mca_coll_tuned_barrier_intra_dec_fixed, +/* NULL, */ mca_coll_tuned_bcast_intra_dec_fixed, /* NULL, */ /* mca_coll_tuned_exscan_intra_dec_fixed, */ @@ -120,8 +120,8 @@ static const mca_coll_base_module_1_0_0_t intra_dynamic = { NULL, /* mca_coll_tuned_gatherv_intra_dec_dynamic, */ NULL, - mca_coll_tuned_reduce_intra_dec_dynamic, -/* NULL, */ +/* mca_coll_tuned_reduce_intra_dec_dynamic, */ + NULL, /* mca_coll_tuned_reduce_scatter_intra_dec_dynamic, */ NULL, /* mca_coll_tuned_scan_intra_dec_dynamic, */