diff --git a/ompi/mca/coll/sync/.windows b/ompi/mca/coll/sync/.windows new file mode 100644 index 0000000000..55362000a1 --- /dev/null +++ b/ompi/mca/coll/sync/.windows @@ -0,0 +1,14 @@ +# +# Copyright (c) 2008 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +# Specific to this module + +mca_dependencies=libmpi + diff --git a/ompi/mca/coll/sync/Makefile.am b/ompi/mca/coll/sync/Makefile.am new file mode 100644 index 0000000000..0d6547ab7f --- /dev/null +++ b/ompi/mca/coll/sync/Makefile.am @@ -0,0 +1,55 @@ +# +# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana +# University Research and Technology +# Corporation. All rights reserved. +# Copyright (c) 2004-2005 The University of Tennessee and The University +# of Tennessee Research Foundation. All rights +# reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# Copyright (c) 2009 Cisco Systems, Inc. All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +sources = \ + coll_sync.h \ + coll_sync_component.c \ + coll_sync_module.c \ + coll_sync_allgather.c \ + coll_sync_allgatherv.c \ + coll_sync_allreduce.c \ + coll_sync_alltoall.c \ + coll_sync_alltoallv.c \ + coll_sync_alltoallw.c \ + coll_sync_bcast.c \ + coll_sync_exscan.c \ + coll_sync_gather.c \ + coll_sync_gatherv.c \ + coll_sync_reduce.c \ + coll_sync_reduce_scatter.c \ + coll_sync_scan.c \ + coll_sync_scatter.c \ + coll_sync_scatterv.c + +if OMPI_BUILD_coll_sync_DSO +component_noinst = +component_install = mca_coll_sync.la +else +component_noinst = libmca_coll_sync.la +component_install = +endif + +mcacomponentdir = $(pkglibdir) +mcacomponent_LTLIBRARIES = $(component_install) +mca_coll_sync_la_SOURCES = $(sources) +mca_coll_sync_la_LDFLAGS = -module -avoid-version + +noinst_LTLIBRARIES = $(component_noinst) +libmca_coll_sync_la_SOURCES =$(sources) +libmca_coll_sync_la_LDFLAGS = -module -avoid-version diff --git a/ompi/mca/coll/sync/coll_sync.h b/ompi/mca/coll/sync/coll_sync.h new file mode 100644 index 0000000000..f6ac2c3d73 --- /dev/null +++ b/ompi/mca/coll/sync/coll_sync.h @@ -0,0 +1,229 @@ +/* + * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2006 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2008-2009 Cisco Systems, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef MCA_COLL_SYNC_EXPORT_H +#define MCA_COLL_SYNC_EXPORT_H + +#include "ompi_config.h" + +#include "mpi.h" + +#include "opal/class/opal_object.h" +#include "opal/mca/mca.h" + +#include "ompi/constants.h" +#include "ompi/mca/coll/coll.h" +#include "ompi/mca/coll/base/base.h" +#include "ompi/communicator/communicator.h" + +BEGIN_C_DECLS + +/* API functions */ + +int mca_coll_sync_init_query(bool enable_progress_threads, + bool enable_mpi_threads); +mca_coll_base_module_t +*mca_coll_sync_comm_query(struct ompi_communicator_t *comm, + int *priority); + +int mca_coll_sync_module_enable(mca_coll_base_module_t *module, + struct ompi_communicator_t *comm); + +int mca_coll_sync_allgather(void *sbuf, int scount, + struct ompi_datatype_t *sdtype, + void *rbuf, int rcount, + struct ompi_datatype_t *rdtype, + struct ompi_communicator_t *comm, + mca_coll_base_module_t *module); + +int mca_coll_sync_allgatherv(void *sbuf, int scount, + struct ompi_datatype_t *sdtype, + void *rbuf, int *rcounts, + int *disps, + struct ompi_datatype_t *rdtype, + struct ompi_communicator_t *comm, + mca_coll_base_module_t *module); + +int mca_coll_sync_allreduce(void *sbuf, void *rbuf, int count, + struct ompi_datatype_t *dtype, + struct ompi_op_t *op, + struct ompi_communicator_t *comm, + mca_coll_base_module_t *module); + +int mca_coll_sync_alltoall(void *sbuf, int scount, + struct ompi_datatype_t *sdtype, + void *rbuf, int rcount, + struct ompi_datatype_t *rdtype, + struct ompi_communicator_t *comm, + mca_coll_base_module_t *module); + +int mca_coll_sync_alltoallv(void *sbuf, int *scounts, + int *sdisps, + struct ompi_datatype_t *sdtype, + void *rbuf, int *rcounts, + int *rdisps, + struct ompi_datatype_t *rdtype, + struct ompi_communicator_t *comm, + mca_coll_base_module_t *module); + +int mca_coll_sync_alltoallw(void *sbuf, int *scounts, + int *sdisps, + struct ompi_datatype_t **sdtypes, + void *rbuf, int *rcounts, + int *rdisps, + struct ompi_datatype_t **rdtypes, + struct ompi_communicator_t *comm, + mca_coll_base_module_t *module); + +int mca_coll_sync_barrier(struct ompi_communicator_t *comm, + mca_coll_base_module_t *module); + +int mca_coll_sync_bcast(void *buff, int count, + struct ompi_datatype_t *datatype, + int root, + struct ompi_communicator_t *comm, + mca_coll_base_module_t *module); + +int mca_coll_sync_exscan(void *sbuf, void *rbuf, int count, + struct ompi_datatype_t *dtype, + struct ompi_op_t *op, + struct ompi_communicator_t *comm, + mca_coll_base_module_t *module); + +int mca_coll_sync_gather(void *sbuf, int scount, + struct ompi_datatype_t *sdtype, + void *rbuf, int rcount, + struct ompi_datatype_t *rdtype, + int root, + struct ompi_communicator_t *comm, + mca_coll_base_module_t *module); + +int mca_coll_sync_gatherv(void *sbuf, int scount, + struct ompi_datatype_t *sdtype, + void *rbuf, int *rcounts, int *disps, + struct ompi_datatype_t *rdtype, + int root, + struct ompi_communicator_t *comm, + mca_coll_base_module_t *module); + +int mca_coll_sync_reduce(void *sbuf, void *rbuf, int count, + struct ompi_datatype_t *dtype, + struct ompi_op_t *op, + int root, + struct ompi_communicator_t *comm, + mca_coll_base_module_t *module); + +int mca_coll_sync_reduce_scatter(void *sbuf, void *rbuf, + int *rcounts, + struct ompi_datatype_t *dtype, + struct ompi_op_t *op, + struct ompi_communicator_t *comm, + mca_coll_base_module_t *module); + +int mca_coll_sync_scan(void *sbuf, void *rbuf, int count, + struct ompi_datatype_t *dtype, + struct ompi_op_t *op, + struct ompi_communicator_t *comm, + mca_coll_base_module_t *module); + +int mca_coll_sync_scatter(void *sbuf, int scount, + struct ompi_datatype_t *sdtype, + void *rbuf, int rcount, + struct ompi_datatype_t *rdtype, + int root, + struct ompi_communicator_t *comm, + mca_coll_base_module_t *module); + +int mca_coll_sync_scatterv(void *sbuf, int *scounts, int *disps, + struct ompi_datatype_t *sdtype, + void *rbuf, int rcount, + struct ompi_datatype_t *rdtype, + int root, + struct ompi_communicator_t *comm, + mca_coll_base_module_t *module); + +int mca_coll_sync_ft_event(int status); + +/* Types */ +/* Module */ + +typedef struct mca_coll_sync_module_t { + mca_coll_base_module_t super; + + /* Pointers to all the "real" collective functions */ + mca_coll_base_comm_coll_t c_coll; + + /* How many ops we've executed */ + int before_num_operations; + + /* How many ops we've executed (it's easier to have 2) */ + int after_num_operations; + + /* Avoid recursion of syncs */ + bool in_operation; +} mca_coll_sync_module_t; + +OBJ_CLASS_DECLARATION(mca_coll_sync_module_t); + +/* Component */ + +typedef struct mca_coll_sync_component_t { + mca_coll_base_component_2_0_0_t super; + + /* Priority of this component */ + int priority; + + /* Do a sync *before* each Nth collective */ + int barrier_before_nops; + + /* Do a sync *after* each Nth collective */ + int barrier_after_nops; +} mca_coll_sync_component_t; + +/* Globally exported variables */ + +OMPI_MODULE_DECLSPEC extern mca_coll_sync_component_t mca_coll_sync_component; + +/* Macro used in most of the collectives */ + +#define COLL_SYNC(module, op) \ +do { \ + int err = MPI_SUCCESS; \ + s->in_operation = true; \ + if (OPAL_UNLIKELY(++s->before_num_operations == \ + mca_coll_sync_component.barrier_before_nops)) { \ + s->before_num_operations = 0; \ + err = s->c_coll.coll_barrier(comm, s->c_coll.coll_barrier_module); \ + } \ + if (OPAL_LIKELY(MPI_SUCCESS == err)) { \ + err = op; \ + } \ + if (OPAL_UNLIKELY(++s->after_num_operations == \ + mca_coll_sync_component.barrier_after_nops) && \ + OPAL_LIKELY(MPI_SUCCESS == err)) { \ + s->after_num_operations = 0; \ + err = s->c_coll.coll_barrier(comm, s->c_coll.coll_barrier_module); \ + } \ + s->in_operation = false; \ + return err; \ +} while(0) + +END_C_DECLS + +#endif /* MCA_COLL_SYNC_EXPORT_H */ diff --git a/ompi/mca/coll/sync/coll_sync_allgather.c b/ompi/mca/coll/sync/coll_sync_allgather.c new file mode 100644 index 0000000000..9d5c239ec2 --- /dev/null +++ b/ompi/mca/coll/sync/coll_sync_allgather.c @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2006 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2009 Cisco Systems, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "ompi_config.h" + +#include "coll_sync.h" + + +/* + * allgather + * + * Function: - allgather using other MPI collections + * Accepts: - same as MPI_Allgather() + * Returns: - MPI_SUCCESS or error code + */ +int mca_coll_sync_allgather(void *sbuf, int scount, + struct ompi_datatype_t *sdtype, void *rbuf, + int rcount, struct ompi_datatype_t *rdtype, + struct ompi_communicator_t *comm, + mca_coll_base_module_t *module) +{ + mca_coll_sync_module_t *s = (mca_coll_sync_module_t*) module; + + if (s->in_operation) { + return s->c_coll.coll_allgather(sbuf, scount, sdtype, rbuf, rcount, + rdtype, comm, + s->c_coll.coll_allgather_module); + } else { + COLL_SYNC(s, s->c_coll.coll_allgather(sbuf, scount, sdtype, rbuf, + rcount, rdtype, comm, + s->c_coll.coll_allgather_module)); + } +} diff --git a/ompi/mca/coll/sync/coll_sync_allgatherv.c b/ompi/mca/coll/sync/coll_sync_allgatherv.c new file mode 100644 index 0000000000..cd2145d310 --- /dev/null +++ b/ompi/mca/coll/sync/coll_sync_allgatherv.c @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2006 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2009 Cisco Systems, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "ompi_config.h" + +#include "coll_sync.h" + + +/* + * allgatherv + * + * Function: - allgatherv + * Accepts: - same as MPI_Allgatherv() + * Returns: - MPI_SUCCESS or error code + */ +int mca_coll_sync_allgatherv(void *sbuf, int scount, + struct ompi_datatype_t *sdtype, + void *rbuf, int *rcounts, int *disps, + struct ompi_datatype_t *rdtype, + struct ompi_communicator_t *comm, + mca_coll_base_module_t *module) +{ + mca_coll_sync_module_t *s = (mca_coll_sync_module_t*) module; + + if (s->in_operation) { + return s->c_coll.coll_allgatherv(sbuf, scount, sdtype, rbuf, rcounts, + disps, rdtype, comm, + s->c_coll.coll_allgatherv_module); + } else { + COLL_SYNC(s, s->c_coll.coll_allgatherv(sbuf, scount, sdtype, + rbuf, rcounts, + disps, rdtype, comm, + s->c_coll.coll_allgatherv_module)); + } +} diff --git a/ompi/mca/coll/sync/coll_sync_allreduce.c b/ompi/mca/coll/sync/coll_sync_allreduce.c new file mode 100644 index 0000000000..e68d8bda6a --- /dev/null +++ b/ompi/mca/coll/sync/coll_sync_allreduce.c @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2009 Cisco Systems, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "ompi_config.h" + +#include "coll_sync.h" + + +/* + * allreduce + * + * Function: - allreduce + * Accepts: - same as MPI_Allreduce() + * Returns: - MPI_SUCCESS or error code + */ +int mca_coll_sync_allreduce(void *sbuf, void *rbuf, int count, + struct ompi_datatype_t *dtype, + struct ompi_op_t *op, + struct ompi_communicator_t *comm, + mca_coll_base_module_t *module) +{ + mca_coll_sync_module_t *s = (mca_coll_sync_module_t*) module; + + if (s->in_operation) { + return s->c_coll.coll_allreduce(sbuf, rbuf, count, dtype, op, comm, + s->c_coll.coll_allreduce_module); + } else { + COLL_SYNC(s, s->c_coll.coll_allreduce(sbuf, rbuf, count, dtype, + op, comm, + s->c_coll.coll_allreduce_module)); + } +} diff --git a/ompi/mca/coll/sync/coll_sync_alltoall.c b/ompi/mca/coll/sync/coll_sync_alltoall.c new file mode 100644 index 0000000000..e2fa3e8c58 --- /dev/null +++ b/ompi/mca/coll/sync/coll_sync_alltoall.c @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2009 Cisco Systems, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "ompi_config.h" + +#include "coll_sync.h" + + +/* + * alltoall_intra + * + * Function: - MPI_Alltoall + * Accepts: - same as MPI_Alltoall() + * Returns: - MPI_SUCCESS or an MPI error code + */ +int mca_coll_sync_alltoall(void *sbuf, int scount, + struct ompi_datatype_t *sdtype, + void *rbuf, int rcount, + struct ompi_datatype_t *rdtype, + struct ompi_communicator_t *comm, + mca_coll_base_module_t *module) +{ + mca_coll_sync_module_t *s = (mca_coll_sync_module_t*) module; + + if (s->in_operation) { + return s->c_coll.coll_alltoall(sbuf, scount, sdtype, rbuf, rcount, + rdtype, comm, + s->c_coll.coll_alltoall_module); + } else { + COLL_SYNC(s, s->c_coll.coll_alltoall(sbuf, scount, sdtype, + rbuf, rcount, + rdtype, comm, + s->c_coll.coll_alltoall_module)); + } +} diff --git a/ompi/mca/coll/sync/coll_sync_alltoallv.c b/ompi/mca/coll/sync/coll_sync_alltoallv.c new file mode 100644 index 0000000000..21fae56e86 --- /dev/null +++ b/ompi/mca/coll/sync/coll_sync_alltoallv.c @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2009 Cisco Systems, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "ompi_config.h" + +#include "coll_sync.h" + + +/* + * alltoallv + * + * Function: - MPI_Alltoallv + * Accepts: - same as MPI_Alltoallv() + * Returns: - MPI_SUCCESS or an MPI error code + */ +int mca_coll_sync_alltoallv(void *sbuf, int *scounts, int *sdisps, + struct ompi_datatype_t *sdtype, + void *rbuf, int *rcounts, int *rdisps, + struct ompi_datatype_t *rdtype, + struct ompi_communicator_t *comm, + mca_coll_base_module_t *module) +{ + mca_coll_sync_module_t *s = (mca_coll_sync_module_t*) module; + + if (s->in_operation) { + return s->c_coll.coll_alltoallv(sbuf, scounts, sdisps, sdtype, + rbuf, rcounts, rdisps, rdtype, comm, + s->c_coll.coll_alltoallv_module); + } else { + COLL_SYNC(s, s->c_coll.coll_alltoallv(sbuf, scounts, sdisps, sdtype, + rbuf, rcounts, rdisps, rdtype, + comm, + s->c_coll.coll_alltoallv_module)); + } +} diff --git a/ompi/mca/coll/sync/coll_sync_alltoallw.c b/ompi/mca/coll/sync/coll_sync_alltoallw.c new file mode 100644 index 0000000000..9fe04749f5 --- /dev/null +++ b/ompi/mca/coll/sync/coll_sync_alltoallw.c @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2009 Cisco Systems, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "ompi_config.h" + +#include "coll_sync.h" + + +/* + * alltoallw + * + * Function: - MPI_Alltoallw + * Accepts: - same as MPI_Alltoallw() + * Returns: - MPI_SUCCESS or an MPI error code + */ +int mca_coll_sync_alltoallw(void *sbuf, int *scounts, int *sdisps, + struct ompi_datatype_t **sdtypes, + void *rbuf, int *rcounts, int *rdisps, + struct ompi_datatype_t **rdtypes, + struct ompi_communicator_t *comm, + mca_coll_base_module_t *module) +{ + mca_coll_sync_module_t *s = (mca_coll_sync_module_t*) module; + + if (s->in_operation) { + return s->c_coll.coll_alltoallw(sbuf, scounts, sdisps, sdtypes, + rbuf, rcounts, rdisps, rdtypes, comm, + s->c_coll.coll_alltoallw_module); + } else { + COLL_SYNC(s, s->c_coll.coll_alltoallw(sbuf, scounts, sdisps, sdtypes, + rbuf, rcounts, rdisps, rdtypes, + comm, + s->c_coll.coll_alltoallw_module)); + } +} diff --git a/ompi/mca/coll/sync/coll_sync_bcast.c b/ompi/mca/coll/sync/coll_sync_bcast.c new file mode 100644 index 0000000000..696f78dbc6 --- /dev/null +++ b/ompi/mca/coll/sync/coll_sync_bcast.c @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2009 Cisco Systems, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "ompi_config.h" + +#include "mpi.h" +#include "coll_sync.h" + + +/* + * bcast + * + * Function: - broadcast + * Accepts: - same arguments as MPI_Bcast() + * Returns: - MPI_SUCCESS or error code + */ +int mca_coll_sync_bcast(void *buff, int count, + struct ompi_datatype_t *datatype, int root, + struct ompi_communicator_t *comm, + mca_coll_base_module_t *module) +{ + mca_coll_sync_module_t *s = (mca_coll_sync_module_t*) module; + + if (s->in_operation) { + return s->c_coll.coll_bcast(buff, count, datatype, root, comm, + s->c_coll.coll_bcast_module); + } else { + COLL_SYNC(s, s->c_coll.coll_bcast(buff, count, datatype, root, comm, + s->c_coll.coll_bcast_module)); + } +} diff --git a/ompi/mca/coll/sync/coll_sync_component.c b/ompi/mca/coll/sync/coll_sync_component.c new file mode 100644 index 0000000000..381fe71551 --- /dev/null +++ b/ompi/mca/coll/sync/coll_sync_component.c @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2008-2009 Cisco Systems, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "ompi_config.h" + +#include + +#include "mpi.h" +#include "ompi/constants.h" +#include "ompi/mca/coll/coll.h" +#include "coll_sync.h" + +/* + * Public string showing the coll ompi_sync component version number + */ +const char *mca_coll_sync_component_version_string = + "Open MPI sync collective MCA component version " OMPI_VERSION; + +/* + * Local function + */ +static int sync_register(void); + +/* + * Instantiate the public struct with all of our public information + * and pointers to our public functions in it + */ + +mca_coll_sync_component_t mca_coll_sync_component = { + { + /* First, the mca_component_t struct containing meta information + * about the component itself */ + + { + MCA_COLL_BASE_VERSION_2_0_0, + + /* Component name and version */ + "sync", + OMPI_MAJOR_VERSION, + OMPI_MINOR_VERSION, + OMPI_RELEASE_VERSION, + + /* Component open and close functions */ + NULL, + NULL, + NULL, + sync_register + }, + { + /* The component is checkpoint ready */ + MCA_BASE_METADATA_PARAM_CHECKPOINT + }, + + /* Initialization / querying functions */ + + mca_coll_sync_init_query, + mca_coll_sync_comm_query + }, + + /* Sync-specific component information */ + + /* Priority: use a low priority, but allow others to be lower */ + 50, + + /* Do a sync *before* each Nth collective */ + 0, + + /* Do a sync *after* each Nth collective */ + 0, +}; + + +static int sync_register(void) +{ + int val; + + val = mca_coll_sync_component.priority; + mca_base_param_reg_int(&mca_coll_sync_component.super.collm_version, + "priority", + "Priority of the sync coll component; only relevant if barrier_before or barrier_after is > 0", + false, false, val, &val); + mca_coll_sync_component.priority = val; + + mca_base_param_reg_int(&mca_coll_sync_component.super.collm_version, + "barrier_before", + "Do a synchronization before each Nth collective", + false, false, + mca_coll_sync_component.barrier_before_nops, + &mca_coll_sync_component.barrier_before_nops); + + mca_base_param_reg_int(&mca_coll_sync_component.super.collm_version, + "barrier_after", + "Do a synchronization after each Nth collective", + false, false, + mca_coll_sync_component.barrier_after_nops, + &mca_coll_sync_component.barrier_after_nops); + + return OMPI_SUCCESS; +} diff --git a/ompi/mca/coll/sync/coll_sync_exscan.c b/ompi/mca/coll/sync/coll_sync_exscan.c new file mode 100644 index 0000000000..259847b607 --- /dev/null +++ b/ompi/mca/coll/sync/coll_sync_exscan.c @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2006 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2009 Cisco Systems, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "ompi_config.h" + +#include "coll_sync.h" + + +/* + * exscan + * + * Function: - exscan + * Accepts: - same arguments as MPI_Exscan() + * Returns: - MPI_SUCCESS or error code + */ +int mca_coll_sync_exscan(void *sbuf, void *rbuf, int count, + struct ompi_datatype_t *dtype, + struct ompi_op_t *op, + struct ompi_communicator_t *comm, + mca_coll_base_module_t *module) +{ + mca_coll_sync_module_t *s = (mca_coll_sync_module_t*) module; + + if (s->in_operation) { + return s->c_coll.coll_exscan(sbuf, rbuf, count, dtype, op, comm, + s->c_coll.coll_exscan_module); + } else { + COLL_SYNC(s, s->c_coll.coll_exscan(sbuf, rbuf, count, dtype, op, comm, + s->c_coll.coll_exscan_module)); + } +} diff --git a/ompi/mca/coll/sync/coll_sync_gather.c b/ompi/mca/coll/sync/coll_sync_gather.c new file mode 100644 index 0000000000..9715d80f62 --- /dev/null +++ b/ompi/mca/coll/sync/coll_sync_gather.c @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2009 Cisco Systems, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "ompi_config.h" + +#include "coll_sync.h" + + +/* + * gather + * + * Function: - gather + * Accepts: - same arguments as MPI_Gather() + * Returns: - MPI_SUCCESS or error code + */ +int mca_coll_sync_gather(void *sbuf, int scount, + struct ompi_datatype_t *sdtype, + void *rbuf, int rcount, + struct ompi_datatype_t *rdtype, + int root, struct ompi_communicator_t *comm, + mca_coll_base_module_t *module) +{ + mca_coll_sync_module_t *s = (mca_coll_sync_module_t*) module; + + if (s->in_operation) { + return s->c_coll.coll_gather(sbuf, scount, sdtype, + rbuf, rcount, rdtype, root, comm, + s->c_coll.coll_gather_module); + } else { + COLL_SYNC(s, s->c_coll.coll_gather(sbuf, scount, sdtype, + rbuf, rcount, rdtype, root, comm, + s->c_coll.coll_gather_module)); + } +} diff --git a/ompi/mca/coll/sync/coll_sync_gatherv.c b/ompi/mca/coll/sync/coll_sync_gatherv.c new file mode 100644 index 0000000000..982c1cce1f --- /dev/null +++ b/ompi/mca/coll/sync/coll_sync_gatherv.c @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2009 Cisco Systems, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "ompi_config.h" + +#include "coll_sync.h" + + +/* + * gatherv + * + * Function: - gatherv + * Accepts: - same arguments as MPI_Gatherv() + * Returns: - MPI_SUCCESS or error code + */ +int mca_coll_sync_gatherv(void *sbuf, int scount, + struct ompi_datatype_t *sdtype, + void *rbuf, int *rcounts, int *disps, + struct ompi_datatype_t *rdtype, int root, + struct ompi_communicator_t *comm, + mca_coll_base_module_t *module) +{ + mca_coll_sync_module_t *s = (mca_coll_sync_module_t*) module; + + if (s->in_operation) { + return s->c_coll.coll_gatherv(sbuf, scount, sdtype, + rbuf, rcounts, disps, rdtype, root, comm, + s->c_coll.coll_gatherv_module); + } else { + COLL_SYNC(s, s->c_coll.coll_gatherv(sbuf, scount, sdtype, + rbuf, rcounts, disps, rdtype, + root, comm, + s->c_coll.coll_gatherv_module)); + } +} diff --git a/ompi/mca/coll/sync/coll_sync_module.c b/ompi/mca/coll/sync/coll_sync_module.c new file mode 100644 index 0000000000..0c777b5805 --- /dev/null +++ b/ompi/mca/coll/sync/coll_sync_module.c @@ -0,0 +1,206 @@ +/* + * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2006 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2009 Cisco Systems, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "ompi_config.h" +#include "coll_sync.h" + +#include + +#include "mpi.h" + +#include "orte/util/show_help.h" + +#include "ompi/constants.h" +#include "ompi/communicator/communicator.h" +#include "ompi/mca/coll/coll.h" +#include "ompi/mca/coll/base/base.h" +#include "coll_sync.h" + + +static void mca_coll_sync_module_construct(mca_coll_sync_module_t *module) +{ + memset(&(module->c_coll), 0, sizeof(module->c_coll)); + module->before_num_operations = 0; + module->after_num_operations = 0; + module->in_operation = false; +} + +static void mca_coll_sync_module_destruct(mca_coll_sync_module_t *module) +{ + OBJ_RELEASE(module->c_coll.coll_allgather_module); + OBJ_RELEASE(module->c_coll.coll_allgatherv_module); + OBJ_RELEASE(module->c_coll.coll_allreduce_module); + OBJ_RELEASE(module->c_coll.coll_alltoall_module); + OBJ_RELEASE(module->c_coll.coll_alltoallv_module); + OBJ_RELEASE(module->c_coll.coll_alltoallw_module); + OBJ_RELEASE(module->c_coll.coll_bcast_module); + OBJ_RELEASE(module->c_coll.coll_gather_module); + OBJ_RELEASE(module->c_coll.coll_gatherv_module); + OBJ_RELEASE(module->c_coll.coll_reduce_module); + OBJ_RELEASE(module->c_coll.coll_reduce_scatter_module); + OBJ_RELEASE(module->c_coll.coll_scatter_module); + OBJ_RELEASE(module->c_coll.coll_scatterv_module); + /* If the exscan module is not NULL, then this was an + intracommunicator, and therefore scan will have a module as + well. */ + if (NULL != module->c_coll.coll_exscan_module) { + OBJ_RELEASE(module->c_coll.coll_exscan_module); + OBJ_RELEASE(module->c_coll.coll_scan_module); + } +} + +OBJ_CLASS_INSTANCE(mca_coll_sync_module_t, mca_coll_base_module_t, + mca_coll_sync_module_construct, + mca_coll_sync_module_destruct); + + +/* + * Initial query function that is invoked during MPI_INIT, allowing + * this component to disqualify itself if it doesn't support the + * required level of thread support. + */ +int mca_coll_sync_init_query(bool enable_progress_threads, + bool enable_mpi_threads) +{ + /* Nothing to do */ + + return OMPI_SUCCESS; +} + + +/* + * Invoked when there's a new communicator that has been created. + * Look at the communicator and decide which set of functions and + * priority we want to return. + */ +mca_coll_base_module_t * +mca_coll_sync_comm_query(struct ompi_communicator_t *comm, + int *priority) +{ + mca_coll_sync_module_t *sync_module; + + sync_module = OBJ_NEW(mca_coll_sync_module_t); + if (NULL == sync_module) { + return NULL; + } + + /* If both MCA params are 0, then disqualify us */ + if (0 == mca_coll_sync_component.barrier_before_nops && + 0 == mca_coll_sync_component.barrier_after_nops) { + return NULL; + } + *priority = mca_coll_sync_component.priority; + + /* Choose whether to use [intra|inter] */ + sync_module->super.coll_module_enable = mca_coll_sync_module_enable; + sync_module->super.ft_event = mca_coll_sync_ft_event; + + sync_module->super.coll_allgather = mca_coll_sync_allgather; + sync_module->super.coll_allgatherv = mca_coll_sync_allgatherv; + sync_module->super.coll_allreduce = mca_coll_sync_allreduce; + sync_module->super.coll_alltoall = mca_coll_sync_alltoall; + sync_module->super.coll_alltoallv = mca_coll_sync_alltoallv; + sync_module->super.coll_alltoallw = mca_coll_sync_alltoallw; + sync_module->super.coll_barrier = NULL; + sync_module->super.coll_bcast = mca_coll_sync_bcast; + sync_module->super.coll_exscan = mca_coll_sync_exscan; + sync_module->super.coll_gather = mca_coll_sync_gather; + sync_module->super.coll_gatherv = mca_coll_sync_gatherv; + sync_module->super.coll_reduce = mca_coll_sync_reduce; + sync_module->super.coll_reduce_scatter = mca_coll_sync_reduce_scatter; + sync_module->super.coll_scan = mca_coll_sync_scan; + sync_module->super.coll_scatter = mca_coll_sync_scatter; + sync_module->super.coll_scatterv = mca_coll_sync_scatterv; + + return &(sync_module->super); +} + + +/* + * Init module on the communicator + */ +int mca_coll_sync_module_enable(mca_coll_base_module_t *module, + struct ompi_communicator_t *comm) +{ + bool good = true; + char *msg = NULL; + mca_coll_sync_module_t *s = (mca_coll_sync_module_t*) module; + + /* Save the prior layer of coll functions */ + s->c_coll = comm->c_coll; + +#define CHECK_AND_RETAIN(name) \ + if (NULL == s->c_coll.coll_ ## name ## _module) { \ + good = false; \ + msg = "##name##"; \ + } else if (good) { \ + OBJ_RETAIN(s->c_coll.coll_ ## name ## _module); \ + } + + CHECK_AND_RETAIN(allgather); + CHECK_AND_RETAIN(allgatherv); + CHECK_AND_RETAIN(allreduce); + CHECK_AND_RETAIN(alltoall); + CHECK_AND_RETAIN(alltoallv); + CHECK_AND_RETAIN(alltoallw); + CHECK_AND_RETAIN(bcast); + CHECK_AND_RETAIN(gather); + CHECK_AND_RETAIN(gatherv); + CHECK_AND_RETAIN(reduce); + CHECK_AND_RETAIN(reduce_scatter); + CHECK_AND_RETAIN(scatter); + CHECK_AND_RETAIN(scatterv); + if (!OMPI_COMM_IS_INTER(comm)) { + /* MPI does not define scan/exscan on intercommunicators */ + CHECK_AND_RETAIN(exscan); + CHECK_AND_RETAIN(scan); + } + + /* All done */ + if (good) { + return OMPI_SUCCESS; + } else { + orte_show_help("help-coll-sync.txt", "missing collective", true, + orte_process_info.nodename, + mca_coll_sync_component.priority, msg); + return OMPI_ERR_NOT_FOUND; + } +} + + +int mca_coll_sync_ft_event(int state) +{ + if (OPAL_CRS_CHECKPOINT == state) { + ; + } + else if (OPAL_CRS_CONTINUE == state) { + ; + } + else if (OPAL_CRS_RESTART == state) { + ; + } + else if (OPAL_CRS_TERM == state ) { + ; + } + else { + ; + } + + return OMPI_SUCCESS; +} diff --git a/ompi/mca/coll/sync/coll_sync_reduce.c b/ompi/mca/coll/sync/coll_sync_reduce.c new file mode 100644 index 0000000000..3b49e72262 --- /dev/null +++ b/ompi/mca/coll/sync/coll_sync_reduce.c @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2006 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "ompi_config.h" + +#include "coll_sync.h" + + +/* + * reduce + * + * Function: - reduce + * Accepts: - same as MPI_Reduce() + * Returns: - MPI_SUCCESS or error code + */ +int mca_coll_sync_reduce(void *sbuf, void *rbuf, int count, + struct ompi_datatype_t *dtype, + struct ompi_op_t *op, + int root, struct ompi_communicator_t *comm, + mca_coll_base_module_t *module) +{ + mca_coll_sync_module_t *s = (mca_coll_sync_module_t*) module; + + if (s->in_operation) { + return s->c_coll.coll_reduce(sbuf, rbuf, count, dtype, op, root, comm, + s->c_coll.coll_reduce_module); + } else { + COLL_SYNC(s, s->c_coll.coll_reduce(sbuf, rbuf, count, dtype, + op, root, comm, + s->c_coll.coll_reduce_module)); + } +} diff --git a/ompi/mca/coll/sync/coll_sync_reduce_scatter.c b/ompi/mca/coll/sync/coll_sync_reduce_scatter.c new file mode 100644 index 0000000000..b45031e2d0 --- /dev/null +++ b/ompi/mca/coll/sync/coll_sync_reduce_scatter.c @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2006 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2008 Sun Microsystems, Inc. All rights reserved. + * Copyright (c) 2009 Cisco Systems, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "ompi_config.h" + +#include "coll_sync.h" + + +/* + * reduce_scatter + * + * Function: - reduce then scatter + * Accepts: - same as MPI_Reduce_scatter() + * Returns: - MPI_SUCCESS or error code + */ +int mca_coll_sync_reduce_scatter(void *sbuf, void *rbuf, int *rcounts, + struct ompi_datatype_t *dtype, + struct ompi_op_t *op, + struct ompi_communicator_t *comm, + mca_coll_base_module_t *module) +{ + mca_coll_sync_module_t *s = (mca_coll_sync_module_t*) module; + + if (s->in_operation) { + return s->c_coll.coll_reduce_scatter(sbuf, rbuf, rcounts, + dtype, op, comm, + s->c_coll.coll_reduce_scatter_module); + } else { + COLL_SYNC(s, s->c_coll.coll_reduce_scatter(sbuf, rbuf, rcounts, + dtype, op, comm, + s->c_coll.coll_reduce_scatter_module)); + } +} diff --git a/ompi/mca/coll/sync/coll_sync_scan.c b/ompi/mca/coll/sync/coll_sync_scan.c new file mode 100644 index 0000000000..2cbf40e4d1 --- /dev/null +++ b/ompi/mca/coll/sync/coll_sync_scan.c @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2006 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "ompi_config.h" + +#include "coll_sync.h" + + +/* + * scan + * + * Function: - scan + * Accepts: - same arguments as MPI_Scan() + * Returns: - MPI_SUCCESS or error code + */ +int mca_coll_sync_scan(void *sbuf, void *rbuf, int count, + struct ompi_datatype_t *dtype, + struct ompi_op_t *op, + struct ompi_communicator_t *comm, + mca_coll_base_module_t *module) +{ + mca_coll_sync_module_t *s = (mca_coll_sync_module_t*) module; + + if (s->in_operation) { + return s->c_coll.coll_scan(sbuf, rbuf, count, dtype, op, comm, + s->c_coll.coll_scan_module); + } else { + COLL_SYNC(s, s->c_coll.coll_scan(sbuf, rbuf, count, dtype, op, comm, + s->c_coll.coll_scan_module)); + } +} diff --git a/ompi/mca/coll/sync/coll_sync_scatter.c b/ompi/mca/coll/sync/coll_sync_scatter.c new file mode 100644 index 0000000000..e21c4370b6 --- /dev/null +++ b/ompi/mca/coll/sync/coll_sync_scatter.c @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2009 Cisco Systems, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "ompi_config.h" + +#include "coll_sync.h" + + +/* + * scatter + * + * Function: - scatter + * Accepts: - same arguments as MPI_Scatter() + * Returns: - MPI_SUCCESS or error code + */ +int mca_coll_sync_scatter(void *sbuf, int scount, + struct ompi_datatype_t *sdtype, + void *rbuf, int rcount, + struct ompi_datatype_t *rdtype, + int root, struct ompi_communicator_t *comm, + mca_coll_base_module_t *module) +{ + mca_coll_sync_module_t *s = (mca_coll_sync_module_t*) module; + + if (s->in_operation) { + return s->c_coll.coll_scatter(sbuf, scount, sdtype, + rbuf, rcount, rdtype, root, comm, + s->c_coll.coll_scatter_module); + } else { + COLL_SYNC(s, s->c_coll.coll_scatter(sbuf, scount, sdtype, + rbuf, rcount, rdtype, root, comm, + s->c_coll.coll_scatter_module)); + } +} diff --git a/ompi/mca/coll/sync/coll_sync_scatterv.c b/ompi/mca/coll/sync/coll_sync_scatterv.c new file mode 100644 index 0000000000..17cd5d79c4 --- /dev/null +++ b/ompi/mca/coll/sync/coll_sync_scatterv.c @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2009 Cisco Systems, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "ompi_config.h" + +#include "coll_sync.h" + + +/* + * scatterv + * + * Function: - scatterv + * Accepts: - same arguments as MPI_Scatterv() + * Returns: - MPI_SUCCESS or error code + */ +int mca_coll_sync_scatterv(void *sbuf, int *scounts, + int *disps, struct ompi_datatype_t *sdtype, + void *rbuf, int rcount, + struct ompi_datatype_t *rdtype, int root, + struct ompi_communicator_t *comm, + mca_coll_base_module_t *module) +{ + mca_coll_sync_module_t *s = (mca_coll_sync_module_t*) module; + + if (s->in_operation) { + return s->c_coll.coll_scatterv(sbuf, scounts, disps, sdtype, + rbuf, rcount, rdtype, root, comm, + s->c_coll.coll_scatterv_module); + } else { + COLL_SYNC(s, s->c_coll.coll_scatterv(sbuf, scounts, disps, sdtype, + rbuf, rcount, rdtype, root, comm, + s->c_coll.coll_scatterv_module)); + } +} diff --git a/ompi/mca/coll/sync/configure.params b/ompi/mca/coll/sync/configure.params new file mode 100644 index 0000000000..1b880fa432 --- /dev/null +++ b/ompi/mca/coll/sync/configure.params @@ -0,0 +1,24 @@ +# -*- shell-script -*- +# +# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana +# University Research and Technology +# Corporation. All rights reserved. +# Copyright (c) 2004-2005 The University of Tennessee and The University +# of Tennessee Research Foundation. All rights +# reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# Copyright (c) 2007 Los Alamos National Security, LLC. All rights +# reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +# Specific to this module + +PARAM_CONFIG_FILES=Makefile diff --git a/ompi/mca/coll/sync/help-coll-sync.txt b/ompi/mca/coll/sync/help-coll-sync.txt new file mode 100644 index 0000000000..18769d3409 --- /dev/null +++ b/ompi/mca/coll/sync/help-coll-sync.txt @@ -0,0 +1,23 @@ +# -*- text -*- +# +# Copyright (c) 2009 Cisco Systems, Inc. All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# +# This is the US/English general help file for Open MPI's sync +# collective component. +# +[missing collective] + +The sync collective component in Open MPI was activated on a +communicator where it did not find an underlying collective operation +defined. This usually means that the sync collective module's +priority was not set high enough. Please try increasing sync's +priority. + + Local host: %s + Sync coll module priority: %d + First discovered missing collective: %s