320 строки
12 KiB
C
320 строки
12 KiB
C
/*
|
|
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
|
* University Research and Technology
|
|
* Corporation. All rights reserved.
|
|
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
|
* of Tennessee Research Foundation. All rights
|
|
* reserved.
|
|
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
|
* University of Stuttgart. All rights reserved.
|
|
* Copyright (c) 2004-2005 The Regents of the University of California.
|
|
* All rights reserved.
|
|
* $COPYRIGHT$
|
|
*
|
|
* Additional copyrights may follow
|
|
*
|
|
* $HEADER$
|
|
*/
|
|
|
|
#ifndef MCA_COLL_HIERARCH_EXPORT_H
|
|
#define MCA_COLL_HIERARCH_EXPORT_H
|
|
|
|
#include "ompi_config.h"
|
|
|
|
#include "mpi.h"
|
|
#include "opal/mca/mca.h"
|
|
#include "ompi/mca/coll/coll.h"
|
|
#include "ompi/request/request.h"
|
|
|
|
#if defined(c_plusplus) || defined(__cplusplus)
|
|
extern "C" {
|
|
#endif
|
|
|
|
/*
|
|
* Globally exported variable
|
|
*/
|
|
|
|
OMPI_MODULE_DECLSPEC extern const mca_coll_base_component_1_0_0_t mca_coll_hierarch_component;
|
|
extern int mca_coll_hierarch_priority_param;
|
|
extern int mca_coll_hierarch_verbose_param;
|
|
extern int mca_coll_hierarch_use_rdma_param;
|
|
extern int mca_coll_hierarch_ignore_sm_param;
|
|
extern int mca_coll_hierarch_symmetric_param;
|
|
|
|
|
|
#define HIER_DEFAULT_NUM_LLEAD 5
|
|
/*
|
|
* Data structure for attaching data to the communicator
|
|
*/
|
|
|
|
/* Clarifying some terminology:
|
|
* comm: the input communicator, consisting of several lower level communicators.
|
|
* lcomm: low level communicator, often refered to as subcommunicator
|
|
* lleader: local leader, a dedicated process of each low level communicator
|
|
ATTENTION: an lleader might be the 'head' of a low level
|
|
communicator of size one!
|
|
* llcomm: local leader communicator, grouping all local leaders of a comm.
|
|
*/
|
|
|
|
struct mca_coll_base_comm_t {
|
|
struct ompi_communicator_t *hier_comm; /* link back to the attached comm */
|
|
struct ompi_communicator_t *hier_lcomm; /* low level communicator */
|
|
ompi_pointer_array_t hier_llead; /* local leader communicator structure */
|
|
int hier_num_lleaders; /* number of local leaders */
|
|
int hier_level; /* level in the hierarchy. For debugging*/
|
|
int hier_num_reqs; /* num. of requests */
|
|
ompi_request_t **hier_reqs; /* list of requests */
|
|
int hier_num_colorarr; /* size of the colorarr array */
|
|
int *hier_llr; /* color array compacted (1 entry per color).
|
|
Array of size hier_num_lleaders */
|
|
int *hier_max_offset; /* Number of processes for each color.
|
|
Array of size hier_num_lleaders */
|
|
int *hier_colorarr; /* array containing the color of all procs */
|
|
};
|
|
|
|
struct mca_coll_hierarch_llead_t {
|
|
struct ompi_communicator_t *llcomm; /* local leader communicator */
|
|
int *lleaders; /* list of local leaders, ranks in comm */
|
|
int my_lleader; /* rank of my lleader in lcomm */
|
|
int am_lleader; /* am I an lleader? */
|
|
int offset; /* Offset used for this llcomm */
|
|
};
|
|
|
|
typedef struct mca_coll_hierarch_llead_t mca_coll_hierarch_llead_t;
|
|
|
|
|
|
static inline int mca_coll_hierarch_count_lleaders ( int size, int *carr)
|
|
{
|
|
|
|
/*
|
|
* Determine the number of local leaders. Please note, that any process
|
|
* with color = MPI_UNDEFINED will be counted as the head of a group of its own.
|
|
* Please note furthermore, that every process with color=MPI_UNDEFINED will be
|
|
* stored in this array on its own...
|
|
*/
|
|
int cnt, i, j, found;
|
|
int *llr=NULL;
|
|
|
|
llr = (int *) malloc ( size * sizeof(int));
|
|
if (NULL == llr ){
|
|
return OMPI_ERR_OUT_OF_RESOURCE;
|
|
}
|
|
|
|
llr[0] = carr[0];
|
|
for (cnt=1, i=1; i<size; i++ ) {
|
|
if ( carr[i] == MPI_UNDEFINED ) {
|
|
llr[cnt++] = carr[i];
|
|
continue;
|
|
}
|
|
for ( found=0, j=0; j<cnt; j++ ) {
|
|
if ( carr[i] == llr[j] ) {
|
|
found = 1;
|
|
break;
|
|
}
|
|
}
|
|
if ( !found ) {
|
|
llr[cnt++] = carr[i];
|
|
}
|
|
}
|
|
|
|
free (llr);
|
|
return cnt;
|
|
}
|
|
|
|
static inline int mca_coll_hierarch_get_offset ( int rank, int size, int *carr)
|
|
{
|
|
int offset, i, color = carr[rank];
|
|
|
|
if ( color == MPI_UNDEFINED ) {
|
|
/* always */
|
|
return 1;
|
|
}
|
|
|
|
for ( offset=0, i=0; i<=rank; i++) {
|
|
if ( carr[i] == color ) {
|
|
offset++;
|
|
}
|
|
}
|
|
|
|
return offset;
|
|
}
|
|
|
|
|
|
|
|
/* This function determine the parameters required in hierarchical
|
|
* collective operations. It is called from the collective operations themselves.
|
|
*
|
|
* @param root (input): rank of the root process in comm
|
|
* @param data (input): coll_base_comm structure of the hierarchical module. Contains
|
|
* all relevant, precomputed data for this set of collectives.
|
|
*
|
|
* @param llroot (output): rank of the root process in llcomm, MPI_UNDEFINED for all
|
|
* processes not being part of the local leader communicator.
|
|
* @param lroot (output): rank of the local leader in the low level communicator,
|
|
* or MPI_UNDEFINED if there is no low level communicator.
|
|
* return value: llcomm (local leader communicator) or MPI_COMM_NULL for
|
|
* all processes not being part of the local leader communicator.
|
|
*/
|
|
|
|
struct ompi_communicator_t* mca_coll_hierarch_get_llcomm (int rroot,
|
|
struct mca_coll_base_comm_t *data,
|
|
int* llroot,
|
|
int* lleader);
|
|
|
|
/* This function is supposed to set up all elements of the mca_coll_base_comm_t
|
|
* structure, including:
|
|
* data->hier_num_lleaders: determine number of local leaders in the comms
|
|
* data->hier_llr: array of size hier_num_lleaders containing the colors
|
|
* data->hier_max_offset: array containing the counter for each color how often
|
|
* it appears in the colorarr array.
|
|
*/
|
|
|
|
int mca_coll_hierarch_get_llr ( struct mca_coll_base_comm_t *data );
|
|
|
|
|
|
/* This function is supposed to set all elements of the llead structure based on the
|
|
* offset and the rank of the process.
|
|
*
|
|
* @param rank(input): rank of the calling process in comm
|
|
* @param data(input): coll_base_comm structure of the hierarchical module. Contains
|
|
* all relevant, precomputed data for this set of collectives.
|
|
* @param llead(output): ptr to the mca_coll_hierarch_llead_t element which should
|
|
* be set
|
|
* @param offset(input): offset which shall be used.
|
|
*/
|
|
|
|
int mca_coll_hierarch_get_all_lleaders ( int rank, struct mca_coll_base_comm_t *data,
|
|
struct mca_coll_hierarch_llead_t *llead,
|
|
int offset );
|
|
|
|
|
|
|
|
/*
|
|
* coll API functions
|
|
*/
|
|
int mca_coll_hierarch_init_query(bool allow_hierarch_user_threads,
|
|
bool have_hidden_threads);
|
|
const struct mca_coll_base_module_1_0_0_t *
|
|
mca_coll_hierarch_comm_query(struct ompi_communicator_t *comm,
|
|
int *priority, struct mca_coll_base_comm_t **data);
|
|
int mca_coll_hierarch_comm_unquery(struct ompi_communicator_t *comm,
|
|
struct mca_coll_base_comm_t *data);
|
|
|
|
const struct mca_coll_base_module_1_0_0_t *
|
|
mca_coll_hierarch_module_init(struct ompi_communicator_t *comm);
|
|
|
|
int mca_coll_hierarch_module_finalize(struct ompi_communicator_t *comm);
|
|
|
|
int mca_coll_hierarch_allgather_intra(void *sbuf, int scount,
|
|
struct ompi_datatype_t *sdtype,
|
|
void *rbuf, int rcount,
|
|
struct ompi_datatype_t *rdtype,
|
|
struct ompi_communicator_t *comm);
|
|
int mca_coll_hierarch_allgatherv_intra(void *sbuf, int scount,
|
|
struct ompi_datatype_t *sdtype,
|
|
void * rbuf, int *rcounts,
|
|
int *disps,
|
|
struct ompi_datatype_t *rdtype,
|
|
struct ompi_communicator_t *comm);
|
|
int mca_coll_hierarch_allreduce_intra(void *sbuf, void *rbuf, int count,
|
|
struct ompi_datatype_t *dtype,
|
|
struct ompi_op_t *op,
|
|
struct ompi_communicator_t *comm);
|
|
int mca_coll_hierarch_alltoall_intra(void *sbuf, int scount,
|
|
struct ompi_datatype_t *sdtype,
|
|
void* rbuf, int rcount,
|
|
struct ompi_datatype_t *rdtype,
|
|
struct ompi_communicator_t *comm);
|
|
int mca_coll_hierarch_alltoallv_intra(void *sbuf, int *scounts,
|
|
int *sdisps,
|
|
struct ompi_datatype_t *sdtype,
|
|
void *rbuf, int *rcounts,
|
|
int *rdisps,
|
|
struct ompi_datatype_t *rdtype,
|
|
struct ompi_communicator_t *comm);
|
|
int mca_coll_hierarch_alltoallw_intra(void *sbuf, int *scounts,
|
|
int *sdisps,
|
|
struct ompi_datatype_t **sdtypes,
|
|
void *rbuf, int *rcounts,
|
|
int *rdisps,
|
|
struct ompi_datatype_t **rdtypes,
|
|
struct ompi_communicator_t *comm);
|
|
int mca_coll_hierarch_barrier_intra(struct ompi_communicator_t *comm);
|
|
int mca_coll_hierarch_bcast_intra(void *buff, int count,
|
|
struct ompi_datatype_t *datatype,
|
|
int root,
|
|
struct ompi_communicator_t *comm);
|
|
int mca_coll_hierarch_exscan_intra(void *sbuf, void *rbuf, int count,
|
|
struct ompi_datatype_t *dtype,
|
|
struct ompi_op_t *op,
|
|
struct ompi_communicator_t *comm);
|
|
int mca_coll_hierarch_gather_intra(void *sbuf, int scount,
|
|
struct ompi_datatype_t *sdtype,
|
|
void *rbuf, int rcount,
|
|
struct ompi_datatype_t *rdtype,
|
|
int root,
|
|
struct ompi_communicator_t *comm);
|
|
int mca_coll_hierarch_gatherv_intra(void *sbuf, int scount,
|
|
struct ompi_datatype_t *sdtype,
|
|
void *rbuf, int *rcounts, int *disps,
|
|
struct ompi_datatype_t *rdtype,
|
|
int root,
|
|
struct ompi_communicator_t *comm);
|
|
int mca_coll_hierarch_reduce_intra(void *sbuf, void* rbuf, int count,
|
|
struct ompi_datatype_t *dtype,
|
|
struct ompi_op_t *op,
|
|
int root,
|
|
struct ompi_communicator_t *comm);
|
|
int mca_coll_hierarch_reduce_scatter_intra(void *sbuf, void *rbuf,
|
|
int *rcounts,
|
|
struct ompi_datatype_t *dtype,
|
|
struct ompi_op_t *op,
|
|
struct ompi_communicator_t *comm);
|
|
int mca_coll_hierarch_scan_intra(void *sbuf, void *rbuf, int count,
|
|
struct ompi_datatype_t *dtype,
|
|
struct ompi_op_t *op,
|
|
struct ompi_communicator_t *comm);
|
|
int mca_coll_hierarch_scatter_intra(void *sbuf, int scount,
|
|
struct ompi_datatype_t *sdtype, void *rbuf,
|
|
int rcount, struct ompi_datatype_t *rdtype,
|
|
int root, struct ompi_communicator_t *comm);
|
|
int mca_coll_hierarch_scatterv_intra(void *sbuf, int *scounts, int *disps,
|
|
struct ompi_datatype_t *sdtype,
|
|
void* rbuf, int rcount,
|
|
struct ompi_datatype_t *rdtype, int root,
|
|
struct ompi_communicator_t *comm);
|
|
|
|
/*
|
|
* These are trivial implementations of these routines used during comm_query/init,
|
|
* since we cannot access any other collectives
|
|
*/
|
|
int mca_coll_hierarch_allgather_tmp(void *sbuf, int scount,
|
|
struct ompi_datatype_t *sdtype,
|
|
void *rbuf, int rcount,
|
|
struct ompi_datatype_t *rdtype,
|
|
struct ompi_communicator_t *comm);
|
|
int mca_coll_hierarch_allreduce_tmp(void *sbuf, void *rbuf, int count,
|
|
struct ompi_datatype_t *dtype,
|
|
struct ompi_op_t *op,
|
|
struct ompi_communicator_t *comm);
|
|
int mca_coll_hierarch_bcast_tmp ( void *buf, int count, struct ompi_datatype_t *dtype,
|
|
int root, struct ompi_communicator_t *comm);
|
|
|
|
int mca_coll_hierarch_gather_tmp(void *sbuf, int scount,
|
|
struct ompi_datatype_t *sdtype,
|
|
void *rbuf, int rcount,
|
|
struct ompi_datatype_t *rdtype,
|
|
int root, struct ompi_communicator_t *comm);
|
|
int mca_coll_hierarch_reduce_tmp(void *sbuf, void *rbuf, int count,
|
|
struct ompi_datatype_t *dtype,
|
|
struct ompi_op_t *op,
|
|
int root, struct ompi_communicator_t *comm);
|
|
|
|
|
|
|
|
|
|
#if defined(c_plusplus) || defined(__cplusplus)
|
|
}
|
|
#endif
|
|
#endif /* MCA_COLL_HIERARCH_EXPORT_H */
|