diff --git a/ompi/mca/coll/fca/coll_fca.h b/ompi/mca/coll/fca/coll_fca.h index 393b96d4c0..944a3f5bbb 100644 --- a/ompi/mca/coll/fca/coll_fca.h +++ b/ompi/mca/coll/fca/coll_fca.h @@ -10,9 +10,8 @@ #ifndef MCA_COLL_FCA_H #define MCA_COLL_FCA_H -#include - #include "ompi_config.h" + #include "mpi.h" #include "opal/mca/mca.h" #include "ompi/mca/coll/coll.h" @@ -21,6 +20,8 @@ #include "ompi/mca/coll/base/coll_tags.h" #include "ompi/communicator/communicator.h" #include "ompi/op/op.h" + +#include "coll_fca_api.h" #include "coll_fca_debug.h" #ifdef OMPI_DATATYPE_MAX_PREDEFINED @@ -47,45 +48,7 @@ BEGIN_C_DECLS * Used to load the library dynamically. */ -struct mca_coll_fca_fca_ops_t { - /* FCA Context operations */ - int (*init)(fca_init_spec_t *spec, fca_t **context); - void (*cleanup)(fca_t *context); - void (*progress)(fca_t *context); - - /* Fabric communicator creation */ - int (*comm_new)(fca_t *context, fca_comm_new_spec_t *spec, fca_comm_desc_t *comm_desc); - int (*comm_end)(fca_t *context, int comm_id); - void* (*get_rank_info)(fca_t *context, int *size); - void (*free_rank_info)(void *rank_info); - - /* Local communicator creation */ - int (*comm_init)(fca_t *context, fca_comm_init_spec_t *spec, fca_comm_t** fca_comm); - void (*comm_destroy)(fca_comm_t *comm); - int (*comm_get_caps)(fca_comm_t *comm, fca_comm_caps_t *caps); - - /* Collectives supported by FCA */ - int (*do_reduce)(fca_comm_t *comm, fca_reduce_spec_t *spec); - int (*do_all_reduce)(fca_comm_t *comm, fca_reduce_spec_t *spec); - int (*do_bcast)(fca_comm_t *comm, fca_bcast_spec_t *spec); - int (*do_allgather)(fca_comm_t *comm, fca_gather_spec_t *spec); - int (*do_allgatherv)(fca_comm_t *comm, fca_gatherv_spec_t *spec); - int (*do_barrier)(fca_comm_t *comm); - - /* Helper functions */ - unsigned long (*get_version)(void); - char * (*get_version_string)(void); - int (*maddr_ib_pton)(const char *mlid_str, const char *mgid_str, fca_mcast_addr_t *dst); - int (*maddr_inet_pton)(int af, const char *src, fca_mcast_addr_t *dst); - fca_init_spec_t *(*parse_spec_file)(char* spec_ini_file); - void (*free_init_spec)(fca_init_spec_t *fca_init_spec); - int (*translate_mpi_op)(char *mpi_op); - int (*translate_mpi_dtype)(char *mpi_dtype); - int (*get_dtype_size)(int dtype); - const char* (*strerror)(int code); -}; -typedef struct mca_coll_fca_fca_ops_t mca_coll_fca_fca_ops_t; /** * FCA data type information @@ -184,8 +147,8 @@ struct mca_coll_fca_component_t { int fca_np; /* FCA global stuff */ - void *fca_lib_handle; /* FCA dynamic library */ - mca_coll_fca_fca_ops_t fca_ops; /* FCA operations */ + void *fca_lib_handle; /* FCA dynamic library */ + mca_coll_fca_ops_t fca_ops; /* FCA operations */ fca_t *fca_context; /* FCA context handle */ mca_coll_fca_dtype_info_t fca_dtypes[FCA_DT_MAX_PREDEFINED]; /* FCA dtype translation */ mca_coll_fca_op_info_t fca_reduce_ops[FCA_MAX_OPS]; /* FCA op translation */ diff --git a/ompi/mca/coll/fca/coll_fca_api.h b/ompi/mca/coll/fca/coll_fca_api.h new file mode 100644 index 0000000000..c62e1c2170 --- /dev/null +++ b/ompi/mca/coll/fca/coll_fca_api.h @@ -0,0 +1,174 @@ +/** + Copyright (c) 2010 Voltaire, Inc. All rights reserved. + $COPYRIGHT$ + + Additional copyrights may follow + + $HEADER$ + */ + +#include "ompi_config.h" + +#include + +/* + * FCA API compatibility layer. + * MPI build must define an FCA version macro. + */ + +#ifdef OMPI_FCA_VERSION_1_2 + +#define FCA_API_ABI_MAJOR 1 +#define FCA_API_ABI_MINOR 2 +#define FCA_MAJOR_BIT 24ul +#define FCA_MINOR_BIT 16ul +#define EUSEMPI 287 + + +typedef struct mca_coll_fca_ops_t { + + /* FCA Context operations */ + int (*init)(fca_init_spec_t *spec, fca_t **context); + void (*cleanup)(fca_t *context); + + /* Fabric communicator creation */ + int (*comm_new)(fca_t *context, fca_comm_new_spec_t *spec, fca_comm_desc_t *comm_desc); + int (*comm_end)(fca_t *context, int comm_id); + void* (*get_rank_info)(fca_t *context, int *size); + void (*free_rank_info)(void *rank_info); + + /* Local communicator creation */ + int (*comm_init)(fca_t *context, int proc_idx, int num_procs, int comm_size, + fca_comm_desc_t *comm_desc, fca_comm_t** fca_comm); + void (*comm_destroy)(fca_comm_t *comm); + int (*comm_get_caps)(fca_comm_t *comm, fca_comm_caps_t *caps); + + /* Collectives supported by FCA */ + int (*do_reduce)(fca_comm_t *comm, fca_reduce_spec_t *spec); + int (*do_all_reduce)(fca_comm_t *comm, fca_reduce_spec_t *spec); + int (*do_bcast)(fca_comm_t *comm, fca_bcast_spec_t *spec); + int (*do_barrier)(fca_comm_t *comm); + + /* Helper functions */ + unsigned long (*get_version)(void); + char * (*get_version_string)(void); + fca_init_spec_t *(*parse_spec_file)(char* spec_ini_file); + void (*free_init_spec)(fca_init_spec_t *fca_init_spec); + int (*translate_mpi_op)(char *mpi_op); + int (*translate_mpi_dtype)(char *mpi_dtype); + int (*get_dtype_size)(int dtype); + const char* (*strerror)(int code); +} mca_coll_fca_ops_t; + + +static inline int mca_coll_fca_comm_init(mca_coll_fca_ops_t *fca_ops, + fca_t *fca_context, int rank, int comm_size, + int local_proc_idx, int num_local_procs, + fca_comm_desc_t *comm_desc, + fca_comm_t **fca_comm) +{ + return fca_ops->comm_init(fca_context, local_proc_idx, num_local_procs, + comm_size, comm_desc, fca_comm); +} + +static inline void mca_coll_fca_get_bcast_root(int root_rank, int *local_ranks, + int num_local_ranks, + fca_bcast_spec_t *spec) +{ + int i; + + for (i = 0; i < num_local_ranks; ++i) { + if (local_ranks[i] == root_rank) { + spec->root_indx = i; + return; + } + } + spec->root_indx = -1; +} + +static inline void mca_coll_fca_get_reduce_root(int root_rank, int my_rank, + fca_reduce_spec_t *spec) +{ + spec->is_root = root_rank == my_rank; +} + +#elif OMPI_FCA_VERSION_2_0 + +#define FCA_API_ABI_MAJOR 2 +#define FCA_API_ABI_MINOR 0 +#define OMPI_FCA_ALLGATHER 1 +#define OMPI_FCA_PROGRESS 1 + + +typedef struct mca_coll_fca_ops_t { + + /* FCA Context operations */ + int (*init)(fca_init_spec_t *spec, fca_t **context); + void (*cleanup)(fca_t *context); + void (*progress)(fca_t *context); + + /* Fabric communicator creation */ + int (*comm_new)(fca_t *context, fca_comm_new_spec_t *spec, fca_comm_desc_t *comm_desc); + int (*comm_end)(fca_t *context, int comm_id); + void* (*get_rank_info)(fca_t *context, int *size); + void (*free_rank_info)(void *rank_info); + + /* Local communicator creation */ + int (*comm_init)(fca_t *context, fca_comm_init_spec_t *spec, fca_comm_t** fca_comm); + void (*comm_destroy)(fca_comm_t *comm); + int (*comm_get_caps)(fca_comm_t *comm, fca_comm_caps_t *caps); + + /* Collectives supported by FCA */ + int (*do_reduce)(fca_comm_t *comm, fca_reduce_spec_t *spec); + int (*do_all_reduce)(fca_comm_t *comm, fca_reduce_spec_t *spec); + int (*do_bcast)(fca_comm_t *comm, fca_bcast_spec_t *spec); + int (*do_barrier)(fca_comm_t *comm); + int (*do_allgather)(fca_comm_t *comm, fca_gather_spec_t *spec); + int (*do_allgatherv)(fca_comm_t *comm, fca_gatherv_spec_t *spec); + + /* Helper functions */ + unsigned long (*get_version)(void); + char * (*get_version_string)(void); + fca_init_spec_t *(*parse_spec_file)(char* spec_ini_file); + void (*free_init_spec)(fca_init_spec_t *fca_init_spec); + int (*translate_mpi_op)(char *mpi_op); + int (*translate_mpi_dtype)(char *mpi_dtype); + int (*get_dtype_size)(int dtype); + const char* (*strerror)(int code); +} mca_coll_fca_ops_t; + + +static inline int mca_coll_fca_comm_init(mca_coll_fca_ops_t *fca_ops, + fca_t *fca_context, int rank, int comm_size, + int local_proc_idx, int num_local_procs, + fca_comm_desc_t *comm_desc, + fca_comm_t **fca_comm) +{ + fca_comm_init_spec_t spec; + + spec.rank = rank; + spec.size = comm_size; + spec.desc = *comm_desc; + spec.proc_idx = local_proc_idx; + spec.num_procs = num_local_procs; + return fca_ops->comm_init(fca_context, &spec, fca_comm); +} + +static inline void mca_coll_fca_get_bcast_root(int root_rank, int *local_ranks, + int num_local_ranks, + fca_bcast_spec_t *spec) +{ + spec->root = root_rank; +} + +static inline void mca_coll_fca_get_reduce_root(int root_rank, int my_rank, + fca_reduce_spec_t *spec) +{ + spec->root = root_rank; +} + +#else + +#error "FCA API version is undefinded" + +#endif diff --git a/ompi/mca/coll/fca/coll_fca_component.c b/ompi/mca/coll/fca/coll_fca_component.c index 7ed75c76b2..6588eb2b79 100644 --- a/ompi/mca/coll/fca/coll_fca_component.c +++ b/ompi/mca/coll/fca/coll_fca_component.c @@ -65,8 +65,6 @@ mca_coll_fca_component_t mca_coll_fca_component = { } }; -#define FCA_API_ABI_MAJOR (2) -#define FCA_API_ABI_MINOR (0) #define FCA_API_CLEAR_MICRO(__x) ((__x>>FCA_MINOR_BIT)<comm; ompi_proc_t* proc; - int rank; + int i, rank; + /* Count the local ranks */ fca_module->num_local_procs = 0; - for (rank = 0; rank < ompi_comm_size(comm); ++rank) { proc = __local_rank_lookup(comm, rank); - if (!FCA_IS_LOCAL_PROCESS(proc->proc_flags)) - continue; - - if (rank == fca_module->rank) { - fca_module->local_proc_idx = fca_module->num_local_procs; + if (FCA_IS_LOCAL_PROCESS(proc->proc_flags)) { + if (rank == fca_module->rank) { + fca_module->local_proc_idx = fca_module->num_local_procs; + } + ++fca_module->num_local_procs; } - ++fca_module->num_local_procs; - } + } + + /* Make a list of local ranks */ + fca_module->local_ranks = calloc(fca_module->num_local_procs, + sizeof *fca_module->local_ranks); + if (!fca_module->local_ranks) { + FCA_ERROR("Failed to allocate memory for %d local ranks", + fca_module->num_local_procs); + return OMPI_ERROR; + } + + i = 0; + for (rank = 0; rank < ompi_comm_size(comm); ++rank) { + proc = __local_rank_lookup(comm, rank); + if (FCA_IS_LOCAL_PROCESS(proc->proc_flags)) { + fca_module->local_ranks[i++] = rank; + } + } FCA_MODULE_VERBOSE(fca_module, 3, "i am %d/%d", fca_module->local_proc_idx, fca_module->num_local_procs); return OMPI_SUCCESS; } - static int __fca_comm_new(mca_coll_fca_module_t *fca_module) { ompi_communicator_t *comm = fca_module->comm; @@ -173,28 +188,24 @@ static int __fca_comm_new(mca_coll_fca_module_t *fca_module) static int __create_fca_comm(mca_coll_fca_module_t *fca_module) { - fca_comm_init_spec_t spec; - int rc, ret; int comm_size; + int rc, ret; rc = __fca_comm_new(fca_module); if (rc != OMPI_SUCCESS) return rc; /* allocate comm_init_spec */ - comm_size = ompi_comm_size(fca_module->comm); - spec.rank = fca_module->rank; - spec.size = comm_size; - spec.desc = fca_module->fca_comm_desc; - spec.proc_idx = fca_module->local_proc_idx; - spec.num_procs = fca_module->num_local_procs; - FCA_MODULE_VERBOSE(fca_module, 1, "Starting COMM_INIT comm_id %d proc_idx %d num_procs %d", fca_module->fca_comm_desc.comm_id, fca_module->local_proc_idx, fca_module->num_local_procs); - ret = mca_coll_fca_component.fca_ops.comm_init(mca_coll_fca_component.fca_context, - &spec, &fca_module->fca_comm); + comm_size = ompi_comm_size(fca_module->comm); + ret = mca_coll_fca_comm_init(&mca_coll_fca_component.fca_ops, + mca_coll_fca_component.fca_context, + fca_module->rank, comm_size, + fca_module->local_proc_idx, fca_module->num_local_procs, + &fca_module->fca_comm_desc, &fca_module->fca_comm); if (ret < 0) { FCA_ERROR("COMM_INIT failed: %s", mca_coll_fca_component.fca_ops.strerror(ret)); return OMPI_ERROR; @@ -304,6 +315,7 @@ static int mca_coll_fca_ft_event(int state) static void mca_coll_fca_module_clear(mca_coll_fca_module_t *fca_module) { fca_module->num_local_procs = 0; + fca_module->local_ranks = NULL; fca_module->fca_comm = NULL; fca_module->previous_barrier = NULL; @@ -343,7 +355,7 @@ static void mca_coll_fca_module_destruct(mca_coll_fca_module_t *fca_module) OBJ_RELEASE(fca_module->previous_reduce_scatter_module); if (fca_module->fca_comm) __destroy_fca_comm(fca_module); - + free(fca_module->local_ranks); mca_coll_fca_module_clear(fca_module); } diff --git a/ompi/mca/coll/fca/coll_fca_ops.c b/ompi/mca/coll/fca/coll_fca_ops.c index 5c0a3cec7c..f276af8482 100644 --- a/ompi/mca/coll/fca/coll_fca_ops.c +++ b/ompi/mca/coll/fca/coll_fca_ops.c @@ -180,7 +180,8 @@ int mca_coll_fca_bcast(void *buff, int count, struct ompi_datatype_t *datatype, FCA_VERBOSE(5,"Using FCA Bcast"); spec.buf = buff; - spec.root = root; + mca_coll_fca_get_bcast_root(root, fca_module->local_ranks, + fca_module->num_local_procs, &spec); ret = mca_coll_fca_component.fca_ops.do_bcast(fca_module->fca_comm, &spec); if (ret < 0) { if (ret == -EUSEMPI) { @@ -208,12 +209,11 @@ int mca_coll_fca_reduce(void *sbuf, void *rbuf, int count, int root, struct ompi_communicator_t *comm, mca_coll_base_module_t *module) { - mca_coll_fca_module_t *fca_module = (mca_coll_fca_module_t*)module; fca_reduce_spec_t spec; int ret; - spec.root = root; + mca_coll_fca_get_reduce_root(root, fca_module->rank, &spec); spec.sbuf = sbuf; spec.rbuf = rbuf; if (mca_coll_fca_fill_reduce_spec(count, dtype, op, &spec, @@ -293,6 +293,7 @@ int mca_coll_fca_allgather(void *sbuf, int scount, struct ompi_datatype_t *sdtyp mca_coll_base_module_t *module) { mca_coll_fca_module_t *fca_module = (mca_coll_fca_module_t*)module; +#ifdef OMPI_FCA_ALLGATHER fca_gather_spec_t spec = {0,}; int ret; @@ -329,6 +330,7 @@ int mca_coll_fca_allgather(void *sbuf, int scount, struct ompi_datatype_t *sdtyp return OMPI_SUCCESS; orig_allgather: +#endif return fca_module->previous_allgather(sbuf, scount, sdtype, rbuf, rcount, rdtype, comm, fca_module->previous_allgather_module); } @@ -342,6 +344,7 @@ int mca_coll_fca_allgatherv(void *sbuf, int scount, mca_coll_base_module_t *module) { mca_coll_fca_module_t *fca_module = (mca_coll_fca_module_t*)module; +#ifdef OMPI_FCA_ALLGATHER fca_gatherv_spec_t spec; int relemsize; int comm_size; @@ -388,6 +391,7 @@ int mca_coll_fca_allgatherv(void *sbuf, int scount, return OMPI_SUCCESS; orig_allgatherv: +#endif return fca_module->previous_allgatherv(sbuf, scount, sdtype, rbuf, rcounts, disps, rdtype, comm, fca_module->previous_allgatherv_module); @@ -419,7 +423,6 @@ int mca_coll_fca_alltoallv(void *sbuf, int *scounts, int *sdisps, comm, fca_module->previous_alltoallv_module); } - int mca_coll_fca_alltoallw(void *sbuf, int *scounts, int *sdisps, struct ompi_datatype_t **sdtypes, void *rbuf, int *rcounts, int *rdisps, @@ -433,7 +436,6 @@ int mca_coll_fca_alltoallw(void *sbuf, int *scounts, int *sdisps, comm, fca_module->previous_alltoallw_module); } - int mca_coll_fca_gather(void *sbuf, int scount, struct ompi_datatype_t *sdtype, void *rbuf, int rcount, @@ -471,5 +473,3 @@ int mca_coll_fca_reduce_scatter(void *sbuf, void *rbuf, int *rcounts, return fca_module->previous_reduce_scatter(sbuf, rbuf, rcounts, dtype, op, comm, fca_module->previous_reduce_scatter_module); } - -