ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

Add FCA 1.2/2.0 backward compatibility, depending on OMPI_FCA_VERSION_xx macro definition.

Просмотр исходного кода 
This commit was SVN r24192.
Этот коммит содержится в:
Mike Dubman 2010-12-27 21:32:34 +00:00
родитель bfe611d3bd
Коммит b339a7a07b
5 изменённых файлов: 228 добавлений и 81 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

47
ompi/mca/coll/fca/coll_fca.h
Просмотреть файл

@ -10,9 +10,8 @@
#ifndef MCA_COLL_FCA_H
#define MCA_COLL_FCA_H
#include <fca_api.h>
#include "ompi_config.h"
#include "mpi.h"
#include "opal/mca/mca.h"
#include "ompi/mca/coll/coll.h"
@ -21,6 +20,8 @@
#include "ompi/mca/coll/base/coll_tags.h"
#include "ompi/communicator/communicator.h"
#include "ompi/op/op.h"
#include "coll_fca_api.h"
#include "coll_fca_debug.h"
#ifdef OMPI_DATATYPE_MAX_PREDEFINED
@ -47,45 +48,7 @@ BEGIN_C_DECLS
* Used to load the library dynamically.
*/
struct mca_coll_fca_fca_ops_t {
/* FCA Context operations */
int (*init)(fca_init_spec_t *spec, fca_t **context);
void (*cleanup)(fca_t *context);
void (*progress)(fca_t *context);
/* Fabric communicator creation */
int (*comm_new)(fca_t *context, fca_comm_new_spec_t *spec, fca_comm_desc_t *comm_desc);
int (*comm_end)(fca_t *context, int comm_id);
void* (*get_rank_info)(fca_t *context, int *size);
void (*free_rank_info)(void *rank_info);
/* Local communicator creation */
int (*comm_init)(fca_t *context, fca_comm_init_spec_t *spec, fca_comm_t** fca_comm);
void (*comm_destroy)(fca_comm_t *comm);
int (*comm_get_caps)(fca_comm_t *comm, fca_comm_caps_t *caps);
/* Collectives supported by FCA */
int (*do_reduce)(fca_comm_t *comm, fca_reduce_spec_t *spec);
int (*do_all_reduce)(fca_comm_t *comm, fca_reduce_spec_t *spec);
int (*do_bcast)(fca_comm_t *comm, fca_bcast_spec_t *spec);
int (*do_allgather)(fca_comm_t *comm, fca_gather_spec_t *spec);
int (*do_allgatherv)(fca_comm_t *comm, fca_gatherv_spec_t *spec);
int (*do_barrier)(fca_comm_t *comm);
/* Helper functions */
unsigned long (*get_version)(void);
char * (*get_version_string)(void);
int (*maddr_ib_pton)(const char *mlid_str, const char *mgid_str, fca_mcast_addr_t *dst);
int (*maddr_inet_pton)(int af, const char *src, fca_mcast_addr_t *dst);
fca_init_spec_t *(*parse_spec_file)(char* spec_ini_file);
void (*free_init_spec)(fca_init_spec_t *fca_init_spec);
int (*translate_mpi_op)(char *mpi_op);
int (*translate_mpi_dtype)(char *mpi_dtype);
int (*get_dtype_size)(int dtype);
const char* (*strerror)(int code);
};
typedef struct mca_coll_fca_fca_ops_t mca_coll_fca_fca_ops_t;
/**
* FCA data type information
@ -184,8 +147,8 @@ struct mca_coll_fca_component_t {
int fca_np;
/* FCA global stuff */
void *fca_lib_handle; /* FCA dynamic library */
mca_coll_fca_fca_ops_t fca_ops; /* FCA operations */
void *fca_lib_handle; /* FCA dynamic library */
mca_coll_fca_ops_t fca_ops; /* FCA operations */
fca_t *fca_context; /* FCA context handle */
mca_coll_fca_dtype_info_t fca_dtypes[FCA_DT_MAX_PREDEFINED]; /* FCA dtype translation */
mca_coll_fca_op_info_t fca_reduce_ops[FCA_MAX_OPS]; /* FCA op translation */

174
ompi/mca/coll/fca/coll_fca_api.h Обычный файл
Просмотреть файл

@ -0,0 +1,174 @@
/**
Copyright (c) 2010 Voltaire, Inc. All rights reserved.
$COPYRIGHT$
Additional copyrights may follow
$HEADER$
*/
#include "ompi_config.h"
#include <fca_api.h>
/*
* FCA API compatibility layer.
* MPI build must define an FCA version macro.
*/
#ifdef OMPI_FCA_VERSION_1_2
#define FCA_API_ABI_MAJOR 1
#define FCA_API_ABI_MINOR 2
#define FCA_MAJOR_BIT 24ul
#define FCA_MINOR_BIT 16ul
#define EUSEMPI 287
typedef struct mca_coll_fca_ops_t {
/* FCA Context operations */
int (*init)(fca_init_spec_t *spec, fca_t **context);
void (*cleanup)(fca_t *context);
/* Fabric communicator creation */
int (*comm_new)(fca_t *context, fca_comm_new_spec_t *spec, fca_comm_desc_t *comm_desc);
int (*comm_end)(fca_t *context, int comm_id);
void* (*get_rank_info)(fca_t *context, int *size);
void (*free_rank_info)(void *rank_info);
/* Local communicator creation */
int (*comm_init)(fca_t *context, int proc_idx, int num_procs, int comm_size,
fca_comm_desc_t *comm_desc, fca_comm_t** fca_comm);
void (*comm_destroy)(fca_comm_t *comm);
int (*comm_get_caps)(fca_comm_t *comm, fca_comm_caps_t *caps);
/* Collectives supported by FCA */
int (*do_reduce)(fca_comm_t *comm, fca_reduce_spec_t *spec);
int (*do_all_reduce)(fca_comm_t *comm, fca_reduce_spec_t *spec);
int (*do_bcast)(fca_comm_t *comm, fca_bcast_spec_t *spec);
int (*do_barrier)(fca_comm_t *comm);
/* Helper functions */
unsigned long (*get_version)(void);
char * (*get_version_string)(void);
fca_init_spec_t *(*parse_spec_file)(char* spec_ini_file);
void (*free_init_spec)(fca_init_spec_t *fca_init_spec);
int (*translate_mpi_op)(char *mpi_op);
int (*translate_mpi_dtype)(char *mpi_dtype);
int (*get_dtype_size)(int dtype);
const char* (*strerror)(int code);
} mca_coll_fca_ops_t;
static inline int mca_coll_fca_comm_init(mca_coll_fca_ops_t *fca_ops,
fca_t *fca_context, int rank, int comm_size,
int local_proc_idx, int num_local_procs,
fca_comm_desc_t *comm_desc,
fca_comm_t **fca_comm)
{
return fca_ops->comm_init(fca_context, local_proc_idx, num_local_procs,
comm_size, comm_desc, fca_comm);
}
static inline void mca_coll_fca_get_bcast_root(int root_rank, int *local_ranks,
int num_local_ranks,
fca_bcast_spec_t *spec)
{
int i;
for (i = 0; i < num_local_ranks; ++i) {
if (local_ranks[i] == root_rank) {
spec->root_indx = i;
return;
}
}
spec->root_indx = -1;
}
static inline void mca_coll_fca_get_reduce_root(int root_rank, int my_rank,
fca_reduce_spec_t *spec)
{
spec->is_root = root_rank == my_rank;
}
#elif OMPI_FCA_VERSION_2_0
#define FCA_API_ABI_MAJOR 2
#define FCA_API_ABI_MINOR 0
#define OMPI_FCA_ALLGATHER 1
#define OMPI_FCA_PROGRESS 1
typedef struct mca_coll_fca_ops_t {
/* FCA Context operations */
int (*init)(fca_init_spec_t *spec, fca_t **context);
void (*cleanup)(fca_t *context);
void (*progress)(fca_t *context);
/* Fabric communicator creation */
int (*comm_new)(fca_t *context, fca_comm_new_spec_t *spec, fca_comm_desc_t *comm_desc);
int (*comm_end)(fca_t *context, int comm_id);
void* (*get_rank_info)(fca_t *context, int *size);
void (*free_rank_info)(void *rank_info);
/* Local communicator creation */
int (*comm_init)(fca_t *context, fca_comm_init_spec_t *spec, fca_comm_t** fca_comm);
void (*comm_destroy)(fca_comm_t *comm);
int (*comm_get_caps)(fca_comm_t *comm, fca_comm_caps_t *caps);
/* Collectives supported by FCA */
int (*do_reduce)(fca_comm_t *comm, fca_reduce_spec_t *spec);
int (*do_all_reduce)(fca_comm_t *comm, fca_reduce_spec_t *spec);
int (*do_bcast)(fca_comm_t *comm, fca_bcast_spec_t *spec);
int (*do_barrier)(fca_comm_t *comm);
int (*do_allgather)(fca_comm_t *comm, fca_gather_spec_t *spec);
int (*do_allgatherv)(fca_comm_t *comm, fca_gatherv_spec_t *spec);
/* Helper functions */
unsigned long (*get_version)(void);
char * (*get_version_string)(void);
fca_init_spec_t *(*parse_spec_file)(char* spec_ini_file);
void (*free_init_spec)(fca_init_spec_t *fca_init_spec);
int (*translate_mpi_op)(char *mpi_op);
int (*translate_mpi_dtype)(char *mpi_dtype);
int (*get_dtype_size)(int dtype);
const char* (*strerror)(int code);
} mca_coll_fca_ops_t;
static inline int mca_coll_fca_comm_init(mca_coll_fca_ops_t *fca_ops,
fca_t *fca_context, int rank, int comm_size,
int local_proc_idx, int num_local_procs,
fca_comm_desc_t *comm_desc,
fca_comm_t **fca_comm)
{
fca_comm_init_spec_t spec;
spec.rank = rank;
spec.size = comm_size;
spec.desc = *comm_desc;
spec.proc_idx = local_proc_idx;
spec.num_procs = num_local_procs;
return fca_ops->comm_init(fca_context, &spec, fca_comm);
}
static inline void mca_coll_fca_get_bcast_root(int root_rank, int *local_ranks,
int num_local_ranks,
fca_bcast_spec_t *spec)
{
spec->root = root_rank;
}
static inline void mca_coll_fca_get_reduce_root(int root_rank, int my_rank,
fca_reduce_spec_t *spec)
{
spec->root = root_rank;
}
#else
#error "FCA API version is undefinded"
#endif

16
ompi/mca/coll/fca/coll_fca_component.c
Просмотреть файл

@ -65,8 +65,6 @@ mca_coll_fca_component_t mca_coll_fca_component = {
}
};
#define FCA_API_ABI_MAJOR (2)
#define FCA_API_ABI_MINOR (0)
#define FCA_API_CLEAR_MICRO(__x) ((__x>>FCA_MINOR_BIT)<<FCA_MINOR_BIT)
#define FCA_API_VER(__major,__minor) (__major<<FCA_MAJOR_BIT | __minor<<FCA_MINOR_BIT)
@ -97,11 +95,13 @@ static void mca_coll_fca_progress_cb(void *arg)
*/
static int mca_coll_fca_mpi_progress_cb(void)
{
#ifdef OMPI_FCA_PROGRESS
if (!mca_coll_fca_component.fca_context)
return 0;
if (mca_coll_fca_component.fca_ops.progress)
mca_coll_fca_component.fca_ops.progress(mca_coll_fca_component.fca_context);
#endif
return 0;
}
@ -124,8 +124,6 @@ static void mca_coll_fca_init_fca_translations(void)
}
}
int mca_coll_fca_get_fca_lib(struct ompi_communicator_t *comm)
{
struct fca_init_spec *spec;
@ -146,8 +144,8 @@ int mca_coll_fca_get_fca_lib(struct ompi_communicator_t *comm)
FCA_VERBOSE(1, "FCA Loaded from: %s", mca_coll_fca_component.fca_lib_path);
GET_FCA_SYM(get_version);
GET_FCA_SYM(get_version_string);
fca_ver = FCA_API_CLEAR_MICRO(mca_coll_fca_component.fca_ops.get_version());
fca_ver = FCA_API_CLEAR_MICRO(mca_coll_fca_component.fca_ops.get_version());
if (fca_ver != FCA_API_VER(FCA_API_ABI_MAJOR,FCA_API_ABI_MINOR)) {
FCA_ERROR("Unsupported FCA version: %s, please update FCA to v%d.%d",
mca_coll_fca_component.fca_ops.get_version_string(),
@ -158,7 +156,9 @@ int mca_coll_fca_get_fca_lib(struct ompi_communicator_t *comm)
GET_FCA_SYM(init);
GET_FCA_SYM(cleanup);
#ifdef OMPI_FCA_PROGRESS
GET_FCA_SYM(progress);
#endif
GET_FCA_SYM(comm_new);
GET_FCA_SYM(comm_end);
GET_FCA_SYM(get_rank_info);
@ -170,10 +170,10 @@ int mca_coll_fca_get_fca_lib(struct ompi_communicator_t *comm)
GET_FCA_SYM(do_all_reduce);
GET_FCA_SYM(do_bcast);
GET_FCA_SYM(do_barrier);
#ifdef OMPI_FCA_ALLGATHER
GET_FCA_SYM(do_allgather);
GET_FCA_SYM(do_allgatherv);
GET_FCA_SYM(maddr_ib_pton);
GET_FCA_SYM(maddr_inet_pton);
#endif
GET_FCA_SYM(parse_spec_file);
GET_FCA_SYM(free_init_spec);
GET_FCA_SYM(translate_mpi_op);
@ -213,8 +213,6 @@ static void mca_coll_fca_close_fca_lib(void)
mca_coll_fca_component.fca_lib_handle = NULL;
}
static int fca_register(void)
{
mca_base_component_t *c;

54
ompi/mca/coll/fca/coll_fca_module.c
Просмотреть файл

@ -50,19 +50,35 @@ static int __get_local_ranks(mca_coll_fca_module_t *fca_module)
{
ompi_communicator_t *comm = fca_module->comm;
ompi_proc_t* proc;
int rank;
int i, rank;
/* Count the local ranks */
fca_module->num_local_procs = 0;
for (rank = 0; rank < ompi_comm_size(comm); ++rank) {
proc = __local_rank_lookup(comm, rank);
if (!FCA_IS_LOCAL_PROCESS(proc->proc_flags))
continue;
if (rank == fca_module->rank) {
fca_module->local_proc_idx = fca_module->num_local_procs;
if (FCA_IS_LOCAL_PROCESS(proc->proc_flags)) {
if (rank == fca_module->rank) {
fca_module->local_proc_idx = fca_module->num_local_procs;
}
++fca_module->num_local_procs;
}
}
/* Make a list of local ranks */
fca_module->local_ranks = calloc(fca_module->num_local_procs,
sizeof *fca_module->local_ranks);
if (!fca_module->local_ranks) {
FCA_ERROR("Failed to allocate memory for %d local ranks",
fca_module->num_local_procs);
return OMPI_ERROR;
}
i = 0;
for (rank = 0; rank < ompi_comm_size(comm); ++rank) {
proc = __local_rank_lookup(comm, rank);
if (FCA_IS_LOCAL_PROCESS(proc->proc_flags)) {
fca_module->local_ranks[i++] = rank;
}
++fca_module->num_local_procs;
}
FCA_MODULE_VERBOSE(fca_module, 3, "i am %d/%d", fca_module->local_proc_idx,
@ -70,7 +86,6 @@ static int __get_local_ranks(mca_coll_fca_module_t *fca_module)
return OMPI_SUCCESS;
}
static int __fca_comm_new(mca_coll_fca_module_t *fca_module)
{
ompi_communicator_t *comm = fca_module->comm;
@ -173,28 +188,24 @@ static int __fca_comm_new(mca_coll_fca_module_t *fca_module)
static int __create_fca_comm(mca_coll_fca_module_t *fca_module)
{
fca_comm_init_spec_t spec;
int rc, ret;
int comm_size;
int rc, ret;
rc = __fca_comm_new(fca_module);
if (rc != OMPI_SUCCESS)
return rc;
/* allocate comm_init_spec */
comm_size = ompi_comm_size(fca_module->comm);
spec.rank = fca_module->rank;
spec.size = comm_size;
spec.desc = fca_module->fca_comm_desc;
spec.proc_idx = fca_module->local_proc_idx;
spec.num_procs = fca_module->num_local_procs;
FCA_MODULE_VERBOSE(fca_module, 1, "Starting COMM_INIT comm_id %d proc_idx %d num_procs %d",
fca_module->fca_comm_desc.comm_id, fca_module->local_proc_idx,
fca_module->num_local_procs);
ret = mca_coll_fca_component.fca_ops.comm_init(mca_coll_fca_component.fca_context,
&spec, &fca_module->fca_comm);
comm_size = ompi_comm_size(fca_module->comm);
ret = mca_coll_fca_comm_init(&mca_coll_fca_component.fca_ops,
mca_coll_fca_component.fca_context,
fca_module->rank, comm_size,
fca_module->local_proc_idx, fca_module->num_local_procs,
&fca_module->fca_comm_desc, &fca_module->fca_comm);
if (ret < 0) {
FCA_ERROR("COMM_INIT failed: %s", mca_coll_fca_component.fca_ops.strerror(ret));
return OMPI_ERROR;
@ -304,6 +315,7 @@ static int mca_coll_fca_ft_event(int state)
static void mca_coll_fca_module_clear(mca_coll_fca_module_t *fca_module)
{
fca_module->num_local_procs = 0;
fca_module->local_ranks = NULL;
fca_module->fca_comm = NULL;
fca_module->previous_barrier = NULL;
@ -343,7 +355,7 @@ static void mca_coll_fca_module_destruct(mca_coll_fca_module_t *fca_module)
OBJ_RELEASE(fca_module->previous_reduce_scatter_module);
if (fca_module->fca_comm)
__destroy_fca_comm(fca_module);
free(fca_module->local_ranks);
mca_coll_fca_module_clear(fca_module);
}

14
ompi/mca/coll/fca/coll_fca_ops.c
Просмотреть файл

@ -180,7 +180,8 @@ int mca_coll_fca_bcast(void *buff, int count, struct ompi_datatype_t *datatype,
FCA_VERBOSE(5,"Using FCA Bcast");
spec.buf = buff;
spec.root = root;
mca_coll_fca_get_bcast_root(root, fca_module->local_ranks,
fca_module->num_local_procs, &spec);
ret = mca_coll_fca_component.fca_ops.do_bcast(fca_module->fca_comm, &spec);
if (ret < 0) {
if (ret == -EUSEMPI) {
@ -208,12 +209,11 @@ int mca_coll_fca_reduce(void *sbuf, void *rbuf, int count,
int root, struct ompi_communicator_t *comm,
mca_coll_base_module_t *module)
{
mca_coll_fca_module_t *fca_module = (mca_coll_fca_module_t*)module;
fca_reduce_spec_t spec;
int ret;
spec.root = root;
mca_coll_fca_get_reduce_root(root, fca_module->rank, &spec);
spec.sbuf = sbuf;
spec.rbuf = rbuf;
if (mca_coll_fca_fill_reduce_spec(count, dtype, op, &spec,
@ -293,6 +293,7 @@ int mca_coll_fca_allgather(void *sbuf, int scount, struct ompi_datatype_t *sdtyp
mca_coll_base_module_t *module)
{
mca_coll_fca_module_t *fca_module = (mca_coll_fca_module_t*)module;
#ifdef OMPI_FCA_ALLGATHER
fca_gather_spec_t spec = {0,};
int ret;
@ -329,6 +330,7 @@ int mca_coll_fca_allgather(void *sbuf, int scount, struct ompi_datatype_t *sdtyp
return OMPI_SUCCESS;
orig_allgather:
#endif
return fca_module->previous_allgather(sbuf, scount, sdtype, rbuf, rcount, rdtype,
comm, fca_module->previous_allgather_module);
}
@ -342,6 +344,7 @@ int mca_coll_fca_allgatherv(void *sbuf, int scount,
mca_coll_base_module_t *module)
{
mca_coll_fca_module_t *fca_module = (mca_coll_fca_module_t*)module;
#ifdef OMPI_FCA_ALLGATHER
fca_gatherv_spec_t spec;
int relemsize;
int comm_size;
@ -388,6 +391,7 @@ int mca_coll_fca_allgatherv(void *sbuf, int scount,
return OMPI_SUCCESS;
orig_allgatherv:
#endif
return fca_module->previous_allgatherv(sbuf, scount, sdtype, rbuf, rcounts,
disps, rdtype, comm,
fca_module->previous_allgatherv_module);
@ -419,7 +423,6 @@ int mca_coll_fca_alltoallv(void *sbuf, int *scounts, int *sdisps,
comm, fca_module->previous_alltoallv_module);
}
int mca_coll_fca_alltoallw(void *sbuf, int *scounts, int *sdisps,
struct ompi_datatype_t **sdtypes,
void *rbuf, int *rcounts, int *rdisps,
@ -433,7 +436,6 @@ int mca_coll_fca_alltoallw(void *sbuf, int *scounts, int *sdisps,
comm, fca_module->previous_alltoallw_module);
}
int mca_coll_fca_gather(void *sbuf, int scount,
struct ompi_datatype_t *sdtype,
void *rbuf, int rcount,
@ -471,5 +473,3 @@ int mca_coll_fca_reduce_scatter(void *sbuf, void *rbuf, int *rcounts,
return fca_module->previous_reduce_scatter(sbuf, rbuf, rcounts, dtype, op,
comm, fca_module->previous_reduce_scatter_module);
}