ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

OSHMEM: add support for MPI collectives

Просмотр исходного кода 
oshmem collective can select MPI collectives modules as provider.

Developed by Elena, reviewed by Miked

cmr=v1.7.5:reviewer=ompi-rm1.7

This commit was SVN r30808.
Этот коммит содержится в:
Mike Dubman 2014-02-25 15:01:10 +00:00
родитель 9ea9bec4ad
Коммит 180e4f06bf
9 изменённых файлов: 754 добавлений и 5 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

34
oshmem/mca/scoll/mpi/Makefile.am Обычный файл
Просмотреть файл

@ -0,0 +1,34 @@
# Copyright (c) 2013 Mellanox Technologies, Inc.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
#
scoll_mpi_sources = \
scoll_mpi.h \
scoll_mpi_debug.h \
scoll_mpi_dtypes.h \
scoll_mpi_module.c \
scoll_mpi_component.c \
scoll_mpi_ops.c
if MCA_BUILD_oshmem_scoll_mpi_DSO
component_noinst =
component_install = mca_scoll_mpi.la
else
component_noinst = libmca_scoll_mpi.la
component_install =
endif
mcacomponentdir = $(pkglibdir)
mcacomponent_LTLIBRARIES = $(component_install)
mca_scoll_mpi_la_SOURCES = $(scoll_mpi_sources)
mca_scoll_mpi_la_LIBADD = $(scoll_mpi_LIBS)
mca_scoll_mpi_la_LDFLAGS = -module -avoid-version $(scoll_mpi_LDFLAGS)
noinst_LTLIBRARIES = $(component_noinst)
libmca_scoll_mpi_la_SOURCES =$(scoll_mpi_sources)
libmca_scoll_mpi_la_LIBADD = $(scoll_mpi_LIBS)
libmca_scoll_mpi_la_LDFLAGS = -module -avoid-version $(scoll_mpi_LDFLAGS)

112
oshmem/mca/scoll/mpi/scoll_mpi.h Обычный файл
Просмотреть файл

@ -0,0 +1,112 @@
/**
Copyright (c) 2011 Mellanox Technologies. All rights reserved.
$COPYRIGHT$
Additional copyrights may follow
$HEADER$
*/
#ifndef MCA_SCOLL_MPI_H
#define MCA_SCOLL_MPI_H
#include "oshmem_config.h"
#include "shmem.h"
#include "opal/mca/mca.h"
#include "oshmem/mca/scoll/scoll.h"
#include "oshmem/request/request.h"
#include "oshmem/util/oshmem_util.h"
#include "oshmem/proc/proc.h"
#include "ompi/communicator/communicator.h"
#include "orte/runtime/orte_globals.h"
#include "scoll_mpi_debug.h"
BEGIN_C_DECLS
/**
* Globally exported structure
*/
struct mca_scoll_mpi_component_t {
/** Base coll component */
mca_scoll_base_component_1_0_0_t super;
/** MCA parameter: Priority of this component */
int mpi_priority;
/** MCA parameter: Verbose level of this component */
int mpi_verbose;
/** MCA parameter: Enable MPI */
int mpi_enable;
/** MCA parameter: Minimal number of processes in the communicator
for the corresponding mpi context to be created */
int mpi_np;
};
typedef struct mca_scoll_mpi_component_t mca_scoll_mpi_component_t;
OMPI_MODULE_DECLSPEC extern mca_scoll_mpi_component_t mca_scoll_mpi_component;
/**
* MPI enabled communicator
*/
struct mca_scoll_mpi_module_t {
mca_scoll_base_module_t super;
ompi_communicator_t *comm;
int rank;
/* Saved handlers - for fallback */
mca_scoll_base_module_reduce_fn_t previous_reduce;
mca_scoll_base_module_t *previous_reduce_module;
mca_scoll_base_module_broadcast_fn_t previous_broadcast;
mca_scoll_base_module_t *previous_broadcast_module;
mca_scoll_base_module_barrier_fn_t previous_barrier;
mca_scoll_base_module_t *previous_barrier_module;
mca_scoll_base_module_collect_fn_t previous_collect;
mca_scoll_base_module_t *previous_collect_module;
};
typedef struct mca_scoll_mpi_module_t mca_scoll_mpi_module_t;
OBJ_CLASS_DECLARATION(mca_scoll_mpi_module_t);
/* API functions */
int mca_scoll_mpi_init_query(bool enable_progress_threads, bool enable_mpi_threads);
mca_scoll_base_module_t* mca_scoll_mpi_comm_query(oshmem_group_t *osh_group, int *priority);
int mca_scoll_mpi_barrier(struct oshmem_group_t *group, long *pSync, int alg);
int mca_scoll_mpi_broadcast(struct oshmem_group_t *group,
int PE_root,
void *target,
const void *source,
size_t nlong,
long *pSync,
int alg);
int mca_scoll_mpi_collect(struct oshmem_group_t *group,
void *target,
const void *source,
size_t nlong,
long *pSync,
bool nlong_type,
int alg);
int mca_scoll_mpi_reduce(struct oshmem_group_t *group,
struct oshmem_op_t *op,
void *target,
const void *source,
size_t nlong,
long *pSync,
void *pWrk,
int alg);
END_C_DECLS
#endif

171
oshmem/mca/scoll/mpi/scoll_mpi_component.c Обычный файл
Просмотреть файл

@ -0,0 +1,171 @@
/**
Copyright (c) 2011 Mellanox Technologies. All rights reserved.
$COPYRIGHT$
Additional copyrights may follow
$HEADER$
*/
#include <stdio.h>
#include <dlfcn.h>
#include <libgen.h>
#include "scoll_mpi.h"
/*
* Public string showing the oshmem scoll_mpi component version number
*/
const char *mca_scoll_mpi_component_version_string =
"OpenSHMEM MPI collective MCA component version " OSHMEM_VERSION;
static int mpi_open(void);
static int mpi_close(void);
static int mpi_register(void);
int mca_scoll_mpi_output = -1;
mca_scoll_mpi_component_t mca_scoll_mpi_component = {
/* First, the mca_component_t struct containing meta information
about the component itfca */
{
{
MCA_SCOLL_BASE_VERSION_2_0_0,
/* Component name and version */
"mpi",
OSHMEM_MAJOR_VERSION,
OSHMEM_MINOR_VERSION,
OSHMEM_RELEASE_VERSION,
/* Component open and close functions */
mpi_open,
mpi_close,
NULL,
mpi_register
},
{
/* The component is not checkpoint ready */
MCA_BASE_METADATA_PARAM_NONE
},
/* Initialization / querying functions */
mca_scoll_mpi_init_query,
mca_scoll_mpi_comm_query,
},
90, /* priority */
0, /* verbose level */
1 /* mpi_enable */
};
/*
* * Local flags
* */
enum {
REGINT_NEG_ONE_OK = 0x01,
REGINT_GE_ZERO = 0x02,
REGINT_GE_ONE = 0x04,
REGINT_NONZERO = 0x08,
REGINT_MAX = 0x88
};
enum {
REGSTR_EMPTY_OK = 0x01,
REGSTR_MAX = 0x88
};
/*
* Utility routine for integer parameter registration
*/
static int reg_int(const char* param_name,
const char* deprecated_param_name,
const char* param_desc,
int default_value, int *storage, int flags)
{
int index;
*storage = default_value;
index = mca_base_component_var_register(
&mca_scoll_mpi_component.super.scoll_version,
param_name, param_desc, MCA_BASE_VAR_TYPE_INT,
NULL, 0, 0,OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY, storage);
if (NULL != deprecated_param_name) {
(void) mca_base_var_register_synonym(index,
"oshmem", "scoll", "mpi", deprecated_param_name,
MCA_BASE_VAR_SYN_FLAG_DEPRECATED);
}
if (0 != (flags & REGINT_NEG_ONE_OK) && -1 == *storage) {
return OSHMEM_SUCCESS;
}
if ((0 != (flags & REGINT_GE_ZERO) && *storage < 0) ||
(0 != (flags & REGINT_GE_ONE) && *storage < 1) ||
(0 != (flags & REGINT_NONZERO) && 0 == *storage)) {
opal_output(0, "Bad parameter value for parameter \"%s\"",
param_name);
return OSHMEM_ERR_BAD_PARAM;
}
return OSHMEM_SUCCESS;
}
static int mpi_register(void)
{
int ret, tmp;
ret = OSHMEM_SUCCESS;
#define CHECK(expr) do { \
tmp = (expr); \
if (OSHMEM_SUCCESS != tmp) ret = tmp; \
} while (0)
CHECK(reg_int("priority",NULL,
"Priority of the mpi coll component",
90,
&mca_scoll_mpi_component.mpi_priority,
0));
CHECK(reg_int("verbose", NULL,
"Verbose level of the mpi coll component",
0,
&mca_scoll_mpi_component.mpi_verbose,
0));
CHECK(reg_int("enable",NULL,
"[1|0|] Enable/Disable MPI scoll component",
1 /*enable by default*/,
&mca_scoll_mpi_component.mpi_enable,
0));
CHECK(reg_int("np",NULL,
"Minimal number of processes in the communicator"
" for the corresponding mpi context to be created (default: 32)",
2 /*enable by default*/,
&mca_scoll_mpi_component.mpi_np,
0));
return ret;
}
static int mpi_open(void)
{
mca_scoll_mpi_component_t *cm;
cm = &mca_scoll_mpi_component;
mca_scoll_mpi_output = opal_output_open(NULL);
opal_output_set_verbosity(mca_scoll_mpi_output, cm->mpi_verbose);
return OSHMEM_SUCCESS;
}
static int mpi_close(void)
{
return OSHMEM_SUCCESS;
}

35
oshmem/mca/scoll/mpi/scoll_mpi_debug.h Обычный файл
Просмотреть файл

@ -0,0 +1,35 @@
/**
Copyright (c) 2011 Mellanox Technologies. All rights reserved.
$COPYRIGHT$
Additional copyrights may follow
$HEADER$
*/
#ifndef SCOLL_MPI_DEBUG_H
#define SCOLL_MPI_DEBUG_H
#include "oshmem_config.h"
#pragma GCC system_header
#ifdef __BASE_FILE__
#define __MPI_FILE__ __BASE_FILE__
#else
#define __MPI_FILE__ __FILE__
#endif
#define MPI_COLL_VERBOSE(level, format, ...) \
opal_output_verbose(level, mca_scoll_mpi_output, "%s:%d - %s() " format, \
__MPI_FILE__, __LINE__, __FUNCTION__, ## __VA_ARGS__)
#define MPI_COLL_ERROR(format, ... ) \
opal_output_verbose(0, mca_scoll_mpi_output, "Error: %s:%d - %s() " format, \
__MPI_FILE__, __LINE__, __FUNCTION__, ## __VA_ARGS__)
#define MPI_MODULE_VERBOSE(mpi_module, level, format, ...) \
MPI_COLL_VERBOSE(level, "[%p:%d] " format, (void*)(mpi_module)->comm, (mpi_module)->rank, ## __VA_ARGS__)
extern int mca_scoll_mpi_output;
#endif // SCOLL_MPI_DEBUG_H

61
oshmem/mca/scoll/mpi/scoll_mpi_dtypes.h Обычный файл
Просмотреть файл

@ -0,0 +1,61 @@
#ifndef SCOLL_MPI_DTYPES_H
#define SCOLL_MPI_DTYPES_H
#include "oshmem/op/op.h"
#include "ompi/datatype/ompi_datatype.h"
#include "ompi/op/op.h"
static struct ompi_datatype_t* shmem_dtype_to_ompi_dtype(oshmem_op_t *op)
{
int dtype = op->dt;
int dtsize = op->dt_size * 8;
switch (dtype) {
case OSHMEM_OP_TYPE_FLOAT:
return &ompi_mpi_float.dt;
case OSHMEM_OP_TYPE_DOUBLE:
return &ompi_mpi_double.dt;
case OSHMEM_OP_TYPE_LDOUBLE:
return &ompi_mpi_long_double.dt;
case OSHMEM_OP_TYPE_FCOMPLEX:
return &ompi_mpi_c_float_complex.dt;
case OSHMEM_OP_TYPE_DCOMPLEX:
return &ompi_mpi_c_double_complex.dt;
default:
switch (dtsize) {
case 64:
return &ompi_mpi_int64_t.dt;
case 32:
return &ompi_mpi_int32_t.dt;
case 16:
return &ompi_mpi_int16_t.dt;
case 8:
return &ompi_mpi_int8_t.dt;
default:
return &ompi_mpi_datatype_null.dt;
}
}
}
static struct ompi_op_t* shmem_op_to_ompi_op(int op)
{
switch (op) {
case OSHMEM_OP_AND:
return &ompi_mpi_op_band;
case OSHMEM_OP_OR:
return &ompi_mpi_op_bor;
case OSHMEM_OP_XOR:
return &ompi_mpi_op_bxor;
case OSHMEM_OP_MAX:
return &ompi_mpi_op_max;
case OSHMEM_OP_MIN:
return &ompi_mpi_op_min;
case OSHMEM_OP_SUM:
return &ompi_mpi_op_sum;
case OSHMEM_OP_PROD:
return &ompi_mpi_op_prod;
default:
return &ompi_mpi_op_null;
}
}
#endif /* SCOLL_MPI_DTYPES_H */

154
oshmem/mca/scoll/mpi/scoll_mpi_module.c Обычный файл
Просмотреть файл

@ -0,0 +1,154 @@
/**
Copyright (c) 2011 Mellanox Technologies. All rights reserved.
$COPYRIGHT$
Additional copyrights may follow
$HEADER$
*/
#include "ompi_config.h"
#include "scoll_mpi.h"
#include "oshmem/proc/proc.h"
#include "ompi/mca/coll/base/base.h"
int mca_scoll_mpi_init_query(bool enable_progress_threads, bool enable_mpi_threads)
{
return OSHMEM_ERROR;
}
static void mca_scoll_mpi_module_clear(mca_scoll_mpi_module_t *mpi_module)
{
mpi_module->previous_barrier = NULL;
mpi_module->previous_broadcast = NULL;
mpi_module->previous_reduce = NULL;
mpi_module->previous_collect = NULL;
}
static void mca_scoll_mpi_module_construct(mca_scoll_mpi_module_t *mpi_module)
{
mca_scoll_mpi_module_clear(mpi_module);
}
static void mca_scoll_mpi_module_destruct(mca_scoll_mpi_module_t *mpi_module)
{
mca_scoll_mpi_module_t *module;
OBJ_RELEASE(mpi_module->previous_barrier_module);
OBJ_RELEASE(mpi_module->previous_broadcast_module);
OBJ_RELEASE(mpi_module->previous_reduce_module);
OBJ_RELEASE(mpi_module->previous_collect_module);
mca_scoll_mpi_module_clear(mpi_module);
/* Free ompi_comm */
if (mpi_module->comm != &(ompi_mpi_comm_world.comm) && (NULL != mpi_module->comm)) {
ompi_comm_free(&mpi_module->comm);
}
}
#define MPI_SAVE_PREV_SCOLL_API(__api) do {\
mpi_module->previous_ ## __api = osh_group->g_scoll.scoll_ ## __api;\
mpi_module->previous_ ## __api ## _module = osh_group->g_scoll.scoll_ ## __api ## _module;\
if (!osh_group->g_scoll.scoll_ ## __api || !osh_group->g_scoll.scoll_ ## __api ## _module) {\
MPI_COLL_VERBOSE(1, "no underlying " # __api"; disqualifying myself");\
return OSHMEM_ERROR;\
}\
OBJ_RETAIN(mpi_module->previous_ ## __api ## _module);\
} while(0)
static int mca_scoll_mpi_save_coll_handlers(mca_scoll_base_module_t *module, oshmem_group_t *osh_group)
{
mca_scoll_mpi_module_t* mpi_module = (mca_scoll_mpi_module_t*) module;
MPI_SAVE_PREV_SCOLL_API(barrier);
MPI_SAVE_PREV_SCOLL_API(broadcast);
MPI_SAVE_PREV_SCOLL_API(reduce);
MPI_SAVE_PREV_SCOLL_API(collect);
return OSHMEM_SUCCESS;
}
/*
* Initialize module on the communicator
*/
static int mca_scoll_mpi_module_enable(mca_scoll_base_module_t *module,
oshmem_group_t *osh_group)
{
int ret;
if (OSHMEM_SUCCESS != mca_scoll_mpi_save_coll_handlers(module, osh_group)){
MPI_COLL_ERROR("scoll_mpi: mca_coll_mpi_save_coll_handlers failed");
return OSHMEM_ERROR;
}
return OSHMEM_SUCCESS;
}
/*
* Invoked when there's a new communicator that has been created.
* Look at the communicator and decide which set of functions and
* priority we want to return.
*/
mca_scoll_base_module_t *
mca_scoll_mpi_comm_query(oshmem_group_t *osh_group, int *priority)
{
mca_scoll_base_module_t *module;
mca_scoll_mpi_module_t *mpi_module;
int err, i;
int tag;
ompi_group_t* parent_group, *new_group;
ompi_communicator_t* newcomm = NULL;
*priority = 0;
mca_scoll_mpi_component_t *cm;
cm = &mca_scoll_mpi_component;
int* ranks;
if (!cm->mpi_enable){
return NULL;
}
if ((osh_group->proc_count < 2) || (osh_group->proc_count < cm->mpi_np)) {
return NULL;
}
/* Create OMPI_Comm object and store ptr to it in group obj*/
if (NULL == oshmem_group_all) {
osh_group->ompi_comm = &(ompi_mpi_comm_world.comm);
} else {
err = ompi_comm_group(&(ompi_mpi_comm_world.comm), &parent_group);
ranks = (int*) malloc(osh_group->proc_count * sizeof(int));
tag = 1;
for (i = 0; i < osh_group->proc_count; i++) {
ranks[i] = osh_group->proc_array[i]->proc_name.vpid;
}
err = ompi_group_incl(parent_group, osh_group->proc_count, ranks, &new_group);
err = ompi_comm_create_group(&(ompi_mpi_comm_world.comm), new_group, tag, &newcomm);
err = ompi_group_free(&new_group);
free(ranks);
osh_group->ompi_comm = newcomm;
}
mpi_module = OBJ_NEW(mca_scoll_mpi_module_t);
if (!mpi_module){
return NULL;
}
mpi_module->comm = osh_group->ompi_comm;
mpi_module->super.scoll_module_enable = mca_scoll_mpi_module_enable;
mpi_module->super.scoll_barrier = mca_scoll_mpi_barrier;
mpi_module->super.scoll_broadcast = mca_scoll_mpi_broadcast;
mpi_module->super.scoll_reduce = mca_scoll_mpi_reduce;
mpi_module->super.scoll_collect = mca_scoll_mpi_collect;
*priority = cm->mpi_priority;
module = &mpi_module->super;
return module;
}
OBJ_CLASS_INSTANCE(mca_scoll_mpi_module_t,
mca_scoll_base_module_t,
mca_scoll_mpi_module_construct,
mca_scoll_mpi_module_destruct);

177
oshmem/mca/scoll/mpi/scoll_mpi_ops.c Обычный файл
Просмотреть файл

@ -0,0 +1,177 @@
/**
Copyright (c) 2011 Mellanox Technologies. All rights reserved.
$COPYRIGHT$
Additional copyrights may follow
$HEADER$
*/
#include "ompi_config.h"
#include "ompi/constants.h"
#include "scoll_mpi.h"
#include "scoll_mpi_dtypes.h"
#define INCOMPATIBLE_SHMEM_OMPI_COLL_APIS 1
int mca_scoll_mpi_barrier(struct oshmem_group_t *group, long *pSync, int alg)
{
mca_scoll_mpi_module_t *mpi_module;
int rc;
MPI_COLL_VERBOSE(20,"RUNNING MPI BARRIER");
mpi_module = (mca_scoll_mpi_module_t *) group->g_scoll.scoll_barrier_module;
rc = mpi_module->comm->c_coll.coll_barrier(mpi_module->comm, mpi_module->comm->c_coll.coll_barrier_module);
if (OMPI_SUCCESS != rc){
MPI_COLL_VERBOSE(20,"RUNNING FALLBACK BARRIER");
rc = mpi_module->previous_barrier(group, pSync, SCOLL_DEFAULT_ALG);
}
return rc;
}
int mca_scoll_mpi_broadcast(struct oshmem_group_t *group,
int PE_root,
void *target,
const void *source,
size_t nlong,
long *pSync,
int alg)
{
mca_scoll_mpi_module_t *mpi_module;
ompi_datatype_t* dtype;
int rc;
void* buf;
int root;
MPI_COLL_VERBOSE(20,"RUNNING MPI BCAST");
mpi_module = (mca_scoll_mpi_module_t *) group->g_scoll.scoll_broadcast_module;
if (group->my_pe == PE_root) {
buf = (void *) source;
} else {
buf = target;
}
dtype = &ompi_mpi_char.dt;
root = oshmem_proc_group_find_id(group, PE_root);
#ifdef INCOMPATIBLE_SHMEM_OMPI_COLL_APIS
if (INT_MAX < nlong) {
MPI_COLL_VERBOSE(20,"RUNNING FALLBACK BCAST");
rc = mpi_module->previous_broadcast(group,
PE_root,
target,
source,
nlong,
pSync,
SCOLL_DEFAULT_ALG);
return rc;
}
rc = mpi_module->comm->c_coll.coll_bcast(buf, (int)nlong, dtype, root, mpi_module->comm, mpi_module->comm->c_coll.coll_bcast_module);
#else
rc = mpi_module->comm->c_coll.coll_bcast(buf, nlong, dtype, root, mpi_module->comm, mpi_module->comm->c_coll.coll_bcast_module);
#endif
if (OMPI_SUCCESS != rc){
MPI_COLL_VERBOSE(20,"RUNNING FALLBACK BCAST");
rc = mpi_module->previous_broadcast(group,
PE_root,
target,
source,
nlong,
pSync,
SCOLL_DEFAULT_ALG);
}
return rc;
}
int mca_scoll_mpi_collect(struct oshmem_group_t *group,
void *target,
const void *source,
size_t nlong,
long *pSync,
bool nlong_type,
int alg)
{
mca_scoll_mpi_module_t *mpi_module;
ompi_datatype_t* stype;
ompi_datatype_t* rtype;
int rc;
void *sbuf, *rbuf;
MPI_COLL_VERBOSE(20,"RUNNING MPI ALLGATHER");
mpi_module = (mca_scoll_mpi_module_t *) group->g_scoll.scoll_collect_module;
if (nlong_type == true) {
sbuf = (void *) source;
rbuf = target;
stype = &ompi_mpi_char.dt;
rtype = &ompi_mpi_char.dt;
#ifdef INCOMPATIBLE_SHMEM_OMPI_COLL_APIS
if (INT_MAX < nlong) {
MPI_COLL_VERBOSE(20,"RUNNING FALLBACK COLLECT");
rc = mpi_module->previous_collect(group, target, source, nlong, pSync, nlong_type, SCOLL_DEFAULT_ALG);
return rc;
}
rc = mpi_module->comm->c_coll.coll_allgather(sbuf, (int)nlong, stype, rbuf, (int)nlong, rtype, mpi_module->comm, mpi_module->comm->c_coll.coll_allgather_module);
#else
rc = mpi_module->comm->c_coll.coll_allgather(sbuf, nlong, stype, rbuf, nlong, rtype, mpi_module->comm, mpi_module->comm->c_coll.coll_allgather_module);
#endif
if (OMPI_SUCCESS != rc){
MPI_COLL_VERBOSE(20,"RUNNING FALLBACK FCOLLECT");
rc = mpi_module->previous_collect(group, target, source, nlong, pSync, nlong_type, SCOLL_DEFAULT_ALG);
}
} else {
MPI_COLL_VERBOSE(20,"RUNNING FALLBACK COLLECT");
rc = mpi_module->previous_collect(group, target, source, nlong, pSync, nlong_type, SCOLL_DEFAULT_ALG);
}
return rc;
}
int mca_scoll_mpi_reduce(struct oshmem_group_t *group,
struct oshmem_op_t *op,
void *target,
const void *source,
size_t nlong,
long *pSync,
void *pWrk,
int alg)
{
mca_scoll_mpi_module_t *mpi_module;
struct ompi_datatype_t* dtype;
struct ompi_op_t *h_op;
int rc;
int dtsize;
size_t count;
MPI_COLL_VERBOSE(20,"RUNNING MPI REDUCE");
void *sbuf, *rbuf;
mpi_module = (mca_scoll_mpi_module_t *) group->g_scoll.scoll_reduce_module;
sbuf = (void *) source;
rbuf = target;
dtype = shmem_dtype_to_ompi_dtype(op);
h_op = shmem_op_to_ompi_op(op->op);
count = nlong/op->dt_size;
#ifdef INCOMPATIBLE_SHMEM_OMPI_COLL_APIS
if (INT_MAX < count) {
MPI_COLL_VERBOSE(20,"RUNNING FALLBACK REDUCE");
rc = mpi_module->previous_reduce(group,
op,
target,
source,
nlong,
pSync,
pWrk,
SCOLL_DEFAULT_ALG);
return rc;
}
rc = mpi_module->comm->c_coll.coll_allreduce(sbuf, rbuf, (int)count, dtype, h_op, mpi_module->comm, mpi_module->comm->c_coll.coll_allreduce_module);
#else
rc = mpi_module->comm->c_coll.coll_allreduce(sbuf, rbuf, count, dtype, h_op, mpi_module->comm, mpi_module->comm->c_coll.coll_allreduce_module);
#endif
if (OMPI_SUCCESS != rc){
MPI_COLL_VERBOSE(20,"RUNNING FALLBACK REDUCE");
rc = mpi_module->previous_reduce(group,
op,
target,
source,
nlong,
pSync,
pWrk,
SCOLL_DEFAULT_ALG);
}
return rc;
}

12
oshmem/proc/proc.c
Просмотреть файл

@ -619,15 +619,17 @@ OSHMEM_DECLSPEC oshmem_group_t* oshmem_proc_group_create(int pe_start,
int pe_stride,
size_t pe_size)
{
int cur_pe, count_pe;
int i;
oshmem_group_t* group = NULL;
oshmem_proc_t** proc_array = NULL;
oshmem_proc_t* proc = NULL;
group = OBJ_NEW(oshmem_group_t);
if (group) {
int cur_pe = 0;
int count_pe = 0;
oshmem_proc_t** proc_array = NULL;
oshmem_proc_t* proc = NULL;
cur_pe = 0;
count_pe = 0;
OPAL_THREAD_LOCK(&oshmem_proc_lock);
@ -657,11 +659,11 @@ OSHMEM_DECLSPEC oshmem_group_t* oshmem_proc_group_create(int pe_start,
}
group->proc_array = proc_array;
group->proc_count = (int) count_pe;
group->ompi_comm = NULL;
/* Prepare peers list */
OBJ_CONSTRUCT(&(group->peer_list), opal_list_t);
{
int i = 0;
orte_namelist_t *peer = NULL;
for (i = 0; i < group->proc_count; i++) {

3
oshmem/proc/proc.h
Просмотреть файл

@ -23,6 +23,8 @@
#include "orte/types.h"
#include "orte/runtime/orte_globals.h"
#include "ompi/communicator/communicator.h"
BEGIN_C_DECLS
/* ******************************************************************** */
@ -87,6 +89,7 @@ struct oshmem_group_t {
/* Collectives module interface and data */
mca_scoll_base_group_scoll_t g_scoll;
ompi_communicator_t* ompi_comm;
};
typedef struct oshmem_group_t oshmem_group_t;
OSHMEM_DECLSPEC OBJ_CLASS_DECLARATION(oshmem_group_t);