ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

* updated copyrights

Просмотр исходного кода 
* added support for non-contig data layout in FCA

This commit was SVN r24702.
Этот коммит содержится в:
Mike Dubman 2011-05-16 14:43:11 +00:00
родитель 4083e23073
Коммит 36db9c6233
11 изменённых файлов: 319 добавлений и 86 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

14
README
Просмотреть файл

@ -9,7 +9,7 @@ Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
Copyright (c) 2004-2007 The Regents of the University of California.
All rights reserved.
Copyright (c) 2006-2010 Cisco Systems, Inc. All rights reserved.
Copyright (c) 2006-2007 Voltaire, Inc. All rights reserved.
Copyright (c) 2006-2011 Mellanox Technologies. All rights reserved.
Copyright (c) 2006-2010 Oracle and/or its affiliates. All rights reserved.
Copyright (c) 2007 Myricom, Inc. All rights reserved.
Copyright (c) 2008 IBM Corporation. All rights reserved.
@ -121,7 +121,7 @@ Detailed Open MPI v1.5 Feature List:
- Performance improvements
- Support for hierarchical collectives (must be activated
manually; see below)
- Support for Voltaire FCA (Fabric Collective Accelerator) technology
- Support for Mellanox FCA (Fabric Collective Accelerator) technology
o Miscellaneous
- MPI 2.1 compliant
@ -440,12 +440,10 @@ Collectives
We would appreciate feedback from the user community about how well
hierarch works for your applications.
- The "fca" coll component: Voltaire Fabric Collective Accelerator (FCA)
- The "fca" coll component: Mellanox Fabric Collective Accelerator (FCA)
is a solution for offloading collective operations from the MPI process
onto Voltaire QDR InfiniBand switch CPUs.
onto Mellanox QDR InfiniBand switch CPUs and HCAs.
See http://www.voltaire.com/Products/Application_Acceleration_Software/voltaire_fabric_collective_accelerator_fca
for details.
Network Support
@ -811,10 +809,10 @@ for a full list); a summary of the more commonly used ones follows:
This is currently disabled by default.
--with-fca=<directory>
Specify the directory where the Voltaire FCA library and
Specify the directory where the Mellanox FCA library and
header files are located.
FCA is the support library for Voltaire QDR switches.
FCA is the support library for Mellanox QDR switches and HCAs.
--disable-mpi-cxx
Disable building the C++ MPI bindings. Note that this does *not*

4
ompi/config/ompi_check_libfca.m4
Просмотреть файл

@ -1,6 +1,6 @@
# -*- shell-script -*-
#
# Copyright (c) 2010 Voltaire Corporation. All rights reserved.
# Copyright (c) 2011 Mellanox Technologies. All rights reserved.
# $COPYRIGHT$
#
@ -17,7 +17,7 @@
AC_DEFUN([OMPI_CHECK_FCA],[
AC_ARG_WITH([fca],
[AC_HELP_STRING([--with-fca(=DIR)],
[Build fca (Voltaire Fabric Collective Accelerator) support, searching for libraries in DIR])])
[Build fca (Mellanox Fabric Collective Accelerator) support, searching for libraries in DIR])])
OMPI_CHECK_WITHDIR([fca], [$with_fca], [lib/libfca.so])
AS_IF([test "$with_fca" != "no"],

3
ompi/mca/coll/fca/Makefile.am
Просмотреть файл

@ -1,7 +1,7 @@
# -*- shell-script -*-
#
#
# Copyright (c) 2010 Voltaire, Inc. All rights reserved.
# Copyright (c) 2011 Mellanox Technologies. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
@ -17,6 +17,7 @@ coll_fca_sources = \
coll_fca.h \
coll_fca_debug.h \
coll_fca_api.h \
coll_fca_convertor.h \
coll_fca_module.c \
coll_fca_component.c \
coll_fca_ops.c

38
ompi/mca/coll/fca/coll_fca.h
Просмотреть файл

@ -1,5 +1,5 @@
/**
Copyright (c) 2010 Voltaire, Inc. All rights reserved.
Copyright (c) 2011 Mellanox Technologies. All rights reserved.
$COPYRIGHT$
Additional copyrights may follow
@ -24,16 +24,40 @@
#include "coll_fca_api.h"
#include "coll_fca_debug.h"
#ifdef OMPI_DATATYPE_MAX_PREDEFINED
#define FCA_DT_MAX_PREDEFINED OMPI_DATATYPE_MAX_PREDEFINED
#define FCA_DT_GET_TRUE_EXTENT ompi_datatype_get_true_extent
#define FCA_DT_IS_CONTIGUOUS_MEMORY_LAYOUT ompi_datatype_is_contiguous_memory_layout
#include "opal/datatype/opal_convertor.h"
#define FCA_CONVERTOR_T opal_convertor_t
#define FCA_CONVERTOR_COPY_AND_PREPARE_FOR_SEND opal_convertor_copy_and_prepare_for_send
#define FCA_CONVERTOR_COPY_AND_PREPARE_FOR_RECV opal_convertor_copy_and_prepare_for_recv
#define FCA_CONVERTOR_CONVERTOR_GET_PACKED_SIZE opal_convertor_get_packed_size
#define FCA_CONVERTOR_CONVERTOR_UNPACK opal_convertor_unpack
#define FCA_CONVERTOR_CONVERTOR_PACK opal_convertor_pack
#define FCA_DT_MAX_PREDEFINED OMPI_DATATYPE_MAX_PREDEFINED
#define FCA_DT_EXTENT ompi_datatype_type_extent
#define FCA_DT_GET_TRUE_EXTENT ompi_datatype_get_true_extent
#define FCA_DT_IS_CONTIGUOUS_MEMORY_LAYOUT(__dtype, __count) \
ompi_datatype_is_contiguous_memory_layout(__dtype, __count)
#else
#define FCA_DT_MAX_PREDEFINED DT_MAX_PREDEFINED
#define FCA_DT_GET_TRUE_EXTENT ompi_ddt_get_true_extent
#define FCA_DT_IS_CONTIGUOUS_MEMORY_LAYOUT ompi_ddt_is_contiguous_memory_layout
#include "ompi/datatype/convertor.h"
#define FCA_CONVERTOR_T ompi_convertor_t
#define FCA_CONVERTOR_COPY_AND_PREPARE_FOR_SEND ompi_convertor_copy_and_prepare_for_send
#define FCA_CONVERTOR_COPY_AND_PREPARE_FOR_RECV ompi_convertor_copy_and_prepare_for_recv
#define FCA_CONVERTOR_CONVERTOR_GET_PACKED_SIZE ompi_convertor_get_packed_size
#define FCA_CONVERTOR_CONVERTOR_UNPACK ompi_convertor_unpack
#define FCA_CONVERTOR_CONVERTOR_PACK ompi_convertor_pack
#define FCA_DT_MAX_PREDEFINED DT_MAX_PREDEFINED
#define FCA_DT_EXTENT ompi_ddt_type_extent
#define FCA_DT_GET_TRUE_EXTENT ompi_ddt_get_true_extent
#define FCA_DT_IS_CONTIGUOUS_MEMORY_LAYOUT(__dtype, __count) \
ompi_ddt_is_contiguous_memory_layout(__dtype, __count)
#endif
#ifdef OMPI_PROC_FLAG_LOCAL
#define FCA_IS_LOCAL_PROCESS(n) ((n) & OMPI_PROC_FLAG_LOCAL)
#else

2
ompi/mca/coll/fca/coll_fca_api.h
Просмотреть файл

@ -1,5 +1,5 @@
/**
Copyright (c) 2010 Voltaire, Inc. All rights reserved.
Copyright (c) 2011 Mellanox Technologies. All rights reserved.
$COPYRIGHT$
Additional copyrights may follow

2
ompi/mca/coll/fca/coll_fca_component.c
Просмотреть файл

@ -1,5 +1,5 @@
/**
Copyright (c) 2010 Voltaire, Inc. All rights reserved.
Copyright (c) 2011 Mellanox Technologies. All rights reserved.
$COPYRIGHT$
Additional copyrights may follow

86
ompi/mca/coll/fca/coll_fca_convertor.h Обычный файл
Просмотреть файл

@ -0,0 +1,86 @@
#ifndef MCA_COLL_FCA_CONVERTOR_H
#define MCA_COLL_FCA_CONVERTOR_H
enum {
MCA_COLL_CONVERTOR_NULL = 0,
MCA_COLL_FCA_CONV_SEND,
MCA_COLL_FCA_CONV_RECV
};
struct mca_coll_fca_convertor {
int type;
FCA_CONVERTOR_T ompic;
size_t size;
void *buf;
};
#define MCA_COLL_FCA_DECLARE_CONVERTOR(__name) \
struct mca_coll_fca_convertor __name = {MCA_COLL_CONVERTOR_NULL}
static inline void mca_coll_fca_convertor_set(struct mca_coll_fca_convertor *conv,
struct ompi_datatype_t *datatype,
void *buffer, int count)
{
if (conv->type == MCA_COLL_FCA_CONV_SEND) {
FCA_CONVERTOR_COPY_AND_PREPARE_FOR_SEND(ompi_mpi_local_convertor,
&datatype->super, count,
buffer, 0, &conv->ompic);
} else if (conv->type == MCA_COLL_FCA_CONV_RECV) {
FCA_CONVERTOR_COPY_AND_PREPARE_FOR_RECV(ompi_mpi_local_convertor,
&datatype->super, count,
buffer, 0, &conv->ompic);
}
}
static inline void mca_coll_fca_convertor_create(struct mca_coll_fca_convertor *conv,
struct ompi_datatype_t *datatype,
int count, void *buffer, int type,
void **tmpbuf, size_t *size)
{
OBJ_CONSTRUCT(&conv->ompic, FCA_CONVERTOR_T);
conv->type = type;
mca_coll_fca_convertor_set(conv, datatype, buffer, count);
FCA_CONVERTOR_CONVERTOR_GET_PACKED_SIZE(&conv->ompic, &conv->size);
conv->buf = malloc(conv->size);
*tmpbuf = conv->buf;
*size = conv->size;
}
static inline int mca_coll_fca_convertor_valid(struct mca_coll_fca_convertor *conv)
{
return conv->type != MCA_COLL_CONVERTOR_NULL;
}
static inline void mca_coll_fca_convertor_destroy(struct mca_coll_fca_convertor *conv)
{
if (mca_coll_fca_convertor_valid(conv)) {
free(conv->buf);
OBJ_DESTRUCT(&conv->ompic);
}
}
static inline int32_t mca_coll_fca_convertor_process(struct mca_coll_fca_convertor *conv,
size_t offset)
{
struct iovec invec;
unsigned iov_count;
size_t size;
iov_count = 1;
invec.iov_base = (char*)conv->buf + offset;
invec.iov_len = conv->size;
size = conv->size;
if (conv->type == MCA_COLL_FCA_CONV_SEND) {
return FCA_CONVERTOR_CONVERTOR_PACK(&conv->ompic, &invec, &iov_count, &size);
} else if (conv->type == MCA_COLL_FCA_CONV_RECV) {
return FCA_CONVERTOR_CONVERTOR_UNPACK(&conv->ompic, &invec, &iov_count, &size);
}
return 0;
}
#endif

2
ompi/mca/coll/fca/coll_fca_debug.h
Просмотреть файл

@ -1,5 +1,5 @@
/**
Copyright (c) 2010 Voltaire, Inc. All rights reserved.
Copyright (c) 2011 Mellanox Technologies. All rights reserved.
$COPYRIGHT$
Additional copyrights may follow

2
ompi/mca/coll/fca/coll_fca_module.c
Просмотреть файл

@ -1,5 +1,5 @@
/**
Copyright (c) 2010 Voltaire, Inc. All rights reserved.
Copyright (c) 2011 Mellanox Technologies. All rights reserved.
$COPYRIGHT$
Additional copyrights may follow

250
ompi/mca/coll/fca/coll_fca_ops.c
Просмотреть файл

@ -1,5 +1,5 @@
/**
Copyright (c) 2010 Voltaire, Inc. All rights reserved.
Copyright (c) 2011 Mellanox Technologies. All rights reserved.
$COPYRIGHT$
Additional copyrights may follow
@ -10,6 +10,7 @@
#include "ompi_config.h"
#include "ompi/constants.h"
#include "coll_fca.h"
#include "coll_fca_convertor.h"
static mca_coll_fca_dtype_info_t* mca_coll_fca_get_dtype(ompi_datatype_t *dtype)
@ -19,18 +20,25 @@ static mca_coll_fca_dtype_info_t* mca_coll_fca_get_dtype(ompi_datatype_t *dtype)
int id = dtype->id;
int fca_dtype;
if (id < 0 || id >= FCA_DT_MAX_PREDEFINED)
if (id < 0 || id >= FCA_DT_MAX_PREDEFINED) {
return NULL;
}
/* Different dtype structures may have the same id. In that case, we assume
* they are aliases.
*/
dtype_info = &mca_coll_fca_component.fca_dtypes[id];
if (dtype_info->mpi_dtype == dtype)
if (dtype_info->mpi_dtype->id == id) {
return dtype_info;
}
/* assert we don't overwrite another datatype */
assert(dtype_info->mpi_dtype == MPI_DATATYPE_NULL);
fca_dtype = mca_coll_fca_component.fca_ops.translate_mpi_dtype(dtype->name);
if (fca_dtype < 0)
if (fca_dtype < 0) {
return NULL;
}
FCA_DT_GET_TRUE_EXTENT(dtype, &lb, &extent);
dtype_info->mpi_dtype = dtype;
@ -72,19 +80,21 @@ static mca_coll_fca_op_info_t *mca_coll_fca_get_op(ompi_op_t *op)
return NULL;
}
static int mca_coll_fca_get_buf_size(ompi_datatype_t *dtype, int count,
int contiguous_count)
/**
* If "datatype" is contiguous when it appears "count" times, return 1 and
* set "*size" to the total buffer size. Otherwise return 0.
*/
static inline int mca_coll_fca_array_size(ompi_datatype_t *dtype, int count, size_t *size)
{
ptrdiff_t true_lb, true_extent;
/* Check that the type in contiguous */
if (!FCA_DT_IS_CONTIGUOUS_MEMORY_LAYOUT(dtype, contiguous_count)) {
FCA_VERBOSE(5, "Unsupported datatype layout, only contiguous is supported now");
return OMPI_ERROR;
}
FCA_DT_GET_TRUE_EXTENT(dtype, &true_lb, &true_extent);
return true_extent * count;
if (FCA_DT_IS_CONTIGUOUS_MEMORY_LAYOUT(dtype, count)) {
FCA_DT_GET_TRUE_EXTENT(dtype, &true_lb, &true_extent);
*size = true_extent * count;
return 1;
} else {
return 0;
}
}
static int mca_coll_fca_fill_reduce_spec(int count, ompi_datatype_t *dtype,
@ -165,31 +175,63 @@ int mca_coll_fca_bcast(void *buff, int count, struct ompi_datatype_t *datatype,
mca_coll_base_module_t *module)
{
mca_coll_fca_module_t *fca_module = (mca_coll_fca_module_t*)module;
MCA_COLL_FCA_DECLARE_CONVERTOR(conv);
fca_bcast_spec_t spec;
size_t size;
int ret;
FCA_VERBOSE(5,"[%d] Calling mca_coll_fca_bcast, root=%d, count=%d",
FCA_VERBOSE(5, "[%d] Calling mca_coll_fca_bcast, root=%d, count=%d",
ompi_comm_rank(comm), root, count);
spec.size = mca_coll_fca_get_buf_size(datatype, count, count);
if (spec.size < 0 || spec.size > fca_module->fca_comm_caps.max_payload) {
FCA_VERBOSE(5, "Unsupported bcast operation, dtype=%s[%d] using fallback\n",
datatype->name, count);
goto orig_bcast;
/* Setup exchange buffer */
spec.root = root;
if (mca_coll_fca_array_size(datatype, count, &size)) {
spec.buf = buff;
} else {
mca_coll_fca_convertor_create(&conv, datatype, count, buff,
(root == fca_module->rank)
? MCA_COLL_FCA_CONV_SEND
: MCA_COLL_FCA_CONV_RECV,
&spec.buf, &size);
}
FCA_VERBOSE(5,"Using FCA Bcast");
spec.buf = buff;
mca_coll_fca_get_bcast_root(root, fca_module->local_ranks,
fca_module->num_local_procs, &spec);
/* Check that operation size does not exceed limit */
spec.size = size;
if (spec.size > fca_module->fca_comm_caps.max_payload) {
FCA_VERBOSE(5, "Unsupported bcast operation size %d, using fallback",
spec.size);
if (spec.buf != buff) {
mca_coll_fca_convertor_destroy(&conv);
}
goto orig_bcast;
}
/* Sender may pack data */
if (spec.buf != buff && root == fca_module->rank) {
mca_coll_fca_convertor_process(&conv, 0);
}
/* Call FCA Bcast */
FCA_VERBOSE(5, "Using FCA Bcast");
ret = mca_coll_fca_component.fca_ops.do_bcast(fca_module->fca_comm, &spec);
/* Destroy convertor if operation failed */
if (ret < 0) {
mca_coll_fca_convertor_destroy(&conv);
if (ret == -EUSEMPI) {
goto orig_bcast;
}
FCA_ERROR("Bcast failed: %s", mca_coll_fca_component.fca_ops.strerror(ret));
return OMPI_ERROR;
}
/* Unpack data and clean up convertor */
if (mca_coll_fca_convertor_valid(&conv)) {
if (root != fca_module->rank) {
mca_coll_fca_convertor_process(&conv, 0);
}
mca_coll_fca_convertor_destroy(&conv);
}
return OMPI_SUCCESS;
orig_bcast:
@ -276,8 +318,36 @@ int mca_coll_fca_allreduce(void *sbuf, void *rbuf, int count,
return OMPI_SUCCESS;
orig_allreduce:
return fca_module->previous_allreduce(sbuf, rbuf, count, dtype, op,
comm, fca_module->previous_allreduce_module);
return fca_module->previous_allreduce(sbuf, rbuf, count, dtype, op, comm,
fca_module->previous_allreduce_module);
}
/*
* Prepare a send buffer for allgather/allgatherv, handle packing and MPI_IN_PLACE.
*/
static size_t __setup_gather_sendbuf(void *sbuf, void *inplace_sbuf, int scount,
struct ompi_datatype_t *sdtype,
struct mca_coll_fca_convertor *sconv,
void **real_sendbuf)
{
size_t ssize;
if (mca_coll_fca_array_size(sdtype, scount, &ssize)) {
*real_sendbuf = (MPI_IN_PLACE == sbuf) ? inplace_sbuf : sbuf;
} else {
FCA_VERBOSE(5, "Packing send buffer");
if (MPI_IN_PLACE == sbuf) {
mca_coll_fca_convertor_create(sconv, sdtype, scount, inplace_sbuf,
MCA_COLL_FCA_CONV_SEND, real_sendbuf,
&ssize);
} else {
mca_coll_fca_convertor_create(sconv, sdtype, scount, sbuf,
MCA_COLL_FCA_CONV_SEND, real_sendbuf,
&ssize);
}
mca_coll_fca_convertor_process(sconv, 0);
}
return ssize;
}
/*
@ -294,39 +364,53 @@ int mca_coll_fca_allgather(void *sbuf, int scount, struct ompi_datatype_t *sdtyp
{
mca_coll_fca_module_t *fca_module = (mca_coll_fca_module_t*)module;
#if OMPI_FCA_ALLGATHER == 1
MCA_COLL_FCA_DECLARE_CONVERTOR(sconv);
MCA_COLL_FCA_DECLARE_CONVERTOR(rconv);
fca_gather_spec_t spec = {0,};
size_t rsize;
ptrdiff_t rdtype_extent;
ssize_t total_rcount;
int ret;
spec.sbuf = sbuf;
spec.rbuf = rbuf;
spec.size = mca_coll_fca_get_buf_size(sdtype, scount, scount);
FCA_DT_EXTENT(rdtype, &rdtype_extent);
if (spec.size < 0 || spec.size > fca_module->fca_comm_caps.max_payload ||
!FCA_DT_IS_CONTIGUOUS_MEMORY_LAYOUT(rdtype, ompi_comm_size(comm))) {
FCA_VERBOSE(5, "Unsupported allgather operation size %d, using fallback\n",
spec.size);
goto orig_allgather;
/* Setup send buffer */
spec.size =
__setup_gather_sendbuf(sbuf, (char *)rbuf + rcount * fca_module->rank * rdtype_extent,
scount, sdtype, &sconv, &spec.sbuf);
/* Setup recv buffer */
total_rcount = ompi_comm_size(comm) * rcount;
if (mca_coll_fca_array_size(rdtype, total_rcount, &rsize)) {
spec.rbuf = rbuf;
} else {
mca_coll_fca_convertor_create(&rconv, rdtype, total_rcount, rbuf,
MCA_COLL_FCA_CONV_RECV, &spec.rbuf, &rsize);
}
if (spec.size != mca_coll_fca_get_buf_size(rdtype, rcount, rcount)) {
FCA_VERBOSE(5, "Unsupported allgather: send_size != recv_size\n");
goto orig_allgather;
}
if (MPI_IN_PLACE == spec.sbuf) {
FCA_VERBOSE(10, "Using MPI_IN_PLACE for sbuf");
spec.sbuf = (char*)spec.rbuf + spec.size * fca_module->rank;
}
FCA_VERBOSE(5,"Using FCA Allgather");
/* Call FCA Allgather */
FCA_VERBOSE(5,"Using FCA Allgather size");
ret = mca_coll_fca_component.fca_ops.do_allgather(fca_module->fca_comm, &spec);
/* Destroy convertors if operation failed */
if (ret < 0) {
mca_coll_fca_convertor_destroy(&sconv);
mca_coll_fca_convertor_destroy(&rconv);
if (ret == -EUSEMPI) {
goto orig_allgather;
}
FCA_ERROR("Allgather failed: %s", mca_coll_fca_component.fca_ops.strerror(ret));
return OMPI_ERROR;
}
/* Unpack data and clean up convertor */
mca_coll_fca_convertor_destroy(&sconv);
if (mca_coll_fca_convertor_valid(&rconv)) {
FCA_VERBOSE(5, "Unpacking Allgather receive buffer");
mca_coll_fca_convertor_process(&rconv, 0);
mca_coll_fca_convertor_destroy(&rconv);
}
return OMPI_SUCCESS;
orig_allgather:
@ -335,7 +419,6 @@ orig_allgather:
comm, fca_module->previous_allgather_module);
}
int mca_coll_fca_allgatherv(void *sbuf, int scount,
struct ompi_datatype_t *sdtype,
void *rbuf, int *rcounts, int *disps,
@ -345,49 +428,90 @@ int mca_coll_fca_allgatherv(void *sbuf, int scount,
{
mca_coll_fca_module_t *fca_module = (mca_coll_fca_module_t*)module;
#if OMPI_FCA_ALLGATHER == 1
MCA_COLL_FCA_DECLARE_CONVERTOR(sconv);
MCA_COLL_FCA_DECLARE_CONVERTOR(rconv);
fca_gatherv_spec_t spec;
int relemsize;
size_t rsize;
int sum_rcounts;
ptrdiff_t rdtype_extent;
int comm_size;
int relemsize;
size_t displ;
int i, ret;
comm_size = ompi_comm_size(fca_module->comm);
FCA_DT_EXTENT(rdtype, &rdtype_extent);
spec.sbuf = sbuf;
spec.rbuf = rbuf;
spec.sendsize = mca_coll_fca_get_buf_size(sdtype, scount, scount);
if (spec.sendsize < 0 || spec.sendsize > fca_module->fca_comm_caps.max_payload ||
!FCA_DT_IS_CONTIGUOUS_MEMORY_LAYOUT(rdtype, ompi_comm_size(comm))) {
FCA_VERBOSE(5, "Unsupported allgatherv operation size %d, using fallback\n",
spec.sendsize);
goto orig_allgatherv;
}
/* Setup send buffer */
spec.sendsize =
__setup_gather_sendbuf(sbuf, (char *)rbuf + disps[fca_module->rank] * rdtype_extent,
scount, sdtype, &sconv, &spec.sbuf);
/* Allocate alternative recvsizes/displs on the stack, which will be in bytes */
spec.recvsizes = alloca(sizeof *spec.recvsizes * comm_size);
spec.displs = alloca(sizeof *spec.displs * comm_size);
/* convert MPI counts which depend on dtype) to FCA sizes (which are in bytes) */
relemsize = mca_coll_fca_get_buf_size(rdtype, 1, comm_size);
/* Calculate the size of receive buffer */
sum_rcounts = 0;
for (i = 0; i < comm_size; ++i) {
spec.recvsizes[i] = rcounts[i] * relemsize;
spec.displs[i] = disps[i] * relemsize;
sum_rcounts += rcounts[i];
}
if (MPI_IN_PLACE == spec.sbuf) {
FCA_VERBOSE(10, "Using MPI_IN_PLACE for sbuf");
spec.sbuf = (char *)spec.rbuf + spec.displs[fca_module->rank];
/* convert MPI counts which depend on dtype) to FCA sizes (which are in bytes) */
if (mca_coll_fca_array_size(rdtype, sum_rcounts, &rsize)) {
spec.rbuf = rbuf;
for (i = 0; i < comm_size; ++i) {
spec.recvsizes[i] = rcounts[i] * rdtype_extent;
spec.displs[i] = disps[i] * rdtype_extent;
}
} else {
/*
* Reorder and remove gaps in displs - we want to allocate as little memory
* as possible, and we should unpack one-by-one anyway.
*/
FCA_VERBOSE(5, "Reordering AllgatherV displacements");
mca_coll_fca_convertor_create(&rconv, rdtype, sum_rcounts, rbuf,
MCA_COLL_FCA_CONV_RECV, &spec.rbuf, &rsize);
assert(rsize % sum_rcounts == 0);
relemsize = rsize / sum_rcounts;
displ = 0;
for (i = 0; i < comm_size; ++i) {
spec.recvsizes[i] = rcounts[i] * relemsize;
spec.displs[i] = displ;
displ += spec.recvsizes[i];
}
assert(displ == rsize);
}
/* Call FCA AllgatherV */
FCA_VERBOSE(5,"Using FCA Allgatherv");
ret = mca_coll_fca_component.fca_ops.do_allgatherv(fca_module->fca_comm, &spec);
/* Destroy convertors if operation failed */
if (ret < 0) {
mca_coll_fca_convertor_destroy(&sconv);
mca_coll_fca_convertor_destroy(&rconv);
if (ret == -EUSEMPI) {
goto orig_allgatherv;
}
FCA_ERROR("Allgatherv failed: %s", mca_coll_fca_component.fca_ops.strerror(ret));
return OMPI_ERROR;
}
/* Unpack data and clean up convertor */
mca_coll_fca_convertor_destroy(&sconv);
if (mca_coll_fca_convertor_valid(&rconv)) {
FCA_VERBOSE(5, "Unpacking AllgatherV receive buffer rdtype_extent=%ld",
rdtype_extent);
for (i = 0; i < comm_size; ++i) {
mca_coll_fca_convertor_set(&rconv, rdtype,
(char*)rbuf + disps[i] * rdtype_extent,
rcounts[i]);
mca_coll_fca_convertor_process(&rconv, spec.displs[i]);
}
mca_coll_fca_convertor_destroy(&rconv);
}
return OMPI_SUCCESS;
orig_allgatherv:

2
ompi/mca/coll/fca/configure.m4
Просмотреть файл

@ -1,7 +1,7 @@
# -*- shell-script -*-
#
#
# Copyright (c) 2010 Voltaire, Inc. All rights reserved.
# Copyright (c) 2011 Mellanox Technologies. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow