From 1bed5d8aee08cf71aba1154d0823b92919bfeb7c Mon Sep 17 00:00:00 2001 From: Igor Ivanov Date: Fri, 18 Mar 2016 12:43:55 +0200 Subject: [PATCH] oshmem: Align OSHMEM API with spec v1.3 (update scoll/basic) --- oshmem/include/shmem.h.in | 23 +--- oshmem/mca/scoll/basic/scoll_basic.h | 3 +- oshmem/mca/scoll/basic/scoll_basic_alltoall.c | 118 ++++++++++++++++++ oshmem/mca/scoll/scoll.h | 3 +- oshmem/shmem/c/shmem_alltoall.c | 4 +- oshmem/shmem/fortran/shmem_alltoall_f.c | 4 +- 6 files changed, 131 insertions(+), 24 deletions(-) create mode 100644 oshmem/mca/scoll/basic/scoll_basic_alltoall.c diff --git a/oshmem/include/shmem.h.in b/oshmem/include/shmem.h.in index 6be14ddb4e..4767ddebda 100644 --- a/oshmem/include/shmem.h.in +++ b/oshmem/include/shmem.h.in @@ -193,24 +193,6 @@ OSHMEM_DECLSPEC void shmem_iput32(void* target, const void* source, ptrdiff_t ts OSHMEM_DECLSPEC void shmem_iput64(void* target, const void* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe); OSHMEM_DECLSPEC void shmem_iput128(void* target, const void* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe); -/* - * Nonblocking put routines - */ -OSHMEM_DECLSPEC void shmem_putmem_nbi(void *target, const void *source, size_t len, int pe); -OSHMEM_DECLSPEC void shmem_char_put_nbi(char *target, const char *source, size_t len, int pe); -OSHMEM_DECLSPEC void shmem_short_put_nbi(short *target, const short *source, size_t len, int pe); -OSHMEM_DECLSPEC void shmem_int_put_nbi(int* target, const int* source, size_t len, int pe); -OSHMEM_DECLSPEC void shmem_long_put_nbi(long *target, const long *source, size_t len, int pe); -OSHMEM_DECLSPEC void shmem_longlong_put_nbi(long long *target, const long long *source, size_t len, int pe); -OSHMEM_DECLSPEC void shmem_float_put_nbi(float *target, const float *source, size_t len, int pe); -OSHMEM_DECLSPEC void shmem_double_put_nbi(double *target, const double *source, size_t len, int pe); -OSHMEM_DECLSPEC void shmem_longdouble_put_nbi(long double *target, const long double *source, size_t len, int pe); -OSHMEM_DECLSPEC void shmem_put8_nbi(void *target, const void *source, size_t len, int pe); -OSHMEM_DECLSPEC void shmem_put16_nbi(void *target, const void *source, size_t len, int pe); -OSHMEM_DECLSPEC void shmem_put32_nbi(void *target, const void *source, size_t len, int pe); -OSHMEM_DECLSPEC void shmem_put64_nbi(void *target, const void *source, size_t len, int pe); -OSHMEM_DECLSPEC void shmem_put128_nbi(void *target, const void *source, size_t len, int pe); - /* * Elemental get routines */ @@ -346,6 +328,11 @@ OSHMEM_DECLSPEC void shmem_collect32(void *target, const void *source, size_t nl OSHMEM_DECLSPEC void shmem_collect64(void *target, const void *source, size_t nlong, int PE_start, int logPE_stride, int PE_size, long *pSync); OSHMEM_DECLSPEC void shmem_fcollect32(void *target, const void *source, size_t nlong, int PE_start, int logPE_stride, int PE_size, long *pSync); OSHMEM_DECLSPEC void shmem_fcollect64(void *target, const void *source, size_t nlong, int PE_start, int logPE_stride, int PE_size, long *pSync); +OSHMEM_DECLSPEC void shmem_alltoall32(void *target, const void *source, size_t nelems, int PE_start, int logPE_stride, int PE_size, long *pSync); +OSHMEM_DECLSPEC void shmem_alltoall64(void *target, const void *source, size_t nelems, int PE_start, int logPE_stride, int PE_size, long *pSync); +OSHMEM_DECLSPEC void shmem_alltoalls32(void *target, const void *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems, int PE_start, int logPE_stride, int PE_size, long *pSync); +OSHMEM_DECLSPEC void shmem_alltoalls64(void *target, const void *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems, int PE_start, int logPE_stride, int PE_size, long *pSync); + /* * Reduction routines diff --git a/oshmem/mca/scoll/basic/scoll_basic.h b/oshmem/mca/scoll/basic/scoll_basic.h index a60eda8046..b45b8380f8 100644 --- a/oshmem/mca/scoll/basic/scoll_basic.h +++ b/oshmem/mca/scoll/basic/scoll_basic.h @@ -81,7 +81,8 @@ int mca_scoll_basic_alltoall(struct oshmem_group_t *group, void *target, const void *source, ptrdiff_t dst, ptrdiff_t sst, - size_t nlong, + size_t nelems, + size_t element_size, long *pSync, int alg); diff --git a/oshmem/mca/scoll/basic/scoll_basic_alltoall.c b/oshmem/mca/scoll/basic/scoll_basic_alltoall.c new file mode 100644 index 0000000000..eeb4964890 --- /dev/null +++ b/oshmem/mca/scoll/basic/scoll_basic_alltoall.c @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2016 Mellanox Technologies, Inc. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "oshmem_config.h" +#include +#include + +#include "oshmem/constants.h" +#include "oshmem/op/op.h" +#include "oshmem/mca/spml/spml.h" +#include "oshmem/mca/scoll/scoll.h" +#include "oshmem/mca/scoll/base/base.h" +#include "scoll_basic.h" + +static int _algorithm_simple(struct oshmem_group_t *group, + void *target, + const void *source, + ptrdiff_t dst, ptrdiff_t sst, + size_t nelems, + size_t element_size, + long *pSync); + +int mca_scoll_basic_alltoall(struct oshmem_group_t *group, + void *target, + const void *source, + ptrdiff_t dst, ptrdiff_t sst, + size_t nelems, + size_t element_size, + long *pSync, + int alg) +{ + int rc = OSHMEM_SUCCESS; + + /* Arguments validation */ + if (!group) { + SCOLL_ERROR("Active set (group) of PE is not defined"); + rc = OSHMEM_ERR_BAD_PARAM; + } + + /* Check if this PE is part of the group */ + if ((rc == OSHMEM_SUCCESS) && oshmem_proc_group_is_member(group)) { + int i = 0; + + if (pSync) { + rc = _algorithm_simple(group, + target, + source, + dst, + sst, + nelems, + element_size, + pSync); + } else { + SCOLL_ERROR("Incorrect argument pSync"); + rc = OSHMEM_ERR_BAD_PARAM; + } + + /* Restore initial values */ + SCOLL_VERBOSE(12, + "PE#%d Restore special synchronization array", + group->my_pe); + for (i = 0; pSync && (i < _SHMEM_ALLTOALL_SYNC_SIZE); i++) { + pSync[i] = _SHMEM_SYNC_VALUE; + } + } + + return rc; +} + +static int _algorithm_simple(struct oshmem_group_t *group, + void *target, + const void *source, + ptrdiff_t tst, ptrdiff_t sst, + size_t nelems, + size_t element_size, + long *pSync) +{ + int rc = OSHMEM_SUCCESS; + int pe_cur; + int i; + int j; + int k; + + SCOLL_VERBOSE(14, + "[#%d] send data to all PE in the group", + group->my_pe); + j = oshmem_proc_group_find_id(group, group->my_pe); + for (i = 0; i < group->proc_count; i++) { + /* index permutation for better distribution of traffic */ + k = (((j)+(i))%(group->proc_count)); + pe_cur = oshmem_proc_pe(group->proc_array[k]); + rc = MCA_SPML_CALL(put( + (void *)((char *)target + j * tst * nelems * element_size), + nelems * element_size, + (void *)((char *)source + i * sst * nelems * element_size), + pe_cur)); + if (OSHMEM_SUCCESS != rc) { + break; + } + } + + /* Wait for operation completion */ + if (rc == OSHMEM_SUCCESS) { + SCOLL_VERBOSE(14, "[#%d] Wait for operation completion", group->my_pe); + rc = BARRIER_FUNC(group, + (pSync + 1), + SCOLL_DEFAULT_ALG); + } + + return rc; +} diff --git a/oshmem/mca/scoll/scoll.h b/oshmem/mca/scoll/scoll.h index d1e016430c..cc6cfe6094 100644 --- a/oshmem/mca/scoll/scoll.h +++ b/oshmem/mca/scoll/scoll.h @@ -142,7 +142,8 @@ typedef int (*mca_scoll_base_module_alltoall_fn_t)(struct oshmem_group_t *group, void *target, const void *source, ptrdiff_t dst, ptrdiff_t sst, - size_t nlong, + size_t nelems, + size_t element_size, long *pSync, int alg); diff --git a/oshmem/shmem/c/shmem_alltoall.c b/oshmem/shmem/c/shmem_alltoall.c index d565ee4d7c..c75aa3ae29 100644 --- a/oshmem/shmem/c/shmem_alltoall.c +++ b/oshmem/shmem/c/shmem_alltoall.c @@ -44,7 +44,7 @@ static void _shmem_alltoall(void *target, RUNTIME_CHECK_ADDR(target); \ RUNTIME_CHECK_ADDR(source); \ \ - _shmem_alltoall(target, source, 1, 1, nelems * element_size, \ + _shmem_alltoall(target, source, 1, 1, nelems, element_size, \ PE_start, logPE_stride, PE_size, \ pSync); \ } @@ -63,7 +63,7 @@ static void _shmem_alltoall(void *target, RUNTIME_CHECK_ADDR(target); \ RUNTIME_CHECK_ADDR(source); \ \ - _shmem_alltoall(target, source, dst, sst, nelems * element_size, \ + _shmem_alltoall(target, source, dst, sst, nelems, element_size, \ PE_start, logPE_stride, PE_size, \ pSync); \ } diff --git a/oshmem/shmem/fortran/shmem_alltoall_f.c b/oshmem/shmem/fortran/shmem_alltoall_f.c index 4a507be6c9..58fd866792 100644 --- a/oshmem/shmem/fortran/shmem_alltoall_f.c +++ b/oshmem/shmem/fortran/shmem_alltoall_f.c @@ -169,8 +169,8 @@ SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void, rc = group->g_scoll.scoll_alltoall( group, \ FPTR_2_VOID_PTR(target), \ FPTR_2_VOID_PTR(source), \ - OMPI_FINT_2_INT(*dst), \ - OMPI_FINT_2_INT(*sst), \ + OMPI_FINT_2_INT(*dst), \ + OMPI_FINT_2_INT(*sst), \ OMPI_FINT_2_INT(*nlong), \ op->dt_size, \ FPTR_2_VOID_PTR(pSync), SCOLL_DEFAULT_ALG );\