Resolve merge conflicts
Signed-off-by: Mikhail Kurnosov <mkurnosov@gmail.com>
Этот коммит содержится в:
Mikhail Kurnosov
Коммит
8cf8553abd
@ -11,7 +11,7 @@ dnl University of Stuttgart. All rights reserved.
|
||||
dnl Copyright (c) 2004-2006 The Regents of the University of California.
|
||||
dnl All rights reserved.
|
||||
dnl Copyright (c) 2009-2017 Cisco Systems, Inc. All rights reserved
|
||||
dnl Copyright (c) 2008-2017 University of Houston. All rights reserved.
|
||||
dnl Copyright (c) 2008-2018 University of Houston. All rights reserved.
|
||||
dnl Copyright (c) 2015 Research Organization for Information Science
|
||||
dnl and Technology (RIST). All rights reserved.
|
||||
dnl $COPYRIGHT$
|
||||
@ -45,23 +45,26 @@ AC_DEFUN([OMPI_CHECK_LUSTRE],[
|
||||
[Build Lustre support, optionally adding DIR/include, DIR/lib, and DIR/lib64 to the search path for headers and libraries])])
|
||||
OPAL_CHECK_WITHDIR([lustre], [$with_lustre], [include/lustre/lustreapi.h])
|
||||
|
||||
AS_IF([test -z "$with_lustre" || test "$with_lustre" = "yes"],
|
||||
[ompi_check_lustre_dir="/usr"],
|
||||
[ompi_check_lustre_dir=$with_lustre])
|
||||
|
||||
if test -e "$ompi_check_lustre_dir/lib64" ; then
|
||||
ompi_check_lustre_libdir="$ompi_check_lustre_dir/lib64"
|
||||
else
|
||||
ompi_check_lustre_libdir="$ompi_check_lustre_dir/lib"
|
||||
fi
|
||||
|
||||
# Add correct -I and -L flags
|
||||
OPAL_CHECK_PACKAGE([$1], [lustre/lustreapi.h], [lustreapi], [llapi_file_create], [],
|
||||
[$ompi_check_lustre_dir], [$ompi_check_lustre_libdir], [ompi_check_lustre_happy="yes"],
|
||||
[ompi_check_lustre_happy="no"])
|
||||
|
||||
AC_MSG_CHECKING([for required lustre data structures])
|
||||
cat > conftest.c <<EOF
|
||||
AS_IF([test "$with_lustre" = "no"],
|
||||
[ompi_check_lustre_happy="no"],
|
||||
[AS_IF([test -z "$with_lustre" || test "$with_lustre" = "yes"],
|
||||
[ompi_check_lustre_dir="/usr"],
|
||||
[ompi_check_lustre_dir=$with_lustre])
|
||||
|
||||
if test -e "$ompi_check_lustre_dir/lib64" ; then
|
||||
ompi_check_lustre_libdir="$ompi_check_lustre_dir/lib64"
|
||||
else
|
||||
ompi_check_lustre_libdir="$ompi_check_lustre_dir/lib"
|
||||
fi
|
||||
|
||||
# Add correct -I and -L flags
|
||||
OPAL_CHECK_PACKAGE([$1], [lustre/lustreapi.h], [lustreapi], [llapi_file_create],
|
||||
[], [$ompi_check_lustre_dir], [$ompi_check_lustre_libdir],
|
||||
[ompi_check_lustre_happy="yes"],
|
||||
[ompi_check_lustre_happy="no"])
|
||||
|
||||
AC_MSG_CHECKING([for required lustre data structures])
|
||||
cat > conftest.c <<EOF
|
||||
#include "lustre/lustreapi.h"
|
||||
void alloc_lum()
|
||||
{
|
||||
@ -82,7 +85,7 @@ OPAL_LOG_COMMAND(
|
||||
)
|
||||
rm -f conftest.c conftest.o
|
||||
AC_MSG_RESULT([$ompi_check_lustre_struct_happy])
|
||||
|
||||
])
|
||||
|
||||
AS_IF([test "$ompi_check_lustre_happy" = "yes"],
|
||||
[$2],
|
||||
|
||||
@ -11,7 +11,7 @@ dnl University of Stuttgart. All rights reserved.
|
||||
dnl Copyright (c) 2004-2006 The Regents of the University of California.
|
||||
dnl All rights reserved.
|
||||
dnl Copyright (c) 2009 Cisco Systems, Inc. All rights reserved.
|
||||
dnl Copyright (c) 2008-2016 University of Houston. All rights reserved.
|
||||
dnl Copyright (c) 2008-2018 University of Houston. All rights reserved.
|
||||
dnl Copyright (c) 2015 Research Organization for Information Science
|
||||
dnl and Technology (RIST). All rights reserved.
|
||||
dnl $COPYRIGHT$
|
||||
@ -42,27 +42,30 @@ AC_DEFUN([OMPI_CHECK_PVFS2],[
|
||||
[Build Pvfs2 support, optionally adding DIR/include, DIR/lib, and DIR/lib64 to the search path for headers and libraries])])
|
||||
OPAL_CHECK_WITHDIR([pvfs2], [$with_pvfs2], [include/pvfs2.h])
|
||||
|
||||
AS_IF([test -z "$with_pvfs2"],
|
||||
[ompi_check_pvfs2_dir="/usr/local"],
|
||||
[ompi_check_pvfs2_dir=$with_pvfs2])
|
||||
AS_IF([test "$with_pvfs2" = "no"],
|
||||
[ompi_check_pvfs2_happy="no"],
|
||||
[AS_IF([test -z "$with_pvfs2"],
|
||||
[ompi_check_pvfs2_dir="/usr/local"],
|
||||
[ompi_check_pvfs2_dir=$with_pvfs2])
|
||||
|
||||
if test -e "$ompi_check_pvfs2_dir/lib64" ; then
|
||||
ompi_check_pvfs2_libdir="$ompi_check_pvfs2_dir/lib64"
|
||||
else
|
||||
ompi_check_pvfs2_libdir="$ompi_check_pvfs2_dir/lib"
|
||||
fi
|
||||
|
||||
# Add correct -I and -L flags
|
||||
OPAL_CHECK_PACKAGE([$1], [pvfs2.h], [pvfs2], [PVFS_util_resolve], [],
|
||||
[$ompi_check_pvfs2_dir], [$ompi_check_pvfs2_libdir], [ompi_check_pvfs2_happy="yes"],
|
||||
[ompi_check_pvfs2_happy="no"])
|
||||
if test -e "$ompi_check_pvfs2_dir/lib64" ; then
|
||||
ompi_check_pvfs2_libdir="$ompi_check_pvfs2_dir/lib64"
|
||||
else
|
||||
ompi_check_pvfs2_libdir="$ompi_check_pvfs2_dir/lib"
|
||||
fi
|
||||
|
||||
# Add correct -I and -L flags
|
||||
OPAL_CHECK_PACKAGE([$1], [pvfs2.h], [pvfs2], [PVFS_util_resolve], [],
|
||||
[$ompi_check_pvfs2_dir], [$ompi_check_pvfs2_libdir],
|
||||
[ompi_check_pvfs2_happy="yes"],
|
||||
[ompi_check_pvfs2_happy="no"])
|
||||
])
|
||||
|
||||
AS_IF([test "$ompi_check_pvfs2_happy" = "yes"],
|
||||
[$2],
|
||||
[AS_IF([test ! -z "$with_pvfs2" && test "$with_pvfs2" != "no"],
|
||||
[echo PVFS2 support not found])
|
||||
$3])
|
||||
|
||||
])
|
||||
[$2],
|
||||
[AS_IF([test ! -z "$with_pvfs2" && test "$with_pvfs2" != "no"],
|
||||
[echo PVFS2 support not found])
|
||||
$3])
|
||||
|
||||
])
|
||||
|
||||
|
||||
@ -93,18 +93,29 @@ AC_DEFUN([OPAL_SETUP_JAVA],[
|
||||
# hard-code a few of the common ones so that users don't have to
|
||||
# specify --with-java-<foo>=LONG_ANNOYING_DIRECTORY.
|
||||
AS_IF([test -z "$with_jdk_bindir"],
|
||||
[ # OS X Snow Leopard and Lion (10.6 and 10.7 -- did not
|
||||
# check prior versions)
|
||||
[ # OS X/macOS
|
||||
opal_java_found=0
|
||||
# The following logic was deliberately decided upon in https://github.com/open-mpi/ompi/pull/5015 specifically to prevent this script and the
|
||||
# rest of Open MPI's build system from getting confused by the somewhat unorthodox Java toolchain layout present on OS X/macOS systems, described
|
||||
# in depth by https://github.com/open-mpi/ompi/pull/5015#issuecomment-379324639, and mishandling OS X/macOS Java toolchain path
|
||||
# detection as a result.
|
||||
AS_IF([test -x /usr/libexec/java_home],
|
||||
[opal_java_dir=`/usr/libexec/java_home`/include],
|
||||
[opal_java_dir=/System/Library/Frameworks/JavaVM.framework/Versions/Current/Headers])
|
||||
AC_MSG_CHECKING([OSX locations])
|
||||
[opal_java_dir=`/usr/libexec/java_home`],
|
||||
[opal_java_dir=/System/Library/Frameworks/JavaVM.framework/Versions/Current])
|
||||
AC_MSG_CHECKING([OS X/macOS locations])
|
||||
AS_IF([test -d $opal_java_dir],
|
||||
[AC_MSG_RESULT([found ($opal_java_dir)])
|
||||
opal_java_found=1
|
||||
with_jdk_headers=$opal_java_dir
|
||||
with_jdk_bindir=/usr/bin],
|
||||
if test -d "$opal_java_dir/Headers" && test -d "$opal_java_dir/Commands"; then
|
||||
with_jdk_headers=$opal_java_dir/Headers
|
||||
with_jdk_bindir=$opal_java_dir/Commands
|
||||
elif test -d "$opal_java_dir/include" && test -d "$opal_java_dir/bin"; then
|
||||
with_jdk_headers=$opal_java_dir/include
|
||||
with_jdk_bindir=$opal_java_dir/bin
|
||||
else
|
||||
AC_MSG_WARN([No recognized directory structure found under $opal_java_dir])
|
||||
AC_MSG_ERROR([Cannot continue])
|
||||
fi],
|
||||
[AC_MSG_RESULT([not found])])
|
||||
|
||||
if test "$opal_java_found" = "0"; then
|
||||
|
||||
@ -1411,6 +1411,7 @@ AC_CONFIG_FILES([
|
||||
test/datatype/Makefile
|
||||
test/dss/Makefile
|
||||
test/class/Makefile
|
||||
test/mpool/Makefile
|
||||
test/support/Makefile
|
||||
test/threads/Makefile
|
||||
test/util/Makefile
|
||||
|
||||
@ -12,7 +12,7 @@
|
||||
# Copyright (c) 2009-2017 Cisco Systems, Inc. All rights reserved.
|
||||
# Copyright (c) 2010 IBM Corporation. All rights reserved.
|
||||
# Copyright (c) 2010-2011 Oak Ridge National Labs. All rights reserved.
|
||||
# Copyright (c) 2013-2016 Los Alamos National Security, Inc. All rights
|
||||
# Copyright (c) 2013-2018 Los Alamos National Security, Inc. All rights
|
||||
# reserved.
|
||||
# Copyright (c) 2013 Intel Corporation. All rights reserved.
|
||||
# Copyright (c) 2017 Amazon.com, Inc. or its affiliates.
|
||||
@ -67,17 +67,21 @@ EXTRA_DIST = \
|
||||
platform/lanl/cray_xc_cle5.2/optimized-common \
|
||||
platform/lanl/cray_xc_cle5.2/optimized-lustre \
|
||||
platform/lanl/cray_xc_cle5.2/optimized-lustre.conf \
|
||||
platform/lanl/toss/debug-common \
|
||||
platform/lanl/toss/debug \
|
||||
platform/lanl/toss/debug.conf \
|
||||
platform/lanl/toss/debug-mlx \
|
||||
platform/lanl/toss/debug-mlx.conf \
|
||||
platform/lanl/toss/optimized-common \
|
||||
platform/lanl/toss/optimized \
|
||||
platform/lanl/toss/optimized.conf \
|
||||
platform/lanl/toss/optimized-mlx \
|
||||
platform/lanl/toss/optimized-mlx.conf \
|
||||
platform/lanl/toss/toss-common \
|
||||
platform/lanl/toss/README \
|
||||
platform/lanl/toss/common \
|
||||
platform/lanl/toss/common-optimized \
|
||||
platform/lanl/toss/cray-lustre-optimized \
|
||||
platform/lanl/toss/cray-lustre-optimized.conf \
|
||||
platform/lanl/toss/toss2-mlx-optimized \
|
||||
platform/lanl/toss/toss2-mlx-optimized.conf \
|
||||
platform/lanl/toss/toss2-qib-optimized \
|
||||
platform/lanl/toss/toss2-qib-optimized.conf \
|
||||
platform/lanl/toss/toss3-hfi-optimized \
|
||||
platform/lanl/toss/toss3-hfi-optimized.conf \
|
||||
platform/lanl/toss/toss3-mlx-optimized \
|
||||
platform/lanl/toss/toss3-mlx-optimized.conf \
|
||||
platform/lanl/toss/toss3-wc-optimized \
|
||||
platform/lanl/toss/toss3-wc-optimized.conf \
|
||||
platform/lanl/darwin/darwin-common \
|
||||
platform/lanl/darwin/debug-common \
|
||||
platform/lanl/darwin/optimized-common \
|
||||
|
||||
99
contrib/platform/lanl/toss/README
Обычный файл
99
contrib/platform/lanl/toss/README
Обычный файл
@ -0,0 +1,99 @@
|
||||
These platform files were created from platform files shipped with the release
|
||||
tarball. Each file has been modified. Here are the details on how they were
|
||||
created.
|
||||
|
||||
- common
|
||||
Copy of contrib/platform/lanl/toss/toss-common. Removed entries in bottom
|
||||
half of file that were specific to TOSS so that it could be used for Cray
|
||||
platforms as well.
|
||||
- common-optimized
|
||||
Copy of contrib/platform/lanl/toss/optimized-common. Used the file as-is.
|
||||
- toss2-qib-optimized
|
||||
Copy of contrib/platform/lanl/toss/optimized with the following changes:
|
||||
- source common and common-optimzed instead of toss-common and
|
||||
optimized-common
|
||||
- added entries that were removed from common:
|
||||
- enable_mca_no_build
|
||||
- with_slurm
|
||||
- with_tm
|
||||
- with_pmi
|
||||
- with_verbs
|
||||
- NOTE: common had "with_devel_headers=yes" in it that was not propagated.
|
||||
This option should not be used in production as per Open MPI developer
|
||||
mailing list guidance.
|
||||
- Changed comment "Disable components not needed on any TOSS platform" to
|
||||
"Disable components not needed on TOSS platforms with high-speed networks"
|
||||
- Changed "enable panasas" to "enable lustre"
|
||||
- toss2-qib-optimized.conf
|
||||
- copy of contrib/platform/lanl/toss/optimized.conf with the following
|
||||
changes:
|
||||
- changed: orte_no_session_dirs = /lustre,/net,/users,/usr/projects
|
||||
- changed: btl = ^openib
|
||||
- removed: hwloc_base_binding_policy = core (outdated setting)
|
||||
- added: rmaps_base_ranking_policy = core (rank by core)
|
||||
- added: ras_base_launch_orted_on_hn = true (run orted on parent node of
|
||||
allocation)
|
||||
- toss2-mlx-optimized
|
||||
- copy of toss2-qib-optimized
|
||||
- toss2-mlx-optimized.conf
|
||||
- copy of toss2-qib-optimized.conf with the following changes:
|
||||
- remove: oob_tcp_if_include = ib0,eth0 (identification of general network
|
||||
device names is problematic in RHEL7. Just let Open MPI figure it out)
|
||||
- change: btl = vader,openib,self
|
||||
- change: btl_openib_receive_queues = X,4096,1024:X,12288,512:X,65536,512
|
||||
(change S to X; make sure numbers match those for the same entry in
|
||||
contrib/platform/lanl/toss/optimized-mlx.conf)
|
||||
- addition: pml = ob1 (disable MXM)
|
||||
- addition: coll = ^hcoll (disable MXM)
|
||||
- toss3-hfi-optimized
|
||||
- copy of toss2-qib-optimized
|
||||
- toss3-hfi-optimized.conf
|
||||
- copy of toss2-qib-optimized.conf with the following changes:
|
||||
- remove: oob_tcp_if_include = ib0,eth0
|
||||
- add: oob_tcp_if_exclude = ib0 (Omnipath is flaky; don't use it for oob)
|
||||
- toss3-wc-optimized (platform file for woodchuck which is an ethernet-only
|
||||
connected cluster)
|
||||
- copy of toss3-hfi-optimized with the following changes:
|
||||
- change: remove "btl-tcp" from the enable_mca_no_build list
|
||||
- change: comment "Disable components not needed on TOSS platforms with
|
||||
high-speed networks" to "Disable components not needed on TOSS Ethernet-
|
||||
connected clusters"
|
||||
- change: with_verbs=no
|
||||
- change: comment "Always build ibverbs support" to "Do not build ibverbs
|
||||
support"
|
||||
- toss3-wc-optimized.conf
|
||||
- copy of toss3-hfi-optimized.conf with the following changes:
|
||||
- change: comment "Add the interface for out-of-band communication and set
|
||||
it up" to "Set up the interface for out-of-band communication"
|
||||
- remove: oob_tcp_if_exclude = ib0
|
||||
- remove: btl (let Open MPI figure out what best to use for ethernet-
|
||||
connected hardware)
|
||||
- remove: btl_openib_want_fork_support (no infiniband)
|
||||
- remove: btl_openib_receive_queues (no infiniband)
|
||||
- cray-lustre-optimized
|
||||
- copy of contrib/platform/lanl/cray_xc_cle5.2/optimized-lustre with the
|
||||
following changes:
|
||||
- remove: whole if/else clause of 'test "$enable_debug" = "yes"'
|
||||
- addition: source ./common
|
||||
- addition: source ./common-optimized
|
||||
- change: with_io_romio_flags="--with-file-system=ufs+nfs+lustre"
|
||||
- remove: with_lustre=/opt/cray/lustre-cray_ari_s/default
|
||||
- additions from platform/lanl/cray_xc_cle5.2/optimized-common that don't
|
||||
go in common-optimzed:
|
||||
- enable_mca_no_build=crs,filem,routed-linear,snapc,pml-dr,pml-crcp2,pml-crcpw,pml-v,pml-example,crcp,pml-cm,ess-cnos,grpcomm-cnos,plm-rsh,btl-tcp,oob-ud,ras-simulator,mpool-fake
|
||||
- enable_mca_static=btl:ugni,btl:self,btl:vader,pml:ob1
|
||||
- enable_mca_directpml-ob1
|
||||
- with_verbs=no
|
||||
- with_tm=no
|
||||
- enable_orte_static_ports=no
|
||||
- enable_pty_support=no
|
||||
- addition: enable_dlopen=yes (change from original platform file as per
|
||||
Nathan Hjelm)
|
||||
- cray-lustre-optimized.conf
|
||||
- copy of contrib/platform/lanl/cray_xc_cle5.2/optimized-lustre.conf with
|
||||
the following changes:
|
||||
- change: orte_no_session_dirs = /lustre,/users,/usr/projects
|
||||
- remove: hwloc_base_binding_policy = core (outdated setting)
|
||||
- addition: rmaps_base_ranking_policy = core (rank by core)
|
||||
|
||||
# vi: filetype=txt
|
||||
20
contrib/platform/lanl/toss/common
Обычный файл
20
contrib/platform/lanl/toss/common
Обычный файл
@ -0,0 +1,20 @@
|
||||
# (c) 2013-1018 Los Alamos National Security, LLC. All rights reserved.
|
||||
# Open MPI common configuration for TOSS/TOSS2 v1.7.x/1.8.x
|
||||
|
||||
enable_binaries=yes
|
||||
enable_heterogeneous=no
|
||||
enable_shared=yes
|
||||
enable_static=yes
|
||||
enable_ipv6=no
|
||||
enable_ft_thread=no
|
||||
enable_per_user_config_files=no
|
||||
enable_memchecker=no
|
||||
with_valgrind=no
|
||||
|
||||
# Enable the fortran bindings
|
||||
enable_mpi_fortran=yes
|
||||
|
||||
# Disable the C++ binding. They were deprecated in MPI-2.2 and removed in MPI-3
|
||||
enable_mpi_cxx=no
|
||||
enable_mpi_cxx_seek=no
|
||||
enable_cxx_exceptions=no
|
||||
@ -1,4 +1,4 @@
|
||||
# (c) 2013 Los Alamos National Security, LLC. All rights reserved.
|
||||
# (c) 2013-2018 Los Alamos National Security, LLC. All rights reserved.
|
||||
# Open MPI common optimized configuration for TOSS/TOSS2 v1.7.x/1.8.x
|
||||
|
||||
enable_mem_debug=no
|
||||
34
contrib/platform/lanl/toss/cray-lustre-optimized
Обычный файл
34
contrib/platform/lanl/toss/cray-lustre-optimized
Обычный файл
@ -0,0 +1,34 @@
|
||||
# (c) 2012-2018 Los Alamos National Security, LLC. All rights reserved.
|
||||
# Open MPI configuration for Cray XC v2.x GNU compiler,
|
||||
# Lustre
|
||||
|
||||
if test "$CC" = "cc" ; then
|
||||
echo "ERROR: Open MPI should not be compiled with Cray's wrapper compilers (cc/CC/ftn)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
source ./common
|
||||
source ./common-optimized
|
||||
|
||||
# enable Lustre in romio
|
||||
with_io_romio_flags="--with-file-system=ufs+nfs+lustre"
|
||||
|
||||
# Disable components not needed
|
||||
enable_mca_no_build=crs,filem,routed-linear,snapc,pml-dr,pml-crcp2,pml-crcpw,pml-v,pml-example,crcp,pml-cm,ess-cnos,grpcomm-cnos,plm-rsh,btl-tcp,oob-ud,ras-simulator,mpool-fake
|
||||
enable_mca_static=btl:ugni,btl:self,btl:vader,pml:ob1
|
||||
|
||||
# enable direct calling for ob1
|
||||
enable_mca_direct=pml-ob1
|
||||
|
||||
# do not use IB verbs
|
||||
with_verbs=no
|
||||
|
||||
# do not use torque
|
||||
with_tm=no
|
||||
|
||||
enable_dlopen=yes
|
||||
|
||||
enable_orte_static_ports=no
|
||||
|
||||
enable_pty_support=no
|
||||
|
||||
@ -10,8 +10,8 @@
|
||||
# Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
# All rights reserved.
|
||||
# Copyright (c) 2006 Cisco Systems, Inc. All rights reserved.
|
||||
# Copyright (c) 2011-2016 Los Alamos National Security, LLC. All rights
|
||||
# reserved.
|
||||
# Copyright (c) 2015-2018 Los Alamos National Security, LLC.
|
||||
# All rights reserved.
|
||||
# $COPYRIGHT$
|
||||
#
|
||||
# Additional copyrights may follow
|
||||
@ -62,43 +62,46 @@
|
||||
|
||||
# Basic behavior to smooth startup
|
||||
mca_base_component_show_load_errors = 0
|
||||
opal_set_max_sys_limits = 1
|
||||
orte_report_launch_progress = 1
|
||||
#orte_report_launch_progress = 1
|
||||
|
||||
# Set line buffering for stdout/stderr
|
||||
ess_base_stream_buffering = 1
|
||||
|
||||
# Define timeout for daemons to report back during launch
|
||||
orte_startup_timeout = 10000
|
||||
orte_startup_timeout = 360
|
||||
|
||||
## Protect the shared file systems
|
||||
orte_no_session_dirs = /panfs,/scratch,/users,/usr/projects
|
||||
orte_tmpdir_base = /tmp
|
||||
orte_no_session_dirs = /lustre,/users,/usr/projects
|
||||
orte_tmpdir_base = /var/tmp
|
||||
|
||||
## Require an allocation to run - protects the frontend
|
||||
## from inadvertent job executions
|
||||
orte_allocation_required = 1
|
||||
|
||||
## Deal with the allocator
|
||||
orte_strip_prefix = nid
|
||||
orte_retain_aliases = 1
|
||||
# 1st alias entry is the stripped node name,
|
||||
# 2nd is the unstripped one
|
||||
orte_hostname_alias_index = 2
|
||||
|
||||
## Add the interface for out-of-band communication
|
||||
## and set it up
|
||||
oob_tcp_if_include=ib0,eth0
|
||||
oob_tcp_if_include=ipogif0
|
||||
oob_tcp_peer_retries = 1000
|
||||
oob_tcp_sndbuf = 32768
|
||||
oob_tcp_rcvbuf = 32768
|
||||
|
||||
## Define the MPI interconnects
|
||||
btl = vader,openib,self
|
||||
btl = self,vader,ugni
|
||||
|
||||
## Setup OpenIB - just in case
|
||||
btl_openib_want_fork_support = 0
|
||||
btl_openib_receive_queues = X,4096,1024:X,12288,512:X,65536,512
|
||||
## Setup Gemini
|
||||
# TODO LANL
|
||||
|
||||
## Disable MXM
|
||||
pml = ob1
|
||||
coll = ^hcoll
|
||||
|
||||
## Enable cpu affinity
|
||||
hwloc_base_binding_policy = core
|
||||
## Rank by core
|
||||
rmaps_base_ranking_policy = core
|
||||
|
||||
## Setup MPI options
|
||||
mpi_show_handle_leaks = 1
|
||||
mpi_warn_on_fork = 1
|
||||
#mpi_abort_print_stack = 1
|
||||
|
||||
@ -1,8 +0,0 @@
|
||||
# (c) 2013-2016 Los Alamos National Security, LLC. All rights reserved.
|
||||
# Open MPI debug configuration for TOSS/TOSS2 v1.7.x/1.8.x
|
||||
|
||||
source ./toss-common
|
||||
source ./debug-common
|
||||
|
||||
# Enable panasas support in romio
|
||||
with_io_romio_flags=--with-file-system=ufs+nfs+lustre
|
||||
@ -1,8 +0,0 @@
|
||||
# (c) 2013 Los Alamos National Security, LLC. All rights reserved.
|
||||
# Open MPI common debug configuration for TOSS/TOSS2 v1.7.x/1.8.x
|
||||
|
||||
enable_mem_debug=yes
|
||||
enable_mem_profile=yes
|
||||
enable_debug_symbols=yes
|
||||
enable_picky=yes
|
||||
enable_debug=yes
|
||||
@ -1,4 +0,0 @@
|
||||
# (c) 2013-2016 Los Alamos National Security, LLC. All rights reserved.
|
||||
# Open MPI debug configuration for TOSS/TOSS2 v1.7.x/1.8.x
|
||||
|
||||
source ./debug
|
||||
@ -1,8 +0,0 @@
|
||||
# (c) 2013-2016 Los Alamos National Security, LLC. All rights reserved.
|
||||
# Open MPI optimized configuration for TOSS/TOSS2 v1.7.x/1.8.x
|
||||
|
||||
source ./toss-common
|
||||
source ./optimized-common
|
||||
|
||||
# Enable panasas support in romio
|
||||
with_io_romio_flags=--with-file-system=ufs+nfs+lustre
|
||||
@ -1,4 +0,0 @@
|
||||
# (c) 2013-2016 Los Alamos National Security, LLC. All rights reserved.
|
||||
# Open MPI optimized configuration for TOSS/TOSS2 v1.7.x/1.8.x
|
||||
|
||||
source ./optimized
|
||||
@ -1,40 +0,0 @@
|
||||
# (c) 2013 Los Alamos National Security, LLC. All rights reserved.
|
||||
# Open MPI common configuration for TOSS/TOSS2 v1.7.x/1.8.x
|
||||
|
||||
enable_binaries=yes
|
||||
enable_heterogeneous=no
|
||||
enable_shared=yes
|
||||
enable_static=yes
|
||||
enable_ipv6=no
|
||||
enable_ft_thread=no
|
||||
enable_per_user_config_files=no
|
||||
enable_memchecker=no
|
||||
with_valgrind=no
|
||||
|
||||
# Enable the fortran bindings
|
||||
enable_mpi_fortran=yes
|
||||
|
||||
# Disable the C++ binding. They were deprecated in MPI-2.2 and removed in MPI-3
|
||||
enable_mpi_cxx=no
|
||||
enable_mpi_cxx_seek=no
|
||||
enable_cxx_exceptions=no
|
||||
|
||||
# Disable components not needed on any TOSS platform
|
||||
enable_mca_no_build=carto,crs,filem,routed-linear,snapc,pml-dr,pml-crcp2,pml-crcpw,pml-v,pml-example,crcp,btl-tcp
|
||||
|
||||
# Enable malloc hooks for mpi_leave_pinned
|
||||
with_memory_manager=linux
|
||||
|
||||
# TOSS2 uses slurm
|
||||
with_slurm=yes
|
||||
with_tm=no
|
||||
|
||||
# Enable PMI support for direct launch
|
||||
with_pmi=yes
|
||||
|
||||
# Always build ibverbs support
|
||||
with_verbs=yes
|
||||
|
||||
# Install the development headers
|
||||
with_devel_headers=yes
|
||||
|
||||
21
contrib/platform/lanl/toss/toss2-mlx-optimized
Обычный файл
21
contrib/platform/lanl/toss/toss2-mlx-optimized
Обычный файл
@ -0,0 +1,21 @@
|
||||
# (c) 2013-2018 Los Alamos National Security, LLC. All rights reserved.
|
||||
# Open MPI optimized configuration for TOSS/TOSS2 v1.7.x/1.8.x
|
||||
|
||||
source ./common
|
||||
source ./common-optimized
|
||||
|
||||
# Disable components not needed on TOSS platforms with high-speed networks
|
||||
enable_mca_no_build=carto,crs,filem,routed-linear,snapc,pml-dr,pml-crcp2,pml-crcpw,pml-v,pml-example,crcp,btl-tcp
|
||||
|
||||
# TOSS2 uses slurm
|
||||
with_slurm=yes
|
||||
with_tm=no
|
||||
|
||||
# Enable PMI support for direct launch
|
||||
with_pmi=yes
|
||||
|
||||
# Enable lustre support in romio
|
||||
with_io_romio_flags=--with-file-system=ufs+nfs+lustre
|
||||
|
||||
# Always build ibverbs support
|
||||
with_verbs=yes
|
||||
@ -10,7 +10,7 @@
|
||||
# Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
# All rights reserved.
|
||||
# Copyright (c) 2006 Cisco Systems, Inc. All rights reserved.
|
||||
# Copyright (c) 2011-2016 Los Alamos National Security, LLC. All rights
|
||||
# Copyright (c) 2011-2018 Los Alamos National Security, LLC. All rights
|
||||
# reserved.
|
||||
# $COPYRIGHT$
|
||||
#
|
||||
@ -69,7 +69,7 @@ orte_report_launch_progress = 1
|
||||
orte_startup_timeout = 10000
|
||||
|
||||
## Protect the shared file systems
|
||||
orte_no_session_dirs = /panfs,/scratch,/users,/usr/projects
|
||||
orte_no_session_dirs = /lustre,/net,/users,/usr/projects
|
||||
orte_tmpdir_base = /tmp
|
||||
|
||||
## Require an allocation to run - protects the frontend
|
||||
@ -88,17 +88,22 @@ btl = vader,openib,self
|
||||
|
||||
## Setup OpenIB - just in case
|
||||
btl_openib_want_fork_support = 0
|
||||
## Use Shared Receive Queues (SRQ). Mellanox ConnectX cards should be able to
|
||||
## use eXtended Reliable Connection receive queues (XRC), but our systems are
|
||||
## missing the needed libraries and headers to support it.
|
||||
btl_openib_receive_queues = S,4096,1024:S,12288,512:S,65536,512
|
||||
|
||||
## Disable MXM
|
||||
pml = ob1
|
||||
coll = ^hcoll
|
||||
|
||||
## Enable cpu affinity
|
||||
hwloc_base_binding_policy = core
|
||||
## Rank by core
|
||||
rmaps_base_ranking_policy = core
|
||||
|
||||
## Setup MPI options
|
||||
mpi_show_handle_leaks = 0
|
||||
mpi_warn_on_fork = 1
|
||||
#mpi_abort_print_stack = 0
|
||||
|
||||
## Run orted on parent node of allocation
|
||||
ras_base_launch_orted_on_hn = true
|
||||
|
||||
## Disable MXM
|
||||
pml = ob1
|
||||
coll = ^hcoll
|
||||
21
contrib/platform/lanl/toss/toss2-qib-optimized
Обычный файл
21
contrib/platform/lanl/toss/toss2-qib-optimized
Обычный файл
@ -0,0 +1,21 @@
|
||||
# (c) 2013-2018 Los Alamos National Security, LLC. All rights reserved.
|
||||
# Open MPI optimized configuration for TOSS/TOSS2 v1.7.x/1.8.x
|
||||
|
||||
source ./common
|
||||
source ./common-optimized
|
||||
|
||||
# Disable components not needed on TOSS platforms with high-speed networks
|
||||
enable_mca_no_build=carto,crs,filem,routed-linear,snapc,pml-dr,pml-crcp2,pml-crcpw,pml-v,pml-example,crcp,btl-tcp
|
||||
|
||||
# TOSS2 uses slurm
|
||||
with_slurm=yes
|
||||
with_tm=no
|
||||
|
||||
# Enable PMI support for direct launch
|
||||
with_pmi=yes
|
||||
|
||||
# Enable lustre support in romio
|
||||
with_io_romio_flags=--with-file-system=ufs+nfs+lustre
|
||||
|
||||
# Always build ibverbs support
|
||||
with_verbs=yes
|
||||
111
contrib/platform/lanl/toss/toss2-qib-optimized.conf
Обычный файл
111
contrib/platform/lanl/toss/toss2-qib-optimized.conf
Обычный файл
@ -0,0 +1,111 @@
|
||||
#
|
||||
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
# University Research and Technology
|
||||
# Corporation. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
# of Tennessee Research Foundation. All rights
|
||||
# reserved.
|
||||
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
# University of Stuttgart. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
# All rights reserved.
|
||||
# Copyright (c) 2006 Cisco Systems, Inc. All rights reserved.
|
||||
# Copyright (c) 2011-2018 Los Alamos National Security, LLC. All rights
|
||||
# reserved.
|
||||
# $COPYRIGHT$
|
||||
#
|
||||
# Additional copyrights may follow
|
||||
#
|
||||
# $HEADER$
|
||||
#
|
||||
|
||||
# This is the default system-wide MCA parameters defaults file.
|
||||
# Specifically, the MCA parameter "mca_param_files" defaults to a
|
||||
# value of
|
||||
# "$HOME/.openmpi/mca-params.conf:$sysconf/openmpi-mca-params.conf"
|
||||
# (this file is the latter of the two). So if the default value of
|
||||
# mca_param_files is not changed, this file is used to set system-wide
|
||||
# MCA parameters. This file can therefore be used to set system-wide
|
||||
# default MCA parameters for all users. Of course, users can override
|
||||
# these values if they want, but this file is an excellent location
|
||||
# for setting system-specific MCA parameters for those users who don't
|
||||
# know / care enough to investigate the proper values for them.
|
||||
|
||||
# Note that this file is only applicable where it is visible (in a
|
||||
# filesystem sense). Specifically, MPI processes each read this file
|
||||
# during their startup to determine what default values for MCA
|
||||
# parameters should be used. mpirun does not bundle up the values in
|
||||
# this file from the node where it was run and send them to all nodes;
|
||||
# the default value decisions are effectively distributed. Hence,
|
||||
# these values are only applicable on nodes that "see" this file. If
|
||||
# $sysconf is a directory on a local disk, it is likely that changes
|
||||
# to this file will need to be propagated to other nodes. If $sysconf
|
||||
# is a directory that is shared via a networked filesystem, changes to
|
||||
# this file will be visible to all nodes that share this $sysconf.
|
||||
|
||||
# The format is straightforward: one per line, mca_param_name =
|
||||
# rvalue. Quoting is ignored (so if you use quotes or escape
|
||||
# characters, they'll be included as part of the value). For example:
|
||||
|
||||
# Disable run-time MPI parameter checking
|
||||
# mpi_param_check = 0
|
||||
|
||||
# Note that the value "~/" will be expanded to the current user's home
|
||||
# directory. For example:
|
||||
|
||||
# Change component loading path
|
||||
# component_path = /usr/local/lib/openmpi:~/my_openmpi_components
|
||||
|
||||
# See "ompi_info --param all all" for a full listing of Open MPI MCA
|
||||
# parameters available and their default values.
|
||||
#
|
||||
|
||||
# Basic behavior to smooth startup
|
||||
mca_base_component_show_load_errors = 0
|
||||
opal_set_max_sys_limits = 1
|
||||
orte_report_launch_progress = 1
|
||||
|
||||
# Define timeout for daemons to report back during launch
|
||||
orte_startup_timeout = 10000
|
||||
|
||||
## Protect the shared file systems
|
||||
orte_no_session_dirs = /lustre,/net,/users,/usr/projects
|
||||
orte_tmpdir_base = /tmp
|
||||
|
||||
## Require an allocation to run - protects the frontend
|
||||
## from inadvertent job executions
|
||||
orte_allocation_required = 1
|
||||
|
||||
## Add the interface for out-of-band communication
|
||||
## and set it up
|
||||
oob_tcp_if_include = ib0,eth0
|
||||
oob_tcp_peer_retries = 1000
|
||||
oob_tcp_sndbuf = 32768
|
||||
oob_tcp_rcvbuf = 32768
|
||||
|
||||
## Define the MPI interconnects
|
||||
btl = ^openib
|
||||
|
||||
## Turn off osc rdma component. This is used in one-sided communication which
|
||||
## isn't supported on psm2 and omnipath interconnects. If this option isn't
|
||||
## used, a runtime error about btl's will result because we are turning off
|
||||
## openib above. The current implementation of one-sided communication by-
|
||||
## passes the loaded components and goes straight for btl's. Since one-sided
|
||||
## communication doesn't really work on psm2 and omnipath, rather than turn
|
||||
## openib back on, we're going to turn off some rdma capability.
|
||||
osc = ^rdma
|
||||
|
||||
## Setup OpenIB - just in case
|
||||
btl_openib_want_fork_support = 0
|
||||
btl_openib_receive_queues = S,4096,1024:S,12288,512:S,65536,512
|
||||
|
||||
## Rank by core
|
||||
rmaps_base_ranking_policy = core
|
||||
|
||||
## Setup MPI options
|
||||
mpi_show_handle_leaks = 0
|
||||
mpi_warn_on_fork = 1
|
||||
#mpi_abort_print_stack = 0
|
||||
|
||||
## Run orted on parent node of allocation
|
||||
ras_base_launch_orted_on_hn = true
|
||||
21
contrib/platform/lanl/toss/toss3-hfi-optimized
Обычный файл
21
contrib/platform/lanl/toss/toss3-hfi-optimized
Обычный файл
@ -0,0 +1,21 @@
|
||||
# (c) 2013-2018 Los Alamos National Security, LLC. All rights reserved.
|
||||
# Open MPI optimized configuration for TOSS/TOSS2 v1.7.x/1.8.x
|
||||
|
||||
source ./common
|
||||
source ./common-optimized
|
||||
|
||||
# Disable components not needed on TOSS platforms with high-speed networks
|
||||
enable_mca_no_build=carto,crs,filem,routed-linear,snapc,pml-dr,pml-crcp2,pml-crcpw,pml-v,pml-example,crcp,btl-tcp
|
||||
|
||||
# TOSS2 uses slurm
|
||||
with_slurm=yes
|
||||
with_tm=no
|
||||
|
||||
# Enable PMI support for direct launch
|
||||
with_pmi=yes
|
||||
|
||||
# Enable lustre support in romio
|
||||
with_io_romio_flags=--with-file-system=ufs+nfs+lustre
|
||||
|
||||
# Always build ibverbs support
|
||||
with_verbs=yes
|
||||
112
contrib/platform/lanl/toss/toss3-hfi-optimized.conf
Обычный файл
112
contrib/platform/lanl/toss/toss3-hfi-optimized.conf
Обычный файл
@ -0,0 +1,112 @@
|
||||
#
|
||||
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
# University Research and Technology
|
||||
# Corporation. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
# of Tennessee Research Foundation. All rights
|
||||
# reserved.
|
||||
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
# University of Stuttgart. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
# All rights reserved.
|
||||
# Copyright (c) 2006 Cisco Systems, Inc. All rights reserved.
|
||||
# Copyright (c) 2011-2018 Los Alamos National Security, LLC. All rights
|
||||
# reserved.
|
||||
# $COPYRIGHT$
|
||||
#
|
||||
# Additional copyrights may follow
|
||||
#
|
||||
# $HEADER$
|
||||
#
|
||||
|
||||
# This is the default system-wide MCA parameters defaults file.
|
||||
# Specifically, the MCA parameter "mca_param_files" defaults to a
|
||||
# value of
|
||||
# "$HOME/.openmpi/mca-params.conf:$sysconf/openmpi-mca-params.conf"
|
||||
# (this file is the latter of the two). So if the default value of
|
||||
# mca_param_files is not changed, this file is used to set system-wide
|
||||
# MCA parameters. This file can therefore be used to set system-wide
|
||||
# default MCA parameters for all users. Of course, users can override
|
||||
# these values if they want, but this file is an excellent location
|
||||
# for setting system-specific MCA parameters for those users who don't
|
||||
# know / care enough to investigate the proper values for them.
|
||||
|
||||
# Note that this file is only applicable where it is visible (in a
|
||||
# filesystem sense). Specifically, MPI processes each read this file
|
||||
# during their startup to determine what default values for MCA
|
||||
# parameters should be used. mpirun does not bundle up the values in
|
||||
# this file from the node where it was run and send them to all nodes;
|
||||
# the default value decisions are effectively distributed. Hence,
|
||||
# these values are only applicable on nodes that "see" this file. If
|
||||
# $sysconf is a directory on a local disk, it is likely that changes
|
||||
# to this file will need to be propagated to other nodes. If $sysconf
|
||||
# is a directory that is shared via a networked filesystem, changes to
|
||||
# this file will be visible to all nodes that share this $sysconf.
|
||||
|
||||
# The format is straightforward: one per line, mca_param_name =
|
||||
# rvalue. Quoting is ignored (so if you use quotes or escape
|
||||
# characters, they'll be included as part of the value). For example:
|
||||
|
||||
# Disable run-time MPI parameter checking
|
||||
# mpi_param_check = 0
|
||||
|
||||
# Note that the value "~/" will be expanded to the current user's home
|
||||
# directory. For example:
|
||||
|
||||
# Change component loading path
|
||||
# component_path = /usr/local/lib/openmpi:~/my_openmpi_components
|
||||
|
||||
# See "ompi_info --param all all" for a full listing of Open MPI MCA
|
||||
# parameters available and their default values.
|
||||
#
|
||||
|
||||
# Basic behavior to smooth startup
|
||||
mca_base_component_show_load_errors = 0
|
||||
opal_set_max_sys_limits = 1
|
||||
orte_report_launch_progress = 1
|
||||
|
||||
# Define timeout for daemons to report back during launch
|
||||
orte_startup_timeout = 10000
|
||||
|
||||
## Protect the shared file systems
|
||||
orte_no_session_dirs = /lustre,/net,/users,/usr/projects
|
||||
orte_tmpdir_base = /tmp
|
||||
|
||||
## Require an allocation to run - protects the frontend
|
||||
## from inadvertent job executions
|
||||
orte_allocation_required = 1
|
||||
|
||||
## Set up out-of-band communication.
|
||||
## For OmniPath connected machines, don't use the ib0 network for
|
||||
## out-of-band communication. It's a little flaky.
|
||||
oob_tcp_if_exclude = ib0
|
||||
oob_tcp_peer_retries = 1000
|
||||
oob_tcp_sndbuf = 32768
|
||||
oob_tcp_rcvbuf = 32768
|
||||
|
||||
## Define the MPI interconnects
|
||||
btl = ^openib
|
||||
|
||||
## Turn off osc rdma component. This is used in one-sided communication which
|
||||
## isn't supported on psm2 and omnipath interconnects. If this option isn't
|
||||
## used, a runtime error about btl's will result because we are turning off
|
||||
## openib above. The current implementation of one-sided communication by-
|
||||
## passes the loaded components and goes straight for btl's. Since one-sided
|
||||
## communication doesn't really work on psm2 and omnipath, rather than turn
|
||||
## openib back on, we're going to turn off some rdma capability.
|
||||
osc = ^rdma
|
||||
|
||||
## Setup OpenIB - just in case
|
||||
btl_openib_want_fork_support = 0
|
||||
btl_openib_receive_queues = S,4096,1024:S,12288,512:S,65536,512
|
||||
|
||||
## Rank by core
|
||||
rmaps_base_ranking_policy = core
|
||||
|
||||
## Setup MPI options
|
||||
mpi_show_handle_leaks = 0
|
||||
mpi_warn_on_fork = 1
|
||||
#mpi_abort_print_stack = 0
|
||||
|
||||
## Run orted on parent node of allocation
|
||||
ras_base_launch_orted_on_hn = true
|
||||
21
contrib/platform/lanl/toss/toss3-mlx-optimized
Обычный файл
21
contrib/platform/lanl/toss/toss3-mlx-optimized
Обычный файл
@ -0,0 +1,21 @@
|
||||
# (c) 2013-2018 Los Alamos National Security, LLC. All rights reserved.
|
||||
# Open MPI optimized configuration for TOSS/TOSS2 v1.7.x/1.8.x
|
||||
|
||||
source ./common
|
||||
source ./common-optimized
|
||||
|
||||
# Disable components not needed on TOSS platforms with high-speed networks
|
||||
enable_mca_no_build=carto,crs,filem,routed-linear,snapc,pml-dr,pml-crcp2,pml-crcpw,pml-v,pml-example,crcp,btl-tcp
|
||||
|
||||
# TOSS2 uses slurm
|
||||
with_slurm=yes
|
||||
with_tm=no
|
||||
|
||||
# Enable PMI support for direct launch
|
||||
with_pmi=yes
|
||||
|
||||
# Enable lustre support in romio
|
||||
with_io_romio_flags=--with-file-system=ufs+nfs+lustre
|
||||
|
||||
# Always build ibverbs support
|
||||
with_verbs=yes
|
||||
@ -10,7 +10,7 @@
|
||||
# Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
# All rights reserved.
|
||||
# Copyright (c) 2006 Cisco Systems, Inc. All rights reserved.
|
||||
# Copyright (c) 2011-2014 Los Alamos National Security, LLC. All rights
|
||||
# Copyright (c) 2011-2018 Los Alamos National Security, LLC. All rights
|
||||
# reserved.
|
||||
# $COPYRIGHT$
|
||||
#
|
||||
@ -69,16 +69,14 @@ orte_report_launch_progress = 1
|
||||
orte_startup_timeout = 10000
|
||||
|
||||
## Protect the shared file systems
|
||||
orte_no_session_dirs = /panfs,/scratch,/users,/usr/projects
|
||||
orte_no_session_dirs = /lustre,/net,/users,/usr/projects
|
||||
orte_tmpdir_base = /tmp
|
||||
|
||||
## Require an allocation to run - protects the frontend
|
||||
## from inadvertent job executions
|
||||
orte_allocation_required = 1
|
||||
|
||||
## Add the interface for out-of-band communication
|
||||
## and set it up
|
||||
oob_tcp_if_include=ib0,eth0
|
||||
## Set up out-of-band communication.
|
||||
oob_tcp_peer_retries = 1000
|
||||
oob_tcp_sndbuf = 32768
|
||||
oob_tcp_rcvbuf = 32768
|
||||
@ -88,13 +86,18 @@ btl = vader,openib,self
|
||||
|
||||
## Setup OpenIB - just in case
|
||||
btl_openib_want_fork_support = 0
|
||||
## Use Shared Receive Queues (SRQ). Mellanox ConnectX cards should be able to
|
||||
## use eXtended Reliable Connection receive queues (XRC), but our systems are
|
||||
## missing the needed libraries and headers to support it.
|
||||
btl_openib_receive_queues = S,4096,1024:S,12288,512:S,65536,512
|
||||
|
||||
## Enable cpu affinity
|
||||
hwloc_base_binding_policy = core
|
||||
## Rank by core
|
||||
rmaps_base_ranking_policy = core
|
||||
|
||||
## Setup MPI options
|
||||
mpi_show_handle_leaks = 1
|
||||
mpi_show_handle_leaks = 0
|
||||
mpi_warn_on_fork = 1
|
||||
#mpi_abort_print_stack = 1
|
||||
#mpi_abort_print_stack = 0
|
||||
|
||||
## Run orted on parent node of allocation
|
||||
ras_base_launch_orted_on_hn = true
|
||||
21
contrib/platform/lanl/toss/toss3-wc-optimized
Обычный файл
21
contrib/platform/lanl/toss/toss3-wc-optimized
Обычный файл
@ -0,0 +1,21 @@
|
||||
# (c) 2013-2018 Los Alamos National Security, LLC. All rights reserved.
|
||||
# Open MPI optimized configuration for TOSS/TOSS2 v1.7.x/1.8.x
|
||||
|
||||
source ./common
|
||||
source ./common-optimized
|
||||
|
||||
# Disable components not needed on TOSS Ethernet-connected clusters
|
||||
enable_mca_no_build=carto,crs,filem,routed-linear,snapc,pml-dr,pml-crcp2,pml-crcpw,pml-v,pml-example,crcp
|
||||
|
||||
# TOSS2 uses slurm
|
||||
with_slurm=yes
|
||||
with_tm=no
|
||||
|
||||
# Enable PMI support for direct launch
|
||||
with_pmi=yes
|
||||
|
||||
# Enable lustre support in romio
|
||||
with_io_romio_flags=--with-file-system=ufs+nfs+lustre
|
||||
|
||||
# Do not build ibverbs support
|
||||
with_verbs=no
|
||||
@ -10,7 +10,7 @@
|
||||
# Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
# All rights reserved.
|
||||
# Copyright (c) 2006 Cisco Systems, Inc. All rights reserved.
|
||||
# Copyright (c) 2011-2014 Los Alamos National Security, LLC. All rights
|
||||
# Copyright (c) 2011-2018 Los Alamos National Security, LLC. All rights
|
||||
# reserved.
|
||||
# $COPYRIGHT$
|
||||
#
|
||||
@ -69,32 +69,25 @@ orte_report_launch_progress = 1
|
||||
orte_startup_timeout = 10000
|
||||
|
||||
## Protect the shared file systems
|
||||
orte_no_session_dirs = /panfs,/scratch,/users,/usr/projects
|
||||
orte_no_session_dirs = /lustre,/net,/users,/usr/projects
|
||||
orte_tmpdir_base = /tmp
|
||||
|
||||
## Require an allocation to run - protects the frontend
|
||||
## from inadvertent job executions
|
||||
orte_allocation_required = 1
|
||||
|
||||
## Add the interface for out-of-band communication
|
||||
## and set it up
|
||||
oob_tcp_if_include = ib0,eth0
|
||||
## Set up the interface for out-of-band communication
|
||||
oob_tcp_peer_retries = 1000
|
||||
oob_tcp_sndbuf = 32768
|
||||
oob_tcp_rcvbuf = 32768
|
||||
|
||||
## Define the MPI interconnects
|
||||
btl = vader,openib,self
|
||||
|
||||
## Setup OpenIB - just in case
|
||||
btl_openib_want_fork_support = 0
|
||||
btl_openib_receive_queues = S,4096,1024:S,12288,512:S,65536,512
|
||||
|
||||
## Enable cpu affinity
|
||||
hwloc_base_binding_policy = core
|
||||
## Rank by core
|
||||
rmaps_base_ranking_policy = core
|
||||
|
||||
## Setup MPI options
|
||||
mpi_show_handle_leaks = 0
|
||||
mpi_warn_on_fork = 1
|
||||
#mpi_abort_print_stack = 0
|
||||
|
||||
## Run orted on parent node of allocation
|
||||
ras_base_launch_orted_on_hn = true
|
||||
@ -27,13 +27,13 @@ libompitrace_la_SOURCES = \
|
||||
add_error_class.c \
|
||||
add_error_code.c \
|
||||
add_error_string.c \
|
||||
address.c \
|
||||
allgather.c \
|
||||
allgatherv.c \
|
||||
alloc_mem.c \
|
||||
allreduce.c \
|
||||
barrier.c \
|
||||
bcast.c \
|
||||
get_address.c \
|
||||
finalize.c \
|
||||
init.c \
|
||||
isend.c \
|
||||
|
||||
@ -10,6 +10,8 @@
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2009 Cisco Systems, Inc. All rights reserved.
|
||||
* Copyright (c) 2018 Los Alamos National Security, LLC. All
|
||||
* rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
@ -24,16 +26,16 @@
|
||||
#include "opal_stdint.h"
|
||||
#include "ompi/mpi/c/bindings.h"
|
||||
|
||||
int MPI_Address(void *location, MPI_Aint *address)
|
||||
int MPI_Get_address(const void *location, MPI_Aint *address)
|
||||
{
|
||||
|
||||
int rank;
|
||||
|
||||
PMPI_Comm_rank(MPI_COMM_WORLD, &rank);
|
||||
|
||||
fprintf(stderr, "MPI_ADDRESS[%d]: location %0" PRIxPTR " address %0" PRIxPTR "\n",
|
||||
fprintf(stderr, "MPI_GET_ADDRESS[%d]: location %0" PRIxPTR " address %0" PRIxPTR "\n",
|
||||
rank, (uintptr_t)location, (uintptr_t)address);
|
||||
fflush(stderr);
|
||||
|
||||
return PMPI_Address(location, address);
|
||||
return PMPI_Get_address(location, address);
|
||||
}
|
||||
@ -178,7 +178,7 @@ OMPI_DECLSPEC int32_t ompi_datatype_destroy( ompi_datatype_t** type);
|
||||
* Datatype creation functions
|
||||
*/
|
||||
static inline int32_t
|
||||
ompi_datatype_add( ompi_datatype_t* pdtBase, const ompi_datatype_t* pdtAdd, uint32_t count,
|
||||
ompi_datatype_add( ompi_datatype_t* pdtBase, const ompi_datatype_t* pdtAdd, size_t count,
|
||||
ptrdiff_t disp, ptrdiff_t extent )
|
||||
{
|
||||
return opal_datatype_add( &pdtBase->super, &pdtAdd->super, count, disp, extent );
|
||||
|
||||
@ -49,7 +49,7 @@ int32_t ompi_datatype_create_vector( int count, int bLength, int stride,
|
||||
|
||||
pData = ompi_datatype_create( oldType->super.desc.used + 2 );
|
||||
if( (bLength == stride) || (1 >= count) ) { /* the elements are contiguous */
|
||||
ompi_datatype_add( pData, oldType, count * bLength, 0, extent );
|
||||
ompi_datatype_add( pData, oldType, (size_t)count * bLength, 0, extent );
|
||||
} else {
|
||||
if( 1 == bLength ) {
|
||||
ompi_datatype_add( pData, oldType, count, 0, extent * stride );
|
||||
|
||||
@ -43,4 +43,6 @@ libmca_coll_la_SOURCES += \
|
||||
base/coll_base_reduce.c \
|
||||
base/coll_base_barrier.c \
|
||||
base/coll_base_reduce_scatter.c \
|
||||
base/coll_base_reduce_scatter_block.c
|
||||
base/coll_base_reduce_scatter_block.c \
|
||||
base/coll_base_exscan.c \
|
||||
base/coll_base_scan.c
|
||||
|
||||
@ -15,6 +15,8 @@
|
||||
* reserved.
|
||||
* Copyright (c) 2015-2017 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
* Copyright (c) 2018 Siberian State University of Telecommunications
|
||||
* and Information Science. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
@ -907,5 +909,335 @@ ompi_coll_base_allreduce_intra_basic_linear(const void *sbuf, void *rbuf, int co
|
||||
return ompi_coll_base_bcast_intra_basic_linear(rbuf, count, dtype, 0, comm, module);
|
||||
}
|
||||
|
||||
/*
|
||||
* ompi_coll_base_allreduce_intra_redscat_allgather
|
||||
*
|
||||
* Function: Allreduce using Rabenseifner's algorithm.
|
||||
* Accepts: Same arguments as MPI_Allreduce
|
||||
* Returns: MPI_SUCCESS or error code
|
||||
*
|
||||
* Description: an implementation of Rabenseifner's allreduce algorithm [1, 2].
|
||||
* [1] Rajeev Thakur, Rolf Rabenseifner and William Gropp.
|
||||
* Optimization of Collective Communication Operations in MPICH //
|
||||
* The Int. Journal of High Performance Computing Applications. Vol 19,
|
||||
* Issue 1, pp. 49--66.
|
||||
* [2] http://www.hlrs.de/mpi/myreduce.html.
|
||||
*
|
||||
* This algorithm is a combination of a reduce-scatter implemented with
|
||||
* recursive vector halving and recursive distance doubling, followed either
|
||||
* by an allgather implemented with recursive doubling [1].
|
||||
*
|
||||
* Step 1. If the number of processes is not a power of two, reduce it to
|
||||
* the nearest lower power of two (p' = 2^{\floor{\log_2 p}})
|
||||
* by removing r = p - p' extra processes as follows. In the first 2r processes
|
||||
* (ranks 0 to 2r - 1), all the even ranks send the second half of the input
|
||||
* vector to their right neighbor (rank + 1), and all the odd ranks send
|
||||
* the first half of the input vector to their left neighbor (rank - 1).
|
||||
* The even ranks compute the reduction on the first half of the vector and
|
||||
* the odd ranks compute the reduction on the second half. The odd ranks then
|
||||
* send the result to their left neighbors (the even ranks). As a result,
|
||||
* the even ranks among the first 2r processes now contain the reduction with
|
||||
* the input vector on their right neighbors (the odd ranks). These odd ranks
|
||||
* do not participate in the rest of the algorithm, which leaves behind
|
||||
* a power-of-two number of processes. The first r even-ranked processes and
|
||||
* the last p - 2r processes are now renumbered from 0 to p' - 1.
|
||||
*
|
||||
* Step 2. The remaining processes now perform a reduce-scatter by using
|
||||
* recursive vector halving and recursive distance doubling. The even-ranked
|
||||
* processes send the second half of their buffer to rank + 1 and the odd-ranked
|
||||
* processes send the first half of their buffer to rank - 1. All processes
|
||||
* then compute the reduction between the local buffer and the received buffer.
|
||||
* In the next log_2(p') - 1 steps, the buffers are recursively halved, and the
|
||||
* distance is doubled. At the end, each of the p' processes has 1 / p' of the
|
||||
* total reduction result.
|
||||
*
|
||||
* Step 3. An allgather is performed by using recursive vector doubling and
|
||||
* distance halving. All exchanges are executed in reverse order relative
|
||||
* to recursive doubling on previous step. If the number of processes is not
|
||||
* a power of two, the total result vector must be sent to the r processes
|
||||
* that were removed in the first step.
|
||||
*
|
||||
* Limitations:
|
||||
* count >= 2^{\floor{\log_2 p}}
|
||||
* commutative operations only
|
||||
* intra-communicators only
|
||||
*
|
||||
* Memory requirements (per process):
|
||||
* count * typesize + 4 * \log_2(p) * sizeof(int) = O(count)
|
||||
*/
|
||||
int ompi_coll_base_allreduce_intra_redscat_allgather(
|
||||
const void *sbuf, void *rbuf, int count, struct ompi_datatype_t *dtype,
|
||||
struct ompi_op_t *op, struct ompi_communicator_t *comm,
|
||||
mca_coll_base_module_t *module)
|
||||
{
|
||||
int *rindex = NULL, *rcount = NULL, *sindex = NULL, *scount = NULL;
|
||||
|
||||
int comm_size = ompi_comm_size(comm);
|
||||
int rank = ompi_comm_rank(comm);
|
||||
OPAL_OUTPUT((ompi_coll_base_framework.framework_output,
|
||||
"coll:base:allreduce_intra_redscat_allgather: rank %d/%d",
|
||||
rank, comm_size));
|
||||
|
||||
/* Find nearest power-of-two less than or equal to comm_size */
|
||||
int nsteps = opal_hibit(comm_size, comm->c_cube_dim + 1); /* ilog2(comm_size) */
|
||||
assert(nsteps >= 0);
|
||||
int nprocs_pof2 = 1 << nsteps; /* flp2(comm_size) */
|
||||
|
||||
if (count < nprocs_pof2 || !ompi_op_is_commute(op)) {
|
||||
OPAL_OUTPUT((ompi_coll_base_framework.framework_output,
|
||||
"coll:base:allreduce_intra_redscat_allgather: rank %d/%d "
|
||||
"count %d switching to basic linear allreduce",
|
||||
rank, comm_size, count));
|
||||
return ompi_coll_base_allreduce_intra_basic_linear(sbuf, rbuf, count, dtype,
|
||||
op, comm, module);
|
||||
}
|
||||
|
||||
int err = MPI_SUCCESS;
|
||||
ptrdiff_t lb, extent, dsize, gap = 0;
|
||||
ompi_datatype_get_extent(dtype, &lb, &extent);
|
||||
dsize = opal_datatype_span(&dtype->super, count, &gap);
|
||||
|
||||
/* Temporary buffer for receiving messages */
|
||||
char *tmp_buf = NULL;
|
||||
char *tmp_buf_raw = (char *)malloc(dsize);
|
||||
if (NULL == tmp_buf_raw)
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
tmp_buf = tmp_buf_raw - gap;
|
||||
|
||||
if (sbuf != MPI_IN_PLACE) {
|
||||
err = ompi_datatype_copy_content_same_ddt(dtype, count, (char *)rbuf,
|
||||
(char *)sbuf);
|
||||
if (MPI_SUCCESS != err) { goto cleanup_and_return; }
|
||||
}
|
||||
|
||||
/*
|
||||
* Step 1. Reduce the number of processes to the nearest lower power of two
|
||||
* p' = 2^{\floor{\log_2 p}} by removing r = p - p' processes.
|
||||
* 1. In the first 2r processes (ranks 0 to 2r - 1), all the even ranks send
|
||||
* the second half of the input vector to their right neighbor (rank + 1)
|
||||
* and all the odd ranks send the first half of the input vector to their
|
||||
* left neighbor (rank - 1).
|
||||
* 2. All 2r processes compute the reduction on their half.
|
||||
* 3. The odd ranks then send the result to their left neighbors
|
||||
* (the even ranks).
|
||||
*
|
||||
* The even ranks (0 to 2r - 1) now contain the reduction with the input
|
||||
* vector on their right neighbors (the odd ranks). The first r even
|
||||
* processes and the p - 2r last processes are renumbered from
|
||||
* 0 to 2^{\floor{\log_2 p}} - 1.
|
||||
*/
|
||||
|
||||
int vrank, step, wsize;
|
||||
int nprocs_rem = comm_size - nprocs_pof2;
|
||||
|
||||
if (rank < 2 * nprocs_rem) {
|
||||
int count_lhalf = count / 2;
|
||||
int count_rhalf = count - count_lhalf;
|
||||
|
||||
if (rank % 2 != 0) {
|
||||
/*
|
||||
* Odd process -- exchange with rank - 1
|
||||
* Send the left half of the input vector to the left neighbor,
|
||||
* Recv the right half of the input vector from the left neighbor
|
||||
*/
|
||||
err = ompi_coll_base_sendrecv(rbuf, count_lhalf, dtype, rank - 1,
|
||||
MCA_COLL_BASE_TAG_ALLREDUCE,
|
||||
(char *)tmp_buf + (ptrdiff_t)count_lhalf * extent,
|
||||
count_rhalf, dtype, rank - 1,
|
||||
MCA_COLL_BASE_TAG_ALLREDUCE, comm,
|
||||
MPI_STATUS_IGNORE, rank);
|
||||
if (MPI_SUCCESS != err) { goto cleanup_and_return; }
|
||||
|
||||
/* Reduce on the right half of the buffers (result in rbuf) */
|
||||
ompi_op_reduce(op, (char *)tmp_buf + (ptrdiff_t)count_lhalf * extent,
|
||||
(char *)rbuf + count_lhalf * extent, count_rhalf, dtype);
|
||||
|
||||
/* Send the right half to the left neighbor */
|
||||
err = MCA_PML_CALL(send((char *)rbuf + (ptrdiff_t)count_lhalf * extent,
|
||||
count_rhalf, dtype, rank - 1,
|
||||
MCA_COLL_BASE_TAG_ALLREDUCE,
|
||||
MCA_PML_BASE_SEND_STANDARD, comm));
|
||||
if (MPI_SUCCESS != err) { goto cleanup_and_return; }
|
||||
|
||||
/* This process does not pariticipate in recursive doubling phase */
|
||||
vrank = -1;
|
||||
|
||||
} else {
|
||||
/*
|
||||
* Even process -- exchange with rank + 1
|
||||
* Send the right half of the input vector to the right neighbor,
|
||||
* Recv the left half of the input vector from the right neighbor
|
||||
*/
|
||||
err = ompi_coll_base_sendrecv((char *)rbuf + (ptrdiff_t)count_lhalf * extent,
|
||||
count_rhalf, dtype, rank + 1,
|
||||
MCA_COLL_BASE_TAG_ALLREDUCE,
|
||||
tmp_buf, count_lhalf, dtype, rank + 1,
|
||||
MCA_COLL_BASE_TAG_ALLREDUCE, comm,
|
||||
MPI_STATUS_IGNORE, rank);
|
||||
if (MPI_SUCCESS != err) { goto cleanup_and_return; }
|
||||
|
||||
/* Reduce on the right half of the buffers (result in rbuf) */
|
||||
ompi_op_reduce(op, tmp_buf, rbuf, count_lhalf, dtype);
|
||||
|
||||
/* Recv the right half from the right neighbor */
|
||||
err = MCA_PML_CALL(recv((char *)rbuf + (ptrdiff_t)count_lhalf * extent,
|
||||
count_rhalf, dtype, rank + 1,
|
||||
MCA_COLL_BASE_TAG_ALLREDUCE, comm,
|
||||
MPI_STATUS_IGNORE));
|
||||
if (MPI_SUCCESS != err) { goto cleanup_and_return; }
|
||||
|
||||
vrank = rank / 2;
|
||||
}
|
||||
} else { /* rank >= 2 * nprocs_rem */
|
||||
vrank = rank - nprocs_rem;
|
||||
}
|
||||
|
||||
/*
|
||||
* Step 2. Reduce-scatter implemented with recursive vector halving and
|
||||
* recursive distance doubling. We have p' = 2^{\floor{\log_2 p}}
|
||||
* power-of-two number of processes with new ranks (vrank) and result in rbuf.
|
||||
*
|
||||
* The even-ranked processes send the right half of their buffer to rank + 1
|
||||
* and the odd-ranked processes send the left half of their buffer to
|
||||
* rank - 1. All processes then compute the reduction between the local
|
||||
* buffer and the received buffer. In the next \log_2(p') - 1 steps, the
|
||||
* buffers are recursively halved, and the distance is doubled. At the end,
|
||||
* each of the p' processes has 1 / p' of the total reduction result.
|
||||
*/
|
||||
rindex = malloc(sizeof(*rindex) * nsteps);
|
||||
sindex = malloc(sizeof(*sindex) * nsteps);
|
||||
rcount = malloc(sizeof(*rcount) * nsteps);
|
||||
scount = malloc(sizeof(*scount) * nsteps);
|
||||
if (NULL == rindex || NULL == sindex || NULL == rcount || NULL == scount) {
|
||||
err = OMPI_ERR_OUT_OF_RESOURCE;
|
||||
goto cleanup_and_return;
|
||||
}
|
||||
|
||||
if (vrank != -1) {
|
||||
step = 0;
|
||||
wsize = count;
|
||||
sindex[0] = rindex[0] = 0;
|
||||
|
||||
for (int mask = 1; mask < nprocs_pof2; mask <<= 1) {
|
||||
/*
|
||||
* On each iteration: rindex[step] = sindex[step] -- begining of the
|
||||
* current window. Length of the current window is storded in wsize.
|
||||
*/
|
||||
int vdest = vrank ^ mask;
|
||||
/* Translate vdest virtual rank to real rank */
|
||||
int dest = (vdest < nprocs_rem) ? vdest * 2 : vdest + nprocs_rem;
|
||||
|
||||
if (rank < dest) {
|
||||
/*
|
||||
* Recv into the left half of the current window, send the right
|
||||
* half of the window to the peer (perform reduce on the left
|
||||
* half of the current window)
|
||||
*/
|
||||
rcount[step] = wsize / 2;
|
||||
scount[step] = wsize - rcount[step];
|
||||
sindex[step] = rindex[step] + rcount[step];
|
||||
} else {
|
||||
/*
|
||||
* Recv into the right half of the current window, send the left
|
||||
* half of the window to the peer (perform reduce on the right
|
||||
* half of the current window)
|
||||
*/
|
||||
scount[step] = wsize / 2;
|
||||
rcount[step] = wsize - scount[step];
|
||||
rindex[step] = sindex[step] + scount[step];
|
||||
}
|
||||
|
||||
/* Send part of data from the rbuf, recv into the tmp_buf */
|
||||
err = ompi_coll_base_sendrecv((char *)rbuf + (ptrdiff_t)sindex[step] * extent,
|
||||
scount[step], dtype, dest,
|
||||
MCA_COLL_BASE_TAG_ALLREDUCE,
|
||||
(char *)tmp_buf + (ptrdiff_t)rindex[step] * extent,
|
||||
rcount[step], dtype, dest,
|
||||
MCA_COLL_BASE_TAG_ALLREDUCE, comm,
|
||||
MPI_STATUS_IGNORE, rank);
|
||||
if (MPI_SUCCESS != err) { goto cleanup_and_return; }
|
||||
|
||||
/* Local reduce: rbuf[] = tmp_buf[] <op> rbuf[] */
|
||||
ompi_op_reduce(op, (char *)tmp_buf + (ptrdiff_t)rindex[step] * extent,
|
||||
(char *)rbuf + (ptrdiff_t)rindex[step] * extent,
|
||||
rcount[step], dtype);
|
||||
|
||||
/* Move the current window to the received message */
|
||||
if (step + 1 < nsteps) {
|
||||
rindex[step + 1] = rindex[step];
|
||||
sindex[step + 1] = rindex[step];
|
||||
wsize = rcount[step];
|
||||
step++;
|
||||
}
|
||||
}
|
||||
/*
|
||||
* Assertion: each process has 1 / p' of the total reduction result:
|
||||
* rcount[nsteps - 1] elements in the rbuf[rindex[nsteps - 1], ...].
|
||||
*/
|
||||
|
||||
/*
|
||||
* Step 3. Allgather by the recursive doubling algorithm.
|
||||
* Each process has 1 / p' of the total reduction result:
|
||||
* rcount[nsteps - 1] elements in the rbuf[rindex[nsteps - 1], ...].
|
||||
* All exchanges are executed in reverse order relative
|
||||
* to recursive doubling (previous step).
|
||||
*/
|
||||
|
||||
step = nsteps - 1;
|
||||
|
||||
for (int mask = nprocs_pof2 >> 1; mask > 0; mask >>= 1) {
|
||||
int vdest = vrank ^ mask;
|
||||
/* Translate vdest virtual rank to real rank */
|
||||
int dest = (vdest < nprocs_rem) ? vdest * 2 : vdest + nprocs_rem;
|
||||
|
||||
/*
|
||||
* Send rcount[step] elements from rbuf[rindex[step]...]
|
||||
* Recv scount[step] elements to rbuf[sindex[step]...]
|
||||
*/
|
||||
err = ompi_coll_base_sendrecv((char *)rbuf + (ptrdiff_t)rindex[step] * extent,
|
||||
rcount[step], dtype, dest,
|
||||
MCA_COLL_BASE_TAG_ALLREDUCE,
|
||||
(char *)rbuf + (ptrdiff_t)sindex[step] * extent,
|
||||
scount[step], dtype, dest,
|
||||
MCA_COLL_BASE_TAG_ALLREDUCE, comm,
|
||||
MPI_STATUS_IGNORE, rank);
|
||||
if (MPI_SUCCESS != err) { goto cleanup_and_return; }
|
||||
step--;
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Step 4. Send total result to excluded odd ranks.
|
||||
*/
|
||||
if (rank < 2 * nprocs_rem) {
|
||||
if (rank % 2 != 0) {
|
||||
/* Odd process -- recv result from rank - 1 */
|
||||
err = MCA_PML_CALL(recv(rbuf, count, dtype, rank - 1,
|
||||
MCA_COLL_BASE_TAG_ALLREDUCE, comm,
|
||||
MPI_STATUS_IGNORE));
|
||||
if (OMPI_SUCCESS != err) { goto cleanup_and_return; }
|
||||
|
||||
} else {
|
||||
/* Even process -- send result to rank + 1 */
|
||||
err = MCA_PML_CALL(send(rbuf, count, dtype, rank + 1,
|
||||
MCA_COLL_BASE_TAG_ALLREDUCE,
|
||||
MCA_PML_BASE_SEND_STANDARD, comm));
|
||||
if (MPI_SUCCESS != err) { goto cleanup_and_return; }
|
||||
}
|
||||
}
|
||||
|
||||
cleanup_and_return:
|
||||
if (NULL != tmp_buf_raw)
|
||||
free(tmp_buf_raw);
|
||||
if (NULL != rindex)
|
||||
free(rindex);
|
||||
if (NULL != sindex)
|
||||
free(sindex);
|
||||
if (NULL != rcount)
|
||||
free(rcount);
|
||||
if (NULL != scount)
|
||||
free(scount);
|
||||
return err;
|
||||
}
|
||||
|
||||
/* copied function (with appropriate renaming) ends here */
|
||||
|
||||
223
ompi/mca/coll/base/coll_base_exscan.c
Обычный файл
223
ompi/mca/coll/base/coll_base_exscan.c
Обычный файл
@ -0,0 +1,223 @@
|
||||
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
|
||||
/*
|
||||
* Copyright (c) 2018 Siberian State University of Telecommunications
|
||||
* and Information Science. All rights reserved.
|
||||
* Copyright (c) 2018 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include "ompi/constants.h"
|
||||
#include "ompi/datatype/ompi_datatype.h"
|
||||
#include "ompi/communicator/communicator.h"
|
||||
#include "ompi/mca/coll/coll.h"
|
||||
#include "ompi/mca/coll/base/coll_base_functions.h"
|
||||
#include "ompi/mca/coll/base/coll_tags.h"
|
||||
#include "ompi/mca/coll/base/coll_base_util.h"
|
||||
#include "ompi/mca/pml/pml.h"
|
||||
#include "ompi/op/op.h"
|
||||
|
||||
/*
|
||||
* ompi_coll_base_exscan_intra_linear
|
||||
*
|
||||
* Function: Linear algorithm for exclusive scan.
|
||||
* Accepts: Same as MPI_Exscan
|
||||
* Returns: MPI_SUCCESS or error code
|
||||
*/
|
||||
int
|
||||
ompi_coll_base_exscan_intra_linear(const void *sbuf, void *rbuf, int count,
|
||||
struct ompi_datatype_t *dtype,
|
||||
struct ompi_op_t *op,
|
||||
struct ompi_communicator_t *comm,
|
||||
mca_coll_base_module_t *module)
|
||||
{
|
||||
int size, rank, err;
|
||||
ptrdiff_t dsize, gap;
|
||||
char *free_buffer = NULL;
|
||||
char *reduce_buffer = NULL;
|
||||
|
||||
rank = ompi_comm_rank(comm);
|
||||
size = ompi_comm_size(comm);
|
||||
|
||||
/* For MPI_IN_PLACE, just adjust send buffer to point to
|
||||
* receive buffer. */
|
||||
if (MPI_IN_PLACE == sbuf) {
|
||||
sbuf = rbuf;
|
||||
}
|
||||
|
||||
/* If we're rank 0, then just send our sbuf to the next rank, and
|
||||
* we are done. */
|
||||
if (0 == rank) {
|
||||
return MCA_PML_CALL(send(sbuf, count, dtype, rank + 1,
|
||||
MCA_COLL_BASE_TAG_EXSCAN,
|
||||
MCA_PML_BASE_SEND_STANDARD, comm));
|
||||
}
|
||||
|
||||
/* If we're the last rank, then just receive the result from the
|
||||
* prior rank, and we are done. */
|
||||
else if ((size - 1) == rank) {
|
||||
return MCA_PML_CALL(recv(rbuf, count, dtype, rank - 1,
|
||||
MCA_COLL_BASE_TAG_EXSCAN, comm,
|
||||
MPI_STATUS_IGNORE));
|
||||
}
|
||||
|
||||
/* Otherwise, get the result from the prior rank, combine it with my
|
||||
* data, and send it to the next rank */
|
||||
|
||||
/* Get a temporary buffer to perform the reduction into. Rationale
|
||||
* for malloc'ing this size is provided in coll_basic_reduce.c. */
|
||||
dsize = opal_datatype_span(&dtype->super, count, &gap);
|
||||
|
||||
free_buffer = (char*)malloc(dsize);
|
||||
if (NULL == free_buffer) {
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
reduce_buffer = free_buffer - gap;
|
||||
err = ompi_datatype_copy_content_same_ddt(dtype, count,
|
||||
reduce_buffer, (char*)sbuf);
|
||||
|
||||
/* Receive the reduced value from the prior rank */
|
||||
err = MCA_PML_CALL(recv(rbuf, count, dtype, rank - 1,
|
||||
MCA_COLL_BASE_TAG_EXSCAN, comm, MPI_STATUS_IGNORE));
|
||||
if (MPI_SUCCESS != err) {
|
||||
goto error;
|
||||
}
|
||||
|
||||
/* Now reduce the prior rank's result with my source buffer. The source
|
||||
* buffer had been previously copied into the temporary reduce_buffer. */
|
||||
ompi_op_reduce(op, rbuf, reduce_buffer, count, dtype);
|
||||
|
||||
/* Send my result off to the next rank */
|
||||
err = MCA_PML_CALL(send(reduce_buffer, count, dtype, rank + 1,
|
||||
MCA_COLL_BASE_TAG_EXSCAN,
|
||||
MCA_PML_BASE_SEND_STANDARD, comm));
|
||||
/* Error */
|
||||
error:
|
||||
free(free_buffer);
|
||||
|
||||
/* All done */
|
||||
return err;
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* ompi_coll_base_exscan_intra_recursivedoubling
|
||||
*
|
||||
* Function: Recursive doubling algorithm for exclusive scan.
|
||||
* Accepts: Same as MPI_Exscan
|
||||
* Returns: MPI_SUCCESS or error code
|
||||
*
|
||||
* Description: Implements recursive doubling algorithm for MPI_Exscan.
|
||||
* The algorithm preserves order of operations so it can
|
||||
* be used both by commutative and non-commutative operations.
|
||||
*
|
||||
* Example for 5 processes and commutative operation MPI_SUM:
|
||||
* Process: 0 1 2 3 4
|
||||
* recvbuf: - - - - -
|
||||
* psend: [0] [1] [2] [3] [4]
|
||||
*
|
||||
* Step 1:
|
||||
* recvbuf: - [0] - [2] -
|
||||
* psend: [1+0] [0+1] [3+2] [2+3] [4]
|
||||
*
|
||||
* Step 2:
|
||||
* recvbuf: - [0] [1+0] [(0+1)+2] -
|
||||
* psend: [(3+2)+(1+0)] [(2+3)+(0+1)] [(1+0)+(3+2)] [(1+0)+(2+3)] [4]
|
||||
*
|
||||
* Step 3:
|
||||
* recvbuf: - [0] [1+0] [(0+1)+2] [(3+2)+(1+0)]
|
||||
* psend: [4+((3+2)+(1+0))] [((3+2)+(1+0))+4]
|
||||
*
|
||||
* Time complexity (worst case): \ceil(\log_2(p))(2\alpha + 2m\beta + 2m\gamma)
|
||||
* Memory requirements (per process): 2 * count * typesize = O(count)
|
||||
* Limitations: intra-communicators only
|
||||
*/
|
||||
int ompi_coll_base_exscan_intra_recursivedoubling(
|
||||
const void *sendbuf, void *recvbuf, int count, struct ompi_datatype_t *datatype,
|
||||
struct ompi_op_t *op, struct ompi_communicator_t *comm,
|
||||
mca_coll_base_module_t *module)
|
||||
{
|
||||
int err = MPI_SUCCESS;
|
||||
char *tmpsend_raw = NULL, *tmprecv_raw = NULL;
|
||||
int comm_size = ompi_comm_size(comm);
|
||||
int rank = ompi_comm_rank(comm);
|
||||
|
||||
OPAL_OUTPUT((ompi_coll_base_framework.framework_output, "coll:base:exscan_intra_recursivedoubling: rank %d/%d",
|
||||
rank, comm_size));
|
||||
if (count == 0)
|
||||
return MPI_SUCCESS;
|
||||
if (comm_size < 2)
|
||||
return MPI_SUCCESS;
|
||||
|
||||
ptrdiff_t dsize, gap;
|
||||
dsize = opal_datatype_span(&datatype->super, count, &gap);
|
||||
tmpsend_raw = malloc(dsize);
|
||||
tmprecv_raw = malloc(dsize);
|
||||
if (NULL == tmpsend_raw || NULL == tmprecv_raw) {
|
||||
err = OMPI_ERR_OUT_OF_RESOURCE;
|
||||
goto cleanup_and_return;
|
||||
}
|
||||
char *psend = tmpsend_raw - gap;
|
||||
char *precv = tmprecv_raw - gap;
|
||||
if (sendbuf != MPI_IN_PLACE) {
|
||||
err = ompi_datatype_copy_content_same_ddt(datatype, count, psend, (char *)sendbuf);
|
||||
if (MPI_SUCCESS != err) { goto cleanup_and_return; }
|
||||
} else {
|
||||
err = ompi_datatype_copy_content_same_ddt(datatype, count, psend, recvbuf);
|
||||
if (MPI_SUCCESS != err) { goto cleanup_and_return; }
|
||||
}
|
||||
int is_commute = ompi_op_is_commute(op);
|
||||
int is_first_block = 1;
|
||||
|
||||
for (int mask = 1; mask < comm_size; mask <<= 1) {
|
||||
int remote = rank ^ mask;
|
||||
if (remote < comm_size) {
|
||||
err = ompi_coll_base_sendrecv(psend, count, datatype, remote,
|
||||
MCA_COLL_BASE_TAG_EXSCAN,
|
||||
precv, count, datatype, remote,
|
||||
MCA_COLL_BASE_TAG_EXSCAN, comm,
|
||||
MPI_STATUS_IGNORE, rank);
|
||||
if (MPI_SUCCESS != err) { goto cleanup_and_return; }
|
||||
|
||||
if (rank > remote) {
|
||||
/* Assertion: rank > 0 and rbuf is valid */
|
||||
if (is_first_block) {
|
||||
err = ompi_datatype_copy_content_same_ddt(datatype, count,
|
||||
recvbuf, precv);
|
||||
if (MPI_SUCCESS != err) { goto cleanup_and_return; }
|
||||
is_first_block = 0;
|
||||
} else {
|
||||
/* Accumulate prefix reduction: recvbuf = precv <op> recvbuf */
|
||||
ompi_op_reduce(op, precv, recvbuf, count, datatype);
|
||||
}
|
||||
/* Partial result: psend = precv <op> psend */
|
||||
ompi_op_reduce(op, precv, psend, count, datatype);
|
||||
} else {
|
||||
if (is_commute) {
|
||||
/* psend = precv <op> psend */
|
||||
ompi_op_reduce(op, precv, psend, count, datatype);
|
||||
} else {
|
||||
/* precv = psend <op> precv */
|
||||
ompi_op_reduce(op, psend, precv, count, datatype);
|
||||
char *tmp = psend;
|
||||
psend = precv;
|
||||
precv = tmp;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
cleanup_and_return:
|
||||
if (NULL != tmpsend_raw)
|
||||
free(tmpsend_raw);
|
||||
if (NULL != tmprecv_raw)
|
||||
free(tmprecv_raw);
|
||||
return err;
|
||||
}
|
||||
@ -14,7 +14,7 @@
|
||||
* Copyright (c) 2008 Cisco Systems, Inc. All rights reserved.
|
||||
* Copyright (c) 2013-2016 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2015 Research Organization for Information Science
|
||||
* Copyright (c) 2015-2018 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
* Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
|
||||
* Copyright (c) 2017 FUJITSU LIMITED. All rights reserved.
|
||||
@ -182,6 +182,7 @@ int ompi_coll_base_allreduce_intra_recursivedoubling(ALLREDUCE_ARGS);
|
||||
int ompi_coll_base_allreduce_intra_ring(ALLREDUCE_ARGS);
|
||||
int ompi_coll_base_allreduce_intra_ring_segmented(ALLREDUCE_ARGS, uint32_t segsize);
|
||||
int ompi_coll_base_allreduce_intra_basic_linear(ALLREDUCE_ARGS);
|
||||
int ompi_coll_base_allreduce_intra_redscat_allgather(ALLREDUCE_ARGS);
|
||||
|
||||
/* AlltoAll */
|
||||
int ompi_coll_base_alltoall_intra_pairwise(ALLTOALL_ARGS);
|
||||
@ -222,6 +223,9 @@ int ompi_coll_base_bcast_intra_bintree(BCAST_ARGS, uint32_t segsize);
|
||||
int ompi_coll_base_bcast_intra_split_bintree(BCAST_ARGS, uint32_t segsize);
|
||||
|
||||
/* Exscan */
|
||||
int ompi_coll_base_exscan_intra_recursivedoubling(EXSCAN_ARGS);
|
||||
int ompi_coll_base_exscan_intra_linear(EXSCAN_ARGS);
|
||||
int ompi_coll_base_exscan_intra_recursivedoubling(EXSCAN_ARGS);
|
||||
|
||||
/* Gather */
|
||||
int ompi_coll_base_gather_intra_basic_linear(GATHER_ARGS);
|
||||
@ -238,6 +242,7 @@ int ompi_coll_base_reduce_intra_pipeline(REDUCE_ARGS, uint32_t segsize, int max_
|
||||
int ompi_coll_base_reduce_intra_binary(REDUCE_ARGS, uint32_t segsize, int max_outstanding_reqs );
|
||||
int ompi_coll_base_reduce_intra_binomial(REDUCE_ARGS, uint32_t segsize, int max_outstanding_reqs );
|
||||
int ompi_coll_base_reduce_intra_in_order_binary(REDUCE_ARGS, uint32_t segsize, int max_outstanding_reqs );
|
||||
int ompi_coll_base_reduce_intra_redscat_gather(REDUCE_ARGS);
|
||||
|
||||
/* Reduce_scatter */
|
||||
int ompi_coll_base_reduce_scatter_intra_nonoverlapping(REDUCESCATTER_ARGS);
|
||||
@ -248,6 +253,9 @@ int ompi_coll_base_reduce_scatter_intra_ring(REDUCESCATTER_ARGS);
|
||||
int ompi_coll_base_reduce_scatter_block_intra_recursivedoubling(REDUCESCATTERBLOCK_ARGS);
|
||||
|
||||
/* Scan */
|
||||
int ompi_coll_base_scan_intra_recursivedoubling(SCAN_ARGS);
|
||||
int ompi_coll_base_scan_intra_linear(SCAN_ARGS);
|
||||
int ompi_coll_base_scan_intra_recursivedoubling(SCAN_ARGS);
|
||||
|
||||
/* Scatter */
|
||||
int ompi_coll_base_scatter_intra_basic_linear(SCATTER_ARGS);
|
||||
|
||||
@ -15,6 +15,8 @@
|
||||
* Copyright (c) 2015-2016 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
* Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
|
||||
* Copyright (c) 2018 Siberian State University of Telecommunications
|
||||
* and Information Science. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
@ -25,6 +27,7 @@
|
||||
#include "ompi_config.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include "opal/util/bit_ops.h"
|
||||
#include "ompi/constants.h"
|
||||
#include "ompi/datatype/ompi_datatype.h"
|
||||
#include "ompi/communicator/communicator.h"
|
||||
@ -34,6 +37,7 @@
|
||||
#include "ompi/op/op.h"
|
||||
#include "ompi/mca/coll/base/coll_base_functions.h"
|
||||
#include "coll_base_topo.h"
|
||||
#include "coll_base_util.h"
|
||||
|
||||
int mca_coll_base_reduce_local(const void *inbuf, void *inoutbuf, int count,
|
||||
struct ompi_datatype_t * dtype, struct ompi_op_t * op,
|
||||
@ -706,3 +710,395 @@ ompi_coll_base_reduce_intra_basic_linear(const void *sbuf, void *rbuf, int count
|
||||
return MPI_SUCCESS;
|
||||
}
|
||||
|
||||
/*
|
||||
* ompi_coll_base_reduce_intra_redscat_gather
|
||||
*
|
||||
* Function: Reduce using Rabenseifner's algorithm.
|
||||
* Accepts: Same arguments as MPI_Reduce
|
||||
* Returns: MPI_SUCCESS or error code
|
||||
*
|
||||
* Description: an implementation of Rabenseifner's reduce algorithm [1, 2].
|
||||
* [1] Rajeev Thakur, Rolf Rabenseifner and William Gropp.
|
||||
* Optimization of Collective Communication Operations in MPICH //
|
||||
* The Int. Journal of High Performance Computing Applications. Vol 19,
|
||||
* Issue 1, pp. 49--66.
|
||||
* [2] http://www.hlrs.de/mpi/myreduce.html.
|
||||
*
|
||||
* This algorithm is a combination of a reduce-scatter implemented with
|
||||
* recursive vector halving and recursive distance doubling, followed either
|
||||
* by a binomial tree gather [1].
|
||||
*
|
||||
* Step 1. If the number of processes is not a power of two, reduce it to
|
||||
* the nearest lower power of two (p' = 2^{\floor{\log_2 p}})
|
||||
* by removing r = p - p' extra processes as follows. In the first 2r processes
|
||||
* (ranks 0 to 2r - 1), all the even ranks send the second half of the input
|
||||
* vector to their right neighbor (rank + 1), and all the odd ranks send
|
||||
* the first half of the input vector to their left neighbor (rank - 1).
|
||||
* The even ranks compute the reduction on the first half of the vector and
|
||||
* the odd ranks compute the reduction on the second half. The odd ranks then
|
||||
* send the result to their left neighbors (the even ranks). As a result,
|
||||
* the even ranks among the first 2r processes now contain the reduction with
|
||||
* the input vector on their right neighbors (the odd ranks). These odd ranks
|
||||
* do not participate in the rest of the algorithm, which leaves behind
|
||||
* a power-of-two number of processes. The first r even-ranked processes and
|
||||
* the last p - 2r processes are now renumbered from 0 to p' - 1.
|
||||
*
|
||||
* Step 2. The remaining processes now perform a reduce-scatter by using
|
||||
* recursive vector halving and recursive distance doubling. The even-ranked
|
||||
* processes send the second half of their buffer to rank + 1 and the odd-ranked
|
||||
* processes send the first half of their buffer to rank - 1. All processes
|
||||
* then compute the reduction between the local buffer and the received buffer.
|
||||
* In the next log_2(p') - 1 steps, the buffers are recursively halved, and the
|
||||
* distance is doubled. At the end, each of the p' processes has 1 / p' of the
|
||||
* total reduction result.
|
||||
*
|
||||
* Step 3. A binomial tree gather is performed by using recursive vector
|
||||
* doubling and distance halving. In the non-power-of-two case, if the root
|
||||
* happens to be one of those odd-ranked processes that would normally
|
||||
* be removed in the first step, then the role of this process and process 0
|
||||
* are interchanged.
|
||||
*
|
||||
* Limitations:
|
||||
* count >= 2^{\floor{\log_2 p}}
|
||||
* commutative operations only
|
||||
* intra-communicators only
|
||||
*
|
||||
* Memory requirements (per process):
|
||||
* rank != root: 2 * count * typesize + 4 * \log_2(p) * sizeof(int) = O(count)
|
||||
* rank == root: count * typesize + 4 * \log_2(p) * sizeof(int) = O(count)
|
||||
*
|
||||
* Recommendations: root = 0, otherwise it is required additional steps
|
||||
* in the root process.
|
||||
*/
|
||||
int ompi_coll_base_reduce_intra_redscat_gather(
|
||||
const void *sbuf, void *rbuf, int count, struct ompi_datatype_t *dtype,
|
||||
struct ompi_op_t *op, int root, struct ompi_communicator_t *comm,
|
||||
mca_coll_base_module_t *module)
|
||||
{
|
||||
int comm_size = ompi_comm_size(comm);
|
||||
int rank = ompi_comm_rank(comm);
|
||||
|
||||
OPAL_OUTPUT((ompi_coll_base_framework.framework_output,
|
||||
"coll:base:reduce_intra_redscat_gather: rank %d/%d, root %d",
|
||||
rank, comm_size, root));
|
||||
|
||||
/* Find nearest power-of-two less than or equal to comm_size */
|
||||
int nsteps = opal_hibit(comm_size, comm->c_cube_dim + 1); /* ilog2(comm_size) */
|
||||
assert(nsteps >= 0);
|
||||
int nprocs_pof2 = 1 << nsteps; /* flp2(comm_size) */
|
||||
|
||||
if (count < nprocs_pof2 || !ompi_op_is_commute(op)) {
|
||||
OPAL_OUTPUT((ompi_coll_base_framework.framework_output,
|
||||
"coll:base:reduce_intra_redscat_gather: rank %d/%d count %d "
|
||||
"switching to basic linear reduce", rank, comm_size, count));
|
||||
return ompi_coll_base_reduce_intra_basic_linear(sbuf, rbuf, count, dtype,
|
||||
op, root, comm, module);
|
||||
}
|
||||
|
||||
int err = MPI_SUCCESS;
|
||||
int *rindex = NULL, *rcount = NULL, *sindex = NULL, *scount = NULL;
|
||||
ptrdiff_t lb, extent, dsize, gap;
|
||||
ompi_datatype_get_extent(dtype, &lb, &extent);
|
||||
dsize = opal_datatype_span(&dtype->super, count, &gap);
|
||||
|
||||
/* Temporary buffers */
|
||||
char *tmp_buf_raw = NULL, *rbuf_raw = NULL;
|
||||
tmp_buf_raw = malloc(dsize);
|
||||
if (NULL == tmp_buf_raw) {
|
||||
err = OMPI_ERR_OUT_OF_RESOURCE;
|
||||
goto cleanup_and_return;
|
||||
}
|
||||
char *tmp_buf = tmp_buf_raw - gap;
|
||||
|
||||
if (rank != root) {
|
||||
rbuf_raw = malloc(dsize);
|
||||
if (NULL == rbuf_raw) {
|
||||
err = OMPI_ERR_OUT_OF_RESOURCE;
|
||||
goto cleanup_and_return;
|
||||
}
|
||||
rbuf = rbuf_raw - gap;
|
||||
}
|
||||
|
||||
if ((rank != root) || (sbuf != MPI_IN_PLACE)) {
|
||||
err = ompi_datatype_copy_content_same_ddt(dtype, count, rbuf,
|
||||
(char *)sbuf);
|
||||
if (MPI_SUCCESS != err) { goto cleanup_and_return; }
|
||||
}
|
||||
|
||||
/*
|
||||
* Step 1. Reduce the number of processes to the nearest lower power of two
|
||||
* p' = 2^{\floor{\log_2 p}} by removing r = p - p' processes.
|
||||
* 1. In the first 2r processes (ranks 0 to 2r - 1), all the even ranks send
|
||||
* the second half of the input vector to their right neighbor (rank + 1)
|
||||
* and all the odd ranks send the first half of the input vector to their
|
||||
* left neighbor (rank - 1).
|
||||
* 2. All 2r processes compute the reduction on their half.
|
||||
* 3. The odd ranks then send the result to their left neighbors
|
||||
* (the even ranks).
|
||||
*
|
||||
* The even ranks (0 to 2r - 1) now contain the reduction with the input
|
||||
* vector on their right neighbors (the odd ranks). The first r even
|
||||
* processes and the p - 2r last processes are renumbered from
|
||||
* 0 to 2^{\floor{\log_2 p}} - 1. These odd ranks do not participate in the
|
||||
* rest of the algorithm.
|
||||
*/
|
||||
|
||||
int vrank, step, wsize;
|
||||
int nprocs_rem = comm_size - nprocs_pof2;
|
||||
|
||||
if (rank < 2 * nprocs_rem) {
|
||||
int count_lhalf = count / 2;
|
||||
int count_rhalf = count - count_lhalf;
|
||||
|
||||
if (rank % 2 != 0) {
|
||||
/*
|
||||
* Odd process -- exchange with rank - 1
|
||||
* Send the left half of the input vector to the left neighbor,
|
||||
* Recv the right half of the input vector from the left neighbor
|
||||
*/
|
||||
err = ompi_coll_base_sendrecv(rbuf, count_lhalf, dtype, rank - 1,
|
||||
MCA_COLL_BASE_TAG_REDUCE,
|
||||
(char *)tmp_buf + (ptrdiff_t)count_lhalf * extent,
|
||||
count_rhalf, dtype, rank - 1,
|
||||
MCA_COLL_BASE_TAG_REDUCE, comm,
|
||||
MPI_STATUS_IGNORE, rank);
|
||||
if (MPI_SUCCESS != err) { goto cleanup_and_return; }
|
||||
|
||||
/* Reduce on the right half of the buffers (result in rbuf) */
|
||||
ompi_op_reduce(op, (char *)tmp_buf + (ptrdiff_t)count_lhalf * extent,
|
||||
(char *)rbuf + count_lhalf * extent, count_rhalf, dtype);
|
||||
|
||||
/* Send the right half to the left neighbor */
|
||||
err = MCA_PML_CALL(send((char *)rbuf + (ptrdiff_t)count_lhalf * extent,
|
||||
count_rhalf, dtype, rank - 1,
|
||||
MCA_COLL_BASE_TAG_REDUCE,
|
||||
MCA_PML_BASE_SEND_STANDARD, comm));
|
||||
if (MPI_SUCCESS != err) { goto cleanup_and_return; }
|
||||
|
||||
/* This process does not pariticipate in recursive doubling phase */
|
||||
vrank = -1;
|
||||
|
||||
} else {
|
||||
/*
|
||||
* Even process -- exchange with rank + 1
|
||||
* Send the right half of the input vector to the right neighbor,
|
||||
* Recv the left half of the input vector from the right neighbor
|
||||
*/
|
||||
err = ompi_coll_base_sendrecv((char *)rbuf + (ptrdiff_t)count_lhalf * extent,
|
||||
count_rhalf, dtype, rank + 1,
|
||||
MCA_COLL_BASE_TAG_REDUCE,
|
||||
tmp_buf, count_lhalf, dtype, rank + 1,
|
||||
MCA_COLL_BASE_TAG_REDUCE, comm,
|
||||
MPI_STATUS_IGNORE, rank);
|
||||
if (MPI_SUCCESS != err) { goto cleanup_and_return; }
|
||||
|
||||
/* Reduce on the right half of the buffers (result in rbuf) */
|
||||
ompi_op_reduce(op, tmp_buf, rbuf, count_lhalf, dtype);
|
||||
|
||||
/* Recv the right half from the right neighbor */
|
||||
err = MCA_PML_CALL(recv((char *)rbuf + (ptrdiff_t)count_lhalf * extent,
|
||||
count_rhalf, dtype, rank + 1,
|
||||
MCA_COLL_BASE_TAG_REDUCE, comm,
|
||||
MPI_STATUS_IGNORE));
|
||||
if (MPI_SUCCESS != err) { goto cleanup_and_return; }
|
||||
|
||||
vrank = rank / 2;
|
||||
}
|
||||
} else { /* rank >= 2 * nprocs_rem */
|
||||
vrank = rank - nprocs_rem;
|
||||
}
|
||||
|
||||
/*
|
||||
* Step 2. Reduce-scatter implemented with recursive vector halving and
|
||||
* recursive distance doubling. We have p' = 2^{\floor{\log_2 p}}
|
||||
* power-of-two number of processes with new ranks (vrank) and result in rbuf.
|
||||
*
|
||||
* The even-ranked processes send the right half of their buffer to rank + 1
|
||||
* and the odd-ranked processes send the left half of their buffer to
|
||||
* rank - 1. All processes then compute the reduction between the local
|
||||
* buffer and the received buffer. In the next \log_2(p') - 1 steps, the
|
||||
* buffers are recursively halved, and the distance is doubled. At the end,
|
||||
* each of the p' processes has 1 / p' of the total reduction result.
|
||||
*/
|
||||
|
||||
rindex = malloc(sizeof(*rindex) * nsteps); /* O(\log_2(p)) */
|
||||
sindex = malloc(sizeof(*sindex) * nsteps);
|
||||
rcount = malloc(sizeof(*rcount) * nsteps);
|
||||
scount = malloc(sizeof(*scount) * nsteps);
|
||||
if (NULL == rindex || NULL == sindex || NULL == rcount || NULL == scount) {
|
||||
err = OMPI_ERR_OUT_OF_RESOURCE;
|
||||
goto cleanup_and_return;
|
||||
}
|
||||
|
||||
if (vrank != -1) {
|
||||
step = 0;
|
||||
wsize = count;
|
||||
sindex[0] = rindex[0] = 0;
|
||||
|
||||
for (int mask = 1; mask < nprocs_pof2; mask <<= 1) {
|
||||
/*
|
||||
* On each iteration: rindex[step] = sindex[step] -- begining of the
|
||||
* current window. Length of the current window is storded in wsize.
|
||||
*/
|
||||
int vdest = vrank ^ mask;
|
||||
/* Translate vdest virtual rank to real rank */
|
||||
int dest = (vdest < nprocs_rem) ? vdest * 2 : vdest + nprocs_rem;
|
||||
|
||||
if (rank < dest) {
|
||||
/*
|
||||
* Recv into the left half of the current window, send the right
|
||||
* half of the window to the peer (perform reduce on the left
|
||||
* half of the current window)
|
||||
*/
|
||||
rcount[step] = wsize / 2;
|
||||
scount[step] = wsize - rcount[step];
|
||||
sindex[step] = rindex[step] + rcount[step];
|
||||
} else {
|
||||
/*
|
||||
* Recv into the right half of the current window, send the left
|
||||
* half of the window to the peer (perform reduce on the right
|
||||
* half of the current window)
|
||||
*/
|
||||
scount[step] = wsize / 2;
|
||||
rcount[step] = wsize - scount[step];
|
||||
rindex[step] = sindex[step] + scount[step];
|
||||
}
|
||||
|
||||
/* Send part of data from the rbuf, recv into the tmp_buf */
|
||||
err = ompi_coll_base_sendrecv((char *)rbuf + (ptrdiff_t)sindex[step] * extent,
|
||||
scount[step], dtype, dest,
|
||||
MCA_COLL_BASE_TAG_REDUCE,
|
||||
(char *)tmp_buf + (ptrdiff_t)rindex[step] * extent,
|
||||
rcount[step], dtype, dest,
|
||||
MCA_COLL_BASE_TAG_REDUCE, comm,
|
||||
MPI_STATUS_IGNORE, rank);
|
||||
if (MPI_SUCCESS != err) { goto cleanup_and_return; }
|
||||
|
||||
/* Local reduce: rbuf[] = tmp_buf[] <op> rbuf[] */
|
||||
ompi_op_reduce(op, (char *)tmp_buf + (ptrdiff_t)rindex[step] * extent,
|
||||
(char *)rbuf + (ptrdiff_t)rindex[step] * extent,
|
||||
rcount[step], dtype);
|
||||
|
||||
/* Move the current window to the received message */
|
||||
if (step + 1 < nsteps) {
|
||||
rindex[step + 1] = rindex[step];
|
||||
sindex[step + 1] = rindex[step];
|
||||
wsize = rcount[step];
|
||||
step++;
|
||||
}
|
||||
}
|
||||
}
|
||||
/*
|
||||
* Assertion: each process has 1 / p' of the total reduction result:
|
||||
* rcount[nsteps - 1] elements in the rbuf[rindex[nsteps - 1], ...].
|
||||
*/
|
||||
|
||||
/*
|
||||
* Setup the root process for gather operation.
|
||||
* Case 1: root < 2r and root is odd -- root process was excluded on step 1
|
||||
* Recv data from process 0, vroot = 0, vrank = 0
|
||||
* Case 2: root < 2r and root is even: vroot = root / 2
|
||||
* Case 3: root >= 2r: vroot = root - r
|
||||
*/
|
||||
int vroot = 0;
|
||||
if (root < 2 * nprocs_rem) {
|
||||
if (root % 2 != 0) {
|
||||
vroot = 0;
|
||||
if (rank == root) {
|
||||
/*
|
||||
* Case 1: root < 2r and root is odd -- root process was
|
||||
* excluded on step 1 (newrank == -1).
|
||||
* Recv a data from the process 0.
|
||||
*/
|
||||
rindex[0] = 0;
|
||||
step = 0, wsize = count;
|
||||
for (int mask = 1; mask < nprocs_pof2; mask *= 2) {
|
||||
rcount[step] = wsize / 2;
|
||||
scount[step] = wsize - rcount[step];
|
||||
rindex[step] = 0;
|
||||
sindex[step] = rcount[step];
|
||||
step++;
|
||||
wsize /= 2;
|
||||
}
|
||||
|
||||
err = MCA_PML_CALL(recv(rbuf, rcount[nsteps - 1], dtype, 0,
|
||||
MCA_COLL_BASE_TAG_REDUCE, comm,
|
||||
MPI_STATUS_IGNORE));
|
||||
if (MPI_SUCCESS != err) { goto cleanup_and_return; }
|
||||
vrank = 0;
|
||||
|
||||
} else if (vrank == 0) {
|
||||
/* Send a data to the root */
|
||||
err = MCA_PML_CALL(send(rbuf, rcount[nsteps - 1], dtype, root,
|
||||
MCA_COLL_BASE_TAG_REDUCE,
|
||||
MCA_PML_BASE_SEND_STANDARD, comm));
|
||||
if (MPI_SUCCESS != err) { goto cleanup_and_return; }
|
||||
vrank = -1;
|
||||
}
|
||||
} else {
|
||||
/* Case 2: root < 2r and a root is even: vroot = root / 2 */
|
||||
vroot = root / 2;
|
||||
}
|
||||
} else {
|
||||
/* Case 3: root >= 2r: newroot = root - r */
|
||||
vroot = root - nprocs_rem;
|
||||
}
|
||||
|
||||
/*
|
||||
* Step 3. Gather result at the vroot by the binomial tree algorithm.
|
||||
* Each process has 1 / p' of the total reduction result:
|
||||
* rcount[nsteps - 1] elements in the rbuf[rindex[nsteps - 1], ...].
|
||||
* All exchanges are executed in reverse order relative
|
||||
* to recursive doubling (previous step).
|
||||
*/
|
||||
|
||||
if (vrank != -1) {
|
||||
int vdest_tree, vroot_tree;
|
||||
step = nsteps - 1; /* step = ilog2(p') - 1 */
|
||||
|
||||
for (int mask = nprocs_pof2 >> 1; mask > 0; mask >>= 1) {
|
||||
int vdest = vrank ^ mask;
|
||||
/* Translate vdest virtual rank to real rank */
|
||||
int dest = (vdest < nprocs_rem) ? vdest * 2 : vdest + nprocs_rem;
|
||||
if ((vdest == 0) && (root < 2 * nprocs_rem) && (root % 2 != 0))
|
||||
dest = root;
|
||||
|
||||
vdest_tree = vdest >> step;
|
||||
vdest_tree <<= step;
|
||||
vroot_tree = vroot >> step;
|
||||
vroot_tree <<= step;
|
||||
if (vdest_tree == vroot_tree) {
|
||||
/* Send data from rbuf and exit */
|
||||
err = MCA_PML_CALL(send((char *)rbuf + (ptrdiff_t)rindex[step] * extent,
|
||||
rcount[step], dtype, dest,
|
||||
MCA_COLL_BASE_TAG_REDUCE,
|
||||
MCA_PML_BASE_SEND_STANDARD, comm));
|
||||
if (MPI_SUCCESS != err) { goto cleanup_and_return; }
|
||||
break;
|
||||
} else {
|
||||
/* Recv and continue */
|
||||
err = MCA_PML_CALL(recv((char *)rbuf + (ptrdiff_t)sindex[step] * extent,
|
||||
scount[step], dtype, dest,
|
||||
MCA_COLL_BASE_TAG_REDUCE, comm,
|
||||
MPI_STATUS_IGNORE));
|
||||
if (MPI_SUCCESS != err) { goto cleanup_and_return; }
|
||||
}
|
||||
step--;
|
||||
}
|
||||
}
|
||||
|
||||
cleanup_and_return:
|
||||
if (NULL != tmp_buf_raw)
|
||||
free(tmp_buf_raw);
|
||||
if (NULL != rbuf_raw)
|
||||
free(rbuf_raw);
|
||||
if (NULL != rindex)
|
||||
free(rindex);
|
||||
if (NULL != sindex)
|
||||
free(sindex);
|
||||
if (NULL != rcount)
|
||||
free(rcount);
|
||||
if (NULL != scount)
|
||||
free(scount);
|
||||
return err;
|
||||
}
|
||||
|
||||
230
ompi/mca/coll/base/coll_base_scan.c
Обычный файл
230
ompi/mca/coll/base/coll_base_scan.c
Обычный файл
@ -0,0 +1,230 @@
|
||||
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
|
||||
/*
|
||||
* Copyright (c) 2018 Siberian State University of Telecommunications
|
||||
* and Information Science. All rights reserved.
|
||||
* Copyright (c) 2018 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include "ompi/constants.h"
|
||||
#include "ompi/datatype/ompi_datatype.h"
|
||||
#include "ompi/communicator/communicator.h"
|
||||
#include "ompi/mca/coll/coll.h"
|
||||
#include "ompi/mca/coll/base/coll_base_functions.h"
|
||||
#include "ompi/mca/coll/base/coll_tags.h"
|
||||
#include "ompi/mca/coll/base/coll_base_util.h"
|
||||
#include "ompi/mca/pml/pml.h"
|
||||
#include "ompi/op/op.h"
|
||||
|
||||
/*
|
||||
* ompi_coll_base_scan_intra_linear
|
||||
*
|
||||
* Function: Linear algorithm for inclusive scan.
|
||||
* Accepts: Same as MPI_Scan
|
||||
* Returns: MPI_SUCCESS or error code
|
||||
*/
|
||||
int
|
||||
ompi_coll_base_scan_intra_linear(const void *sbuf, void *rbuf, int count,
|
||||
struct ompi_datatype_t *dtype,
|
||||
struct ompi_op_t *op,
|
||||
struct ompi_communicator_t *comm,
|
||||
mca_coll_base_module_t *module)
|
||||
{
|
||||
int size, rank, err;
|
||||
ptrdiff_t dsize, gap;
|
||||
char *free_buffer = NULL;
|
||||
char *pml_buffer = NULL;
|
||||
|
||||
/* Initialize */
|
||||
|
||||
rank = ompi_comm_rank(comm);
|
||||
size = ompi_comm_size(comm);
|
||||
|
||||
/* If I'm rank 0, just copy into the receive buffer */
|
||||
|
||||
if (0 == rank) {
|
||||
if (MPI_IN_PLACE != sbuf) {
|
||||
err = ompi_datatype_copy_content_same_ddt(dtype, count, (char*)rbuf, (char*)sbuf);
|
||||
if (MPI_SUCCESS != err) {
|
||||
return err;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Otherwise receive previous buffer and reduce. */
|
||||
|
||||
else {
|
||||
/* Allocate a temporary buffer. Rationale for this size is
|
||||
* listed in coll_basic_reduce.c. Use this temporary buffer to
|
||||
* receive into, later. */
|
||||
|
||||
dsize = opal_datatype_span(&dtype->super, count, &gap);
|
||||
free_buffer = malloc(dsize);
|
||||
if (NULL == free_buffer) {
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
pml_buffer = free_buffer - gap;
|
||||
|
||||
/* Copy the send buffer into the receive buffer. */
|
||||
|
||||
if (MPI_IN_PLACE != sbuf) {
|
||||
err = ompi_datatype_copy_content_same_ddt(dtype, count, (char*)rbuf, (char*)sbuf);
|
||||
if (MPI_SUCCESS != err) {
|
||||
if (NULL != free_buffer) {
|
||||
free(free_buffer);
|
||||
}
|
||||
return err;
|
||||
}
|
||||
}
|
||||
|
||||
/* Receive the prior answer */
|
||||
|
||||
err = MCA_PML_CALL(recv(pml_buffer, count, dtype,
|
||||
rank - 1, MCA_COLL_BASE_TAG_SCAN, comm,
|
||||
MPI_STATUS_IGNORE));
|
||||
if (MPI_SUCCESS != err) {
|
||||
if (NULL != free_buffer) {
|
||||
free(free_buffer);
|
||||
}
|
||||
return err;
|
||||
}
|
||||
|
||||
/* Perform the operation */
|
||||
|
||||
ompi_op_reduce(op, pml_buffer, rbuf, count, dtype);
|
||||
|
||||
/* All done */
|
||||
|
||||
if (NULL != free_buffer) {
|
||||
free(free_buffer);
|
||||
}
|
||||
}
|
||||
|
||||
/* Send result to next process. */
|
||||
|
||||
if (rank < (size - 1)) {
|
||||
return MCA_PML_CALL(send(rbuf, count, dtype, rank + 1,
|
||||
MCA_COLL_BASE_TAG_SCAN,
|
||||
MCA_PML_BASE_SEND_STANDARD, comm));
|
||||
}
|
||||
|
||||
/* All done */
|
||||
|
||||
return MPI_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* ompi_coll_base_scan_intra_recursivedoubling
|
||||
*
|
||||
* Function: Recursive doubling algorithm for inclusive scan.
|
||||
* Accepts: Same as MPI_Scan
|
||||
* Returns: MPI_SUCCESS or error code
|
||||
*
|
||||
* Description: Implements recursive doubling algorithm for MPI_Scan.
|
||||
* The algorithm preserves order of operations so it can
|
||||
* be used both by commutative and non-commutative operations.
|
||||
*
|
||||
* Example for 5 processes and commutative operation MPI_SUM:
|
||||
* Process: 0 1 2 3 4
|
||||
* recvbuf: [0] [1] [2] [3] [4]
|
||||
* psend: [0] [1] [2] [3] [4]
|
||||
*
|
||||
* Step 1:
|
||||
* recvbuf: [0] [0+1] [2] [2+3] [4]
|
||||
* psend: [1+0] [0+1] [3+2] [2+3] [4]
|
||||
*
|
||||
* Step 2:
|
||||
* recvbuf: [0] [0+1] [(1+0)+2] [(1+0)+(2+3)] [4]
|
||||
* psend: [(3+2)+(1+0)] [(2+3)+(0+1)] [(1+0)+(3+2)] [(1+0)+(2+3)] [4]
|
||||
*
|
||||
* Step 3:
|
||||
* recvbuf: [0] [0+1] [(1+0)+2] [(1+0)+(2+3)] [((3+2)+(1+0))+4]
|
||||
* psend: [4+((3+2)+(1+0))] [((3+2)+(1+0))+4]
|
||||
*
|
||||
* Time complexity (worst case): \ceil(\log_2(p))(2\alpha + 2m\beta + 2m\gamma)
|
||||
* Memory requirements (per process): 2 * count * typesize = O(count)
|
||||
* Limitations: intra-communicators only
|
||||
*/
|
||||
int ompi_coll_base_scan_intra_recursivedoubling(
|
||||
const void *sendbuf, void *recvbuf, int count, struct ompi_datatype_t *datatype,
|
||||
struct ompi_op_t *op, struct ompi_communicator_t *comm,
|
||||
mca_coll_base_module_t *module)
|
||||
{
|
||||
int err = MPI_SUCCESS;
|
||||
char *tmpsend_raw = NULL, *tmprecv_raw = NULL;
|
||||
int comm_size = ompi_comm_size(comm);
|
||||
int rank = ompi_comm_rank(comm);
|
||||
|
||||
OPAL_OUTPUT((ompi_coll_base_framework.framework_output,
|
||||
"coll:base:scan_intra_recursivedoubling: rank %d/%d",
|
||||
rank, comm_size));
|
||||
if (count == 0)
|
||||
return MPI_SUCCESS;
|
||||
|
||||
if (sendbuf != MPI_IN_PLACE) {
|
||||
err = ompi_datatype_copy_content_same_ddt(datatype, count, recvbuf, (char *)sendbuf);
|
||||
if (MPI_SUCCESS != err) { goto cleanup_and_return; }
|
||||
}
|
||||
if (comm_size < 2)
|
||||
return MPI_SUCCESS;
|
||||
|
||||
ptrdiff_t dsize, gap;
|
||||
dsize = opal_datatype_span(&datatype->super, count, &gap);
|
||||
tmpsend_raw = malloc(dsize);
|
||||
tmprecv_raw = malloc(dsize);
|
||||
if (NULL == tmpsend_raw || NULL == tmprecv_raw) {
|
||||
err = OMPI_ERR_OUT_OF_RESOURCE;
|
||||
goto cleanup_and_return;
|
||||
}
|
||||
char *psend = tmpsend_raw - gap;
|
||||
char *precv = tmprecv_raw - gap;
|
||||
err = ompi_datatype_copy_content_same_ddt(datatype, count, psend, recvbuf);
|
||||
if (MPI_SUCCESS != err) { goto cleanup_and_return; }
|
||||
int is_commute = ompi_op_is_commute(op);
|
||||
|
||||
for (int mask = 1; mask < comm_size; mask <<= 1) {
|
||||
int remote = rank ^ mask;
|
||||
if (remote < comm_size) {
|
||||
err = ompi_coll_base_sendrecv(psend, count, datatype, remote,
|
||||
MCA_COLL_BASE_TAG_SCAN,
|
||||
precv, count, datatype, remote,
|
||||
MCA_COLL_BASE_TAG_SCAN, comm,
|
||||
MPI_STATUS_IGNORE, rank);
|
||||
if (MPI_SUCCESS != err) { goto cleanup_and_return; }
|
||||
|
||||
if (rank > remote) {
|
||||
/* Accumulate prefix reduction: recvbuf = precv <op> recvbuf */
|
||||
ompi_op_reduce(op, precv, recvbuf, count, datatype);
|
||||
/* Partial result: psend = precv <op> psend */
|
||||
ompi_op_reduce(op, precv, psend, count, datatype);
|
||||
} else {
|
||||
if (is_commute) {
|
||||
/* psend = precv <op> psend */
|
||||
ompi_op_reduce(op, precv, psend, count, datatype);
|
||||
} else {
|
||||
/* precv = psend <op> precv */
|
||||
ompi_op_reduce(op, psend, precv, count, datatype);
|
||||
char *tmp = psend;
|
||||
psend = precv;
|
||||
precv = tmp;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
cleanup_and_return:
|
||||
if (NULL != tmpsend_raw)
|
||||
free(tmpsend_raw);
|
||||
if (NULL != tmprecv_raw)
|
||||
free(tmprecv_raw);
|
||||
return err;
|
||||
}
|
||||
@ -10,7 +10,7 @@
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2011 NVIDIA Corporation. All rights reserved.
|
||||
* Copyright (c) 2015 Research Organization for Information Science
|
||||
* Copyright (c) 2015-2018 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
@ -48,72 +48,7 @@ mca_coll_basic_exscan_intra(const void *sbuf, void *rbuf, int count,
|
||||
struct ompi_communicator_t *comm,
|
||||
mca_coll_base_module_t *module)
|
||||
{
|
||||
int size, rank, err;
|
||||
ptrdiff_t dsize, gap;
|
||||
char *free_buffer = NULL;
|
||||
char *reduce_buffer = NULL;
|
||||
|
||||
rank = ompi_comm_rank(comm);
|
||||
size = ompi_comm_size(comm);
|
||||
|
||||
/* For MPI_IN_PLACE, just adjust send buffer to point to
|
||||
* receive buffer. */
|
||||
if (MPI_IN_PLACE == sbuf) {
|
||||
sbuf = rbuf;
|
||||
}
|
||||
|
||||
/* If we're rank 0, then just send our sbuf to the next rank, and
|
||||
* we are done. */
|
||||
if (0 == rank) {
|
||||
return MCA_PML_CALL(send(sbuf, count, dtype, rank + 1,
|
||||
MCA_COLL_BASE_TAG_EXSCAN,
|
||||
MCA_PML_BASE_SEND_STANDARD, comm));
|
||||
}
|
||||
|
||||
/* If we're the last rank, then just receive the result from the
|
||||
* prior rank, and we are done. */
|
||||
else if ((size - 1) == rank) {
|
||||
return MCA_PML_CALL(recv(rbuf, count, dtype, rank - 1,
|
||||
MCA_COLL_BASE_TAG_EXSCAN, comm,
|
||||
MPI_STATUS_IGNORE));
|
||||
}
|
||||
|
||||
/* Otherwise, get the result from the prior rank, combine it with my
|
||||
* data, and send it to the next rank */
|
||||
|
||||
/* Get a temporary buffer to perform the reduction into. Rationale
|
||||
* for malloc'ing this size is provided in coll_basic_reduce.c. */
|
||||
dsize = opal_datatype_span(&dtype->super, count, &gap);
|
||||
|
||||
free_buffer = (char*)malloc(dsize);
|
||||
if (NULL == free_buffer) {
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
reduce_buffer = free_buffer - gap;
|
||||
err = ompi_datatype_copy_content_same_ddt(dtype, count,
|
||||
reduce_buffer, (char*)sbuf);
|
||||
|
||||
/* Receive the reduced value from the prior rank */
|
||||
err = MCA_PML_CALL(recv(rbuf, count, dtype, rank - 1,
|
||||
MCA_COLL_BASE_TAG_EXSCAN, comm, MPI_STATUS_IGNORE));
|
||||
if (MPI_SUCCESS != err) {
|
||||
goto error;
|
||||
}
|
||||
|
||||
/* Now reduce the prior rank's result with my source buffer. The source
|
||||
* buffer had been previously copied into the temporary reduce_buffer. */
|
||||
ompi_op_reduce(op, rbuf, reduce_buffer, count, dtype);
|
||||
|
||||
/* Send my result off to the next rank */
|
||||
err = MCA_PML_CALL(send(reduce_buffer, count, dtype, rank + 1,
|
||||
MCA_COLL_BASE_TAG_EXSCAN,
|
||||
MCA_PML_BASE_SEND_STANDARD, comm));
|
||||
/* Error */
|
||||
error:
|
||||
free(free_buffer);
|
||||
|
||||
/* All done */
|
||||
return err;
|
||||
return ompi_coll_base_exscan_intra_linear(sbuf, rbuf, count, dtype, op, comm, module);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -9,7 +9,7 @@
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2015 Research Organization for Information Science
|
||||
* Copyright (c) 2015-2018 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
@ -46,85 +46,5 @@ mca_coll_basic_scan_intra(const void *sbuf, void *rbuf, int count,
|
||||
struct ompi_communicator_t *comm,
|
||||
mca_coll_base_module_t *module)
|
||||
{
|
||||
int size, rank, err;
|
||||
ptrdiff_t dsize, gap;
|
||||
char *free_buffer = NULL;
|
||||
char *pml_buffer = NULL;
|
||||
|
||||
/* Initialize */
|
||||
|
||||
rank = ompi_comm_rank(comm);
|
||||
size = ompi_comm_size(comm);
|
||||
|
||||
/* If I'm rank 0, just copy into the receive buffer */
|
||||
|
||||
if (0 == rank) {
|
||||
if (MPI_IN_PLACE != sbuf) {
|
||||
err = ompi_datatype_copy_content_same_ddt(dtype, count, (char*)rbuf, (char*)sbuf);
|
||||
if (MPI_SUCCESS != err) {
|
||||
return err;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Otherwise receive previous buffer and reduce. */
|
||||
|
||||
else {
|
||||
/* Allocate a temporary buffer. Rationale for this size is
|
||||
* listed in coll_basic_reduce.c. Use this temporary buffer to
|
||||
* receive into, later. */
|
||||
|
||||
dsize = opal_datatype_span(&dtype->super, count, &gap);
|
||||
free_buffer = malloc(dsize);
|
||||
if (NULL == free_buffer) {
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
pml_buffer = free_buffer - gap;
|
||||
|
||||
/* Copy the send buffer into the receive buffer. */
|
||||
|
||||
if (MPI_IN_PLACE != sbuf) {
|
||||
err = ompi_datatype_copy_content_same_ddt(dtype, count, (char*)rbuf, (char*)sbuf);
|
||||
if (MPI_SUCCESS != err) {
|
||||
if (NULL != free_buffer) {
|
||||
free(free_buffer);
|
||||
}
|
||||
return err;
|
||||
}
|
||||
}
|
||||
|
||||
/* Receive the prior answer */
|
||||
|
||||
err = MCA_PML_CALL(recv(pml_buffer, count, dtype,
|
||||
rank - 1, MCA_COLL_BASE_TAG_SCAN, comm,
|
||||
MPI_STATUS_IGNORE));
|
||||
if (MPI_SUCCESS != err) {
|
||||
if (NULL != free_buffer) {
|
||||
free(free_buffer);
|
||||
}
|
||||
return err;
|
||||
}
|
||||
|
||||
/* Perform the operation */
|
||||
|
||||
ompi_op_reduce(op, pml_buffer, rbuf, count, dtype);
|
||||
|
||||
/* All done */
|
||||
|
||||
if (NULL != free_buffer) {
|
||||
free(free_buffer);
|
||||
}
|
||||
}
|
||||
|
||||
/* Send result to next process. */
|
||||
|
||||
if (rank < (size - 1)) {
|
||||
return MCA_PML_CALL(send(rbuf, count, dtype, rank + 1,
|
||||
MCA_COLL_BASE_TAG_SCAN,
|
||||
MCA_PML_BASE_SEND_STANDARD, comm));
|
||||
}
|
||||
|
||||
/* All done */
|
||||
|
||||
return MPI_SUCCESS;
|
||||
return ompi_coll_base_scan_intra_linear(sbuf, rbuf, count, dtype, op, comm, module);
|
||||
}
|
||||
|
||||
@ -1,37 +0,0 @@
|
||||
#
|
||||
# Copyright (c) 2017 IBM Corporation. All rights reserved.
|
||||
# $COPYRIGHT$
|
||||
#
|
||||
# Additional copyrights may follow
|
||||
#
|
||||
# $HEADER$
|
||||
#
|
||||
|
||||
sources = \
|
||||
coll_spacc.h \
|
||||
coll_spacc_component.c \
|
||||
coll_spacc_module.c \
|
||||
coll_spacc_allreduce.c \
|
||||
coll_spacc_reduce.c
|
||||
|
||||
# Make the output library in this directory, and name it either
|
||||
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
|
||||
# (for static builds).
|
||||
|
||||
if MCA_BUILD_ompi_coll_spacc_DSO
|
||||
component_noinst =
|
||||
component_install = mca_coll_spacc.la
|
||||
else
|
||||
component_noinst = libmca_coll_spacc.la
|
||||
component_install =
|
||||
endif
|
||||
|
||||
mcacomponentdir = $(ompilibdir)
|
||||
mcacomponent_LTLIBRARIES = $(component_install)
|
||||
mca_coll_spacc_la_SOURCES = $(sources)
|
||||
mca_coll_spacc_la_LDFLAGS = -module -avoid-version
|
||||
mca_coll_spacc_la_LIBADD = $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la
|
||||
|
||||
noinst_LTLIBRARIES = $(component_noinst)
|
||||
libmca_coll_spacc_la_SOURCES =$(sources)
|
||||
libmca_coll_spacc_la_LDFLAGS = -module -avoid-version
|
||||
@ -1,84 +0,0 @@
|
||||
/*
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#ifndef MCA_COLL_SPACC_EXPORT_H
|
||||
#define MCA_COLL_SPACC_EXPORT_H
|
||||
|
||||
#include "ompi_config.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include "ompi/mca/coll/coll.h"
|
||||
|
||||
BEGIN_C_DECLS
|
||||
|
||||
/* Globally exported variables */
|
||||
extern int mca_coll_spacc_stream;
|
||||
extern int mca_coll_spacc_priority;
|
||||
extern int mca_coll_spacc_verbose;
|
||||
|
||||
/* API functions */
|
||||
|
||||
int mca_coll_spacc_init_query(bool enable_progress_threads,
|
||||
bool enable_mpi_threads);
|
||||
mca_coll_base_module_t
|
||||
*mca_coll_spacc_comm_query(struct ompi_communicator_t *comm, int *priority);
|
||||
|
||||
int mca_coll_spacc_module_enable(mca_coll_base_module_t *module,
|
||||
struct ompi_communicator_t *comm);
|
||||
|
||||
int mca_coll_spacc_allreduce_intra_redscat_allgather(
|
||||
const void *sbuf, void *rbuf, int count, struct ompi_datatype_t *dtype,
|
||||
struct ompi_op_t *op, struct ompi_communicator_t *comm,
|
||||
mca_coll_base_module_t *module);
|
||||
|
||||
int mca_coll_spacc_reduce_intra_redscat_gather(
|
||||
const void *sbuf, void *rbuf, int count, struct ompi_datatype_t *dtype,
|
||||
struct ompi_op_t *op, int root, struct ompi_communicator_t *comm,
|
||||
mca_coll_base_module_t *module);
|
||||
|
||||
/*
|
||||
* coll API functions
|
||||
*/
|
||||
|
||||
/* API functions */
|
||||
|
||||
int ompi_coll_spacc_init_query(bool enable_progress_threads,
|
||||
bool enable_mpi_threads);
|
||||
|
||||
mca_coll_base_module_t *
|
||||
ompi_coll_spacc_comm_query(struct ompi_communicator_t *comm, int *priority);
|
||||
|
||||
struct mca_coll_spacc_component_t {
|
||||
/* Base coll component */
|
||||
mca_coll_base_component_2_0_0_t super;
|
||||
|
||||
/* MCA parameter: priority of this component */
|
||||
int spacc_priority;
|
||||
|
||||
/* global stuff that I need the component to store */
|
||||
|
||||
/* MCA parameters first */
|
||||
};
|
||||
|
||||
/*
|
||||
* Convenience typedef
|
||||
*/
|
||||
typedef struct mca_coll_spacc_component_t mca_coll_spacc_component_t;
|
||||
|
||||
/*
|
||||
* Global component instance
|
||||
*/
|
||||
OMPI_MODULE_DECLSPEC extern mca_coll_spacc_component_t mca_coll_spacc_component;
|
||||
|
||||
struct mca_coll_spacc_module_t {
|
||||
mca_coll_base_module_t super;
|
||||
};
|
||||
typedef struct mca_coll_spacc_module_t mca_coll_spacc_module_t;
|
||||
OBJ_CLASS_DECLARATION(mca_coll_spacc_module_t);
|
||||
|
||||
#endif /* MCA_COLL_SPACC_EXPORT_H */
|
||||
@ -1,355 +0,0 @@
|
||||
/*
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "coll_spacc.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include "ompi/constants.h"
|
||||
#include "opal/util/bit_ops.h"
|
||||
#include "ompi/datatype/ompi_datatype.h"
|
||||
#include "ompi/communicator/communicator.h"
|
||||
#include "ompi/mca/coll/coll.h"
|
||||
#include "ompi/mca/coll/base/coll_base_functions.h"
|
||||
#include "ompi/mca/coll/base/coll_tags.h"
|
||||
#include "ompi/mca/coll/base/coll_base_util.h"
|
||||
#include "ompi/mca/pml/pml.h"
|
||||
#include "ompi/op/op.h"
|
||||
|
||||
/*
|
||||
* mca_coll_spacc_allreduce_intra_redscat_gather
|
||||
*
|
||||
* Function: Allreduce using Rabenseifner's algorithm.
|
||||
* Accepts: Same arguments as MPI_Allreduce
|
||||
* Returns: MPI_SUCCESS or error code
|
||||
*
|
||||
* Description: an implementation of Rabenseifner's allreduce algorithm [1, 2].
|
||||
* [1] Rajeev Thakur, Rolf Rabenseifner and William Gropp.
|
||||
* Optimization of Collective Communication Operations in MPICH //
|
||||
* The Int. Journal of High Performance Computing Applications. Vol 19,
|
||||
* Issue 1, pp. 49--66.
|
||||
* [2] http://www.hlrs.de/mpi/myreduce.html.
|
||||
*
|
||||
* This algorithm is a combination of a reduce-scatter implemented with
|
||||
* recursive vector halving and recursive distance doubling, followed either
|
||||
* by an allgather implemented with recursive doubling [1].
|
||||
*
|
||||
* Step 1. If the number of processes is not a power of two, reduce it to
|
||||
* the nearest lower power of two (p' = 2^{\floor{\log_2 p}})
|
||||
* by removing r = p - p' extra processes as follows. In the first 2r processes
|
||||
* (ranks 0 to 2r - 1), all the even ranks send the second half of the input
|
||||
* vector to their right neighbor (rank + 1), and all the odd ranks send
|
||||
* the first half of the input vector to their left neighbor (rank - 1).
|
||||
* The even ranks compute the reduction on the first half of the vector and
|
||||
* the odd ranks compute the reduction on the second half. The odd ranks then
|
||||
* send the result to their left neighbors (the even ranks). As a result,
|
||||
* the even ranks among the first 2r processes now contain the reduction with
|
||||
* the input vector on their right neighbors (the odd ranks). These odd ranks
|
||||
* do not participate in the rest of the algorithm, which leaves behind
|
||||
* a power-of-two number of processes. The first r even-ranked processes and
|
||||
* the last p - 2r processes are now renumbered from 0 to p' - 1.
|
||||
*
|
||||
* Step 2. The remaining processes now perform a reduce-scatter by using
|
||||
* recursive vector halving and recursive distance doubling. The even-ranked
|
||||
* processes send the second half of their buffer to rank + 1 and the odd-ranked
|
||||
* processes send the first half of their buffer to rank - 1. All processes
|
||||
* then compute the reduction between the local buffer and the received buffer.
|
||||
* In the next log_2(p') - 1 steps, the buffers are recursively halved, and the
|
||||
* distance is doubled. At the end, each of the p' processes has 1 / p' of the
|
||||
* total reduction result.
|
||||
*
|
||||
* Step 3. An allgather is performed by using recursive vector doubling and
|
||||
* distance halving. All exchanges are executed in reverse order relative
|
||||
* to recursive doubling on previous step. If the number of processes is not
|
||||
* a power of two, the total result vector must be sent to the r processes
|
||||
* that were removed in the first step.
|
||||
*
|
||||
* Limitations:
|
||||
* count >= 2^{\floor{\log_2 p}}
|
||||
* commutative operations only
|
||||
* intra-communicators only
|
||||
*
|
||||
* Memory requirements (per process):
|
||||
* count * typesize + 4 * log_2(p) * sizeof(int) = O(count)
|
||||
*/
|
||||
int mca_coll_spacc_allreduce_intra_redscat_allgather(
|
||||
const void *sbuf, void *rbuf, int count, struct ompi_datatype_t *dtype,
|
||||
struct ompi_op_t *op, struct ompi_communicator_t *comm,
|
||||
mca_coll_base_module_t *module)
|
||||
{
|
||||
int *rindex = NULL, *rcount = NULL, *sindex = NULL, *scount = NULL;
|
||||
|
||||
int comm_size = ompi_comm_size(comm);
|
||||
int rank = ompi_comm_rank(comm);
|
||||
|
||||
opal_output_verbose(30, mca_coll_spacc_stream,
|
||||
"coll:spacc:allreduce_intra_redscat_allgather: rank %d/%d",
|
||||
rank, comm_size);
|
||||
|
||||
/* Find nearest power-of-two less than or equal to comm_size */
|
||||
int nsteps = opal_hibit(comm_size, comm->c_cube_dim + 1); /* ilog2(comm_size) */
|
||||
assert(nsteps >= 0);
|
||||
int nprocs_pof2 = 1 << nsteps; /* flp2(comm_size) */
|
||||
|
||||
if (count < nprocs_pof2 || !ompi_op_is_commute(op)) {
|
||||
opal_output_verbose(20, mca_coll_spacc_stream,
|
||||
"coll:spacc:allreduce_intra_redscat_allgather: rank %d/%d count %d switching to base allreduce",
|
||||
rank, comm_size, count);
|
||||
return ompi_coll_base_allreduce_intra_basic_linear(sbuf, rbuf, count, dtype,
|
||||
op, comm, module);
|
||||
}
|
||||
|
||||
int err = MPI_SUCCESS;
|
||||
|
||||
ptrdiff_t lb, extent, dsize, gap = 0;
|
||||
ompi_datatype_get_extent(dtype, &lb, &extent);
|
||||
dsize = opal_datatype_span(&dtype->super, count, &gap);
|
||||
|
||||
/* Temporary buffer for receiving messages */
|
||||
char *tmp_buf = NULL;
|
||||
char *tmp_buf_raw = (char *)malloc(dsize);
|
||||
if (NULL == tmp_buf_raw)
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
tmp_buf = tmp_buf_raw - gap;
|
||||
|
||||
if (sbuf != MPI_IN_PLACE) {
|
||||
/* Copy sbuf to rbuf */
|
||||
err = ompi_datatype_copy_content_same_ddt(dtype, count, (char *)rbuf,
|
||||
(char *)sbuf);
|
||||
}
|
||||
|
||||
/*
|
||||
* Step 1. Reduce the number of processes to the nearest lower power of two
|
||||
* p' = 2^{\floor{\log_2 p}} by removing r = p - p' processes.
|
||||
* 1. In the first 2r processes (ranks 0 to 2r - 1), all the even ranks send
|
||||
* the second half of the input vector to their right neighbor (rank + 1)
|
||||
* and all the odd ranks send the first half of the input vector to their
|
||||
* left neighbor (rank - 1).
|
||||
* 2. All 2r processes compute the reduction on their half.
|
||||
* 3. The odd ranks then send the result to their left neighbors
|
||||
* (the even ranks).
|
||||
*
|
||||
* The even ranks (0 to 2r - 1) now contain the reduction with the input
|
||||
* vector on their right neighbors (the odd ranks). The first r even
|
||||
* processes and the p - 2r last processes are renumbered from
|
||||
* 0 to 2^{\floor{\log_2 p}} - 1.
|
||||
*/
|
||||
|
||||
int vrank, step, wsize;
|
||||
int nprocs_rem = comm_size - nprocs_pof2;
|
||||
|
||||
if (rank < 2 * nprocs_rem) {
|
||||
int count_lhalf = count / 2;
|
||||
int count_rhalf = count - count_lhalf;
|
||||
|
||||
if (rank % 2 != 0) {
|
||||
/*
|
||||
* Odd process -- exchange with rank - 1
|
||||
* Send the left half of the input vector to the left neighbor,
|
||||
* Recv the right half of the input vector from the left neighbor
|
||||
*/
|
||||
err = ompi_coll_base_sendrecv(rbuf, count_lhalf, dtype, rank - 1,
|
||||
MCA_COLL_BASE_TAG_ALLREDUCE,
|
||||
(char *)tmp_buf + (ptrdiff_t)count_lhalf * extent,
|
||||
count_rhalf, dtype, rank - 1,
|
||||
MCA_COLL_BASE_TAG_ALLREDUCE, comm,
|
||||
MPI_STATUS_IGNORE, rank);
|
||||
if (MPI_SUCCESS != err) { goto cleanup_and_return; }
|
||||
|
||||
/* Reduce on the right half of the buffers (result in rbuf) */
|
||||
ompi_op_reduce(op, (char *)tmp_buf + (ptrdiff_t)count_lhalf * extent,
|
||||
(char *)rbuf + count_lhalf * extent, count_rhalf, dtype);
|
||||
|
||||
/* Send the right half to the left neighbor */
|
||||
err = MCA_PML_CALL(send((char *)rbuf + (ptrdiff_t)count_lhalf * extent,
|
||||
count_rhalf, dtype, rank - 1,
|
||||
MCA_COLL_BASE_TAG_ALLREDUCE,
|
||||
MCA_PML_BASE_SEND_STANDARD, comm));
|
||||
if (MPI_SUCCESS != err) { goto cleanup_and_return; }
|
||||
|
||||
/* This process does not pariticipate in recursive doubling phase */
|
||||
vrank = -1;
|
||||
|
||||
} else {
|
||||
/*
|
||||
* Even process -- exchange with rank + 1
|
||||
* Send the right half of the input vector to the right neighbor,
|
||||
* Recv the left half of the input vector from the right neighbor
|
||||
*/
|
||||
err = ompi_coll_base_sendrecv((char *)rbuf + (ptrdiff_t)count_lhalf * extent,
|
||||
count_rhalf, dtype, rank + 1,
|
||||
MCA_COLL_BASE_TAG_ALLREDUCE,
|
||||
tmp_buf, count_lhalf, dtype, rank + 1,
|
||||
MCA_COLL_BASE_TAG_ALLREDUCE, comm,
|
||||
MPI_STATUS_IGNORE, rank);
|
||||
if (MPI_SUCCESS != err) { goto cleanup_and_return; }
|
||||
|
||||
/* Reduce on the right half of the buffers (result in rbuf) */
|
||||
ompi_op_reduce(op, tmp_buf, rbuf, count_lhalf, dtype);
|
||||
|
||||
/* Recv the right half from the right neighbor */
|
||||
err = MCA_PML_CALL(recv((char *)rbuf + (ptrdiff_t)count_lhalf * extent,
|
||||
count_rhalf, dtype, rank + 1,
|
||||
MCA_COLL_BASE_TAG_ALLREDUCE, comm,
|
||||
MPI_STATUS_IGNORE));
|
||||
if (MPI_SUCCESS != err) { goto cleanup_and_return; }
|
||||
|
||||
vrank = rank / 2;
|
||||
}
|
||||
} else { /* rank >= 2 * nprocs_rem */
|
||||
vrank = rank - nprocs_rem;
|
||||
}
|
||||
|
||||
/*
|
||||
* Step 2. Reduce-scatter implemented with recursive vector halving and
|
||||
* recursive distance doubling. We have p' = 2^{\floor{\log_2 p}}
|
||||
* power-of-two number of processes with new ranks (vrank) and result in rbuf.
|
||||
*
|
||||
* The even-ranked processes send the right half of their buffer to rank + 1
|
||||
* and the odd-ranked processes send the left half of their buffer to
|
||||
* rank - 1. All processes then compute the reduction between the local
|
||||
* buffer and the received buffer. In the next \log_2(p') - 1 steps, the
|
||||
* buffers are recursively halved, and the distance is doubled. At the end,
|
||||
* each of the p' processes has 1 / p' of the total reduction result.
|
||||
*/
|
||||
rindex = malloc(sizeof(*rindex) * nsteps);
|
||||
sindex = malloc(sizeof(*sindex) * nsteps);
|
||||
rcount = malloc(sizeof(*rcount) * nsteps);
|
||||
scount = malloc(sizeof(*scount) * nsteps);
|
||||
if (NULL == rindex || NULL == sindex || NULL == rcount || NULL == scount) {
|
||||
err = OMPI_ERR_OUT_OF_RESOURCE;
|
||||
goto cleanup_and_return;
|
||||
}
|
||||
|
||||
if (vrank != -1) {
|
||||
step = 0;
|
||||
wsize = count;
|
||||
sindex[0] = rindex[0] = 0;
|
||||
|
||||
for (int mask = 1; mask < nprocs_pof2; mask <<= 1) {
|
||||
/*
|
||||
* On each iteration: rindex[step] = sindex[step] -- begining of the
|
||||
* current window. Length of the current window is storded in wsize.
|
||||
*/
|
||||
int vdest = vrank ^ mask;
|
||||
/* Translate vdest virtual rank to real rank */
|
||||
int dest = (vdest < nprocs_rem) ? vdest * 2 : vdest + nprocs_rem;
|
||||
|
||||
if (rank < dest) {
|
||||
/*
|
||||
* Recv into the left half of the current window, send the right
|
||||
* half of the window to the peer (perform reduce on the left
|
||||
* half of the current window)
|
||||
*/
|
||||
rcount[step] = wsize / 2;
|
||||
scount[step] = wsize - rcount[step];
|
||||
sindex[step] = rindex[step] + rcount[step];
|
||||
} else {
|
||||
/*
|
||||
* Recv into the right half of the current window, send the left
|
||||
* half of the window to the peer (perform reduce on the right
|
||||
* half of the current window)
|
||||
*/
|
||||
scount[step] = wsize / 2;
|
||||
rcount[step] = wsize - scount[step];
|
||||
rindex[step] = sindex[step] + scount[step];
|
||||
}
|
||||
|
||||
/* Send part of data from the rbuf, recv into the tmp_buf */
|
||||
err = ompi_coll_base_sendrecv((char *)rbuf + (ptrdiff_t)sindex[step] * extent,
|
||||
scount[step], dtype, dest,
|
||||
MCA_COLL_BASE_TAG_ALLREDUCE,
|
||||
(char *)tmp_buf + (ptrdiff_t)rindex[step] * extent,
|
||||
rcount[step], dtype, dest,
|
||||
MCA_COLL_BASE_TAG_ALLREDUCE, comm,
|
||||
MPI_STATUS_IGNORE, rank);
|
||||
if (MPI_SUCCESS != err) { goto cleanup_and_return; }
|
||||
|
||||
/* Local reduce: rbuf[] = tmp_buf[] <op> rbuf[] */
|
||||
ompi_op_reduce(op, (char *)tmp_buf + (ptrdiff_t)rindex[step] * extent,
|
||||
(char *)rbuf + (ptrdiff_t)rindex[step] * extent,
|
||||
rcount[step], dtype);
|
||||
|
||||
/* Move the current window to the received message */
|
||||
if (step + 1 < nsteps) {
|
||||
rindex[step + 1] = rindex[step];
|
||||
sindex[step + 1] = rindex[step];
|
||||
wsize = rcount[step];
|
||||
step++;
|
||||
}
|
||||
}
|
||||
/*
|
||||
* Assertion: each process has 1 / p' of the total reduction result:
|
||||
* rcount[nsteps - 1] elements in the rbuf[rindex[nsteps - 1], ...].
|
||||
*/
|
||||
|
||||
/*
|
||||
* Step 3. Allgather by the recursive doubling algorithm.
|
||||
* Each process has 1 / p' of the total reduction result:
|
||||
* rcount[nsteps - 1] elements in the rbuf[rindex[nsteps - 1], ...].
|
||||
* All exchanges are executed in reverse order relative
|
||||
* to recursive doubling (previous step).
|
||||
*/
|
||||
|
||||
step--;
|
||||
|
||||
for (int mask = nprocs_pof2 >> 1; mask > 0; mask >>= 1) {
|
||||
int vdest = vrank ^ mask;
|
||||
/* Translate vdest virtual rank to real rank */
|
||||
int dest = (vdest < nprocs_rem) ? vdest * 2 : vdest + nprocs_rem;
|
||||
|
||||
/*
|
||||
* Send rcount[step] elements from rbuf[rindex[step]...]
|
||||
* Recv scount[step] elements to rbuf[sindex[step]...]
|
||||
*/
|
||||
err = ompi_coll_base_sendrecv((char *)rbuf + (ptrdiff_t)rindex[step] * extent,
|
||||
rcount[step], dtype, dest,
|
||||
MCA_COLL_BASE_TAG_ALLREDUCE,
|
||||
(char *)rbuf + (ptrdiff_t)sindex[step] * extent,
|
||||
scount[step], dtype, dest,
|
||||
MCA_COLL_BASE_TAG_ALLREDUCE, comm,
|
||||
MPI_STATUS_IGNORE, rank);
|
||||
if (MPI_SUCCESS != err) { goto cleanup_and_return; }
|
||||
step--;
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Step 4. Send total result to excluded odd ranks.
|
||||
*/
|
||||
if (rank < 2 * nprocs_rem) {
|
||||
if (rank % 2 != 0) {
|
||||
/* Odd process -- recv result from rank - 1 */
|
||||
err = MCA_PML_CALL(recv(rbuf, count, dtype, rank - 1,
|
||||
MCA_COLL_BASE_TAG_ALLREDUCE, comm,
|
||||
MPI_STATUS_IGNORE));
|
||||
if (OMPI_SUCCESS != err) { goto cleanup_and_return; }
|
||||
|
||||
} else {
|
||||
/* Even process -- send result to rank + 1 */
|
||||
err = MCA_PML_CALL(send(rbuf, count, dtype, rank + 1,
|
||||
MCA_COLL_BASE_TAG_ALLREDUCE,
|
||||
MCA_PML_BASE_SEND_STANDARD, comm));
|
||||
if (MPI_SUCCESS != err) { goto cleanup_and_return; }
|
||||
}
|
||||
}
|
||||
|
||||
cleanup_and_return:
|
||||
if (NULL != tmp_buf_raw)
|
||||
free(tmp_buf_raw);
|
||||
if (NULL != rindex)
|
||||
free(rindex);
|
||||
if (NULL != sindex)
|
||||
free(sindex);
|
||||
if (NULL != rcount)
|
||||
free(rcount);
|
||||
if (NULL != scount)
|
||||
free(scount);
|
||||
|
||||
return err;
|
||||
}
|
||||
@ -1,100 +0,0 @@
|
||||
/*
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include "ompi/mca/coll/coll.h"
|
||||
#include "coll_spacc.h"
|
||||
|
||||
/*
|
||||
* Public string showing the coll ompi_spacc component version number
|
||||
*/
|
||||
const char *ompi_coll_spacc_component_version_string =
|
||||
"Open MPI SPACC collective MCA component version " OMPI_VERSION;
|
||||
|
||||
/*
|
||||
* Global variable
|
||||
*/
|
||||
int mca_coll_spacc_priority = 5;
|
||||
int mca_coll_spacc_stream = -1;
|
||||
int mca_coll_spacc_verbose = 0;
|
||||
|
||||
/*
|
||||
* Local function
|
||||
*/
|
||||
static int spacc_register(void);
|
||||
static int spacc_open(void);
|
||||
static int spacc_close(void);
|
||||
|
||||
/*
|
||||
* Instantiate the public struct with all of our public information
|
||||
* and pointers to our public functions in it
|
||||
*/
|
||||
mca_coll_spacc_component_t mca_coll_spacc_component = {
|
||||
/* First, fill in the super */
|
||||
{
|
||||
/* First, the mca_component_t struct containing meta information
|
||||
about the component itself */
|
||||
.collm_version = {
|
||||
MCA_COLL_BASE_VERSION_2_0_0,
|
||||
|
||||
/* Component name and version */
|
||||
.mca_component_name = "spacc",
|
||||
MCA_BASE_MAKE_VERSION(component, OMPI_MAJOR_VERSION, OMPI_MINOR_VERSION,
|
||||
OMPI_RELEASE_VERSION),
|
||||
|
||||
/* Component open and close functions */
|
||||
.mca_open_component = spacc_open,
|
||||
.mca_close_component = spacc_close,
|
||||
.mca_register_component_params = spacc_register,
|
||||
},
|
||||
.collm_data = {
|
||||
/* The component is checkpoint ready */
|
||||
MCA_BASE_METADATA_PARAM_CHECKPOINT
|
||||
},
|
||||
|
||||
/* Initialization / querying functions */
|
||||
.collm_init_query = ompi_coll_spacc_init_query,
|
||||
.collm_comm_query = ompi_coll_spacc_comm_query,
|
||||
}
|
||||
};
|
||||
|
||||
static int spacc_register(void)
|
||||
{
|
||||
/* Use a low priority, but allow other components to be lower */
|
||||
mca_coll_spacc_priority = 5;
|
||||
(void)mca_base_component_var_register(&mca_coll_spacc_component.super.collm_version,
|
||||
"priority", "Priority of the spacc coll component",
|
||||
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
|
||||
OPAL_INFO_LVL_9,
|
||||
MCA_BASE_VAR_SCOPE_READONLY,
|
||||
&mca_coll_spacc_priority);
|
||||
|
||||
(void)mca_base_component_var_register(&mca_coll_spacc_component.super.collm_version,
|
||||
"verbose", "Verbose level of the spacc coll component",
|
||||
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
|
||||
OPAL_INFO_LVL_9,
|
||||
MCA_BASE_VAR_SCOPE_READONLY,
|
||||
&mca_coll_spacc_verbose);
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
static int spacc_open(void)
|
||||
{
|
||||
mca_coll_spacc_stream = opal_output_open(NULL);
|
||||
opal_output_set_verbosity(mca_coll_spacc_stream, mca_coll_spacc_verbose);
|
||||
opal_output_verbose(30, mca_coll_spacc_stream, "coll:spacc:component_open: done");
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
static int spacc_close(void)
|
||||
{
|
||||
opal_output_verbose(30, mca_coll_spacc_stream, "coll:spacc:component_close: done");
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
@ -1,99 +0,0 @@
|
||||
/*
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include "ompi/communicator/communicator.h"
|
||||
#include "ompi/mca/coll/base/base.h"
|
||||
#include "ompi/mca/coll/coll.h"
|
||||
#include "coll_spacc.h"
|
||||
|
||||
static int spacc_module_enable(mca_coll_base_module_t *module,
|
||||
struct ompi_communicator_t *comm);
|
||||
/*
|
||||
* Initial query function that is invoked during MPI_INIT, allowing
|
||||
* this component to disqualify itself if it doesn't support the
|
||||
* required level of thread support.
|
||||
*/
|
||||
int ompi_coll_spacc_init_query(bool enable_progress_threads,
|
||||
bool enable_mpi_threads)
|
||||
{
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
/*
|
||||
* Invoked when there's a new communicator that has been created.
|
||||
* Look at the communicator and decide which set of functions and
|
||||
* priority we want to return.
|
||||
*/
|
||||
mca_coll_base_module_t *ompi_coll_spacc_comm_query(
|
||||
struct ompi_communicator_t *comm, int *priority)
|
||||
{
|
||||
mca_coll_spacc_module_t *spacc_module;
|
||||
|
||||
opal_output_verbose(30, mca_coll_spacc_stream, "coll:spacc:module_comm_query called");
|
||||
|
||||
if (OMPI_COMM_IS_INTER(comm)) {
|
||||
opal_output_verbose(20, mca_coll_spacc_stream,
|
||||
"coll:spacc:module_comm_query: spacc does not support inter-communicators");
|
||||
*priority = 0;
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (OMPI_COMM_IS_INTRA(comm) && ompi_comm_size(comm) < 2) {
|
||||
*priority = 0;
|
||||
return NULL;
|
||||
}
|
||||
|
||||
spacc_module = OBJ_NEW(mca_coll_spacc_module_t);
|
||||
if (NULL == spacc_module)
|
||||
return NULL;
|
||||
|
||||
*priority = mca_coll_spacc_priority;
|
||||
|
||||
spacc_module->super.coll_module_enable = spacc_module_enable;
|
||||
spacc_module->super.ft_event = NULL;
|
||||
spacc_module->super.coll_allgather = NULL;
|
||||
spacc_module->super.coll_allgatherv = NULL;
|
||||
spacc_module->super.coll_allreduce = mca_coll_spacc_allreduce_intra_redscat_allgather;
|
||||
spacc_module->super.coll_alltoall = NULL;
|
||||
spacc_module->super.coll_alltoallv = NULL;
|
||||
spacc_module->super.coll_alltoallw = NULL;
|
||||
spacc_module->super.coll_barrier = NULL;
|
||||
spacc_module->super.coll_bcast = NULL;
|
||||
spacc_module->super.coll_exscan = NULL;
|
||||
spacc_module->super.coll_gather = NULL;
|
||||
spacc_module->super.coll_gatherv = NULL;
|
||||
spacc_module->super.coll_reduce = mca_coll_spacc_reduce_intra_redscat_gather;
|
||||
spacc_module->super.coll_reduce_scatter_block = NULL;
|
||||
spacc_module->super.coll_reduce_scatter = NULL;
|
||||
spacc_module->super.coll_scan = NULL;
|
||||
spacc_module->super.coll_scatter = NULL;
|
||||
spacc_module->super.coll_scatterv = NULL;
|
||||
|
||||
return &(spacc_module->super);
|
||||
}
|
||||
|
||||
/*
|
||||
* Init module on the communicator
|
||||
*/
|
||||
static int spacc_module_enable(mca_coll_base_module_t *module,
|
||||
struct ompi_communicator_t *comm)
|
||||
{
|
||||
opal_output_verbose(30, mca_coll_spacc_stream, "coll:spacc:module_enable called");
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
static void mca_coll_spacc_module_construct(mca_coll_spacc_module_t *module)
|
||||
{
|
||||
/* mca_coll_spacc_module_t *spacc_module = (mca_coll_spacc_module_t*)module; */
|
||||
}
|
||||
|
||||
OBJ_CLASS_INSTANCE(mca_coll_spacc_module_t, mca_coll_base_module_t,
|
||||
mca_coll_spacc_module_construct, NULL);
|
||||
@ -1,416 +0,0 @@
|
||||
/*
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "coll_spacc.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include "ompi/constants.h"
|
||||
#include "opal/util/bit_ops.h"
|
||||
#include "ompi/datatype/ompi_datatype.h"
|
||||
#include "ompi/communicator/communicator.h"
|
||||
#include "ompi/mca/coll/coll.h"
|
||||
#include "ompi/mca/coll/base/coll_base_functions.h"
|
||||
#include "ompi/mca/coll/base/coll_tags.h"
|
||||
#include "ompi/mca/coll/base/coll_base_util.h"
|
||||
#include "ompi/mca/pml/pml.h"
|
||||
#include "ompi/op/op.h"
|
||||
|
||||
/*
|
||||
* mca_coll_spacc_reduce_intra_redscat_gather
|
||||
*
|
||||
* Function: Reduce using Rabenseifner's algorithm.
|
||||
* Accepts: Same arguments as MPI_Reduce
|
||||
* Returns: MPI_SUCCESS or error code
|
||||
*
|
||||
* Description: an implementation of Rabenseifner's reduce algorithm [1, 2].
|
||||
* [1] Rajeev Thakur, Rolf Rabenseifner and William Gropp.
|
||||
* Optimization of Collective Communication Operations in MPICH //
|
||||
* The Int. Journal of High Performance Computing Applications. Vol 19,
|
||||
* Issue 1, pp. 49--66.
|
||||
* [2] http://www.hlrs.de/mpi/myreduce.html.
|
||||
*
|
||||
* This algorithm is a combination of a reduce-scatter implemented with
|
||||
* recursive vector halving and recursive distance doubling, followed either
|
||||
* by a binomial tree gather [1].
|
||||
*
|
||||
* Step 1. If the number of processes is not a power of two, reduce it to
|
||||
* the nearest lower power of two (p' = 2^{\floor{\log_2 p}})
|
||||
* by removing r = p - p' extra processes as follows. In the first 2r processes
|
||||
* (ranks 0 to 2r - 1), all the even ranks send the second half of the input
|
||||
* vector to their right neighbor (rank + 1), and all the odd ranks send
|
||||
* the first half of the input vector to their left neighbor (rank - 1).
|
||||
* The even ranks compute the reduction on the first half of the vector and
|
||||
* the odd ranks compute the reduction on the second half. The odd ranks then
|
||||
* send the result to their left neighbors (the even ranks). As a result,
|
||||
* the even ranks among the first 2r processes now contain the reduction with
|
||||
* the input vector on their right neighbors (the odd ranks). These odd ranks
|
||||
* do not participate in the rest of the algorithm, which leaves behind
|
||||
* a power-of-two number of processes. The first r even-ranked processes and
|
||||
* the last p - 2r processes are now renumbered from 0 to p' - 1.
|
||||
*
|
||||
* Step 2. The remaining processes now perform a reduce-scatter by using
|
||||
* recursive vector halving and recursive distance doubling. The even-ranked
|
||||
* processes send the second half of their buffer to rank + 1 and the odd-ranked
|
||||
* processes send the first half of their buffer to rank - 1. All processes
|
||||
* then compute the reduction between the local buffer and the received buffer.
|
||||
* In the next log_2(p') - 1 steps, the buffers are recursively halved, and the
|
||||
* distance is doubled. At the end, each of the p' processes has 1 / p' of the
|
||||
* total reduction result.
|
||||
*
|
||||
* Step 3. A binomial tree gather is performed by using recursive vector
|
||||
* doubling and distance halving. In the non-power-of-two case, if the root
|
||||
* happens to be one of those odd-ranked processes that would normally
|
||||
* be removed in the first step, then the role of this process and process 0
|
||||
* are interchanged.
|
||||
*
|
||||
* Limitations:
|
||||
* count >= 2^{\floor{\log_2 p}}
|
||||
* commutative operations only
|
||||
* intra-communicators only
|
||||
*
|
||||
* Memory requirements (per process):
|
||||
* rank != root: 2 * count * typesize + 4 * log_2(p) * sizeof(int) = O(count)
|
||||
* rank == root: count * typesize + 4 * log_2(p) * sizeof(int) = O(count)
|
||||
*
|
||||
* Recommendations: root = 0, otherwise it is required additional steps
|
||||
* in the root process.
|
||||
*/
|
||||
int mca_coll_spacc_reduce_intra_redscat_gather(
|
||||
const void *sbuf, void *rbuf, int count, struct ompi_datatype_t *dtype,
|
||||
struct ompi_op_t *op, int root, struct ompi_communicator_t *comm,
|
||||
mca_coll_base_module_t *module)
|
||||
{
|
||||
int comm_size = ompi_comm_size(comm);
|
||||
int rank = ompi_comm_rank(comm);
|
||||
|
||||
opal_output_verbose(30, mca_coll_spacc_stream,
|
||||
"coll:spacc:reduce_intra_redscat_gather: rank %d/%d, root %d",
|
||||
rank, comm_size, root);
|
||||
|
||||
/* Find nearest power-of-two less than or equal to comm_size */
|
||||
int nsteps = opal_hibit(comm_size, comm->c_cube_dim + 1); /* ilog2(comm_size) */
|
||||
assert(nsteps >= 0);
|
||||
int nprocs_pof2 = 1 << nsteps; /* flp2(comm_size) */
|
||||
|
||||
if (count < nprocs_pof2 || !ompi_op_is_commute(op)) {
|
||||
opal_output_verbose(20, mca_coll_spacc_stream,
|
||||
"coll:spacc:reduce_intra_redscat_gather: rank %d/%d count %d switching to base reduce",
|
||||
rank, comm_size, count);
|
||||
return ompi_coll_base_reduce_intra_basic_linear(sbuf, rbuf, count, dtype,
|
||||
op, root, comm, module);
|
||||
}
|
||||
|
||||
int err = MPI_SUCCESS;
|
||||
int *rindex = NULL, *rcount = NULL, *sindex = NULL, *scount = NULL;
|
||||
|
||||
ptrdiff_t lb, extent, dsize, gap;
|
||||
ompi_datatype_get_extent(dtype, &lb, &extent);
|
||||
dsize = opal_datatype_span(&dtype->super, count, &gap);
|
||||
|
||||
/* Temporary buffer for receiving messages */
|
||||
char *tmp_buf = NULL;
|
||||
char *tmp_buf_raw = (char *)malloc(dsize);
|
||||
if (NULL == tmp_buf_raw)
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
tmp_buf = tmp_buf_raw - gap;
|
||||
|
||||
char *rbuf_raw = NULL;
|
||||
if (rank != root) {
|
||||
rbuf_raw = (char *)malloc(dsize);
|
||||
if (NULL == rbuf_raw) {
|
||||
err = OMPI_ERR_OUT_OF_RESOURCE;
|
||||
goto cleanup_and_return;
|
||||
}
|
||||
rbuf = rbuf_raw - gap;
|
||||
}
|
||||
|
||||
if ((rank != root) || (sbuf != MPI_IN_PLACE)) {
|
||||
/* Copy sbuf to rbuf */
|
||||
err = ompi_datatype_copy_content_same_ddt(dtype, count, (char *)rbuf,
|
||||
(char *)sbuf);
|
||||
}
|
||||
|
||||
/*
|
||||
* Step 1. Reduce the number of processes to the nearest lower power of two
|
||||
* p' = 2^{\floor{\log_2 p}} by removing r = p - p' processes.
|
||||
* 1. In the first 2r processes (ranks 0 to 2r - 1), all the even ranks send
|
||||
* the second half of the input vector to their right neighbor (rank + 1)
|
||||
* and all the odd ranks send the first half of the input vector to their
|
||||
* left neighbor (rank - 1).
|
||||
* 2. All 2r processes compute the reduction on their half.
|
||||
* 3. The odd ranks then send the result to their left neighbors
|
||||
* (the even ranks).
|
||||
*
|
||||
* The even ranks (0 to 2r - 1) now contain the reduction with the input
|
||||
* vector on their right neighbors (the odd ranks). The first r even
|
||||
* processes and the p - 2r last processes are renumbered from
|
||||
* 0 to 2^{\floor{\log_2 p}} - 1. These odd ranks do not participate in the
|
||||
* rest of the algorithm.
|
||||
*/
|
||||
|
||||
int vrank, step, wsize;
|
||||
int nprocs_rem = comm_size - nprocs_pof2;
|
||||
|
||||
if (rank < 2 * nprocs_rem) {
|
||||
int count_lhalf = count / 2;
|
||||
int count_rhalf = count - count_lhalf;
|
||||
|
||||
if (rank % 2 != 0) {
|
||||
/*
|
||||
* Odd process -- exchange with rank - 1
|
||||
* Send the left half of the input vector to the left neighbor,
|
||||
* Recv the right half of the input vector from the left neighbor
|
||||
*/
|
||||
err = ompi_coll_base_sendrecv(rbuf, count_lhalf, dtype, rank - 1,
|
||||
MCA_COLL_BASE_TAG_REDUCE,
|
||||
(char *)tmp_buf + (ptrdiff_t)count_lhalf * extent,
|
||||
count_rhalf, dtype, rank - 1,
|
||||
MCA_COLL_BASE_TAG_REDUCE, comm,
|
||||
MPI_STATUS_IGNORE, rank);
|
||||
if (MPI_SUCCESS != err) { goto cleanup_and_return; }
|
||||
|
||||
/* Reduce on the right half of the buffers (result in rbuf) */
|
||||
ompi_op_reduce(op, (char *)tmp_buf + (ptrdiff_t)count_lhalf * extent,
|
||||
(char *)rbuf + count_lhalf * extent, count_rhalf, dtype);
|
||||
|
||||
/* Send the right half to the left neighbor */
|
||||
err = MCA_PML_CALL(send((char *)rbuf + (ptrdiff_t)count_lhalf * extent,
|
||||
count_rhalf, dtype, rank - 1,
|
||||
MCA_COLL_BASE_TAG_REDUCE,
|
||||
MCA_PML_BASE_SEND_STANDARD, comm));
|
||||
if (MPI_SUCCESS != err) { goto cleanup_and_return; }
|
||||
|
||||
/* This process does not pariticipate in recursive doubling phase */
|
||||
vrank = -1;
|
||||
|
||||
} else {
|
||||
/*
|
||||
* Even process -- exchange with rank + 1
|
||||
* Send the right half of the input vector to the right neighbor,
|
||||
* Recv the left half of the input vector from the right neighbor
|
||||
*/
|
||||
err = ompi_coll_base_sendrecv((char *)rbuf + (ptrdiff_t)count_lhalf * extent,
|
||||
count_rhalf, dtype, rank + 1,
|
||||
MCA_COLL_BASE_TAG_REDUCE,
|
||||
tmp_buf, count_lhalf, dtype, rank + 1,
|
||||
MCA_COLL_BASE_TAG_REDUCE, comm,
|
||||
MPI_STATUS_IGNORE, rank);
|
||||
if (MPI_SUCCESS != err) { goto cleanup_and_return; }
|
||||
|
||||
/* Reduce on the right half of the buffers (result in rbuf) */
|
||||
ompi_op_reduce(op, tmp_buf, rbuf, count_lhalf, dtype);
|
||||
|
||||
/* Recv the right half from the right neighbor */
|
||||
err = MCA_PML_CALL(recv((char *)rbuf + (ptrdiff_t)count_lhalf * extent,
|
||||
count_rhalf, dtype, rank + 1,
|
||||
MCA_COLL_BASE_TAG_REDUCE, comm,
|
||||
MPI_STATUS_IGNORE));
|
||||
if (MPI_SUCCESS != err) { goto cleanup_and_return; }
|
||||
|
||||
vrank = rank / 2;
|
||||
}
|
||||
} else { /* rank >= 2 * nprocs_rem */
|
||||
vrank = rank - nprocs_rem;
|
||||
}
|
||||
|
||||
/*
|
||||
* Step 2. Reduce-scatter implemented with recursive vector halving and
|
||||
* recursive distance doubling. We have p' = 2^{\floor{\log_2 p}}
|
||||
* power-of-two number of processes with new ranks (vrank) and result in rbuf.
|
||||
*
|
||||
* The even-ranked processes send the right half of their buffer to rank + 1
|
||||
* and the odd-ranked processes send the left half of their buffer to
|
||||
* rank - 1. All processes then compute the reduction between the local
|
||||
* buffer and the received buffer. In the next \log_2(p') - 1 steps, the
|
||||
* buffers are recursively halved, and the distance is doubled. At the end,
|
||||
* each of the p' processes has 1 / p' of the total reduction result.
|
||||
*/
|
||||
|
||||
rindex = malloc(sizeof(*rindex) * nsteps); /* O(\log_2(p)) */
|
||||
sindex = malloc(sizeof(*sindex) * nsteps);
|
||||
rcount = malloc(sizeof(*rcount) * nsteps);
|
||||
scount = malloc(sizeof(*scount) * nsteps);
|
||||
if (NULL == rindex || NULL == sindex || NULL == rcount || NULL == scount) {
|
||||
err = OMPI_ERR_OUT_OF_RESOURCE;
|
||||
goto cleanup_and_return;
|
||||
}
|
||||
|
||||
if (vrank != -1) {
|
||||
step = 0;
|
||||
wsize = count;
|
||||
sindex[0] = rindex[0] = 0;
|
||||
|
||||
for (int mask = 1; mask < nprocs_pof2; mask <<= 1) {
|
||||
/*
|
||||
* On each iteration: rindex[step] = sindex[step] -- begining of the
|
||||
* current window. Length of the current window is storded in wsize.
|
||||
*/
|
||||
int vdest = vrank ^ mask;
|
||||
/* Translate vdest virtual rank to real rank */
|
||||
int dest = (vdest < nprocs_rem) ? vdest * 2 : vdest + nprocs_rem;
|
||||
|
||||
if (rank < dest) {
|
||||
/*
|
||||
* Recv into the left half of the current window, send the right
|
||||
* half of the window to the peer (perform reduce on the left
|
||||
* half of the current window)
|
||||
*/
|
||||
rcount[step] = wsize / 2;
|
||||
scount[step] = wsize - rcount[step];
|
||||
sindex[step] = rindex[step] + rcount[step];
|
||||
} else {
|
||||
/*
|
||||
* Recv into the right half of the current window, send the left
|
||||
* half of the window to the peer (perform reduce on the right
|
||||
* half of the current window)
|
||||
*/
|
||||
scount[step] = wsize / 2;
|
||||
rcount[step] = wsize - scount[step];
|
||||
rindex[step] = sindex[step] + scount[step];
|
||||
}
|
||||
|
||||
/* Send part of data from the rbuf, recv into the tmp_buf */
|
||||
err = ompi_coll_base_sendrecv((char *)rbuf + (ptrdiff_t)sindex[step] * extent,
|
||||
scount[step], dtype, dest,
|
||||
MCA_COLL_BASE_TAG_REDUCE,
|
||||
(char *)tmp_buf + (ptrdiff_t)rindex[step] * extent,
|
||||
rcount[step], dtype, dest,
|
||||
MCA_COLL_BASE_TAG_REDUCE, comm,
|
||||
MPI_STATUS_IGNORE, rank);
|
||||
if (MPI_SUCCESS != err) { goto cleanup_and_return; }
|
||||
|
||||
/* Local reduce: rbuf[] = tmp_buf[] <op> rbuf[] */
|
||||
ompi_op_reduce(op, (char *)tmp_buf + (ptrdiff_t)rindex[step] * extent,
|
||||
(char *)rbuf + (ptrdiff_t)rindex[step] * extent,
|
||||
rcount[step], dtype);
|
||||
|
||||
/* Move the current window to the received message */
|
||||
if (step + 1 < nsteps) {
|
||||
rindex[step + 1] = rindex[step];
|
||||
sindex[step + 1] = rindex[step];
|
||||
wsize = rcount[step];
|
||||
step++;
|
||||
}
|
||||
}
|
||||
}
|
||||
/*
|
||||
* Assertion: each process has 1 / p' of the total reduction result:
|
||||
* rcount[nsteps - 1] elements in the rbuf[rindex[nsteps - 1], ...].
|
||||
*/
|
||||
|
||||
/*
|
||||
* Setup the root process for gather operation.
|
||||
* Case 1: root < 2r and root is odd -- root process was excluded on step 1
|
||||
* Recv data from process 0, vroot = 0, vrank = 0
|
||||
* Case 2: root < 2r and root is even: vroot = root / 2
|
||||
* Case 3: root >= 2r: vroot = root - r
|
||||
*/
|
||||
int vroot = 0;
|
||||
if (root < 2 * nprocs_rem) {
|
||||
if (root % 2 != 0) {
|
||||
vroot = 0;
|
||||
if (rank == root) {
|
||||
/*
|
||||
* Case 1: root < 2r and root is odd -- root process was
|
||||
* excluded on step 1 (newrank == -1).
|
||||
* Recv a data from the process 0.
|
||||
*/
|
||||
rindex[0] = 0;
|
||||
step = 0, wsize = count;
|
||||
for (int mask = 1; mask < nprocs_pof2; mask *= 2) {
|
||||
rcount[step] = wsize / 2;
|
||||
scount[step] = wsize - rcount[step];
|
||||
rindex[step] = 0;
|
||||
sindex[step] = rcount[step];
|
||||
step++;
|
||||
wsize /= 2;
|
||||
}
|
||||
|
||||
err = MCA_PML_CALL(recv(rbuf, rcount[nsteps - 1], dtype, 0,
|
||||
MCA_COLL_BASE_TAG_REDUCE, comm,
|
||||
MPI_STATUS_IGNORE));
|
||||
if (MPI_SUCCESS != err) { goto cleanup_and_return; }
|
||||
vrank = 0;
|
||||
|
||||
} else if (vrank == 0) {
|
||||
/* Send a data to the root */
|
||||
err = MCA_PML_CALL(send(rbuf, rcount[nsteps - 1], dtype, root,
|
||||
MCA_COLL_BASE_TAG_REDUCE,
|
||||
MCA_PML_BASE_SEND_STANDARD, comm));
|
||||
if (MPI_SUCCESS != err) { goto cleanup_and_return; }
|
||||
vrank = -1;
|
||||
}
|
||||
} else {
|
||||
/* Case 2: root < 2r and a root is even: vroot = root / 2 */
|
||||
vroot = root / 2;
|
||||
}
|
||||
} else {
|
||||
/* Case 3: root >= 2r: newroot = root - r */
|
||||
vroot = root - nprocs_rem;
|
||||
}
|
||||
|
||||
/*
|
||||
* Step 3. Gather result at the vroot by the binomial tree algorithm.
|
||||
* Each process has 1 / p' of the total reduction result:
|
||||
* rcount[nsteps - 1] elements in the rbuf[rindex[nsteps - 1], ...].
|
||||
* All exchanges are executed in reverse order relative
|
||||
* to recursive doubling (previous step).
|
||||
*/
|
||||
|
||||
if (vrank != -1) {
|
||||
int vdest_tree, vroot_tree;
|
||||
step = nsteps - 1; /* step = ilog2(p') - 1 */
|
||||
|
||||
for (int mask = nprocs_pof2 >> 1; mask > 0; mask >>= 1) {
|
||||
int vdest = vrank ^ mask;
|
||||
/* Translate vdest virtual rank to real rank */
|
||||
int dest = (vdest < nprocs_rem) ? vdest * 2 : vdest + nprocs_rem;
|
||||
if ((vdest == 0) && (root < 2 * nprocs_rem) && (root % 2 != 0))
|
||||
dest = root;
|
||||
|
||||
vdest_tree = vdest >> step;
|
||||
vdest_tree <<= step;
|
||||
vroot_tree = vroot >> step;
|
||||
vroot_tree <<= step;
|
||||
if (vdest_tree == vroot_tree) {
|
||||
/* Send data from rbuf and exit */
|
||||
err = MCA_PML_CALL(send((char *)rbuf + (ptrdiff_t)rindex[step] * extent,
|
||||
rcount[step], dtype, dest,
|
||||
MCA_COLL_BASE_TAG_REDUCE,
|
||||
MCA_PML_BASE_SEND_STANDARD, comm));
|
||||
if (MPI_SUCCESS != err) { goto cleanup_and_return; }
|
||||
break;
|
||||
} else {
|
||||
/* Recv and continue */
|
||||
err = MCA_PML_CALL(recv((char *)rbuf + (ptrdiff_t)sindex[step] * extent,
|
||||
scount[step], dtype, dest,
|
||||
MCA_COLL_BASE_TAG_REDUCE, comm,
|
||||
MPI_STATUS_IGNORE));
|
||||
if (MPI_SUCCESS != err) { goto cleanup_and_return; }
|
||||
}
|
||||
step--;
|
||||
}
|
||||
}
|
||||
|
||||
cleanup_and_return:
|
||||
if (NULL != tmp_buf_raw)
|
||||
free(tmp_buf_raw);
|
||||
if (NULL != rbuf_raw)
|
||||
free(rbuf_raw);
|
||||
if (NULL != rindex)
|
||||
free(rindex);
|
||||
if (NULL != sindex)
|
||||
free(sindex);
|
||||
if (NULL != rcount)
|
||||
free(rcount);
|
||||
if (NULL != scount)
|
||||
free(scount);
|
||||
|
||||
return err;
|
||||
}
|
||||
@ -11,6 +11,8 @@
|
||||
# All rights reserved.
|
||||
# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
|
||||
# Copyright (c) 2017 IBM Corporation. All rights reserved.
|
||||
# Copyright (c) 2018 Research Organization for Information Science
|
||||
# and Technology (RIST). All rights reserved.
|
||||
# $COPYRIGHT$
|
||||
#
|
||||
# Additional copyrights may follow
|
||||
@ -39,7 +41,9 @@ sources = \
|
||||
coll_tuned_bcast_decision.c \
|
||||
coll_tuned_reduce_scatter_decision.c \
|
||||
coll_tuned_scatter_decision.c \
|
||||
coll_tuned_reduce_scatter_block_decision.c
|
||||
coll_tuned_reduce_scatter_block_decision.c \
|
||||
coll_tuned_exscan_decision.c \
|
||||
coll_tuned_scan_decision.c
|
||||
|
||||
# Make the output library in this directory, and name it either
|
||||
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
|
||||
|
||||
@ -3,8 +3,8 @@
|
||||
* Copyright (c) 2004-2015 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2015 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
* Copyright (c) 2015-2018 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
@ -162,6 +162,18 @@ int ompi_coll_tuned_scatter_intra_dec_dynamic(SCATTER_ARGS);
|
||||
int ompi_coll_tuned_scatter_intra_do_this(SCATTER_ARGS, int algorithm, int faninout, int segsize);
|
||||
int ompi_coll_tuned_scatter_intra_check_forced_init (coll_tuned_force_algorithm_mca_param_indices_t *mca_param_indices);
|
||||
|
||||
/* Exscan */
|
||||
int ompi_coll_tuned_exscan_intra_dec_fixed(EXSCAN_ARGS);
|
||||
int ompi_coll_tuned_exscan_intra_dec_dynamic(EXSCAN_ARGS);
|
||||
int ompi_coll_tuned_exscan_intra_do_this(EXSCAN_ARGS, int algorithm);
|
||||
int ompi_coll_tuned_exscan_intra_check_forced_init (coll_tuned_force_algorithm_mca_param_indices_t *mca_param_indices);
|
||||
|
||||
/* Scan */
|
||||
int ompi_coll_tuned_scan_intra_dec_fixed(SCAN_ARGS);
|
||||
int ompi_coll_tuned_scan_intra_dec_dynamic(SCAN_ARGS);
|
||||
int ompi_coll_tuned_scan_intra_do_this(SCAN_ARGS, int algorithm);
|
||||
int ompi_coll_tuned_scan_intra_check_forced_init (coll_tuned_force_algorithm_mca_param_indices_t *mca_param_indices);
|
||||
|
||||
int mca_coll_tuned_ft_event(int state);
|
||||
|
||||
struct mca_coll_tuned_component_t {
|
||||
|
||||
@ -3,8 +3,8 @@
|
||||
* Copyright (c) 2004-2017 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2015 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
* Copyright (c) 2015-2018 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
@ -41,6 +41,7 @@ static mca_base_var_enum_value_t allreduce_algorithms[] = {
|
||||
{3, "recursive_doubling"},
|
||||
{4, "ring"},
|
||||
{5, "segmented_ring"},
|
||||
{6, "rabenseifner"},
|
||||
{0, NULL}
|
||||
};
|
||||
|
||||
@ -142,6 +143,8 @@ int ompi_coll_tuned_allreduce_intra_do_this(const void *sbuf, void *rbuf, int co
|
||||
return ompi_coll_base_allreduce_intra_ring(sbuf, rbuf, count, dtype, op, comm, module);
|
||||
case (5):
|
||||
return ompi_coll_base_allreduce_intra_ring_segmented(sbuf, rbuf, count, dtype, op, comm, module, segsize);
|
||||
case (6):
|
||||
return ompi_coll_base_allreduce_intra_redscat_allgather(sbuf, rbuf, count, dtype, op, comm, module);
|
||||
} /* switch */
|
||||
OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:allreduce_intra_do_this attempt to select algorithm %d when only 0-%d is valid?",
|
||||
algorithm, ompi_coll_tuned_forced_max_algorithms[ALLREDUCE]));
|
||||
|
||||
@ -14,7 +14,7 @@
|
||||
* Copyright (c) 2008 Cisco Systems, Inc. All rights reserved.
|
||||
* Copyright (c) 2015 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2015 Research Organization for Information Science
|
||||
* Copyright (c) 2015-2018 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
@ -190,6 +190,8 @@ static int tuned_register(void)
|
||||
ompi_coll_tuned_reduce_scatter_block_intra_check_forced_init(&ompi_coll_tuned_forced_params[REDUCESCATTERBLOCK]);
|
||||
ompi_coll_tuned_gather_intra_check_forced_init(&ompi_coll_tuned_forced_params[GATHER]);
|
||||
ompi_coll_tuned_scatter_intra_check_forced_init(&ompi_coll_tuned_forced_params[SCATTER]);
|
||||
ompi_coll_tuned_exscan_intra_check_forced_init(&ompi_coll_tuned_forced_params[EXSCAN]);
|
||||
ompi_coll_tuned_scan_intra_check_forced_init(&ompi_coll_tuned_forced_params[SCAN]);
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
@ -10,7 +10,7 @@
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008 Sun Microsystems, Inc. All rights reserved.
|
||||
* Copyright (c) 2015 Research Organization for Information Science
|
||||
* Copyright (c) 2015-2018 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
@ -662,3 +662,89 @@ int ompi_coll_tuned_scatter_intra_dec_dynamic(const void *sbuf, int scount,
|
||||
rbuf, rcount, rdtype,
|
||||
root, comm, module);
|
||||
}
|
||||
|
||||
int ompi_coll_tuned_exscan_intra_dec_dynamic(const void *sbuf, void* rbuf, int count,
|
||||
struct ompi_datatype_t *dtype,
|
||||
struct ompi_op_t *op,
|
||||
struct ompi_communicator_t *comm,
|
||||
mca_coll_base_module_t *module)
|
||||
{
|
||||
mca_coll_tuned_module_t *tuned_module = (mca_coll_tuned_module_t*) module;
|
||||
|
||||
OPAL_OUTPUT((ompi_coll_tuned_stream,
|
||||
"ompi_coll_tuned_exscan_intra_dec_dynamic"));
|
||||
|
||||
/**
|
||||
* check to see if we have some filebased rules.
|
||||
*/
|
||||
if (tuned_module->com_rules[EXSCAN]) {
|
||||
int comsize, alg, faninout, segsize, max_requests;
|
||||
size_t dsize;
|
||||
|
||||
comsize = ompi_comm_size(comm);
|
||||
ompi_datatype_type_size (dtype, &dsize);
|
||||
dsize *= comsize;
|
||||
|
||||
alg = ompi_coll_tuned_get_target_method_params (tuned_module->com_rules[EXSCAN],
|
||||
dsize, &faninout, &segsize, &max_requests);
|
||||
|
||||
if (alg) {
|
||||
/* we have found a valid choice from the file based rules for this message size */
|
||||
return ompi_coll_tuned_exscan_intra_do_this (sbuf, rbuf, count, dtype,
|
||||
op, comm, module,
|
||||
alg);
|
||||
} /* found a method */
|
||||
} /*end if any com rules to check */
|
||||
|
||||
if (tuned_module->user_forced[EXSCAN].algorithm) {
|
||||
return ompi_coll_tuned_exscan_intra_do_this(sbuf, rbuf, count, dtype,
|
||||
op, comm, module,
|
||||
tuned_module->user_forced[EXSCAN].algorithm);
|
||||
}
|
||||
|
||||
return ompi_coll_base_exscan_intra_linear(sbuf, rbuf, count, dtype,
|
||||
op, comm, module);
|
||||
}
|
||||
|
||||
int ompi_coll_tuned_scan_intra_dec_dynamic(const void *sbuf, void* rbuf, int count,
|
||||
struct ompi_datatype_t *dtype,
|
||||
struct ompi_op_t *op,
|
||||
struct ompi_communicator_t *comm,
|
||||
mca_coll_base_module_t *module)
|
||||
{
|
||||
mca_coll_tuned_module_t *tuned_module = (mca_coll_tuned_module_t*) module;
|
||||
|
||||
OPAL_OUTPUT((ompi_coll_tuned_stream,
|
||||
"ompi_coll_tuned_scan_intra_dec_dynamic"));
|
||||
|
||||
/**
|
||||
* check to see if we have some filebased rules.
|
||||
*/
|
||||
if (tuned_module->com_rules[SCAN]) {
|
||||
int comsize, alg, faninout, segsize, max_requests;
|
||||
size_t dsize;
|
||||
|
||||
comsize = ompi_comm_size(comm);
|
||||
ompi_datatype_type_size (dtype, &dsize);
|
||||
dsize *= comsize;
|
||||
|
||||
alg = ompi_coll_tuned_get_target_method_params (tuned_module->com_rules[SCAN],
|
||||
dsize, &faninout, &segsize, &max_requests);
|
||||
|
||||
if (alg) {
|
||||
/* we have found a valid choice from the file based rules for this message size */
|
||||
return ompi_coll_tuned_scan_intra_do_this (sbuf, rbuf, count, dtype,
|
||||
op, comm, module,
|
||||
alg);
|
||||
} /* found a method */
|
||||
} /*end if any com rules to check */
|
||||
|
||||
if (tuned_module->user_forced[SCAN].algorithm) {
|
||||
return ompi_coll_tuned_scan_intra_do_this(sbuf, rbuf, count, dtype,
|
||||
op, comm, module,
|
||||
tuned_module->user_forced[SCAN].algorithm);
|
||||
}
|
||||
|
||||
return ompi_coll_base_scan_intra_linear(sbuf, rbuf, count, dtype,
|
||||
op, comm, module);
|
||||
}
|
||||
|
||||
104
ompi/mca/coll/tuned/coll_tuned_exscan_decision.c
Обычный файл
104
ompi/mca/coll/tuned/coll_tuned_exscan_decision.c
Обычный файл
@ -0,0 +1,104 @@
|
||||
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
|
||||
/*
|
||||
* Copyright (c) 2018 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include "ompi/constants.h"
|
||||
#include "ompi/datatype/ompi_datatype.h"
|
||||
#include "ompi/communicator/communicator.h"
|
||||
#include "ompi/mca/coll/coll.h"
|
||||
#include "ompi/mca/coll/base/coll_base_topo.h"
|
||||
#include "ompi/mca/coll/base/coll_tags.h"
|
||||
#include "ompi/mca/pml/pml.h"
|
||||
#include "ompi/op/op.h"
|
||||
#include "coll_tuned.h"
|
||||
|
||||
/* exscan algorithm variables */
|
||||
static int coll_tuned_exscan_forced_algorithm = 0;
|
||||
|
||||
/* valid values for coll_tuned_exscan_forced_algorithm */
|
||||
static mca_base_var_enum_value_t exscan_algorithms[] = {
|
||||
{0, "ignore"},
|
||||
{1, "linear"},
|
||||
{2, "recursive_doubling"},
|
||||
{0, NULL}
|
||||
};
|
||||
|
||||
/**
|
||||
* The following are used by dynamic and forced rules
|
||||
*
|
||||
* publish details of each algorithm and if its forced/fixed/locked in
|
||||
* as you add methods/algorithms you must update this and the query/map routines
|
||||
*
|
||||
* this routine is called by the component only
|
||||
* this makes sure that the mca parameters are set to their initial values and
|
||||
* perms module does not call this they call the forced_getvalues routine
|
||||
* instead.
|
||||
*/
|
||||
|
||||
int ompi_coll_tuned_exscan_intra_check_forced_init (coll_tuned_force_algorithm_mca_param_indices_t *mca_param_indices)
|
||||
{
|
||||
mca_base_var_enum_t*new_enum;
|
||||
int cnt;
|
||||
|
||||
for( cnt = 0; NULL != exscan_algorithms[cnt].string; cnt++ );
|
||||
ompi_coll_tuned_forced_max_algorithms[EXSCAN] = cnt;
|
||||
|
||||
(void) mca_base_component_var_register(&mca_coll_tuned_component.super.collm_version,
|
||||
"exscan_algorithm_count",
|
||||
"Number of exscan algorithms available",
|
||||
MCA_BASE_VAR_TYPE_INT, NULL, 0,
|
||||
MCA_BASE_VAR_FLAG_DEFAULT_ONLY,
|
||||
OPAL_INFO_LVL_5,
|
||||
MCA_BASE_VAR_SCOPE_CONSTANT,
|
||||
&ompi_coll_tuned_forced_max_algorithms[EXSCAN]);
|
||||
|
||||
/* MPI_T: This variable should eventually be bound to a communicator */
|
||||
coll_tuned_exscan_forced_algorithm = 0;
|
||||
(void) mca_base_var_enum_create("coll_tuned_exscan_algorithms", exscan_algorithms, &new_enum);
|
||||
mca_param_indices->algorithm_param_index =
|
||||
mca_base_component_var_register(&mca_coll_tuned_component.super.collm_version,
|
||||
"exscan_algorithm",
|
||||
"Which exscan algorithm is used. Can be locked down to choice of: 0 ignore, 1 linear, 2 recursive_doubling",
|
||||
MCA_BASE_VAR_TYPE_INT, new_enum, 0, MCA_BASE_VAR_FLAG_SETTABLE,
|
||||
OPAL_INFO_LVL_5,
|
||||
MCA_BASE_VAR_SCOPE_ALL,
|
||||
&coll_tuned_exscan_forced_algorithm);
|
||||
OBJ_RELEASE(new_enum);
|
||||
if (mca_param_indices->algorithm_param_index < 0) {
|
||||
return mca_param_indices->algorithm_param_index;
|
||||
}
|
||||
|
||||
return (MPI_SUCCESS);
|
||||
}
|
||||
|
||||
int ompi_coll_tuned_exscan_intra_do_this(const void *sbuf, void* rbuf, int count,
|
||||
struct ompi_datatype_t *dtype,
|
||||
struct ompi_op_t *op,
|
||||
struct ompi_communicator_t *comm,
|
||||
mca_coll_base_module_t *module,
|
||||
int algorithm)
|
||||
{
|
||||
OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:exscan_intra_do_this selected algorithm %d",
|
||||
algorithm));
|
||||
|
||||
switch (algorithm) {
|
||||
case (0):
|
||||
case (1): return ompi_coll_base_exscan_intra_linear(sbuf, rbuf, count, dtype,
|
||||
op, comm, module);
|
||||
case (2): return ompi_coll_base_exscan_intra_recursivedoubling(sbuf, rbuf, count, dtype,
|
||||
op, comm, module);
|
||||
} /* switch */
|
||||
OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:exscan_intra_do_this attempt to select algorithm %d when only 0-%d is valid?",
|
||||
algorithm, ompi_coll_tuned_forced_max_algorithms[EXSCAN]));
|
||||
return (MPI_ERR_ARG);
|
||||
}
|
||||
@ -11,6 +11,8 @@
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008 Sun Microsystems, Inc. All rights reserved.
|
||||
* Copyright (c) 2016 Intel, Inc. All rights reserved.
|
||||
* Copyright (c) 2018 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
@ -230,7 +232,7 @@ tuned_module_enable( mca_coll_base_module_t *module,
|
||||
COLL_TUNED_EXECUTE_IF_DYNAMIC(tuned_module, BCAST,
|
||||
tuned_module->super.coll_bcast = ompi_coll_tuned_bcast_intra_dec_dynamic);
|
||||
COLL_TUNED_EXECUTE_IF_DYNAMIC(tuned_module, EXSCAN,
|
||||
tuned_module->super.coll_exscan = NULL);
|
||||
tuned_module->super.coll_exscan = ompi_coll_tuned_exscan_intra_dec_dynamic);
|
||||
COLL_TUNED_EXECUTE_IF_DYNAMIC(tuned_module, GATHER,
|
||||
tuned_module->super.coll_gather = ompi_coll_tuned_gather_intra_dec_dynamic);
|
||||
COLL_TUNED_EXECUTE_IF_DYNAMIC(tuned_module, GATHERV,
|
||||
@ -242,7 +244,7 @@ tuned_module_enable( mca_coll_base_module_t *module,
|
||||
COLL_TUNED_EXECUTE_IF_DYNAMIC(tuned_module, REDUCESCATTERBLOCK,
|
||||
tuned_module->super.coll_reduce_scatter_block = ompi_coll_tuned_reduce_scatter_block_intra_dec_dynamic);
|
||||
COLL_TUNED_EXECUTE_IF_DYNAMIC(tuned_module, SCAN,
|
||||
tuned_module->super.coll_scan = NULL);
|
||||
tuned_module->super.coll_scan = ompi_coll_tuned_scan_intra_dec_dynamic);
|
||||
COLL_TUNED_EXECUTE_IF_DYNAMIC(tuned_module, SCATTER,
|
||||
tuned_module->super.coll_scatter = ompi_coll_tuned_scatter_intra_dec_dynamic);
|
||||
COLL_TUNED_EXECUTE_IF_DYNAMIC(tuned_module, SCATTERV,
|
||||
|
||||
@ -3,8 +3,8 @@
|
||||
* Copyright (c) 2004-2017 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2015 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
* Copyright (c) 2015-2018 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
@ -41,6 +41,7 @@ static mca_base_var_enum_value_t reduce_algorithms[] = {
|
||||
{4, "binary"},
|
||||
{5, "binomial"},
|
||||
{6, "in-order_binary"},
|
||||
{7, "rabenseifner"},
|
||||
{0, NULL}
|
||||
};
|
||||
|
||||
@ -79,7 +80,7 @@ int ompi_coll_tuned_reduce_intra_check_forced_init (coll_tuned_force_algorithm_m
|
||||
mca_param_indices->algorithm_param_index =
|
||||
mca_base_component_var_register(&mca_coll_tuned_component.super.collm_version,
|
||||
"reduce_algorithm",
|
||||
"Which reduce algorithm is used. Can be locked down to choice of: 0 ignore, 1 linear, 2 chain, 3 pipeline, 4 binary, 5 binomial, 6 in-order binary",
|
||||
"Which reduce algorithm is used. Can be locked down to choice of: 0 ignore, 1 linear, 2 chain, 3 pipeline, 4 binary, 5 binomial, 6 in-order binary, 7 rabenseifner",
|
||||
MCA_BASE_VAR_TYPE_INT, new_enum, 0, MCA_BASE_VAR_FLAG_SETTABLE,
|
||||
OPAL_INFO_LVL_5,
|
||||
MCA_BASE_VAR_SCOPE_ALL,
|
||||
@ -173,6 +174,8 @@ int ompi_coll_tuned_reduce_intra_do_this(const void *sbuf, void* rbuf, int count
|
||||
case (6): return ompi_coll_base_reduce_intra_in_order_binary(sbuf, rbuf, count, dtype,
|
||||
op, root, comm, module,
|
||||
segsize, max_requests);
|
||||
case (7): return ompi_coll_base_reduce_intra_redscat_gather(sbuf, rbuf, count, dtype,
|
||||
op, root, comm, module);
|
||||
} /* switch */
|
||||
OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:reduce_intra_do_this attempt to select algorithm %d when only 0-%d is valid?",
|
||||
algorithm, ompi_coll_tuned_forced_max_algorithms[REDUCE]));
|
||||
|
||||
104
ompi/mca/coll/tuned/coll_tuned_scan_decision.c
Обычный файл
104
ompi/mca/coll/tuned/coll_tuned_scan_decision.c
Обычный файл
@ -0,0 +1,104 @@
|
||||
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
|
||||
/*
|
||||
* Copyright (c) 2018 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include "ompi/constants.h"
|
||||
#include "ompi/datatype/ompi_datatype.h"
|
||||
#include "ompi/communicator/communicator.h"
|
||||
#include "ompi/mca/coll/coll.h"
|
||||
#include "ompi/mca/coll/base/coll_base_topo.h"
|
||||
#include "ompi/mca/coll/base/coll_tags.h"
|
||||
#include "ompi/mca/pml/pml.h"
|
||||
#include "ompi/op/op.h"
|
||||
#include "coll_tuned.h"
|
||||
|
||||
/* scan algorithm variables */
|
||||
static int coll_tuned_scan_forced_algorithm = 0;
|
||||
|
||||
/* valid values for coll_tuned_scan_forced_algorithm */
|
||||
static mca_base_var_enum_value_t scan_algorithms[] = {
|
||||
{0, "ignore"},
|
||||
{1, "linear"},
|
||||
{2, "recursive_doubling"},
|
||||
{0, NULL}
|
||||
};
|
||||
|
||||
/**
|
||||
* The following are used by dynamic and forced rules
|
||||
*
|
||||
* publish details of each algorithm and if its forced/fixed/locked in
|
||||
* as you add methods/algorithms you must update this and the query/map routines
|
||||
*
|
||||
* this routine is called by the component only
|
||||
* this makes sure that the mca parameters are set to their initial values and
|
||||
* perms module does not call this they call the forced_getvalues routine
|
||||
* instead.
|
||||
*/
|
||||
|
||||
int ompi_coll_tuned_scan_intra_check_forced_init (coll_tuned_force_algorithm_mca_param_indices_t *mca_param_indices)
|
||||
{
|
||||
mca_base_var_enum_t*new_enum;
|
||||
int cnt;
|
||||
|
||||
for( cnt = 0; NULL != scan_algorithms[cnt].string; cnt++ );
|
||||
ompi_coll_tuned_forced_max_algorithms[SCAN] = cnt;
|
||||
|
||||
(void) mca_base_component_var_register(&mca_coll_tuned_component.super.collm_version,
|
||||
"scan_algorithm_count",
|
||||
"Number of scan algorithms available",
|
||||
MCA_BASE_VAR_TYPE_INT, NULL, 0,
|
||||
MCA_BASE_VAR_FLAG_DEFAULT_ONLY,
|
||||
OPAL_INFO_LVL_5,
|
||||
MCA_BASE_VAR_SCOPE_CONSTANT,
|
||||
&ompi_coll_tuned_forced_max_algorithms[SCAN]);
|
||||
|
||||
/* MPI_T: This variable should eventually be bound to a communicator */
|
||||
coll_tuned_scan_forced_algorithm = 0;
|
||||
(void) mca_base_var_enum_create("coll_tuned_scan_algorithms", scan_algorithms, &new_enum);
|
||||
mca_param_indices->algorithm_param_index =
|
||||
mca_base_component_var_register(&mca_coll_tuned_component.super.collm_version,
|
||||
"scan_algorithm",
|
||||
"Which scan algorithm is used. Can be locked down to choice of: 0 ignore, 1 linear, 2 recursive_doubling",
|
||||
MCA_BASE_VAR_TYPE_INT, new_enum, 0, MCA_BASE_VAR_FLAG_SETTABLE,
|
||||
OPAL_INFO_LVL_5,
|
||||
MCA_BASE_VAR_SCOPE_ALL,
|
||||
&coll_tuned_scan_forced_algorithm);
|
||||
OBJ_RELEASE(new_enum);
|
||||
if (mca_param_indices->algorithm_param_index < 0) {
|
||||
return mca_param_indices->algorithm_param_index;
|
||||
}
|
||||
|
||||
return (MPI_SUCCESS);
|
||||
}
|
||||
|
||||
int ompi_coll_tuned_scan_intra_do_this(const void *sbuf, void* rbuf, int count,
|
||||
struct ompi_datatype_t *dtype,
|
||||
struct ompi_op_t *op,
|
||||
struct ompi_communicator_t *comm,
|
||||
mca_coll_base_module_t *module,
|
||||
int algorithm)
|
||||
{
|
||||
OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:scan_intra_do_this selected algorithm %d",
|
||||
algorithm));
|
||||
|
||||
switch (algorithm) {
|
||||
case (0):
|
||||
case (1): return ompi_coll_base_scan_intra_linear(sbuf, rbuf, count, dtype,
|
||||
op, comm, module);
|
||||
case (2): return ompi_coll_base_scan_intra_recursivedoubling(sbuf, rbuf, count, dtype,
|
||||
op, comm, module);
|
||||
} /* switch */
|
||||
OPAL_OUTPUT((ompi_coll_tuned_stream,"coll:tuned:scan_intra_do_this attempt to select algorithm %d when only 0-%d is valid?",
|
||||
algorithm, ompi_coll_tuned_forced_max_algorithms[SCAN]));
|
||||
return (MPI_ERR_ARG);
|
||||
}
|
||||
@ -53,12 +53,13 @@ foreach $id (sort {$a <=> $b} keys %phaseid) {
|
||||
|
||||
sub aggregate{
|
||||
$phase = $_[0];
|
||||
|
||||
#Aggregating all files of given phase in files array.This should be done
|
||||
# before creating $phase.prof to avoid adding $phase.prof to files array
|
||||
@files = glob ($phase."*");
|
||||
print "Building $phase.prof\n";
|
||||
|
||||
open OUT,">$phase.prof";
|
||||
|
||||
@files = glob ($phase."*");
|
||||
|
||||
foreach $file ( @files) {
|
||||
open IN,$file;
|
||||
|
||||
@ -47,6 +47,7 @@ typedef struct mca_io_ompio_local_io_array{
|
||||
|
||||
typedef struct mca_io_ompio_aggregator_data {
|
||||
int *disp_index, *sorted, *fview_count, n;
|
||||
int *max_disp_index;
|
||||
int **blocklen_per_process;
|
||||
MPI_Aint **displs_per_process, total_bytes, bytes_per_cycle, total_bytes_written;
|
||||
MPI_Comm comm;
|
||||
@ -499,6 +500,13 @@ int mca_fcoll_dynamic_gen2_file_write_all (mca_io_ompio_file_t *fh,
|
||||
goto exit;
|
||||
}
|
||||
|
||||
aggr_data[i]->max_disp_index = (int *)calloc (fh->f_procs_per_group, sizeof (int));
|
||||
if (NULL == aggr_data[i]->max_disp_index) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
ret = OMPI_ERR_OUT_OF_RESOURCE;
|
||||
goto exit;
|
||||
}
|
||||
|
||||
aggr_data[i]->blocklen_per_process = (int **)calloc (fh->f_procs_per_group, sizeof (int*));
|
||||
if (NULL == aggr_data[i]->blocklen_per_process) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
@ -679,6 +687,7 @@ exit :
|
||||
}
|
||||
|
||||
free (aggr_data[i]->disp_index);
|
||||
free (aggr_data[i]->max_disp_index);
|
||||
free (aggr_data[i]->global_buf);
|
||||
free (aggr_data[i]->prev_global_buf);
|
||||
for(l=0;l<aggr_data[i]->procs_per_group;l++){
|
||||
@ -791,16 +800,20 @@ static int shuffle_init ( int index, int cycles, int aggregator, int rank, mca_i
|
||||
|
||||
for(l=0;l<data->procs_per_group;l++){
|
||||
data->disp_index[l] = 1;
|
||||
|
||||
free(data->blocklen_per_process[l]);
|
||||
free(data->displs_per_process[l]);
|
||||
|
||||
data->blocklen_per_process[l] = (int *) calloc (1, sizeof(int));
|
||||
data->displs_per_process[l] = (MPI_Aint *) calloc (1, sizeof(MPI_Aint));
|
||||
if (NULL == data->displs_per_process[l] || NULL == data->blocklen_per_process[l]){
|
||||
opal_output (1, "OUT OF MEMORY for displs\n");
|
||||
ret = OMPI_ERR_OUT_OF_RESOURCE;
|
||||
goto exit;
|
||||
|
||||
if(data->max_disp_index[l] == 0) {
|
||||
data->blocklen_per_process[l] = (int *) calloc (INIT_LEN, sizeof(int));
|
||||
data->displs_per_process[l] = (MPI_Aint *) calloc (INIT_LEN, sizeof(MPI_Aint));
|
||||
if (NULL == data->displs_per_process[l] || NULL == data->blocklen_per_process[l]){
|
||||
opal_output (1, "OUT OF MEMORY for displs\n");
|
||||
ret = OMPI_ERR_OUT_OF_RESOURCE;
|
||||
goto exit;
|
||||
}
|
||||
data->max_disp_index[l] = INIT_LEN;
|
||||
}
|
||||
else {
|
||||
memset ( data->blocklen_per_process[l], 0, data->max_disp_index[l]*sizeof(int) );
|
||||
memset ( data->displs_per_process[l], 0, data->max_disp_index[l]*sizeof(MPI_Aint) );
|
||||
}
|
||||
}
|
||||
} /* (aggregator == rank */
|
||||
@ -871,15 +884,22 @@ static int shuffle_init ( int index, int cycles, int aggregator, int rank, mca_i
|
||||
(data->global_iov_array[data->sorted[data->current_index]].iov_len
|
||||
- data->bytes_remaining);
|
||||
|
||||
data->disp_index[data->n] += 1;
|
||||
|
||||
/* In this cases the length is consumed so allocating for
|
||||
next displacement and blocklength*/
|
||||
data->blocklen_per_process[data->n] = (int *) realloc
|
||||
((void *)data->blocklen_per_process[data->n], (data->disp_index[data->n]+1)*sizeof(int));
|
||||
data->displs_per_process[data->n] = (MPI_Aint *) realloc
|
||||
((void *)data->displs_per_process[data->n], (data->disp_index[data->n]+1)*sizeof(MPI_Aint));
|
||||
if ( data->disp_index[data->n] == data->max_disp_index[data->n] ) {
|
||||
data->max_disp_index[data->n] *= 2;
|
||||
data->blocklen_per_process[data->n] = (int *) realloc(
|
||||
(void *)data->blocklen_per_process[data->n],
|
||||
(data->max_disp_index[data->n])*sizeof(int));
|
||||
data->displs_per_process[data->n] = (MPI_Aint *) realloc(
|
||||
(void *)data->displs_per_process[data->n],
|
||||
(data->max_disp_index[data->n])*sizeof(MPI_Aint));
|
||||
}
|
||||
|
||||
data->blocklen_per_process[data->n][data->disp_index[data->n]] = 0;
|
||||
data->displs_per_process[data->n][data->disp_index[data->n]] = 0;
|
||||
data->disp_index[data->n] += 1;
|
||||
}
|
||||
if (data->procs_in_group[data->n] == rank) {
|
||||
bytes_sent += data->bytes_remaining;
|
||||
@ -887,7 +907,6 @@ static int shuffle_init ( int index, int cycles, int aggregator, int rank, mca_i
|
||||
data->current_index ++;
|
||||
data->bytes_to_write_in_cycle -= data->bytes_remaining;
|
||||
data->bytes_remaining = 0;
|
||||
// continue;
|
||||
}
|
||||
else {
|
||||
/* the remaining data from the previous cycle is larger than the
|
||||
@ -935,16 +954,22 @@ static int shuffle_init ( int index, int cycles, int aggregator, int rank, mca_i
|
||||
data->displs_per_process[data->n][data->disp_index[data->n] - 1] = (ptrdiff_t)
|
||||
data->global_iov_array[data->sorted[data->current_index]].iov_base;
|
||||
|
||||
data->disp_index[data->n] += 1;
|
||||
|
||||
/*realloc for next blocklength
|
||||
and assign this displacement and check for next displs as
|
||||
the total length of this entry has been consumed!*/
|
||||
data->blocklen_per_process[data->n] =
|
||||
(int *) realloc ((void *)data->blocklen_per_process[data->n], (data->disp_index[data->n]+1)*sizeof(int));
|
||||
data->displs_per_process[data->n] = (MPI_Aint *)realloc
|
||||
((void *)data->displs_per_process[data->n], (data->disp_index[data->n]+1)*sizeof(MPI_Aint));
|
||||
if ( data->disp_index[data->n] == data->max_disp_index[data->n] ) {
|
||||
data->max_disp_index[data->n] *= 2;
|
||||
data->blocklen_per_process[data->n] = (int *) realloc(
|
||||
(void *)data->blocklen_per_process[data->n],
|
||||
(data->max_disp_index[data->n]*sizeof(int)));
|
||||
data->displs_per_process[data->n] = (MPI_Aint *)realloc(
|
||||
(void *)data->displs_per_process[data->n],
|
||||
(data->max_disp_index[data->n]*sizeof(MPI_Aint)));
|
||||
}
|
||||
data->blocklen_per_process[data->n][data->disp_index[data->n]] = 0;
|
||||
data->displs_per_process[data->n][data->disp_index[data->n]] = 0;
|
||||
data->disp_index[data->n] += 1;
|
||||
}
|
||||
if (data->procs_in_group[data->n] == rank) {
|
||||
bytes_sent += data->global_iov_array[data->sorted[data->current_index]].iov_len;
|
||||
@ -952,7 +977,6 @@ static int shuffle_init ( int index, int cycles, int aggregator, int rank, mca_i
|
||||
data->bytes_to_write_in_cycle -=
|
||||
data->global_iov_array[data->sorted[data->current_index]].iov_len;
|
||||
data->current_index ++;
|
||||
// continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -34,6 +34,8 @@ AC_DEFUN([MCA_ompi_fs_lustre_CONFIG],[
|
||||
[$1],
|
||||
[$2])
|
||||
|
||||
OPAL_SUMMARY_ADD([[OMPIO File Systems]],[[Lustre]],[$1],[$fs_lustre_happy])
|
||||
|
||||
# substitute in the things needed to build lustre
|
||||
AC_SUBST([fs_lustre_CPPFLAGS])
|
||||
AC_SUBST([fs_lustre_LDFLAGS])
|
||||
|
||||
@ -30,6 +30,7 @@ AC_DEFUN([MCA_ompi_fs_pvfs2_CONFIG],[
|
||||
[fs_pvfs2_happy="yes"],
|
||||
[fs_pvfs2_happy="no"])
|
||||
|
||||
OPAL_SUMMARY_ADD([[OMPIO File Systems]],[[PVFS2/OrangeFS]],[$1],[$fs_pvfs2_happy])
|
||||
AS_IF([test "$fs_pvfs2_happy" = "yes"],
|
||||
[$1],
|
||||
[$2])
|
||||
|
||||
29
ompi/mca/fs/ufs/configure.m4
Обычный файл
29
ompi/mca/fs/ufs/configure.m4
Обычный файл
@ -0,0 +1,29 @@
|
||||
# -*- shell-script -*-
|
||||
#
|
||||
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
# University Research and Technology
|
||||
# Corporation. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
# of Tennessee Research Foundation. All rights
|
||||
# reserved.
|
||||
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
# University of Stuttgart. All rights reserved.
|
||||
# Copyright (c) 2004-2012 The Regents of the University of California.
|
||||
# All rights reserved.
|
||||
# Copyright (c) 2010-2014 Cisco Systems, Inc. All rights reserved.
|
||||
# Copyright (c) 2008-2018 University of Houston. All rights reserved.
|
||||
# $COPYRIGHT$
|
||||
#
|
||||
# Additional copyrights may follow
|
||||
#
|
||||
# $HEADER$
|
||||
#
|
||||
|
||||
# MCA_fbtl_posix_CONFIG(action-if-can-compile,
|
||||
# [action-if-cant-compile])
|
||||
# ------------------------------------------------
|
||||
AC_DEFUN([MCA_ompi_fs_ufs_CONFIG],[
|
||||
AC_CONFIG_FILES([ompi/mca/fs/ufs/Makefile])
|
||||
|
||||
OPAL_SUMMARY_ADD([[OMPIO File Systems]],[[Generic Unix FS]],[$1],[yes])
|
||||
])dnl
|
||||
@ -164,7 +164,7 @@ void ADIOI_NFS_ReadStrided(ADIO_File fd, void *buf, int count,
|
||||
ADIO_Offset abs_off_in_filetype=0;
|
||||
int req_len, partial_read;
|
||||
MPI_Count filetype_size, etype_size, buftype_size;
|
||||
MPI_Aint filetype_extent, buftype_extent;
|
||||
MPI_Aint filetype_extent, buftype_extent, lb;
|
||||
int buf_count, buftype_is_contig, filetype_is_contig;
|
||||
ADIO_Offset userbuf_off;
|
||||
ADIO_Offset off, req_off, disp, end_offset=0, readbuf_off, start_off;
|
||||
@ -187,9 +187,9 @@ void ADIOI_NFS_ReadStrided(ADIO_File fd, void *buf, int count,
|
||||
return;
|
||||
}
|
||||
|
||||
MPI_Type_extent(fd->filetype, &filetype_extent);
|
||||
MPI_Type_get_extent(fd->filetype, &lb, &filetype_extent);
|
||||
MPI_Type_size_x(datatype, &buftype_size);
|
||||
MPI_Type_extent(datatype, &buftype_extent);
|
||||
MPI_Type_get_extent(datatype, &lb, &buftype_extent);
|
||||
etype_size = fd->etype_size;
|
||||
|
||||
bufsize = buftype_size * count;
|
||||
|
||||
@ -267,7 +267,7 @@ void ADIOI_NFS_WriteStrided(ADIO_File fd, const void *buf, int count,
|
||||
ADIO_Offset abs_off_in_filetype=0;
|
||||
int req_len;
|
||||
MPI_Count filetype_size, etype_size, buftype_size;
|
||||
MPI_Aint filetype_extent, buftype_extent;
|
||||
MPI_Aint filetype_extent, buftype_extent, lb;
|
||||
int buf_count, buftype_is_contig, filetype_is_contig;
|
||||
ADIO_Offset userbuf_off;
|
||||
ADIO_Offset off, req_off, disp, end_offset=0, writebuf_off, start_off;
|
||||
@ -289,9 +289,9 @@ void ADIOI_NFS_WriteStrided(ADIO_File fd, const void *buf, int count,
|
||||
return;
|
||||
}
|
||||
|
||||
MPI_Type_extent(fd->filetype, &filetype_extent);
|
||||
MPI_Type_get_extent(fd->filetype, &lb, &filetype_extent);
|
||||
MPI_Type_size_x(datatype, &buftype_size);
|
||||
MPI_Type_extent(datatype, &buftype_extent);
|
||||
MPI_Type_get_extent(datatype, &lb, &buftype_extent);
|
||||
etype_size = fd->etype_size;
|
||||
|
||||
bufsize = buftype_size * count;
|
||||
|
||||
@ -30,7 +30,7 @@ ADIO_Offset ADIOI_TESTFS_SeekIndividual(ADIO_File fd, ADIO_Offset offset,
|
||||
int size_in_filetype;
|
||||
int filetype_is_contig;
|
||||
MPI_Count filetype_size;
|
||||
MPI_Aint etype_size, filetype_extent;
|
||||
MPI_Aint etype_size, filetype_extent, lb;
|
||||
|
||||
*error_code = MPI_SUCCESS;
|
||||
|
||||
@ -47,7 +47,7 @@ ADIO_Offset ADIOI_TESTFS_SeekIndividual(ADIO_File fd, ADIO_Offset offset,
|
||||
flat_file = ADIOI_Flatlist;
|
||||
while (flat_file->type != fd->filetype) flat_file = flat_file->next;
|
||||
|
||||
MPI_Type_extent(fd->filetype, &filetype_extent);
|
||||
MPI_Type_get_extent(fd->filetype, &lb, &filetype_extent);
|
||||
MPI_Type_size_x(fd->filetype, &filetype_size);
|
||||
if ( ! filetype_size ) {
|
||||
*error_code = MPI_SUCCESS;
|
||||
|
||||
@ -237,18 +237,18 @@ void ADIOI_Calc_file_realms_fsize (ADIO_File fd, int nprocs_for_coll,
|
||||
void ADIOI_Create_fr_simpletype (int size, int nprocs_for_coll,
|
||||
MPI_Datatype *simpletype)
|
||||
{
|
||||
int count=2, blocklens[2];
|
||||
MPI_Aint indices[2];
|
||||
MPI_Datatype old_types[2];
|
||||
int count=1, blocklens[1];
|
||||
MPI_Aint indices[1];
|
||||
MPI_Datatype old_types[1];
|
||||
MPI_Datatype inttype;
|
||||
|
||||
blocklens[0] = size;
|
||||
blocklens[1] = 1;
|
||||
indices[0] = 0;
|
||||
indices[1] = size*nprocs_for_coll;
|
||||
old_types[0] = MPI_BYTE;
|
||||
old_types[1] = MPI_UB;
|
||||
|
||||
MPI_Type_struct (count, blocklens, indices, old_types, simpletype);
|
||||
MPI_Type_create_struct (count, blocklens, indices, old_types, &inttype);
|
||||
MPI_Type_create_resized (inttype, 0, size*nprocs_for_coll, simpletype);
|
||||
MPI_Type_free (&inttype);
|
||||
|
||||
MPI_Type_commit (simpletype);
|
||||
}
|
||||
|
||||
@ -279,7 +279,7 @@ static inline int get_next_fr_off(ADIO_File fd,
|
||||
ADIO_Offset *fr_next_off_p,
|
||||
ADIO_Offset *fr_max_len_p)
|
||||
{
|
||||
MPI_Aint fr_extent = -1;
|
||||
MPI_Aint fr_extent = -1, lb;
|
||||
ADIO_Offset tmp_off, off_rem;
|
||||
ADIOI_Flatlist_node *fr_node_p = ADIOI_Flatlist;
|
||||
int i = -1, fr_dtype_ct = 0;
|
||||
@ -299,7 +299,7 @@ static inline int get_next_fr_off(ADIO_File fd,
|
||||
|
||||
/* Calculate how many times to loop through the fr_type
|
||||
* and where the next fr_off is. */
|
||||
MPI_Type_extent(*fr_type_p, &fr_extent);
|
||||
MPI_Type_get_extent(*fr_type_p, &lb, &fr_extent);
|
||||
tmp_off = off - fr_st_off;
|
||||
fr_dtype_ct = tmp_off / fr_extent;
|
||||
off_rem = tmp_off % fr_extent;
|
||||
|
||||
@ -127,7 +127,7 @@ void ADIOI_Exch_file_views(int myrank, int nprocs, int file_ptr_type,
|
||||
MPI_Request *send_req_arr = NULL, *recv_req_arr = NULL;
|
||||
MPI_Status *statuses = NULL;
|
||||
ADIO_Offset disp_off_sz_ext_typesz[6];
|
||||
MPI_Aint memtype_extent, filetype_extent;
|
||||
MPI_Aint memtype_extent, filetype_extent, lb;
|
||||
int ret = -1;
|
||||
|
||||
/* parameters for datatypes */
|
||||
@ -143,7 +143,7 @@ void ADIOI_Exch_file_views(int myrank, int nprocs, int file_ptr_type,
|
||||
* freed in the close and should have been flattened in the file
|
||||
* view. */
|
||||
MPI_Type_size_x(datatype, &memtype_sz);
|
||||
MPI_Type_extent(datatype, &memtype_extent);
|
||||
MPI_Type_get_extent(datatype, &lb, &memtype_extent);
|
||||
if (memtype_sz == memtype_extent) {
|
||||
memtype_is_contig = 1;
|
||||
flat_mem_p = ADIOI_Add_contig_flattened(datatype);
|
||||
@ -156,7 +156,7 @@ void ADIOI_Exch_file_views(int myrank, int nprocs, int file_ptr_type,
|
||||
flat_mem_p = flat_mem_p->next;
|
||||
}
|
||||
|
||||
MPI_Type_extent(fd->filetype, &filetype_extent);
|
||||
MPI_Type_get_extent(fd->filetype, &lb, &filetype_extent);
|
||||
MPI_Type_size_x(fd->filetype, &filetype_sz);
|
||||
if (filetype_extent == filetype_sz) {
|
||||
flat_file_p = ADIOI_Add_contig_flattened(fd->filetype);
|
||||
|
||||
@ -24,11 +24,11 @@ int ADIO_Type_create_darray(int size, int rank, int ndims,
|
||||
int order, MPI_Datatype oldtype,
|
||||
MPI_Datatype *newtype)
|
||||
{
|
||||
MPI_Datatype type_old, type_new=MPI_DATATYPE_NULL, types[3];
|
||||
int procs, tmp_rank, i, tmp_size, blklens[3], *coords;
|
||||
MPI_Aint *st_offsets, orig_extent, disps[3];
|
||||
MPI_Datatype type_old, type_new=MPI_DATATYPE_NULL, inttype;
|
||||
int procs, tmp_rank, i, tmp_size, blklen, *coords;
|
||||
MPI_Aint *st_offsets, orig_extent, disp, ub, lb;
|
||||
|
||||
MPI_Type_extent(oldtype, &orig_extent);
|
||||
MPI_Type_get_extent(oldtype, &lb, &orig_extent);
|
||||
|
||||
/* calculate position in Cartesian grid as MPI would (row-major
|
||||
ordering) */
|
||||
@ -77,11 +77,11 @@ int ADIO_Type_create_darray(int size, int rank, int ndims,
|
||||
}
|
||||
|
||||
/* add displacement and UB */
|
||||
disps[1] = st_offsets[0];
|
||||
disp = st_offsets[0];
|
||||
tmp_size = 1;
|
||||
for (i=1; i<ndims; i++) {
|
||||
tmp_size *= array_of_gsizes[i-1];
|
||||
disps[1] += (MPI_Aint)tmp_size*st_offsets[i];
|
||||
disp += (MPI_Aint)tmp_size*st_offsets[i];
|
||||
}
|
||||
/* rest done below for both Fortran and C order */
|
||||
}
|
||||
@ -115,26 +115,24 @@ int ADIO_Type_create_darray(int size, int rank, int ndims,
|
||||
}
|
||||
|
||||
/* add displacement and UB */
|
||||
disps[1] = st_offsets[ndims-1];
|
||||
disp = st_offsets[ndims-1];
|
||||
tmp_size = 1;
|
||||
for (i=ndims-2; i>=0; i--) {
|
||||
tmp_size *= array_of_gsizes[i+1];
|
||||
disps[1] += (MPI_Aint)tmp_size*st_offsets[i];
|
||||
disp += (MPI_Aint)tmp_size*st_offsets[i];
|
||||
}
|
||||
}
|
||||
|
||||
disps[1] *= orig_extent;
|
||||
disp *= orig_extent;
|
||||
|
||||
disps[2] = orig_extent;
|
||||
for (i=0; i<ndims; i++) disps[2] *= (MPI_Aint)array_of_gsizes[i];
|
||||
ub = orig_extent;
|
||||
for (i=0; i<ndims; i++) ub *= (MPI_Aint)array_of_gsizes[i];
|
||||
|
||||
disps[0] = 0;
|
||||
blklens[0] = blklens[1] = blklens[2] = 1;
|
||||
types[0] = MPI_LB;
|
||||
types[1] = type_new;
|
||||
types[2] = MPI_UB;
|
||||
blklen = 1;
|
||||
|
||||
MPI_Type_struct(3, blklens, disps, types, newtype);
|
||||
MPI_Type_create_struct(1, &blklen, &disp, &type_new, &inttype);
|
||||
MPI_Type_create_resized (inttype, 0, ub, newtype);
|
||||
MPI_Type_free (&inttype);
|
||||
|
||||
MPI_Type_free(&type_new);
|
||||
ADIOI_Free(st_offsets);
|
||||
@ -184,7 +182,7 @@ static int MPIOI_Type_block(int *array_of_gsizes, int dim, int ndims, int nprocs
|
||||
MPI_Type_contiguous(mysize, type_old, type_new);
|
||||
else {
|
||||
for (i=0; i<dim; i++) stride *= (MPI_Aint)array_of_gsizes[i];
|
||||
MPI_Type_hvector(mysize, 1, stride, type_old, type_new);
|
||||
MPI_Type_create_hvector(mysize, 1, stride, type_old, type_new);
|
||||
}
|
||||
}
|
||||
else {
|
||||
@ -192,7 +190,7 @@ static int MPIOI_Type_block(int *array_of_gsizes, int dim, int ndims, int nprocs
|
||||
MPI_Type_contiguous(mysize, type_old, type_new);
|
||||
else {
|
||||
for (i=ndims-1; i>dim; i--) stride *= (MPI_Aint)array_of_gsizes[i];
|
||||
MPI_Type_hvector(mysize, 1, stride, type_old, type_new);
|
||||
MPI_Type_create_hvector(mysize, 1, stride, type_old, type_new);
|
||||
}
|
||||
|
||||
}
|
||||
@ -217,7 +215,7 @@ static int MPIOI_Type_cyclic(int *array_of_gsizes, int dim, int ndims, int nproc
|
||||
rank = coordinate of this process in dimension dim */
|
||||
int blksize, i, blklens[3], st_index, end_index, local_size, rem, count;
|
||||
MPI_Aint stride, disps[3];
|
||||
MPI_Datatype type_tmp, types[3];
|
||||
MPI_Datatype type_tmp, type_tmp1, types[3];
|
||||
|
||||
if (darg == MPI_DISTRIBUTE_DFLT_DARG) blksize = 1;
|
||||
else blksize = darg;
|
||||
@ -246,7 +244,7 @@ static int MPIOI_Type_cyclic(int *array_of_gsizes, int dim, int ndims, int nproc
|
||||
for (i=0; i<dim; i++) stride *= (MPI_Aint)array_of_gsizes[i];
|
||||
else for (i=ndims-1; i>dim; i--) stride *= (MPI_Aint)array_of_gsizes[i];
|
||||
|
||||
MPI_Type_hvector(count, blksize, stride, type_old, type_new);
|
||||
MPI_Type_create_hvector(count, blksize, stride, type_old, type_new);
|
||||
|
||||
if (rem) {
|
||||
/* if the last block is of size less than blksize, include
|
||||
@ -259,7 +257,7 @@ static int MPIOI_Type_cyclic(int *array_of_gsizes, int dim, int ndims, int nproc
|
||||
blklens[0] = 1;
|
||||
blklens[1] = rem;
|
||||
|
||||
MPI_Type_struct(2, blklens, disps, types, &type_tmp);
|
||||
MPI_Type_create_struct(2, blklens, disps, types, &type_tmp);
|
||||
|
||||
MPI_Type_free(type_new);
|
||||
*type_new = type_tmp;
|
||||
@ -269,14 +267,12 @@ static int MPIOI_Type_cyclic(int *array_of_gsizes, int dim, int ndims, int nproc
|
||||
dimension correctly. */
|
||||
if ( ((order == MPI_ORDER_FORTRAN) && (dim == 0)) ||
|
||||
((order == MPI_ORDER_C) && (dim == ndims-1)) ) {
|
||||
types[0] = MPI_LB;
|
||||
disps[0] = 0;
|
||||
types[1] = *type_new;
|
||||
disps[1] = (MPI_Aint)rank * (MPI_Aint)blksize * orig_extent;
|
||||
types[2] = MPI_UB;
|
||||
disps[2] = orig_extent * (MPI_Aint)array_of_gsizes[dim];
|
||||
blklens[0] = blklens[1] = blklens[2] = 1;
|
||||
MPI_Type_struct(3, blklens, disps, types, &type_tmp);
|
||||
types[0] = *type_new;
|
||||
disps[0] = (MPI_Aint)rank * (MPI_Aint)blksize * orig_extent;
|
||||
blklens[0] = 1;
|
||||
MPI_Type_create_struct(1, blklens, disps, types, &type_tmp1);
|
||||
MPI_Type_create_resized (type_tmp1, 0, orig_extent * (MPI_Aint)array_of_gsizes[dim], &type_tmp);
|
||||
MPI_Type_free(&type_tmp1);
|
||||
MPI_Type_free(type_new);
|
||||
*type_new = type_tmp;
|
||||
|
||||
|
||||
@ -72,13 +72,13 @@ int ADIOI_End_call(MPI_Comm comm, int keyval, void *attribute_val, void
|
||||
ADIOI_UNREFERENCED_ARG(attribute_val);
|
||||
ADIOI_UNREFERENCED_ARG(extra_state);
|
||||
|
||||
MPI_Keyval_free(&keyval);
|
||||
MPI_Comm_free_keyval (&keyval);
|
||||
|
||||
/* The end call will be called after all possible uses of this keyval, even
|
||||
* if a file was opened with MPI_COMM_SELF. Note, this assumes LIFO
|
||||
* MPI_COMM_SELF attribute destruction behavior mandated by MPI-2.2. */
|
||||
if (ADIOI_cb_config_list_keyval != MPI_KEYVAL_INVALID)
|
||||
MPI_Keyval_free(&ADIOI_cb_config_list_keyval);
|
||||
MPI_Comm_free_keyval (&ADIOI_cb_config_list_keyval);
|
||||
|
||||
ADIO_End(&error_code);
|
||||
return error_code;
|
||||
|
||||
@ -54,7 +54,7 @@ void ADIOI_IOStridedColl (ADIO_File fd, void *buf, int count, int rdwr,
|
||||
int interleave_count = 0, i, nprocs, myrank, nprocs_for_coll;
|
||||
int cb_enable;
|
||||
ADIO_Offset bufsize;
|
||||
MPI_Aint extent;
|
||||
MPI_Aint extent, lb;
|
||||
#ifdef DEBUG2
|
||||
MPI_Aint bufextent;
|
||||
#endif
|
||||
@ -191,7 +191,7 @@ void ADIOI_IOStridedColl (ADIO_File fd, void *buf, int count, int rdwr,
|
||||
return;
|
||||
}
|
||||
|
||||
MPI_Type_extent(datatype, &extent);
|
||||
MPI_Type_get_extent(datatype, &lb, &extent);
|
||||
#ifdef DEBUG2
|
||||
bufextent = extent * count;
|
||||
#endif
|
||||
@ -702,7 +702,7 @@ void ADIOI_Calc_bounds (ADIO_File fd, int count, MPI_Datatype buftype,
|
||||
{
|
||||
MPI_Count filetype_size, buftype_size, etype_size;
|
||||
int sum;
|
||||
MPI_Aint filetype_extent;
|
||||
MPI_Aint filetype_extent, lb;
|
||||
ADIO_Offset total_io;
|
||||
int filetype_is_contig;
|
||||
ADIO_Offset i, remainder;
|
||||
@ -726,7 +726,7 @@ void ADIOI_Calc_bounds (ADIO_File fd, int count, MPI_Datatype buftype,
|
||||
ADIOI_Datatype_iscontig (fd->filetype, &filetype_is_contig);
|
||||
|
||||
MPI_Type_size_x (fd->filetype, &filetype_size);
|
||||
MPI_Type_extent (fd->filetype, &filetype_extent);
|
||||
MPI_Type_get_extent (fd->filetype, &lb, &filetype_extent);
|
||||
MPI_Type_size_x (fd->etype, &etype_size);
|
||||
MPI_Type_size_x (buftype, &buftype_size);
|
||||
|
||||
@ -884,7 +884,7 @@ void ADIOI_IOFiletype(ADIO_File fd, void *buf, int count,
|
||||
int user_ind_rd_buffer_size;
|
||||
int f_is_contig, m_is_contig;
|
||||
int user_ds_read, user_ds_write;
|
||||
MPI_Aint f_extent;
|
||||
MPI_Aint f_extent, lb;
|
||||
MPI_Count f_size;
|
||||
int f_ds_percent; /* size/extent */
|
||||
|
||||
@ -894,7 +894,7 @@ void ADIOI_IOFiletype(ADIO_File fd, void *buf, int count,
|
||||
else
|
||||
MPE_Log_event(5008, 0, NULL);
|
||||
#endif
|
||||
MPI_Type_extent(custom_ftype, &f_extent);
|
||||
MPI_Type_get_extent(custom_ftype, &lb, &f_extent);
|
||||
MPI_Type_size_x(custom_ftype, &f_size);
|
||||
f_ds_percent = 100 * f_size / f_extent;
|
||||
|
||||
|
||||
@ -307,8 +307,7 @@ void ADIOI_Calc_my_off_len(ADIO_File fd, int bufcount, MPI_Datatype
|
||||
ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
|
||||
|
||||
MPI_Type_size_x(fd->filetype, &filetype_size);
|
||||
MPI_Type_extent(fd->filetype, &filetype_extent);
|
||||
MPI_Type_lb(fd->filetype, &filetype_lb);
|
||||
MPI_Type_get_extent(fd->filetype, &filetype_lb, &filetype_extent);
|
||||
MPI_Type_size_x(datatype, &buftype_size);
|
||||
etype_size = fd->etype_size;
|
||||
|
||||
@ -524,7 +523,7 @@ static void ADIOI_Read_and_exch(ADIO_File fd, void *buf, MPI_Datatype
|
||||
int req_len, flag, rank;
|
||||
MPI_Status status;
|
||||
ADIOI_Flatlist_node *flat_buf=NULL;
|
||||
MPI_Aint buftype_extent;
|
||||
MPI_Aint buftype_extent, lb;
|
||||
int coll_bufsize;
|
||||
|
||||
*error_code = MPI_SUCCESS; /* changed below if error */
|
||||
@ -605,7 +604,7 @@ static void ADIOI_Read_and_exch(ADIO_File fd, void *buf, MPI_Datatype
|
||||
flat_buf = ADIOI_Flatlist;
|
||||
while (flat_buf->type != datatype) flat_buf = flat_buf->next;
|
||||
}
|
||||
MPI_Type_extent(datatype, &buftype_extent);
|
||||
MPI_Type_get_extent(datatype, &lb, &buftype_extent);
|
||||
|
||||
done = 0;
|
||||
off = st_loc;
|
||||
@ -685,7 +684,7 @@ static void ADIOI_Read_and_exch(ADIO_File fd, void *buf, MPI_Datatype
|
||||
if (req_off < real_off + real_size) {
|
||||
count[i]++;
|
||||
ADIOI_Assert((((ADIO_Offset)(MPIR_Upint)read_buf)+req_off-real_off) == (ADIO_Offset)(MPIR_Upint)(read_buf+req_off-real_off));
|
||||
MPI_Address(read_buf+req_off-real_off,
|
||||
MPI_Get_address(read_buf+req_off-real_off,
|
||||
&(others_req[i].mem_ptrs[j]));
|
||||
ADIOI_Assert((real_off + real_size - req_off) == (int)(real_off + real_size - req_off));
|
||||
send_size[i] += (int)(ADIOI_MIN(real_off + real_size - req_off,
|
||||
|
||||
@ -56,7 +56,7 @@ void ADIOI_GEN_ReadStrided(ADIO_File fd, void *buf, int count,
|
||||
ADIO_Offset n_filetypes, etype_in_filetype, st_n_filetypes, size_in_filetype;
|
||||
ADIO_Offset abs_off_in_filetype=0, new_frd_size, frd_size=0, st_frd_size;
|
||||
MPI_Count filetype_size, etype_size, buftype_size, partial_read;
|
||||
MPI_Aint filetype_extent, buftype_extent;
|
||||
MPI_Aint filetype_extent, buftype_extent, lb;
|
||||
int buf_count, buftype_is_contig, filetype_is_contig;
|
||||
ADIO_Offset userbuf_off, req_len, sum;
|
||||
ADIO_Offset off, req_off, disp, end_offset=0, readbuf_off, start_off;
|
||||
@ -94,9 +94,9 @@ void ADIOI_GEN_ReadStrided(ADIO_File fd, void *buf, int count,
|
||||
return;
|
||||
}
|
||||
|
||||
MPI_Type_extent(fd->filetype, &filetype_extent);
|
||||
MPI_Type_get_extent(fd->filetype, &lb, &filetype_extent);
|
||||
MPI_Type_size_x(datatype, &buftype_size);
|
||||
MPI_Type_extent(datatype, &buftype_extent);
|
||||
MPI_Type_get_extent(datatype, &lb, &buftype_extent);
|
||||
etype_size = fd->etype_size;
|
||||
|
||||
ADIOI_Assert((buftype_size * count) == ((ADIO_Offset)(MPI_Count)buftype_size * (ADIO_Offset)count));
|
||||
|
||||
@ -23,7 +23,7 @@ void ADIOI_GEN_ReadStrided_naive(ADIO_File fd, void *buf, int count,
|
||||
ADIO_Offset abs_off_in_filetype=0;
|
||||
MPI_Count bufsize, filetype_size, buftype_size, size_in_filetype;
|
||||
ADIO_Offset etype_size;
|
||||
MPI_Aint filetype_extent, buftype_extent;
|
||||
MPI_Aint filetype_extent, buftype_extent, lb;
|
||||
int buf_count, buftype_is_contig, filetype_is_contig;
|
||||
ADIO_Offset userbuf_off;
|
||||
ADIO_Offset off, req_off, disp, end_offset=0, start_off;
|
||||
@ -43,9 +43,9 @@ void ADIOI_GEN_ReadStrided_naive(ADIO_File fd, void *buf, int count,
|
||||
return;
|
||||
}
|
||||
|
||||
MPI_Type_extent(fd->filetype, &filetype_extent);
|
||||
MPI_Type_get_extent(fd->filetype, &lb, &filetype_extent);
|
||||
MPI_Type_size_x(buftype, &buftype_size);
|
||||
MPI_Type_extent(buftype, &buftype_extent);
|
||||
MPI_Type_get_extent(buftype, &lb, &buftype_extent);
|
||||
etype_size = fd->etype_size;
|
||||
|
||||
ADIOI_Assert((buftype_size * count) == ((ADIO_Offset)buftype_size * (ADIO_Offset)count));
|
||||
|
||||
@ -28,7 +28,7 @@ ADIO_Offset ADIOI_GEN_SeekIndividual(ADIO_File fd, ADIO_Offset offset,
|
||||
ADIO_Offset size_in_filetype, sum;
|
||||
MPI_Count filetype_size, etype_size;
|
||||
int filetype_is_contig;
|
||||
MPI_Aint filetype_extent;
|
||||
MPI_Aint filetype_extent, lb;
|
||||
|
||||
ADIOI_UNREFERENCED_ARG(whence);
|
||||
|
||||
@ -40,7 +40,7 @@ ADIO_Offset ADIOI_GEN_SeekIndividual(ADIO_File fd, ADIO_Offset offset,
|
||||
flat_file = ADIOI_Flatlist;
|
||||
while (flat_file->type != fd->filetype) flat_file = flat_file->next;
|
||||
|
||||
MPI_Type_extent(fd->filetype, &filetype_extent);
|
||||
MPI_Type_get_extent(fd->filetype, &lb, &filetype_extent);
|
||||
MPI_Type_size_x(fd->filetype, &filetype_size);
|
||||
if ( ! filetype_size ) {
|
||||
/* Since offset relative to the filetype size, we can't
|
||||
|
||||
@ -16,11 +16,11 @@ int ADIO_Type_create_subarray(int ndims,
|
||||
MPI_Datatype oldtype,
|
||||
MPI_Datatype *newtype)
|
||||
{
|
||||
MPI_Aint extent, disps[3], size;
|
||||
int i, blklens[3];
|
||||
MPI_Datatype tmp1, tmp2, types[3];
|
||||
MPI_Aint extent, disp, size, lb, ub;
|
||||
int i, blklen;
|
||||
MPI_Datatype tmp1, tmp2, inttype;
|
||||
|
||||
MPI_Type_extent(oldtype, &extent);
|
||||
MPI_Type_get_extent(oldtype, &lb, &extent);
|
||||
|
||||
if (order == MPI_ORDER_FORTRAN) {
|
||||
/* dimension 0 changes fastest */
|
||||
@ -35,18 +35,18 @@ int ADIO_Type_create_subarray(int ndims,
|
||||
size = (MPI_Aint)array_of_sizes[0]*extent;
|
||||
for (i=2; i<ndims; i++) {
|
||||
size *= (MPI_Aint)array_of_sizes[i-1];
|
||||
MPI_Type_hvector(array_of_subsizes[i], 1, size, tmp1, &tmp2);
|
||||
MPI_Type_create_hvector(array_of_subsizes[i], 1, size, tmp1, &tmp2);
|
||||
MPI_Type_free(&tmp1);
|
||||
tmp1 = tmp2;
|
||||
}
|
||||
}
|
||||
|
||||
/* add displacement and UB */
|
||||
disps[1] = array_of_starts[0];
|
||||
disp = array_of_starts[0];
|
||||
size = 1;
|
||||
for (i=1; i<ndims; i++) {
|
||||
size *= (MPI_Aint)array_of_sizes[i-1];
|
||||
disps[1] += size*(MPI_Aint)array_of_starts[i];
|
||||
disp += size*(MPI_Aint)array_of_starts[i];
|
||||
}
|
||||
/* rest done below for both Fortran and C order */
|
||||
}
|
||||
@ -64,33 +64,31 @@ int ADIO_Type_create_subarray(int ndims,
|
||||
size = (MPI_Aint)array_of_sizes[ndims-1]*extent;
|
||||
for (i=ndims-3; i>=0; i--) {
|
||||
size *= (MPI_Aint)array_of_sizes[i+1];
|
||||
MPI_Type_hvector(array_of_subsizes[i], 1, size, tmp1, &tmp2);
|
||||
MPI_Type_create_hvector(array_of_subsizes[i], 1, size, tmp1, &tmp2);
|
||||
MPI_Type_free(&tmp1);
|
||||
tmp1 = tmp2;
|
||||
}
|
||||
}
|
||||
|
||||
/* add displacement and UB */
|
||||
disps[1] = array_of_starts[ndims-1];
|
||||
disp = array_of_starts[ndims-1];
|
||||
size = 1;
|
||||
for (i=ndims-2; i>=0; i--) {
|
||||
size *= (MPI_Aint)array_of_sizes[i+1];
|
||||
disps[1] += size*(MPI_Aint)array_of_starts[i];
|
||||
disp += size*(MPI_Aint)array_of_starts[i];
|
||||
}
|
||||
}
|
||||
|
||||
disps[1] *= extent;
|
||||
disp *= extent;
|
||||
|
||||
disps[2] = extent;
|
||||
for (i=0; i<ndims; i++) disps[2] *= (MPI_Aint)array_of_sizes[i];
|
||||
ub = extent;
|
||||
for (i=0; i<ndims; i++) ub *= (MPI_Aint)array_of_sizes[i];
|
||||
|
||||
disps[0] = 0;
|
||||
blklens[0] = blklens[1] = blklens[2] = 1;
|
||||
types[0] = MPI_LB;
|
||||
types[1] = tmp1;
|
||||
types[2] = MPI_UB;
|
||||
blklen = 1;
|
||||
|
||||
MPI_Type_struct(3, blklens, disps, types, newtype);
|
||||
MPI_Type_create_struct(1, &blklen, &disp, &tmp1, &inttype);
|
||||
MPI_Type_create_resized (inttype, 0, ub, newtype);
|
||||
MPI_Type_free(&inttype);
|
||||
|
||||
MPI_Type_free(&tmp1);
|
||||
|
||||
|
||||
@ -321,7 +321,7 @@ static void ADIOI_Exch_and_write(ADIO_File fd, void *buf, MPI_Datatype
|
||||
int *send_buf_idx, *curr_to_proc, *done_to_proc;
|
||||
MPI_Status status;
|
||||
ADIOI_Flatlist_node *flat_buf=NULL;
|
||||
MPI_Aint buftype_extent;
|
||||
MPI_Aint buftype_extent, lb;
|
||||
int info_flag, coll_bufsize;
|
||||
char *value;
|
||||
static char myname[] = "ADIOI_EXCH_AND_WRITE";
|
||||
@ -406,7 +406,7 @@ static void ADIOI_Exch_and_write(ADIO_File fd, void *buf, MPI_Datatype
|
||||
flat_buf = ADIOI_Flatlist;
|
||||
while (flat_buf->type != datatype) flat_buf = flat_buf->next;
|
||||
}
|
||||
MPI_Type_extent(datatype, &buftype_extent);
|
||||
MPI_Type_get_extent(datatype, &lb, &buftype_extent);
|
||||
|
||||
|
||||
/* I need to check if there are any outstanding nonblocking writes to
|
||||
@ -468,7 +468,7 @@ static void ADIOI_Exch_and_write(ADIO_File fd, void *buf, MPI_Datatype
|
||||
if (req_off < off + size) {
|
||||
count[i]++;
|
||||
ADIOI_Assert((((ADIO_Offset)(MPIR_Upint)write_buf)+req_off-off) == (ADIO_Offset)(MPIR_Upint)(write_buf+req_off-off));
|
||||
MPI_Address(write_buf+req_off-off,
|
||||
MPI_Get_address(write_buf+req_off-off,
|
||||
&(others_req[i].mem_ptrs[j]));
|
||||
ADIOI_Assert((off + size - req_off) == (int)(off + size - req_off));
|
||||
recv_size[i] += (int)(ADIOI_MIN(off + size - req_off,
|
||||
|
||||
@ -35,7 +35,7 @@ void ADIOI_NOLOCK_WriteStrided(ADIO_File fd, const void *buf, int count,
|
||||
ADIO_Offset n_filetypes, etype_in_filetype, size, sum;
|
||||
ADIO_Offset abs_off_in_filetype=0, size_in_filetype;
|
||||
MPI_Count filetype_size, etype_size, buftype_size;
|
||||
MPI_Aint filetype_extent, buftype_extent, indx;
|
||||
MPI_Aint filetype_extent, buftype_extent, indx, lb;
|
||||
int buf_count, buftype_is_contig, filetype_is_contig;
|
||||
ADIO_Offset off, disp;
|
||||
int flag, err_flag=0;
|
||||
@ -71,9 +71,9 @@ void ADIOI_NOLOCK_WriteStrided(ADIO_File fd, const void *buf, int count,
|
||||
MPI_Comm_size(fd->comm, &nprocs);
|
||||
#endif
|
||||
|
||||
MPI_Type_extent(fd->filetype, &filetype_extent);
|
||||
MPI_Type_get_extent(fd->filetype, &lb, &filetype_extent);
|
||||
MPI_Type_size_x(datatype, &buftype_size);
|
||||
MPI_Type_extent(datatype, &buftype_extent);
|
||||
MPI_Type_get_extent(datatype, &lb, &buftype_extent);
|
||||
etype_size = fd->etype_size;
|
||||
|
||||
ADIOI_Assert((buftype_size * count) == ((ADIO_Offset)(unsigned)buftype_size * (ADIO_Offset)count));
|
||||
|
||||
@ -122,7 +122,7 @@ void ADIOI_GEN_WriteStrided(ADIO_File fd, const void *buf, int count,
|
||||
ADIO_Offset num, size, n_filetypes, etype_in_filetype, st_n_filetypes;
|
||||
ADIO_Offset n_etypes_in_filetype, abs_off_in_filetype=0;
|
||||
MPI_Count filetype_size, etype_size, buftype_size;
|
||||
MPI_Aint filetype_extent, buftype_extent;
|
||||
MPI_Aint filetype_extent, buftype_extent, lb;
|
||||
int buf_count, buftype_is_contig, filetype_is_contig;
|
||||
ADIO_Offset userbuf_off;
|
||||
ADIO_Offset off, req_off, disp, end_offset=0, writebuf_off, start_off;
|
||||
@ -164,9 +164,9 @@ void ADIOI_GEN_WriteStrided(ADIO_File fd, const void *buf, int count,
|
||||
return;
|
||||
}
|
||||
|
||||
MPI_Type_extent(fd->filetype, &filetype_extent);
|
||||
MPI_Type_get_extent(fd->filetype, &lb, &filetype_extent);
|
||||
MPI_Type_size_x(datatype, &buftype_size);
|
||||
MPI_Type_extent(datatype, &buftype_extent);
|
||||
MPI_Type_get_extent(datatype, &lb, &buftype_extent);
|
||||
etype_size = fd->etype_size;
|
||||
|
||||
ADIOI_Assert((buftype_size * count) == ((MPI_Count)buftype_size * (ADIO_Offset)count));
|
||||
|
||||
@ -24,7 +24,7 @@ void ADIOI_GEN_WriteStrided_naive(ADIO_File fd, const void *buf, int count,
|
||||
ADIO_Offset size, n_filetypes, etype_in_filetype;
|
||||
ADIO_Offset abs_off_in_filetype=0, req_len;
|
||||
MPI_Count filetype_size, etype_size, buftype_size;
|
||||
MPI_Aint filetype_extent, buftype_extent;
|
||||
MPI_Aint filetype_extent, buftype_extent, lb;
|
||||
int buf_count, buftype_is_contig, filetype_is_contig;
|
||||
ADIO_Offset userbuf_off;
|
||||
ADIO_Offset off, req_off, disp, end_offset=0, start_off;
|
||||
@ -44,9 +44,9 @@ void ADIOI_GEN_WriteStrided_naive(ADIO_File fd, const void *buf, int count,
|
||||
return;
|
||||
}
|
||||
|
||||
MPI_Type_extent(fd->filetype, &filetype_extent);
|
||||
MPI_Type_get_extent(fd->filetype, &lb, &filetype_extent);
|
||||
MPI_Type_size_x(buftype, &buftype_size);
|
||||
MPI_Type_extent(buftype, &buftype_extent);
|
||||
MPI_Type_get_extent(buftype, &lb, &buftype_extent);
|
||||
etype_size = fd->etype_size;
|
||||
|
||||
ADIOI_Assert((buftype_size * count) == ((ADIO_Offset)(unsigned)buftype_size * (ADIO_Offset)count));
|
||||
|
||||
@ -18,7 +18,7 @@ void ADIOI_Get_byte_offset(ADIO_File fd, ADIO_Offset offset, ADIO_Offset *disp)
|
||||
ADIO_Offset n_filetypes, etype_in_filetype, sum, abs_off_in_filetype=0, size_in_filetype;
|
||||
MPI_Count n_etypes_in_filetype, filetype_size, etype_size;
|
||||
int filetype_is_contig;
|
||||
MPI_Aint filetype_extent;
|
||||
MPI_Aint filetype_extent, lb;
|
||||
|
||||
ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
|
||||
etype_size = fd->etype_size;
|
||||
@ -46,7 +46,7 @@ void ADIOI_Get_byte_offset(ADIO_File fd, ADIO_Offset offset, ADIO_Offset *disp)
|
||||
}
|
||||
|
||||
/* abs. offset in bytes in the file */
|
||||
MPI_Type_extent(fd->filetype, &filetype_extent);
|
||||
MPI_Type_get_extent(fd->filetype, &lb, &filetype_extent);
|
||||
*disp = fd->disp + n_filetypes * ADIOI_AINT_CAST_TO_OFFSET filetype_extent + abs_off_in_filetype;
|
||||
}
|
||||
}
|
||||
|
||||
@ -135,12 +135,12 @@ int ADIOI_cb_gather_name_array(MPI_Comm comm,
|
||||
|
||||
if (ADIOI_cb_config_list_keyval == MPI_KEYVAL_INVALID) {
|
||||
/* cleaned up by ADIOI_End_call */
|
||||
MPI_Keyval_create((MPI_Copy_function *) ADIOI_cb_copy_name_array,
|
||||
(MPI_Delete_function *) ADIOI_cb_delete_name_array,
|
||||
MPI_Comm_create_keyval((MPI_Comm_copy_attr_function *) ADIOI_cb_copy_name_array,
|
||||
(MPI_Comm_delete_attr_function *) ADIOI_cb_delete_name_array,
|
||||
&ADIOI_cb_config_list_keyval, NULL);
|
||||
}
|
||||
else {
|
||||
MPI_Attr_get(comm, ADIOI_cb_config_list_keyval, (void *) &array, &found);
|
||||
MPI_Comm_get_attr(comm, ADIOI_cb_config_list_keyval, (void *) &array, &found);
|
||||
if (found) {
|
||||
ADIOI_Assert(array != NULL);
|
||||
*arrayp = array;
|
||||
@ -255,8 +255,8 @@ int ADIOI_cb_gather_name_array(MPI_Comm comm,
|
||||
* it next time an open is performed on this same comm, and on the
|
||||
* dupcomm, so we can use it in I/O operations.
|
||||
*/
|
||||
MPI_Attr_put(comm, ADIOI_cb_config_list_keyval, array);
|
||||
MPI_Attr_put(dupcomm, ADIOI_cb_config_list_keyval, array);
|
||||
MPI_Comm_set_attr (comm, ADIOI_cb_config_list_keyval, array);
|
||||
MPI_Comm_set_attr (dupcomm, ADIOI_cb_config_list_keyval, array);
|
||||
*arrayp = array;
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -18,7 +18,7 @@ void ADIOI_Get_eof_offset(ADIO_File fd, ADIO_Offset *eof_offset)
|
||||
ADIO_Offset fsize, disp, sum=0, size_in_file, n_filetypes, rem, etype_size;
|
||||
int flag, i;
|
||||
ADIO_Fcntl_t *fcntl_struct;
|
||||
MPI_Aint filetype_extent;
|
||||
MPI_Aint filetype_extent, lb;
|
||||
ADIOI_Flatlist_node *flat_file;
|
||||
|
||||
/* find the eof in bytes */
|
||||
@ -45,7 +45,7 @@ void ADIOI_Get_eof_offset(ADIO_File fd, ADIO_Offset *eof_offset)
|
||||
flat_file = flat_file->next;
|
||||
|
||||
MPI_Type_size_x(fd->filetype, &filetype_size);
|
||||
MPI_Type_extent(fd->filetype, &filetype_extent);
|
||||
MPI_Type_get_extent(fd->filetype, &lb, &filetype_extent);
|
||||
|
||||
disp = fd->disp;
|
||||
n_filetypes = -1;
|
||||
|
||||
@ -198,7 +198,7 @@ void ADIOI_Flatten(MPI_Datatype datatype, ADIOI_Flatlist_node *flat,
|
||||
avoid >2G integer arithmetic problems */
|
||||
ADIO_Offset top_count;
|
||||
MPI_Count j, old_size, prev_index, num;
|
||||
MPI_Aint old_extent;/* Assume extents are non-negative */
|
||||
MPI_Aint old_extent, lb;/* Assume extents are non-negative */
|
||||
int *ints;
|
||||
MPI_Aint *adds; /* Make no assumptions about +/- sign on these */
|
||||
MPI_Datatype *types;
|
||||
@ -331,7 +331,7 @@ void ADIOI_Flatten(MPI_Datatype datatype, ADIOI_Flatlist_node *flat,
|
||||
num = *curr_index - prev_index;
|
||||
|
||||
/* The noncontiguous types have to be replicated count times */
|
||||
MPI_Type_extent(types[0], &old_extent);
|
||||
MPI_Type_get_extent(types[0], &lb, &old_extent);
|
||||
for (m=1; m<top_count; m++) {
|
||||
for (i=0; i<num; i++) {
|
||||
flat->indices[j] = flat->indices[j-num] + ADIOI_AINT_CAST_TO_OFFSET old_extent;
|
||||
@ -385,7 +385,7 @@ void ADIOI_Flatten(MPI_Datatype datatype, ADIOI_Flatlist_node *flat,
|
||||
|
||||
/* The noncontiguous types have to be replicated blocklen times
|
||||
and then strided. Replicate the first one. */
|
||||
MPI_Type_extent(types[0], &old_extent);
|
||||
MPI_Type_get_extent(types[0], &lb, &old_extent);
|
||||
for (m=1; m<blocklength; m++) {
|
||||
for (i=0; i<num; i++) {
|
||||
flat->indices[j] = flat->indices[j-num] + ADIOI_AINT_CAST_TO_OFFSET old_extent;
|
||||
@ -448,7 +448,7 @@ void ADIOI_Flatten(MPI_Datatype datatype, ADIOI_Flatlist_node *flat,
|
||||
|
||||
/* The noncontiguous types have to be replicated blocklen times
|
||||
and then strided. Replicate the first one. */
|
||||
MPI_Type_extent(types[0], &old_extent);
|
||||
MPI_Type_get_extent(types[0], &lb, &old_extent);
|
||||
for (m=1; m<blocklength; m++) {
|
||||
for (i=0; i<num; i++) {
|
||||
flat->indices[j] = flat->indices[j-num] + ADIOI_AINT_CAST_TO_OFFSET old_extent;
|
||||
@ -479,7 +479,7 @@ void ADIOI_Flatten(MPI_Datatype datatype, ADIOI_Flatlist_node *flat,
|
||||
ADIOI_Type_get_envelope(types[0], &old_nints, &old_nadds,
|
||||
&old_ntypes, &old_combiner);
|
||||
ADIOI_Datatype_iscontig(types[0], &old_is_contig);
|
||||
MPI_Type_extent(types[0], &old_extent);
|
||||
MPI_Type_get_extent(types[0], &lb, &old_extent);
|
||||
|
||||
prev_index = *curr_index;
|
||||
if ((old_combiner != MPI_COMBINER_NAMED) && (!old_is_contig))
|
||||
@ -585,7 +585,7 @@ void ADIOI_Flatten(MPI_Datatype datatype, ADIOI_Flatlist_node *flat,
|
||||
ADIOI_Type_get_envelope(types[0], &old_nints, &old_nadds,
|
||||
&old_ntypes, &old_combiner);
|
||||
ADIOI_Datatype_iscontig(types[0], &old_is_contig);
|
||||
MPI_Type_extent(types[0], &old_extent);
|
||||
MPI_Type_get_extent(types[0], &lb, &old_extent);
|
||||
|
||||
prev_index = *curr_index;
|
||||
if ((old_combiner != MPI_COMBINER_NAMED) && (!old_is_contig))
|
||||
@ -633,7 +633,7 @@ void ADIOI_Flatten(MPI_Datatype datatype, ADIOI_Flatlist_node *flat,
|
||||
if (is_hindexed_block) {
|
||||
/* this is the one place the hindexed case uses the
|
||||
* extent of a type */
|
||||
MPI_Type_extent(types[0], &old_extent);
|
||||
MPI_Type_get_extent(types[0], &lb, &old_extent);
|
||||
}
|
||||
flat->indices[j] = flat->indices[j-num] +
|
||||
ADIOI_AINT_CAST_TO_OFFSET old_extent;
|
||||
@ -708,7 +708,7 @@ void ADIOI_Flatten(MPI_Datatype datatype, ADIOI_Flatlist_node *flat,
|
||||
|
||||
/* The noncontiguous types have to be replicated blocklens[i] times
|
||||
and then strided. Replicate the first one. */
|
||||
MPI_Type_extent(types[0], &old_extent);
|
||||
MPI_Type_get_extent(types[0], &lb, &old_extent);
|
||||
for (m=1; m<ints[1]; m++) {
|
||||
for (i=0, nonzeroth=j; i<num; i++) {
|
||||
if (flat->blocklens[j-num] > 0) {
|
||||
@ -775,7 +775,7 @@ void ADIOI_Flatten(MPI_Datatype datatype, ADIOI_Flatlist_node *flat,
|
||||
/* simplest case, current type is basic or contiguous types */
|
||||
/* By using ADIO_Offset we preserve +/- sign and
|
||||
avoid >2G integer arithmetic problems */
|
||||
if (ints[1+n] > 0 || types[n] == MPI_LB || types[n] == MPI_UB) {
|
||||
if (ints[1+n] > 0) {
|
||||
ADIO_Offset blocklength = ints[1+n];
|
||||
j = *curr_index;
|
||||
flat->indices[j] = st_offset + adds[n];
|
||||
@ -794,7 +794,7 @@ void ADIOI_Flatten(MPI_Datatype datatype, ADIOI_Flatlist_node *flat,
|
||||
num = *curr_index - prev_index;
|
||||
|
||||
/* The current type has to be replicated blocklens[n] times */
|
||||
MPI_Type_extent(types[n], &old_extent);
|
||||
MPI_Type_get_extent(types[n], &lb, &old_extent);
|
||||
for (m=1; m<ints[1+n]; m++) {
|
||||
for (i=0; i<num; i++) {
|
||||
flat->indices[j] =
|
||||
|
||||
@ -18,7 +18,7 @@ void ADIOI_Get_position(ADIO_File fd, ADIO_Offset *offset)
|
||||
int i, flag;
|
||||
MPI_Count filetype_size, etype_size;
|
||||
int filetype_is_contig;
|
||||
MPI_Aint filetype_extent;
|
||||
MPI_Aint filetype_extent, lb;
|
||||
ADIO_Offset disp, byte_offset, sum=0, size_in_file, n_filetypes, frd_size;
|
||||
|
||||
ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
|
||||
@ -31,7 +31,7 @@ void ADIOI_Get_position(ADIO_File fd, ADIO_Offset *offset)
|
||||
while (flat_file->type != fd->filetype) flat_file = flat_file->next;
|
||||
|
||||
MPI_Type_size_x(fd->filetype, &filetype_size);
|
||||
MPI_Type_extent(fd->filetype, &filetype_extent);
|
||||
MPI_Type_get_extent(fd->filetype, &lb, &filetype_extent);
|
||||
|
||||
disp = fd->disp;
|
||||
byte_offset = fd->fp_ind;
|
||||
|
||||
@ -12,8 +12,6 @@
|
||||
|
||||
#undef MPI_Abort
|
||||
#define MPI_Abort PMPI_Abort
|
||||
#undef MPI_Address
|
||||
#define MPI_Address PMPI_Address
|
||||
#undef MPI_Allgather
|
||||
#define MPI_Allgather PMPI_Allgather
|
||||
#undef MPI_Allgatherv
|
||||
@ -30,8 +28,6 @@
|
||||
#define MPI_Attr_delete PMPI_Attr_delete
|
||||
#undef MPI_Attr_get
|
||||
#define MPI_Attr_get PMPI_Attr_get
|
||||
#undef MPI_Attr_put
|
||||
#define MPI_Attr_put PMPI_Attr_put
|
||||
#undef MPI_Barrier
|
||||
#define MPI_Barrier PMPI_Barrier
|
||||
#undef MPI_Bcast
|
||||
@ -68,10 +64,14 @@
|
||||
#define MPI_Comm_compare PMPI_Comm_compare
|
||||
#undef MPI_Comm_create
|
||||
#define MPI_Comm_create PMPI_Comm_create
|
||||
#undef MPI_Comm_create_keyval
|
||||
#define MPI_Comm_create_keyval PMPI_Comm_create_keyval
|
||||
#undef MPI_Comm_dup
|
||||
#define MPI_Comm_dup PMPI_Comm_dup
|
||||
#undef MPI_Comm_free
|
||||
#define MPI_Comm_free PMPI_Comm_free
|
||||
#undef MPI_Comm_free_keyval
|
||||
#define MPI_Comm_free_keyval PMPI_Comm_free_keyval
|
||||
#undef MPI_Comm_group
|
||||
#define MPI_Comm_group PMPI_Comm_group
|
||||
#undef MPI_Comm_rank
|
||||
@ -80,6 +80,8 @@
|
||||
#define MPI_Comm_remote_group PMPI_Comm_remote_group
|
||||
#undef MPI_Comm_remote_size
|
||||
#define MPI_Comm_remote_size PMPI_Comm_remote_size
|
||||
#undef MPI_Comm_set_attr
|
||||
#define MPI_Comm_set_attr PMPI_Comm_set_attr
|
||||
#undef MPI_Comm_size
|
||||
#define MPI_Comm_size PMPI_Comm_size
|
||||
#undef MPI_Comm_split
|
||||
@ -106,6 +108,8 @@
|
||||
#define MPI_Gather PMPI_Gather
|
||||
#undef MPI_Gatherv
|
||||
#define MPI_Gatherv PMPI_Gatherv
|
||||
#undef MPI_Get_address
|
||||
#define MPI_Get_address PMPI_Get_address
|
||||
#undef MPI_Get_count
|
||||
#define MPI_Get_count PMPI_Get_count
|
||||
#undef MPI_Get_elements
|
||||
@ -170,10 +174,6 @@
|
||||
#define MPI_Isend PMPI_Isend
|
||||
#undef MPI_Issend
|
||||
#define MPI_Issend PMPI_Issend
|
||||
#undef MPI_Keyval_create
|
||||
#define MPI_Keyval_create PMPI_Keyval_create
|
||||
#undef MPI_Keyval_free
|
||||
#define MPI_Keyval_free PMPI_Keyval_free
|
||||
#undef MPI_Name_get
|
||||
#define MPI_Name_get PMPI_Name_get
|
||||
#undef MPI_Name_put
|
||||
@ -248,14 +248,22 @@
|
||||
#define MPI_Type_contiguous PMPI_Type_contiguous
|
||||
#undef MPI_Type_count
|
||||
#define MPI_Type_count PMPI_Type_count
|
||||
#undef MPI_Type_create_struct
|
||||
#define MPI_Type_create_struct PMPI_Type_create_struct
|
||||
#undef MPI_Type_create_resized
|
||||
#define MPI_Type_create_resized PMPI_Type_create_resized
|
||||
/* #define MPI_Type_create_darray PMPI_Type_create_darray */
|
||||
#undef MPI_Type_create_indexed_block
|
||||
#define MPI_Type_create_indexed_block PMPI_Type_create_indexed_block
|
||||
#undef MPI_Type_create_hindexed
|
||||
#define MPI_Type_create_hindexed PMPI_Type_create_hindexed
|
||||
#undef MPI_Type_create_hindexed_block
|
||||
#define MPI_Type_create_hindexed_block PMPI_Type_create_hindexed_block
|
||||
#undef MPI_Type_create_hvector
|
||||
#define MPI_Type_create_hvector PMPI_Type_create_hvector
|
||||
/* #define MPI_Type_create_subarray PMPI_Type_create_subarray */
|
||||
#undef MPI_Type_extent
|
||||
#define MPI_Type_extent PMPI_Type_extent
|
||||
#undef MPI_Type_get_extent
|
||||
#define MPI_Type_get_extent PMPI_Type_get_extent
|
||||
#undef MPI_Type_free
|
||||
#define MPI_Type_free PMPI_Type_free
|
||||
#undef MPI_Type_get_contents
|
||||
@ -264,20 +272,10 @@
|
||||
#define MPI_Type_get_envelope PMPI_Type_get_envelope
|
||||
#undef MPI_Type_get_true_extent
|
||||
#define MPI_Type_get_true_extent PMPI_Type_get_true_extent
|
||||
#undef MPI_Type_hindexed
|
||||
#define MPI_Type_hindexed PMPI_Type_hindexed
|
||||
#undef MPI_Type_hvector
|
||||
#define MPI_Type_hvector PMPI_Type_hvector
|
||||
#undef MPI_Type_indexed
|
||||
#define MPI_Type_indexed PMPI_Type_indexed
|
||||
#undef MPI_Type_lb
|
||||
#define MPI_Type_lb PMPI_Type_lb
|
||||
#undef MPI_Type_size
|
||||
#define MPI_Type_size PMPI_Type_size
|
||||
#undef MPI_Type_struct
|
||||
#define MPI_Type_struct PMPI_Type_struct
|
||||
#undef MPI_Type_ub
|
||||
#define MPI_Type_ub PMPI_Type_ub
|
||||
#undef MPI_Type_vector
|
||||
#define MPI_Type_vector PMPI_Type_vector
|
||||
#undef MPI_Unpack
|
||||
|
||||
@ -42,6 +42,7 @@ int MPI_File_get_type_extent(MPI_File fh, MPI_Datatype datatype, MPI_Aint *exten
|
||||
int error_code;
|
||||
ADIO_File adio_fh;
|
||||
static char myname[] = "MPI_FILE_GET_TYPE_EXTENT";
|
||||
MPI_Aint lb;
|
||||
|
||||
adio_fh = MPIO_File_resolve(fh);
|
||||
|
||||
@ -52,7 +53,7 @@ int MPI_File_get_type_extent(MPI_File fh, MPI_Datatype datatype, MPI_Aint *exten
|
||||
|
||||
/* FIXME: handle other file data representations */
|
||||
|
||||
error_code = MPI_Type_extent(datatype, extent);
|
||||
error_code = MPI_Type_get_extent(datatype, &lb, extent);
|
||||
|
||||
fn_exit:
|
||||
return error_code;
|
||||
|
||||
@ -58,7 +58,7 @@ struct MPIR_Info {
|
||||
|
||||
#define MPIR_INFO_COOKIE 5835657
|
||||
|
||||
MPI_Delete_function ADIOI_End_call;
|
||||
MPI_Comm_delete_attr_function ADIOI_End_call;
|
||||
|
||||
/* common initialization routine */
|
||||
void MPIR_MPIOInit(int * error_code);
|
||||
|
||||
@ -36,15 +36,15 @@ void MPIR_MPIOInit(int * error_code) {
|
||||
}
|
||||
/* --END ERROR HANDLING-- */
|
||||
|
||||
MPI_Keyval_create(MPI_NULL_COPY_FN, ADIOI_End_call, &ADIO_Init_keyval,
|
||||
(void *) 0);
|
||||
MPI_Comm_create_keyval (MPI_COMM_NULL_COPY_FN, ADIOI_End_call, &ADIO_Init_keyval,
|
||||
(void *) 0);
|
||||
|
||||
/* put a dummy attribute on MPI_COMM_SELF, because we want the delete
|
||||
function to be called when MPI_COMM_SELF is freed. Clarified
|
||||
in MPI-2 section 4.8, the standard mandates that attributes on
|
||||
MPI_COMM_SELF get cleaned up early in MPI_Finalize */
|
||||
|
||||
MPI_Attr_put(MPI_COMM_SELF, ADIO_Init_keyval, (void *) 0);
|
||||
MPI_Comm_set_attr (MPI_COMM_SELF, ADIO_Init_keyval, (void *) 0);
|
||||
|
||||
/* initialize ADIO */
|
||||
ADIO_Init( (int *)0, (char ***)0, error_code);
|
||||
|
||||
@ -10,7 +10,7 @@
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2006 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2015 Los Alamos National Security, LLC. All rights
|
||||
* Copyright (c) 2015-2018 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
@ -108,6 +108,7 @@ ompi_mtl_base_close(void)
|
||||
{
|
||||
/* NTH: Should we be freeing the mtl module here? */
|
||||
ompi_mtl = NULL;
|
||||
ompi_mtl_base_selected_component = NULL;
|
||||
|
||||
/* Close all remaining available modules (may be one if this is a
|
||||
OMPI RTE program, or [possibly] multiple if this is ompi_info) */
|
||||
|
||||
@ -16,10 +16,13 @@
|
||||
#include "ompi/communicator/communicator.h"
|
||||
|
||||
#define OMPI_OSC_UCX_POST_PEER_MAX 32
|
||||
#define OMPI_OSC_UCX_ATTACH_MAX 32
|
||||
#define OMPI_OSC_UCX_RKEY_BUF_MAX 1024
|
||||
|
||||
typedef struct ompi_osc_ucx_win_info {
|
||||
ucp_rkey_h rkey;
|
||||
uint64_t addr;
|
||||
bool rkey_init;
|
||||
} ompi_osc_ucx_win_info_t;
|
||||
|
||||
typedef struct ompi_osc_ucx_component {
|
||||
@ -29,6 +32,7 @@ typedef struct ompi_osc_ucx_component {
|
||||
bool enable_mpi_threads;
|
||||
opal_free_list_t requests; /* request free list for the r* communication variants */
|
||||
int num_incomplete_req_ops;
|
||||
unsigned int priority;
|
||||
} ompi_osc_ucx_component_t;
|
||||
|
||||
OMPI_DECLSPEC extern ompi_osc_ucx_component_t mca_osc_ucx_component;
|
||||
@ -59,6 +63,18 @@ typedef struct ompi_osc_ucx_epoch_type {
|
||||
#define OSC_UCX_STATE_COMPLETE_COUNT_OFFSET (sizeof(uint64_t) * 3)
|
||||
#define OSC_UCX_STATE_POST_INDEX_OFFSET (sizeof(uint64_t) * 4)
|
||||
#define OSC_UCX_STATE_POST_STATE_OFFSET (sizeof(uint64_t) * 5)
|
||||
#define OSC_UCX_STATE_DYNAMIC_WIN_CNT_OFFSET (sizeof(uint64_t) * (5 + OMPI_OSC_UCX_POST_PEER_MAX))
|
||||
|
||||
typedef struct ompi_osc_dynamic_win_info {
|
||||
uint64_t base;
|
||||
size_t size;
|
||||
char rkey_buffer[OMPI_OSC_UCX_RKEY_BUF_MAX];
|
||||
} ompi_osc_dynamic_win_info_t;
|
||||
|
||||
typedef struct ompi_osc_local_dynamic_win_info {
|
||||
ucp_mem_h memh;
|
||||
int refcnt;
|
||||
} ompi_osc_local_dynamic_win_info_t;
|
||||
|
||||
typedef struct ompi_osc_ucx_state {
|
||||
volatile uint64_t lock;
|
||||
@ -67,12 +83,16 @@ typedef struct ompi_osc_ucx_state {
|
||||
volatile uint64_t complete_count; /* # msgs received from complete processes */
|
||||
volatile uint64_t post_index;
|
||||
volatile uint64_t post_state[OMPI_OSC_UCX_POST_PEER_MAX];
|
||||
volatile uint64_t dynamic_win_count;
|
||||
volatile ompi_osc_dynamic_win_info_t dynamic_wins[OMPI_OSC_UCX_ATTACH_MAX];
|
||||
} ompi_osc_ucx_state_t;
|
||||
|
||||
typedef struct ompi_osc_ucx_module {
|
||||
ompi_osc_base_module_t super;
|
||||
struct ompi_communicator_t *comm;
|
||||
ucp_mem_h memh; /* remote accessible memory */
|
||||
int flavor;
|
||||
size_t size;
|
||||
ucp_mem_h state_memh;
|
||||
ompi_osc_ucx_win_info_t *win_info_array;
|
||||
ompi_osc_ucx_win_info_t *state_info_array;
|
||||
@ -82,6 +102,7 @@ typedef struct ompi_osc_ucx_module {
|
||||
int *disp_units;
|
||||
|
||||
ompi_osc_ucx_state_t state; /* remote accessible flags */
|
||||
ompi_osc_local_dynamic_win_info_t local_dynamic_win_info[OMPI_OSC_UCX_ATTACH_MAX];
|
||||
ompi_osc_ucx_epoch_type_t epoch_type;
|
||||
ompi_group_t *start_group;
|
||||
ompi_group_t *post_group;
|
||||
@ -184,6 +205,10 @@ int ompi_osc_ucx_flush_all(struct ompi_win_t *win);
|
||||
int ompi_osc_ucx_flush_local(int target, struct ompi_win_t *win);
|
||||
int ompi_osc_ucx_flush_local_all(struct ompi_win_t *win);
|
||||
|
||||
int ompi_osc_find_attached_region_position(ompi_osc_dynamic_win_info_t *dynamic_wins,
|
||||
int min_index, int max_index,
|
||||
uint64_t base, size_t len, int *insert);
|
||||
|
||||
void req_completion(void *request, ucs_status_t status);
|
||||
void internal_req_init(void *request);
|
||||
|
||||
|
||||
@ -325,13 +325,68 @@ static inline int end_atomicity(ompi_osc_ucx_module_t *module, ucp_ep_h ep, int
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
static inline int get_dynamic_win_info(uint64_t remote_addr, ompi_osc_ucx_module_t *module,
|
||||
ucp_ep_h ep, int target) {
|
||||
ucp_rkey_h state_rkey = (module->state_info_array)[target].rkey;
|
||||
uint64_t remote_state_addr = (module->state_info_array)[target].addr + OSC_UCX_STATE_DYNAMIC_WIN_CNT_OFFSET;
|
||||
size_t len = sizeof(uint64_t) + sizeof(ompi_osc_dynamic_win_info_t) * OMPI_OSC_UCX_ATTACH_MAX;
|
||||
char *temp_buf = malloc(len);
|
||||
ompi_osc_dynamic_win_info_t *temp_dynamic_wins;
|
||||
int win_count, contain, insert = -1;
|
||||
ucs_status_t status;
|
||||
|
||||
if ((module->win_info_array[target]).rkey_init == true) {
|
||||
ucp_rkey_destroy((module->win_info_array[target]).rkey);
|
||||
(module->win_info_array[target]).rkey_init == false;
|
||||
}
|
||||
|
||||
status = ucp_get_nbi(ep, (void *)temp_buf, len, remote_state_addr, state_rkey);
|
||||
if (status != UCS_OK && status != UCS_INPROGRESS) {
|
||||
opal_output_verbose(1, ompi_osc_base_framework.framework_output,
|
||||
"%s:%d: ucp_get_nbi failed: %d\n",
|
||||
__FILE__, __LINE__, status);
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
status = ucp_ep_flush(ep);
|
||||
if (status != UCS_OK) {
|
||||
opal_output_verbose(1, ompi_osc_base_framework.framework_output,
|
||||
"%s:%d: ucp_ep_flush failed: %d\n",
|
||||
__FILE__, __LINE__, status);
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
memcpy(&win_count, temp_buf, sizeof(uint64_t));
|
||||
assert(win_count > 0 && win_count <= OMPI_OSC_UCX_ATTACH_MAX);
|
||||
|
||||
temp_dynamic_wins = (ompi_osc_dynamic_win_info_t *)(temp_buf + sizeof(uint64_t));
|
||||
contain = ompi_osc_find_attached_region_position(temp_dynamic_wins, 0, win_count,
|
||||
remote_addr, 1, &insert);
|
||||
assert(contain >= 0 && contain < win_count);
|
||||
|
||||
status = ucp_ep_rkey_unpack(ep, temp_dynamic_wins[contain].rkey_buffer,
|
||||
&((module->win_info_array[target]).rkey));
|
||||
if (status != UCS_OK) {
|
||||
opal_output_verbose(1, ompi_osc_base_framework.framework_output,
|
||||
"%s:%d: ucp_ep_rkey_unpack failed: %d\n",
|
||||
__FILE__, __LINE__, status);
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
(module->win_info_array[target]).rkey_init = true;
|
||||
|
||||
free(temp_buf);
|
||||
|
||||
return status;
|
||||
}
|
||||
|
||||
int ompi_osc_ucx_put(const void *origin_addr, int origin_count, struct ompi_datatype_t *origin_dt,
|
||||
int target, ptrdiff_t target_disp, int target_count,
|
||||
struct ompi_datatype_t *target_dt, struct ompi_win_t *win) {
|
||||
ompi_osc_ucx_module_t *module = (ompi_osc_ucx_module_t*) win->w_osc_module;
|
||||
ucp_ep_h ep = OSC_UCX_GET_EP(module->comm, target);
|
||||
uint64_t remote_addr = (module->win_info_array[target]).addr + target_disp * OSC_UCX_GET_DISP(module, target);
|
||||
ucp_rkey_h rkey = (module->win_info_array[target]).rkey;
|
||||
ucp_rkey_h rkey;
|
||||
bool is_origin_contig = false, is_target_contig = false;
|
||||
ptrdiff_t origin_lb, origin_extent, target_lb, target_extent;
|
||||
ucs_status_t status;
|
||||
@ -342,6 +397,15 @@ int ompi_osc_ucx_put(const void *origin_addr, int origin_count, struct ompi_data
|
||||
return ret;
|
||||
}
|
||||
|
||||
if (module->flavor == MPI_WIN_FLAVOR_DYNAMIC) {
|
||||
status = get_dynamic_win_info(remote_addr, module, ep, target);
|
||||
if (status != UCS_OK) {
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
}
|
||||
|
||||
rkey = (module->win_info_array[target]).rkey;
|
||||
|
||||
ompi_datatype_get_true_extent(origin_dt, &origin_lb, &origin_extent);
|
||||
ompi_datatype_get_true_extent(target_dt, &target_lb, &target_extent);
|
||||
|
||||
@ -378,7 +442,7 @@ int ompi_osc_ucx_get(void *origin_addr, int origin_count,
|
||||
ompi_osc_ucx_module_t *module = (ompi_osc_ucx_module_t*) win->w_osc_module;
|
||||
ucp_ep_h ep = OSC_UCX_GET_EP(module->comm, target);
|
||||
uint64_t remote_addr = (module->win_info_array[target]).addr + target_disp * OSC_UCX_GET_DISP(module, target);
|
||||
ucp_rkey_h rkey = (module->win_info_array[target]).rkey;
|
||||
ucp_rkey_h rkey;
|
||||
ptrdiff_t origin_lb, origin_extent, target_lb, target_extent;
|
||||
bool is_origin_contig = false, is_target_contig = false;
|
||||
ucs_status_t status;
|
||||
@ -389,6 +453,15 @@ int ompi_osc_ucx_get(void *origin_addr, int origin_count,
|
||||
return ret;
|
||||
}
|
||||
|
||||
if (module->flavor == MPI_WIN_FLAVOR_DYNAMIC) {
|
||||
status = get_dynamic_win_info(remote_addr, module, ep, target);
|
||||
if (status != UCS_OK) {
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
}
|
||||
|
||||
rkey = (module->win_info_array[target]).rkey;
|
||||
|
||||
ompi_datatype_get_true_extent(origin_dt, &origin_lb, &origin_extent);
|
||||
ompi_datatype_get_true_extent(target_dt, &target_lb, &target_extent);
|
||||
|
||||
@ -557,10 +630,11 @@ int ompi_osc_ucx_compare_and_swap(const void *origin_addr, const void *compare_a
|
||||
ompi_osc_ucx_module_t *module = (ompi_osc_ucx_module_t *)win->w_osc_module;
|
||||
ucp_ep_h ep = OSC_UCX_GET_EP(module->comm, target);
|
||||
uint64_t remote_addr = (module->win_info_array[target]).addr + target_disp * OSC_UCX_GET_DISP(module, target);
|
||||
ucp_rkey_h rkey = (module->win_info_array[target]).rkey;
|
||||
ucp_rkey_h rkey;
|
||||
size_t dt_bytes;
|
||||
ompi_osc_ucx_internal_request_t *req = NULL;
|
||||
int ret = OMPI_SUCCESS;
|
||||
ucs_status_t status;
|
||||
|
||||
ret = check_sync_state(module, target, false);
|
||||
if (ret != OMPI_SUCCESS) {
|
||||
@ -572,6 +646,15 @@ int ompi_osc_ucx_compare_and_swap(const void *origin_addr, const void *compare_a
|
||||
return ret;
|
||||
}
|
||||
|
||||
if (module->flavor == MPI_WIN_FLAVOR_DYNAMIC) {
|
||||
status = get_dynamic_win_info(remote_addr, module, ep, target);
|
||||
if (status != UCS_OK) {
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
}
|
||||
|
||||
rkey = (module->win_info_array[target]).rkey;
|
||||
|
||||
ompi_datatype_type_size(dt, &dt_bytes);
|
||||
memcpy(result_addr, origin_addr, dt_bytes);
|
||||
req = ucp_atomic_fetch_nb(ep, UCP_ATOMIC_FETCH_OP_CSWAP, *(uint64_t *)compare_addr,
|
||||
@ -604,17 +687,27 @@ int ompi_osc_ucx_fetch_and_op(const void *origin_addr, void *result_addr,
|
||||
op == &ompi_mpi_op_sum.op) {
|
||||
ucp_ep_h ep = OSC_UCX_GET_EP(module->comm, target);
|
||||
uint64_t remote_addr = (module->win_info_array[target]).addr + target_disp * OSC_UCX_GET_DISP(module, target);
|
||||
ucp_rkey_h rkey = (module->win_info_array[target]).rkey;
|
||||
ucp_rkey_h rkey;
|
||||
uint64_t value = *(uint64_t *)origin_addr;
|
||||
ucp_atomic_fetch_op_t opcode;
|
||||
size_t dt_bytes;
|
||||
ompi_osc_ucx_internal_request_t *req = NULL;
|
||||
ucs_status_t status;
|
||||
|
||||
ret = start_atomicity(module, ep, target);
|
||||
if (ret != OMPI_SUCCESS) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
if (module->flavor == MPI_WIN_FLAVOR_DYNAMIC) {
|
||||
status = get_dynamic_win_info(remote_addr, module, ep, target);
|
||||
if (status != UCS_OK) {
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
}
|
||||
|
||||
rkey = (module->win_info_array[target]).rkey;
|
||||
|
||||
ompi_datatype_type_size(dt, &dt_bytes);
|
||||
|
||||
if (op == &ompi_mpi_op_replace.op) {
|
||||
@ -789,7 +882,7 @@ int ompi_osc_ucx_rput(const void *origin_addr, int origin_count,
|
||||
ompi_osc_ucx_module_t *module = (ompi_osc_ucx_module_t*) win->w_osc_module;
|
||||
ucp_ep_h ep = OSC_UCX_GET_EP(module->comm, target);
|
||||
uint64_t remote_addr = (module->state_info_array[target]).addr + OSC_UCX_STATE_REQ_FLAG_OFFSET;
|
||||
ucp_rkey_h rkey = (module->state_info_array[target]).rkey;
|
||||
ucp_rkey_h rkey;
|
||||
ompi_osc_ucx_request_t *ucx_req = NULL;
|
||||
ompi_osc_ucx_internal_request_t *internal_req = NULL;
|
||||
ucs_status_t status;
|
||||
@ -800,6 +893,15 @@ int ompi_osc_ucx_rput(const void *origin_addr, int origin_count,
|
||||
return ret;
|
||||
}
|
||||
|
||||
if (module->flavor == MPI_WIN_FLAVOR_DYNAMIC) {
|
||||
status = get_dynamic_win_info(remote_addr, module, ep, target);
|
||||
if (status != UCS_OK) {
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
}
|
||||
|
||||
rkey = (module->win_info_array[target]).rkey;
|
||||
|
||||
OMPI_OSC_UCX_REQUEST_ALLOC(win, ucx_req);
|
||||
if (NULL == ucx_req) {
|
||||
return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
|
||||
@ -843,7 +945,7 @@ int ompi_osc_ucx_rget(void *origin_addr, int origin_count,
|
||||
ompi_osc_ucx_module_t *module = (ompi_osc_ucx_module_t*) win->w_osc_module;
|
||||
ucp_ep_h ep = OSC_UCX_GET_EP(module->comm, target);
|
||||
uint64_t remote_addr = (module->state_info_array[target]).addr + OSC_UCX_STATE_REQ_FLAG_OFFSET;
|
||||
ucp_rkey_h rkey = (module->state_info_array[target]).rkey;
|
||||
ucp_rkey_h rkey;
|
||||
ompi_osc_ucx_request_t *ucx_req = NULL;
|
||||
ompi_osc_ucx_internal_request_t *internal_req = NULL;
|
||||
ucs_status_t status;
|
||||
@ -854,6 +956,15 @@ int ompi_osc_ucx_rget(void *origin_addr, int origin_count,
|
||||
return ret;
|
||||
}
|
||||
|
||||
if (module->flavor == MPI_WIN_FLAVOR_DYNAMIC) {
|
||||
status = get_dynamic_win_info(remote_addr, module, ep, target);
|
||||
if (status != UCS_OK) {
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
}
|
||||
|
||||
rkey = (module->win_info_array[target]).rkey;
|
||||
|
||||
OMPI_OSC_UCX_REQUEST_ALLOC(win, ucx_req);
|
||||
if (NULL == ucx_req) {
|
||||
return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
|
||||
|
||||
@ -91,6 +91,15 @@ static int component_open(void) {
|
||||
}
|
||||
|
||||
static int component_register(void) {
|
||||
char *description_str;
|
||||
mca_osc_ucx_component.priority = 0;
|
||||
asprintf(&description_str, "Priority of the osc/ucx component (default: %d)",
|
||||
mca_osc_ucx_component.priority);
|
||||
(void) mca_base_component_var_register(&mca_osc_ucx_component.super.osc_version, "priority", description_str,
|
||||
MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL, 0, 0, OPAL_INFO_LVL_3,
|
||||
MCA_BASE_VAR_SCOPE_GROUP, &mca_osc_ucx_component.priority);
|
||||
free(description_str);
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
@ -201,7 +210,7 @@ static int component_finalize(void) {
|
||||
static int component_query(struct ompi_win_t *win, void **base, size_t size, int disp_unit,
|
||||
struct ompi_communicator_t *comm, struct opal_info_t *info, int flavor) {
|
||||
if (MPI_WIN_FLAVOR_SHARED == flavor) return -1;
|
||||
return 100;
|
||||
return mca_osc_ucx_component.priority;
|
||||
}
|
||||
|
||||
static inline int allgather_len_and_info(void *my_info, int my_info_len, char **recv_info,
|
||||
@ -243,7 +252,10 @@ static inline int mem_map(void **base, size_t size, ucp_mem_h *memh_ptr,
|
||||
ucs_status_t status;
|
||||
int ret = OMPI_SUCCESS;
|
||||
|
||||
assert(flavor == MPI_WIN_FLAVOR_ALLOCATE || flavor == MPI_WIN_FLAVOR_CREATE);
|
||||
if (!(flavor == MPI_WIN_FLAVOR_ALLOCATE || flavor == MPI_WIN_FLAVOR_CREATE)
|
||||
|| size == 0) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
memset(&mem_params, 0, sizeof(ucp_mem_map_params_t));
|
||||
mem_params.field_mask = UCP_MEM_MAP_PARAM_FIELD_ADDRESS |
|
||||
@ -312,6 +324,7 @@ static int component_select(struct ompi_win_t *win, void **base, size_t size, in
|
||||
size_t my_info_len;
|
||||
int disps[comm_size];
|
||||
int rkey_sizes[comm_size];
|
||||
uint64_t zero = 0;
|
||||
|
||||
/* the osc/sm component is the exclusive provider for support for
|
||||
* shared memory windows */
|
||||
@ -376,10 +389,14 @@ static int component_select(struct ompi_win_t *win, void **base, size_t size, in
|
||||
goto error;
|
||||
}
|
||||
|
||||
*model = MPI_WIN_UNIFIED;
|
||||
asprintf(&name, "ucx window %d", ompi_comm_get_cid(module->comm));
|
||||
ompi_win_set_name(win, name);
|
||||
free(name);
|
||||
|
||||
module->flavor = flavor;
|
||||
module->size = size;
|
||||
|
||||
/* share everyone's displacement units. Only do an allgather if
|
||||
strictly necessary, since it requires O(p) state. */
|
||||
values[0] = disp_unit;
|
||||
@ -497,14 +514,18 @@ static int component_select(struct ompi_win_t *win, void **base, size_t size, in
|
||||
goto error;
|
||||
}
|
||||
|
||||
status = ucp_rkey_pack(mca_osc_ucx_component.ucp_context, module->memh,
|
||||
&rkey_buffer, &rkey_buffer_size);
|
||||
if (status != UCS_OK) {
|
||||
opal_output_verbose(1, ompi_osc_base_framework.framework_output,
|
||||
"%s:%d: ucp_rkey_pack failed: %d\n",
|
||||
__FILE__, __LINE__, status);
|
||||
ret = OMPI_ERROR;
|
||||
goto error;
|
||||
if (size > 0 && (flavor == MPI_WIN_FLAVOR_ALLOCATE || flavor == MPI_WIN_FLAVOR_CREATE)) {
|
||||
status = ucp_rkey_pack(mca_osc_ucx_component.ucp_context, module->memh,
|
||||
&rkey_buffer, &rkey_buffer_size);
|
||||
if (status != UCS_OK) {
|
||||
opal_output_verbose(1, ompi_osc_base_framework.framework_output,
|
||||
"%s:%d: ucp_rkey_pack failed: %d\n",
|
||||
__FILE__, __LINE__, status);
|
||||
ret = OMPI_ERROR;
|
||||
goto error;
|
||||
}
|
||||
} else {
|
||||
rkey_buffer_size = 0;
|
||||
}
|
||||
|
||||
status = ucp_rkey_pack(mca_osc_ucx_component.ucp_context, module->state_memh,
|
||||
@ -524,7 +545,11 @@ static int component_select(struct ompi_win_t *win, void **base, size_t size, in
|
||||
goto error;
|
||||
}
|
||||
|
||||
memcpy(my_info, base, sizeof(uint64_t));
|
||||
if (flavor == MPI_WIN_FLAVOR_ALLOCATE || flavor == MPI_WIN_FLAVOR_CREATE) {
|
||||
memcpy(my_info, base, sizeof(uint64_t));
|
||||
} else {
|
||||
memcpy(my_info, &zero, sizeof(uint64_t));
|
||||
}
|
||||
memcpy((void *)((char *)my_info + sizeof(uint64_t)), &state_base, sizeof(uint64_t));
|
||||
memcpy((void *)((char *)my_info + 2 * sizeof(uint64_t)), rkey_buffer, rkey_buffer_size);
|
||||
memcpy((void *)((char *)my_info + 2 * sizeof(uint64_t) + rkey_buffer_size),
|
||||
@ -550,14 +575,18 @@ static int component_select(struct ompi_win_t *win, void **base, size_t size, in
|
||||
memcpy(&(module->state_info_array[i]).addr, &recv_buf[disps[i] + sizeof(uint64_t)],
|
||||
sizeof(uint64_t));
|
||||
|
||||
status = ucp_ep_rkey_unpack(ep, &(recv_buf[disps[i] + 2 * sizeof(uint64_t)]),
|
||||
&((module->win_info_array[i]).rkey));
|
||||
if (status != UCS_OK) {
|
||||
opal_output_verbose(1, ompi_osc_base_framework.framework_output,
|
||||
"%s:%d: ucp_ep_rkey_unpack failed: %d\n",
|
||||
__FILE__, __LINE__, status);
|
||||
ret = OMPI_ERROR;
|
||||
goto error;
|
||||
(module->win_info_array[i]).rkey_init = false;
|
||||
if (size > 0 && (flavor == MPI_WIN_FLAVOR_ALLOCATE || flavor == MPI_WIN_FLAVOR_CREATE)) {
|
||||
status = ucp_ep_rkey_unpack(ep, &(recv_buf[disps[i] + 2 * sizeof(uint64_t)]),
|
||||
&((module->win_info_array[i]).rkey));
|
||||
if (status != UCS_OK) {
|
||||
opal_output_verbose(1, ompi_osc_base_framework.framework_output,
|
||||
"%s:%d: ucp_ep_rkey_unpack failed: %d\n",
|
||||
__FILE__, __LINE__, status);
|
||||
ret = OMPI_ERROR;
|
||||
goto error;
|
||||
}
|
||||
(module->win_info_array[i]).rkey_init = true;
|
||||
}
|
||||
|
||||
status = ucp_ep_rkey_unpack(ep, &(recv_buf[disps[i] + 2 * sizeof(uint64_t) + rkey_sizes[i]]),
|
||||
@ -569,12 +598,15 @@ static int component_select(struct ompi_win_t *win, void **base, size_t size, in
|
||||
ret = OMPI_ERROR;
|
||||
goto error;
|
||||
}
|
||||
(module->state_info_array[i]).rkey_init = true;
|
||||
}
|
||||
|
||||
free(my_info);
|
||||
free(recv_buf);
|
||||
|
||||
ucp_rkey_buffer_release(rkey_buffer);
|
||||
if (rkey_buffer_size != 0) {
|
||||
ucp_rkey_buffer_release(rkey_buffer);
|
||||
}
|
||||
ucp_rkey_buffer_release(state_rkey_buffer);
|
||||
|
||||
module->state.lock = TARGET_LOCK_UNLOCKED;
|
||||
@ -583,6 +615,10 @@ static int component_select(struct ompi_win_t *win, void **base, size_t size, in
|
||||
module->state.complete_count = 0;
|
||||
module->state.req_flag = 0;
|
||||
module->state.acc_lock = TARGET_LOCK_UNLOCKED;
|
||||
module->state.dynamic_win_count = 0;
|
||||
for (i = 0; i < OMPI_OSC_UCX_ATTACH_MAX; i++) {
|
||||
module->local_dynamic_win_info[i].refcnt = 0;
|
||||
}
|
||||
module->epoch_type.access = NONE_EPOCH;
|
||||
module->epoch_type.exposure = NONE_EPOCH;
|
||||
module->lock_count = 0;
|
||||
@ -643,11 +679,116 @@ static int component_select(struct ompi_win_t *win, void **base, size_t size, in
|
||||
return ret;
|
||||
}
|
||||
|
||||
int ompi_osc_find_attached_region_position(ompi_osc_dynamic_win_info_t *dynamic_wins,
|
||||
int min_index, int max_index,
|
||||
uint64_t base, size_t len, int *insert) {
|
||||
int mid_index = (max_index + min_index) >> 1;
|
||||
|
||||
if (min_index > max_index) {
|
||||
(*insert) = min_index;
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (dynamic_wins[mid_index].base > base) {
|
||||
return ompi_osc_find_attached_region_position(dynamic_wins, min_index, mid_index-1,
|
||||
base, len, insert);
|
||||
} else if (base + len < dynamic_wins[mid_index].base + dynamic_wins[mid_index].size) {
|
||||
return mid_index;
|
||||
} else {
|
||||
return ompi_osc_find_attached_region_position(dynamic_wins, mid_index+1, max_index,
|
||||
base, len, insert);
|
||||
}
|
||||
}
|
||||
|
||||
int ompi_osc_ucx_win_attach(struct ompi_win_t *win, void *base, size_t len) {
|
||||
return OMPI_SUCCESS;
|
||||
ompi_osc_ucx_module_t *module = (ompi_osc_ucx_module_t*) win->w_osc_module;
|
||||
int insert_index = -1, contain_index;
|
||||
void *rkey_buffer;
|
||||
size_t rkey_buffer_size;
|
||||
int ret = OMPI_SUCCESS;
|
||||
ucs_status_t status;
|
||||
|
||||
if (module->state.dynamic_win_count >= OMPI_OSC_UCX_ATTACH_MAX) {
|
||||
return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
|
||||
}
|
||||
|
||||
if (module->state.dynamic_win_count > 0) {
|
||||
contain_index = ompi_osc_find_attached_region_position((ompi_osc_dynamic_win_info_t *)module->state.dynamic_wins,
|
||||
0, (int)module->state.dynamic_win_count,
|
||||
(uint64_t)base, len, &insert_index);
|
||||
if (contain_index >= 0) {
|
||||
module->local_dynamic_win_info[contain_index].refcnt++;
|
||||
return ret;
|
||||
}
|
||||
|
||||
assert(insert_index >= 0 && insert_index < module->state.dynamic_win_count);
|
||||
|
||||
memmove((void *)&module->local_dynamic_win_info[insert_index+1],
|
||||
(void *)&module->local_dynamic_win_info[insert_index],
|
||||
(OMPI_OSC_UCX_ATTACH_MAX - (insert_index + 1)) * sizeof(ompi_osc_local_dynamic_win_info_t));
|
||||
memmove((void *)&module->state.dynamic_wins[insert_index+1],
|
||||
(void *)&module->state.dynamic_wins[insert_index],
|
||||
(OMPI_OSC_UCX_ATTACH_MAX - (insert_index + 1)) * sizeof(ompi_osc_dynamic_win_info_t));
|
||||
} else {
|
||||
insert_index = 0;
|
||||
}
|
||||
|
||||
ret = mem_map(&base, len, &(module->local_dynamic_win_info[insert_index].memh),
|
||||
module, MPI_WIN_FLAVOR_CREATE);
|
||||
if (ret != OMPI_SUCCESS) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
module->state.dynamic_wins[insert_index].base = (uint64_t)base;
|
||||
module->state.dynamic_wins[insert_index].size = len;
|
||||
|
||||
status = ucp_rkey_pack(mca_osc_ucx_component.ucp_context,
|
||||
module->local_dynamic_win_info[insert_index].memh,
|
||||
&rkey_buffer, (size_t *)&rkey_buffer_size);
|
||||
if (status != UCS_OK) {
|
||||
opal_output_verbose(1, ompi_osc_base_framework.framework_output,
|
||||
"%s:%d: ucp_rkey_pack failed: %d\n",
|
||||
__FILE__, __LINE__, status);
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
assert(rkey_buffer_size <= OMPI_OSC_UCX_RKEY_BUF_MAX);
|
||||
memcpy((char *)(module->state.dynamic_wins[insert_index].rkey_buffer),
|
||||
(char *)rkey_buffer, rkey_buffer_size);
|
||||
|
||||
module->local_dynamic_win_info[insert_index].refcnt++;
|
||||
module->state.dynamic_win_count++;
|
||||
|
||||
ucp_rkey_buffer_release(rkey_buffer);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
int ompi_osc_ucx_win_detach(struct ompi_win_t *win, const void *base) {
|
||||
ompi_osc_ucx_module_t *module = (ompi_osc_ucx_module_t*) win->w_osc_module;
|
||||
int insert, contain;
|
||||
|
||||
assert(module->state.dynamic_win_count > 0);
|
||||
|
||||
contain = ompi_osc_find_attached_region_position((ompi_osc_dynamic_win_info_t *)module->state.dynamic_wins,
|
||||
0, (int)module->state.dynamic_win_count,
|
||||
(uint64_t)base, 1, &insert);
|
||||
assert(contain >= 0 && contain < module->state.dynamic_win_count);
|
||||
|
||||
module->local_dynamic_win_info[contain].refcnt--;
|
||||
if (module->local_dynamic_win_info[contain].refcnt == 0) {
|
||||
ucp_mem_unmap(mca_osc_ucx_component.ucp_context,
|
||||
module->local_dynamic_win_info[contain].memh);
|
||||
memmove((void *)&(module->local_dynamic_win_info[contain]),
|
||||
(void *)&(module->local_dynamic_win_info[contain+1]),
|
||||
(OMPI_OSC_UCX_ATTACH_MAX - (contain + 1)) * sizeof(ompi_osc_local_dynamic_win_info_t));
|
||||
memmove((void *)&module->state.dynamic_wins[contain],
|
||||
(void *)&module->state.dynamic_wins[contain+1],
|
||||
(OMPI_OSC_UCX_ATTACH_MAX - (contain + 1)) * sizeof(ompi_osc_dynamic_win_info_t));
|
||||
|
||||
module->state.dynamic_win_count--;
|
||||
}
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
@ -679,7 +820,10 @@ int ompi_osc_ucx_free(struct ompi_win_t *win) {
|
||||
module->comm->c_coll->coll_barrier_module);
|
||||
|
||||
for (i = 0; i < ompi_comm_size(module->comm); i++) {
|
||||
ucp_rkey_destroy((module->win_info_array[i]).rkey);
|
||||
if ((module->win_info_array[i]).rkey_init == true) {
|
||||
ucp_rkey_destroy((module->win_info_array[i]).rkey);
|
||||
(module->win_info_array[i]).rkey_init == false;
|
||||
}
|
||||
ucp_rkey_destroy((module->state_info_array[i]).rkey);
|
||||
}
|
||||
free(module->win_info_array);
|
||||
@ -687,7 +831,10 @@ int ompi_osc_ucx_free(struct ompi_win_t *win) {
|
||||
|
||||
free(module->per_target_ops_nums);
|
||||
|
||||
ucp_mem_unmap(mca_osc_ucx_component.ucp_context, module->memh);
|
||||
if ((module->flavor == MPI_WIN_FLAVOR_ALLOCATE || module->flavor == MPI_WIN_FLAVOR_CREATE)
|
||||
&& module->size > 0) {
|
||||
ucp_mem_unmap(mca_osc_ucx_component.ucp_context, module->memh);
|
||||
}
|
||||
ucp_mem_unmap(mca_osc_ucx_component.ucp_context, module->state_memh);
|
||||
|
||||
if (module->disp_units) free(module->disp_units);
|
||||
|
||||
@ -78,6 +78,7 @@ mca_pml_ucx_module_t ompi_pml_ucx = {
|
||||
#define PML_UCX_REQ_ALLOCA() \
|
||||
((char *)alloca(ompi_pml_ucx.request_size) + ompi_pml_ucx.request_size);
|
||||
|
||||
|
||||
static int mca_pml_ucx_send_worker_address(void)
|
||||
{
|
||||
ucp_address_t *address;
|
||||
@ -111,9 +112,10 @@ static int mca_pml_ucx_recv_worker_address(ompi_proc_t *proc,
|
||||
|
||||
*address_p = NULL;
|
||||
OPAL_MODEX_RECV(ret, &mca_pml_ucx_component.pmlm_version, &proc->super.proc_name,
|
||||
(void**)address_p, addrlen_p);
|
||||
(void**)address_p, addrlen_p);
|
||||
if (ret < 0) {
|
||||
PML_UCX_ERROR("Failed to receive EP address");
|
||||
PML_UCX_ERROR("Failed to receive UCX worker address: %s (%d)",
|
||||
opal_strerror(ret), ret);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
@ -267,7 +269,7 @@ int mca_pml_ucx_cleanup(void)
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
ucp_ep_h mca_pml_ucx_add_proc(ompi_communicator_t *comm, int dst)
|
||||
static ucp_ep_h mca_pml_ucx_add_proc_common(ompi_proc_t *proc)
|
||||
{
|
||||
ucp_ep_params_t ep_params;
|
||||
ucp_address_t *address;
|
||||
@ -276,23 +278,12 @@ ucp_ep_h mca_pml_ucx_add_proc(ompi_communicator_t *comm, int dst)
|
||||
ucp_ep_h ep;
|
||||
int ret;
|
||||
|
||||
ompi_proc_t *proc0 = ompi_comm_peer_lookup(comm, 0);
|
||||
ompi_proc_t *proc_peer = ompi_comm_peer_lookup(comm, dst);
|
||||
|
||||
/* Note, mca_pml_base_pml_check_selected, doesn't use 3rd argument */
|
||||
if (OMPI_SUCCESS != (ret = mca_pml_base_pml_check_selected("ucx",
|
||||
&proc0,
|
||||
dst))) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
ret = mca_pml_ucx_recv_worker_address(proc_peer, &address, &addrlen);
|
||||
ret = mca_pml_ucx_recv_worker_address(proc, &address, &addrlen);
|
||||
if (ret < 0) {
|
||||
PML_UCX_ERROR("Failed to receive worker address from proc: %d", proc_peer->super.proc_name.vpid);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
PML_UCX_VERBOSE(2, "connecting to proc. %d", proc_peer->super.proc_name.vpid);
|
||||
PML_UCX_VERBOSE(2, "connecting to proc. %d", proc->super.proc_name.vpid);
|
||||
|
||||
ep_params.field_mask = UCP_EP_PARAM_FIELD_REMOTE_ADDRESS;
|
||||
ep_params.address = address;
|
||||
@ -300,68 +291,80 @@ ucp_ep_h mca_pml_ucx_add_proc(ompi_communicator_t *comm, int dst)
|
||||
status = ucp_ep_create(ompi_pml_ucx.ucp_worker, &ep_params, &ep);
|
||||
free(address);
|
||||
if (UCS_OK != status) {
|
||||
PML_UCX_ERROR("Failed to connect to proc: %d, %s", proc_peer->super.proc_name.vpid,
|
||||
ucs_status_string(status));
|
||||
PML_UCX_ERROR("ucp_ep_create(proc=%d) failed: %s",
|
||||
proc->super.proc_name.vpid,
|
||||
ucs_status_string(status));
|
||||
return NULL;
|
||||
}
|
||||
|
||||
proc_peer->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_PML] = ep;
|
||||
|
||||
proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_PML] = ep;
|
||||
return ep;
|
||||
}
|
||||
|
||||
static ucp_ep_h mca_pml_ucx_add_proc(ompi_communicator_t *comm, int dst)
|
||||
{
|
||||
ompi_proc_t *proc0 = ompi_comm_peer_lookup(comm, 0);
|
||||
ompi_proc_t *proc_peer = ompi_comm_peer_lookup(comm, dst);
|
||||
int ret;
|
||||
|
||||
/* Note, mca_pml_base_pml_check_selected, doesn't use 3rd argument */
|
||||
if (OMPI_SUCCESS != (ret = mca_pml_base_pml_check_selected("ucx",
|
||||
&proc0,
|
||||
dst))) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
return mca_pml_ucx_add_proc_common(proc_peer);
|
||||
}
|
||||
|
||||
int mca_pml_ucx_add_procs(struct ompi_proc_t **procs, size_t nprocs)
|
||||
{
|
||||
ucp_ep_params_t ep_params;
|
||||
ucp_address_t *address;
|
||||
ucs_status_t status;
|
||||
ompi_proc_t *proc;
|
||||
size_t addrlen;
|
||||
ucp_ep_h ep;
|
||||
size_t i;
|
||||
int ret;
|
||||
|
||||
if (OMPI_SUCCESS != (ret = mca_pml_base_pml_check_selected("ucx",
|
||||
procs,
|
||||
nprocs))) {
|
||||
procs,
|
||||
nprocs))) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
for (i = 0; i < nprocs; ++i) {
|
||||
proc = procs[(i + OMPI_PROC_MY_NAME->vpid) % nprocs];
|
||||
|
||||
ret = mca_pml_ucx_recv_worker_address(proc, &address, &addrlen);
|
||||
if (ret < 0) {
|
||||
PML_UCX_ERROR("Failed to receive worker address from proc: %d",
|
||||
proc->super.proc_name.vpid);
|
||||
return ret;
|
||||
}
|
||||
|
||||
if (proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_PML]) {
|
||||
PML_UCX_VERBOSE(3, "already connected to proc. %d", proc->super.proc_name.vpid);
|
||||
continue;
|
||||
}
|
||||
|
||||
PML_UCX_VERBOSE(2, "connecting to proc. %d", proc->super.proc_name.vpid);
|
||||
|
||||
ep_params.field_mask = UCP_EP_PARAM_FIELD_REMOTE_ADDRESS;
|
||||
ep_params.address = address;
|
||||
|
||||
status = ucp_ep_create(ompi_pml_ucx.ucp_worker, &ep_params, &ep);
|
||||
free(address);
|
||||
|
||||
if (UCS_OK != status) {
|
||||
PML_UCX_ERROR("Failed to connect to proc: %d, %s", proc->super.proc_name.vpid,
|
||||
ucs_status_string(status));
|
||||
ep = mca_pml_ucx_add_proc_common(proc);
|
||||
if (ep == NULL) {
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_PML] = ep;
|
||||
}
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
static inline ucp_ep_h mca_pml_ucx_get_ep(ompi_communicator_t *comm, int rank)
|
||||
{
|
||||
ucp_ep_h ep;
|
||||
|
||||
ep = ompi_comm_peer_lookup(comm, rank)->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_PML];
|
||||
if (OPAL_LIKELY(ep != NULL)) {
|
||||
return ep;
|
||||
}
|
||||
|
||||
ep = mca_pml_ucx_add_proc(comm, rank);
|
||||
if (OPAL_LIKELY(ep != NULL)) {
|
||||
return ep;
|
||||
}
|
||||
|
||||
if (rank >= ompi_comm_size(comm)) {
|
||||
PML_UCX_ERROR("Rank number (%d) is larger than communicator size (%d)",
|
||||
rank, ompi_comm_size(comm));
|
||||
} else {
|
||||
PML_UCX_ERROR("Failed to resolve UCX endpoint for rank %d", rank);
|
||||
}
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static void mca_pml_ucx_waitall(void **reqs, size_t *count_p)
|
||||
{
|
||||
ucs_status_t status;
|
||||
@ -581,7 +584,6 @@ int mca_pml_ucx_isend_init(const void *buf, size_t count, ompi_datatype_t *datat
|
||||
|
||||
ep = mca_pml_ucx_get_ep(comm, dst);
|
||||
if (OPAL_UNLIKELY(NULL == ep)) {
|
||||
PML_UCX_ERROR("Failed to get ep for rank %d", dst);
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
@ -695,7 +697,6 @@ int mca_pml_ucx_isend(const void *buf, size_t count, ompi_datatype_t *datatype,
|
||||
|
||||
ep = mca_pml_ucx_get_ep(comm, dst);
|
||||
if (OPAL_UNLIKELY(NULL == ep)) {
|
||||
PML_UCX_ERROR("Failed to get ep for rank %d", dst);
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
@ -779,7 +780,6 @@ int mca_pml_ucx_send(const void *buf, size_t count, ompi_datatype_t *datatype, i
|
||||
|
||||
ep = mca_pml_ucx_get_ep(comm, dst);
|
||||
if (OPAL_UNLIKELY(NULL == ep)) {
|
||||
PML_UCX_ERROR("Failed to get ep for rank %d", dst);
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
|
||||
@ -87,7 +87,6 @@ int mca_pml_ucx_close(void);
|
||||
int mca_pml_ucx_init(void);
|
||||
int mca_pml_ucx_cleanup(void);
|
||||
|
||||
ucp_ep_h mca_pml_ucx_add_proc(ompi_communicator_t *comm, int dst);
|
||||
int mca_pml_ucx_add_procs(struct ompi_proc_t **procs, size_t nprocs);
|
||||
int mca_pml_ucx_del_procs(struct ompi_proc_t **procs, size_t nprocs);
|
||||
|
||||
|
||||
@ -136,16 +136,6 @@ void mca_pml_ucx_request_init(void *request);
|
||||
void mca_pml_ucx_request_cleanup(void *request);
|
||||
|
||||
|
||||
static inline ucp_ep_h mca_pml_ucx_get_ep(ompi_communicator_t *comm, int dst)
|
||||
{
|
||||
ucp_ep_h ep = ompi_comm_peer_lookup(comm,dst)->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_PML];
|
||||
if (OPAL_UNLIKELY(NULL == ep)) {
|
||||
ep = mca_pml_ucx_add_proc(comm, dst);
|
||||
}
|
||||
|
||||
return ep;
|
||||
}
|
||||
|
||||
static inline void mca_pml_ucx_request_reset(ompi_request_t *req)
|
||||
{
|
||||
req->req_complete = REQUEST_PENDING;
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
|
||||
/*
|
||||
* Copyright (c) 2012-2013 The University of Tennessee and The University
|
||||
* Copyright (c) 2012-2018 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2012-2013 Inria. All rights reserved.
|
||||
@ -62,7 +62,7 @@ int MPI_Dist_graph_create(MPI_Comm comm_old, int n, const int sources[],
|
||||
/* Ensure the arrays are full of valid-valued integers */
|
||||
comm_size = ompi_comm_size(comm_old);
|
||||
for( i = index = 0; i < n; ++i ) {
|
||||
if (sources[i] < 0 || sources[i] >= comm_size) {
|
||||
if (((sources[i] < 0) && (sources[i] != MPI_PROC_NULL)) || sources[i] >= comm_size) {
|
||||
return OMPI_ERRHANDLER_INVOKE(comm_old, MPI_ERR_ARG,
|
||||
FUNC_NAME);
|
||||
} else if (degrees[i] < 0) {
|
||||
@ -70,7 +70,7 @@ int MPI_Dist_graph_create(MPI_Comm comm_old, int n, const int sources[],
|
||||
FUNC_NAME);
|
||||
}
|
||||
for( j = 0; j < degrees[i]; ++j ) {
|
||||
if (destinations[index] < 0 || destinations[index] >= comm_size) {
|
||||
if (((destinations[index] < 0) && (destinations[index] != MPI_PROC_NULL)) || destinations[index] >= comm_size) {
|
||||
return OMPI_ERRHANDLER_INVOKE(comm_old, MPI_ERR_ARG,
|
||||
FUNC_NAME);
|
||||
} else if (MPI_UNWEIGHTED != weights && weights[index] < 0) {
|
||||
|
||||
@ -3,7 +3,7 @@
|
||||
* Copyright (c) 2008 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2011-2013 The University of Tennessee and The University
|
||||
* Copyright (c) 2011-2018 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2009 Cisco Systems, Inc. All rights reserved.
|
||||
@ -74,7 +74,7 @@ int MPI_Dist_graph_create_adjacent(MPI_Comm comm_old,
|
||||
}
|
||||
comm_size = ompi_comm_size(comm_old);
|
||||
for (i = 0; i < indegree; ++i) {
|
||||
if (sources[i] < 0 || sources[i] >= comm_size) {
|
||||
if (((sources[i] < 0) && (sources[i] != MPI_PROC_NULL)) || sources[i] >= comm_size) {
|
||||
return OMPI_ERRHANDLER_INVOKE(comm_old, MPI_ERR_ARG,
|
||||
"MPI_Dist_graph_create_adjacent invalid sources");
|
||||
} else if (MPI_UNWEIGHTED != sourceweights && sourceweights[i] < 0) {
|
||||
@ -83,7 +83,7 @@ int MPI_Dist_graph_create_adjacent(MPI_Comm comm_old,
|
||||
}
|
||||
}
|
||||
for (i = 0; i < outdegree; ++i) {
|
||||
if (destinations[i] < 0 || destinations[i] >= comm_size) {
|
||||
if (((destinations[i] < 0) && (destinations[i] != MPI_PROC_NULL)) || destinations[i] >= comm_size) {
|
||||
return OMPI_ERRHANDLER_INVOKE(comm_old, MPI_ERR_ARG,
|
||||
"MPI_Dist_graph_create_adjacent invalid destinations");
|
||||
} else if (MPI_UNWEIGHTED != destweights && destweights[i] < 0) {
|
||||
|
||||
@ -4,8 +4,8 @@
|
||||
! Copyright (c) 2006-2014 Cisco Systems, Inc. All rights reserved.
|
||||
! Copyright (c) 2013 Los Alamos National Security, LLC. All rights
|
||||
! reserved.
|
||||
! Copyright (c) 2015 Research Organization for Information Science
|
||||
! and Technology (RIST). All rights reserved.
|
||||
! Copyright (c) 2015-2018 Research Organization for Information Science
|
||||
! and Technology (RIST). All rights reserved.
|
||||
! $COPYRIGHT$
|
||||
!
|
||||
! Additional copyrights may follow
|
||||
@ -84,35 +84,35 @@ interface
|
||||
|
||||
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
||||
|
||||
subroutine MPI_TYPE_DUP_FN( oldtype, type_keyval, extra_state, &
|
||||
subroutine MPI_TYPE_DUP_FN( datatype, type_keyval, extra_state, &
|
||||
attribute_val_in, attribute_val_out, &
|
||||
flag, ierr )
|
||||
implicit none
|
||||
include 'mpif-config.h'
|
||||
integer :: oldtype
|
||||
integer :: datatype
|
||||
integer :: type_keyval
|
||||
integer(KIND=MPI_ADDRESS_KIND) :: extra_state, attribute_val_in, attribute_val_out
|
||||
logical :: flag
|
||||
integer :: ierr
|
||||
end subroutine MPI_TYPE_DUP_FN
|
||||
|
||||
subroutine MPI_TYPE_NULL_COPY_FN( type, type_keyval, extra_state, &
|
||||
subroutine MPI_TYPE_NULL_COPY_FN( datatype, type_keyval, extra_state, &
|
||||
attribute_val_in, attribute_val_out, &
|
||||
flag, ierr )
|
||||
implicit none
|
||||
include 'mpif-config.h'
|
||||
integer :: type
|
||||
integer :: datatype
|
||||
integer :: type_keyval
|
||||
integer(kind=MPI_ADDRESS_KIND) :: extra_state, attribute_val_in, attribute_val_out
|
||||
integer :: ierr
|
||||
logical :: flag
|
||||
end subroutine MPI_TYPE_NULL_COPY_FN
|
||||
|
||||
subroutine MPI_TYPE_NULL_DELETE_FN( type, type_keyval, attribute_val_out, &
|
||||
subroutine MPI_TYPE_NULL_DELETE_FN( datatype, type_keyval, attribute_val_out, &
|
||||
extra_state, ierr )
|
||||
implicit none
|
||||
include 'mpif-config.h'
|
||||
integer :: type
|
||||
integer :: datatype
|
||||
integer :: type_keyval
|
||||
integer(kind=MPI_ADDRESS_KIND) :: attribute_val_out, extra_state
|
||||
integer :: ierr
|
||||
|
||||
@ -3,7 +3,7 @@
|
||||
! Copyright (c) 2006-2014 Cisco Systems, Inc. All rights reserved.
|
||||
! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
|
||||
! All rights reserved.
|
||||
! Copyright (c) 2017 Research Organization for Information Science
|
||||
! Copyright (c) 2017-2018 Research Organization for Information Science
|
||||
! and Technology (RIST). All rights reserved.
|
||||
!
|
||||
! $COPYRIGHT$
|
||||
@ -46,6 +46,8 @@
|
||||
! Line 2 of the ignore TKR syntax
|
||||
#define OMPI_FORTRAN_IGNORE_TKR_TYPE @OMPI_FORTRAN_IGNORE_TKR_TYPE@
|
||||
|
||||
|
||||
#define OMPI_FORTRAN_BUILD_SIZEOF @OMPI_FORTRAN_BUILD_SIZEOF@
|
||||
! Integers
|
||||
|
||||
#define OMPI_HAVE_FORTRAN_INTEGER1 @OMPI_HAVE_FORTRAN_INTEGER1@
|
||||
|
||||
@ -2,8 +2,8 @@
|
||||
! Copyright (c) 2009-2013 Cisco Systems, Inc. All rights reserved.
|
||||
! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
|
||||
! All rights reserved.
|
||||
! Copyright (c) 2015-2017 Research Organization for Information Science
|
||||
! and Technology (RIST). All rights reserved.
|
||||
! Copyright (c) 2015-2018 Research Organization for Information Science
|
||||
! and Technology (RIST). All rights reserved.
|
||||
! $COPYRIGHT$
|
||||
|
||||
#include "ompi/mpi/fortran/configure-fortran-output.h"
|
||||
@ -23,13 +23,13 @@ END INTERFACE
|
||||
|
||||
!Example of a user defined callback function
|
||||
!
|
||||
! subroutine my_user_function( invec, inoutvec, len, type ) bind(c)
|
||||
! subroutine my_user_function( invec, inoutvec, len, datatype ) bind(c)
|
||||
! use, intrinsic :: iso_c_binding, only : c_ptr, c_f_pointer
|
||||
! type(c_ptr), value :: invec, inoutvec
|
||||
! integer, intent(in) :: len
|
||||
! type(MPI_Datatype) :: type
|
||||
! type(MPI_Datatype) :: datatype
|
||||
! real, pointer :: invec_r(:), inoutvec_r(:)
|
||||
! if (type%MPI_VAL == MPI_REAL%MPI_VAL) then
|
||||
! if (datatype%MPI_VAL == MPI_REAL%MPI_VAL) then
|
||||
! call c_f_pointer(invec, invec_r, (/ len /) )
|
||||
! call c_f_pointer(inoutvec, inoutvec_r, (/ len /) )
|
||||
! inoutvec_r = invec_r + inoutvec_r
|
||||
|
||||
@ -7,8 +7,8 @@
|
||||
! of Tennessee Research Foundation. All rights
|
||||
! reserved.
|
||||
! Copyright (c) 2012 Inria. All rights reserved.
|
||||
! Copyright (c) 2015-2017 Research Organization for Information Science
|
||||
! and Technology (RIST). All rights reserved.
|
||||
! Copyright (c) 2015-2018 Research Organization for Information Science
|
||||
! and Technology (RIST). All rights reserved.
|
||||
! $COPYRIGHT$
|
||||
!
|
||||
! This file provides the interface specifications for the MPI Fortran
|
||||
@ -655,10 +655,10 @@ subroutine ompi_type_create_subarray_f(ndims,array_of_sizes, &
|
||||
INTEGER, INTENT(OUT) :: ierror
|
||||
end subroutine ompi_type_create_subarray_f
|
||||
|
||||
subroutine ompi_type_dup_f(type,newtype,ierror) &
|
||||
subroutine ompi_type_dup_f(oldtype,newtype,ierror) &
|
||||
BIND(C, name="ompi_type_dup_f")
|
||||
implicit none
|
||||
INTEGER, INTENT(IN) :: type
|
||||
INTEGER, INTENT(IN) :: oldtype
|
||||
INTEGER, INTENT(OUT) :: newtype
|
||||
INTEGER, INTENT(OUT) :: ierror
|
||||
end subroutine ompi_type_dup_f
|
||||
@ -1536,10 +1536,10 @@ subroutine ompi_type_create_keyval_f(type_copy_attr_fn,type_delete_attr_fn, &
|
||||
INTEGER, INTENT(OUT) :: ierror
|
||||
end subroutine ompi_type_create_keyval_f
|
||||
|
||||
subroutine ompi_type_delete_attr_f(type,type_keyval,ierror) &
|
||||
subroutine ompi_type_delete_attr_f(datatype,type_keyval,ierror) &
|
||||
BIND(C, name="ompi_type_delete_attr_f")
|
||||
implicit none
|
||||
INTEGER, INTENT(IN) :: type
|
||||
INTEGER, INTENT(IN) :: datatype
|
||||
INTEGER, INTENT(IN) :: type_keyval
|
||||
INTEGER, INTENT(OUT) :: ierror
|
||||
end subroutine ompi_type_delete_attr_f
|
||||
@ -1551,32 +1551,32 @@ subroutine ompi_type_free_keyval_f(type_keyval,ierror) &
|
||||
INTEGER, INTENT(OUT) :: ierror
|
||||
end subroutine ompi_type_free_keyval_f
|
||||
|
||||
subroutine ompi_type_get_name_f(type,type_name,resultlen,ierror,type_name_len) &
|
||||
subroutine ompi_type_get_name_f(datatype,type_name,resultlen,ierror,type_name_len) &
|
||||
BIND(C, name="ompi_type_get_name_f")
|
||||
use, intrinsic :: ISO_C_BINDING, only : C_CHAR
|
||||
implicit none
|
||||
INTEGER, INTENT(IN) :: type
|
||||
INTEGER, INTENT(IN) :: datatype
|
||||
CHARACTER(KIND=C_CHAR), DIMENSION(*), INTENT(OUT) :: type_name
|
||||
INTEGER, INTENT(OUT) :: resultlen
|
||||
INTEGER, INTENT(OUT) :: ierror
|
||||
INTEGER, VALUE, INTENT(IN) :: type_name_len
|
||||
end subroutine ompi_type_get_name_f
|
||||
|
||||
subroutine ompi_type_set_attr_f(type,type_keyval,attribute_val,ierror) &
|
||||
subroutine ompi_type_set_attr_f(datatype,type_keyval,attribute_val,ierror) &
|
||||
BIND(C, name="ompi_type_set_attr_f")
|
||||
use :: mpi_f08_types, only : MPI_ADDRESS_KIND
|
||||
implicit none
|
||||
INTEGER, INTENT(IN) :: type
|
||||
INTEGER, INTENT(IN) :: datatype
|
||||
INTEGER, INTENT(IN) :: type_keyval
|
||||
INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: attribute_val
|
||||
INTEGER, INTENT(OUT) :: ierror
|
||||
end subroutine ompi_type_set_attr_f
|
||||
|
||||
subroutine ompi_type_set_name_f(type,type_name,ierror,type_name_len) &
|
||||
subroutine ompi_type_set_name_f(datatype,type_name,ierror,type_name_len) &
|
||||
BIND(C, name="ompi_type_set_name_f")
|
||||
use, intrinsic :: ISO_C_BINDING, only : C_CHAR
|
||||
implicit none
|
||||
INTEGER, INTENT(IN) :: type
|
||||
INTEGER, INTENT(IN) :: datatype
|
||||
CHARACTER(KIND=C_CHAR), DIMENSION(*), INTENT(IN) :: type_name
|
||||
INTEGER, INTENT(OUT) :: ierror
|
||||
INTEGER, VALUE, INTENT(IN) :: type_name_len
|
||||
@ -3246,11 +3246,11 @@ subroutine ompi_type_create_f90_real_f(p,r,newtype,ierror) &
|
||||
INTEGER, INTENT(OUT) :: ierror
|
||||
end subroutine ompi_type_create_f90_real_f
|
||||
|
||||
subroutine ompi_type_match_size_f(typeclass,size,type,ierror) &
|
||||
subroutine ompi_type_match_size_f(typeclass,size,datatype,ierror) &
|
||||
BIND(C, name="ompi_type_match_size_f")
|
||||
implicit none
|
||||
INTEGER, INTENT(IN) :: typeclass, size
|
||||
INTEGER, INTENT(OUT) :: type
|
||||
INTEGER, INTENT(OUT) :: datatype
|
||||
INTEGER, INTENT(OUT) :: ierror
|
||||
end subroutine ompi_type_match_size_f
|
||||
|
||||
|
||||
@ -7,8 +7,8 @@
|
||||
! of Tennessee Research Foundation. All rights
|
||||
! reserved.
|
||||
! Copyright (c) 2012 Inria. All rights reserved.
|
||||
! Copyright (c) 2015-2017 Research Organization for Information Science
|
||||
! and Technology (RIST). All rights reserved.
|
||||
! Copyright (c) 2015-2018 Research Organization for Information Science
|
||||
! and Technology (RIST). All rights reserved.
|
||||
! $COPYRIGHT$
|
||||
!
|
||||
! This file provides the interface specifications for the MPI Fortran
|
||||
@ -560,10 +560,10 @@ subroutine pompi_type_create_subarray_f(ndims,array_of_sizes, &
|
||||
INTEGER, INTENT(OUT) :: ierror
|
||||
end subroutine pompi_type_create_subarray_f
|
||||
|
||||
subroutine pompi_type_dup_f(type,newtype,ierror) &
|
||||
subroutine pompi_type_dup_f(oldtype,newtype,ierror) &
|
||||
BIND(C, name="pompi_type_dup_f")
|
||||
implicit none
|
||||
INTEGER, INTENT(IN) :: type
|
||||
INTEGER, INTENT(IN) :: oldtype
|
||||
INTEGER, INTENT(OUT) :: newtype
|
||||
INTEGER, INTENT(OUT) :: ierror
|
||||
end subroutine pompi_type_dup_f
|
||||
@ -1370,10 +1370,10 @@ subroutine pompi_type_create_keyval_f(type_copy_attr_fn,type_delete_attr_fn, &
|
||||
INTEGER, INTENT(OUT) :: ierror
|
||||
end subroutine pompi_type_create_keyval_f
|
||||
|
||||
subroutine pompi_type_delete_attr_f(type,type_keyval,ierror) &
|
||||
subroutine pompi_type_delete_attr_f(datatype,type_keyval,ierror) &
|
||||
BIND(C, name="pompi_type_delete_attr_f")
|
||||
implicit none
|
||||
INTEGER, INTENT(IN) :: type
|
||||
INTEGER, INTENT(IN) :: datatype
|
||||
INTEGER, INTENT(IN) :: type_keyval
|
||||
INTEGER, INTENT(OUT) :: ierror
|
||||
end subroutine pompi_type_delete_attr_f
|
||||
@ -1385,32 +1385,32 @@ subroutine pompi_type_free_keyval_f(type_keyval,ierror) &
|
||||
INTEGER, INTENT(OUT) :: ierror
|
||||
end subroutine pompi_type_free_keyval_f
|
||||
|
||||
subroutine pompi_type_get_name_f(type,type_name,resultlen,ierror,type_name_len) &
|
||||
subroutine pompi_type_get_name_f(datatype,type_name,resultlen,ierror,type_name_len) &
|
||||
BIND(C, name="pompi_type_get_name_f")
|
||||
use, intrinsic :: ISO_C_BINDING, only : C_CHAR
|
||||
implicit none
|
||||
INTEGER, INTENT(IN) :: type
|
||||
INTEGER, INTENT(IN) :: datatype
|
||||
CHARACTER(KIND=C_CHAR), DIMENSION(*), INTENT(OUT) :: type_name
|
||||
INTEGER, INTENT(OUT) :: resultlen
|
||||
INTEGER, INTENT(OUT) :: ierror
|
||||
INTEGER, VALUE, INTENT(IN) :: type_name_len
|
||||
end subroutine pompi_type_get_name_f
|
||||
|
||||
subroutine pompi_type_set_attr_f(type,type_keyval,attribute_val,ierror) &
|
||||
subroutine pompi_type_set_attr_f(datatype,type_keyval,attribute_val,ierror) &
|
||||
BIND(C, name="pompi_type_set_attr_f")
|
||||
use :: mpi_f08_types, only : MPI_ADDRESS_KIND
|
||||
implicit none
|
||||
INTEGER, INTENT(IN) :: type
|
||||
INTEGER, INTENT(IN) :: datatype
|
||||
INTEGER, INTENT(IN) :: type_keyval
|
||||
INTEGER(MPI_ADDRESS_KIND), INTENT(IN) :: attribute_val
|
||||
INTEGER, INTENT(OUT) :: ierror
|
||||
end subroutine pompi_type_set_attr_f
|
||||
|
||||
subroutine pompi_type_set_name_f(type,type_name,ierror,type_name_len) &
|
||||
subroutine pompi_type_set_name_f(datatype,type_name,ierror,type_name_len) &
|
||||
BIND(C, name="pompi_type_set_name_f")
|
||||
use, intrinsic :: ISO_C_BINDING, only : C_CHAR
|
||||
implicit none
|
||||
INTEGER, INTENT(IN) :: type
|
||||
INTEGER, INTENT(IN) :: datatype
|
||||
CHARACTER(KIND=C_CHAR), DIMENSION(*), INTENT(IN) :: type_name
|
||||
INTEGER, INTENT(OUT) :: ierror
|
||||
INTEGER, VALUE, INTENT(IN) :: type_name_len
|
||||
@ -3029,11 +3029,11 @@ subroutine pompi_type_create_f90_real_f(p,r,newtype,ierror) &
|
||||
INTEGER, INTENT(OUT) :: ierror
|
||||
end subroutine pompi_type_create_f90_real_f
|
||||
|
||||
subroutine pompi_type_match_size_f(typeclass,size,type,ierror) &
|
||||
subroutine pompi_type_match_size_f(typeclass,size,datatype,ierror) &
|
||||
BIND(C, name="pompi_type_match_size_f")
|
||||
implicit none
|
||||
INTEGER, INTENT(IN) :: typeclass, size
|
||||
INTEGER, INTENT(OUT) :: type
|
||||
INTEGER, INTENT(OUT) :: datatype
|
||||
INTEGER, INTENT(OUT) :: ierror
|
||||
end subroutine pompi_type_match_size_f
|
||||
|
||||
|
||||
@ -3,18 +3,20 @@
|
||||
! Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved.
|
||||
! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
|
||||
! All rights reserved.
|
||||
! Copyright (c) 2018 Research Organization for Information Science
|
||||
! and Technology (RIST). All rights reserved.
|
||||
! $COPYRIGHT$
|
||||
|
||||
subroutine PMPI_Type_delete_attr_f08(type,type_keyval,ierror)
|
||||
subroutine PMPI_Type_delete_attr_f08(datatype,type_keyval,ierror)
|
||||
use :: mpi_f08_types, only : MPI_Datatype
|
||||
use :: mpi_f08, only : ompi_type_delete_attr_f
|
||||
implicit none
|
||||
TYPE(MPI_Datatype), INTENT(IN) :: type
|
||||
TYPE(MPI_Datatype), INTENT(IN) :: datatype
|
||||
INTEGER, INTENT(IN) :: type_keyval
|
||||
INTEGER, OPTIONAL, INTENT(OUT) :: ierror
|
||||
integer :: c_ierror
|
||||
|
||||
call ompi_type_delete_attr_f(type%MPI_VAL,type_keyval,c_ierror)
|
||||
call ompi_type_delete_attr_f(datatype%MPI_VAL,type_keyval,c_ierror)
|
||||
if (present(ierror)) ierror = c_ierror
|
||||
|
||||
end subroutine PMPI_Type_delete_attr_f08
|
||||
|
||||
Некоторые файлы не были показаны из-за слишком большого количества измененных файлов Показать больше
Загрузка…
x
Ссылка в новой задаче
Block a user