ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

Backing out the Open SHMEM project

Просмотр исходного кода 
This commit was SVN r28050.
Этот коммит содержится в:
Joshua Ladd 2013-02-12 17:45:27 +00:00
родитель ff384daab4
Коммит 70ad711337
351 изменённых файлов: 27 добавлений и 37807 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

7
CMakeLists.txt
Просмотреть файл

@ -1,8 +1,6 @@
#
# Copyright (c) 2007-2011 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2012 Mellanox Technologies, Inc.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
@ -70,8 +68,6 @@ INCLUDE_DIRECTORIES (
${OpenMPI_SOURCE_DIR}/opal/include
${OpenMPI_SOURCE_DIR}/ompi
${OpenMPI_SOURCE_DIR}/ompi/include
${OpenMPI_SOURCE_DIR}/oshmem
${OpenMPI_SOURCE_DIR}/oshmem/include
${OpenMPI_SOURCE_DIR}/orte
${OpenMPI_SOURCE_DIR}/orte/include
${OpenMPI_BINARY_DIR}/
@ -79,8 +75,6 @@ INCLUDE_DIRECTORIES (
${OpenMPI_BINARY_DIR}/opal/include
${OpenMPI_BINARY_DIR}/ompi
${OpenMPI_BINARY_DIR}/ompi/include
${OpenMPI_BINARY_DIR}/oshmem
${OpenMPI_BINARY_DIR}/oshmem/include
${OpenMPI_BINARY_DIR}/orte
${OpenMPI_BINARY_DIR}/orte/include
)
@ -113,7 +107,6 @@ ADD_SUBDIRECTORY(opal)
# set up the libevent and hwloc include dirs
INCLUDE_DIRECTORIES (${LIBEVENT_INCLUDE_DIRS} ${HWLOC_PATH}/hwloc/include)
ADD_SUBDIRECTORY(ompi)
ADD_SUBDIRECTORY(oshmem)
ADD_SUBDIRECTORY(orte)
ADD_SUBDIRECTORY(contrib/platform/win32/examples)

11
Makefile.am
Просмотреть файл

@ -11,8 +11,6 @@
# All rights reserved.
# Copyright (c) 2006-2010 Cisco Systems, Inc. All rights reserved.
# Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved.
# Copyright (c) 2012 Mellanox Technologies, Inc.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
@ -30,12 +28,3 @@ dist-hook:
csh "$(top_srcdir)/config/distscript.csh" "$(top_srcdir)" "$(distdir)" "$(OMPI_VERSION)" "$(OMPI_SVN_R)"
ACLOCAL_AMFLAGS = -I config
if OSHMEM_SUPPORT
DISTCHECK_CONFIGURE_FLAGS = --with-oshmem --disable-debug
cov:
PATH=/hpc/local/commercial/coverity/cov-sa/bin:$$PATH && rm -rf $(PWD)/cov-build && make clean && cov-build --dir $(PWD)/cov-build make all && cov-analyze --dir $(PWD)/cov-build && cov-format-errors --dir $(PWD)/cov-build
get_tarball:
cp $(distdir).tar.gz $(to)
endif

36
README-SHMEM-WITH-VALGRIND.txt
Просмотреть файл

@ -1,36 +0,0 @@
Copyright (c) 2012 Mellanox Technologies, Inc.
All rights reserved.
In order to enable valgrind support:
1) download and build (into the same dir for convenience) two libraries: libmlx4 and libibverbs with valgrind support.
- http://www.openfabrics.org/downloads/libmlx4/
- ./configure --prefix=/PATH_TO_LIBS --with-valgrind && make && make install
- http://www.openfabrics.org/downloads/libibverbs/
- ./configure --prefix=/PATH_TO_LIBS --with-valgrind && make && make install
NOTE: libmlx4 and libibvers should match your ofed version
2) build shmem with valgrind, memchecker module and these two libs:
./autogen.sh && ./configure --prefix=$PWD/install --with-openib=/usr --with-oshmem --with-openib-libdir=PATH_TO_LIBS --with-valgrind=PATH_TO_VALGRIND --enable-memchecker --disable-dlopen && make clean && make && make install
- --with-openib=/usr - the place where to look for infiniband
- --with-openib-libdir=PATH_TO_LIBS - two libs previously installed
- --with-valgrind - the --prefix to the valgrind install. that must contain $prefix/include/valgrind/valgrind.h and $prefix/valgrind/memchecker.h. Should be /usr if valgrind installed from rpm
==8511== Conditional jump or move depends on uninitialised value(s)
==8511== at 0x3849A17486: index (in /lib64/ld-2.12.so)
==8511== by 0x3849A06254: expand_dynamic_string_token (in /lib64/ld-2.12.so)
==8511== by 0x3849A07CAF: _dl_map_object (in /lib64/ld-2.12.so)
==8511== by 0x3849A016EA: map_doit (in /lib64/ld-2.12.so)
==8511== by 0x3849A0E0A5: _dl_catch_error (in /lib64/ld-2.12.so)
==8511== by 0x3849A015EE: do_preload (in /lib64/ld-2.12.so)
==8511== by 0x3849A03BAA: dl_main (in /lib64/ld-2.12.so)
==8511== by 0x3849A15A7D: _dl_sysdep_start (in /lib64/ld-2.12.so)
==8511== by 0x3849A01493: _dl_start (in /lib64/ld-2.12.so)
==8511== by 0x3849A00AF7: ??? (in /lib64/ld-2.12.so)
==8511== by 0x2: ???
==8511== by 0x7FF0000A2: ???
==8511== Uninitialised value was created by a stack allocation
==8511== at 0x3849A0328D: dl_main (in /lib64/ld-2.12.so)
In order to get rid of them use an extra command line option to valgrind: --suppression=${prefix}/share/openshmem/openmpi-valgrind.supp --suppressions=shmem_suppressions, where shmem_suppressions is the file in the root of the shmem code and ${prefix} is shmem install prefix as set by configure.

19
README-SHMEM.txt
Просмотреть файл

@ -1,19 +0,0 @@
Copyright (c) 2012 Mellanox Technologies, Inc.
All rights reserved.
Build SHMEM
-----------
./autogen.sh
./configure --prefix=$PWD/install --with-oshmem -enable-contrib-no-build=libnbc,vt --with-openib
make
make install
export SHMEM_HOME=$PWD/install
Run SHMEM
---------
$SHMEM_HOME/bin/shmemrun -np 10 --host amd1,amd5,amd7 -mca btl openib,self hello_shmem.exe

3
VERSION
Просмотреть файл

@ -1,8 +1,6 @@
# Copyright (c) 2008 Sun Microsystems, Inc. All rights reserved.
# Copyright (c) 2008-2011 Cisco Systems, Inc. All rights reserved.
# Copyright (c) 2011 NVIDIA Corporation. All rights reserved.
# Copyright (c) 2012 Mellanox Technologies, Inc.
# All rights reserved.
# This is the VERSION file for Open MPI, describing the precise
# version of Open MPI in this distribution. The various components of
@ -98,7 +96,6 @@ libmpi_usempi_ignore_tkr_so_version=0:0:0
libopen_rte_so_version=0:0:0
libopen_pal_so_version=0:0:0
libmpi_java_so_version=0:0:0
libshmem_so_version=0:0:0
# "Common" components install standalone libraries that are run-time
# linked by one or more components. So they need to be versioned as

13
autogen.pl
Просмотреть файл

@ -2,8 +2,7 @@
#
# Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved.
# Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved.
# Copyright (c) 2012 Mellanox Technologies, Inc.
# All rights reserved.
#
# $COPYRIGHT$
#
# Additional copyrights may follow
@ -42,7 +41,6 @@ my @subdirs;
# Command line parameters
my $no_ompi_arg = 0;
my $no_orte_arg = 0;
my $no_oshmem_arg = 0;
my $quiet_arg = 0;
my $debug_arg = 0;
my $help_arg = 0;
@ -937,7 +935,6 @@ sub patch_autotools_output {
my $ok = Getopt::Long::GetOptions("no-ompi" => \$no_ompi_arg,
"no-orte" => \$no_orte_arg,
"no-oshmem" => \$no_oshmem_arg,
"quiet|q" => \$quiet_arg,
"debug|d" => \$debug_arg,
"help|h" => \$help_arg,
@ -952,7 +949,6 @@ if (!$ok || $help_arg) {
print "Options:
--no-ompi | -no-ompi Do not build the Open MPI layer
--no-orte | -no-orte Do not build the ORTE layer
--no-oshmem | -no-oshmem Do not build the OSHMEM layer
--quiet | -q Do not display normal verbose output
--debug | -d Output lots of debug information
--help | -h This help list
@ -981,10 +977,6 @@ if (! -e "orte") {
$no_orte_arg = 1;
debug "No orte subdirectory found - will not build ORTE\n";
}
if (! -e "oshmem") {
$no_oshmem_arg = 1;
debug "No oshmem subdirectory found - will not build OSHMEM\n";
}
if ($no_ompi_arg == 1 && $no_orte_arg == 0) {
$project_name_long = "Open MPI Run Time Environment";
@ -1151,9 +1143,6 @@ push(@{$projects}, { name => "orte", dir => "orte", need_base => 1 })
if (!$no_orte_arg);
push(@{$projects}, { name => "ompi", dir => "ompi", need_base => 1 })
if (!$no_ompi_arg);
push(@{$projects}, { name => "oshmem", dir => "oshmem", need_base => 1 })
if (!$no_ompi_arg && !$no_orte_arg && !$no_oshmem_arg);
# Save the list of projects in the m4 file
my $str;

12
config/opal_check_attributes.m4
Просмотреть файл

@ -12,8 +12,6 @@
# All rights reserved.
# Copyright (c) 2009 Oak Ridge National Labs. All rights reserved.
# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
# Copyright (c) 2012 Mellanox Technologies, Inc.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
@ -217,7 +215,6 @@ AC_DEFUN([OPAL_CHECK_ATTRIBUTES], [
opal_cv___attribute__visibility=0
opal_cv___attribute__warn_unused_result=0
opal_cv___attribute__weak_alias=0
opal_cv___attribute__destructor=0
else
AC_MSG_RESULT([yes])
@ -536,13 +533,6 @@ AC_DEFUN([OPAL_CHECK_ATTRIBUTES], [
[],
[])
_OPAL_CHECK_SPECIFIC_ATTRIBUTE([destructor],
[
void foo(void) __attribute__ ((__destructor__));
void foo(void) { return ; }
],
[],
[])
fi
# Now that all the values are set, define them
@ -591,6 +581,4 @@ AC_DEFUN([OPAL_CHECK_ATTRIBUTES], [
[Whether your compiler has __attribute__ warn unused result or not])
AC_DEFINE_UNQUOTED(OPAL_HAVE_ATTRIBUTE_WEAK_ALIAS, [$opal_cv___attribute__weak_alias],
[Whether your compiler has __attribute__ weak alias or not])
AC_DEFINE_UNQUOTED(OPAL_HAVE_ATTRIBUTE_DESTRUCTOR, [$opal_cv___attribute__destructor],
[Whether your compiler has __attribute__ destructor or not])
])

26
config/oshmem_config_files.m4
Просмотреть файл

@ -1,26 +0,0 @@
# -*- shell-script -*-
#
# Copyright (c) 2012 Mellanox Technologies, Inc.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
AC_DEFUN([OSHMEM_CONFIG_FILES],[
AC_CONFIG_FILES([
oshmem/Makefile
oshmem/include/Makefile
oshmem/shmem/c/Makefile
oshmem/shmem/f77/Makefile
oshmem/shmem/c/profile/Makefile
oshmem/tools/wrappers/Makefile
oshmem/tools/wrappers/shmemcc-wrapper-data.txt
oshmem/tools/wrappers/shmemf77-wrapper-data.txt
oshmem/tools/wrappers/shmemf90-wrapper-data.txt
])
])

188
config/oshmem_configure_options.m4
Просмотреть файл

@ -1,188 +0,0 @@
dnl -*- shell-script -*-
dnl
dnl Copyright (c) 2012 Mellanox Technologies, Inc.
dnl All rights reserved.
dnl
dnl $COPYRIGHT$
dnl
dnl Additional copyrights may follow
dnl
dnl $HEADER$
dnl
AC_DEFUN([OSHMEM_CONFIGURE_OPTIONS],[
ompi_show_subtitle "OSHMEM Configuration options"
AC_SUBST(OSHMEM_LIBSHMEM_EXTRA_LIBS)
AC_SUBST(OSHMEM_LIBSHMEM_EXTRA_LDFLAGS)
#
# OSHMEM support
#
AC_MSG_CHECKING([if want OSHMEM support])
AC_ARG_WITH([oshmem],
[AC_HELP_STRING([--with-oshmem],
[Build with OSHMEM support (default=no)])])
if test "$with_oshmem" = "no"; then
AC_MSG_RESULT([no])
oshmem_with_support=0
else
AC_MSG_RESULT([yes])
oshmem_with_support=1
fi
AM_CONDITIONAL(OSHMEM_SUPPORT, test "$oshmem_with_support" = 1)
AC_DEFINE_UNQUOTED([OSHMEM_ENABLED], [$oshmem_with_support],
[Whether user wants OSHMEM support or not])
#
# Enable compatibility mode
#
AC_MSG_CHECKING([if want SGI/Quadrix compatibility mode])
AC_ARG_ENABLE(oshmem-compat,
AC_HELP_STRING([--enable-oshmem-compat],
[enable compatibility mode (default: enabled)]))
if test "$enable_oshmem_compat" != "no"; then
AC_MSG_RESULT([yes])
OSHMEM_SPEC_COMPAT=1
else
AC_MSG_RESULT([no])
OSHMEM_SPEC_COMPAT=0
fi
AC_DEFINE_UNQUOTED([OSHMEM_SPEC_COMPAT], [$OSHMEM_SPEC_COMPAT],
[Whether user wants OSHMEM in compatibility mode or not])
#
# Do we want to disable OSHMEM parameter checking at run-time?
#
AC_MSG_CHECKING([if want SHMEM API parameter checking])
AC_ARG_WITH(shmem-param-check,
AC_HELP_STRING([--shmem-param-check(=VALUE)],
[behavior of SHMEM function parameter checking. Valid values are: always, never. If --with-shmem-param-check is specified with no VALUE argument, it is equivalent to a VALUE of "always"; --without-shmem-param-check is equivalent to "never" (default: always).]))
shmem_param_check=1
if test "$with_shmem_param_check" = "no" -o \
"$with_shmem_param_check" = "never"; then
shmem_param_check=0
AC_MSG_RESULT([never])
elif test "$with_shmem_param_check" = "yes" -o \
"$with_shmem_param_check" = "always" -o \
-z "$with_shmem_param_check"; then
shmem_param_check=1
AC_MSG_RESULT([always])
else
AC_MSG_RESULT([unknown])
AC_MSG_WARN([*** Unrecognized --with-shmem-param-check value])
AC_MSG_WARN([*** See "configure --help" output])
AC_MSG_WARN([*** Defaulting to "runtime"])
fi
AC_DEFINE_UNQUOTED(OSHMEM_PARAM_CHECK, $shmem_param_check,
[Whether we want to check SHMEM parameters always or never])
#
# OSHMEM profiling support
#
AC_MSG_CHECKING([if want pshmem_])
AC_ARG_ENABLE(oshmem-profile,
AC_HELP_STRING([--enable-oshmem-profile],
[enable OSHMEM profiling (default: enabled)]))
if test "$enable_oshmem_profile" != "no"; then
AC_MSG_RESULT([yes])
oshmem_progiling_support=1
else
AC_MSG_RESULT([no])
oshmem_progiling_support=0
fi
AM_CONDITIONAL(OSHMEM_PROFILING, test "$oshmem_progiling_support" = 1)
#AC_DEFINE_UNQUOTED([OSHMEM_PROFILING], [$oshmem_progiling_support],
# [Whether user wants OSHMEM profiling])
])
AC_DEFUN([OSHMEM_SETUP_CFLAGS],[
OMPI_C_COMPILER_VENDOR([oshmem_c_vendor])
#
# OSHMEM force warnings as errors
#
#
# Since SHMEM libraries are not fully ISO99 C compliant
# -pedantic and -Wundef raise a bunch of warnings, so
# we just strip them off for this component
AC_MSG_WARN([Removed -pedantic and -Wundef from CFLAGS for OSHMEM])
oshmem_CFLAGS="$CFLAGS"
# Strip off problematic arguments
oshmem_CFLAGS="`echo $oshmem_CFLAGS | sed 's/-pedantic//g'`"
oshmem_CFLAGS="`echo $oshmem_CFLAGS | sed 's/-Wundef//g'`"
oshmem_CFLAGS="`echo $oshmem_CFLAGS | sed 's/-Wno-long-double//g'`"
CFLAGS="$oshmem_CFLAGS"
case "$oshmem_c_vendor" in
gnu)
OSHMEM_CFLAGS=" -Werror "
OSHMEM_TEST_CFLAGS="$CFLAGS -Wall -Wundef -Werror "
;;
intel)
# we want specifically the warning on format string conversion
OSHMEM_CFLAGS=" -Werror "
OSHMEM_TEST_CFLAGS="$CFLAGS -Wall -Werror -wd188,981,1419,810"
;;
esac
AC_SUBST([OSHMEM_CFLAGS])
AC_SUBST([OSHMEM_TEST_CFLAGS])
OMPI_CHECK_OPENFABRICS([openib],
[openib_happy="yes"],
[openib_happy="no"])
# substitute in the things needed to build MEMHEAP BASE
AC_SUBST([openib_CFLAGS])
AC_SUBST([openib_CPPFLAGS])
AC_SUBST([openib_LDFLAGS])
AC_SUBST([openib_LIBS])
# If we have the openib stuff available, find out what we've got
AS_IF(
[test "$openib_happy" = "yes"],
[
OSHMEM_LIBSHMEM_EXTRA_LDFLAGS="$OSHMEM_LIBSHMEM_EXTRA_LDFLAGS $openib_LDFLAGS"
OSHMEM_LIBSHMEM_EXTRA_LIBS="$OSHMEM_LIBSHMEM_EXTRA_LIBS $openib_LIBS"
# ibv_reg_shared_mr was added in MOFED 1.8
oshmem_have_mpage=0
openib_save_CPPFLAGS="$CPPFLAGS"
openib_save_LDFLAGS="$LDFLAGS"
openib_save_LIBS="$LIBS"
CPPFLAGS="$CPPFLAGS $openib_CPPFLAGS"
LDFLAGS="$LDFLAGS $openib_LDFLAGS"
LIBS="$LIBS $openib_LIBS"
AC_CHECK_DECLS([IBV_ACCESS_ALLOCATE_MR],
[oshmem_have_mpage=1], [],
[#include <infiniband/verbs.h>])
AC_CHECK_LIB([ibverbs], [ibv_reg_shared_mr], [oshmem_have_mpage=2])
CPPFLAGS="$openib_save_CPPFLAGS"
LDFLAGS="$openib_save_LDFLAGS"
LIBS="$openib_save_LIBS"
AC_DEFINE_UNQUOTED(MPAGE_ENABLE, $oshmem_have_mpage,
[Whether we can use M-PAGE supported since MOFED 1.8])
])
])dnl

25
configure.ac
Просмотреть файл

@ -17,8 +17,6 @@
# Copyright (c) 2009 Oak Ridge National Labs. All rights reserved.
# Copyright (c) 2011 NVIDIA Corporation. All rights reserved.
# Copyright (c) 2012 Oracle and/or its affiliates. All rights reserved.
# Copyright (c) 2012 Mellanox Technologies, Inc.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
@ -101,8 +99,6 @@ AC_SUBST([CONFIGURE_DEPENDENCIES], ['$(top_srcdir)/VERSION'])
# Set up project specific AM_CONDITIONALs
AM_CONDITIONAL([PROJECT_OMPI], m4_ifdef([project_ompi], [true], [false]))
AM_CONDITIONAL([PROJECT_ORTE], m4_ifdef([project_orte], [true], [false]))
AM_CONDITIONAL([PROJECT_OSHMEM], m4_ifdef([project_oshmem], [true], [false]))
ompi_show_subtitle "Checking versions"
@ -117,11 +113,6 @@ m4_ifdef([project_orte],
[$srcdir/VERSION],
[orte/include/orte/version.h])])
m4_ifdef([project_oshmem],
[OPAL_SAVE_VERSION([OSHMEM], [Open SHMEM],
[$srcdir/VERSION],
[oshmem/include/oshmem/version.h])])
OPAL_SAVE_VERSION([OPAL], [Open Portable Access Layer], [$srcdir/VERSION],
[opal/include/opal/version.h])
@ -146,8 +137,6 @@ m4_ifdef([project_ompi],
AC_SUBST(libmca_common_portals_so_version)])
m4_ifdef([project_orte],
[AC_SUBST(libopen_rte_so_version)])
m4_ifdef([project_oshmem],
[AC_SUBST(libshmem_so_version)])
AC_SUBST(libmca_opal_common_hwloc_so_version)
AC_SUBST(libopen_pal_so_version)
@ -171,8 +160,6 @@ m4_ifdef([project_orte],
[AC_CONFIG_HEADERS([orte/include/orte_config.h])])
m4_ifdef([project_ompi],
[AC_CONFIG_HEADERS([ompi/include/ompi_config.h ompi/include/mpi.h])])
m4_ifdef([project_oshmem],
[AC_CONFIG_HEADER([oshmem/include/oshmem_config.h oshmem/include/shmem.h oshmem/include/shmem_portable_platform.h])])
# override/fixup the version numbers set by AC_INIT, since on
# developer builds, there's no good way to know what the version is
@ -252,7 +239,6 @@ AC_SUBST(top_ompi_builddir)
OPAL_CONFIGURE_OPTIONS
m4_ifdef([project_orte], [ORTE_CONFIGURE_OPTIONS])
m4_ifdef([project_ompi], [OMPI_CONFIGURE_OPTIONS])
m4_ifdef([project_oshmem], [OSHMEM_CONFIGURE_OPTIONS])
if test "$enable_binaries" = "no" -a "$enable_dist" = "yes"; then
AC_MSG_WARN([--disable-binaries is incompatible with --enable dist])
@ -685,7 +671,6 @@ ompi_show_title "Type tests"
# Size of pid_t
AC_CHECK_SIZEOF(pid_t)
AC_CHECK_SIZEOF(long)
AC_CHECK_TYPES([socklen_t, struct sockaddr_in, struct sockaddr_in6,
struct sockaddr_storage],
@ -860,12 +845,6 @@ AC_DEFINE_UNQUOTED(OMPI_MPI_OFFSET_TYPE, $MPI_OFFSET_TYPE, [Type of MPI_Offset -
AC_DEFINE_UNQUOTED(OMPI_MPI_OFFSET_SIZE, $MPI_OFFSET_SIZE, [Size of the MPI_Offset])
AC_DEFINE_UNQUOTED(OMPI_OFFSET_DATATYPE, $MPI_OFFSET_DATATYPE, [MPI datatype corresponding to MPI_Offset])
AC_DEFINE_UNQUOTED(OSHMEM_SHMEM_OFFSET_TYPE, $MPI_OFFSET_TYPE, [Type of SHMEM_Offset -- has to be defined here and typedef'ed later because shmem.h does not get AC SUBST's])
AC_DEFINE_UNQUOTED(OSHMEM_SHMEM_OFFSET_SIZE, $MPI_OFFSET_SIZE, [Size of the SHMEM_Offset])
AC_DEFINE_UNQUOTED(OSHMEM_OFFSET_DATATYPE, $MPI_OFFSET_DATATYPE, [SHMEM datatype corresponding to SHMEM_Offset])
AC_DEFINE_UNQUOTED(OPAL_SIZEOF_LONG, $ac_cv_sizeof_long, "Size of 'long' type")
#
# Check for MPI_Aint type. Yes, there are platforms where
# sizeof(void*) != sizeof(long) (64 bit Windows, apparently).
@ -1193,7 +1172,7 @@ if test "$OMPI_TOP_BUILDDIR" != "$OMPI_TOP_SRCDIR"; then
# rather than have successive assignments to these shell
# variables, lest the $(foo) names try to get evaluated here.
# Yuck!
CPPFLAGS='-I$(top_srcdir) -I$(top_builddir) -I$(top_srcdir)/opal/include m4_ifdef([project_orte], [-I$(top_srcdir)/orte/include]) m4_ifdef([project_ompi], [-I$(top_srcdir)/ompi/include]) m4_ifdef([project_oshmem], [-I$(top_srcdir)/oshmem/include])'" $CPPFLAGS"
CPPFLAGS='-I$(top_srcdir) -I$(top_builddir) -I$(top_srcdir)/opal/include m4_ifdef([project_orte], [-I$(top_srcdir)/orte/include]) m4_ifdef([project_ompi], [-I$(top_srcdir)/ompi/include])'" $CPPFLAGS"
# C++ is only relevant if we're building OMPI
m4_ifdef([project_ompi], [CXXCPPFLAGS='-I$(top_srcdir) -I$(top_builddir) -I$(top_srcdir)/opal/include -I$(top_srcdir)/orte/include -I$(top_srcdir)/ompi/include'" $CXXCPPFLAGS"])
else
@ -1207,7 +1186,6 @@ fi
m4_ifdef([project_orte], [ORTE_SETUP_DEBUGGER_FLAGS],
[m4_ifdef([project_ompi], [ORTE_SETUP_DEBUGGER_FLAGS])])
m4_ifdef([project_oshmem], [OSHMEM_SETUP_CFLAGS])
#
# Delayed the substitution of CFLAGS and CXXFLAGS until now because
@ -1314,6 +1292,5 @@ AC_CONFIG_FILES([
OPAL_CONFIG_FILES
m4_ifdef([project_orte], [ORTE_CONFIG_FILES])
m4_ifdef([project_ompi], [OMPI_CONFIG_FILES])
m4_ifdef([project_oshmem], [OSHMEM_CONFIG_FILES])
AC_OUTPUT

10
contrib/Makefile.am
Просмотреть файл

@ -12,9 +12,6 @@
# Copyright (c) 2009 Cisco Systems, Inc. All rights reserved.
# Copyright (c) 2010 IBM Corporation. All rights reserved.
# Copyright (c) 2010-2011 Oak Ridge National Labs. All rights reserved.
# Copyright (c) 2012 Mellanox Technologies, Inc.
# All rights reserved.
#
# $COPYRIGHT$
#
# Additional copyrights may follow
@ -163,10 +160,3 @@ EXTRA_DIST = \
platform/greenplum/mrplus/linux-optimized.conf
dist_pkgdata_DATA = openmpi-valgrind.supp
if OSHMEM_SUPPORT
dist_pkgdata_DATA += \
tau/shmem_wiki_tau.sh \
tau/tau_openshmem.patch \
tau/readme
endif

11
contrib/platform/win32/CMakeModules/opal_get_version.cmake
Просмотреть файл

@ -1,8 +1,6 @@
#
# Copyright (c) 2007-2012 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2012 Mellanox Technologies, Inc.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
@ -109,15 +107,6 @@ OMPI_DEF(OMPI_MINOR_VERSION ${MINOR_VERSION} "Minor release number of Open MPI."
OMPI_DEF(OMPI_RELEASE_VERSION ${RELEASE_VERSION} "Release number of Open MPI." 0 1)
OMPI_DEF(OMPI_VERSION ${VERSION_STRING} "Complete release number of Open MPI." 1 1)
# Set OSHMEM versions
OMPI_DEF(OSHMEM_WANT_REPO_REV ${WANT_REPO_REV} "SVN verstion of OSHMEM" 1 1)
OMPI_DEF(OSHMEM_REPO_REV "${SVN_VERSION}" "SVN verstion of OSHMEM" 1 1)
OMPI_DEF(OSHMEM_GREEK_VERSION "${GREEK_VERSION}" "Greek - alpha, beta, etc - release number of Open Portable Access Layer." 1 1)
OMPI_DEF(OSHMEM_MAJOR_VERSION ${MAJOR_VERSION} "Major release number of OSHMEM." 0 1)
OMPI_DEF(OSHMEM_MINOR_VERSION ${MINOR_VERSION} "Minor release number of OSHMEM." 0 1)
OMPI_DEF(OSHMEM_RELEASE_VERSION ${RELEASE_VERSION} "Release number of OSHMEM." 0 1)
OMPI_DEF(OSHMEM_VERSION ${VERSION_STRING} "Complete release number of OSHMEM." 1 1)
# Set ORTE versions
OMPI_DEF(ORTE_WANT_REPO_REV ${WANT_REPO_REV} "SVN verstion of ORTE" 1 1)
OMPI_DEF(ORTE_REPO_REV "${SVN_VERSION}" "SVN verstion of ORTE" 1 1)

178
contrib/tau/readme
Просмотреть файл

@ -1,178 +0,0 @@
Copyright (c) 2012 Mellanox Technologies, Inc.
All rights reserved.
22 May 2012
Description
===========
Using TAU with OpenSHMEM
========================
Building PDT
=============
Download, configure and build PDT:
-----
wget -nc http://tau.uoregon.edu/pdt_releases/pdtoolkit-3.17.tar.gz
tar -xzf pdtoolkit-3.17.tar.gz
cd pdtoolkit-*
PDT_INST=$PWD
./configure
make install
cd ..
-----
Building TAU
============
Download:
-----
wget -nc http://www.cs.uoregon.edu/research/paracomp/tau/tauprofile/dist/tau_latest.tar.gztar -xzf pdtoolkit-3.17.tar.gz
tar -xzf tau_latest.tar.gz
cd tau-*
-----
Accept patch:
-----
patch -p1 -i tau_openshmem.patch
-----
Note:
Patch is needed to define profiling API that is not part of official openshmem.org standard.
Configure and build:
-----
TAU_INST=$PWD/inst-tau-shmem
./configure -prefix=$TAU_INST -shmem -tag=oshmem -cc=gcc -pdt=$PDT_INST -PROFILEPARAM -useropt="-I$OSHMEM_INST/include/mpp" -shmemlib=$OSHMEM_INST/lib -shmemlibrary=-lshmem#-lpmi
make install
cd ..
-----
Note:
-useropt : specifies additional user options such as -g or -I. For multiple options, the options list should be enclosed in a single quote.
-lpmi : SLURM support
-shmeminc : is not set due to bug in TAU (http://nic.uoregon.edu/pipermail/tau-users/2011-December/000599.html)
TAU documentation link http://www.cs.uoregon.edu/Research/tau/docs/old/bk04ch01.html
Building Application
====================
Add OpenSHMEM installation location to the PATH environment:
-----
export PATH=$OSHMEM_INST/bin:$PATH
-----
Set TAU environmemnt variables:
-----
export PATH=$TAU_INST/x86_64/bin:$PATH
export TAU_MAKEFILE=$TAU_INST/x86_64/lib/Makefile.tau-oshmem-param-shmem-pdt
export TAU_OPTIONS=
-----
Note:
TAU provides shell scripts tau_cxx.sh, tau_f90.sh and tau_cc.sh for C++,F90 and C compilers. These are typically installed in <taudir>/<arch>/bin >directory. You may replace the compiler in your Makefiles with these scripts. These scripts may also be used on the command line. Each configuration of TAU has a stub makefile associated with it. For instance,
inst-tau-shmem/x86_64/lib/Makefile.tau-oshmem-param-shmem-pdt
This makefile specifies the measurement components that have been configured with it. In this case it shows that TAU's MPI/SHMEM wrapper interposition library and PDT have been configured. You need to set the environment variable:
TAU_MAKEFILE
to point to this stub makefile. You can also pass arguments to the four stages of compilation using the environment variable
TAU_OPTIONS
Build and run example:
-----
tau_cc.sh $TAU_INST/../examples/shmem/c/simple.c -o simple-tau.out
env LD_LIBRARY_PATH=$OSHMEM_INST/lib:$LD_LIBRARY_PATH srun -n 2 simple-tau.out
-----
After job completes, view simple report:
-----
pprof
paraprof --pack example_shmem_tau.ppk
-----
Output example
==============
-----
$pprof
Reading Profile files in profile.*
NODE 0;CONTEXT 0;THREAD 0:
---------------------------------------------------------------------------------------
%Time Exclusive Inclusive #Call #Subrs Inclusive Name
msec total msec usec/call
---------------------------------------------------------------------------------------
100.0 0.102 10,479 1 5 10479786 int main(int, char **) C
100.0 10,479 10,479 1 0 10479221 void start_pes(int) C
0.0 0.423 0.423 1 0 423 void shmem_barrier_all(void) C
0.0 0.04 0.04 1 0 40 void shmem_long_put(long *, const long *, size_t, int) C
0.0 0.04 0.04 1 0 40 void shmem_long_put(long *, const long *, size_t, int) C [ <message size> = <80> ]
0.0 0 0 2 0 0 int shmem_my_pe(void) C
---------------------------------------------------------------------------------------
USER EVENTS Profile :NODE 0, CONTEXT 0, THREAD 0
---------------------------------------------------------------------------------------
NumSamples MaxValue MinValue MeanValue Std. Dev. Event Name
---------------------------------------------------------------------------------------
1 80 80 80 0 Message size sent to all nodes
---------------------------------------------------------------------------------------
NODE 1;CONTEXT 0;THREAD 0:
---------------------------------------------------------------------------------------
%Time Exclusive Inclusive #Call #Subrs Inclusive Name
msec total msec usec/call
---------------------------------------------------------------------------------------
100.0 0.042 9,270 1 4 9270413 int main(int, char **) C
100.0 9,270 9,270 1 0 9270317 void start_pes(int) C
0.0 0.053 0.053 1 0 53 void shmem_barrier_all(void) C
0.0 0.001 0.001 2 0 0 int shmem_my_pe(void) C
FUNCTION SUMMARY (total):
---------------------------------------------------------------------------------------
%Time Exclusive Inclusive #Call #Subrs Inclusive Name
msec total msec usec/call
---------------------------------------------------------------------------------------
100.0 0.144 19,750 2 9 9875100 int main(int, char **) C
100.0 19,749 19,749 2 0 9874769 void start_pes(int) C
0.0 0.476 0.476 2 0 238 void shmem_barrier_all(void) C
0.0 0.04 0.04 1 0 40 void shmem_long_put(long *, const long *, size_t, int) C
0.0 0.04 0.04 1 0 40 void shmem_long_put(long *, const long *, size_t, int) C [ <message size> = <80> ]
0.0 0.001 0.001 4 0 0 int shmem_my_pe(void) C
FUNCTION SUMMARY (mean):
---------------------------------------------------------------------------------------
%Time Exclusive Inclusive #Call #Subrs Inclusive Name
msec total msec usec/call
---------------------------------------------------------------------------------------
100.0 0.072 9,875 1 4.5 9875100 int main(int, char **) C
100.0 9,874 9,874 1 0 9874769 void start_pes(int) C
0.0 0.238 0.238 1 0 238 void shmem_barrier_all(void) C
0.0 0.02 0.02 0.5 0 40 void shmem_long_put(long *, const long *, size_t, int) C
0.0 0.02 0.02 0.5 0 40 void shmem_long_put(long *, const long *, size_t, int) C [ <message size> = <80> ]
0.0 0.0005 0.0005 2 0 0 int shmem_my_pe(void) C
-----
A sample session
=========== =====
Following shell script can be launched to play with OpenSHMEM and TAU as shmem_wiki_tau.sh
This script download, configure and install all needed tools, compile simple open shmem application, run one using SLURM and show the result using pprof and prepares result to view using paraprof also.
ParaProf - User's Manual: http://www.cs.uoregon.edu/Research/tau/docs/paraprof/index.html
-----
./shmem_wiki_tau.sh <path to openshmem installation>
-----
Enjoy.

176
contrib/tau/shmem_wiki_tau.sh
Просмотреть файл

@ -1,176 +0,0 @@
#!/bin/bash
# Copyright (c) 2012 Mellanox Technologies, Inc.
# All rights reserved.
if [[ -n "$1" ]]; then
OSHMEM_INST=$1
else
echo "Path to OpenShmem install should be passed as an argument and built with --with-pmi"
fi
export PATH=$OSHMEM_INST/bin:$PATH
OSHMEM_TAU_PATCH=`pwd`/tau_openshmem.patch
# TAU expects having shmcc compiler name for Open SMEM
#ln -s $OSHMEM_INST/bin/shmemcc $OSHMEM_INST/bin/oshcc
#ln -s $OSHMEM_INST/share/openshmem/shmemcc-wrapper-data.txt $OSHMEM_INST/share/openshmem/oshcc-wrapper-data.txt
# download PDT sources
wget -nc http://tau.uoregon.edu/pdt_releases/pdtoolkit-3.17.tar.gz
# build PDT
tar -xzf pdtoolkit-3.17.tar.gz
cd pdtoolkit-*
PDT_INST=$PWD
./configure
make install
cd ..
function install_openshmem_tau_patch
{
cat > ${OSHMEM_TAU_PATCH} <<END_MSG
*** tau-2.21.2/src/Profile/TauShmemOpenShmemC.c 2012-01-27 20:43:12.000000000 +0200
--- new/src/Profile/TauShmemOpenShmemC.c 2012-05-21 14:14:51.000000000 +0300
***************
*** 6,11 ****
--- 6,99 ----
#define TAU_SHMEM_TAGID tau_shmem_tagid_f=tau_shmem_tagid_f%250
#define TAU_SHMEM_TAGID_NEXT (++tau_shmem_tagid_f) % 250
+
+ /* This section contains old API that are not part of openshmem.org specification
+ *
+ */
+ void pshmem_init (void)
+ {
+ fprintf(stderr, "Dummy %s\n", __FUNCTION__);
+ return ;
+ }
+
+ void pshmem_finalize (void)
+ {
+ fprintf(stderr, "Dummy %s\n", __FUNCTION__);
+ return ;
+ }
+
+ char *pshmem_nodename (void)
+ {
+ fprintf(stderr, "Dummy %s\n", __FUNCTION__);
+ return NULL;
+ }
+
+ int pshmem_version (int *major, int *minor)
+ {
+ fprintf(stderr, "Dummy %s\n", __FUNCTION__);
+ return 0;
+ }
+
+ void *pshmem_malloc (size_t size)
+ {
+ fprintf(stderr, "Dummy %s\n", __FUNCTION__);
+ return NULL;
+ }
+
+ void pshmem_free (void *ptr)
+ {
+ fprintf(stderr, "Dummy %s\n", __FUNCTION__);
+ return ;
+ }
+
+ void *pshmem_realloc (void *ptr, size_t size)
+ {
+ fprintf(stderr, "Dummy %s\n", __FUNCTION__);
+ return NULL;
+ }
+
+ void *pshmem_memalign (size_t alignment, size_t size)
+ {
+ fprintf(stderr, "Dummy %s\n", __FUNCTION__);
+ return NULL;
+ }
+
+ char *psherror (void)
+ {
+ fprintf(stderr, "Dummy %s\n", __FUNCTION__);
+ return NULL;
+ }
+
+ char *pshmem_error (void)
+ {
+ fprintf(stderr, "Dummy %s\n", __FUNCTION__);
+ return NULL;
+ }
+
+ void pshmem_sync_init (long *pSync)
+ {
+ fprintf(stderr, "Dummy %s\n", __FUNCTION__);
+ return ;
+ }
+
+ #ifdef __cplusplus
+ # include <complex>
+ # define COMPLEXIFY(T) std::complex<T>
+ #else /* _cplusplus */
+ # include <complex.h>
+ # define COMPLEXIFY(T) T complex
+ #endif /* __cplusplus */
+ void pshmem_complexd_put (COMPLEXIFY (double) * dest,
+ const COMPLEXIFY (double) * src,
+ size_t nelems, int pe)
+ {
+ fprintf(stderr, "Dummy %s\n", __FUNCTION__);
+ return ;
+ }
+
+ /* Old API */
+
+
/**********************************************************
start_pes
**********************************************************/
END_MSG
patch --dry-run -p1 -i ${OSHMEM_TAU_PATCH}
patch -p1 -i ${OSHMEM_TAU_PATCH}
return 0
}
# download TAU sources
wget -nc http://www.cs.uoregon.edu/research/paracomp/tau/tauprofile/dist/tau_latest.tar.gz
#wget http://tau.uoregon.edu/tau.tgz
tar -xzf tau_latest.tar.gz
cd tau-*
install_openshmem_tau_patch
TAU_INST=$PWD/inst-tau-shmem
./configure -prefix=$TAU_INST -shmem -tag=oshmem -cc=gcc -pdt=$PDT_INST -PROFILEPARAM -useropt="-g" -shmemlib=$OSHMEM_INST/lib -shmemlibrary="-lshmem -lpmi"
make install
cd ..
export PATH=$TAU_INST/x86_64/bin:$PATH
export TAU_MAKEFILE=$TAU_INST/x86_64/lib/Makefile.tau-oshmem-param-shmem-pdt
export TAU_OPTIONS=
# Example
# Note:
# srun reports error message as ORTE_ERROR_LOG: A message is attempting to be sent to a process whose contact information is unknown in file
# in case shmem library is built w/o option --with-pmi
mkdir example
cd example
if [[ -n "$2" ]]; then
tau_cc.sh $2/test/shmem/vs/osu_latency.c -DOSHMEM -o osu_latency-tau.out
env LD_LIBRARY_PATH=$OSHMEM_INST/lib:$LD_LIBRARY_PATH srun -n 2 osu_latency-tau.out
cd ..
else
tau_cc.sh $TAU_INST/../examples/shmem/c/simple.c -o simple-tau.out
env LD_LIBRARY_PATH=$OSHMEM_INST/lib:$LD_LIBRARY_PATH srun -n 2 simple-tau.out
fi
pprof
paraprof --pack example_shmem_tau.ppk
cd ..

101
contrib/tau/tau_openshmem.patch
Просмотреть файл

@ -1,101 +0,0 @@
# Copyright (c) 2012 Mellanox Technologies, Inc.
# All rights reserved.
*** tau-2.21.2/src/Profile/TauShmemOpenShmemC.c 2012-01-27 20:43:12.000000000 +0200
--- new/src/Profile/TauShmemOpenShmemC.c 2012-05-21 14:14:51.000000000 +0300
***************
*** 6,11 ****
--- 6,99 ----
#define TAU_SHMEM_TAGID tau_shmem_tagid_f=tau_shmem_tagid_f%250
#define TAU_SHMEM_TAGID_NEXT (++tau_shmem_tagid_f) % 250
+
+ /* This section contains old API that are not part of openshmem.org specification
+ *
+ */
+ void pshmem_init (void)
+ {
+ fprintf(stderr, "Dummy %s\n", __FUNCTION__);
+ return ;
+ }
+
+ void pshmem_finalize (void)
+ {
+ fprintf(stderr, "Dummy %s\n", __FUNCTION__);
+ return ;
+ }
+
+ char *pshmem_nodename (void)
+ {
+ fprintf(stderr, "Dummy %s\n", __FUNCTION__);
+ return NULL;
+ }
+
+ int pshmem_version (int *major, int *minor)
+ {
+ fprintf(stderr, "Dummy %s\n", __FUNCTION__);
+ return 0;
+ }
+
+ void *pshmem_malloc (size_t size)
+ {
+ fprintf(stderr, "Dummy %s\n", __FUNCTION__);
+ return NULL;
+ }
+
+ void pshmem_free (void *ptr)
+ {
+ fprintf(stderr, "Dummy %s\n", __FUNCTION__);
+ return ;
+ }
+
+ void *pshmem_realloc (void *ptr, size_t size)
+ {
+ fprintf(stderr, "Dummy %s\n", __FUNCTION__);
+ return NULL;
+ }
+
+ void *pshmem_memalign (size_t alignment, size_t size)
+ {
+ fprintf(stderr, "Dummy %s\n", __FUNCTION__);
+ return NULL;
+ }
+
+ char *psherror (void)
+ {
+ fprintf(stderr, "Dummy %s\n", __FUNCTION__);
+ return NULL;
+ }
+
+ char *pshmem_error (void)
+ {
+ fprintf(stderr, "Dummy %s\n", __FUNCTION__);
+ return NULL;
+ }
+
+ void pshmem_sync_init (long *pSync)
+ {
+ fprintf(stderr, "Dummy %s\n", __FUNCTION__);
+ return ;
+ }
+
+ #ifdef __cplusplus
+ # include <complex>
+ # define COMPLEXIFY(T) std::complex<T>
+ #else /* _cplusplus */
+ # include <complex.h>
+ # define COMPLEXIFY(T) T complex
+ #endif /* __cplusplus */
+ void pshmem_complexd_put (COMPLEXIFY (double) * dest,
+ const COMPLEXIFY (double) * src,
+ size_t nelems, int pe)
+ {
+ fprintf(stderr, "Dummy %s\n", __FUNCTION__);
+ return ;
+ }
+
+ /* Old API */
+
+
/**********************************************************
start_pes
**********************************************************/

24
distr/README
Просмотреть файл

@ -1,24 +0,0 @@
Copyright (c) 2012 Mellanox Technologies, Inc.
All rights reserved.
Files oshmem.spec and buildrpm.sh provide a way to create binary and source rpms of the OSHMEM.
In order to create rpms:
./buildrpm.sh
Created rpms are located in ./rpm-dest/.
Several parameters are hardcoded in the buildshmem.sh: package name, version, release. Apart fromit, one can specify wheter to create source rpm or not.
See comments in buildrpm.sh.
Binary rpms created by the script are relocatable, implying that they can be installed in user specified directory with:
sudo rpm -ihv --prefix=<USER DIR> <RPM>
Default params:
package_name=openshmem
version=1.1
release=1
_prefix=/opt/<package_name=openshmem> - place to install
In order to create binary tarball from binary rpm:
./buildtarball.sh <rpm>
The output will appear in ./tarballs/

13
distr/build_knem_rpm.sh
Просмотреть файл

@ -1,13 +0,0 @@
#!/bin/sh
# Copyright (c) 2012 Mellanox Technologies, Inc.
# All rights reserved.
mydir=$(cd `dirname $0`;pwd)
rpmdir=$mydir/rpm-dist/`hostname`
rpmdist=$rpmdir/build
knem_tgz="$mydir/knem-0.9.7.tar.gz"
mkdir -p $rpmdir $rpmdist
rpmbuild --define="_rpmdir $rpmdir" --define="_srcrpmdir $rpmdir" --define="_sourcedir $rpmdist" --define="_specdir $rpmdist" --define="_builddir $rpmdist" -tb $knem_tgz

165
distr/buildrpm.sh
Просмотреть файл

@ -1,165 +0,0 @@
#!/bin/bash
# Copyright (c) 2012 Mellanox Technologies, Inc.
# All rights reserved.
#this three parameters define the name of rpm package: <package_name>-<version>-<release>
package_name=openshmem
#version - defined from VERSION file
#release - define from VERSION file
if [ -d /opt/knem-0.9.7mlnx1 ];then
KNEM_FOLDER="/opt/knem-0.9.7mlnx1"
else
KNEM_FOLDER="/opt/knem-0.9.7"
fi
fca='no'
debug='no'
extra_cflags=""
slurm_dir=""
while getopts :k:hf:dmw:scl: OPTION
do
case $OPTION in
k)KNEM_FOLDER=$OPTARG
;;
h)echo "`basename $0` -[h] [-k <knem folder>] [-f <spec file>] [-d (debug)] [-w <number> (-DOSHMEM_WAIT_COMPLETION_DEBUG=number)] [-s (-DOSHMEM_SM_PUT_SYNC_MODE)]"
exit 0;
;;
l)slurm_dir=$OPTARG
if [ -d "$slurm_dir/include" ]; then
echo using pmi/slurm
oshmem_configure_params="$oshmem_configure_params --with-pmi=$slurm_dir --with-slurm=$slurm_dir"
else
echo PMI selected but not found
exit -1
fi
;;
f)rpmspec=$OPTARG
;;
\?)echo "`basename $0` -[h] [-k <knem folder>] [-f <spec file>] [-d (debug)] [-w <number> (-DOSHMEM_WAIT_COMPLETION_DEBUG=number)] [-s (-DOSHMEM_SM_PUT_SYNC_MODE)]"
;;
d)debug='yes'
;;
c)fca='yes'
;;
m)mxm='yes'
;;
w) if [ $OPTARG -lt 0 ]; then
echo '-w key should be > 0';
exit 1;
fi
extra_cflags="$extra_cflags -DOSHMEM_WAIT_COMPLETION_DEBUG=$OPTARG"
;;
s)extra_cflags="$extra_cflags -DOSHMEM_SM_PUT_SYNC_MODE"
;;
esac
done
if [ $debug == "yes" ]; then
DEBUG_CONF="--enable-debug"
oshmem_name_prefix="debug-"
extra_cflags="$extra_cflags -g -O0"
elif [ $debug == "no" ]; then
DEBUG_CONF="--disable-debug"
extra_cflags="$extra_cflags -g -O2"
oshmem_name_prefix="";
fi
if [ "x" == "x$rpmspec" ]; then
echo "please provide a spec file: -f <filename>";
exit 1;
fi
if [ ! -f "$rpmspec" ]; then
echo "$rpmspec does not exist";
exit 1;
fi
if [ ! -d "$KNEM_FOLDER" ]; then
echo "$KNEM_FOLDER does not exist";
exit 1;
fi
#parameters that are passed to the ./configure script
oshmem_configure_params="$oshmem_configure_params --with-oshmem --enable-mpirun-prefix-by-default $DEBUG_CONF --with-knem=$KNEM_FOLDER"
if [ $fca == "yes" ]; then
echo using fca
oshmem_configure_params="$oshmem_configure_params --with-fca=/opt/mellanox/fca"
fi
if [ $mxm == "yes" ]; then
echo using mxm
oshmem_configure_params="$oshmem_configure_params --with-mxm=/opt/mellanox/mxm"
fi
#build the binary rpm only
build_binary_rpm=${build_binary_rpm:='no'}
#build both: binary and source rpms
build_source_rpm=${build_source_rpm:='no'}
which rpmbuild &> /dev/null;
if [ $? -ne 0 ]; then \
echo "*** This make target requires an rpm-based linux distribution."; \
(exit 1); exit 1; \
fi
mkdir -p rpm-dist sources build
work_dir=$(cd `dirname $0` && pwd)
version="`grep major= $work_dir/../VERSION | sed -e s/major=//`"."`grep minor= $work_dir/../VERSION | sed -e s/minor=//`"
release="`grep release= $work_dir/../VERSION | sed -e s/release=//`"
build=$(hg id -n | sed -e s/\+//g)
release=$build
echo oshmem version defined: $version-$release build: $build
echo Making source tarball...
(cd $work_dir/.. && ./autogen.sh && ./configure $oshmem_configure_params)
make -C $work_dir/.. distcheck && make -C $work_dir/.. get_tarball to=$work_dir/sources
rpmmacros="--define='_rpmdir $work_dir/rpm-dist' --define='_srcrpmdir $work_dir/rpm-dist' --define='_sourcedir $work_dir/sources' --define='_specdir $work_dir' --define='_builddir $work_dir/build'"
rpmopts="--nodeps --buildroot=$work_dir/_rpm"
# Generate spec file for rpm
echo Generating oshmem.spec file
if [ ! -z "$oshmem_configure_params" ]
then
oshmem_configure_params=${oshmem_configure_params//\//\\\/}
fi
if [ ! -z "$extra_cflags" ]
then
extra_cflags=${extra_cflags//\//\\\/}
fi
sed -e s/@OSHMEM_NAME@/openshmem/ -e s/@OSHMEM_CONFIGURE_PARAMS@/"$oshmem_configure_params"/ \
-e s/@OSHMEM_VERSION@/"$version"/ -e s/@OSHMEM_RELEASE@/"$release"/ -e s/@OSHMEM_CFLAGS@/"$extra_cflags"/ \
-e s/@OSHMEM_BUILD@/"$build"/ -e s/@OSHMEM_NAME_PREFIX@/"$oshmem_names_prefix"/ \
$rpmspec > oshmem.spec
echo ${version}-${build} > latest.txt
if [ $build_source_rpm == 'yes' ]
then
echo running source rpmbuild...
echo -ba -v $rpmmacros $rpmopts oshmem.spec | xargs rpmbuild
if [ $? -ne 0 ]; then \
(exit 1); exit 1; \
fi
exit 0;
fi
if [ $build_binary_rpm == 'yes' ]
then
echo running binary rpmbuild...
# echo "$rpmmacros $rpmopts $rpmspec"
echo -bb -v $rpmmacros $rpmopts oshmem.spec | xargs rpmbuild
if [ $? -ne 0 ]; then \
(exit 1); exit 1; \
fi
fi

22
distr/buildtarball.sh
Просмотреть файл

@ -1,22 +0,0 @@
#!/bin/bash
# Copyright (c) 2012 Mellanox Technologies, Inc.
# All rights reserved.
rpmname="$1"
if test "$rpmname" = ""; then
echo "Usage: buildtarball.sh <rpm>"
exit 1
fi
if test ! -f $rpmname; then
echo "Can't find $rpmname"
exit 1
fi
echo "--> Found rpm: $rpmname"
version="`rpm -qp $rpmname | sed s/openshmem-//`"
mkdir -p tarballs
rpm2cpio $rpmname | cpio -id
cd opt/mellanox
tar jcvf ../../tarballs/openshmem-$version.tar.bz openshmem
cd ../..
rm -rf opt/

151
distr/oshmem.spec.in
Просмотреть файл

@ -1,151 +0,0 @@
# Copyright (c) 2012 Mellanox Technologies, Inc.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# don't stop with an error if we don't pack all files at once
%define _unpackaged_files_terminate_build 0
# avoid this error:
# /usr/lib/rpm/debugedit: canonicalization unexpectedly shrank by one character
%define debug_package %{nil}
# variables replaced by buildrpm.sh script
%define oshmem_name @OSHMEM_NAME@
%define oshmem_configure_params @OSHMEM_CONFIGURE_PARAMS@
%define oshmem_version @OSHMEM_VERSION@
%define oshmem_release @OSHMEM_RELEASE@
%define oshmem_cflags @OSHMEM_CFLAGS@
%define oshmem_build @OSHMEM_BUILD@
%{!?configure_opts: %define configure_opts %{nil}}
#
# global Open SHMEM stuff
#
%{!?oshmem_name: %define oshmem_name openshmem}
%{!?oshmem_version: %define oshmem_version 2.0}
%{!?oshmem_release: %define oshmem_release 0}
%{!?oshmem_build: %define oshmem_build %{nil}}
%{!?oshmem_name_prefix: %define oshmem_name_prefix %{nil}}
%{!?oshmem_prefix: %define oshmem_prefix /opt/mellanox/%{oshmem_name_prefix}%{oshmem_name}/%{oshmem_version}}
%define oshmem_long_ver %{oshmem_name}-%{oshmem_version}.%{oshmem_build}
%define oshmem_source %{oshmem_long_ver}.tar.gz
%define shell_scripts_path %{_bindir}
%define shell_scripts_basename shmemvars
# fix configure
#
%{!?extra_cflags: %define extra_cflags %{nil}}
#
%define _prefix %{oshmem_prefix}
%define _sysconfdir %{_prefix}/etc
%define _libdir %{_prefix}/lib
%define _includedir %{_prefix}/include
#
# compiler settings
#
%define oshmem_compiler default
%define oshmem_cc " "
%define oshmem_cxx " "
%define oshmem_f77 " "
%define oshmem_fc " "
######################################################################
#
# Build section
#
######################################################################
Summary: Mellanox SHMEM parallel programming library.
Name: %{oshmem_name_prefix}%{oshmem_name}
Version: %{oshmem_version}
Release: %{oshmem_release}
License: Proprietary
Group: Development/Libraries
URL: http://www.mellanox.com
Source0: %{oshmem_source}
BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX)
Requires: libibverbs mxm fca
BuildRequires: gcc-c++ libstdc++ libstdc++-devel libibverbs-devel fca mxm
Provides: shmem
Packager: mellanox
Vendor: mellanox
Prefix: %{_prefix}
%description
The Mellanox SHMEM library provides fast inter-processor communication for large
messages using data passing and one-sided communication techniques.
The Mellanox SHMEM API based on OpenSHMEM standard from http://www.openshmem.org/
%prep
rm -rf $RPM_BUILD_ROOT
%setup -q -n %{oshmem_long_ver}
%build
OSHMEM_CONFIGURE_FLAGS="%{oshmem_configure_params}"
EXTRA_CFLAGS="%{extra_cflags}"
if [ "%{oshmem_compiler}" != "default" ]; then
OSHMEM_CONFIGURE_FLAGS="$OSHMEM_CONFIGURE_FLAGS CC=%{oshmem_cc} CXX=%{oshmem_cxx} F77=%{oshmem_f77} FC=%{oshmem_fc}"
fi
CFLAGS="$EXTRA_CFLAGS" && %configure $OSHMEM_CONFIGURE_FLAGS %{configure_opts}
make -j4
%install
rm -rf $RPM_BUILD_ROOT
%{__make} -j4 install DESTDIR=$RPM_BUILD_ROOT
%{__mkdir_p} $RPM_BUILD_ROOT/%{shell_scripts_path}
cat <<EOF > $RPM_BUILD_ROOT/%{shell_scripts_path}/%{shell_scripts_basename}.sh
# NOTE: This is an automatically-generated file! (generated by the
# Open MPI RPM). Any changes made here will be lost if the RPM is
# uninstalled or upgraded.
# PATH
if test -z "\`echo \$PATH | grep %{_bindir}\`"; then
PATH=%{_bindir}:\${PATH}
export PATH
fi
# LD_LIBRARY_PATH
if test -z "\`echo \$LD_LIBRARY_PATH | grep %{_libdir}\`"; then
LD_LIBRARY_PATH=%{_libdir}\${LD_LIBRARY_PATH:+:}\${LD_LIBRARY_PATH}
export LD_LIBRARY_PATH
fi
# MANPATH
if test -z "\`echo \$MANPATH | grep %{_mandir}\`"; then
MANPATH=%{_mandir}:\${MANPATH}
export MANPATH
fi
# MPI_ROOT
MPI_ROOT=%{_prefix}
export MPI_ROOT
EOF
cat>>$RPM_BUILD_ROOT/%{_sysconfdir}/openmpi-mca-params.conf<<EOF
opal_event_include=epoll
coll_fca_enable = 0
scoll_fca_enable = 0
mca_component_show_load_errors = 0
EOF
%clean
#test "x$RPM_SOURCE_DIR" != "x" && rm -rf $RPM_SOURCE_DIR
test "x$RPM_BUILD_DIR" != "x" && rm -rf $RPM_BUILD_DIR/*
test "x$RPM_BUILD_ROOT" != "x" && rm -rf $RPM_BUILD_ROOT
%files
%defattr(-,root,root,-)
%{oshmem_prefix}

118
distr/rpm2cpio
Просмотреть файл

@ -1,118 +0,0 @@
#!/usr/bin/perl
# Copyright (c) 2012 Mellanox Technologies, Inc.
# All rights reserved.
#
# Copyright (C) 1997,1998,1999 Roger Espel Llima
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and any associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# SOFTWARE'S COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE
# (whew, that's done!)
# why does the world need another rpm2cpio? because the existing one
# won't build unless you have half a ton of things that aren't really
# required for it, since it uses the same library used to extract RPM's.
# in particular, it won't build on the HPsUX box i'm on.
# sw 2002-Mar-6 Don't slurp the whole file
# Stefan 2010-Mar-10 Add bzip2 support
# add a path if desired
$gzip = "gzip";
$bzip2 = "bzip2";
sub printhelp {
print <<HERE;
rpm2cpio, perl version by Roger Espel Llima
dumps the contents to stdout as a cpio archive
use: rpm2cpio [file.rpm] > file.cpio
Here's how to use cpio:
list of contents: cpio -t -i < /file/name
extract files: cpio -d -i < /file/name
HERE
exit 0;
}
if ($#ARGV == -1) {
printhelp if -t STDIN;
$f = "STDIN";
} elsif ($#ARGV == 0) {
open(F, "< $ARGV[0]") or die "Can't read file $ARGV[0]\n";
$f = 'F';
} else {
printhelp;
}
printhelp if -t STDOUT;
read $f, $rpm, 96;
($magic, $major, $minor, $crap) = unpack("NCC C90", $rpm);
die "Not an RPM\n" if $magic != 0xedabeedb;
die "Not a version 3 or 4 RPM\n" if $major != 3 && $major != 4;
while (!eof($f)) {
$pos = tell($f);
read $f,$rpm,16;
$smagic = unpack("n", $rpm);
last if $smagic == 0x1f8b; # ZIP file magic, end of headers.
last if $smagic == 0x425a; # BZIP2 file magic, end of headers.
# Turns out that every header except the start of the gzip one is
# padded to an 8 bytes boundary.
if ($pos & 0x7) {
$pos += 7;
$pos &= ~0x7; # Round to 8 byte boundary
seek $f, $pos, 0;
read $f,$rpm,16;
}
($magic, $crap, $sections, $bytes) = unpack("N4", $rpm);
die "Error: header not recognized\n" if $magic != 0x8eade801;
$pos += 16; # for header
$pos += 16 * $sections;
$pos += $bytes;
seek $f, $pos, 0;
}
if (eof($f)) {
die "bogus RPM\n";
}
if ($smagic == 0x1f8b) {
open(ZCAT, "|$gzip -cd") || die "can't pipe to gzip\n";
} elsif ($smagic == 0x425a) {
open(ZCAT, "|$bzip2 -cd") || die "can't pipe to bzip2\n";
} else {
die "Unknown magic '$smagic' for compressed file contents. Expected 0x1f8b (gzip) or 0x425a (bzip2).";
}
print STDERR "CPIO archive found!\n";
print ZCAT $rpm;
while (read($f, ($_=''), 16384) > 0) {
print ZCAT;
}
close ZCAT;

16
knem_patch/README-knem-patch
Просмотреть файл

@ -1,16 +0,0 @@
#Copyright (c) 2012 Mellanox Technologies, Inc.
# All rights reserved.
#$COPYRIGHT$
#
#Additional copyrights may follow
#
#$HEADER$
#
File shmem_knem.patch contains the modification to the Knem kernel module version 0.9.7 that is required for the correct work of SM BTL within SHMEM which is based on Knem.
Install:
1. Get the Knem-0.9.7 source tarball from http://runtime.bordeaux.inria.fr/knem/download/knem-0.9.7.tar.gz
2. Untar the archive and cope shmem_knem.patch to the top directory of the Knem source tree
3. patch -p1 -i shmem_knem.patch
4. Build and install Knem kernel module as usual.

88
knem_patch/shmem_knem.patch
Просмотреть файл

@ -1,88 +0,0 @@
# Copyright (c) 2012 Mellanox Technologies, Inc.
# All rights reserved.
diff -cr knem-0.9.7/driver/linux/knem_main.c knem-fixed/driver/linux/knem_main.c
*** knem-0.9.7/driver/linux/knem_main.c 2011-07-28 20:06:58.000000000 +0300
--- knem-fixed/driver/linux/knem_main.c 2011-10-04 13:25:12.000000000 +0200
***************
*** 1145,1150 ****
--- 1145,1195 ----
* Copying between pinned iovecs
*/
+ #define KNEM_FOR_SHMEM 1
+
+ #if KNEM_FOR_SHMEM
+ static void perform_copy(void *dst, void *src, unsigned long length)
+ {
+ int i;
+ int num64 = length / sizeof(uint64_t);
+ int num32 = (length % sizeof(uint64_t))/sizeof(uint32_t);
+ int num16 = (length % sizeof(uint32_t))/sizeof(uint16_t);
+ int num8 = (length % sizeof(uint16_t))/sizeof(uint8_t);
+ uint64_t *dst64 = (uint64_t*)dst;
+ uint64_t *src64 = (uint64_t*)src;
+ uint32_t *dst32;
+ uint32_t *src32;
+ uint16_t *src16;
+ uint16_t *dst16;
+ uint8_t *dst8;
+ uint8_t *src8;
+
+ for (i=0; i<num64; i++)
+ {
+ *dst64++ = *src64++;
+ }
+ dst32 = (uint32_t *)dst64;
+ src32 = (uint32_t *)src64;
+ for (i=0; i<num32; i++)
+ {
+ *dst32++ = *src32++;
+ }
+ dst16 = (uint16_t *)dst32;
+ src16 = (uint16_t *)src32;
+ for (i=0; i<num16; i++)
+ {
+ *dst16++ = *src16++;
+ }
+ dst8 = (uint8_t *)dst16;
+ src8 = (uint8_t *)src16;
+ for (i=0; i<num8; i++)
+ {
+ *dst8++ = *src8++;
+ }
+
+ // printk(KERN_ERR "MEMCPY IS DONE IN KNEM, dst = %p. src = %p, len = %lu\n",(void *)dst,(void *)src,length);
+ }
+ #endif
static void
knem_memcpy_pinned(struct page * const * dst_page, unsigned dst_first_page_offset,
struct page * const * src_page, unsigned src_first_page_offset,
***************
*** 1159,1169 ****
chunk = PAGE_SIZE - src_first_page_offset;
if (likely(dst_first_page_offset + chunk > PAGE_SIZE))
chunk = PAGE_SIZE - dst_first_page_offset;
!
memcpy(dst_addr + dst_first_page_offset,
src_addr + src_first_page_offset,
chunk);
!
remaining -= chunk;
if (unlikely(!remaining))
break;
--- 1204,1218 ----
chunk = PAGE_SIZE - src_first_page_offset;
if (likely(dst_first_page_offset + chunk > PAGE_SIZE))
chunk = PAGE_SIZE - dst_first_page_offset;
! #if (!KNEM_FOR_SHMEM)
memcpy(dst_addr + dst_first_page_offset,
src_addr + src_first_page_offset,
chunk);
! #else
! perform_copy(dst_addr + dst_first_page_offset,
! src_addr + src_first_page_offset,
! chunk);
! #endif
remaining -= chunk;
if (unlikely(!remaining))
break;

4
ompi/class/ompi_free_list.c
Просмотреть файл

@ -10,12 +10,11 @@
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006-2007 Mellanox Technologies. All rights reserved.
* Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2011 NVIDIA Corporation. All rights reserved.
* Copyright (c) 2012 Los Alamos National Security, LLC. All rights
* reserved.
* Copyright (c) 2012 Mellanox Technologies, Inc.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -195,7 +194,6 @@ int ompi_free_list_grow(ompi_free_list_t* flist, size_t num_elements)
if(NULL == alloc_ptr)
return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
memset(alloc_ptr,0,alloc_size);
if (0 != flist->fl_payload_buffer_size) {
elem_size = OPAL_ALIGN(flist->fl_payload_buffer_size,

9
ompi/mca/btl/btl.h
Просмотреть файл

@ -14,8 +14,6 @@
* reserved.
* Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2012 NVIDIA Corporation. All rights reserved.
* Copyright (c) 2012 Mellanox Technologies, Inc.
* All rights reserved
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -311,13 +309,6 @@ OMPI_DECLSPEC OBJ_CLASS_DECLARATION(mca_btl_base_descriptor_t);
#define MCA_BTL_DES_FLAGS_PUT 0x0010
#define MCA_BTL_DES_FLAGS_GET 0x0020
#ifdef OSHMEM_ENABLED
#define MCA_BTL_DES_FLAGS_SHMEM_REQUEST 0x0030
#define BTL_SM_HDR_TYPE_PUT_AS_SEND (MCA_BTL_TAG_USR + 0x0A)
#define BTL_SM_HDR_TYPE_GET_AS_SEND (MCA_BTL_TAG_USR + 0x0B)
#endif /* OSHMEM_ENABLED */
/**
* Maximum number of allowed segments in src/dst fields of a descriptor.
*/

33
ompi/mca/btl/openib/btl_openib.c
Просмотреть файл

@ -11,13 +11,12 @@
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2007-2012 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2006-2009 Mellanox Technologies. All rights reserved.
* Copyright (c) 2006-2012 Los Alamos National Security, LLC. All rights
* reserved.
* Copyright (c) 2006-2007 Voltaire All rights reserved.
* Copyright (c) 2008-2012 Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2009 IBM Corporation. All rights reserved.
* Copyright (c) 2012 Mellanox Technologies, Inc.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -1348,12 +1347,11 @@ mca_btl_base_descriptor_t* mca_btl_openib_prepare_dst(
return NULL;
}
#if OSHMEM_ENABLED
#else
/* max_msg_sz is the maximum message size of the HCA (hw limitation)
set the minimum between local max_msg_sz and the remote */
max_msg_sz = MIN(openib_btl->ib_port_attr.max_msg_sz,
endpoint->endpoint_btl->ib_port_attr.max_msg_sz);
/* check if user has explicitly limited the max message size */
if (openib_component->max_hw_msg_size > 0 &&
max_msg_sz > (size_t)openib_component->max_hw_msg_size) {
@ -1365,7 +1363,7 @@ mca_btl_base_descriptor_t* mca_btl_openib_prepare_dst(
*size = (size_t)max_msg_sz;
BTL_VERBOSE(("message size limited to %" PRIsize_t "\n", *size));
}
#endif /* OSHMEM_ENABLED */
opal_convertor_get_current_pointer(convertor, &buffer);
if(NULL == registration){
@ -1390,12 +1388,6 @@ mca_btl_base_descriptor_t* mca_btl_openib_prepare_dst(
to_base_frag(frag)->segment.base.seg_addr.lval = (uint64_t)(uintptr_t) buffer;
to_base_frag(frag)->segment.base.seg_len = *size;
to_base_frag(frag)->segment.key = openib_reg->mr->rkey;
#if OSHMEM_ENABLED
/** Keep lkey of pre-registered user buffer */
to_base_frag(frag)->segment.lkey = frag->sg_entry.lkey;
#endif /* OSHMEM_ENABLED */
to_base_frag(frag)->base.order = order;
to_base_frag(frag)->base.des_flags = flags;
@ -1805,12 +1797,6 @@ int mca_btl_openib_put( mca_btl_base_module_t* btl,
to_com_frag(frag)->sg_entry.addr = src_seg->base.seg_addr.lval;
to_com_frag(frag)->sg_entry.length = src_seg->base.seg_len;
#if OSHMEM_ENABLED
/** Keep lkey of pre-registered user buffer */
to_com_frag(frag)->sg_entry.lkey = src_seg->key;
#endif /* OSHMEM_ENABLED */
to_com_frag(frag)->endpoint = ep;
#if HAVE_XRC
if (MCA_BTL_XRC_ENABLED && BTL_OPENIB_QP_TYPE_XRC(qp))
@ -1848,9 +1834,7 @@ int mca_btl_openib_get(mca_btl_base_module_t* btl,
uint64_t rem_addr = src_seg->base.seg_addr.lval;
uint32_t rkey = src_seg->key;
#if OSHMEM_ENABLED == 0
assert(openib_frag_type(frag) == MCA_BTL_OPENIB_FRAG_RECV_USER);
#endif /* OSHMEM_ENABLED */
descriptor->des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
@ -1899,12 +1883,6 @@ int mca_btl_openib_get(mca_btl_base_module_t* btl,
to_com_frag(frag)->sg_entry.addr = dst_seg->base.seg_addr.lval;
to_com_frag(frag)->sg_entry.length = dst_seg->base.seg_len;
#if OSHMEM_ENABLED
/** Keep lkey of pre-registered user buffer */
to_com_frag(frag)->sg_entry.lkey = dst_seg->lkey;
#endif /* OSHMEM_ENABLED */
to_com_frag(frag)->endpoint = ep;
#if HAVE_XRC
@ -1916,11 +1894,6 @@ int mca_btl_openib_get(mca_btl_base_module_t* btl,
qp_inflight_wqe_to_frag(ep, qp, to_com_frag(frag));
qp_reset_signal_count(ep, qp);
#if OSHMEM_ENABLED
frag->sr_desc.opcode = IBV_WR_RDMA_READ;
frag->sr_desc.send_flags = IBV_SEND_SIGNALED;
#endif /* OSHMEM_ENABLED */
if(ibv_post_send(ep->qps[qp].qp->lcl_qp, &frag->sr_desc, &bad_wr))
return OMPI_ERROR;

5
ompi/mca/btl/openib/btl_openib_async.c
Просмотреть файл

@ -1,9 +1,8 @@
/*
* Copyright (c) 2008-2009 Mellanox Technologies. All rights reserved.
* Copyright (c) 2007-2009 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2006-2007 Voltaire All rights reserved.
* Copyright (c) 2009-2010 Oracle and/or its affiliates. All rights reserved
* Copyright (c) 2012 Mellanox Technologies, Inc.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -532,7 +531,7 @@ void* btl_openib_async_thread(void * async)
int btl_openib_async_command_done(int exp)
{
int comp = 0;
int comp;
if (read(mca_btl_openib_component.async_comp_pipe[0], &comp,
sizeof(int)) < 0){
BTL_ERROR(("Failed to read from pipe"));

8
ompi/mca/btl/openib/btl_openib_endpoint.c
Просмотреть файл

@ -14,10 +14,9 @@
* Copyright (c) 2006-2012 Los Alamos National Security, LLC. All rights
* reserved.
* Copyright (c) 2006-2007 Voltaire All rights reserved.
* Copyright (c) 2006-2009 Mellanox Technologies, Inc. All rights reserved.
* Copyright (c) 2010-2011 IBM Corporation. All rights reserved.
* Copyright (c) 2010-2011 Oracle and/or its affiliates. All rights reserved
* Copyright (c) 2012 Mellanox Technologies, Inc.
* All rights reserved.
*
* $COPYRIGHT$
*
@ -347,11 +346,6 @@ static void mca_btl_openib_endpoint_construct(mca_btl_base_endpoint_t* endpoint)
endpoint->rem_info.rem_srqs = NULL;
}
#ifdef OSHMEM_ENABLED
endpoint->rdma_qp = mca_btl_openib_component.rdma_qp;
#endif /* OSHMEM_ENABLED */
endpoint->ib_addr = NULL;
endpoint->xrc_recv_qp_num = 0;
endpoint->endpoint_btl = 0;

18
ompi/mca/btl/openib/btl_openib_endpoint.h
Просмотреть файл

@ -13,9 +13,8 @@
* Copyright (c) 2006-2007 Los Alamos National Security, LLC. All rights
* reserved.
* Copyright (c) 2006-2007 Voltaire All rights reserved.
* Copyright (c) 2007-2009 Mellanox Technologies. All rights reserved.
* Copyright (c) 2010-2012 Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2012 Mellanox Technologies, Inc.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -258,9 +257,6 @@ struct mca_btl_base_endpoint_t {
/** Whether we've send out CTS to the peer or not (only used in
CTS protocol) */
bool endpoint_cts_sent;
#ifdef OSHMEM_ENABLED
int rdma_qp;
#endif /* OSHMEM_ENABLED */
};
typedef struct mca_btl_base_endpoint_t mca_btl_base_endpoint_t;
@ -513,18 +509,6 @@ static inline int check_endpoint_state(mca_btl_openib_endpoint_t *ep,
return rc;
}
#if OSHMEM_ENABLED
/* hack, since there is no clean way to find inline limits */
#include "ompi/mca/bml/bml.h"
static inline int
mca_btl_openib_rdma_inline_size(mca_bml_base_btl_t *bml_btl)
{
mca_btl_base_endpoint_t *ep = bml_btl->btl_endpoint;
return ep->qps[ep->rdma_qp].ib_inline_max;
}
#endif /* OSHMEM_ENABLED */
static inline __opal_attribute_always_inline__ int
ib_send_flags(uint32_t size, mca_btl_openib_endpoint_qp_t *qp, int do_signal)
{

5
ompi/mca/btl/openib/btl_openib_frag.h
Просмотреть файл

@ -14,8 +14,6 @@
* reserved.
* Copyright (c) 2006-2007 Voltaire All rights reserved.
* Copyright (c) 2010-2012 Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2012 Mellanox Technologies, Inc.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -289,9 +287,6 @@ typedef enum mca_btl_openib_frag_type_t mca_btl_openib_frag_type_t;
typedef struct mca_btl_openib_segment_t {
mca_btl_base_segment_t base;
uint32_t key;
#if OSHMEM_ENABLED
uint32_t lkey;
#endif
} mca_btl_openib_segment_t;
/* base openib frag */

6
ompi/mca/btl/openib/connect/base.h
Просмотреть файл

@ -1,8 +1,6 @@
/*
* Copyright (c) 2007-2008 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2012 Mellanox Technologies, Inc.
* All rights reserved.
*
* Copyright (c) 2009 Mellanox Technologies. All rights reserved.
*
* $COPYRIGHT$
*
@ -14,7 +12,7 @@
#ifndef BTL_OPENIB_CONNECT_BASE_H
#define BTL_OPENIB_CONNECT_BASE_H
#include "ompi/mca/btl/openib/connect/connect.h"
#include "connect/connect.h"
#ifdef OMPI_HAVE_RDMAOE
#define BTL_OPENIB_CONNECT_BASE_CHECK_IF_NOT_IB(btl) \

51
ompi/mca/btl/openib/connect/btl_openib_connect_oob.c
Просмотреть файл

@ -12,8 +12,7 @@
* Copyright (c) 2006-2012 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2006-2012 Los Alamos National Security, LLC. All rights
* reserved.
* Copyright (c) 2012 Mellanox Technologies, Inc.
* All rights reserved.
* Copyright (c) 2008-2011 Mellanox Technologies. All rights reserved.
* Copyright (c) 2009-2011 IBM Corporation. All rights reserved.
* Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved
*
@ -372,40 +371,6 @@ static int qp_connect_all(mca_btl_openib_endpoint_t *endpoint)
}
static void permute_array(int *permuted_qps, int nqps)
{
int i;
int idx;
int tmp;
int control[nqps];
for (i = 0; i < nqps; i++) {
permuted_qps[i] = i;
control[i] = 0;
}
for (i = 0; i < nqps - 1; i++) {
idx = i + random() % (nqps - i);
tmp = permuted_qps[i];
permuted_qps[i] = permuted_qps[idx];
permuted_qps[idx] = tmp;
}
/* verify that permutation is ok: */
for (i = 0; i < nqps; i++) {
control[permuted_qps[i]] ++;
}
for (i = 0; i < nqps; i++) {
if (control[i] != 1) {
printf("bad permutation detected: ");
for (i = 0; i < nqps; i++) printf("%d ", permuted_qps[i]);
printf("\n");
abort();
}
}
}
/*
* Create the local side of all the qp's. The remote sides will be
* connected later.
@ -415,12 +380,6 @@ static int qp_create_all(mca_btl_base_endpoint_t* endpoint)
int qp, rc, pp_qp_num = 0;
int32_t rd_rsv_total = 0;
int rand_qpns[mca_btl_openib_component.num_qps];
int i;
permute_array(rand_qpns, mca_btl_openib_component.num_qps);
for (qp = 0; qp < mca_btl_openib_component.num_qps; ++qp)
if(BTL_OPENIB_QP_TYPE_PP(qp)) {
rd_rsv_total +=
@ -433,12 +392,11 @@ static int qp_create_all(mca_btl_base_endpoint_t* endpoint)
if(0 == pp_qp_num && true == endpoint->use_eager_rdma)
pp_qp_num = 1;
for (i = 0; i < mca_btl_openib_component.num_qps; ++i) {
for (qp = 0; qp < mca_btl_openib_component.num_qps; ++qp) {
struct ibv_srq *srq = NULL;
uint32_t max_recv_wr, max_send_wr;
int32_t rd_rsv, rd_num_credits;
qp = rand_qpns[i];
/* QP used for SW flow control need some additional recourses */
if(qp == mca_btl_openib_component.credits_qp) {
rd_rsv = rd_rsv_total;
@ -505,12 +463,7 @@ static int qp_create_one(mca_btl_base_endpoint_t* endpoint, int qp,
memset(&attr, 0, sizeof(attr));
init_attr.qp_type = IBV_QPT_RC;
#if OSHMEM_ENABLED
// bump priority for rmda qp
init_attr.send_cq = openib_btl->device->ib_cq[BTL_OPENIB_RDMA_QP(qp) ? BTL_OPENIB_HP_CQ: BTL_OPENIB_LP_CQ];
#else
init_attr.send_cq = openib_btl->device->ib_cq[BTL_OPENIB_LP_CQ];
#endif /* OSHMEM_ENABLED */
init_attr.recv_cq = openib_btl->device->ib_cq[qp_cq_prio(qp)];
init_attr.srq = srq;
init_attr.cap.max_inline_data = req_inline =

116
ompi/mca/btl/sm/btl_sm.c
Просмотреть файл

@ -15,8 +15,6 @@
* All rights reserved.
* Copyright (c) 2010-2012 IBM Corporation. All rights reserved.
* Copyright (c) 2012 Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2012 Mellanox Technologies, Inc.
* All rights reserved
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -787,9 +785,6 @@ struct mca_btl_base_descriptor_t* mca_btl_sm_prepare_src(
} else {
#if OMPI_BTL_SM_HAVE_KNEM
struct knem_cmd_create_region knem_cr;
#if OSHMEM_ENABLED
memset(&knem_cr,0,sizeof(struct knem_cmd_create_region));
#endif /* OSHMEM_ENABLED */
struct knem_cmd_param_iovec knem_iov;
#endif /* OMPI_BTL_SM_HAVE_KNEM */
MCA_BTL_SM_FRAG_ALLOC_USER(frag, rc);
@ -812,13 +807,8 @@ struct mca_btl_base_descriptor_t* mca_btl_sm_prepare_src(
knem_iov.len = max_data;
knem_cr.iovec_array = (uintptr_t)&knem_iov;
knem_cr.iovec_nr = iov_count;
#if OSHMEM_ENABLED
knem_cr.protection = PROT_READ | PROT_WRITE;
knem_cr.flags = 0;
#else
knem_cr.protection = PROT_READ;
knem_cr.flags = KNEM_FLAG_SINGLEUSE;
#endif /* OSHMEM_ENABLED */
if (OPAL_UNLIKELY(ioctl(sm_btl->knem_fd, KNEM_CMD_CREATE_REGION, &knem_cr) < 0)) {
return NULL;
}
@ -984,20 +974,7 @@ int mca_btl_sm_send( struct mca_btl_base_module_t* btl,
frag->hdr->len = frag->segment.base.seg_len;
/* type of message, pt-2-pt, one-sided, etc */
frag->hdr->tag = tag;
#if OSHMEM_ENABLED
/*
* This ugly hack is done to support following configuration as:
* OSHMEM + SM => put/get for small messages using send()
*/
if (frag->hdr->tag == BTL_SM_HDR_TYPE_GET_AS_SEND) /*this is shmem_get request */
{
frag->hdr->src_addr = (void *)descriptor->des_src->seg_addr.lval; /*from where to take the data for get operation*/
}
else if (frag->hdr->tag == BTL_SM_HDR_TYPE_PUT_AS_SEND)/*this is shmem_put operation*/
{
frag->hdr->dst_addr = descriptor->des_dst->seg_addr.pval;
}
#endif /* OSHMEM_ENABLED */
MCA_BTL_SM_TOUCH_DATA_TILL_CACHELINE_BOUNDARY(frag);
frag->endpoint = endpoint;
@ -1080,12 +1057,6 @@ int mca_btl_sm_get_sync(struct mca_btl_base_module_t* btl,
icopy.local_iovec_nr = 1;
icopy.remote_cookie = src->key;
icopy.remote_offset = 0;
#if OSHMEM_ENABLED
if (des->des_flags & MCA_BTL_DES_FLAGS_SHMEM_REQUEST)
{
icopy.remote_offset = dst->key;
}
#endif
icopy.write = 0;
/* Use the DMA flag if knem supports it *and* the segment length
@ -1105,13 +1076,6 @@ int mca_btl_sm_get_sync(struct mca_btl_base_module_t* btl,
return OMPI_ERROR;
}
#if OSHMEM_ENABLED
if ( icopy.current_status == KNEM_STATUS_FAILED)
{
opal_output(0,"KNEM FAILED\n");
return OMPI_ERROR;
}
#endif /* OSHMEM_ENABLED */
/* FIXME: what if icopy.current_status == KNEM_STATUS_FAILED? */
}
#endif /* OMPI_BTL_SM_HAVE_KNEM */
@ -1170,84 +1134,6 @@ int mca_btl_sm_get_sync(struct mca_btl_base_module_t* btl,
#if OMPI_BTL_SM_HAVE_KNEM
/* No support async_get for CMA yet */
#if OSHMEM_ENABLED
int mca_btl_sm_put_async(struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* endpoint,
struct mca_btl_base_descriptor_t* des)
{
int btl_ownership;
mca_btl_sm_t* sm_btl = (mca_btl_sm_t*) btl;
mca_btl_sm_frag_t* frag = (mca_btl_sm_frag_t*)des;
mca_btl_sm_segment_t *src = (mca_btl_sm_segment_t*)des->des_src;
mca_btl_sm_segment_t *dst = (mca_btl_sm_segment_t*)des->des_dst;
struct knem_cmd_inline_copy icopy;
struct knem_cmd_param_iovec recv_iovec;
/* If we have no knem slots available, return
TEMP_OUT_OF_RESOURCE */
#ifndef OSHMEM_SM_PUT_SYNC_MODE
if (sm_btl->knem_status_num_used >=
mca_btl_sm_component.knem_max_simultaneous) {
return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
}
#endif /* OSHMEM_SM_PUT_SYNC_MODE */
/* We have a slot, so fill in the data fields. Bump the
first_avail and num_used counters. */
recv_iovec.base = (uintptr_t) src->base.seg_addr.pval;
recv_iovec.len = src->base.seg_len;
icopy.local_iovec_array = (uintptr_t)&recv_iovec;
icopy.local_iovec_nr = 1;
icopy.write = 1;
icopy.remote_cookie = dst->key;
icopy.remote_offset = src->key;
#ifndef OSHMEM_SM_PUT_SYNC_MODE
icopy.async_status_index = sm_btl->knem_status_first_avail++;
if (sm_btl->knem_status_first_avail >=
mca_btl_sm_component.knem_max_simultaneous) {
sm_btl->knem_status_first_avail = 0;
}
++sm_btl->knem_status_num_used;
/* Use the DMA flag if knem supports it *and* the segment length
is greater than the cutoff */
sm_btl->knem_frag_array[icopy.async_status_index] = frag;
icopy.flags = KNEM_FLAG_ASYNCDMACOMPLETE;
#else
icopy.flags = 0;
#endif /* OSHMEM_SM_PUT_SYNC_MODE */
if (mca_btl_sm_component.knem_dma_min <= dst->base.seg_len) {
icopy.flags = mca_btl_sm_component.knem_dma_flag;
}
if (OPAL_LIKELY(0 == ioctl(sm_btl->knem_fd,
KNEM_CMD_INLINE_COPY, &icopy))) {
#ifndef OSHMEM_SM_PUT_SYNC_MODE
if (icopy.current_status != KNEM_STATUS_PENDING) {
/* request completed synchronously */
/* FIXME: what if icopy.current_status == KNEM_STATUS_FAILED? */
#endif /* OSHMEM_SM_PUT_SYNC_MODE */
btl_ownership = (frag->base.des_flags & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP);
if (0 != (MCA_BTL_DES_SEND_ALWAYS_CALLBACK & frag->base.des_flags)) {
frag->base.des_cbfunc(&mca_btl_sm.super,
frag->endpoint, &frag->base,
OMPI_SUCCESS);
}
if (btl_ownership) {
MCA_BTL_SM_FRAG_RETURN(frag);
}
#ifndef OSHMEM_SM_PUT_SYNC_MODE
--sm_btl->knem_status_num_used;
++sm_btl->knem_status_first_used;
if (sm_btl->knem_status_first_used >=
mca_btl_sm_component.knem_max_simultaneous) {
sm_btl->knem_status_first_used = 0;
}
}
#endif /* OSHMEM_SM_PUT_SYNC_MODE */
return OMPI_SUCCESS;
} else {
return OMPI_ERROR;
}
}
#endif /* OSHMEM_ENABLED */
/**
* Initiate an asynchronous get.
*

11
ompi/mca/btl/sm/btl_sm.h
Просмотреть файл

@ -14,8 +14,6 @@
* Copyright (c) 2010-2013 Los Alamos National Security, LLC.
* All rights reserved.
* Copyright (c) 2010-2012 IBM Corporation. All rights reserved.
* Copyright (c) 2012 Mellanox Technologies, Inc.
* All rights reserved
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -184,9 +182,6 @@ struct mca_btl_sm_component_t {
#if OMPI_BTL_SM_HAVE_KNEM
/* Knem capabilities info */
struct knem_cmd_info knem_info;
#if OSHMEM_ENABLED
unsigned int knem_threshold;
#endif /* OSHMEM_ENABLED */
#endif /* OMPI_BTL_SM_HAVE_KNEM */
/** MCA: should we be using knem or not? neg=try but continue if
@ -513,12 +508,6 @@ extern int mca_btl_sm_get_sync(
struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* endpoint,
struct mca_btl_base_descriptor_t* des );
#if OSHMEM_ENABLED
extern int mca_btl_sm_put_async(
struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* endpoint,
struct mca_btl_base_descriptor_t* des );
#endif /* OSHMEM_ENABLED */
extern struct mca_btl_base_descriptor_t* mca_btl_sm_prepare_dst(
struct mca_btl_base_module_t* btl,

65
ompi/mca/btl/sm/btl_sm_component.c
Просмотреть файл

@ -15,8 +15,6 @@
* All rights reserved.
* Copyright (c) 2011 NVIDIA Corporation. All rights reserved.
* Copyright (c) 2010-2012 IBM Corporation. All rights reserved.
* Copyright (c) 2012 Mellanox Technologies, Inc.
* All rights reserved
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -170,25 +168,6 @@ static int sm_register(void)
}
mca_btl_sm_component.use_knem = 0;
}
#if OSHMEM_ENABLED
#if OMPI_BTL_SM_HAVE_KNEM
mca_btl_sm.knem_fd = -1;
mca_base_param_reg_int(&mca_btl_sm_component.super.btl_version,
"knem_threshold", "Messages of the size greater than this value are sent via KNEM others go through SM Send",
false, false, 12228, &i);
if (i < 0)
{
opal_output(0,"Error: knem threshold has to be positive value or zero; using default value: 12228 bytes");
i = 12228;
}
mca_btl_sm_component.knem_threshold = (unsigned int)i;
mca_base_param_reg_int(&mca_btl_sm_component.super.btl_version,
"component_use_knem_value", NULL,
true, false, mca_btl_sm_component.use_knem, NULL);
#endif /* OMPI_BTL_SM_HAVE_KNEM */
#endif /* OSHMEM_ENABLED */
/* Currently disabling DMA mode by default; it's not clear that
this is useful in all applications and architectures. */
mca_base_param_reg_int(&mca_btl_sm_component.super.btl_version,
@ -200,7 +179,7 @@ static int sm_register(void)
mca_base_param_reg_int(&mca_btl_sm_component.super.btl_version,
"knem_max_simultaneous",
"Max number of simultaneous ongoing knem operations to support (0 = do everything synchronously, which probably gives the best large message latency; >0 means to do all operations asynchronously, which supports better overlap for simultaneous large message sends)",
false, false, 100,
false, false, 0,
&mca_btl_sm_component.knem_max_simultaneous);
/* CMA parameters */
@ -244,9 +223,6 @@ static int sm_register(void)
#if OMPI_BTL_SM_HAVE_KNEM || OMPI_BTL_SM_HAVE_CMA
if (mca_btl_sm_component.use_knem || mca_btl_sm_component.use_cma) {
mca_btl_sm.super.btl_flags |= MCA_BTL_FLAGS_GET;
#if OSHMEM_ENABLED
mca_btl_sm.super.btl_flags |= MCA_BTL_FLAGS_PUT;
#endif /* OSHMEM_ENABLED */
}
if (mca_btl_sm_component.use_knem && mca_btl_sm_component.use_cma) {
@ -275,9 +251,6 @@ static int sm_register(void)
static int mca_btl_sm_component_open(void)
{
mca_btl_sm_component.sm_max_btls = 1;
#if OSHMEM_ENABLED
OBJ_CONSTRUCT(&mca_btl_sm_component.sm_lock, opal_mutex_t);
#endif /* OSHMEM_ENABLED */
/* make sure the number of fifos is a power of 2 */
mca_btl_sm_component.nfifos = opal_next_poweroftwo_inclusive (mca_btl_sm_component.nfifos);
@ -292,9 +265,7 @@ static int mca_btl_sm_component_open(void)
mca_btl_sm_component.eager_limit = mca_btl_sm.super.btl_eager_limit;
/* initialize objects */
#if !OSHMEM_ENABLED
OBJ_CONSTRUCT(&mca_btl_sm_component.sm_lock, opal_mutex_t);
#endif /* !OSHMEM_ENABLED */
OBJ_CONSTRUCT(&mca_btl_sm_component.sm_frags_eager, ompi_free_list_t);
OBJ_CONSTRUCT(&mca_btl_sm_component.sm_frags_max, ompi_free_list_t);
OBJ_CONSTRUCT(&mca_btl_sm_component.sm_frags_user, ompi_free_list_t);
@ -736,12 +707,6 @@ mca_btl_sm_component_init(int *num_btls,
ompi_node_rank_t my_node_rank = OMPI_NODE_RANK_INVALID;
#if OMPI_BTL_SM_HAVE_KNEM
int rc;
#if OSHMEM_ENABLED
if (mca_btl_sm_component.use_knem == 0)
{
opal_output_verbose(1, mca_btl_base_output, "SM BTL will operate without KNEM kernel module");
}
#endif /* OSHMEM_ENABLED */
#endif /* OMPI_BTL_SM_HAVE_KNEM */
*num_btls = 0;
@ -914,11 +879,6 @@ mca_btl_sm_component_init(int *num_btls,
}
}
}
#if OSHMEM_ENABLED
/*always use sync get and sync put according to shmem logic*/
mca_btl_sm.super.btl_get = mca_btl_sm_get_sync;
mca_btl_sm.super.btl_put = mca_btl_sm_put_async;
#else
/* Set the BTL get function pointer if we're supporting KNEM;
choose between synchronous and asynchronous. */
if (mca_btl_sm_component.knem_max_simultaneous > 0) {
@ -926,7 +886,6 @@ mca_btl_sm_component_init(int *num_btls,
} else {
mca_btl_sm.super.btl_get = mca_btl_sm_get_sync;
}
#endif /* OSHMEM_ENABLED */
}
#endif /* OMPI_BTL_SM_HAVE_KNEM */
@ -961,9 +920,6 @@ mca_btl_sm_component_init(int *num_btls,
/* If "use_knem" is positive, then it's an error if knem support
is not available -- deactivate the sm btl. */
if (mca_btl_sm_component.use_knem > 0) {
#if OSHMEM_ENABLED
opal_output(0,"Error: SM BTL was explicitly requested to operate with KNEM support but KNEM module is not available! SM BTL will be deactivated");
#endif
return NULL;
}
@ -1095,27 +1051,8 @@ int mca_btl_sm_component_progress(void)
seg.seg_len = hdr->len;
Frag.base.des_dst_cnt = 1;
Frag.base.des_dst = &seg;
#if OSHMEM_ENABLED
/*
* This ugly hack is done to support following configuration as:
* OSHMEM + SM => put/get for small messages using send()
*/
if (hdr->tag == (MCA_BTL_TAG_USR + 0xA))
{
memcpy(hdr->dst_addr, seg.seg_addr.pval, hdr->len);
}
else if (hdr->tag == (MCA_BTL_TAG_USR + 0xB))
{
memcpy(seg.seg_addr.pval, hdr->src_addr, hdr->len);
}
else{
reg->cbfunc(&mca_btl_sm.super, hdr->tag, &(Frag.base),
reg->cbdata);
}
#else
reg->cbfunc(&mca_btl_sm.super, hdr->tag, &(Frag.base),
reg->cbdata);
#endif /* OSHMEM_ENABLED */
/* return the fragment */
MCA_BTL_SM_FIFO_WRITE(
mca_btl_sm_component.sm_peers[peer_smp_rank],

9
ompi/mca/btl/sm/btl_sm_frag.h
Просмотреть файл

@ -11,8 +11,6 @@
* All rights reserved.
* Copyright (c) 2008 Sun Microsystems, Inc. All rights reserved.
* Copyright (c) 2009 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2012 Mellanox Technologies, Inc.
* All rights reserved
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -44,10 +42,6 @@ struct mca_btl_sm_hdr_t {
size_t len;
int my_smp_rank;
mca_btl_base_tag_t tag;
#if OSHMEM_ENABLED
void *dst_addr;
void *src_addr;
#endif /* OSHMEM_ENABLED */
};
typedef struct mca_btl_sm_hdr_t mca_btl_sm_hdr_t;
@ -55,9 +49,6 @@ struct mca_btl_sm_segment_t {
mca_btl_base_segment_t base;
#if OMPI_BTL_SM_HAVE_KNEM || OMPI_BTL_SM_HAVE_CMA
uint64_t key;
#if OSHMEM_ENABLED
uint32_t lkey;
#endif /* OSHMEM_ENABLED */
#endif /* OMPI_BTL_SM_HAVE_KNEM || OMPI_BTL_SM_HAVE_CMA */
};
typedef struct mca_btl_sm_segment_t mca_btl_sm_segment_t;

7
ompi/mca/mpool/grdma/mpool_grdma_module.c
Просмотреть файл

@ -12,11 +12,10 @@
* All rights reserved.
* Copyright (c) 2006-2009 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2006 Voltaire. All rights reserved.
* Copyright (c) 2007 Mellanox Technologies. All rights reserved.
* Copyright (c) 2010 IBM Corporation. All rights reserved.
* Copyright (c) 2011-2012 Los Alamos National Security, LLC. All rights
* reserved.
* Copyright (c) 2012 Mellanox Technologies, Inc.
* All rights reserved.
*
* $COPYRIGHT$
*
@ -141,13 +140,13 @@ void* mca_mpool_grdma_alloc(mca_mpool_base_module_t *mpool, size_t size,
#ifdef HAVE_POSIX_MEMALIGN
if((errno = posix_memalign(&base_addr, align, size)) != 0)
return NULL;
memset(base_addr,0,size);
addr = base_addr;
#else
base_addr = malloc(size + align);
if(NULL == base_addr)
return NULL;
memset(base_addr,0,size + align);
addr = (void*)OPAL_ALIGN((uintptr_t)base_addr, align, uintptr_t);
#endif

7
ompi/mca/rte/rte.h
Просмотреть файл

@ -1,7 +1,5 @@
/*
* Copyright (c) 2012 Los Alamos National Security, LLC. All rights reserved.
* Copyright (c) 2012 Mellanox Technologies, Inc.
* All rights reserved.
*
* $COPYRIGHT$
*
@ -215,11 +213,6 @@ BEGIN_C_DECLS
#define OMPI_RML_TAG_OFACM OMPI_RML_TAG_BASE+11
#define OMPI_RML_TAG_XOFACM OMPI_RML_TAG_BASE+12
#if OSHMEM_ENABLED
/* open shmem oob communication */
#define OMPI_RML_TAG_SHMEM OMPI_RML_TAG_BASE+13
#endif
#define OMPI_RML_TAG_DYNAMIC OMPI_RML_TAG_BASE+200
typedef struct {

5
ompi/mca/sbgp/ibnet/Makefile.am
Просмотреть файл

@ -43,7 +43,7 @@ mca_sbgp_ibnet_la_LDFLAGS = -module -avoid-version $(sbgp_ibnet_LDFLAGS) $(btl_o
mca_sbgp_ibnet_la_LIBADD = $(sbgp_ibnet_LIBS) $(btl_openib_LIBS) \
$(top_ompi_builddir)/ompi/mca/common/verbs/libmca_common_verbs.la \
$(top_ompi_builddir)/ompi/mca/common/ofacm/libmca_common_ofacm.la
#ompi/mca/sbgp/ibnet/Makefile.am
noinst_LTLIBRARIES = $(component_noinst)
libmca_sbgp_ibnet_la_SOURCES =$(sources)
libmca_sbgp_ibnet_la_LDFLAGS = -module -avoid-version
@ -54,7 +54,4 @@ $(top_ompi_builddir)/ompi/mca/common/ofacm/libmca_common_ofacm.la: foo.c
$(top_ompi_builddir)/ompi/mca/common/ofautils/libmca_common_ofautils.la: foo.c
cd $(top_ompi_builddir)/ompi/mca/common/ofautils && $(MAKE)
#$(top_ompi_builddir)/ompi/mca/common/commpatterns/libmca_common_commpatterns.la: foo.c
# cd $(top_ompi_builddir)/ompi/mca/common/commpatterns && $(MAKE)
foo.c:

1
ompi/mca/sbgp/ibnet/sbgp_ibnet_module.c
Просмотреть файл

@ -25,6 +25,7 @@
#include "ompi/mca/sbgp/ibnet/sbgp_ibnet.h"
#include "ompi/mca/common/ofacm/base.h"
#include "ompi/mca/common/ofacm/connect.h"
#include "ompi/patterns/comm/coll_ops.h"
/*
* Unused
static int ibnet_module_enable(mca_sbgp_base_module_t *module,

8
ompi/mpi/c/finalize.c
Просмотреть файл

@ -9,8 +9,6 @@
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2012 Mellanox Technologies, Inc.
* All rights reserved
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -24,11 +22,6 @@
#include "ompi/runtime/params.h"
#include "ompi/errhandler/errhandler.h"
#if OSHMEM_ENABLED
/* Do nothing because SHMEM has made MPI ready and destroy
*/
#pragma weak MPI_Finalize
#else
#if OPAL_HAVE_WEAK_SYMBOLS && OMPI_PROFILING_DEFINES
#pragma weak MPI_Finalize = PMPI_Finalize
#endif
@ -36,7 +29,6 @@
#if OMPI_PROFILING_DEFINES
#include "ompi/mpi/c/profile/defines.h"
#endif
#endif /* OSHMEM_ENABLED */
static const char FUNC_NAME[] = "MPI_Finalize";

10
ompi/mpi/c/init.c
Просмотреть файл

@ -11,8 +11,6 @@
* All rights reserved.
* Copyright (c) 2007 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2007-2008 Sun Microsystems, Inc. All rights reserved.
* Copyright (c) 2012 Mellanox Technologies, Inc.
* All rights reserved
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -29,13 +27,6 @@
#include "ompi/errhandler/errhandler.h"
#include "ompi/constants.h"
#if OSHMEM_ENABLED
/* start_pes() is called instead of MPI_Init() in case OpenSHMEM usage
* Do nothing because SHMEM has made MPI ready
*/
#pragma weak MPI_Init
#else
#if OPAL_HAVE_WEAK_SYMBOLS && OMPI_PROFILING_DEFINES
#pragma weak MPI_Init = PMPI_Init
#endif
@ -43,7 +34,6 @@
#if OMPI_PROFILING_DEFINES
#include "ompi/mpi/c/profile/defines.h"
#endif
#endif /* OSHMEM_ENABLED */
static const char FUNC_NAME[] = "MPI_Init";

11
ompi/mpi/c/init_thread.c
Просмотреть файл

@ -10,8 +10,6 @@
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2010 Oak Ridge National Labs. All rights reserved.
* Copyright (c) 2012 Mellanox Technologies, Inc.
* All rights reserved
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -27,14 +25,6 @@
#include "ompi/errhandler/errhandler.h"
#include "ompi/constants.h"
#if OSHMEM_ENABLED
/* start_pes() is called instead of MPI_Init_thread() in case OpenSHMEM usage
* Do nothing because SHMEM has made MPI ready
*/
#pragma weak MPI_Init_thread
#else
#if OPAL_HAVE_WEAK_SYMBOLS && OMPI_PROFILING_DEFINES
#pragma weak MPI_Init_thread = PMPI_Init_thread
#endif
@ -42,7 +32,6 @@
#if OMPI_PROFILING_DEFINES
#include "ompi/mpi/c/profile/defines.h"
#endif
#endif /* OSHMEM_ENABLED */
static const char FUNC_NAME[] = "MPI_Init_thread";

16
ompi/tools/Makefile.am
Просмотреть файл

@ -10,9 +10,6 @@
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2012 Mellanox Technologies, Inc.
# All rights reserved
#
# $COPYRIGHT$
#
# Additional copyrights may follow
@ -25,18 +22,9 @@
EXTRA_DIST += tools/win_makefile tools/CMakeLists.txt
SUBDIRS += \
tools/ompi_info \
tools/wrappers
DIST_SUBDIRS += \
tools/ompi_info \
tools/wrappers
# Don`t build this tool during OMPI build
# it is built as part of OSHMEM build
if OSHMEM_SUPPORT
else
SUBDIRS += \
tools/ompi_info
DIST_SUBDIRS += \
tools/ompi_info
endif

7
ompi/tools/ompi_info/Makefile.am
Просмотреть файл

@ -13,8 +13,6 @@
# Copyright (c) 2008 Sun Microsystems, Inc. All rights reserved.
# Copyright (c) 2012 Los Alamos National Security, LLC.
# All rights reserved.
# Copyright (c) 2012 Mellanox Technologies, Inc.
# All rights reserved
# $COPYRIGHT$
#
# Additional copyrights may follow
@ -64,11 +62,8 @@ ompi_info_SOURCES = \
param.c \
components.c \
version.c
if OSHMEM_SUPPORT
ompi_info_LDADD = $(top_builddir)/oshmem/libshmem.la
else
ompi_info_LDADD = $(top_builddir)/ompi/libmpi.la
endif
if OMPI_RTE_ORTE
ompi_info_LDADD += $(top_builddir)/orte/libopen-rte.la
endif

57
ompi/tools/ompi_info/components.c
Просмотреть файл

@ -13,8 +13,6 @@
* Copyright (c) 2010-2012 Los Alamos National Security, LLC.
* All rights reserved.
* Copyright (c) 2011-2012 University of Houston. All rights reserved.
* Copyright (c) 2012 Mellanox Technologies, Inc.
* All rights reserved
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -63,23 +61,11 @@
#include "ompi/mca/sharedfp/base/base.h"
#include "ompi/runtime/params.h"
#if OSHMEM_ENABLED
#include "oshmem/runtime/runtime.h"
#endif
#if OPAL_ENABLE_FT_CR == 1
#include "ompi/mca/crcp/crcp.h"
#include "ompi/mca/crcp/base/base.h"
#endif
#if OSHMEM_ENABLED
#include "oshmem/mca/scoll/base/base.h"
#include "oshmem/mca/spml/base/base.h"
#include "oshmem/mca/memheap/base/base.h"
#include "oshmem/mca/atomic/base/base.h"
#endif
#include "ompi/tools/ompi_info/ompi_info.h"
@ -140,13 +126,6 @@ int ompi_info_register_components(opal_pointer_array_t *mca_types,
goto error;
}
#if OSHMEM_ENABLED
if (OMPI_SUCCESS != oshmem_shmem_register_params()) {
str = "oshmem_shmem_Register_params failed";
goto error;
}
#endif
/* Find / open all components */
map = OBJ_NEW(opal_info_component_map_t);
map->type = strdup("base");
@ -181,32 +160,6 @@ int ompi_info_register_components(opal_pointer_array_t *mca_types,
goto breakout;
}
#if OSHMEM_ENABLED
if (OMPI_SUCCESS != mca_scoll_base_open()) {
goto error;
}
map = OBJ_NEW(opal_info_component_map_t);
map->type = strdup("scoll");
map->components = &mca_scoll_base_components_opened;
opal_pointer_array_add(component_map, map);
if (OMPI_SUCCESS != mca_memheap_base_open()) {
goto error;
}
map = OBJ_NEW(opal_info_component_map_t);
map->type = strdup("memheap");
map->components = &mca_memheap_base_components_opened;
opal_pointer_array_add(component_map, map);
if (OMPI_SUCCESS != mca_atomic_base_open()) {
goto error;
}
map = OBJ_NEW(opal_info_component_map_t);
map->type = strdup("atomic");
map->components = &mca_atomic_base_components_opened;
opal_pointer_array_add(component_map, map);
#endif
if (OMPI_SUCCESS != (rc = mca_coll_base_open()) &&
OMPI_ERR_BAD_PARAM != rc) {
str = "coll open";
@ -363,16 +316,6 @@ int ompi_info_register_components(opal_pointer_array_t *mca_types,
goto breakout;
}
#if OSHMEM_ENABLED
if (OMPI_SUCCESS != mca_spml_base_open()) {
goto error;
}
map = OBJ_NEW(opal_info_component_map_t);
map->type = strdup("spml");
map->components = &mca_spml_base_components_available;
opal_pointer_array_add(component_map, map);
#endif
/* No need to call the bml_base_open() because the ob1 pml calls it.
* mca_bml_base_open();
*/

12
ompi/tools/ompi_info/ompi_info.c
Просмотреть файл

@ -13,8 +13,6 @@
* Copyright (c) 2011-2012 University of Houston. All rights reserved.
* Copyright (c) 2010-2012 Los Alamos National Security, LLC.
* All rights reserved.
* Copyright (c) 2012 Mellanox Technologies, Inc.
* All rights reserved
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -117,16 +115,6 @@ int main(int argc, char *argv[])
opal_pointer_array_add(&mca_types, "btl");
opal_pointer_array_add(&mca_types, "coll");
opal_pointer_array_add(&mca_types, "common");
#if OSHMEM_ENABLED
opal_pointer_array_add(&mca_types, "scoll");
opal_pointer_array_add(&mca_types, "scoll_basic");
opal_pointer_array_add(&mca_types, "scoll_fca");
opal_pointer_array_add(&mca_types, "spml");
opal_pointer_array_add(&mca_types, "memheap");
opal_pointer_array_add(&mca_types, "shmalloc");
opal_pointer_array_add(&mca_types, "atomic");
opal_pointer_array_add(&mca_types, "shmem");
#endif
#if OPAL_ENABLE_FT_CR == 1
opal_pointer_array_add(&mca_types, "crcp");
#endif

30
ompi/tools/ompi_info/version.c
Просмотреть файл

@ -13,8 +13,6 @@
* Copyright (c) 2008-2011 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2012 Los Alamos National Security, LLC.
* All rights reserved.
* Copyright (c) 2012 Mellanox Technologies, Inc.
* All rights reserved
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -29,10 +27,6 @@
#include "mpi.h"
#if OSHMEM_ENABLED
#include "shmem.h"
#endif
#include "opal/version.h"
#if OMPI_RTE_ORTE
#include "orte/version.h"
@ -135,36 +129,19 @@ void ompi_info_show_ompi_version(const char *scope)
opal_info_out("Package", "package", OPAL_PACKAGE_STRING);
(void)asprintf(&tmp, "%s:version:full", ompi_info_type_ompi);
#if OSHMEM_ENABLED
tmp2 = opal_info_make_version_str(scope,
OSHMEM_MAJOR_VERSION, OSHMEM_MINOR_VERSION,
OSHMEM_RELEASE_VERSION,
OSHMEM_GREEK_VERSION,
OSHMEM_WANT_REPO_REV, OSHMEM_REPO_REV);
opal_info_out("Open SHMEM", tmp, tmp2);
#else
tmp2 = opal_info_make_version_str(scope,
OMPI_MAJOR_VERSION, OMPI_MINOR_VERSION,
OMPI_RELEASE_VERSION,
OMPI_GREEK_VERSION,
OMPI_WANT_REPO_REV, OMPI_REPO_REV);
opal_info_out("Open MPI", tmp, tmp2);
#endif
free(tmp);
free(tmp2);
(void)asprintf(&tmp, "%s:version:repo", ompi_info_type_ompi);
#if OSHMEM_ENABLED
opal_info_out("Open SHMEM repo revision", tmp, OSHMEM_REPO_REV);
#else
opal_info_out("Open MPI repo revision", tmp, OMPI_REPO_REV);
#endif
free(tmp);
(void)asprintf(&tmp, "%s:version:release_date", ompi_info_type_ompi);
#if OSHMEM_ENABLED
opal_info_out("Open SHMEM release date", tmp, OSHMEM_RELEASE_DATE);
#else
opal_info_out("Open MPI release date", tmp, OMPI_RELEASE_DATE);
#endif
free(tmp);
#if OMPI_RTE_ORTE
@ -175,17 +152,10 @@ void ompi_info_show_ompi_version(const char *scope)
/* show the opal version */
opal_info_show_opal_version(scope);
#if OSHMEM_ENABLED
tmp2 = opal_info_make_version_str(scope,
SHMEM_VERSION, SHMEM_SUBVERSION,
0, "", 0, "");
opal_info_out("SHMEM API", "shmem-api:version:full", tmp2);
#else
tmp2 = opal_info_make_version_str(scope,
MPI_VERSION, MPI_SUBVERSION,
0, "", 0, "");
opal_info_out("MPI API", "mpi-api:version:full", tmp2);
#endif
free(tmp2);
opal_info_out("Ident string", "ident", OPAL_IDENT_STRING);

5
opal/etc/openmpi-mca-params.conf
Просмотреть файл

@ -10,8 +10,6 @@
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2006 Cisco Systems, Inc. All rights reserved.
# Copyright (c) 2012 Mellanox Technologies, Inc.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
@ -58,6 +56,3 @@
# See "ompi_info --param all all" for a full listing of Open MPI MCA
# parameters available and their default values.
coll_fca_enable = 0
scoll_fca_enable = 0
mca_component_show_load_errors = 0

8
opal/include/opal_config_bottom.h
Просмотреть файл

@ -11,8 +11,6 @@
* All rights reserved.
* Copyright (c) 2009 Sun Microsystems, Inc. All rights reserved.
* Copyright (c) 2009-2011 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2012 Mellanox Technologies, Inc.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -222,12 +220,6 @@
# define __opal_attribute_weak_alias__(a)
#endif
#if OPAL_HAVE_ATTRIBUTE_DESTRUCTOR
# define __opal_attribute_destructor__ __attribute__((__destructor__))
#else
# define __opal_attribute_destructor__
#endif
/***********************************************************************
*
* Windows library interface declaration code

7
opal/threads/condition.c
Просмотреть файл

@ -9,8 +9,6 @@
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2012 Mellanox Technologies, Inc.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -29,11 +27,6 @@ static void opal_condition_construct(opal_condition_t *c)
c->c_signaled = 0;
#if OPAL_HAVE_POSIX_THREADS
pthread_cond_init(&c->c_cond, NULL);
#endif
#ifdef OSHMEM_WAIT_COMPLETION_DEBUG
c->my_pe = -1;
c->puts_counter_sm = 0;
c->puts_counter_openib = 0;
#endif
c->name = NULL;
}

42
opal/threads/condition.h
Просмотреть файл

@ -11,8 +11,6 @@
* All rights reserved.
* Copyright (c) 2007 Los Alamos National Security, LLC. All rights
* reserved.
* Copyright (c) 2012 Mellanox Technologies, Inc.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -56,39 +54,11 @@ struct opal_condition_t {
pthread_cond_t c_cond;
#elif OPAL_HAVE_SOLARIS_THREADS
cond_t c_cond;
#endif
#ifdef OSHMEM_WAIT_COMPLETION_DEBUG
int my_pe;
int *pe_dest;
int *msg_length;
char **btl_name;
char **op_name;
int puts_counter_sm;
int puts_counter_openib;
uint64_t *src;
uint64_t *dst;
#endif
char *name;
};
typedef struct opal_condition_t opal_condition_t;
#ifdef OSHMEM_WAIT_COMPLETION_DEBUG
static void print_oshmem_wait_condition_dbg_info(opal_condition_t *c)
{
char output[10000];
int i;
int stride=0;
stride += sprintf(&(output[stride]),"rank=%i, puts_count_sm=%i, puts_count_openib=%i:\n",c->my_pe,c->puts_counter_sm,c->puts_counter_openib);
for (i=OSHMEM_WAIT_COMPLETION_DEBUG-1; i>=0; i--)
{
stride += sprintf(&(output[stride]),"\t%i: %s, pe_dst=%i, btl=%s, msg_length=%i, src=%p, dst=%p\n",OSHMEM_WAIT_COMPLETION_DEBUG-1-i,
c->op_name[i],c->pe_dest[i],c->btl_name[i],c->msg_length[i],(void *)(uintptr_t)c->src[i],(void *)(uintptr_t)c->dst[i]);
}
fprintf(stderr,"%s",output);
fflush(stderr);
}
#endif
OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_condition_t);
@ -126,19 +96,7 @@ static inline int opal_condition_wait(opal_condition_t *c, opal_mutex_t *m)
}
#endif
} else {
#ifdef OSHMEM_WAIT_COMPLETION_DEBUG
time_t wait_time = time(NULL);
time_t show_dbg_info_timeout = 60*5; //wait for 5 minutes
#endif
while (c->c_signaled == 0) {
#ifdef OSHMEM_WAIT_COMPLETION_DEBUG
if (wait_time && (time(NULL)-wait_time > show_dbg_info_timeout) && (c->my_pe >= 0))
{
wait_time = 0;
print_oshmem_wait_condition_dbg_info(c);
}
#endif
opal_progress();
OPAL_CR_TEST_CHECKPOINT_READY_STALL();
}

22
orte/etc/Makefile.am
Просмотреть файл

@ -28,22 +28,6 @@ EXTRA_DIST = $(orte_config_files)
# (http://www.open-mpi.org/community/lists/devel/2008/06/4080.php) for
# details why the mkdir is in install-data-local.
module_file=$(DESTDIR)$(sysconfdir)/shmem_modulefile
module_file_text="\#%Module\n\
\n\
\# NOTE: This is an automatically-generated file! Any changes\n\
\# made here will be lost a) if the RPM is uninstalled, or\n\
\# if the RPM is upgraded or uninstalled.\n\
\n\
proc ModulesHelp { } {\n\
puts stderr \"This module adds Mellanox ScalableSHMEM to various paths\"\n\
}\n\
\n\
module-whatis \"Sets up Mellanox ScalableSHMEM in your enviornment\"\n\
\n\
prepend-path PATH \"$(bindir)\"\n\
prepend-path LD_LIBRARY_PATH \"$(libdir)\""
install-data-local:
$(MKDIR_P) $(DESTDIR)$(sysconfdir)
@ p="$(orte_config_files)"; \
@ -60,9 +44,6 @@ install-data-local:
$(INSTALL_DATA) $$d$$file $(DESTDIR)$(sysconfdir)/$$f; \
fi; \
done
if OSHMEM_SUPPORT
echo -e $(module_file_text)>$(module_file)
endif
# Only remove if exactly the same as what in our tree
# NOTE TO READER: Bourne shell if ... fi evaluates the body if
@ -79,6 +60,3 @@ uninstall-local:
fi ; \
fi ; \
done
if OSHMEM_SUPPORT
rm $(module_file)
endif

128
oshmem/CMakeLists.txt
Просмотреть файл

@ -1,128 +0,0 @@
# Copyright (c) 2012 Mellanox Technologies, Inc.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# The source code is compiled as C++ for dynamic build
# and compiled as C for static build
PROJECT (OSHMEM)
# Recuresive search sub directories excluding mca, mpi and tools.
# Add sources in different source groups.
INCLUDE(list_subdirs)
CHECK_SUBDIRS("${PROJECT_SOURCE_DIR}" OSHMEM_SUBDIRS)
SET(OSHMEM_EXCLUDE_SUBDIRS mca shmem class info)
FOREACH(OSHMEM_SUBDIR ${OSHMEM_SUBDIRS})
LIST(FIND OSHMEM_EXCLUDE_SUBDIRS ${OSHMEM_SUBDIR} OSHMEM_EXCLUDE_SUBDIR)
IF(${OSHMEM_EXCLUDE_SUBDIR} EQUAL -1)
FILE(GLOB_RECURSE OSHMEM_${OSHMEM_SUBDIR}_FILES
"${OSHMEM_SUBDIR}/*.h" "${OSHMEM_SUBDIR}/*.c" "${OSHMEM_SUBDIR}/*.cc" "${OSHMEM_SUBDIR}/*.cpp")
SET (OSHMEM_SOURCE_FILES
${OSHMEM_SOURCE_FILES}
${OSHMEM_${OSHMEM_SUBDIR}_FILES}
)
SOURCE_GROUP("${OSHMEM_SUBDIR}" FILES ${OSHMEM_${OSHMEM_SUBDIR}_FILES})
ENDIF(${OSHMEM_EXCLUDE_SUBDIR} EQUAL -1)
ENDFOREACH(OSHMEM_SUBDIR ${OSHMEM_SUBDIRS})
# Add MPI C files
FILE(GLOB OSHMEM_SHMEM_C_FILES "shmem/c/*.h" "shmem/c/*.c")
SET (OSHMEM_SOURCE_FILES ${OSHMEM_SOURCE_FILES} ${OSHMEM_SHMEM_C_FILES})
SOURCE_GROUP(shmem"")
SOURCE_GROUP(shmem\\c FILES ${OSHMEM_SHMEM_C_FILES})
IF(OMPI_WANT_F77_BINDINGS)
# A handful of files in mpi/f77/base must be included in libshmem, in order to build the
# Fortran 77 glue into libshmem
FILE(GLOB OMPI_F77_BASE_FILES "mpi/f77/base/*.c")
SET_SOURCE_FILES_PROPERTIES(${OSHMEM_F77_BASE_FILES}
PROPERTIES COMPILE_FLAGS "-DOMPI_COMPILING_F77_WRAPPERS=1 -DOMPI_PROFILE_LAYER=0")
SET(OSHMEM_SOURCE_FILES ${OSHMEM_SOURCE_FILES} ${OMPI_F77_BASE_FILES})
SOURCE_GROUP(mpi\\f77\\base FILES ${OMPI_F77_BASE_FILES})
ENDIF(OMPI_WANT_F77_BINDINGS)
INCLUDE (check_mca_subdirs)
SET (OSHMEM_SOURCE_FILES ${OSHMEM_SOURCE_FILES} ${MCA_FILES})
ADD_LIBRARY (libshmem ${OSHMEM_SOURCE_FILES})
SET_TARGET_PROPERTIES(libshmem PROPERTIES COMPILE_FLAGS
"${OSHMEM_C_DEF_PRE}OMPI_MPIHANDLES_DLL_PREFIX=liboshmem_dbg_mpihandles
${OSHMEM_C_DEF_PRE}OMPI_MSGQ_DLL_PREFIX=libompi_dbg_msgq
${OSHMEM_C_DEF_PRE}OMPI_BUILDING
${OSHMEM_C_DEF_PRE}OMPI_BUILDING_CXX_BINDINGS_LIBRARY")
# Set compile flags for this target
IF (BUILD_SHARED_LIBS)
ADD_DEPENDENCIES (libshmem libopen-pal libopen-rte)
SET_TARGET_PROPERTIES(libshmem PROPERTIES
COMPILE_FLAGS "-D_USRDLL -DHAVE_CONFIG_H -DOSHMEM_EXPORTS /TP")
TARGET_LINK_LIBRARIES (libshmem Ws2_32.lib shlwapi.lib)
ELSE (BUILD_SHARED_LIBS)
SET_TARGET_PROPERTIES(libshmem PROPERTIES COMPILE_FLAGS "-D_LIB")
ENDIF(BUILD_SHARED_LIBS)
# generate shmem.h
CONFIGURE_FILE(${OpenMPI_SOURCE_DIR}/oshmem/include/shmem.h.in ${OpenMPI_BINARY_DIR}/oshmem/include/shmem.h)
# generate oshmem_config.h
CONFIGURE_FILE(${OpenMPI_SOURCE_DIR}/oshmem/include/oshmem_config.h.in ${OpenMPI_BINARY_DIR}/oshmem/include/oshmem_config.h)
# generate shmem_portable_platform.h
CONFIGURE_FILE(${OpenMPI_SOURCE_DIR}/oshmem/include/shmem_portable_platform.h.in ${OpenMPI_BINARY_DIR}/oshmem/include/shmem_portable_platform.h)
# generate version.h
CONFIGURE_FILE(${OpenMPI_SOURCE_DIR}/oshmem/include/oshmem/version.h.in ${OpenMPI_BINARY_DIR}/oshmem/include/oshmem/version.h)
ADD_SUBDIRECTORY(shmem)
# Install libraries headers, and shared files
INSTALL(TARGETS libshmem
RUNTIME DESTINATION bin
LIBRARY DESTINATION lib
ARCHIVE DESTINATION lib)
INSTALL(FILES ${PROJECT_BINARY_DIR}/include/shmem.h ${PROJECT_BINARY_DIR}/include/shmem_portable_platform.h
DESTINATION include)
IF (OMPI_DEBUG_BUILD)
INSTALL(FILES ${OpenMPI_BINARY_DIR}/Debug/libmpi${CMAKE_DEBUG_POSTFIX}.pdb
DESTINATION bin)
ENDIF (OMPI_DEBUG_BUILD)
IF(OMPI_WANT_F77_BINDINGS)
INSTALL(FILES ${PROJECT_BINARY_DIR}/include/mpif.h
${PROJECT_BINARY_DIR}/include/mpif-config.h
${PROJECT_SOURCE_DIR}/include/mpif-common.h
${PROJECT_SOURCE_DIR}/include/mpif-mpi-io.h
DESTINATION include)
ENDIF(OMPI_WANT_F77_BINDINGS)
INSTALL(FILES ${PROJECT_SOURCE_DIR}/runtime/help-shmem-runtime.txt ${PROJECT_SOURCE_DIR}/shmem/help-shmem-api.txt
DESTINATION share/openmpi)
INSTALL(DIRECTORY shmem/cxx/ DESTINATION include/openshmem/oshmem/shmem/cxx
FILES_MATCHING PATTERN "*.h" PATTERN ".svn" EXCLUDE)

130
oshmem/Makefile.am
Просмотреть файл

@ -1,130 +0,0 @@
#
# Copyright (c) 2012 Mellanox Technologies, Inc.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
if OSHMEM_SUPPORT
AM_CFLAGS = $(OSHMEM_CFLAGS)
EXTRA_DIST = CMakeLists.txt
# Do we have profiling?
if OSHMEM_PROFILING
c_pshmem_lib = shmem/c/profile/libshmem_c_pshmem.la
else
c_pshmem_lib =
endif
#Irit
f77_shmem_lib = shmem/f77/libshmem_f77.la
# Note that the ordering of "." in SUBDIRS is important: the C++, F77,
# and F90 bindings are all in standalone .la files that depend on
# libshmem.la. So we must fully build libshmem.la first.
# NOTE: A handful of files in mpi/f77/base must be included in
# libshmem.la. But we wanted to keep all the Fortran sources together
# in the same tree, so we moved those sources to a separate
# subdirectory with its own Makefile.include that is included in this
# Makefile.am (NOTE: it did *not* work to put all the files -- base
# q
# and non-base -- into mpi/f77 and have both a regular Makefile.am for
# building the f77 bindings library and a separate Makefile.include
# that was included in this top-level Makefile.am; problems occurred
# with "make distclean" and files in the ompi/mpi/f77/.deps directory
# -- it's not clear whether this is an AM bug or whether this behavior
# is simply not supported). This ompi/mpi/f77/base/Makefile.include
# file makes a convenience LT library that is then sucked into
# libshmem.la (the ompi/mpi/f77/base sources must be compiled with
# special CPPFLAGS; we can't just add the raw sources to
# libshmem_la_SOURCES, unfortunately).
# The end of the result is that libshmem.la -- including a few sources
# from mpi/f77/base -- is fully built before the C++, F77, and F90
# bindings are built. Therefore, the C++, F77 and F90 bindings
# libraries can all link against libshmem.la.
SUBDIRS = \
include \
shmem/c \
shmem/f77 \
$(EXT_oshmem_FRAMEWORKS_SUBDIRS) \
$(EXT_oshmem_FRAMEWORK_COMPONENT_STATIC_SUBDIRS) \
$(MCA_oshmem_FRAMEWORKS_SUBDIRS) \
$(MCA_oshmem_FRAMEWORK_COMPONENT_STATIC_SUBDIRS) \
. \
$(MCA_oshmem_FRAMEWORK_COMPONENT_DSO_SUBDIRS)
DIST_SUBDIRS = \
include \
shmem/c \
shmem/f77 \
$(EXT_oshmem_FRAMEWORKS_SUBDIRS) \
$(EXT_oshmem_FRAMEWORK_COMPONENT_ALL_SUBDIRS) \
$(MCA_oshmem_FRAMEWORKS_SUBDIRS) \
$(MCA_oshmem_FRAMEWORK_COMPONENT_ALL_SUBDIRS)
#Build The main SHMEM library
lib_LTLIBRARIES = libshmem.la
libshmem_la_SOURCES =
libshmem_la_LIBADD = \
shmem/c/libshmem_c.la \
$(c_pshmem_lib) \
$(f77_shmem_lib) \
$(MCA_oshmem_FRAMEWORK_LIBS) \
$(top_ompi_builddir)/ompi/libmpi.la
libshmem_la_DEPENDENCIES = $(libshmem_la_LIBADD)
libshmem_la_LDFLAGS = \
-version-info $(libshmem_so_version) \
$(OSHMEM_LIBSHMEM_EXTRA_LDFLAGS)
# included subdirectory Makefile.am's and appended-to variables
headers =
noinst_LTLIBRARIES =
include_HEADERS =
nobase_oshmem_HEADERS =
dist_pkgdata_DATA =
libshmem_la_SOURCES += $(headers)
nodist_man_MANS =
# Conditionally install the header files
if WANT_INSTALL_HEADERS
oshmemdir = $(includedir)/oshmem/oshmem
nobase_oshmem_HEADERS += $(headers)
else
oshmemdir = $(includedir)
endif
include op/Makefile.am
include proc/Makefile.am
include request/Makefile.am
include runtime/Makefile.am
include shmem/Makefile.am
include tools/Makefile.am
# Ensure that the man page directory exists before we try to make man
# page files (because oshmem/shmem/man/man3 has no config.status-generated
# Makefile)
dir_stamp = $(top_builddir)/$(subdir)/shmem/man/man3/.dir-stamp
# Also ensure that the man pages are rebuilt if the opal_config.h file
# changes (e.g., configure was run again, meaning that the release
# date or version may have changed)
$(nodist_man_MANS): $(dir_stamp) $(top_builddir)/opal/include/opal_config.h
$(dir_stamp):
$(mkdir_p) `dirname $@`
touch "$@"
# Remove the generated man pages
distclean-local:
rm -f $(nodist_man_MANS) $(dir_stamp)
endif # OSHMEM_SUPPORT

67
oshmem/include/Makefile.am
Просмотреть файл

@ -1,67 +0,0 @@
# Copyright (c) 2012 Mellanox Technologies, Inc.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# mpif-common.h is not generated, but mpif.h and mpif-config.h are.
# See big comments in these files for an explanation.
# note - headers and nodist_headers will go in ${includedir}/openmpi,
# not ${includedir}/
headers =
nodist_headers = \
oshmem_config.h
# Install these in $(includedir)
include_HEADERS =
# Install these in $(includedir)
mppincludedir = $(includedir)/mpp
mppinclude_HEADERS = mpp/shmem.h \
mpp/shmem.fh
# Always install these in $(pkgincludedir)
pkginclude_HEADERS =
include_HEADERS += shmem.fh
# pasha # if OMPI_WANT_F77_BINDINGS
# pasha # include_HEADERS =
# pasha #
# pasha # #include_HEADERS += \
# pasha # # mpif-common.h \
# pasha # # mpif-mpi-io.h
# pasha # endif
# These files are always installed in $(includedir), but shouldn't be
# shipped since they are generated by configure from their .in
# counterparts (which AM automatically ships).
nodist_include_HEADERS = \
shmem.h \
shmem_portable_platform.h
# pasha # if OMPI_WANT_F77_BINDINGS
# pasha # nodist_include_HEADERS =
# pasha #
# pasha # #nodist_include_HEADERS += \
# pasha # # mpif.h \
# pasha # # mpif-config.h
# pasha # endif
if WANT_INSTALL_HEADERS
oshmemdir = $(includedir)/oshmem
nobase_dist_oshmem_HEADERS = $(headers)
nobase_nodist_oshmem_HEADERS = $(nodist_headers)
else
oshmemdir = $(includedir)
nobase_dist_noinst_HEADERS = $(headers)
nobase_nodist_noinst_HEADERS = $(nodist_headers)
endif
distclean-local:
include oshmem/Makefile.am

457
oshmem/include/mpif-common.h
Просмотреть файл

@ -1,457 +0,0 @@
!
! Copyright (c) 2012 Mellanox Technologies, Inc.
! All rights reserved.
! $COPYRIGHT$
!
! Additional copyrights may follow
!
! $HEADER$
!
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
! WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
!
! Do ***not*** copy this file to the directory where your Fortran
! fortran application is compiled unless it is absolutely necessary! Most
! modern Fortran compilers now support the -I command line flag, which
! tells the compiler where to find .h files (specifically, this one). For
! example:
!
! shell$ mpif77 foo.f -o foo -I$OMPI_HOME/include
!
! will probably do the trick (assuming that you have set OMPI_HOME
! properly).
!
! That being said, OMPI's "mpif77" wrapper compiler should
! automatically include the -I option for you. The following command
! should be equivalent to the command listed above:
!
! shell$ mpif77 foo.f -o foo
!
! You should not copy this file to your local directory because it is
! possible that this file will be changed between versions of Open MPI.
! Indeed, this mpif.h is incompatible with the mpif.f of other
! implementations of MPI. Using this mpif.h with other implementations
! of MPI, or with other versions of Open MPI will result in undefined
! behavior (to include incorrect results, segmentation faults,
! unexplainable "hanging" in your application, etc.). Always use the
! -I command line option instead (or let mpif77 do it for you).
!
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
! WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
!
! This file contains the bulk of the Open MPI Fortran interface. It
! is included as a back-end file to both mpif.h (i.e., the
! standardized MPI Fortran header file) and mpi.f90 (the MPI-2
! Fortran module source file, found in ompi/mpi/f90).
!
! This file is marginally different than mpif.h. mpif.h includes
! some "external" statements that are not suitable for use with the
! MPI-2 F90 module, and therefore cannot be included in the mpi.f90
! source file. Hence, this file is essentially everything that
! needs to be in the standardized mpif.h *except* the "external"
! statements, and is therefore suitable to be included in mpi.f90.
!
! First, however, include some output from configure.
!
include 'mpif-config.h'
!
! MPI version
!
integer MPI_VERSION, MPI_SUBVERSION
parameter (MPI_VERSION=2)
parameter (MPI_SUBVERSION=1)
!
! Miscellaneous constants
!
integer MPI_ANY_SOURCE, MPI_ANY_TAG
integer MPI_PROC_NULL
integer MPI_ROOT
integer MPI_UNDEFINED
integer MPI_CART, MPI_GRAPH, MPI_KEYVAL_INVALID
integer MPI_SOURCE, MPI_TAG, MPI_ERROR
integer MPI_TAG_UB, MPI_HOST, MPI_IO, MPI_WTIME_IS_GLOBAL
integer MPI_APPNUM, MPI_LASTUSEDCODE, MPI_UNIVERSE_SIZE
integer IMPI_CLIENT_SIZE, IMPI_CLIENT_COLOR
integer IMPI_HOST_SIZE, IMPI_HOST_COLOR
integer MPI_BSEND_OVERHEAD
integer MPI_ORDER_C, MPI_ORDER_FORTRAN
integer MPI_DISTRIBUTE_BLOCK, MPI_DISTRIBUTE_CYCLIC
integer MPI_DISTRIBUTE_NONE, MPI_DISTRIBUTE_DFLT_DARG
integer MPI_TYPECLASS_INTEGER, MPI_TYPECLASS_REAL
integer MPI_TYPECLASS_COMPLEX
integer MPI_MODE_NOCHECK, MPI_MODE_NOPRECEDE, MPI_MODE_NOPUT
integer MPI_MODE_NOSTORE, MPI_MODE_NOSUCCEED
integer MPI_LOCK_EXCLUSIVE, MPI_LOCK_SHARED
integer MPI_WIN_BASE, MPI_WIN_SIZE, MPI_WIN_DISP_UNIT
parameter (MPI_ANY_SOURCE=-1)
parameter (MPI_ANY_TAG=-1)
parameter (MPI_PROC_NULL=-2)
parameter (MPI_ROOT=-4)
parameter (MPI_UNDEFINED=-32766)
parameter (MPI_CART=1)
parameter (MPI_GRAPH=2)
parameter (MPI_KEYVAL_INVALID=-1)
parameter (MPI_SOURCE=1)
parameter (MPI_TAG=2)
parameter (MPI_ERROR=3)
parameter (MPI_TAG_UB=0)
parameter (MPI_HOST=1)
parameter (MPI_IO=2)
parameter (MPI_WTIME_IS_GLOBAL=3)
parameter (MPI_APPNUM=4)
parameter (MPI_LASTUSEDCODE=5)
parameter (MPI_UNIVERSE_SIZE=6)
parameter (MPI_WIN_BASE=7)
parameter (MPI_WIN_SIZE=8)
parameter (MPI_WIN_DISP_UNIT=9)
parameter (IMPI_CLIENT_SIZE=10)
parameter (IMPI_CLIENT_COLOR=11)
parameter (IMPI_HOST_SIZE=12)
parameter (IMPI_HOST_COLOR=13)
parameter (MPI_BSEND_OVERHEAD=128)
parameter (MPI_ORDER_C=0)
parameter (MPI_ORDER_FORTRAN=1)
parameter (MPI_DISTRIBUTE_BLOCK=0)
parameter (MPI_DISTRIBUTE_CYCLIC=1)
parameter (MPI_DISTRIBUTE_NONE=2)
parameter (MPI_DISTRIBUTE_DFLT_DARG=-1)
parameter (MPI_TYPECLASS_INTEGER=1)
parameter (MPI_TYPECLASS_REAL=2)
parameter (MPI_TYPECLASS_COMPLEX=3)
parameter (MPI_MODE_NOCHECK=1)
parameter (MPI_MODE_NOPRECEDE=2)
parameter (MPI_MODE_NOPUT=4)
parameter (MPI_MODE_NOSTORE=8)
parameter (MPI_MODE_NOSUCCEED=16)
parameter (MPI_LOCK_EXCLUSIVE=1)
parameter (MPI_LOCK_SHARED=2)
!
! MPI sentinel values
!
! Several of these types were chosen with care to match specific
! overloaded functions in the F90 bindings. They should also match
! the types of their corresponding C variables. Do not arbitrarily
! change their types without also updating the F90 bindings and
! their corresponding types in ompi/mpi/f77/constants.h and
! ompi/mpi/runtime/ompi_init.c!
!
! MPI_BOTTOM is only used where choice buffers can be used (meaning
! that we already have overloaded F90 bindings for all available
! types), so any type is fine.
integer MPI_BOTTOM
! MPI_IN_PLACE has the same rationale as MPI_BOTTOM.
integer MPI_IN_PLACE
! Making MPI_ARGV_NULL be the same type as the parameter that is
! exepected in the F90 binding for MPI_COMM_SPAWN means that we
! don't need another binding for MPI_COMM_SPAWN.
character MPI_ARGV_NULL(1)
! The array_of_argv parameter in the F90 bindings for
! MPI_COMM_SPAWN_MULTIPLE takes a variable number of dimensions
! (specified by the "count" parameter), so it's not possible to have
! a single variable match all possible values. Hence, make it an
! entirely different type (one that would never likely be used by a
! correct program, e.g., double) and have a separate F90 binding for
! matching just this type.
double precision MPI_ARGVS_NULL
! MPI_ERRCODES_IGNORE has similar rationale to MPI_ARGV_NULL. The
! F77 functions are all smart enough to check that the errcodes
! parameter is not ERRCODES_IGNORE before assigning values into it
! (hence, the fact that this is an array of only 1 element does not
! matter -- we'll never overrun it because we never assign values
! into it).
integer MPI_ERRCODES_IGNORE(1)
! MPI_STATUS_IGNORE has similar rationale to MPI_ERRCODES_IGNORE.
integer MPI_STATUS_IGNORE(MPI_STATUS_SIZE)
! MPI_STATUSES_IGNORE has similar rationale to MPI_ARGVS_NULL.
double precision MPI_STATUSES_IGNORE
common/mpi_fortran_bottom/MPI_BOTTOM
common/mpi_fortran_in_place/MPI_IN_PLACE
common/mpi_fortran_argv_null/MPI_ARGV_NULL
common/mpi_fortran_argvs_null/MPI_ARGVS_NULL
common/mpi_fortran_errcodes_ignore/MPI_ERRCODES_IGNORE
common/mpi_fortran_status_ignore/MPI_STATUS_IGNORE
common/mpi_fortran_statuses_ignore/MPI_STATUSES_IGNORE
!
! NULL "handles" (indices)
!
integer MPI_GROUP_NULL, MPI_COMM_NULL, MPI_DATATYPE_NULL
integer MPI_REQUEST_NULL, MPI_OP_NULL, MPI_ERRHANDLER_NULL
integer MPI_INFO_NULL, MPI_WIN_NULL
parameter (MPI_GROUP_NULL=0)
parameter (MPI_COMM_NULL=2)
parameter (MPI_DATATYPE_NULL=0)
parameter (MPI_REQUEST_NULL=0)
parameter (MPI_OP_NULL=0)
parameter (MPI_ERRHANDLER_NULL=0)
parameter (MPI_INFO_NULL=0)
parameter (MPI_WIN_NULL=0)
!
! MPI_Init_thread constants
!
integer MPI_THREAD_SINGLE, MPI_THREAD_FUNNELED
integer MPI_THREAD_SERIALIZED, MPI_THREAD_MULTIPLE
parameter (MPI_THREAD_SINGLE=0)
parameter (MPI_THREAD_FUNNELED=1)
parameter (MPI_THREAD_SERIALIZED=2)
parameter (MPI_THREAD_MULTIPLE=3)
!
! error classes
!
integer SHMEM_SUCCESS
integer SHMEM_ERR_BUFFER
integer SHMEM_ERR_COUNT
integer SHMEM_ERR_TYPE
integer SHMEM_ERR_TAG
integer SHMEM_ERR_COMM
integer SHMEM_ERR_RANK
integer SHMEM_ERR_REQUEST
integer SHMEM_ERR_ROOT
integer SHMEM_ERR_GROUP
integer SHMEM_ERR_OP
integer SHMEM_ERR_TOPOLOGY
integer SHMEM_ERR_DIMS
integer SHMEM_ERR_ARG
integer SHMEM_ERR_UNKNOWN
integer SHMEM_ERR_TRUNCATE
integer SHMEM_ERR_OTHER
integer SHMEM_ERR_INTERN
integer SHMEM_ERR_IN_STATUS
integer SHMEM_ERR_PENDING
integer SHMEM_ERR_ACCESS
integer SHMEM_ERR_AMODE
integer SHMEM_ERR_ASSERT
integer SHMEM_ERR_BAD_FILE
integer SHMEM_ERR_BASE
integer SHMEM_ERR_CONVERSION
integer SHMEM_ERR_DISP
integer SHMEM_ERR_DUP_DATAREP
integer SHMEM_ERR_FILE_EXISTS
integer SHMEM_ERR_FILE_IN_USE
integer SHMEM_ERR_FILE
integer SHMEM_ERR_INFO_KEY
integer SHMEM_ERR_INFO_NOKEY
integer SHMEM_ERR_INFO_VALUE
integer SHMEM_ERR_INFO
integer SHMEM_ERR_IO
integer SHMEM_ERR_KEYVAL
integer SHMEM_ERR_LOCKTYPE
integer SHMEM_ERR_NAME
integer SHMEM_ERR_NO_MEM
integer SHMEM_ERR_NOT_SAME
integer SHMEM_ERR_NO_SPACE
integer SHMEM_ERR_NO_SUCH_FILE
integer SHMEM_ERR_PORT
integer SHMEM_ERR_QUOTA
integer SHMEM_ERR_READ_ONLY
integer SHMEM_ERR_RMA_CONFLICT
integer SHMEM_ERR_RMA_SYNC
integer SHMEM_ERR_SERVICE
integer SHMEM_ERR_SIZE
integer SHMEM_ERR_SPAWN
integer SHMEM_ERR_UNSUPPORTED_DATAREP
integer SHMEM_ERR_UNSUPPORTED_OPERATION
integer SHMEM_ERR_WIN
integer SHMEM_ERR_SYSRESOURCE
integer SHMEM_ERR_LASTCODE
parameter( SHMEM_SUCCESS = 0)
parameter( SHMEM_ERR_BUFFER = 1)
parameter( SHMEM_ERR_COUNT = 2)
parameter( SHMEM_ERR_TYPE = 3)
parameter( SHMEM_ERR_TAG = 4)
parameter( SHMEM_ERR_COMM = 5)
parameter( SHMEM_ERR_RANK = 6)
parameter( SHMEM_ERR_REQUEST = 7)
parameter( SHMEM_ERR_ROOT = 8)
parameter( SHMEM_ERR_GROUP = 9)
parameter( SHMEM_ERR_OP = 10)
parameter( SHMEM_ERR_TOPOLOGY = 11)
parameter( SHMEM_ERR_DIMS = 12)
parameter( SHMEM_ERR_ARG = 13)
parameter( SHMEM_ERR_UNKNOWN = 14)
parameter( SHMEM_ERR_TRUNCATE = 15)
parameter( SHMEM_ERR_OTHER = 16)
parameter( SHMEM_ERR_INTERN = 17)
parameter( SHMEM_ERR_IN_STATUS = 18)
parameter( SHMEM_ERR_PENDING = 19)
parameter( SHMEM_ERR_ACCESS = 20)
parameter( SHMEM_ERR_AMODE = 21)
parameter( SHMEM_ERR_ASSERT = 22)
parameter( SHMEM_ERR_BAD_FILE = 23)
parameter( SHMEM_ERR_BASE = 24)
parameter( SHMEM_ERR_CONVERSION = 25)
parameter( SHMEM_ERR_DISP = 26)
parameter( SHMEM_ERR_DUP_DATAREP = 27)
parameter( SHMEM_ERR_FILE_EXISTS = 28)
parameter( SHMEM_ERR_FILE_IN_USE = 29)
parameter( SHMEM_ERR_FILE = 30)
parameter( SHMEM_ERR_INFO_KEY = 31)
parameter( SHMEM_ERR_INFO_NOKEY = 32)
parameter( SHMEM_ERR_INFO_VALUE = 33)
parameter( SHMEM_ERR_INFO = 34)
parameter( SHMEM_ERR_IO = 35)
parameter( SHMEM_ERR_KEYVAL = 36)
parameter( SHMEM_ERR_LOCKTYPE = 37)
parameter( SHMEM_ERR_NAME = 38)
parameter( SHMEM_ERR_NO_MEM = 39)
parameter( SHMEM_ERR_NOT_SAME = 40)
parameter( SHMEM_ERR_NO_SPACE = 41)
parameter( SHMEM_ERR_NO_SUCH_FILE = 42)
parameter( SHMEM_ERR_PORT = 43)
parameter( SHMEM_ERR_QUOTA = 44)
parameter( SHMEM_ERR_READ_ONLY = 45)
parameter( SHMEM_ERR_RMA_CONFLICT = 46)
parameter( SHMEM_ERR_RMA_SYNC = 47)
parameter( SHMEM_ERR_SERVICE = 48)
parameter( SHMEM_ERR_SIZE = 49)
parameter( SHMEM_ERR_SPAWN = 50)
parameter( SHMEM_ERR_UNSUPPORTED_DATAREP = 51)
parameter( SHMEM_ERR_UNSUPPORTED_OPERATION= 52)
parameter( SHMEM_ERR_WIN = 53)
parameter( SHMEM_ERR_SYSRESOURCE = -2)
parameter( SHMEM_ERR_LASTCODE = 54)
!
! comparison results
!
integer MPI_IDENT, MPI_CONGRUENT, MPI_SIMILAR, MPI_UNEQUAL
parameter (MPI_IDENT=0)
parameter (MPI_CONGRUENT=1)
parameter (MPI_SIMILAR=2)
parameter (MPI_UNEQUAL=3)
!
! datatype combiners
!
integer MPI_COMBINER_NAMED
integer MPI_COMBINER_DUP
integer MPI_COMBINER_CONTIGUOUS
integer MPI_COMBINER_VECTOR
integer MPI_COMBINER_HVECTOR_INTEGER
integer MPI_COMBINER_HVECTOR
integer MPI_COMBINER_INDEXED
integer MPI_COMBINER_HINDEXED_INTEGER
integer MPI_COMBINER_HINDEXED
integer MPI_COMBINER_INDEXED_BLOCK
integer MPI_COMBINER_STRUCT_INTEGER
integer MPI_COMBINER_STRUCT
integer MPI_COMBINER_SUBARRAY
integer MPI_COMBINER_DARRAY
integer MPI_COMBINER_F90_REAL
integer MPI_COMBINER_F90_COMPLEX
integer MPI_COMBINER_F90_INTEGER
integer MPI_COMBINER_RESIZED
parameter (MPI_COMBINER_NAMED=0)
parameter (MPI_COMBINER_DUP=1)
parameter (MPI_COMBINER_CONTIGUOUS=2)
parameter (MPI_COMBINER_VECTOR=3)
parameter (MPI_COMBINER_HVECTOR_INTEGER=4)
parameter (MPI_COMBINER_HVECTOR=5)
parameter (MPI_COMBINER_INDEXED=6)
parameter (MPI_COMBINER_HINDEXED_INTEGER=7)
parameter (MPI_COMBINER_HINDEXED=8)
parameter (MPI_COMBINER_INDEXED_BLOCK=9)
parameter (MPI_COMBINER_STRUCT_INTEGER=10)
parameter (MPI_COMBINER_STRUCT=11)
parameter (MPI_COMBINER_SUBARRAY=12)
parameter (MPI_COMBINER_DARRAY=13)
parameter (MPI_COMBINER_F90_REAL=14)
parameter (MPI_COMBINER_F90_COMPLEX=15)
parameter (MPI_COMBINER_F90_INTEGER=16)
parameter (MPI_COMBINER_RESIZED=17)
!
! lookup table indices
!
integer MPI_COMM_WORLD, MPI_COMM_SELF
integer MPI_GROUP_EMPTY
integer MPI_ERRORS_ARE_FATAL, MPI_ERRORS_RETURN
parameter (MPI_COMM_WORLD=0)
parameter (MPI_COMM_SELF=1)
parameter (MPI_GROUP_EMPTY=1)
parameter (MPI_ERRORS_ARE_FATAL=1)
parameter (MPI_ERRORS_RETURN=2)
integer MPI_BYTE, MPI_PACKED, MPI_UB, MPI_LB
integer MPI_CHARACTER, MPI_LOGICAL
integer MPI_INTEGER, MPI_INTEGER1, MPI_INTEGER2, MPI_INTEGER4
integer MPI_INTEGER8, MPI_INTEGER16
integer MPI_REAL, MPI_REAL2, MPI_REAL4, MPI_REAL8, MPI_REAL16
integer MPI_DOUBLE_PRECISION
integer MPI_COMPLEX, MPI_COMPLEX8, MPI_COMPLEX16, MPI_COMPLEX32
integer MPI_DOUBLE_COMPLEX
integer MPI_2REAL, MPI_2DOUBLE_PRECISION, MPI_2INTEGER
integer MPI_2COMPLEX, MPI_2DOUBLE_COMPLEX
! Note that MPI_LOGICALx are not defined by the MPI spec, but there are
! other MPI implementations that have them, so it's good for us to have
! as well.
integer MPI_LOGICAL1, MPI_LOGICAL2, MPI_LOGICAL4, MPI_LOGICAL8
!
! Do NOT change the order of these parameters
!
parameter (MPI_BYTE=1)
parameter (MPI_PACKED=2)
parameter (MPI_UB=3)
parameter (MPI_LB=4)
parameter (MPI_CHARACTER=5)
parameter (MPI_LOGICAL=6)
parameter (MPI_INTEGER=7)
parameter (MPI_INTEGER1=8)
parameter (MPI_INTEGER2=9)
parameter (MPI_INTEGER4=10)
parameter (MPI_INTEGER8=11)
parameter (MPI_INTEGER16=12)
parameter (MPI_REAL=13)
parameter (MPI_REAL4=14)
parameter (MPI_REAL8=15)
parameter (MPI_REAL16=16)
parameter (MPI_DOUBLE_PRECISION=17)
parameter (MPI_COMPLEX=18)
parameter (MPI_COMPLEX8=19)
parameter (MPI_COMPLEX16=20)
parameter (MPI_COMPLEX32=21)
parameter (MPI_DOUBLE_COMPLEX=22)
parameter (MPI_2REAL=23)
parameter (MPI_2DOUBLE_PRECISION=24)
parameter (MPI_2INTEGER=25)
parameter (MPI_2COMPLEX=26)
parameter (MPI_2DOUBLE_COMPLEX=27)
parameter (MPI_REAL2=28)
parameter (MPI_LOGICAL1=29)
parameter (MPI_LOGICAL2=30)
parameter (MPI_LOGICAL4=31)
parameter (MPI_LOGICAL8=32)
integer MPI_MAX, MPI_MIN, MPI_SUM, MPI_PROD, MPI_LAND
integer MPI_BAND, MPI_LOR, MPI_BOR, MPI_LXOR, MPI_BXOR
integer MPI_MAXLOC, MPI_MINLOC, MPI_REPLACE
parameter (MPI_MAX=1)
parameter (MPI_MIN=2)
parameter (MPI_SUM=3)
parameter (MPI_PROD=4)
parameter (MPI_LAND=5)
parameter (MPI_BAND=6)
parameter (MPI_LOR=7)
parameter (MPI_BOR=8)
parameter (MPI_LXOR=9)
parameter (MPI_BXOR=10)
parameter (MPI_MAXLOC=11)
parameter (MPI_MINLOC=12)
parameter (MPI_REPLACE=13)

99
oshmem/include/mpif-config.h.in
Просмотреть файл

@ -1,99 +0,0 @@
! -*- fortran -*-
!
! Copyright (c) 2012 Mellanox Technologies, Inc.
! All rights reserved.
! $COPYRIGHT$
!
! Additional copyrights may follow
!
! $HEADER$
!
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
! WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
!
! Do ***not*** copy this file to the directory where your Fortran
! fortran application is compiled unless it is absolutely necessary! Most
! modern Fortran compilers now support the -I command line flag, which
! tells the compiler where to find .h files (specifically, this one). For
! example:
!
! shell$ mpif77 foo.f -o foo -I$OMPI_HOME/include
!
! will probably do the trick (assuming that you have set OMPI_HOME
! properly).
!
! That being said, OMPI's "mpif77" wrapper compiler should
! automatically include the -I option for you. The following command
! should be equivalent to the command listed above:
!
! shell$ mpif77 foo.f -o foo
!
! You should not copy this file to your local directory because it is
! possible that this file will be changed between versions of Open MPI.
! Indeed, this mpif.h is incompatible with the mpif.f of other
! implementations of MPI. Using this mpif.h with other implementations
! of MPI, or with other versions of Open MPI will result in undefined
! behavior (to include incorrect results, segmentation faults,
! unexplainable "hanging" in your application, etc.). Always use the
! -I command line option instead (or let mpif77 do it for you).
!
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
! WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
!
! This file is included as a back-end file to both mpif.h (i.e., the
! standardized MPI Fortran header file) and a bunch of the MPI
! Fortran 90 subroutine implementations found in ompi/mpi/f90.
!
! This file contains the output from configure that is relevant for
! Fortran applications (both 77 and 90) and a few values that are
! necessary to compile the F90 module (e.g., MPI_STATUS_SIZE).
!
! Include the MPI I/O stuff, if needed
@OMPI_MPIF_MPI_IO_INCLUDE@
!
! OMPI version
! This file is generated from configure; do not edit it manually.
!
integer OMPI_MAJOR_VERSION, OMPI_MINOR_VERSION
integer OMPI_RELEASE_VERSION
character*32 OMPI_GREEK_VERSION
character*32 OMPI_SVN_VERSION
parameter (OMPI_MAJOR_VERSION=@OMPI_MAJOR_VERSION@)
parameter (OMPI_MINOR_VERSION=@OMPI_MINOR_VERSION@)
parameter (OMPI_RELEASE_VERSION=@OMPI_RELEASE_VERSION@)
parameter (OMPI_GREEK_VERSION="@OMPI_GREEK_VERSION@")
parameter (OMPI_SVN_VERSION="@OMPI_SVN_R@")
!
! Kind parameters
!
integer MPI_OFFSET_KIND, MPI_ADDRESS_KIND, MPI_INTEGER_KIND
parameter (MPI_INTEGER_KIND=@OMPI_MPI_INTEGER_KIND@)
parameter (MPI_ADDRESS_KIND=@OMPI_MPI_ADDRESS_KIND@)
parameter (MPI_OFFSET_KIND=@OMPI_MPI_OFFSET_KIND@)
!
! Miscellaneous constants
!
integer MPI_STATUS_SIZE
parameter (MPI_STATUS_SIZE=5)
!
! Configurable length constants
!
integer MPI_MAX_PROCESSOR_NAME
integer MPI_MAX_ERROR_STRING
integer MPI_MAX_OBJECT_NAME
integer MPI_MAX_INFO_KEY
integer MPI_MAX_INFO_VAL
integer MPI_MAX_PORT_NAME
integer MPI_MAX_DATAREP_STRING
parameter (MPI_MAX_PROCESSOR_NAME=@OPAL_MAX_PROCESSOR_NAME@-1)
parameter (MPI_MAX_ERROR_STRING=@OPAL_MAX_ERROR_STRING@-1)
parameter (MPI_MAX_OBJECT_NAME=@OPAL_MAX_OBJECT_NAME@-1)
parameter (MPI_MAX_INFO_KEY=@OPAL_MAX_INFO_KEY@-1)
parameter (MPI_MAX_INFO_VAL=@OPAL_MAX_INFO_VAL@-1)
parameter (MPI_MAX_PORT_NAME=@OPAL_MAX_PORT_NAME@-1)
parameter (MPI_MAX_DATAREP_STRING=@OPAL_MAX_DATAREP_STRING@-1)

74
oshmem/include/mpif-mpi-io.h
Просмотреть файл

@ -1,74 +0,0 @@
!Copyright (c) 2012 Mellanox Technologies, Inc.
! All rights reserved.
! $COPYRIGHT$
!
! Additional copyrights may follow
!
! $HEADER$
!
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
! WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
!
! Do ***not*** copy this file to the directory where your Fortran
! fortran application is compiled unless it is absolutely necessary! Most
! modern Fortran compilers now support the -I command line flag, which
! tells the compiler where to find .h files (specifically, this one). For
! example:
!
! shell$ mpif77 foo.f -o foo -I$OMPI_HOME/include
!
! will probably do the trick (assuming that you have set OMPI_HOME
! properly).
!
! That being said, OMPI's "mpif77" wrapper compiler should
! automatically include the -I option for you. The following command
! should be equivalent to the command listed above:
!
! shell$ mpif77 foo.f -o foo
!
! You should not copy this file to your local directory because it is
! possible that this file will be changed between versions of Open MPI.
! Indeed, this mpif.h is incompatible with the mpif.f of other
! implementations of MPI. Using this mpif.h with other implementations
! of MPI, or with other versions of Open MPI will result in undefined
! behavior (to include incorrect results, segmentation faults,
! unexplainable "hanging" in your application, etc.). Always use the
! -I command line option instead (or let mpif77 do it for you).
!
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
! WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
!
! This file is included as a back-end file to both mpif.h (i.e., the
! standardized MPI Fortran header file) and a bunch of the MPI
! Fortran 90 subroutine implementations found in ompi/mpi/f90.
!
! This file contains the output from configure that is relevant for
! Fortran applications (both 77 and 90) and a few values that are
! necessary to compile the F90 module (e.g., MPI_STATUS_SIZE).
!
integer MPI_FILE_NULL
integer MPI_SEEK_SET, MPI_SEEK_CUR, MPI_SEEK_END
integer MPI_MODE_CREATE
integer MPI_MODE_RDONLY, MPI_MODE_WRONLY, MPI_MODE_RDWR
integer MPI_MODE_DELETE_ON_CLOSE, MPI_MODE_UNIQUE_OPEN
integer MPI_MODE_EXCL, MPI_MODE_APPEND, MPI_MODE_SEQUENTIAL
integer MPI_DISPLACEMENT_CURRENT
parameter (MPI_FILE_NULL=0)
parameter (MPI_SEEK_SET=600)
parameter (MPI_SEEK_CUR=602)
parameter (MPI_SEEK_END=604)
parameter (MPI_MODE_CREATE=1)
parameter (MPI_MODE_RDONLY=2)
parameter (MPI_MODE_WRONLY=4)
parameter (MPI_MODE_RDWR=8)
parameter (MPI_MODE_DELETE_ON_CLOSE=16)
parameter (MPI_MODE_UNIQUE_OPEN=32)
parameter (MPI_MODE_EXCL=64)
parameter (MPI_MODE_APPEND=128)
parameter (MPI_MODE_SEQUENTIAL=256)
parameter (MPI_DISPLACEMENT_CURRENT=-54278278)

76
oshmem/include/mpif.h.in
Просмотреть файл

@ -1,76 +0,0 @@
! -*- fortran -*-
!
! Copyright (c) 2012 Mellanox Technologies, Inc.
! All rights reserved.
! $COPYRIGHT$
!
! Additional copyrights may follow
!
! $HEADER$
!
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
! WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
!
! Do ***not*** copy this file to the directory where your Fortran
! fortran application is compiled unless it is absolutely necessary! Most
! modern Fortran compilers now support the -I command line flag, which
! tells the compiler where to find .h files (specifically, this one). For
! example:
!
! shell$ mpif77 foo.f -o foo -I$OMPI_HOME/include
!
! will probably do the trick (assuming that you have set OMPI_HOME
! properly).
!
! That being said, OMPI's "mpif77" wrapper compiler should
! automatically include the -I option for you. The following command
! should be equivalent to the command listed above:
!
! shell$ mpif77 foo.f -o foo
!
! You should not copy this file to your local directory because it is
! possible that this file will be changed between versions of Open MPI.
! Indeed, this mpif.h is incompatible with the mpif.f of other
! implementations of MPI. Using this mpif.h with other implementations
! of MPI, or with other versions of Open MPI will result in undefined
! behavior (to include incorrect results, segmentation faults,
! unexplainable "hanging" in your application, etc.). Always use the
! -I command line option instead (or let mpif77 do it for you).
!
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
! WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
!
! Include the back-end file that has the bulk of the MPI Fortran
! interface.
!
include 'mpif-common.h'
!
! These "external" statements are specific to the MPI F77 interface
! (and are toxic to the MPI F90 interface), and are therefore in the
! MPI F77-specific header file (i.e., this one).
!
external MPI_NULL_COPY_FN, MPI_NULL_DELETE_FN
external MPI_COMM_NULL_COPY_FN, MPI_COMM_NULL_DELETE_FN
external MPI_TYPE_NULL_COPY_FN, MPI_TYPE_NULL_DELETE_FN
external MPI_DUP_FN, MPI_COMM_DUP_FN, MPI_TYPE_DUP_FN
external MPI_WIN_NULL_COPY_FN
external MPI_WIN_NULL_DELETE_FN
external MPI_WIN_DUP_FN
! Note that MPI_CONVERSION_FN_NULL is a "constant" (it is only ever
! checked for comparison; it is never invoked), but it is passed as
! a function pointer (to MPI_REGISTER_DATAREP) and therefore must be
! the same size/type. It is therefore external'ed here, and not
! defined with an integer value in mpif-common.h.
external MPI_CONVERSION_FN_NULL
!
! double precision functions
!
external MPI_WTIME, MPI_WTICK @MPIF_H_PMPI_W_FUNCS@
double precision MPI_WTIME, MPI_WTICK @MPIF_H_PMPI_W_FUNCS@

11
oshmem/include/mpp/shmem.fh
Просмотреть файл

@ -1,11 +0,0 @@
!
! Copyright (c) 2012 Mellanox Technologies, Inc.
! All rights reserved.
! $COPYRIGHT$
!
! Additional copyrights may follow
!
! $HEADER$
!
include 'shmem.fh'

16
oshmem/include/mpp/shmem.h
Просмотреть файл

@ -1,16 +0,0 @@
/*
* Copyright (c) 2012 Mellanox Technologies, Inc.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef __MPP_SHMEM_H__
#define __MPP_SHMEM_H__
#include <shmem.h>
#endif

15
oshmem/include/oshmem/Makefile.am
Просмотреть файл

@ -1,15 +0,0 @@
# Copyright (c) 2012 Mellanox Technologies, Inc.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
headers += \
oshmem/constants.h \
oshmem/types.h
nodist_headers += \
oshmem/version.h

127
oshmem/include/oshmem/constants.h
Просмотреть файл

@ -1,127 +0,0 @@
/*
* Copyright (c) 2012 Mellanox Technologies, Inc.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef OSHMEM_CONSTANTS_H
#define OSHMEM_CONSTANTS_H
#include "orte/constants.h"
#if defined(OSHMEM_PROFILING) && (OSHMEM_PROFILING == 1)
#include "oshmem/shmem/c/profile/defines.h"
#endif
#include "oshmem/include/shmem.h"
#define OSHMEM_ERR_BASE ORTE_ERR_MAX
/* error codes */
enum {
/* Error codes inherited from ORTE/OPAL. Still enum values so
that we might get nice debugger help */
OSHMEM_SUCCESS = ORTE_SUCCESS,
OSHMEM_ERROR = ORTE_ERROR,
OSHMEM_ERR_OUT_OF_RESOURCE = ORTE_ERR_OUT_OF_RESOURCE,
OSHMEM_ERR_TEMP_OUT_OF_RESOURCE = ORTE_ERR_TEMP_OUT_OF_RESOURCE,
OSHMEM_ERR_RESOURCE_BUSY = ORTE_ERR_RESOURCE_BUSY,
OSHMEM_ERR_BAD_PARAM = ORTE_ERR_BAD_PARAM,
OSHMEM_ERR_FATAL = ORTE_ERR_FATAL,
OSHMEM_ERR_NOT_IMPLEMENTED = ORTE_ERR_NOT_IMPLEMENTED,
OSHMEM_ERR_NOT_SUPPORTED = ORTE_ERR_NOT_SUPPORTED,
OSHMEM_ERR_INTERUPTED = ORTE_ERR_INTERUPTED,
OSHMEM_ERR_WOULD_BLOCK = ORTE_ERR_WOULD_BLOCK,
OSHMEM_ERR_IN_ERRNO = ORTE_ERR_IN_ERRNO,
OSHMEM_ERR_UNREACH = ORTE_ERR_UNREACH,
OSHMEM_ERR_NOT_FOUND = ORTE_ERR_NOT_FOUND,
OSHMEM_EXISTS = ORTE_EXISTS, /* indicates that the specified object already exists */
OSHMEM_ERR_TIMEOUT = ORTE_ERR_TIMEOUT,
OSHMEM_ERR_NOT_AVAILABLE = ORTE_ERR_NOT_AVAILABLE,
OSHMEM_ERR_PERM = ORTE_ERR_PERM,
OSHMEM_ERR_VALUE_OUT_OF_BOUNDS = ORTE_ERR_VALUE_OUT_OF_BOUNDS,
OSHMEM_ERR_FILE_READ_FAILURE = ORTE_ERR_FILE_READ_FAILURE,
OSHMEM_ERR_FILE_WRITE_FAILURE = ORTE_ERR_FILE_WRITE_FAILURE,
OSHMEM_ERR_FILE_OPEN_FAILURE = ORTE_ERR_FILE_OPEN_FAILURE,
OSHMEM_ERR_RECV_LESS_THAN_POSTED = ORTE_ERR_RECV_LESS_THAN_POSTED,
OSHMEM_ERR_RECV_MORE_THAN_POSTED = ORTE_ERR_RECV_MORE_THAN_POSTED,
OSHMEM_ERR_NO_MATCH_YET = ORTE_ERR_NO_MATCH_YET,
OSHMEM_ERR_BUFFER = ORTE_ERR_BUFFER,
OSHMEM_ERR_REQUEST = ORTE_ERR_REQUEST,
OSHMEM_ERR_NO_CONNECTION_ALLOWED = ORTE_ERR_NO_CONNECTION_ALLOWED,
OSHMEM_ERR_CONNECTION_REFUSED = ORTE_ERR_CONNECTION_REFUSED ,
OSHMEM_ERR_CONNECTION_FAILED = ORTE_ERR_CONNECTION_FAILED,
OSHMEM_PACK_MISMATCH = ORTE_ERR_PACK_MISMATCH,
OSHMEM_ERR_PACK_FAILURE = ORTE_ERR_PACK_FAILURE,
OSHMEM_ERR_UNPACK_FAILURE = ORTE_ERR_UNPACK_FAILURE,
OSHMEM_ERR_COMM_FAILURE = ORTE_ERR_COMM_FAILURE,
OSHMEM_UNPACK_INADEQUATE_SPACE = ORTE_ERR_UNPACK_INADEQUATE_SPACE,
OSHMEM_UNPACK_READ_PAST_END_OF_BUFFER = ORTE_ERR_UNPACK_READ_PAST_END_OF_BUFFER,
OSHMEM_ERR_TYPE_MISMATCH = ORTE_ERR_TYPE_MISMATCH,
OSHMEM_ERR_COMPARE_FAILURE = ORTE_ERR_COMPARE_FAILURE,
OSHMEM_ERR_COPY_FAILURE = ORTE_ERR_COPY_FAILURE,
OSHMEM_ERR_UNKNOWN_DATA_TYPE = ORTE_ERR_UNKNOWN_DATA_TYPE,
OSHMEM_ERR_DATA_TYPE_REDEF = ORTE_ERR_DATA_TYPE_REDEF,
OSHMEM_ERR_DATA_OVERWRITE_ATTEMPT = ORTE_ERR_DATA_OVERWRITE_ATTEMPT
};
#define OSHMEM_ERR_MAX (OSHMEM_ERR_BASE - 1)
/* C datatypes */
/*
* SHMEM_Init_thread constants
* Do not change the order of these without also modifying mpif.h.in.
*/
enum {
SHMEM_NULL = 0,
SHMEM_CHAR,
SHMEM_UCHAR,
SHMEM_SHORT,
SHMEM_USHORT,
SHMEM_INT,
SHMEM_UINT,
SHMEM_LONG,
SHMEM_ULONG,
SHMEM_LLONG,
SHMEM_ULLONG,
SHMEM_FLOAT,
SHMEM_DOUBLE,
SHMEM_LDOUBLE,
SHMEM_FINT,
SHMEM_FINT4,
SHMEM_FINT8
};
/*
* Miscellaneous constants
*/
#define SHMEM_ANY_SOURCE -1 /* match any source rank */
#define SHMEM_PROC_NULL -2 /* rank of null process */
#define SHMEM_UNDEFINED -32766 /* undefined stuff */
#ifndef UNREFERENCED_PARAMETER
#define UNREFERENCED_PARAMETER(P) ((void)P)
#endif
#define OSHMEM_PREDEFINED_GLOBAL(type, global) ((type) ((void *) &(global)))
#if OMPI_WANT_MEMCHECKER
#define MEMCHECKER(x) do { \
x; \
} while(0)
#else
#define MEMCHECKER(x)
#endif /* OMPI_WANT_MEMCHECKER */
#endif /* OSHMEM_CONSTANTS_H */

23
oshmem/include/oshmem/types.h
Просмотреть файл

@ -1,23 +0,0 @@
/*
* Copyright (c) 2012 Mellanox Technologies, Inc.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef SHMEM_TYPES_H
#define SHMEM_TYPES_H
/*
* Predefine some internal types so we dont need all the include
* dependencies.
*/
struct oshmem_proc_t;
struct oshmem_group_t;
struct oshmem_op_t;
#endif

30
oshmem/include/oshmem/version.h.in
Просмотреть файл

@ -1,30 +0,0 @@
/*
* Copyright (c) 2012 Mellanox Technologies, Inc.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*
* This file should be included by any file that needs full
* version information for the OSHMEM project
*/
#ifndef OSHMEM_VERSIONS_H
#define OSHMEM_VERSIONS_H
#define OSHMEM_MAJOR_VERSION @OSHMEM_MAJOR_VERSION@
#define OSHMEM_MINOR_VERSION @OSHMEM_MINOR_VERSION@
#define OSHMEM_RELEASE_VERSION @OSHMEM_RELEASE_VERSION@
#define OSHMEM_GREEK_VERSION "@OSHMEM_GREEK_VERSION@"
#define OSHMEM_WANT_REPO_REV @OSHMEM_WANT_REPO_REV@
#define OSHMEM_REPO_REV "@OSHMEM_REPO_REV@"
#ifdef OSHMEM_VERSION
/* If we included version.h, we want the real version, not the
stripped (no-r number) version */
#undef OSHMEM_VERSION
#endif
#define OSHMEM_VERSION "@OSHMEM_VERSION@"
#endif

125
oshmem/include/oshmem_config.h.in
Просмотреть файл

@ -1,125 +0,0 @@
/* -*- c -*-
*
* Copyright (c) 2012 Mellanox Technologies, Inc.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*
* Function: - OS, CPU and compiler dependent configuration
*/
#ifndef OSHMEM_CONFIG_H
#define OSHMEM_CONFIG_H
#include "opal_config.h"
#define OSHMEM_IDENT_STRING OPAL_IDENT_STRING
/***********************************************************************
*
* OMPI-specific Fortran code that should be in ompi_config.h, but not
* in the other projects.
*
**********************************************************************/
/* MPI_Fint is the same as ompi_fortran_INTEGER_t */
#define MPI_Fint ompi_fortran_integer_t
#if OMPI_HAVE_FORTRAN_REAL && OMPI_HAVE_FORTRAN_COMPLEX
/* * C type for Fortran COMPLEX */
/*typedef struct {
ompi_fortran_real_t real;
ompi_fortran_real_t imag;
} ompi_fortran_complex_t;*/
#endif
#if OMPI_HAVE_FORTRAN_REAL4 && OMPI_HAVE_FORTRAN_COMPLEX8
/* * C type for Fortran COMPLEX*8 */
/*typedef struct {
ompi_fortran_real4_t real;
ompi_fortran_real4_t imag;
} ompi_fortran_complex8_t;*/
#endif
#if OMPI_HAVE_FORTRAN_REAL8 && OMPI_HAVE_FORTRAN_COMPLEX16
/* * C type for Fortran COMPLEX*16 */
/*typedef struct {
ompi_fortran_real8_t real;
ompi_fortran_real8_t imag;
} ompi_fortran_complex16_t;*/
#endif
#if OMPI_HAVE_FORTRAN_REAL16 && OMPI_HAVE_FORTRAN_COMPLEX32
/* * C type for Fortran COMPLEX*32 */
/*typedef struct {
ompi_fortran_real16_t real;
ompi_fortran_real16_t imag;
} ompi_fortran_complex32_t;*/
#endif
#if OMPI_HAVE_FORTRAN_DOUBLE_PRECISION
/* * C type for Fortran DOUBLE COMPLEX */
/*typedef struct {
ompi_fortran_double_precision_t real;
ompi_fortran_double_precision_t imag;
} ompi_fortran_double_complex_t;*/
#endif
#if OPAL_HAVE_ATTRIBUTE_DESTRUCTOR
# define __opal_attribute_destructor__ __attribute__((__destructor__))
#else
# define __opal_attribute_destructor__
#endif
#if defined(__WINDOWS__)
# if defined(_USRDLL) /* building shared libraries (.DLL) */
# if defined(OSHMEM_EXPORTS)
# define OSHMEM_DECLSPEC __declspec(dllexport)
# define OSHMEM_MODULE_DECLSPEC
# else
# define OSHMEM_DECLSPEC __declspec(dllimport)
# if defined(OSHMEM_MODULE_EXPORTS)
# define OSHMEM_MODULE_DECLSPEC __declspec(dllexport)
# else
# define OSHMEM_MODULE_DECLSPEC __declspec(dllimport)
# endif /* defined(OSHMEM_MODULE_EXPORTS) */
# endif /* defined(OSHMEM_EXPORTS) */
# else /* building static library */
# if defined(OSHMEM_IMPORTS)
# define OSHMEM_DECLSPEC __declspec(dllimport)
# else
# define OSHMEM_DECLSPEC
# endif /* defined(OSHMEM_IMPORTS) */
# define OSHMEM_MODULE_DECLSPEC
# endif /* defined(_USRDLL) */
#else
# if OPAL_C_HAVE_VISIBILITY
# ifndef OSHMEM_DECLSPEC
# define OSHMEM_DECLSPEC __opal_attribute_visibility__("default")
# endif
# ifndef OSHMEM_MODULE_DECLSPEC
# define OSHMEM_MODULE_DECLSPEC __opal_attribute_visibility__("default")
# endif
# ifndef OSHMEM_DESTRUCTOR
# define OSHMEM_DESTRUCTOR __opal_attribute_destructor__
# endif
# else
# ifndef OSHMEM_DECLSPEC
# define OSHMEM_DECLSPEC
# endif
# ifndef OSHMEM_MODULE_DECLSPEC
# define OSHMEM_MODULE_DECLSPEC
# endif
# ifndef OSHMEM_DESTRUCTOR
# define OSHMEM_DESTRUCTOR
# endif
# endif
#endif /* defined(__WINDOWS__) */
#endif

55
oshmem/include/shmem.fh
Просмотреть файл

@ -1,55 +0,0 @@
! Emacs: -*- mode: fortran; -*-
!
! Copyright (c) 2012 Mellanox Technologies, Inc.
! All rights reserved.
! $COPYRIGHT$
!
! Additional copyrights may follow
!
! $HEADER$
!
!
! TODO: exact values should be found during configuration
!
integer SHMEM_BARRIER_SYNC_SIZE
parameter ( SHMEM_BARRIER_SYNC_SIZE = 4 )
integer SHMEM_BCAST_SYNC_SIZE
parameter ( SHMEM_BCAST_SYNC_SIZE = 8 )
integer SHMEM_COLLECT_SYNC_SIZE
parameter ( SHMEM_COLLECT_SYNC_SIZE = 8 )
integer SHMEM_REDUCE_SYNC_SIZE
parameter ( SHMEM_REDUCE_SYNC_SIZE = 8 )
integer SHMEM_SYNC_VALUE
parameter ( SHMEM_SYNC_VALUE = -1 )
integer SHMEM_REDUCE_MIN_WRKDATA_SIZE
parameter ( SHMEM_REDUCE_MIN_WRKDATA_SIZE = 8 )
!
! waits
!
integer SHMEM_CMP_EQ
parameter ( SHMEM_CMP_EQ = 0 )
integer SHMEM_CMP_NE
parameter ( SHMEM_CMP_NE = 1 )
integer SHMEM_CMP_GT
parameter ( SHMEM_CMP_GT = 2 )
integer SHMEM_CMP_LE
parameter ( SHMEM_CMP_LE = 3 )
integer SHMEM_CMP_LT
parameter ( SHMEM_CMP_LT = 4 )
integer SHMEM_CMP_GE
parameter ( SHMEM_CMP_GE = 5 )
logical shmem_pe_accessible
logical shmem_addr_accessible
integer*8 shmem_ptr

389
oshmem/include/shmem.h.in
Просмотреть файл

@ -1,389 +0,0 @@
/*
* Copyright (c) 2012 Mellanox Technologies, Inc.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef OSHMEM_SHMEM_H
#define OSHMEM_SHMEM_H
#include <stddef.h> /* include for ptrdiff_t */
#if defined(WIN32) || defined(_WIN32)
# define OSHMEM_COMPLEX_TYPE(type)
#else
# if defined(c_plusplus) || defined(__cplusplus)
# include <complex>
# define OSHMEM_COMPLEX_TYPE(type) std::complex<type>
# else
# include <complex.h>
# define OSHMEM_COMPLEX_TYPE(type) type complex
# endif
#endif
/*
* SHMEM version
*/
#define SHMEM_VERSION 1
#define SHMEM_SUBVERSION 5
#ifndef OSHMEM_DECLSPEC
# if defined(WIN32) || defined(_WIN32)
# if defined(OSHMEM_IMPORTS)
# define OSHMEM_DECLSPEC __declspec(dllimport)
# else
# define OSHMEM_DECLSPEC
# endif /* defined(OSHMEM_IMPORTS) */
# else
# if defined(OPAL_C_HAVE_VISIBILITY) && (OPAL_C_HAVE_VISIBILITY == 1)
# define OSHMEM_DECLSPEC __attribute__((visibility("default")))
# else
# define OSHMEM_DECLSPEC
# endif
# endif
#endif
#ifndef OSHMEM_DESTRUCTOR
# if defined(OPAL_C_HAVE_VISIBILITY) && (OPAL_C_HAVE_VISIBILITY == 1)
# define OSHMEM_DESTRUCTOR __attribute__((__destructor__))
# else
# define OSHMEM_DESTRUCTOR
# endif
#endif
#if defined(c_plusplus) || defined(__cplusplus)
extern "C" {
#endif
/*
* OpenSHMEM API (www.openshmem.org)
*/
/*
* Environment variables
*/
/* size of symmetric heap in bytes.
* Can be qualified with the letter 'K', 'M', 'G' or 'T'
*/
#define SHMEM_HEAP_SIZE "SHMEM_SYMMETRIC_HEAP_SIZE"
/* Following environment variables are Mellanox extension */
/*
* Type of allocator used by symmetric heap
*/
#define SHMEM_HEAP_TYPE "SHMEM_SYMMETRIC_HEAP_ALLOCATOR"
/*
* Constants and definitions
*/
enum shmem_wait_ops {
SHMEM_CMP_EQ,
SHMEM_CMP_NE,
SHMEM_CMP_GT,
SHMEM_CMP_LE,
SHMEM_CMP_LT,
SHMEM_CMP_GE
};
#define _SHMEM_BARRIER_SYNC_SIZE (1)
#define _SHMEM_BCAST_SYNC_SIZE (1 + _SHMEM_BARRIER_SYNC_SIZE)
#define _SHMEM_COLLECT_SYNC_SIZE (1 + _SHMEM_BCAST_SYNC_SIZE)
#define _SHMEM_REDUCE_SYNC_SIZE (1 + _SHMEM_BCAST_SYNC_SIZE)
#define _SHMEM_REDUCE_MIN_WRKDATA_SIZE (1)
#define _SHMEM_SYNC_VALUE (-1)
#define SHMEM_BARRIER_SYNC_SIZE _SHMEM_BARRIER_SYNC_SIZE
#define SHMEM_BCAST_SYNC_SIZE _SHMEM_BCAST_SYNC_SIZE
#define SHMEM_COLLECT_SYNC_SIZE _SHMEM_COLLECT_SYNC_SIZE
#define SHMEM_REDUCE_SYNC_SIZE _SHMEM_REDUCE_SYNC_SIZE
#define SHMEM_REDUCE_MIN_WRKDATA_SIZE _SHMEM_REDUCE_MIN_WRKDATA_SIZE
#define SHMEM_SYNC_VALUE _SHMEM_SYNC_VALUE
/*
* Initialization routines
*/
OSHMEM_DECLSPEC void start_pes(int npes);
/*
* Query routines
*/
OSHMEM_DECLSPEC int _num_pes(void);
OSHMEM_DECLSPEC int _my_pe(void);
/*
* Accessability routines
*/
OSHMEM_DECLSPEC int shmem_pe_accessible(int pe);
OSHMEM_DECLSPEC int shmem_addr_accessible(void *addr, int pe);
/*
* Symmetric heap routines
*/
OSHMEM_DECLSPEC void* shmalloc(size_t size);
OSHMEM_DECLSPEC void* shmemalign(size_t align, size_t size);
OSHMEM_DECLSPEC void* shrealloc(void *ptr, size_t size);
OSHMEM_DECLSPEC void shfree(void* ptr);
/*
* Remote pointer operations
*/
OSHMEM_DECLSPEC void *shmem_ptr(void *ptr, int pe);
/*
* Elemental put routines
*/
OSHMEM_DECLSPEC void shmem_short_p(short* addr, short value, int pe);
OSHMEM_DECLSPEC void shmem_int_p(int* addr, int value, int pe);
OSHMEM_DECLSPEC void shmem_long_p(long* addr, long value, int pe);
OSHMEM_DECLSPEC void shmem_float_p(float* addr, float value, int pe);
OSHMEM_DECLSPEC void shmem_double_p(double* addr, double value, int pe);
OSHMEM_DECLSPEC void shmem_longlong_p(long long* addr, long long value, int pe);
OSHMEM_DECLSPEC void shmem_longdouble_p(long double* addr, long double value, int pe);
/*
* Block data put routines
*/
OSHMEM_DECLSPEC void shmem_char_put(char *target, const char *source, size_t len, int pe);
OSHMEM_DECLSPEC void shmem_short_put(short *target, const short *source, size_t len, int pe);
OSHMEM_DECLSPEC void shmem_int_put(int* target, const int* source, size_t len, int pe);
OSHMEM_DECLSPEC void shmem_long_put(long *target, const long *source, size_t len, int pe);
OSHMEM_DECLSPEC void shmem_float_put(float *target, const float *source, size_t len, int pe);
OSHMEM_DECLSPEC void shmem_double_put(double *target, const double *source, size_t len, int pe);
OSHMEM_DECLSPEC void shmem_longlong_put(long long *target, const long long *source, size_t len, int pe);
OSHMEM_DECLSPEC void shmem_longdouble_put(long double *target, const long double *source, size_t len, int pe);
OSHMEM_DECLSPEC void shmem_put32(void *target, const void *source, size_t len, int pe);
OSHMEM_DECLSPEC void shmem_put64(void *target, const void *source, size_t len, int pe);
OSHMEM_DECLSPEC void shmem_put128(void *target, const void *source, size_t len, int pe);
OSHMEM_DECLSPEC void shmem_putmem(void *target, const void *source, size_t len, int pe);
/*
* Strided put routines
*/
OSHMEM_DECLSPEC void shmem_int_iput(int* target, const int* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
OSHMEM_DECLSPEC void shmem_short_iput(short* target, const short* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
OSHMEM_DECLSPEC void shmem_float_iput(float* target, const float* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
OSHMEM_DECLSPEC void shmem_double_iput(double* target, const double* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
OSHMEM_DECLSPEC void shmem_longlong_iput(long long* target, const long long* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
OSHMEM_DECLSPEC void shmem_longdouble_iput(long double* target, const long double* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
OSHMEM_DECLSPEC void shmem_long_iput(long* target, const long* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
OSHMEM_DECLSPEC void shmem_iput32(void* target, const void* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
OSHMEM_DECLSPEC void shmem_iput64(void* target, const void* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
OSHMEM_DECLSPEC void shmem_iput128(void* target, const void* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
/*
* Elemental get routines
*/
OSHMEM_DECLSPEC short shmem_short_g(short* addr, int pe);
OSHMEM_DECLSPEC int shmem_int_g(int* addr, int pe);
OSHMEM_DECLSPEC long shmem_long_g(long* addr, int pe);
OSHMEM_DECLSPEC float shmem_float_g(float* addr, int pe);
OSHMEM_DECLSPEC double shmem_double_g(double* addr, int pe);
OSHMEM_DECLSPEC long long shmem_longlong_g(long long* addr, int pe);
OSHMEM_DECLSPEC long double shmem_longdouble_g(long double* addr, int pe);
/*
* Block data get routines
*/
OSHMEM_DECLSPEC void shmem_char_get(char *target, const char *source, size_t len, int pe);
OSHMEM_DECLSPEC void shmem_short_get(short *target, const short *source, size_t len, int pe);
OSHMEM_DECLSPEC void shmem_int_get(int *target, const int *source, size_t len, int pe);
OSHMEM_DECLSPEC void shmem_long_get(long *target, const long *source, size_t len, int pe);
OSHMEM_DECLSPEC void shmem_float_get(float *target, const float *source, size_t len, int pe);
OSHMEM_DECLSPEC void shmem_double_get(double *target, const double *source, size_t len, int pe);
OSHMEM_DECLSPEC void shmem_longlong_get(long long *target, const long long *source, size_t len, int pe);
OSHMEM_DECLSPEC void shmem_longdouble_get(long double *target, const long double *source, size_t len, int pe);
OSHMEM_DECLSPEC void shmem_get32(void *target, const void *source, size_t len, int pe);
OSHMEM_DECLSPEC void shmem_get64(void *target, const void *source, size_t len, int pe);
OSHMEM_DECLSPEC void shmem_get128(void *target, const void *source, size_t len, int pe);
OSHMEM_DECLSPEC void shmem_getmem(void *target, const void *source, size_t len, int pe);
/*
* Strided get routines
*/
OSHMEM_DECLSPEC void shmem_int_iget(int* target, const int* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
OSHMEM_DECLSPEC void shmem_short_iget(short* target, const short* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
OSHMEM_DECLSPEC void shmem_float_iget(float* target, const float* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
OSHMEM_DECLSPEC void shmem_double_iget(double* target, const double* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
OSHMEM_DECLSPEC void shmem_longlong_iget(long long* target, const long long* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
OSHMEM_DECLSPEC void shmem_longdouble_iget(long double* target, const long double* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
OSHMEM_DECLSPEC void shmem_long_iget(long* target, const long* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
OSHMEM_DECLSPEC void shmem_iget32(void* target, const void* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
OSHMEM_DECLSPEC void shmem_iget64(void* target, const void* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
OSHMEM_DECLSPEC void shmem_iget128(void* target, const void* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
/*
* Atomic operations
*/
/* Atomic swap */
OSHMEM_DECLSPEC long shmem_swap(long *target, long value, int pe);
OSHMEM_DECLSPEC double shmem_double_swap(double *target, double value, int pe);
OSHMEM_DECLSPEC float shmem_float_swap(float *target, float value, int pe);
OSHMEM_DECLSPEC int shmem_int_swap(int *target, int value, int pe);
OSHMEM_DECLSPEC long shmem_long_swap(long *target, long value, int pe);
OSHMEM_DECLSPEC long long shmem_longlong_swap(long long*target, long long value, int pe);
/* Atomic conditional swap */
OSHMEM_DECLSPEC int shmem_int_cswap(int *target, int cond, int value, int pe);
OSHMEM_DECLSPEC long shmem_long_cswap(long *target, long cond, long value, int pe);
OSHMEM_DECLSPEC long long shmem_longlong_cswap(long long *target, long long cond, long long value, int pe);
/* Atomic Fetch&Add */
OSHMEM_DECLSPEC int shmem_int_fadd(int *target, int value, int pe);
OSHMEM_DECLSPEC long shmem_long_fadd(long *target, long value, int pe);
OSHMEM_DECLSPEC long long shmem_longlong_fadd(long long *target, long long value, int pe);
/* Atomic Fetch&Inc */
OSHMEM_DECLSPEC int shmem_int_finc(int *target, int pe);
OSHMEM_DECLSPEC long shmem_long_finc(long *target, int pe);
OSHMEM_DECLSPEC long long shmem_longlong_finc(long long *target, int pe);
/* Atomic Add*/
OSHMEM_DECLSPEC void shmem_int_add(int *target, int value, int pe);
OSHMEM_DECLSPEC void shmem_long_add(long *target, long value, int pe);
OSHMEM_DECLSPEC void shmem_longlong_add(long long *target, long long value, int pe);
/* Atomic Inc */
OSHMEM_DECLSPEC void shmem_int_inc(int *target, int pe);
OSHMEM_DECLSPEC void shmem_long_inc(long *target, int pe);
OSHMEM_DECLSPEC void shmem_longlong_inc(long long *target, int pe);
/*
* Lock functions
*/
OSHMEM_DECLSPEC void shmem_set_lock(long *lock);
OSHMEM_DECLSPEC void shmem_clear_lock(long *lock);
OSHMEM_DECLSPEC int shmem_test_lock(long *lock);
/*
* P2P sync routines
*/
OSHMEM_DECLSPEC void shmem_short_wait(short *addr, short value);
OSHMEM_DECLSPEC void shmem_int_wait(int *addr, int value);
OSHMEM_DECLSPEC void shmem_long_wait(long *addr, long value);
OSHMEM_DECLSPEC void shmem_longlong_wait(long long *addr, long long value);
OSHMEM_DECLSPEC void shmem_wait(long *addr, long value);
OSHMEM_DECLSPEC void shmem_short_wait_until(short *addr, int cmp, short value);
OSHMEM_DECLSPEC void shmem_int_wait_until(int *addr, int cmp, int value);
OSHMEM_DECLSPEC void shmem_long_wait_until(long *addr, int cmp, long value);
OSHMEM_DECLSPEC void shmem_longlong_wait_until(long long *addr, int cmp, long long value);
OSHMEM_DECLSPEC void shmem_wait_until(long *addr, int cmp, long value);
/*
* Barrier sync routines
*/
OSHMEM_DECLSPEC void shmem_barrier(int PE_start, int logPE_stride, int PE_size, long *pSync);
OSHMEM_DECLSPEC void shmem_barrier_all(void);
OSHMEM_DECLSPEC void shmem_fence(void);
OSHMEM_DECLSPEC void shmem_quiet(void);
/*
* Collective routines
*/
OSHMEM_DECLSPEC void shmem_broadcast32(void *target, const void *source, size_t nlong, int PE_root, int PE_start, int logPE_stride, int PE_size, long *pSync);
OSHMEM_DECLSPEC void shmem_broadcast64(void *target, const void *source, size_t nlong, int PE_root, int PE_start, int logPE_stride, int PE_size, long *pSync);
OSHMEM_DECLSPEC void shmem_collect32(void *target, const void *source, size_t nlong, int PE_start, int logPE_stride, int PE_size, long *pSync);
OSHMEM_DECLSPEC void shmem_collect64(void *target, const void *source, size_t nlong, int PE_start, int logPE_stride, int PE_size, long *pSync);
OSHMEM_DECLSPEC void shmem_fcollect32(void *target, const void *source, size_t nlong, int PE_start, int logPE_stride, int PE_size, long *pSync);
OSHMEM_DECLSPEC void shmem_fcollect64(void *target, const void *source, size_t nlong, int PE_start, int logPE_stride, int PE_size, long *pSync);
/*
* Reduction routines
*/
OSHMEM_DECLSPEC void shmem_short_and_to_all(short *target, short *source, int nreduce, int PE_start, int logPE_stride, int PE_size, short *pWrk, long *pSync);
OSHMEM_DECLSPEC void shmem_int_and_to_all(int *target, int *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int *pWrk, long *pSync);
OSHMEM_DECLSPEC void shmem_long_and_to_all(long *target, long *source, int nreduce, int PE_start, int logPE_stride, int PE_size, long *pWrk, long *pSync);
OSHMEM_DECLSPEC void shmem_longlong_and_to_all(long long *target, long long *source, int nreduce, int PE_start, int logPE_stride, int PE_size, long long *pWrk, long *pSync);
OSHMEM_DECLSPEC void shmem_short_or_to_all(short *target, short *source, int nreduce, int PE_start, int logPE_stride, int PE_size, short *pWrk, long *pSync);
OSHMEM_DECLSPEC void shmem_int_or_to_all(int *target, int *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int *pWrk, long *pSync);
OSHMEM_DECLSPEC void shmem_long_or_to_all(long *target, long *source, int nreduce, int PE_start, int logPE_stride, int PE_size, long *pWrk, long *pSync);
OSHMEM_DECLSPEC void shmem_longlong_or_to_all(long long *target, long long *source, int nreduce, int PE_start, int logPE_stride, int PE_size, long long *pWrk, long *pSync);
OSHMEM_DECLSPEC void shmem_short_xor_to_all(short *target, short *source, int nreduce, int PE_start, int logPE_stride, int PE_size, short *pWrk, long *pSync);
OSHMEM_DECLSPEC void shmem_int_xor_to_all(int *target, int *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int *pWrk, long *pSync);
OSHMEM_DECLSPEC void shmem_long_xor_to_all(long *target, long *source, int nreduce, int PE_start, int logPE_stride, int PE_size, long *pWrk, long *pSync);
OSHMEM_DECLSPEC void shmem_longlong_xor_to_all(long long *target, long long *source, int nreduce, int PE_start, int logPE_stride, int PE_size, long long *pWrk, long *pSync);
OSHMEM_DECLSPEC void shmem_short_max_to_all(short *target, short *source, int nreduce, int PE_start, int logPE_stride, int PE_size, short *pWrk, long *pSync);
OSHMEM_DECLSPEC void shmem_int_max_to_all(int *target, int *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int *pWrk, long *pSync);
OSHMEM_DECLSPEC void shmem_long_max_to_all(long *target, long *source, int nreduce, int PE_start, int logPE_stride, int PE_size, long *pWrk, long *pSync);
OSHMEM_DECLSPEC void shmem_longlong_max_to_all(long long *target, long long *source, int nreduce, int PE_start, int logPE_stride, int PE_size, long long *pWrk, long *pSync);
OSHMEM_DECLSPEC void shmem_float_max_to_all(float *target, float *source, int nreduce, int PE_start, int logPE_stride, int PE_size, float *pWrk, long *pSync);
OSHMEM_DECLSPEC void shmem_double_max_to_all(double *target, double *source, int nreduce, int PE_start, int logPE_stride, int PE_size, double *pWrk, long *pSync);
OSHMEM_DECLSPEC void shmem_longdouble_max_to_all(long double *target, long double *source, int nreduce, int PE_start, int logPE_stride, int PE_size, long double *pWrk, long *pSync);
OSHMEM_DECLSPEC void shmem_short_min_to_all(short *target, short *source, int nreduce, int PE_start, int logPE_stride, int PE_size, short *pWrk, long *pSync);
OSHMEM_DECLSPEC void shmem_int_min_to_all(int *target, int *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int *pWrk, long *pSync);
OSHMEM_DECLSPEC void shmem_long_min_to_all(long *target, long *source, int nreduce, int PE_start, int logPE_stride, int PE_size, long *pWrk, long *pSync);
OSHMEM_DECLSPEC void shmem_longlong_min_to_all(long long *target, long long *source, int nreduce, int PE_start, int logPE_stride, int PE_size, long long *pWrk, long *pSync);
OSHMEM_DECLSPEC void shmem_float_min_to_all(float *target, float *source, int nreduce, int PE_start, int logPE_stride, int PE_size, float *pWrk, long *pSync);
OSHMEM_DECLSPEC void shmem_double_min_to_all(double *target, double *source, int nreduce, int PE_start, int logPE_stride, int PE_size, double *pWrk, long *pSync);
OSHMEM_DECLSPEC void shmem_longdouble_min_to_all(long double *target, long double *source, int nreduce, int PE_start, int logPE_stride, int PE_size, long double *pWrk, long *pSync);
OSHMEM_DECLSPEC void shmem_short_sum_to_all(short *target, short *source, int nreduce, int PE_start, int logPE_stride, int PE_size, short *pWrk, long *pSync);
OSHMEM_DECLSPEC void shmem_int_sum_to_all(int *target, int *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int *pWrk, long *pSync);
OSHMEM_DECLSPEC void shmem_long_sum_to_all(long *target, long *source, int nreduce, int PE_start, int logPE_stride, int PE_size, long *pWrk, long *pSync);
OSHMEM_DECLSPEC void shmem_longlong_sum_to_all(long long *target, long long *source, int nreduce, int PE_start, int logPE_stride, int PE_size, long long *pWrk, long *pSync);
OSHMEM_DECLSPEC void shmem_float_sum_to_all(float *target, float *source, int nreduce, int PE_start, int logPE_stride, int PE_size, float *pWrk, long *pSync);
OSHMEM_DECLSPEC void shmem_double_sum_to_all(double *target, double *source, int nreduce, int PE_start, int logPE_stride, int PE_size, double *pWrk, long *pSync);
OSHMEM_DECLSPEC void shmem_longdouble_sum_to_all(long double *target, long double *source, int nreduce, int PE_start, int logPE_stride, int PE_size, long double *pWrk, long *pSync);
OSHMEM_DECLSPEC void shmem_complexf_sum_to_all(OSHMEM_COMPLEX_TYPE(float) *target, OSHMEM_COMPLEX_TYPE(float) *source, int nreduce, int PE_start, int logPE_stride, int PE_size, OSHMEM_COMPLEX_TYPE(float) *pWrk, long *pSync);
OSHMEM_DECLSPEC void shmem_complexd_sum_to_all(OSHMEM_COMPLEX_TYPE(double) *target, OSHMEM_COMPLEX_TYPE(double) *source, int nreduce, int PE_start, int logPE_stride, int PE_size, OSHMEM_COMPLEX_TYPE(double) *pWrk, long *pSync);
OSHMEM_DECLSPEC void shmem_short_prod_to_all(short *target, short *source, int nreduce, int PE_start, int logPE_stride, int PE_size, short *pWrk, long *pSync);
OSHMEM_DECLSPEC void shmem_int_prod_to_all(int *target, int *source, int nreduce, int PE_start, int logPE_stride, int PE_size, int *pWrk, long *pSync);
OSHMEM_DECLSPEC void shmem_long_prod_to_all(long *target, long *source, int nreduce, int PE_start, int logPE_stride, int PE_size, long *pWrk, long *pSync);
OSHMEM_DECLSPEC void shmem_longlong_prod_to_all(long long *target, long long *source, int nreduce, int PE_start, int logPE_stride, int PE_size, long long *pWrk, long *pSync);
OSHMEM_DECLSPEC void shmem_float_prod_to_all(float *target, float *source, int nreduce, int PE_start, int logPE_stride, int PE_size, float *pWrk, long *pSync);
OSHMEM_DECLSPEC void shmem_double_prod_to_all(double *target, double *source, int nreduce, int PE_start, int logPE_stride, int PE_size, double *pWrk, long *pSync);
OSHMEM_DECLSPEC void shmem_longdouble_prod_to_all(long double *target, long double *source, int nreduce, int PE_start, int logPE_stride, int PE_size, long double *pWrk, long *pSync);
OSHMEM_DECLSPEC void shmem_complexf_prod_to_all(OSHMEM_COMPLEX_TYPE(float) *target, OSHMEM_COMPLEX_TYPE(float) *source, int nreduce, int PE_start, int logPE_stride, int PE_size, OSHMEM_COMPLEX_TYPE(float) *pWrk, long *pSync);
OSHMEM_DECLSPEC void shmem_complexd_prod_to_all(OSHMEM_COMPLEX_TYPE(double) *target, OSHMEM_COMPLEX_TYPE(double) *source, int nreduce, int PE_start, int logPE_stride, int PE_size, OSHMEM_COMPLEX_TYPE(double) *pWrk, long *pSync);
/*
* Platform specific cache management routines
*/
OSHMEM_DECLSPEC void shmem_udcflush(void);
OSHMEM_DECLSPEC void shmem_udcflush_line(void* target);
OSHMEM_DECLSPEC void shmem_set_cache_inv(void);
OSHMEM_DECLSPEC void shmem_set_cache_line_inv(void* target);
OSHMEM_DECLSPEC void shmem_clear_cache_inv(void);
OSHMEM_DECLSPEC void shmem_clear_cache_line_inv(void* target);
/*
* Legacy API
*/
OSHMEM_DECLSPEC int num_pes(void);
OSHMEM_DECLSPEC int my_pe(void);
/* old init/destruct functions - not in the open shmem spec but still supported */
OSHMEM_DECLSPEC void shmem_init(void);
OSHMEM_DECLSPEC int shmem_finalize(void) OSHMEM_DESTRUCTOR;
OSHMEM_DECLSPEC int shmem_n_pes(void);
OSHMEM_DECLSPEC int shmem_my_pe(void);
OSHMEM_DECLSPEC void shmem_put(void *target, const void *source, size_t len, int pe);
OSHMEM_DECLSPEC void shmem_get(void *target, const void *source, size_t len, int pe);
#if defined(c_plusplus) || defined(__cplusplus)
}
#endif
#endif /* OSHMEM_SHMEM_H */

401
oshmem/include/shmem_portable_platform.h.in
Просмотреть файл

@ -1,401 +0,0 @@
/*
* Header file with preprocessor magic to figure out, which compiler the user has been calling!
*
* This code is adapted from the file other/portable_platform.h of GASnet-1.12.0:
* - Ripping out the required parts.
* - Get rid of brackets as it messes up autoconf
* - Delete version tests for older PGI versions (#include "omp.h" not acceptabe)
* - Indent ('#' should be in column 0)
*
* External packages (vt, romio) depend on top_build_dir/ompi/include, therefore
* although this is not changed in the configure process, this has to be set as
* a .in file...
* ---------------------------------------------------------------------------
*/
#ifndef MPI_PORTABLE_PLATFORM_H
#define MPI_PORTABLE_PLATFORM_H
/* All files in this directory and all sub-directories (except where otherwise noted)
* are subject to the following licensing terms:
*
* ---------------------------------------------------------------------------
* "Copyright (c) 2012 Mellanox Technologies, Inc.
* " All rights reserved.
*
* Permission to use, copy, modify, and distribute this software and its
* documentation for any purpose, without fee, and without written agreement is
* hereby granted, provided that the above copyright notice and the following
* two paragraphs appear in all copies of this software.
*
* IN NO EVENT SHALL THE UNIVERSITY OF CALIFORNIA BE LIABLE TO ANY PARTY FOR
* DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING OUT
* OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF THE UNIVERSITY OF
* CALIFORNIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* THE UNIVERSITY OF CALIFORNIA SPECIFICALLY DISCLAIMS ANY WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
* AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS
* ON AN "AS IS" BASIS, AND THE UNIVERSITY OF CALIFORNIA HAS NO OBLIGATION TO
* PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS."
* ---------------------------------------------------------------------------
*
* Please see the license.txt files within the gm-conduit, lapi-conduit and
* vapi-conduit directories for the licensing terms governing those
* contributed components.
*
* The authors/contributors of GASNet include:
*
* Dan Bonachea <bonachea@cs.berkeley.edu>:
* General infrastructure & documentation
* mpi-conduit
* elan-conduit
* smp-conduit
* udp-conduit
* extended-ref
* template-conduit
* Christian Bell <csbell@cs.berkeley.edu>: gm-conduit, shmem-conduit
* Mike Welcome <mlwelcome@lbl.gov>: lapi-conduit, portals-conduit
* Paul H. Hargrove <phhargrove@lbl.gov>: vapi-conduit, ibv-conduit
* Rajesh Nishtala <rajeshn@cs.berkeley.edu>: collectives, dcmf-conduit
* Parry Husbands (PJRHusbands@lbl.gov): lapi-conduit
*
* For more information about GASNet, visit our home page at:
* http://gasnet.cs.berkeley.edu/
* Or send email to:
* <upc@lbl.gov>
*
* Source code contributions (fixes, patches, extensions etc.) should be
* sent to <upc@lbl.gov> to be reviewed for acceptance into the primary
* distribution. Contributions are most likely to be accepted if they
* are provided as public domain, or under a BSD-style license such as
* the one above.
*
*/
#ifndef _STRINGIFY
#define _STRINGIFY_HELPER(x) #x
#define _STRINGIFY(x) _STRINGIFY_HELPER(x)
#endif
#if defined(__INTEL_COMPILER)
# define PLATFORM_COMPILER_FAMILYNAME INTEL
# define PLATFORM_COMPILER_FAMILYID 2
# ifdef __cplusplus
# define PLATFORM_COMPILER_INTEL_CXX 1
# else
# define PLATFORM_COMPILER_INTEL_C 1
# endif
# define _PLATFORM_COMPILER_INTEL_MIN_BUILDDATE 19700000 /* year 1970: predates most intel products :) */
# ifdef __INTEL_COMPILER_BUILD_DATE
# define _PLATFORM_INTEL_COMPILER_BUILD_DATE __INTEL_COMPILER_BUILD_DATE
# else
# define _PLATFORM_INTEL_COMPILER_BUILD_DATE _PLATFORM_COMPILER_INTEL_MIN_BUILDDATE
# endif
/* patch number is a decimal build date: YYYYMMDD */
# define PLATFORM_COMPILER_VERSION_INT(maj,min,pat) \
(((((maj) * 10) | (min)) << 20) | \
((pat) < _PLATFORM_COMPILER_INTEL_MIN_BUILDDATE ? \
_PLATFORM_COMPILER_INTEL_MIN_BUILDDATE : ((pat)-_PLATFORM_COMPILER_INTEL_MIN_BUILDDATE)))
# define PLATFORM_COMPILER_VERSION \
PLATFORM_COMPILER_VERSION_INT(__INTEL_COMPILER/10, __INTEL_COMPILER/100, _PLATFORM_INTEL_COMPILER_BUILD_DATE)
# define PLATFORM_COMPILER_VERSION_STR \
_STRINGIFY(__INTEL_COMPILER)"."_STRINGIFY(_PLATFORM_INTEL_COMPILER_BUILD_DATE)
#elif defined(__PATHSCALE__)
# define PLATFORM_COMPILER_PATHSCALE 1
# define PLATFORM_COMPILER_FAMILYNAME PATHSCALE
# define PLATFORM_COMPILER_FAMILYID 3
# ifdef __cplusplus
# define PLATFORM_COMPILER_PATHSCALE_CXX 1
# else
# define PLATFORM_COMPILER_PATHSCALE_C 1
# endif
# define PLATFORM_COMPILER_VERSION \
PLATFORM_COMPILER_VERSION_INT(__PATHCC__,__PATHCC_MINOR__,__PATHCC_PATCHLEVEL__)
# define PLATFORM_COMPILER_VERSION_STR __PATHSCALE__
#elif defined(__PGI)
# define PLATFORM_COMPILER_PGI 1
# define PLATFORM_COMPILER_FAMILYNAME PGI
# define PLATFORM_COMPILER_FAMILYID 4
# ifdef __cplusplus
# define PLATFORM_COMPILER_PGI_CXX 1
# else
# define PLATFORM_COMPILER_PGI_C 1
# endif
# if __PGIC__ == 99
/* bug 2230: PGI versioning was broken for some platforms in 7.0
no way to know exact version, but provide something slightly more accurate */
# define PLATFORM_COMPILER_VERSION 0x070000
# define PLATFORM_COMPILER_VERSION_STR "7.?-?"
# elif defined(__PGIC__) && defined(__PGIC_MINOR__) && defined(__PGIC_PATCHLEVEL__)
# define PLATFORM_COMPILER_VERSION \
PLATFORM_COMPILER_VERSION_INT(__PGIC__,__PGIC_MINOR__,__PGIC_PATCHLEVEL__)
# define PLATFORM_COMPILER_VERSION_STR \
_STRINGIFY(__PGIC__)"."_STRINGIFY(__PGIC_MINOR__)"-"_STRINGIFY(__PGIC_PATCHLEVEL__)
# else
/* PGI before 6.1-4 lacks any version ID preprocessor macros - so use this filthy hack */
/* XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
* We cannot do these within mpi.h.in, as we should not include ompi.h
* Hopefully, compilers with integrated preprocessors will not analyse code within the #if 0-block
* XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
*/
#if 0
# ifdef PLATFORM_PGI_IS_ANCIENT
/* Include below might fail for ancient versions lacking this header, but testing shows it
works back to at least 5.1-3 (Nov 2003), and based on docs probably back to 3.2 (Sep 2000) */
# define PLATFORM_COMPILER_VERSION 0
# elif defined(__x86_64__) /* bug 1753 - 64-bit omp.h upgrade happenned in <6.0-8,6.1-1) */
# include "omp.h"
# if defined(_PGOMP_H)
/* 6.1.1 or newer */
# define PLATFORM_COMPILER_VERSION 0x060101
# define PLATFORM_COMPILER_VERSION_STR ">=6.1-1"
# else
/* 6.0.8 or older */
# define PLATFORM_COMPILER_VERSION 0
# define PLATFORM_COMPILER_VERSION_STR "<=6.0-8"
# endif
# else /* 32-bit omp.h upgrade happenned in <5.2-4,6.0-8 */
# include "omp.h"
# if defined(_PGOMP_H)
/* 6.0-8 or newer */
# define PLATFORM_COMPILER_VERSION 0x060008
# define PLATFORM_COMPILER_VERSION_STR ">=6.0-8"
# else
/* 5.2-4 or older */
# define PLATFORM_COMPILER_VERSION 0
# define PLATFORM_COMPILER_VERSION_STR "<=5.2-4"
# endif
# endif
#endif /* 0 */
/* XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX */
# endif
#elif defined(__xlC__)
# define PLATFORM_COMPILER_XLC 1
# define PLATFORM_COMPILER_FAMILYNAME XLC
# define PLATFORM_COMPILER_FAMILYID 5
# ifdef __cplusplus
# define PLATFORM_COMPILER_XLC_CXX 1
# else
# define PLATFORM_COMPILER_XLC_C 1
# endif
# define PLATFORM_COMPILER_VERSION __xlC__
# define PLATFORM_COMPILER_VERSION_INT(maj,min,pat) \
( ((maj) << 8) | ((min) << 4) | (pat) )
#elif defined(__DECC) || defined(__DECCXX)
# define PLATFORM_COMPILER_COMPAQ 1
# define PLATFORM_COMPILER_FAMILYNAME COMPAQ
# define PLATFORM_COMPILER_FAMILYID 6
# ifdef __cplusplus
# define PLATFORM_COMPILER_COMPAQ_CXX 1
# else
# define PLATFORM_COMPILER_COMPAQ_C 1
# endif
# if defined(__DECC_VER)
# define PLATFORM_COMPILER_VERSION __DECC_VER
# elif defined(__DECCXX_VER)
# define PLATFORM_COMPILER_VERSION __DECCXX_VER
# endif
# define PLATFORM_COMPILER_VERSION_INT(maj,min,pat) \
( ((maj) * 10000000) + ((min) * 100000) + (90000) + (pat) )
/* 90000 = official ver, 80000 = customer special ver, 60000 = field test ver */
#elif defined(__SUNPRO_C) || defined(__SUNPRO_CC)
# define PLATFORM_COMPILER_SUN 1
# define PLATFORM_COMPILER_FAMILYNAME SUN
# define PLATFORM_COMPILER_FAMILYID 7
# ifdef __cplusplus
# define PLATFORM_COMPILER_SUN_CXX 1
# else
# define PLATFORM_COMPILER_SUN_C 1
# endif
# if defined(__SUNPRO_C) && __SUNPRO_C > 0
# define PLATFORM_COMPILER_VERSION __SUNPRO_C
# elif defined(__SUNPRO_CC) && __SUNPRO_CC > 0
# define PLATFORM_COMPILER_VERSION __SUNPRO_CC
# endif
# define PLATFORM_COMPILER_VERSION_INT(maj,min,pat) \
( ((maj) << 8) | ((min) << 4) | (pat) )
#elif defined(__HP_cc) || defined(__HP_aCC)
# define PLATFORM_COMPILER_HP 1
# define PLATFORM_COMPILER_FAMILYNAME HP
# define PLATFORM_COMPILER_FAMILYID 8
# ifdef __cplusplus
# define PLATFORM_COMPILER_HP_CXX 1
# else
# define PLATFORM_COMPILER_HP_C 1
# endif
# if defined(__HP_cc) && __HP_cc > 0
# define PLATFORM_COMPILER_VERSION __HP_cc
# elif defined(__HP_aCC) && __HP_aCC > 0
# define PLATFORM_COMPILER_VERSION __HP_aCC
# endif
# define PLATFORM_COMPILER_VERSION_INT(maj,min,pat) \
( ((maj) << 16) | ((min) << 8) | (pat) )
#elif defined(_SGI_COMPILER_VERSION) || \
(defined(_COMPILER_VERSION) && defined(__sgi) && !defined(__GNUC__)) /* 7.3.0 and earlier lack _SGI_COMPILER_VERSION */
# define PLATFORM_COMPILER_SGI 1
# define PLATFORM_COMPILER_FAMILYNAME SGI
# define PLATFORM_COMPILER_FAMILYID 9
# ifdef __cplusplus
# define PLATFORM_COMPILER_SGI_CXX 1
# else
# define PLATFORM_COMPILER_SGI_C 1
# endif
# if defined(_SGI_COMPILER_VERSION) && _SGI_COMPILER_VERSION > 0
# define PLATFORM_COMPILER_VERSION _SGI_COMPILER_VERSION
# elif defined(_COMPILER_VERSION) && _COMPILER_VERSION > 0
# define PLATFORM_COMPILER_VERSION _COMPILER_VERSION
# endif
# define PLATFORM_COMPILER_VERSION_INT(maj,min,pat) \
( ((maj) << 8) | ((min) << 4) | (pat) )
#elif defined(_CRAYC)
# define PLATFORM_COMPILER_CRAY 1
# define PLATFORM_COMPILER_FAMILYNAME CRAY
# define PLATFORM_COMPILER_FAMILYID 10
# ifdef __cplusplus
# define PLATFORM_COMPILER_CRAY_CXX 1
# else
# define PLATFORM_COMPILER_CRAY_C 1
# endif
# if defined(_RELEASE) && defined(_RELEASE_MINOR) /* X1 */
# define PLATFORM_COMPILER_VERSION \
PLATFORM_COMPILER_VERSION_INT(_RELEASE,_RELEASE_MINOR,0)
# elif defined(_RELEASE) /* T3E */
# define PLATFORM_COMPILER_VERSION \
PLATFORM_COMPILER_VERSION_INT(_RELEASE,0,0)
# endif
# ifdef _RELEASE_STRING /* X1 */
# define PLATFORM_COMPILER_VERSION_STR _RELEASE_STRING
# endif
#elif defined(__KCC)
# define PLATFORM_COMPILER_KAI 1
# define PLATFORM_COMPILER_FAMILYNAME KAI
# define PLATFORM_COMPILER_FAMILYID 11
# ifdef __cplusplus
# define PLATFORM_COMPILER_KAI_CXX 1
# else
# define PLATFORM_COMPILER_KAI_C 1
# endif
#elif defined(__MTA__)
# define PLATFORM_COMPILER_MTA 1
# define PLATFORM_COMPILER_FAMILYNAME MTA
# define PLATFORM_COMPILER_FAMILYID 12
# ifdef __cplusplus
# define PLATFORM_COMPILER_MTA_CXX 1
# else
# define PLATFORM_COMPILER_MTA_C 1
# endif
#elif defined(_SX)
# define PLATFORM_COMPILER_NECSX 1
# define PLATFORM_COMPILER_FAMILYNAME NECSX
# define PLATFORM_COMPILER_FAMILYID 13
# ifdef __cplusplus
# define PLATFORM_COMPILER_NECSX_CXX 1
# else
# define PLATFORM_COMPILER_NECSX_C 1
# endif
#elif defined(_MSC_VER)
# define PLATFORM_COMPILER_MICROSOFT 1
# define PLATFORM_COMPILER_FAMILYNAME MICROSOFT
# define PLATFORM_COMPILER_FAMILYID 14
# ifdef __cplusplus
# define PLATFORM_COMPILER_MICROSOFT_CXX 1
# else
# define PLATFORM_COMPILER_MICROSOFT_C 1
# endif
# define PLATFORM_COMPILER_VERSION _MSC_VER
#elif defined(__TINYC__)
# define PLATFORM_COMPILER_TINY 1
# define PLATFORM_COMPILER_FAMILYNAME TINY
# define PLATFORM_COMPILER_FAMILYID 15
# ifdef __cplusplus
# define PLATFORM_COMPILER_TINY_CXX 1
# else
# define PLATFORM_COMPILER_TINY_C 1
# endif
#elif defined(__LCC__)
# define PLATFORM_COMPILER_LCC 1
# define PLATFORM_COMPILER_FAMILYNAME LCC
# define PLATFORM_COMPILER_FAMILYID 16
# ifdef __cplusplus
# define PLATFORM_COMPILER_LCC_CXX 1
# else
# define PLATFORM_COMPILER_LCC_C 1
# endif
#else /* unknown compiler */
# define PLATFORM_COMPILER_UNKNOWN 1
#endif
/* this stanza comes last, because many vendor compilers lie and claim
to be GNU C for compatibility reasons and/or because they share a frontend */
#if defined(__GNUC__)
# undef PLATFORM_COMPILER_UNKNOWN
# ifndef PLATFORM_COMPILER_FAMILYID
# define PLATFORM_COMPILER_GNU 1
# define PLATFORM_COMPILER_FAMILYNAME GNU
# define PLATFORM_COMPILER_FAMILYID 1
# ifdef __cplusplus
# define PLATFORM_COMPILER_GNU_CXX 1
# else
# define PLATFORM_COMPILER_GNU_C 1
# endif
# if defined(__GNUC_MINOR__) && defined(__GNUC_PATCHLEVEL__)
# define PLATFORM_COMPILER_VERSION \
PLATFORM_COMPILER_VERSION_INT(__GNUC__,__GNUC_MINOR__,__GNUC_PATCHLEVEL__)
# elif defined(__GNUC_MINOR__) /* older versions of egcs lack __GNUC_PATCHLEVEL__ */
# define PLATFORM_COMPILER_VERSION \
PLATFORM_COMPILER_VERSION_INT(__GNUC__,__GNUC_MINOR__,0)
# else
# define PLATFORM_COMPILER_VERSION \
PLATFORM_COMPILER_VERSION_INT(__GNUC__,0,0)
# endif
# define PLATFORM_COMPILER_VERSION_STR __PLATFORM_COMPILER_GNU_VERSION_STR
# else
# define _PLATFORM_COMPILER_GNU_VERSION_STR __PLATFORM_COMPILER_GNU_VERSION_STR
# endif
/* gather any advertised GNU version number info, even for non-gcc compilers */
# if defined(__GNUC_MINOR__) && defined(__GNUC_PATCHLEVEL__)
# define __PLATFORM_COMPILER_GNU_VERSION_STR \
_STRINGIFY(__GNUC__)"."_STRINGIFY(__GNUC_MINOR__)"."_STRINGIFY(__GNUC_PATCHLEVEL__)
# elif defined(__GNUC_MINOR__)
# define __PLATFORM_COMPILER_GNU_VERSION_STR \
_STRINGIFY(__GNUC__)"."_STRINGIFY(__GNUC_MINOR__)".?"
# else
# define __PLATFORM_COMPILER_GNU_VERSION_STR \
_STRINGIFY(__GNUC__)".?.?"
# endif
#elif defined(PLATFORM_COMPILER_UNKNOWN) /* unknown compiler */
# define PLATFORM_COMPILER_FAMILYNAME UNKNOWN
# define PLATFORM_COMPILER_FAMILYID 0
#endif
/* Default Values */
#ifndef PLATFORM_COMPILER_VERSION
# define PLATFORM_COMPILER_VERSION 0 /* don't know */
#endif
#ifndef PLATFORM_COMPILER_VERSION_STR
# define PLATFORM_COMPILER_VERSION_STR _STRINGIFY(PLATFORM_COMPILER_VERSION)
#endif
#ifndef PLATFORM_COMPILER_VERSION_INT
# define PLATFORM_COMPILER_VERSION_INT(maj,min,pat) \
(((maj) << 16) | ((min) << 8) | (pat))
#endif
#endif /* MPI_PORTABLE_PLATFORM_H */

35
oshmem/mca/atomic/Makefile.am
Просмотреть файл

@ -1,35 +0,0 @@
#
# Copyright (c) 2012 Mellanox Technologies, Inc.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# main library setup
noinst_LTLIBRARIES = libmca_atomic.la
libmca_atomic_la_SOURCES =
# header setup
nobase_oshmem_HEADERS =
nobase_nodist_oshmem_HEADERS =
# local files
headers = atomic.h
libmca_atomic_la_SOURCES += $(headers) $(nodist_headers)
# Conditionally install the header files
if WANT_INSTALL_HEADERS
nobase_oshmem_HEADERS += $(headers)
nobase_nodist_oshmem_HEADERS += $(nodist_headers)
oshmemdir = $(includedir)/oshmem/oshmem/mca/atomic
else
oshmemdir = $(includedir)
endif
include base/Makefile.am
distclean-local:
rm -f base/static-components.h

124
oshmem/mca/atomic/atomic.h
Просмотреть файл

@ -1,124 +0,0 @@
/*
* Copyright (c) 2012 Mellanox Technologies, Inc.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*
* Atomic Operations Interface
*
*/
#ifndef OSHMEM_MCA_ATOMIC_H
#define OSHMEM_MCA_ATOMIC_H
#include "oshmem_config.h"
#include "oshmem/types.h"
#include "oshmem/constants.h"
#include "opal/util/output.h"
#include "mpi.h"
#include "opal/mca/mca.h"
#include "opal/mca/base/base.h"
#include "oshmem/mca/atomic/base/base.h"
BEGIN_C_DECLS
/* ******************************************************************** */
struct oshmem_op_t;
/* ******************************************************************** */
typedef int (*mca_atomic_base_component_init_fn_t)
(bool enable_progress_threads, bool enable_threads);
typedef int (*mca_atomic_base_component_finalize_fn_t)(void);
typedef struct mca_atomic_base_module_1_0_0_t* (*mca_atomic_base_component_query_fn_t)
(int *priority);
/* ******************************************************************** */
/**
* Atomic component interface
*
* Component interface for the atomic framework. A public
* instance of this structure, called
* mca_atomic_[component_name]_component, must exist in any atomic
* component.
*/
struct mca_atomic_base_component_1_0_0_t {
/** Base component description */
mca_base_component_t atomic_version;
/** Base component data block */
mca_base_component_data_t atomic_data;
/** Component initialization function */
mca_atomic_base_component_init_fn_t atomic_init;
mca_atomic_base_component_finalize_fn_t atomic_finalize;
mca_atomic_base_component_query_fn_t atomic_query;
};
typedef struct mca_atomic_base_component_1_0_0_t mca_atomic_base_component_1_0_0_t;
/** Per guidence in mca.h, use the unversioned struct name if you just
want to always keep up with the most recent version of the
interace. */
typedef struct mca_atomic_base_component_1_0_0_t mca_atomic_base_component_t;
/**
* Atomic module interface
*
*/
struct mca_atomic_base_module_1_0_0_t {
/** Collective modules all inherit from opal_object */
opal_object_t super;
/* Collective function pointers */
int (*atomic_fadd)(void *target, void *prev, const void *value, size_t nlong, int pe, struct oshmem_op_t *op);
int (*atomic_cswap)(void *target, void *prev, const void *cond, const void *value, size_t nlong, int pe);
};
typedef struct mca_atomic_base_module_1_0_0_t mca_atomic_base_module_1_0_0_t;
/** Per guidence in mca.h, use the unversioned struct name if you just
want to always keep up with the most recent version of the
interace. */
typedef struct mca_atomic_base_module_1_0_0_t mca_atomic_base_module_t;
OSHMEM_DECLSPEC OBJ_CLASS_DECLARATION(mca_atomic_base_module_t);
/* ******************************************************************** */
/*
* Macro for use in components
*/
#define MCA_ATOMIC_BASE_VERSION_2_0_0 \
MCA_BASE_VERSION_2_0_0, \
"atomic", 1, 0, 0
/* ******************************************************************** */
OSHMEM_DECLSPEC extern mca_atomic_base_component_t mca_atomic_base_selected_component;
OSHMEM_DECLSPEC extern mca_atomic_base_module_t mca_atomic;
#define MCA_ATOMIC_CALL(a) mca_atomic.atomic_ ## a
END_C_DECLS
#endif /* OSHMEM_MCA_ATOMIC_H */

20
oshmem/mca/atomic/base/Makefile.am
Просмотреть файл

@ -1,20 +0,0 @@
#
# Copyright (c) 2012 Mellanox Technologies, Inc.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
AM_CFLAGS = $(OSHMEM_CFLAGS)
headers += \
base/base.h
libmca_atomic_la_SOURCES += \
base/atomic_base_close.c \
base/atomic_base_available.c \
base/atomic_base_select.c \
base/atomic_base_open.c

180
oshmem/mca/atomic/base/atomic_base_available.c
Просмотреть файл

@ -1,180 +0,0 @@
/*
* Copyright (c) 2012 Mellanox Technologies, Inc.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "oshmem_config.h"
#include "orte/util/show_help.h"
#include "opal/class/opal_list.h"
#include "opal/util/output.h"
#include "opal/mca/mca.h"
#include "opal/mca/base/base.h"
#include "opal/mca/base/mca_base_component_repository.h"
#include "oshmem/constants.h"
#include "oshmem/mca/atomic/atomic.h"
#include "oshmem/mca/atomic/base/base.h"
/*
* Global variables
*/
bool mca_atomic_base_components_available_valid = false;
opal_list_t mca_atomic_base_components_available;
/*
* Private functions
*/
static int init_query(const mca_base_component_t * ls,
mca_base_component_priority_list_item_t * entry,
bool enable_progress_threads,
bool enable_threads);
/*
* Scan down the list of successfully opened components and query each of
* them (the opened list will be one or more components. If the user
* requested a specific component, it will be the only component in the
* opened list). Create and populate the available list of all
* components who indicate that they want to be considered for selection.
* Close all components who do not want to be considered for selection,
* and destroy the opened list.
*
* Also find the basic component while we're doing all of this, and save
* it in a global variable so that we can find it easily later (e.g.,
* during scope selection).
*/
int mca_atomic_base_find_available(bool enable_progress_threads,
bool enable_threads)
{
bool found = false;
mca_base_component_priority_list_item_t *entry;
opal_list_item_t *p;
const mca_base_component_t *component;
/* Initialize the list */
OBJ_CONSTRUCT(&mca_atomic_base_components_available, opal_list_t);
mca_atomic_base_components_available_valid = true;
/* The list of components that we should check has already been
established in mca_coll_base_open. */
for (found = false,
p = opal_list_remove_first(&mca_atomic_base_components_opened);
p != NULL;
p = opal_list_remove_first(&mca_atomic_base_components_opened)) {
component = ((mca_base_component_list_item_t *) p)->cli_component;
/* Call a subroutine to do the work, because the component may
represent different versions of the coll MCA. */
entry = OBJ_NEW(mca_base_component_priority_list_item_t);
entry->super.cli_component = component;
entry->cpli_priority = 0;
if (OSHMEM_SUCCESS == init_query(component, entry,
enable_progress_threads,
enable_threads)) {
opal_list_append(&mca_atomic_base_components_available,
(opal_list_item_t *) entry);
found = true;
} else {
/* If the component doesn't want to run, then close it.
It's already had its close() method invoked; now close
it out of the DSO repository (if it's there). */
mca_base_component_repository_release(component);
OBJ_RELEASE(entry);
}
/* Free the entry from the "opened" list */
OBJ_RELEASE(p);
}
/* The opened list is now no longer useful and we can free it */
OBJ_DESTRUCT(&mca_atomic_base_components_opened);
mca_atomic_base_components_opened_valid = false;
/* If we have no atomic components available, it's an error.
Thanks for playing! */
if (!found) {
/* Need to free all items in the list */
OBJ_DESTRUCT(&mca_atomic_base_components_available);
mca_atomic_base_components_available_valid = false;
ATOMIC_VERBOSE(10,"atomic:find_available: no atomic components available!");
return OSHMEM_ERROR;
}
/* All done */
return mca_atomic_base_select();
}
/*
* Query a component, see if it wants to run at all. If it does, save
* some information. If it doesn't, close it.
*/
static int init_query(const mca_base_component_t * component,
mca_base_component_priority_list_item_t * entry,
bool enable_progress_threads, bool enable_threads)
{
int ret;
ATOMIC_VERBOSE(10,"atomic:find_available: querying atomic component %s",
component->mca_component_name);
/* This component has already been successfully opened. So now
query it. */
if (1 == component->mca_type_major_version &&
0 == component->mca_type_minor_version &&
0 == component->mca_type_release_version) {
mca_atomic_base_component_t *atomic =
(mca_atomic_base_component_t *) component;
ret = atomic->atomic_init(enable_progress_threads,
enable_threads);
} else {
/* Unrecognized coll API version */
ATOMIC_VERBOSE(10,"atomic:find_available: unrecognized atomic API version (%d.%d.%d, ignored)",
component->mca_type_major_version,
component->mca_type_minor_version,
component->mca_type_release_version);
return OSHMEM_ERROR;
}
/* Query done -- look at the return value to see what happened */
if (OSHMEM_SUCCESS != ret) {
ATOMIC_VERBOSE(10,"atomic:find_available: atomic component %s is not available",
component->mca_component_name);
if (NULL != component->mca_close_component) {
component->mca_close_component();
}
} else {
ATOMIC_VERBOSE(10,"atomic:find_available: atomic component %s is available",
component->mca_component_name);
}
/* All done */
return ret;
}

50
oshmem/mca/atomic/base/atomic_base_close.c
Просмотреть файл

@ -1,50 +0,0 @@
/*
* Copyright (c) 2012 Mellanox Technologies, Inc.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include <stdio.h>
#include "oshmem_config.h"
#include "opal/mca/mca.h"
#include "opal/mca/base/base.h"
#include "oshmem/constants.h"
#include "oshmem/mca/atomic/atomic.h"
#include "oshmem/mca/atomic/base/base.h"
int mca_atomic_base_close(void)
{
int ret = OSHMEM_SUCCESS;
if (NULL != mca_atomic_base_selected_component.atomic_finalize) {
ret = mca_atomic_base_selected_component.atomic_finalize();
}
/* Close all components that are still open. This may be the opened
* list (if we're in ompi_info), or it may be the available list (if
* we're anywhere else). */
if (mca_atomic_base_components_opened_valid) {
mca_base_components_close(mca_atomic_base_output,
&mca_atomic_base_components_opened, NULL);
OBJ_DESTRUCT(&mca_atomic_base_components_opened);
mca_atomic_base_components_opened_valid = false;
} else if (mca_atomic_base_components_available_valid) {
mca_base_components_close(mca_atomic_base_output,
&mca_atomic_base_components_available,
NULL);
OBJ_DESTRUCT(&mca_atomic_base_components_available);
mca_atomic_base_components_available_valid = false;
}
/* All done */
return ret;
}

70
oshmem/mca/atomic/base/atomic_base_open.c
Просмотреть файл

@ -1,70 +0,0 @@
/*
* Copyright (c) 2012 Mellanox Technologies, Inc.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include <stdio.h>
#include "oshmem_config.h"
#include "opal/mca/mca.h"
#include "opal/util/output.h"
#include "opal/mca/base/base.h"
#include "opal/mca/base/mca_base_param.h"
#include "oshmem/constants.h"
#include "oshmem/mca/atomic/atomic.h"
#include "oshmem/mca/atomic/base/base.h"
/*
* The following file was created by configure. It contains extern
* statements and the definition of an array of pointers to each
* component's public mca_base_component_t struct.
*/
#include "oshmem/mca/atomic/base/static-components.h"
/*
* Global variables; most of which are loaded by back-ends of MCA
* variables
*/
int mca_atomic_base_output = -1;
bool mca_atomic_base_components_opened_valid = false;
opal_list_t mca_atomic_base_components_opened;
OBJ_CLASS_INSTANCE(mca_atomic_base_module_t, opal_object_t, NULL, NULL);
/*
* Function for finding and opening either all MCA components, or the one
* that was specifically requested via a MCA parameter.
*/
int mca_atomic_base_open(void)
{
/* Open an output stream for this framework */
int value = -1;
mca_atomic_base_output = opal_output_open(NULL);
mca_base_param_reg_int_name("atomic_base","verbose",
"Verbose level of the shmem atomic component",false,false,0,&value);
opal_output_set_verbosity(mca_atomic_base_output, value);
/* Open up all available components */
if (OSHMEM_SUCCESS !=
mca_base_components_open("atomic", mca_atomic_base_output,
mca_atomic_base_static_components,
&mca_atomic_base_components_opened, true)) {
return OSHMEM_ERROR;
}
mca_atomic_base_components_opened_valid = true;
/* All done */
return OSHMEM_SUCCESS;
}

259
oshmem/mca/atomic/base/atomic_base_select.c
Просмотреть файл

@ -1,259 +0,0 @@
/*
* Copyright (c) 2012 Mellanox Technologies, Inc.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "oshmem_config.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "oshmem/constants.h"
#include "opal/class/opal_list.h"
#include "opal/util/output.h"
#include "orte/util/show_help.h"
#include "opal/mca/mca.h"
#include "opal/mca/base/base.h"
#include "opal/mca/base/mca_base_component_repository.h"
#include "oshmem/mca/atomic/atomic.h"
#include "oshmem/mca/atomic/base/base.h"
/*
* Global variables; most of which are loaded by back-ends of MCA
* variables
*/
mca_atomic_base_module_t mca_atomic;
mca_atomic_base_component_t mca_atomic_base_selected_component;
/*
* Local types
*/
struct avail_atomic_t {
opal_list_item_t super;
int ac_priority;
mca_atomic_base_component_t *ac_component;
mca_atomic_base_module_t *ac_module;
};
typedef struct avail_atomic_t avail_atomic_t;
/*
* Local functions
*/
static opal_list_t *check_components(opal_list_t * components);
static int check_one_component(const mca_base_component_t * component,
mca_atomic_base_module_1_0_0_t ** module);
static int query(const mca_base_component_t * component,
int *priority,
mca_atomic_base_module_1_0_0_t ** module);
static int query_1_0_0(const mca_atomic_base_component_1_0_0_t * atomic_component,
int *priority,
mca_atomic_base_module_1_0_0_t ** module);
/*
* Stuff for the OBJ interface
*/
static OBJ_CLASS_INSTANCE(avail_atomic_t, opal_list_item_t, NULL, NULL);
/*
* This function is called at the initialization.
* It is used to select which atomic component will be
* active for a given group.
*/
int mca_atomic_base_select(void)
{
opal_list_t *selectable;
opal_list_item_t *item;
/* Announce */
ATOMIC_VERBOSE(10,"atomic:base:atomic_select: Checking all available modules");
selectable = check_components(&mca_atomic_base_components_available);
/* Upon return from the above, the modules list will contain the
list of modules that returned (priority >= 0). If we have no
atomic modules available, then print error and return. */
if (NULL == selectable) {
/* There's no modules available */
return OSHMEM_ERROR;
}
/* do the selection loop */
for (item = opal_list_remove_first(selectable);
NULL != item; item = opal_list_remove_first(selectable))
{
avail_atomic_t *avail = (avail_atomic_t *)item;
/* Set module having the highest priority */
memcpy(&mca_atomic, avail->ac_module, sizeof(mca_atomic));
memcpy(&mca_atomic_base_selected_component, avail->ac_component, sizeof(mca_atomic_base_selected_component));
OBJ_RELEASE(avail->ac_module);
OBJ_RELEASE(avail);
/* check correctness */
if ( !(mca_atomic.atomic_fadd) ||
!(mca_atomic.atomic_cswap) )
{
return OSHMEM_ERR_NOT_FOUND;
}
}
/* Done with the list from the check_components() call so release it. */
OBJ_RELEASE(selectable);
return OSHMEM_SUCCESS;
}
/*
* For each module in the list, check and see if it wants to run, and
* do the resulting priority comparison. Make a list of modules to be
* only those who returned that they want to run, and put them in
* priority order.
*/
static opal_list_t *check_components(opal_list_t * components)
{
int priority;
const mca_base_component_t *component;
opal_list_item_t *item, *item2;
mca_atomic_base_module_1_0_0_t *module;
opal_list_t *selectable;
avail_atomic_t *avail, *avail2;
/* Make a list of the components that query successfully */
selectable = OBJ_NEW(opal_list_t);
/* Scan through the list of components. This nested loop is
O(N^2), but we should never have too many components, so this
*hopefully* shouldn't matter... */
for (item = opal_list_get_first(components);
((item != opal_list_get_end(components)) && (item != NULL));
item = opal_list_get_next(item)) {
component = ((mca_base_component_priority_list_item_t *)
item)->super.cli_component;
priority = check_one_component(component, &module);
if (priority >= 0) {
/* We have a component that indicated that it wants to run
by giving us a module */
avail = OBJ_NEW(avail_atomic_t);
avail->ac_priority = priority;
avail->ac_module = module;
avail->ac_component = (mca_atomic_base_component_t *)component;
/* Put this item on the list in priority order (lowest
priority first). Should it go first? */
for (item2 = opal_list_get_first(selectable);
item2 != opal_list_get_end(selectable);
item2 = opal_list_get_next(item2)) {
avail2 = (avail_atomic_t *) item2;
if (avail->ac_priority < avail2->ac_priority) {
opal_list_insert_pos(selectable,
item2,
(opal_list_item_t *) avail);
break;
}
}
if (opal_list_get_end(selectable) == item2) {
opal_list_append(selectable,
(opal_list_item_t *) avail);
}
}
}
/*TODO: copy over any of the pointers */
/* If we didn't find any available components, return an error */
if (0 == opal_list_get_size(selectable)) {
OBJ_RELEASE(selectable);
return NULL;
}
/* All done */
return selectable;
}
/*
* Check a single component
*/
static int check_one_component(const mca_base_component_t * component,
mca_atomic_base_module_1_0_0_t ** module)
{
int err;
int priority = -1;
err = query(component, &priority, module);
if (OSHMEM_SUCCESS == err) {
priority = (priority < 100) ? priority : 100;
ATOMIC_VERBOSE(10,"atomic:base:atomic_select: component available: %s, priority: %d",
component->mca_component_name, priority);
} else {
priority = -1;
ATOMIC_VERBOSE(10,"atomic:base:atomic_select: component not available: %s",
component->mca_component_name);
}
return priority;
}
/**************************************************************************
* Query functions
**************************************************************************/
/*
* Take any version of a atomic module, query it, and return the right
* module struct
*/
static int query(const mca_base_component_t * component,
int *priority, mca_atomic_base_module_1_0_0_t ** module)
{
*module = NULL;
if (1 == component->mca_type_major_version &&
0 == component->mca_type_minor_version &&
0 == component->mca_type_release_version) {
const mca_atomic_base_component_1_0_0_t *atomic100 =
(mca_atomic_base_component_1_0_0_t *) component;
return query_1_0_0(atomic100, priority, module);
}
/* Unknown atomic API version -- return error */
return OSHMEM_ERROR;
}
static int query_1_0_0(const mca_atomic_base_component_1_0_0_t * component,
int *priority,
mca_atomic_base_module_1_0_0_t ** module)
{
mca_atomic_base_module_1_0_0_t *ret;
/* There's currently no need for conversion */
ret = component->atomic_query(priority);
if (NULL != ret) {
*module = ret;
return OSHMEM_SUCCESS;
}
return OSHMEM_ERROR;
}

123
oshmem/mca/atomic/base/base.h
Просмотреть файл

@ -1,123 +0,0 @@
/*
* Copyright (c) 2012 Mellanox Technologies, Inc.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef MCA_ATOMIC_BASE_H
#define MCA_ATOMIC_BASE_H
#include "oshmem_config.h"
#include "oshmem/mca/atomic/atomic.h"
#include "opal/class/opal_list.h"
/*
* Global functions for MCA overall atomic open and close
*/
BEGIN_C_DECLS
/**
* Initialize the atomic MCA framework
*
* @retval OSHEM_SUCCESS Upon success
* @retval OSHMEM_ERROR Upon failure
*
*/
OSHMEM_DECLSPEC int mca_atomic_base_open(void);
/**
* Create list of available atomic components.
*
* @param allow_multi_user_threads Will be set to true if any of the
* available components will allow multiple user threads
* @param have_hidden_threads Will be set to true if any of the
* available components have hidden threads.
*
* @retval OSHMEM_SUCCESS If one or more atomic components are available.
* @retval OSHMEM_ERROR If no atomic components are found to be available.
*
*/
int mca_atomic_base_find_available(bool enable_progress_threads,
bool enable_threads);
int mca_atomic_base_select(void);
/**
* Shut down the atomic MCA framework.
*
* @retval OSHMEM_SUCCESS Always
*
* This function shuts down everything in the atomic MCA framework,
* and is called during oshmem_shmem_finalize().
*
* It must be the last function invoked on the atomic MCA framework.
*/
OSHMEM_DECLSPEC int mca_atomic_base_close(void);
/*
* Globals
*/
/**
* ATOMIC framework debugging stream ID used with opal_output() and
* opal_output_verbose().
*/
OSHMEM_DECLSPEC extern int mca_atomic_base_output;
/**
* Indicator as to whether the list of opened atomic components is valid or
* not.
*/
extern bool mca_atomic_base_components_opened_valid;
/**
* List of all opened components; created when the atomic framework is
* initialized and destroyed when we reduce the list to all available
* coll components.
*/
OSHMEM_DECLSPEC extern opal_list_t mca_atomic_base_components_opened;
/**
* Indicator as to whether the list of available atomic components is valid
* or not.
*/
extern bool mca_atomic_base_components_available_valid;
/**
* List of all available components; created by reducing the list of open
* components to all those who indicate that they may run during this
* process.
*/
extern opal_list_t mca_atomic_base_components_available;
/* ******************************************************************** */
#ifdef __BASE_FILE__
#define __ATOMIC_FILE__ __BASE_FILE__
#else
#define __ATOMIC_FILE__ __FILE__
#endif
#define ATOMIC_VERBOSE(level, format, ...) \
opal_output_verbose(level, mca_atomic_base_output, "%s:%d - %s() " format, \
__ATOMIC_FILE__, __LINE__, __FUNCTION__, ## __VA_ARGS__)
#define ATOMIC_ERROR(format, ... ) \
opal_output_verbose(0, mca_atomic_base_output, "Error: %s:%d - %s() " format, \
__SCOLL_FILE__, __LINE__, __FUNCTION__, ## __VA_ARGS__)
END_C_DECLS
#endif /* MCA_ATOMIC_BASE_H */

12
oshmem/mca/atomic/basic/.windows
Просмотреть файл

@ -1,12 +0,0 @@
#
# Copyright (c) 2012 Mellanox Technologies, Inc.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# Specific to this module
mca_link_libraries=libshmem

42
oshmem/mca/atomic/basic/Makefile.am
Просмотреть файл

@ -1,42 +0,0 @@
#
# Copyright (c) 2012 Mellanox Technologies, Inc.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
EXTRA_DIST = .windows
AM_CFLAGS = $(OSHMEM_CFLAGS)
sources = \
atomic_basic.h \
atomic_basic_module.c \
atomic_basic_component.c \
atomic_basic_fadd.c \
atomic_basic_cswap.c
# Make the output library in this directory, and name it either
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
# (for static builds).
if MCA_BUILD_oshmem_atomic_basic_DSO
component_noinst =
component_install = mca_atomic_basic.la
else
component_noinst = libmca_atomic_basic.la
component_install =
endif
mcacomponentdir = $(pkglibdir)
mcacomponent_LTLIBRARIES = $(component_install)
mca_atomic_basic_la_SOURCES = $(sources)
mca_atomic_basic_la_LDFLAGS = -module -avoid-version
noinst_LTLIBRARIES = $(component_noinst)
libmca_atomic_basic_la_SOURCES =$(sources)
libmca_atomic_basic_la_LDFLAGS = -module -avoid-version

51
oshmem/mca/atomic/basic/atomic_basic.h
Просмотреть файл

@ -1,51 +0,0 @@
/*
* Copyright (c) 2012 Mellanox Technologies, Inc.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef MCA_ATOMIC_BASIC_H
#define MCA_ATOMIC_BASIC_H
#include "oshmem_config.h"
#include "opal/mca/mca.h"
#include "oshmem/mca/atomic/atomic.h"
BEGIN_C_DECLS
/* Globally exported variables */
OSHMEM_MODULE_DECLSPEC extern mca_atomic_base_component_1_0_0_t
mca_atomic_basic_component;
extern int mca_atomic_basic_priority_param;
OSHMEM_DECLSPEC void atomic_basic_lock(int pe);
OSHMEM_DECLSPEC void atomic_basic_unlock(int pe);
/* API functions */
int mca_atomic_basic_init(bool enable_progress_threads,
bool enable_threads);
int mca_atomic_basic_finalize(void);
mca_atomic_base_module_t*
mca_atomic_basic_query(int *priority);
int mca_atomic_basic_fadd(void *target, void *prev, const void *value, size_t nlong, int pe, struct oshmem_op_t *op);
int mca_atomic_basic_cswap(void *target, void *prev, const void *cond, const void *value, size_t nlong, int pe);
struct mca_atomic_basic_module_t {
mca_atomic_base_module_t super;
};
typedef struct mca_atomic_basic_module_t mca_atomic_basic_module_t;
OBJ_CLASS_DECLARATION(mca_atomic_basic_module_t);
END_C_DECLS
#endif /* MCA_ATOMIC_BASIC_H */

88
oshmem/mca/atomic/basic/atomic_basic_component.c
Просмотреть файл

@ -1,88 +0,0 @@
/*
* Copyright (c) 2012 Mellanox Technologies, Inc.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "oshmem_config.h"
#include "oshmem/constants.h"
#include "oshmem/mca/atomic/atomic.h"
#include "oshmem/mca/atomic/base/base.h"
#include "atomic_basic.h"
/*
* Public string showing the scoll basic component version number
*/
const char *mca_atomic_basic_component_version_string =
"Open SHMEM basic atomic MCA component version " OSHMEM_VERSION;
/*
* Global variable
*/
int mca_atomic_basic_priority_param = -1;
/*
* Local function
*/
static int __basic_open(void);
/*
* Instantiate the public struct with all of our public information
* and pointers to our public functions in it
*/
mca_atomic_base_component_t mca_atomic_basic_component = {
/* First, the mca_component_t struct containing meta information
about the component itself */
{
MCA_ATOMIC_BASE_VERSION_2_0_0,
/* Component name and version */
"basic",
OSHMEM_MAJOR_VERSION,
OSHMEM_MINOR_VERSION,
OSHMEM_RELEASE_VERSION,
/* Component open and close functions */
__basic_open,
NULL
},
{
/* The component is checkpoint ready */
MCA_BASE_METADATA_PARAM_CHECKPOINT
},
/* Initialization / querying functions */
mca_atomic_basic_init,
mca_atomic_basic_finalize,
mca_atomic_basic_query
};
static int __basic_open(void)
{
/* We'll always be picked if there's only one process in the
communicator */
int default_value = 75;
int param_value = default_value;
mca_atomic_basic_priority_param =
mca_base_param_reg_int(&mca_atomic_basic_component.atomic_version, "priority", NULL, false, false, default_value, &param_value);
return OSHMEM_SUCCESS;
}
OBJ_CLASS_INSTANCE(mca_atomic_basic_module_t,
mca_atomic_base_module_t,
NULL, NULL);

47
oshmem/mca/atomic/basic/atomic_basic_cswap.c
Просмотреть файл

@ -1,47 +0,0 @@
/*
* Copyright (c) 2012 Mellanox Technologies, Inc.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "oshmem_config.h"
#include <stdio.h>
#include <stdlib.h>
#include "oshmem/constants.h"
#include "oshmem/mca/spml/spml.h"
#include "oshmem/mca/atomic/atomic.h"
#include "oshmem/mca/atomic/base/base.h"
#include "atomic_basic.h"
int mca_atomic_basic_cswap(void *target, void *prev, const void *cond, const void *value, size_t nlong, int pe)
{
int rc = OSHMEM_SUCCESS;
if (!prev)
{
rc = OSHMEM_ERROR;
}
if ( rc == OSHMEM_SUCCESS )
{
atomic_basic_lock(pe);
rc = MCA_SPML_CALL(get(target, nlong, prev, pe));
if (( rc == OSHMEM_SUCCESS ) && (!cond || !memcmp(prev, cond, nlong)))
{
rc = MCA_SPML_CALL(put(target, nlong, (void*)value, pe));
shmem_quiet();
}
atomic_basic_unlock(pe);
}
return rc;
}

54
oshmem/mca/atomic/basic/atomic_basic_fadd.c
Просмотреть файл

@ -1,54 +0,0 @@
/*
* Copyright (c) 2012 Mellanox Technologies, Inc.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "oshmem_config.h"
#include <stdio.h>
#include <stdlib.h>
#include "oshmem/constants.h"
#include "oshmem/op/op.h"
#include "oshmem/mca/spml/spml.h"
#include "oshmem/mca/atomic/atomic.h"
#include "oshmem/mca/atomic/base/base.h"
#include "atomic_basic.h"
int mca_atomic_basic_fadd(void *target, void *prev, const void *value, size_t nlong, int pe, struct oshmem_op_t *op)
{
int rc = OSHMEM_SUCCESS;
if (!target || !value)
{
rc = OSHMEM_ERROR;
}
if ( rc == OSHMEM_SUCCESS )
{
long long temp_value = 0;
atomic_basic_lock(pe);
rc = MCA_SPML_CALL(get(target, nlong, (void*)&temp_value, pe));
if (prev) memcpy(prev, (void*)&temp_value, nlong);
op->o_func.c_fn((void*)value, (void*)&temp_value, nlong / op->dt_size);
if (rc == OSHMEM_SUCCESS)
{
rc = MCA_SPML_CALL(put(target, nlong, (void*)&temp_value, pe));
shmem_quiet();
}
atomic_basic_unlock(pe);
}
return rc;
}

211
oshmem/mca/atomic/basic/atomic_basic_module.c
Просмотреть файл

@ -1,211 +0,0 @@
/*
* Copyright (c) 2012 Mellanox Technologies, Inc.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "oshmem_config.h"
#include <stdio.h>
#include "opal/mca/base/mca_base_param.h"
#include "oshmem/constants.h"
#include "oshmem/mca/atomic/atomic.h"
#include "oshmem/mca/spml/spml.h"
#include "oshmem/mca/memheap/memheap.h"
#include "oshmem/proc/proc.h"
#include "atomic_basic.h"
static char *atomic_lock_sync;
static int *atomic_lock_turn;
static char *local_lock_sync;
static int *local_lock_turn;
enum
{
ATOMIC_LOCK_IDLE = 0,
ATOMIC_LOCK_WAITING = 1,
ATOMIC_LOCK_ACTIVE = 2
};
/*
* Initial query function that is invoked during initialization, allowing
* this module to indicate what level of thread support it provides.
*/
int mca_atomic_basic_init(bool enable_progress_threads,
bool enable_threads)
{
int rc = OSHMEM_SUCCESS;
void* ptr = NULL;
int num_pe = oshmem_num_procs();
rc = MCA_MEMHEAP_CALL(private_alloc((num_pe * sizeof(char)), &ptr));
if(rc == OSHMEM_SUCCESS)
{
atomic_lock_sync = (char*)ptr;
memset(atomic_lock_sync, ATOMIC_LOCK_IDLE, sizeof(char) * num_pe);
rc = MCA_MEMHEAP_CALL(private_alloc(sizeof(int), &ptr));
if(rc == OSHMEM_SUCCESS)
{
atomic_lock_turn = (int*)ptr;
*atomic_lock_turn = 0;
if(rc == OSHMEM_SUCCESS)
{
local_lock_sync = (char*)malloc(num_pe * sizeof(char));
local_lock_turn = (int*)malloc(sizeof(int));
if (!local_lock_sync || !local_lock_turn)
{
rc = OSHMEM_ERR_OUT_OF_RESOURCE;
}
else
{
memcpy((void*)local_lock_sync, (void*)atomic_lock_sync, sizeof(char) * num_pe);
*local_lock_turn = *atomic_lock_turn;
}
}
}
}
return rc;
}
int mca_atomic_basic_finalize(void)
{
void* ptr = NULL;
ptr = (void*)atomic_lock_sync;
MCA_MEMHEAP_CALL(private_free(ptr));
atomic_lock_sync = NULL;
ptr = (void*)atomic_lock_turn;
MCA_MEMHEAP_CALL(private_free(ptr));
atomic_lock_turn = NULL;
if (local_lock_sync)
{
free((void*)local_lock_sync);
local_lock_sync = NULL;
}
if (local_lock_turn)
{
free((void*)local_lock_turn);
local_lock_turn = NULL;
}
return OSHMEM_SUCCESS;
}
mca_atomic_base_module_t *
mca_atomic_basic_query(int *priority)
{
mca_atomic_basic_module_t *module;
if (OSHMEM_SUCCESS ==
mca_base_param_lookup_int(mca_atomic_basic_priority_param,
priority))
{
module = OBJ_NEW(mca_atomic_basic_module_t);
if (module)
{
module->super.atomic_fadd = mca_atomic_basic_fadd;
module->super.atomic_cswap = mca_atomic_basic_cswap;
return &(module->super);
}
}
return NULL;
}
void atomic_basic_lock(int pe)
{
int rc = OSHMEM_SUCCESS;
int index = -1;
int me = oshmem_my_proc_id();
int num_pe = oshmem_num_procs();
char lock_required = ATOMIC_LOCK_WAITING;
char lock_active = ATOMIC_LOCK_ACTIVE;
int root_pe = pe;
do
{
/* announce that we need the resource */
do
{
rc = MCA_SPML_CALL(put((void*)(atomic_lock_sync + me), sizeof(lock_required), (void*)&lock_required, root_pe));
//MCA_SPML_CALL(fence()); /* quiet */
MCA_SPML_CALL(get((void*)atomic_lock_sync, num_pe * sizeof(*atomic_lock_sync), (void*)local_lock_sync, root_pe));
} while (local_lock_sync[me] != lock_required);
MCA_SPML_CALL(get((void*)atomic_lock_turn, sizeof(index), (void*)&index, root_pe));
while (index != me)
{
if (local_lock_sync[index] != ATOMIC_LOCK_IDLE)
{
MCA_SPML_CALL(get((void*)atomic_lock_turn, sizeof(index), (void*)&index, root_pe));
MCA_SPML_CALL(get((void*)atomic_lock_sync, num_pe * sizeof(*atomic_lock_sync), (void*)local_lock_sync, root_pe));
}
else
{
index = (index + 1) % num_pe;
}
}
/* now tentatively claim the resource */
do
{
rc = MCA_SPML_CALL(put((void*)(atomic_lock_sync + me), sizeof(lock_active), (void*)&lock_active, root_pe));
//MCA_SPML_CALL(fence()); /* quiet */
MCA_SPML_CALL(get((void*)atomic_lock_sync, num_pe * sizeof(*atomic_lock_sync), (void*)local_lock_sync, root_pe));
} while (local_lock_sync[me] != lock_active);
index = 0;
while((index < num_pe)&&
((index == me)||(local_lock_sync[index] != ATOMIC_LOCK_ACTIVE)))
{
index = index + 1;
}
MCA_SPML_CALL(get((void*)atomic_lock_turn, sizeof(*atomic_lock_turn), (void*)local_lock_turn, root_pe));
} while (!((index >= num_pe) &&
((*local_lock_turn == me)||(local_lock_sync[*local_lock_turn] == ATOMIC_LOCK_IDLE))));
rc = MCA_SPML_CALL(put((void*)atomic_lock_turn, sizeof(me), (void*)&me, root_pe));
//MCA_SPML_CALL(fence()); /* quiet */
}
void atomic_basic_unlock(int pe)
{
int rc = OSHMEM_SUCCESS;
int index = -1;
int me = oshmem_my_proc_id();
int num_pe = oshmem_num_procs();
char lock_idle = ATOMIC_LOCK_IDLE;
int root_pe = pe;
MCA_SPML_CALL(get((void*)atomic_lock_sync, num_pe * sizeof(*atomic_lock_sync), (void*)local_lock_sync, root_pe));
MCA_SPML_CALL(get((void*)atomic_lock_turn, sizeof(index), (void*)&index, root_pe));
do
{
index = (index + 1) % num_pe;
} while (local_lock_sync[index] == ATOMIC_LOCK_IDLE);
rc = MCA_SPML_CALL(put((void*)atomic_lock_turn, sizeof(index), (void*)&index, root_pe));
do
{
rc = MCA_SPML_CALL(put((void*)(atomic_lock_sync + me), sizeof(lock_idle), (void*)&lock_idle, root_pe));
//MCA_SPML_CALL(fence()); /* quiet */
MCA_SPML_CALL(get((void*)atomic_lock_sync, num_pe * sizeof(*atomic_lock_sync), (void*)local_lock_sync, root_pe));
} while (local_lock_sync[me] != lock_idle);
}

13
oshmem/mca/atomic/basic/configure.params
Просмотреть файл

@ -1,13 +0,0 @@
# -*- shell-script -*-
# Copyright (c) 2012 Mellanox Technologies, Inc.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# Specific to this module
PARAM_CONFIG_FILES="Makefile"

39
oshmem/mca/memheap/Makefile.am
Просмотреть файл

@ -1,39 +0,0 @@
#
# Copyright (c) 2012 Mellanox Technologies, Inc.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# main library setup
noinst_LTLIBRARIES = libmca_memheap.la
libmca_memheap_la_SOURCES =
libmca_memheap_la_LDFLAGS =
libmca_memheap_la_LIBADD =
# header setup
nobase_oshmem_HEADERS =
nobase_nodist_oshmem_HEADERS =
dist_pkgdata_DATA =
# local files
headers = memheap.h
libmca_memheap_la_SOURCES += $(headers) $(nodist_headers)
# Conditionally install the header files
if WANT_INSTALL_HEADERS
nobase_oshmem_HEADERS += $(headers)
nobase_nodist_oshmem_HEADERS += $(nodist_headers)
oshmemdir = $(includedir)/oshmem/oshmem/mca/memheap
else
oshmemdir = $(includedir)
endif
include base/Makefile.am
distclean-local:
rm -f base/static-components.h

50
oshmem/mca/memheap/README
Просмотреть файл

@ -1,50 +0,0 @@
# Copyright (c) 2012 Mellanox Technologies, Inc.
# All rights reserved
# $COPYRIGHT$
MEMHEAP Infrustructure documentation
------------------------------------
MEMHEAP Infrustructure is responsible for managing the symmetric heap.
The framework currently has following components: buddy and ptmalloc. buddy which uses a buddy allocator in order to manage the Memory allocations on the symmetric heap. Ptmalloc is an adaptation of ptmalloc3.
Additional components may be added easily to the framework by defining the component's and the module's base and extended structures, and their funtionalities.
The buddy allocator has the following data structures:
1. Base component - of type struct mca_memheap_base_component_2_0_0_t
2. Base module - of type struct mca_memheap_base_module_t
3. Buddy component - of type struct mca_memheap_base_component_2_0_0_t
4. Buddy module - of type struct mca_memheap_buddy_module_t extending the base module (struct mca_memheap_base_module_t)
Each data structure includes the following fields:
1. Base component - memheap_version, memheap_data and memheap_init
2. Base module - Holds pointers to the base component and to the functions: alloc, free and finalize
3. Buddy component - is a base component.
4. Buddy module - Extends the base module and holds additional data on the components's priority, buddy allocator,
maximal order of the symmetric heap, symmetric heap, pointer to the symmetric heap and hashtable maintaining the size of each allocated address.
In the case that the user decides to implement additional components, the Memheap infrastructure chooses a component with the maximal priority.
Handling the component opening is done under the base directory, in three stages:
1. Open all available components. Implemented by memheap_base_open.c and called from shmem_init.
2. Select the maximal priority component. This procedure involves the initialization of all components and then their
finalization except to the chosen component. It is implemented by memheap_base_select.c and called from shmem_init.
3. Close the max priority active cmponent. Implemented by memheap_base_close.c and called from shmem finalize.
Buddy Component/Module
----------------------
Responsible for handling the entire activities of the symmetric heap.
The supported activities are:
- buddy_init (Initialization)
- buddy_alloc (Allocates a variable on the symmetric heap)
- buddy_free (frees a variable previously allocated on the symetric heap)
- buddy_finalize (Finalization).
Data members of buddy module: - priority. The module's priority.
- buddy allocator: bits, num_free, lock and the maximal order (log2 of the maximal size)
of a variable on the symmetric heap. Buddy Allocator gives the offset in the symmetric heap
where a variable should be allocated.
- symmetric_heap: a range of reserved addresses (equal in all executing PE's) dedicated to "shared memory" allocation.
- symmetric_heap_hashtable (holding the size of an allocated variable on the symmetric heap.
used to free an allocated variable on the symmetric heap)

28
oshmem/mca/memheap/base/Makefile.am
Просмотреть файл

@ -1,28 +0,0 @@
# Copyright (c) 2012 Mellanox Technologies, Inc.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
AM_CFLAGS = $(OSHMEM_CFLAGS)
AM_CPPFLAGS = $(openib_CPPFLAGS)
dist_pkgdata_DATA += base/help-shmem-mca.txt
headers += \
base/base.h
libmca_memheap_la_SOURCES += \
base/memheap_base_open.c \
base/memheap_base_select.c \
base/memheap_base_alloc.c \
base/memheap_base_static.c \
base/memheap_base_register.c \
base/memheap_base_mkey.c \
base/memheap_base_close.c
libmca_memheap_la_LDFLAGS += -module -avoid-version $(openib_LDFLAGS)
libmca_memheap_la_LIBADD += $(openib_LIBS)

200
oshmem/mca/memheap/base/base.h
Просмотреть файл

@ -1,200 +0,0 @@
/*
* Copyright (c) 2012 Mellanox Technologies, Inc.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*/
#ifndef MCA_MEMHEAP_BASE_H
#define MCA_MEMHEAP_BASE_H
#include "oshmem_config.h"
#include "opal/class/opal_list.h"
#include "opal/class/opal_value_array.h"
#include "opal/mca/mca.h"
#include "oshmem/mca/memheap/memheap.h"
BEGIN_C_DECLS
/*
* Global functions for MCA: overall MEMHEAP open and close
*/
OSHMEM_DECLSPEC int mca_memheap_base_open(void);
OSHMEM_DECLSPEC int mca_memheap_base_select(void);
OSHMEM_DECLSPEC int mca_memheap_base_close(void);
/*
* Globals
*/
OSHMEM_DECLSPEC extern int mca_memheap_base_output;
OSHMEM_DECLSPEC extern opal_list_t mca_memheap_base_components_opened;
OSHMEM_DECLSPEC extern struct mca_memheap_base_module_t* mca_memheap_base_module_initialized;
/* only used within base -- no need to DECLSPEC */
#define MEMHEAP_BASE_START_ADDRESS 0xFF000000
#define MEMHEAP_BASE_MIN_ORDER 3 /* forces 64 bit alignment */
#define MEMHEAP_BASE_PAGE_ORDER 21
#define MEMHEAP_BASE_PRIVATE_SIZE (1ULL << MEMHEAP_BASE_PAGE_ORDER) /* should be at least the same as a huge page size */
#define MEMHEAP_BASE_MIN_SIZE (1ULL << MEMHEAP_BASE_PAGE_ORDER) /* must fit into at least one huge page */
extern char* mca_memheap_base_include;
extern char* mca_memheap_base_exclude;
extern int mca_memheap_base_already_opened;
extern int mca_memheap_base_shmalloc_use_hugepages;
extern int mca_memheap_buddy_use_modex;
extern int mca_memheap_base_mr_interleave_factor;
#define MCA_MEMHEAP_MAX_SEGMENTS 256
#define HEAP_SEG_INDEX 0
#define SYMB_SEG_INDEX 1
#define MEMHEAP_SHM_INVALID (-1)
#define MEMHEAP_SHM_CODE( type, id ) ((((uint64_t)(type)) << 32) | ((uint32_t)(id)))
#define MEMHEAP_SHM_GET_TYPE( x ) (((uint32_t)((x) >> 32)) & 0xFFFFFFFF)
#define MEMHEAP_SHM_GET_ID( x ) ((uint32_t)((x) & 0xFFFFFFFF))
typedef enum {
MAP_SEGMENT_STATIC = 0,
MAP_SEGMENT_ALLOC_MMAP,
MAP_SEGMENT_ALLOC_SHM,
MAP_SEGMENT_ALLOC_IBV,
MAP_SEGMENT_UNKNOWN
} segment_type_t;
#if defined(MPAGE_ENABLE) && (MPAGE_ENABLE > 0)
#include <infiniband/verbs.h>
typedef struct openib_device_t {
struct ibv_device **ib_devs;
struct ibv_device *ib_dev;
struct ibv_context *ib_dev_context;
struct ibv_device_attr ib_dev_attr;
struct ibv_pd *ib_pd;
opal_value_array_t ib_mr_array;;
struct ibv_mr *ib_mr_shared;
} openib_device_t;
#endif /* MPAGE_ENABLE */
typedef struct map_segment_t {
mca_spml_mkey_t **mkeys_cache; /* includes remote segment bases in va_base */
mca_spml_mkey_t *mkeys; /* includes local segment bases in va_base */
int is_active; /* enable/disable flag */
int shmid;
uint64_t start; /* base address of the segment */
uint64_t end; /* final address of the segment */
size_t size; /* length of the segment */
segment_type_t type; /* type of the segment */
void *context; /* additional data related the segment */
} map_segment_t;
typedef struct mca_memheap_map {
map_segment_t mem_segs[MCA_MEMHEAP_MAX_SEGMENTS]; /* TODO: change into pointer array */
int n_segments;
int num_transports;
} mca_memheap_map_t;
extern mca_memheap_map_t mca_memheap_base_map;
OSHMEM_DECLSPEC int mca_memheap_base_alloc_init(mca_memheap_map_t *, size_t);
OSHMEM_DECLSPEC void mca_memheap_base_alloc_exit(mca_memheap_map_t *);
OSHMEM_DECLSPEC int mca_memheap_base_static_init(mca_memheap_map_t *);
OSHMEM_DECLSPEC void mca_memheap_base_static_exit(mca_memheap_map_t *);
OSHMEM_DECLSPEC int mca_memheap_base_register(mca_memheap_map_t *);
OSHMEM_DECLSPEC int mca_memheap_base_deregister(mca_memheap_map_t *);
OSHMEM_DECLSPEC int memheap_oob_init(mca_memheap_map_t *);
OSHMEM_DECLSPEC void memheap_oob_destruct(void);
OSHMEM_DECLSPEC uint64_t mca_memheap_base_find_offset(int pe, int tr_id, unsigned long va, uint64_t rva);
OSHMEM_DECLSPEC int mca_memheap_base_is_symmetric_addr(unsigned long va);
OSHMEM_DECLSPEC mca_spml_mkey_t *mca_memheap_base_get_mkey(unsigned long va, int tr_id);
OSHMEM_DECLSPEC mca_spml_mkey_t * mca_memheap_base_get_cached_mkey(int pe, unsigned long va, int btl_id, uint64_t *rva);
OSHMEM_DECLSPEC void mca_memheap_modex_recv_all(void);
/* This function is for internal usage only
* return value:
* 0 - addr is not symmetric address
* 1 - addr is part of user memheap
* 2 - addr is part of private memheap
* 3 - addr is static variable
*/
typedef enum {
ADDR_INVALID = 0,
ADDR_USER,
ADDR_PRIVATE,
ADDR_STATIC,
} addr_type_t;
OSHMEM_DECLSPEC int mca_memheap_base_detect_addr_type(unsigned long va);
static inline unsigned memheap_log2(unsigned long long val)
{
/* add 1 if val is NOT a power of 2 (to do the ceil) */
unsigned int count = (val & (val-1) ? 1 : 0);
while(val > 0)
{
val = val >> 1;
count++;
}
return count > 0 ? count-1: 0;
}
static inline void *memheap_down_align_addr(void* addr, unsigned int shift)
{
return (void*) (((intptr_t) addr) & (~(intptr_t) 0) << shift);
}
static inline void *memheap_up_align_addr(void*addr, unsigned int shift)
{
return (void*) ((((intptr_t) addr) | ~((~(intptr_t) 0) << shift)));
}
static inline unsigned long long memheap_align(unsigned long top)
{
return ((top + MEMHEAP_BASE_MIN_SIZE - 1) & ~(MEMHEAP_BASE_MIN_SIZE - 1));
}
/*----------------------------------------------------------------------------------*/
/*logger macros*/
#ifdef __BASE_FILE__
#define __SPML_FILE__ __BASE_FILE__
#else
#define __SPML_FILE__ __FILE__
#endif
#define MEMHEAP_VERBOSE(level, format, ...) \
opal_output_verbose(level, mca_memheap_base_output, "%s:%d - %s() " format, \
__SPML_FILE__, __LINE__, __FUNCTION__, ## __VA_ARGS__)
#define MEMHEAP_ERROR(format, ... ) \
opal_output_verbose(0, mca_memheap_base_output, "Error: %s:%d - %s() " format, \
__SPML_FILE__, __LINE__, __FUNCTION__, ## __VA_ARGS__)
#define MEMHEAP_WARN(format, ... ) \
opal_output_verbose(0, mca_memheap_base_output, "Warning: %s:%d - %s() " format, \
__SPML_FILE__, __LINE__, __FUNCTION__, ## __VA_ARGS__)
END_C_DECLS
#endif /* MCA_MEMHEAP_BASE_H */

23
oshmem/mca/memheap/base/help-shmem-mca.txt
Просмотреть файл

@ -1,23 +0,0 @@
# -*- text -*-
#
# Copyright (c) 2012 Mellanox Technologies, Inc.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# This is the US/English help file for Open SHMEM MCA error messages.
#
[find-available:none-found]
No available %s components were found!
This means that there are no components of this type installed on your
system or all the components reported that they could not be used.
This is a fatal error; your SHMEM process is likely to abort. Check the
output of the "ompi_info" command and ensure that components of this
type are available on your system. You may also wish to check the
value of the "component_path" MCA parameter and ensure that it has at
least one directory that contains valid MCA components.

529
oshmem/mca/memheap/base/memheap_base_alloc.c
Просмотреть файл

@ -1,529 +0,0 @@
/*
* Copyright (c) 2012 Mellanox Technologies, Inc.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "oshmem_config.h"
#include "opal/util/output.h"
#include "orte/util/show_help.h"
#include "oshmem/mca/memheap/memheap.h"
#include "oshmem/mca/memheap/base/base.h"
#ifdef HAVE_SYS_MMAN_H
#include <sys/mman.h>
#endif
#include <sys/ipc.h>
#include <sys/shm.h>
#if defined(MPAGE_ENABLE) && (MPAGE_ENABLE > 0)
#include <infiniband/verbs.h>
#endif /* MPAGE_ENABLE */
static int __shm_attach(map_segment_t *, size_t, int, int);
static void __shm_detach(map_segment_t *);
static int __mmap_attach(map_segment_t *, size_t);
static void __mmap_detach(map_segment_t *);
#if defined(MPAGE_ENABLE) && (MPAGE_ENABLE > 0)
static int __ibv_attach(map_segment_t *, size_t);
static void __ibv_detach(map_segment_t *);
#endif /* MPAGE_ENABLE */
static int __adaptive_attach(map_segment_t *, size_t);
int mca_memheap_base_alloc_init(mca_memheap_map_t *map, size_t size)
{
int ret = OSHMEM_SUCCESS;
int value = mca_memheap_base_shmalloc_use_hugepages;
assert(map);
assert(HEAP_SEG_INDEX == map->n_segments);
MEMHEAP_VERBOSE(5,"memheap method : %d",
mca_memheap_base_shmalloc_use_hugepages);
map_segment_t *s = &map->mem_segs[map->n_segments];
memset(s, 0, sizeof(*s));
s->is_active = 0;
s->shmid = MEMHEAP_SHM_INVALID;
s->start = 0;
s->end = 0;
s->size = 0;
s->type = MAP_SEGMENT_UNKNOWN;
s->context = NULL;
switch(value) {
case 0:
/* use sysv alloc without hugepages */
ret = __shm_attach(s, size, 0, 1);
break;
case 1:
ret = __shm_attach(s, size, 1, 1);
if (OSHMEM_SUCCESS != ret)
ret = __shm_attach(s, size, 0, 1);
break;
case 2:
/* huge pages only */
ret = __shm_attach(s, size, 1, 1);
if (OSHMEM_SUCCESS != ret)
MEMHEAP_ERROR("FAILED to allocated symmetric heap using hugepages fallback is disabled, errno=%d", errno);
break;
case 3:
/* huge pages only + cleanup shmid */
ret = __shm_attach(s, size, 1, 0);
if (OSHMEM_SUCCESS != ret)
MEMHEAP_ERROR("FAILED to allocated symmetric heap using hugepages fallback is disabled, errno=%d", errno);
break;
case 4:
/* use sysv alloc without hugepages */
ret = __shm_attach(s, size, 0, 0);
break;
#if defined(MPAGE_ENABLE) && (MPAGE_ENABLE > 0)
case 5:
/* use shared memory registration (mpages) */
ret = __ibv_attach(s, size);
break;
#endif /* MPAGE_ENABLE */
case 100:
/* use mmap. It will severaly impact performance of intra node communication */
ret = __mmap_attach(s, size);
MEMHEAP_VERBOSE(1, "mmap() memheap allocation will severely impact performance of intra node communication");
break;
case 101:
ret = __shm_attach(s, size, 1, 1);
if (OSHMEM_SUCCESS != ret) {
MEMHEAP_ERROR("Failed to allocate hugepages. Falling back on regular allocation");
ret = __mmap_attach(s, size);
}
else {
s->shmid = MEMHEAP_SHM_INVALID;
}
MEMHEAP_VERBOSE(1, "SM BTL will be always used for intranode comm\n");
break;
case 102:
ret = __shm_attach(s, size, 1, 1);
if (OSHMEM_SUCCESS != ret) {
MEMHEAP_ERROR("FAILED to allocated symmetric heap using hugepages fallback is disabled, errno=%d", errno);
}
else {
s->shmid = MEMHEAP_SHM_INVALID;
}
break;
default:
ret = __adaptive_attach(s, size);
// MEMHEAP_ERROR("Unknown memheap allocation method: %d", value);
// ret = OSHMEM_ERR_BAD_PARAM;
}
if (OSHMEM_SUCCESS == ret)
{
map->n_segments++;
MEMHEAP_VERBOSE(1, "Memheap alloc memory: %llu byte(s), %d segments by method: %d",
(unsigned long long)size, map->n_segments, s->type);
}
return ret;
}
void mca_memheap_base_alloc_exit(mca_memheap_map_t *map)
{
if (map)
{
map_segment_t *s = &map->mem_segs[HEAP_SEG_INDEX];
assert(s);
switch(s->type) {
case MAP_SEGMENT_ALLOC_SHM:
__shm_detach(s);
break;
case MAP_SEGMENT_ALLOC_MMAP:
__mmap_detach(s);
break;
#if defined(MPAGE_ENABLE) && (MPAGE_ENABLE > 0)
case MAP_SEGMENT_ALLOC_IBV:
__ibv_detach(s);
break;
#endif /* MPAGE_ENABLE */
default:
MEMHEAP_ERROR("Unknown segment type: %d", (int)s->type);
}
}
}
static int __adaptive_attach(map_segment_t *s, size_t size)
{
int rc = OSHMEM_SUCCESS;
#if defined(MPAGE_ENABLE) && (MPAGE_ENABLE > 0)
rc = __ibv_attach(s, size);
#endif /* MPAGE_ENABLE */
if (rc)
{
rc = __shm_attach(s, size, 1, 1);
}
if (rc)
{
rc = __shm_attach(s, size, 0, 1);
}
if (rc)
{
rc = __shm_attach(s, size, 0, 0);
}
if (rc)
{
rc = __mmap_attach(s, size);
}
return rc;
}
static int __shm_attach(map_segment_t *s, size_t size, int use_hp, int do_rmid)
{
static int shm_context = 0;;
void *addr = NULL;
int shmid = MEMHEAP_SHM_INVALID;
int flags;
assert(s);
shm_context = use_hp;
flags = IPC_CREAT | IPC_EXCL | SHM_R | SHM_W;
flags |= ( use_hp ? SHM_HUGETLB : 0 );
/* Create a new shared memory segment and save the shmid. */
shmid = shmget(IPC_PRIVATE, size, flags);
if (shmid == MEMHEAP_SHM_INVALID) {
MEMHEAP_VERBOSE(1, "Failed to get shm segment (errno=%d)", errno);
return OSHMEM_ERROR;
}
/* Attach to the sement */
addr = shmat(shmid, (void *)MEMHEAP_BASE_START_ADDRESS, 0);
if (addr == (void *)-1L) {
MEMHEAP_VERBOSE(1, "Failed to attach to shm segment (errno=%d)", errno);
shmctl(shmid, IPC_RMID, NULL);
return OSHMEM_ERR_OUT_OF_RESOURCE;
}
MEMHEAP_VERBOSE(5, "got shmid %d", shmid);
if (do_rmid)
shmctl(shmid, IPC_RMID, NULL);
s->type = MAP_SEGMENT_ALLOC_SHM;
s->shmid = shmid;
s->start = (uintptr_t)addr;
s->size = size;
s->end = s->start + s->size;
s->context = &shm_context;
return OSHMEM_SUCCESS;
}
static void __shm_detach(map_segment_t *s)
{
assert(s);
if (s->shmid != MEMHEAP_SHM_INVALID)
{
shmctl(s->shmid, IPC_RMID, NULL);
}
if (s->context && (*((int *)(s->context))) > 0) {
/**
* Workaround kernel panic when detaching huge pages from user space simultanously from several processes
* dont detach here instead let kernel do it during process cleanup
*/
//shmdt((void *)s->start);
}
}
static int __mmap_attach(map_segment_t *s, size_t size)
{
void *addr = NULL;
assert(s);
addr = mmap((void *)MEMHEAP_BASE_START_ADDRESS,
size,
PROT_READ|PROT_WRITE,
MAP_SHARED|MAP_ANONYMOUS|MAP_FIXED, 0, 0);
if (MAP_FAILED == addr) {
MEMHEAP_ERROR("Failed to mmap() %llu bytes (errno=%d)", (unsigned long long)size, errno);
return OSHMEM_ERR_OUT_OF_RESOURCE;
}
s->type = MAP_SEGMENT_ALLOC_MMAP;
s->shmid = MEMHEAP_SHM_INVALID;
s->start = (uintptr_t)addr;
s->size = size;
s->end = s->start + s->size;
s->context = NULL;
return OSHMEM_SUCCESS;
}
static void __mmap_detach(map_segment_t *s)
{
assert(s);
munmap((void *)s->start, s->size);
}
#if defined(MPAGE_ENABLE) && (MPAGE_ENABLE > 0)
static int __ibv_attach(map_segment_t *s, size_t size)
{
int rc = OSHMEM_SUCCESS;
static openib_device_t memheap_device;
openib_device_t *device = &memheap_device;
int num_devs = 0;
assert(s);
memset(device, 0, sizeof(*device));
#ifdef HAVE_IBV_GET_DEVICE_LIST
device->ib_devs = ibv_get_device_list(&num_devs);
#else
#error unsupported ibv_get_device_list in infiniband/verbs.h
#endif
if (num_devs == 0 || !device->ib_devs)
{
rc = OSHMEM_ERR_NOT_SUPPORTED;
}
/* Open device */
if (!rc)
{
int i = 0;
if (num_devs > 1)
MEMHEAP_VERBOSE(5, "found %d HCAs, choosing the first", num_devs);
for (i = 0; i < num_devs; i++)
{
device->ib_dev = device->ib_devs[i];
device->ib_dev_context = ibv_open_device(device->ib_dev);
if (NULL == device->ib_dev_context)
{
MEMHEAP_ERROR("error obtaining device context for %s errno says %d: %s",
ibv_get_device_name(device->ib_dev), errno, strerror(errno));
rc = OSHMEM_ERR_RESOURCE_BUSY;
}
else
{
MEMHEAP_VERBOSE(5, "selected %s as %d of %d", ibv_get_device_name(device->ib_dev), i, num_devs);
rc = OSHMEM_SUCCESS;
break;
}
}
}
/* Obtain device attributes */
if (!rc)
{
if (ibv_query_device(device->ib_dev_context, &device->ib_dev_attr))
{
MEMHEAP_ERROR("error obtaining device attributes for %s errno says %d: %s",
ibv_get_device_name(device->ib_dev), errno, strerror(errno));
rc = OSHMEM_ERR_RESOURCE_BUSY;
}
else
{
MEMHEAP_VERBOSE(5, "ibv device %s",
ibv_get_device_name(device->ib_dev));
}
}
/* Allocate the protection domain for the device */
if (!rc)
{
device->ib_pd = ibv_alloc_pd(device->ib_dev_context);
if (NULL == device->ib_pd)
{
MEMHEAP_ERROR("error allocating protection domain for %s errno says %d: %s",
ibv_get_device_name(device->ib_dev), errno, strerror(errno));
rc = OSHMEM_ERR_RESOURCE_BUSY;
}
}
/* Allocate memory */
if (!rc)
{
void *addr = NULL;
struct ibv_mr *ib_mr = NULL;
int access_flag = IBV_ACCESS_LOCAL_WRITE |
IBV_ACCESS_REMOTE_WRITE |
IBV_ACCESS_REMOTE_READ;
OBJ_CONSTRUCT(&device->ib_mr_array, opal_value_array_t);
opal_value_array_init(&device->ib_mr_array, sizeof(struct ibv_mr *));
#if defined(MPAGE_ENABLE) && ((MPAGE_ENABLE == 1) || (MPAGE_ENABLE == 2))
access_flag |= IBV_ACCESS_ALLOCATE_MR |
IBV_ACCESS_SHARED_MR_USER_READ |
IBV_ACCESS_SHARED_MR_USER_WRITE;
#endif /* MPAGE_ENABLE */
ib_mr = ibv_reg_mr(device->ib_pd, addr, size, access_flag);
if (NULL == ib_mr)
{
MEMHEAP_ERROR("error to ibv_reg_mr() %llu bytes errno says %d: %s",
(unsigned long long)size, errno, strerror(errno));
rc = OSHMEM_ERR_OUT_OF_RESOURCE;
}
else
{
device->ib_mr_shared = ib_mr;
opal_value_array_append_item(&device->ib_mr_array, &ib_mr);
}
#if defined(MPAGE_ENABLE) && (MPAGE_ENABLE == 2)
if (!rc)
{
access_flag = IBV_ACCESS_LOCAL_WRITE |
IBV_ACCESS_REMOTE_WRITE |
IBV_ACCESS_REMOTE_READ|
IBV_ACCESS_NO_RDMA;
addr = (void *)MEMHEAP_BASE_START_ADDRESS;
ib_mr = ibv_reg_shared_mr(device->ib_mr_shared->handle,
device->ib_pd, addr, access_flag);
if (NULL == ib_mr)
{
MEMHEAP_ERROR("error to ibv_reg_shared_mr() %llu bytes errno says %d: %s",
(unsigned long long)size, errno, strerror(errno));
rc = OSHMEM_ERR_OUT_OF_RESOURCE;
}
else
{
opal_value_array_append_item(&device->ib_mr_array, &ib_mr);
}
}
#endif /* MPAGE_ENABLE */
if (!rc)
{
assert(size == device->ib_mr_shared->length);
s->type = MAP_SEGMENT_ALLOC_IBV;
s->shmid = device->ib_mr_shared->handle;
s->start = (intptr_t)ib_mr->addr;
s->size = size;
s->end = s->start + s->size;
s->context = &memheap_device;
}
}
return rc;
}
static void __ibv_detach(map_segment_t *s)
{
int rc = OSHMEM_SUCCESS;
openib_device_t *device = NULL;
assert(s);
device = (openib_device_t *)s->context;
if (device)
{
if(!rc && opal_value_array_get_size(&device->ib_mr_array))
{
struct ibv_mr** array;
struct ibv_mr* ib_mr = NULL;
array = OPAL_VALUE_ARRAY_GET_BASE(&device->ib_mr_array, struct ibv_mr *);
while (opal_value_array_get_size(&device->ib_mr_array) > 0)
{
ib_mr = array[0];
if(ibv_dereg_mr(ib_mr))
{
MEMHEAP_ERROR("error ibv_dereg_mr(): %d: %s", errno, strerror(errno));
rc = OSHMEM_ERROR;
}
opal_value_array_remove_item(&device->ib_mr_array, 0);
}
if(!rc && device->ib_mr_shared)
{
device->ib_mr_shared = NULL;
}
OBJ_DESTRUCT(&device->ib_mr_array);
}
if(!rc && device->ib_pd)
{
if(ibv_dealloc_pd(device->ib_pd))
{
MEMHEAP_ERROR("error ibv_dealloc_pd(): %d: %s", errno, strerror(errno));
rc = OSHMEM_ERROR;
}
else
{
device->ib_pd = NULL;
}
}
if(!rc && device->ib_dev_context)
{
if(ibv_close_device(device->ib_dev_context))
{
MEMHEAP_ERROR("error ibv_close_device(): %d: %s", errno, strerror(errno));
rc = OSHMEM_ERROR;
}
else
{
device->ib_dev_context = NULL;
}
}
if(!rc && device->ib_devs)
{
ibv_free_device_list(device->ib_devs);
device->ib_devs = NULL;
}
}
}
#endif /* MPAGE_ENABLE */

75
oshmem/mca/memheap/base/memheap_base_close.c
Просмотреть файл

@ -1,75 +0,0 @@
/*
* Copyright (c) 2012 Mellanox Technologies, Inc.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "oshmem_config.h"
#include <stdio.h>
#include "opal/util/argv.h"
#include "opal/util/output.h"
#include "orte/util/show_help.h"
#include "opal/mca/base/base.h"
#include "opal/mca/base/mca_base_component_repository.h"
#include "opal/runtime/opal.h"
#include "oshmem/constants.h"
#include "opal/mca/mca.h"
#include "oshmem/mca/memheap/base/base.h"
int mca_memheap_base_close(void)
{
int rc = OSHMEM_SUCCESS;
if( mca_memheap_base_already_opened <= 0 ) {
return OSHMEM_ERROR;
}
mca_memheap_base_already_opened--;
if (mca_memheap_base_already_opened > 0) {
return OSHMEM_SUCCESS;
}
#if 0
/* disable event processing while cleaning up memheaps */
opal_event_disable();
#endif
/* Free allocated module */
/*free(mca_memheap_base_module_initialized);
if(NULL != mca_memheap_base_module_initialized){
rc = mca_memheap_base_module_initialized->memheap_finalize(mca_memheap_base_module_initialized);
if (OSHMEM_SUCCESS != rc){
return OSHMEM_ERROR;
}
}*/
memheap_oob_destruct();
rc = mca_memheap_base_deregister(&mca_memheap_base_map);
mca_memheap_base_alloc_exit(&mca_memheap_base_map);
mca_memheap_base_static_exit(&mca_memheap_base_map);
/* Close the maximal priority component which is the only component remained opened */
if (0 != opal_list_get_size(&mca_memheap_base_components_opened)) {
mca_base_components_close(mca_memheap_base_output,
&mca_memheap_base_components_opened, NULL);
}
/* cleanup */
if(NULL != mca_memheap_base_include)
free(mca_memheap_base_include);
if(NULL != mca_memheap_base_exclude)
free(mca_memheap_base_exclude);
#if 0
/* restore event processing */
opal_event_enable();
#endif
/* All done */
return OSHMEM_SUCCESS;
}

684
oshmem/mca/memheap/base/memheap_base_mkey.c
Просмотреть файл

@ -1,684 +0,0 @@
/*
* Copyright (c) 2012 Mellanox Technologies, Inc.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "oshmem_config.h"
#include "opal/util/output.h"
#include "opal/dss/dss.h"
#include "orte/mca/rml/rml.h"
#include "orte/mca/rml/rml_types.h"
#include "orte/mca/grpcomm/grpcomm.h"
#include "orte/mca/errmgr/errmgr.h"
#include "orte/runtime/orte_globals.h"
#include "ompi/mca/bml/bml.h"
#include "ompi/mca/dpm/dpm.h"
#include "oshmem/proc/proc.h"
#include "oshmem/runtime/runtime.h"
#include "oshmem/mca/memheap/memheap.h"
#include "oshmem/mca/memheap/base/base.h"
#ifdef HAVE_SYS_MMAN_H
#include <sys/mman.h>
#endif
#include <sys/ipc.h>
#include <sys/shm.h>
#if defined(MPAGE_ENABLE) && (MPAGE_ENABLE > 0)
#include <infiniband/verbs.h>
#endif /* MPAGE_ENABLE */
#define MEMHEAP_RKEY_REQ 0xA1
#define MEMHEAP_RKEY_RESP 0xA2
#define MEMHEAP_RKEY_RESP_FAIL 0xA3
struct oob_comm {
opal_mutex_t lck;
opal_condition_t cond;
mca_spml_mkey_t *mkeys;
int mkeys_rcvd;
};
#define MEMHEAP_VERBOSE_FASTPATH(...)
static mca_memheap_map_t* memheap_map = NULL;
struct oob_comm memheap_oob;
/* pickup list of rkeys and remote va */
static int memheap_oob_get_mkeys(int pe, uint32_t va_seg_num, mca_spml_mkey_t *mkey);
static inline unsigned long __seg2base_va(int seg)
{
return memheap_map->mem_segs[seg].start;
}
static int __seg_cmp(const void *k, const void *v)
{
unsigned long va = (unsigned long)k;
map_segment_t *s = (map_segment_t *)v;
if (va < s->start)
return -1;
if (va >= s->end)
return 1;
return 0;
}
static inline map_segment_t *__find_va(unsigned long va)
{
map_segment_t *s;
if ( OPAL_LIKELY(va >= (unsigned long)memheap_map->mem_segs[HEAP_SEG_INDEX].start &&
va < (unsigned long)memheap_map->mem_segs[HEAP_SEG_INDEX].end))
{
s = &memheap_map->mem_segs[HEAP_SEG_INDEX];
}
else
{
s = bsearch((const void *)va, &memheap_map->mem_segs[SYMB_SEG_INDEX], memheap_map->n_segments - 1, sizeof(*s), __seg_cmp);
}
#if 0
if (s) {
MEMHEAP_VERBOSE(5, "match seg#%02ld: 0x%llX - 0x%llX %llu bytes va=%p",
s - memheap_map->mem_segs,
(long long)s->start,
(long long)s->end,
(long long)(s->end - s->start),
(void *)va);
}
#endif
return s;
}
static int do_mkey_req(opal_buffer_t *msg, int pe, int seg)
{
uint8_t msg_type;
oshmem_proc_t *proc;
int i, n, tr_id;
mca_spml_mkey_t *mkey;
msg_type = MEMHEAP_RKEY_RESP;
opal_dss.pack(msg, &msg_type, 1, OPAL_UINT8);
/* go over all transports to remote pe and pack mkeys */
n = oshmem_get_transport_count(pe);
proc = oshmem_proc_group_find(oshmem_group_all, pe);
opal_dss.pack(msg, &n, 1, OPAL_UINT32);
MEMHEAP_VERBOSE(5, "found %d transports to %d", n, pe);
for (i = 0; i < n; i++) {
tr_id = proc->transport_ids[i];
mkey = mca_memheap_base_get_mkey(__seg2base_va(seg), tr_id);
if (!mkey) {
MEMHEAP_ERROR("seg#%d tr_id: %d failed to find local mkey", seg, tr_id);
return OSHMEM_ERROR;
}
opal_dss.pack(msg, &tr_id, 1, OPAL_UINT32);
opal_dss.pack(msg, &mkey->key, 1, OPAL_UINT64);
opal_dss.pack(msg, &mkey->va_base, 1, OPAL_UINT64);
MEMHEAP_VERBOSE(5, "seg#%d tr_id: %d key %llx base_va %llx",
seg, tr_id,
(unsigned long long)mkey->key,
(unsigned long long)mkey->va_base);
}
return OSHMEM_SUCCESS;
}
static void memheap_attach_segment(mca_spml_mkey_t *mkey, int tr_id)
{
/* process special case when va was got using shmget(IPC_PRIVATE)
* this case is notable for:
* - key is set as (type|shmid);
* - va_base is set as 0;
*/
if (!mkey->va_base && ((int)MEMHEAP_SHM_GET_ID(mkey->key) != MEMHEAP_SHM_INVALID))
{
MEMHEAP_VERBOSE(5, "shared memory usage tr_id: %d key %llx base_va %llx shmid 0x%X|0x%X",
tr_id,
(unsigned long long)mkey->key,
(unsigned long long)mkey->va_base,
MEMHEAP_SHM_GET_TYPE(mkey->key),
MEMHEAP_SHM_GET_ID(mkey->key));
if (MEMHEAP_SHM_GET_TYPE(mkey->key) == MAP_SEGMENT_ALLOC_SHM)
{
mkey->va_base = (intptr_t)shmat(MEMHEAP_SHM_GET_ID(mkey->key), 0, 0);
}
else if (MEMHEAP_SHM_GET_TYPE(mkey->key) == MAP_SEGMENT_ALLOC_IBV)
{
#if defined(MPAGE_ENABLE) && (MPAGE_ENABLE == 2)
openib_device_t *device = NULL;
struct ibv_mr *ib_mr;
void *addr;
static int mr_count;
int access_flag = IBV_ACCESS_LOCAL_WRITE |
IBV_ACCESS_REMOTE_WRITE |
IBV_ACCESS_REMOTE_READ |
IBV_ACCESS_NO_RDMA;
device = (openib_device_t *)memheap_map->mem_segs[HEAP_SEG_INDEX].context;
assert(device);
/* workaround mtt problem - request aligned addresses */
++mr_count;
addr = (void *)(MEMHEAP_BASE_START_ADDRESS + mca_memheap_base_mr_interleave_factor*1024ULL*1024ULL*1024ULL*mr_count);
ib_mr = ibv_reg_shared_mr(MEMHEAP_SHM_GET_ID(mkey->key),
device->ib_pd, addr, access_flag);
if (NULL == ib_mr)
{
mkey->va_base = -1;
MEMHEAP_ERROR("error to ibv_reg_shared_mr() errno says %d: %s",
errno, strerror(errno));
}
else
{
if (ib_mr->addr != addr) {
MEMHEAP_WARN("Failed to map shared region to address %p got addr %p. Try to increase 'memheap_mr_interleave_factor' from %d", addr, ib_mr->addr, mca_memheap_base_mr_interleave_factor);
}
opal_value_array_append_item(&device->ib_mr_array, &ib_mr);
mkey->va_base = (intptr_t)ib_mr->addr;
}
#endif /* MPAGE_ENABLE */
}
else
{
MEMHEAP_ERROR("tr_id: %d key %llx attach failed: incorrect shmid 0x%X|0x%X",
tr_id,
(unsigned long long)mkey->key,
MEMHEAP_SHM_GET_TYPE(mkey->key),
MEMHEAP_SHM_GET_ID(mkey->key));
oshmem_shmem_abort(-1);
}
if ((void *)-1 == (void *)mkey->va_base)
{
MEMHEAP_ERROR("tr_id: %d key %llx attach failed: errno = %d",
tr_id,
(unsigned long long)mkey->key,
errno);
oshmem_shmem_abort(-1);
}
}
}
static void do_mkey_resp(opal_buffer_t *msg)
{
int32_t cnt;
int32_t n;
int32_t tr_id;
int i;
cnt = 1;
opal_dss.unpack(msg, &n, &cnt, OPAL_UINT32);
for (i = 0; i < n; i++) {
opal_dss.unpack(msg, &tr_id, &cnt, OPAL_UINT32);
opal_dss.unpack(msg, &memheap_oob.mkeys[tr_id].key, &cnt, OPAL_UINT64);
opal_dss.unpack(msg, &memheap_oob.mkeys[tr_id].va_base, &cnt, OPAL_UINT64);
memheap_attach_segment(&memheap_oob.mkeys[tr_id], tr_id);
MEMHEAP_VERBOSE(5, "tr_id: %d key %llx base_va %llx",
tr_id,
(unsigned long long)memheap_oob.mkeys[tr_id].key,
(unsigned long long)memheap_oob.mkeys[tr_id].va_base);
}
}
static void memheap_buddy_rml_recv_cb(int status, orte_process_name_t* process_name,
opal_buffer_t* buffer, orte_rml_tag_t tag,
void* cbdata)
{
MEMHEAP_VERBOSE(5,"**** get request from %u:%d", process_name->jobid, process_name->vpid);
int32_t cnt = 1;
int rc;
opal_buffer_t *msg;
uint8_t msg_type;
uint32_t seg;
MEMHEAP_VERBOSE(5,"unpacking %d of %d", cnt, OPAL_UINT8);
rc = opal_dss.unpack(buffer, &msg_type, &cnt, OPAL_UINT8);
if (ORTE_SUCCESS != rc) {
ORTE_ERROR_LOG(rc);
goto send_fail;
}
switch (msg_type) {
case MEMHEAP_RKEY_REQ:
cnt = 1;
rc = opal_dss.unpack(buffer, &seg, &cnt, OPAL_UINT32);
if (ORTE_SUCCESS != rc) {
MEMHEAP_ERROR("bad RKEY_REQ msg");
goto send_fail;
}
MEMHEAP_VERBOSE(5,"*** RKEY REQ");
msg = OBJ_NEW(opal_buffer_t);
if (!msg) {
MEMHEAP_ERROR("failed to get msg buffer");
ORTE_ERROR_LOG(rc);
return;
}
if (OSHMEM_SUCCESS != do_mkey_req(msg, process_name->vpid, seg)) {
OBJ_RELEASE(msg);
goto send_fail;
}
rc = orte_rml.send_buffer(process_name, msg, OMPI_RML_TAG_SHMEM, 0);
OBJ_RELEASE(msg);
if (0 >= rc) {
MEMHEAP_ERROR("FAILED to send rml message %d", rc);
ORTE_ERROR_LOG(rc);
goto send_fail;
}
break;
case MEMHEAP_RKEY_RESP:
MEMHEAP_VERBOSE(5,"*** RKEY RESP");
OPAL_THREAD_LOCK(&memheap_oob.lck);
do_mkey_resp(buffer);
memheap_oob.mkeys_rcvd = MEMHEAP_RKEY_RESP;
opal_condition_broadcast(&memheap_oob.cond);
OPAL_THREAD_UNLOCK(&memheap_oob.lck);
break;
case MEMHEAP_RKEY_RESP_FAIL:
MEMHEAP_VERBOSE(5,"*** RKEY RESP FAIL");
memheap_oob.mkeys_rcvd = MEMHEAP_RKEY_RESP_FAIL;
opal_condition_broadcast(&memheap_oob.cond);
OPAL_THREAD_UNLOCK(&memheap_oob.lck);
break;
default:
MEMHEAP_VERBOSE(5,"Unknown message type %x",msg_type);
goto send_fail;
}
return;
send_fail:
msg = OBJ_NEW(opal_buffer_t);
if (!msg) {
MEMHEAP_ERROR("failed to get msg buffer");
ORTE_ERROR_LOG(rc);
return;
}
msg_type = MEMHEAP_RKEY_RESP_FAIL;
opal_dss.pack(msg, &msg_type, 1, OPAL_UINT8);
rc = orte_rml.send_buffer(process_name, msg, OMPI_RML_TAG_SHMEM, 0);
if (0 >= rc) {
MEMHEAP_ERROR("FAILED to send rml message %d", rc);
ORTE_ERROR_LOG(rc);
}
OBJ_RELEASE(msg);
}
int memheap_oob_init(mca_memheap_map_t *map)
{
int rc;
memheap_map = map;
OBJ_CONSTRUCT(&memheap_oob.lck, opal_mutex_t);
OBJ_CONSTRUCT(&memheap_oob.cond, opal_condition_t);
rc = orte_rml.recv_buffer_nb(ORTE_NAME_WILDCARD,
OMPI_RML_TAG_SHMEM,
ORTE_RML_PERSISTENT,
memheap_buddy_rml_recv_cb,
NULL);
return rc;
}
void memheap_oob_destruct(void)
{
orte_rml.recv_cancel(ORTE_NAME_WILDCARD, OMPI_RML_TAG_SHMEM);
OBJ_DESTRUCT(&memheap_oob.lck);
OBJ_DESTRUCT(&memheap_oob.cond);
}
static int memheap_oob_get_mkeys(int pe, uint32_t seg, mca_spml_mkey_t *mkeys)
{
orte_process_name_t name;
opal_buffer_t *msg;
int rc;
uint8_t cmd;
int i;
if (OSHMEM_SUCCESS == MCA_SPML_CALL(oob_get_mkeys(pe, seg, mkeys))) {
for (i = 0; i < memheap_map->num_transports; i++) {
mkeys[i].va_base = __seg2base_va(seg);
MEMHEAP_VERBOSE(5, "MKEY CALCULATED BY LOCAL SPML: pe: %d tr_id: %d key %llx base_va %llx",
pe,
i,
(unsigned long long)mkeys[i].key,
(unsigned long long)mkeys[i].va_base);
}
return OSHMEM_SUCCESS;
}
OPAL_THREAD_LOCK(&memheap_oob.lck);
memheap_oob.mkeys = mkeys;
memheap_oob.mkeys_rcvd = 0;
name.jobid = ORTE_PROC_MY_NAME->jobid;
name.vpid = pe;
msg = OBJ_NEW(opal_buffer_t);
if (!msg) {
OPAL_THREAD_UNLOCK(&memheap_oob.lck);
MEMHEAP_ERROR("failed to get msg buffer");
return OSHMEM_ERROR;
}
OPAL_THREAD_LOCK(&memheap_oob.lck);
cmd = MEMHEAP_RKEY_REQ;
opal_dss.pack(msg, &cmd, 1, OPAL_UINT8);
opal_dss.pack(msg, &seg, 1, OPAL_UINT32);
rc = orte_rml.send_buffer(&name, msg, OMPI_RML_TAG_SHMEM, 0);
if (0 >= rc) {
OBJ_RELEASE(msg);
OPAL_THREAD_UNLOCK(&memheap_oob.lck);
MEMHEAP_ERROR("FAILED to send rml message %d", rc);
return OSHMEM_ERROR;
}
MEMHEAP_VERBOSE(5,"message sent: %d bytes!", rc);
while (!memheap_oob.mkeys_rcvd) {
opal_condition_wait(&memheap_oob.cond, &memheap_oob.lck);
}
if (MEMHEAP_RKEY_RESP == memheap_oob.mkeys_rcvd) {
rc = OSHMEM_SUCCESS;
}
else {
MEMHEAP_ERROR("failed to get rkey seg#%d pe=%d", seg, pe);
rc = OSHMEM_ERROR;
}
OBJ_RELEASE(msg);
OPAL_THREAD_UNLOCK(&memheap_oob.lck);
return rc;
}
#if 0 /* disable till we figure out double modex&grpcomm.bad problem */
static void memheap_modex_mkey_exchange(void)
{
/* disable till we figure out double modex&grpcomm.bad problem */
return;
if (!mca_memheap_buddy_use_modex)
return;
if (OMPI_SUCCESS != ompi_modex_send(&mca_memheap_buddy_component.memheap_version,
memheap_map->mem_segs[HEAP_SEG_INDEX].mkeys,
sizeof(mca_spml_mkey_t) * memheap_map->num_transports)) {
MEMHEAP_VERBOSE(1, "FAILED to modex_send() my mkeys");
return;
}
if (OMPI_SUCCESS != orte_grpcomm.modex(0)) {
MEMHEAP_VERBOSE(1, "FAILED to do modex()");
}
}
#endif
#if 0
static int memheap_modex_recv(int pe, mca_spml_mkey_t *mkeys, int num_transports)
{
oshmem_proc_t *proc;
size_t size;
int rc;
mca_spml_mkey_t *new_mkeys;
int i;
uint64_t dummy_rva;
proc = oshmem_proc_group_find(oshmem_group_all, pe);
/* hack till we get modex 4 static memheap */
/* force exchange for bss/data segments */
mca_memheap_base_get_cached_mkey(pe, __seg2base_va(SYMB_SEG_INDEX), proc->transport_ids[0], &dummy_rva);
rc = ompi_modex_recv(&mca_memheap_buddy_component.memheap_version, (ompi_proc_t *)proc,
(void**)&new_mkeys, &size);
if (OMPI_SUCCESS != rc)
return OSHMEM_ERROR;
if (size != num_transports * sizeof(mca_spml_mkey_t)) {
MEMHEAP_VERBOSE(1, "modex echanges size mismatch: wanted %d, got %d",
(int)(num_transports * sizeof(mca_spml_mkey_t)),
(int)size);
return OSHMEM_ERROR;
}
memcpy(mkeys, new_mkeys, num_transports * sizeof(mca_spml_mkey_t));
free(new_mkeys);
for (i = 0; i < num_transports; i++) {
if (mkeys[i].key == 0 && mkeys[i].va_base == 0)
continue;
/* don not try to attach to segments that are not connected to us */
if (proc->transport_ids[0] != i)
continue;
MEMHEAP_VERBOSE(5, "pe: %d tr_id: %d key %llx base_va %llx",
pe, i,
(unsigned long long)mkeys[i].key,
(unsigned long long)mkeys[i].va_base);
memheap_attach_segment(&mkeys[i], i);
}
return rc;
}
#endif
void mca_memheap_modex_recv_all(void)
{
int i;
int j;
int nprocs, my_pe;
oshmem_proc_t *proc;
mca_spml_mkey_t *mkey;
uint64_t dummy_rva;
if (!mca_memheap_buddy_use_modex)
return;
/* init rkey cache */
nprocs = oshmem_num_procs();
my_pe = oshmem_my_proc_id();
/* do exchange via rml till we figure out problem with grpcomm.modex and barrier */
for (i = 0; i < nprocs; i++) {
if (i == my_pe)
continue;
proc = oshmem_proc_group_find(oshmem_group_all, i);
for (j = 0; j < memheap_map->n_segments; j++)
{
mkey = mca_memheap_base_get_cached_mkey( i,
memheap_map->mem_segs[j].start,
proc->transport_ids[0],
&dummy_rva);
if (!mkey)
{
MEMHEAP_ERROR("Failed to receive mkeys");
oshmem_shmem_abort(-1);
}
}
}
#if 0
for (i = 0; i < nprocs; i++) {
if (i == my_pe)
continue;
memheap_map->mem_segs[HEAP_SEG_INDEX].mkeys_cache[i] = (mca_spml_mkey_t *)calloc(memheap_map->num_transports, sizeof(mca_spml_mkey_t));
if (!memheap_map->mem_segs[HEAP_SEG_INDEX].mkeys_cache[i]) {
MEMHEAP_ERROR("Failed to allocated mkey cache memory");
oshmem_shmem_abort(-1);
}
if (OSHMEM_SUCCESS != memheap_modex_recv(i, memheap_map->mem_segs[HEAP_SEG_INDEX].mkeys_cache[i], memheap_map->num_transports))
oshmem_shmem_abort(-1);
}
#endif
/*
* There is an issue with orte_grpcomm.barrier usage as
* ess/pmi directs to use grpcomm/pmi in case slurm srun() call grpcomm/pmi calls PMI_Barrier()
* that is a function of external library.
* There is no opal_progress() in such way. As a result slow PEs send a request (MEMHEAP_RKEY_REQ) to
* fast PEs waiting on barrier and do not get a respond (MEMHEAP_RKEY_RESP).
*
* there are following ways to solve one:
* 1. calculate requests from remote PEs and do ORTE_PROGRESSED_WAIT waiting for expected value;
* 2. use shmem_barrier_all();
* 3. rework pmi/barrier to use opal_progress();
* 4. use orte_grpcomm.barrier carefully;
*
* It seems there is no need to use orte_grpcomm.barrier here
*/
if ( memheap_map->mem_segs[HEAP_SEG_INDEX].shmid != MEMHEAP_SHM_INVALID) {
/* unfortunately we must do barrier here to assure that everyone are attached to our segment
* good thing that this code path only invoked on older linuxes (-mca shmalloc_use_hugepages 3|4)
* try to minimize damage here by waiting 5 seconds and doing progress
*/
shmem_barrier_all();
/* keys exchanged, segments attached, now we can safely cleanup */
if (memheap_map->mem_segs[HEAP_SEG_INDEX].type == MAP_SEGMENT_ALLOC_SHM)
{
shmctl(memheap_map->mem_segs[HEAP_SEG_INDEX].shmid, IPC_RMID, NULL);
}
}
}
static inline uint64_t va2rva(unsigned long va, uint64_t local_base, uint64_t remote_base)
{
return remote_base > local_base ? va + (remote_base - local_base) : va - (local_base - remote_base);
}
mca_spml_mkey_t * mca_memheap_base_get_cached_mkey(int pe, unsigned long va, int btl_id, uint64_t *rva)
{
map_segment_t *s;
int rc;
mca_spml_mkey_t *mkey;
MEMHEAP_VERBOSE_FASTPATH(10, "rkey: pe=%d va=%p", pe, (void *)va);
s = __find_va(va);
if (NULL == s)
return NULL;
if (!s->is_active)
return NULL;
if (pe == oshmem_my_proc_id()) {
*rva = va;
MEMHEAP_VERBOSE_FASTPATH(10, "rkey: pe=%d va=%p -> (local) %lx %p", pe, (void *)va,
s->mkeys[btl_id].key, (void *)*rva);
return &s->mkeys[btl_id];
}
if (OPAL_LIKELY(s->mkeys_cache[pe])) {
mkey = &s->mkeys_cache[pe][btl_id];
*rva = va2rva(va, s->start, mkey->va_base);
MEMHEAP_VERBOSE_FASTPATH(10, "rkey: pe=%d va=%p -> (cached) %lx %p", pe, (void *)va, mkey->key, (void *)*rva);
return mkey;
}
s->mkeys_cache[pe] = (mca_spml_mkey_t *)calloc(memheap_map->num_transports, sizeof(mca_spml_mkey_t));
if (!s->mkeys_cache[pe])
return NULL;
rc = memheap_oob_get_mkeys(pe, s - memheap_map->mem_segs, s->mkeys_cache[pe]);
if (OSHMEM_SUCCESS != rc)
return NULL;
mkey = &s->mkeys_cache[pe][btl_id];
*rva = va2rva(va, s->start, mkey->va_base);
MEMHEAP_VERBOSE_FASTPATH(5, "rkey: pe=%d va=%p -> (remote lookup) %lx %p", pe, (void *)va, mkey->key, (void *)*rva);
return mkey;
}
mca_spml_mkey_t *mca_memheap_base_get_mkey(unsigned long va, int tr_id)
{
map_segment_t *s;
s = __find_va(va);
return ( (s && s->is_active) ? &s->mkeys[tr_id] : NULL);
}
uint64_t mca_memheap_base_find_offset(int pe, int tr_id, unsigned long va, uint64_t rva)
{
map_segment_t *s;
s = __find_va(va);
return ( (s && s->is_active) ? (rva - s->mkeys_cache[pe][tr_id].va_base) : 0 );
}
int mca_memheap_base_is_symmetric_addr(unsigned long va)
{
return ( __find_va(va) ? 1 : 0 );
}
int mca_memheap_base_detect_addr_type(unsigned long va)
{
int addr_type = ADDR_INVALID;
map_segment_t *s;
s = __find_va(va);
if (s)
{
if (s->type == MAP_SEGMENT_STATIC)
{
addr_type = ADDR_STATIC;
}
else if ( va >= (unsigned long)s->start &&
va < (unsigned long)(s->start + mca_memheap.memheap_size) )
{
addr_type = ADDR_USER;
}
else
{
assert( va >= (unsigned long)(s->start + mca_memheap.memheap_size) &&
va < (unsigned long)s->end );
addr_type = ADDR_PRIVATE;
}
}
return addr_type;
}

123
oshmem/mca/memheap/base/memheap_base_open.c
Просмотреть файл

@ -1,123 +0,0 @@
/*
* Copyright (c) 2012 Mellanox Technologies, Inc.
* All rights reserved.
*
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "oshmem_config.h"
#include <stdio.h>
#include "opal/mca/mca.h"
#include "opal/util/output.h"
#include "opal/mca/base/base.h"
#include "opal/mca/base/mca_base_param.h"
#include "oshmem/mca/memheap/memheap.h"
#include "oshmem/mca/memheap/base/base.h"
/*
* The following file was created by configure. It contains extern
* statements and the definition of an array of pointers to each
* component's public mca_base_component_t struct.
*/
#include "oshmem/mca/memheap/base/static-components.h"
#if defined(MPAGE_ENABLE) && (MPAGE_ENABLE > 0)
int mca_memheap_base_shmalloc_use_hugepages = 5;
#else
int mca_memheap_base_shmalloc_use_hugepages = 1;
#endif /* MPAGE_ENABLE */
int mca_memheap_base_output = -1;
int mca_memheap_buddy_use_modex = 1;
int mca_memheap_base_mr_interleave_factor = 2;
char* mca_memheap_base_include = NULL;
char* mca_memheap_base_exclude = NULL;
opal_list_t mca_memheap_base_components_opened;
struct mca_memheap_base_module_t* mca_memheap_base_module_initialized = NULL;
int mca_memheap_base_already_opened = 0;
mca_memheap_map_t mca_memheap_base_map;
/**
* Function for finding and opening either all MCA components, or the one
* that was specifically requested via a MCA parameter.
*/
int mca_memheap_base_open(void)
{
int value = -1;
mca_memheap_base_already_opened = mca_memheap_base_already_opened + 1;
if( mca_memheap_base_already_opened > 1 ){
return OSHMEM_SUCCESS;
}
mca_memheap_base_output = opal_output_open(NULL);
mca_base_param_reg_int_name("memheap",
"base_verbose",
"Verbosity level of the MEMHEAP framework",
false, false,
0, &value);
opal_output_set_verbosity(mca_memheap_base_output, value);
#if defined(MPAGE_ENABLE) && (MPAGE_ENABLE > 0)
mca_base_param_reg_int_name("shmalloc",
"use_hugepages",
"0|1|2|5 - disabled, enabled with fallback to mmap(), do not fallback to mmap(), enabled mpages(default)",
false, false,
mca_memheap_base_shmalloc_use_hugepages, &mca_memheap_base_shmalloc_use_hugepages);
#else
mca_base_param_reg_int_name("shmalloc",
"use_hugepages",
"0|1|2 - disabled, enabled(default) with fallback to mmap(), do not fallback to mmap()",
false, false,
mca_memheap_base_shmalloc_use_hugepages, &mca_memheap_base_shmalloc_use_hugepages);
#endif /* MPAGE_ENABLE */
mca_base_param_reg_int_name("shmalloc",
"use_modex",
"0|1 - disabled, enabled(default) use modex to facilitate memory registration exchange",
false, false,
1, &mca_memheap_buddy_use_modex);
mca_base_param_reg_int_name("memheap",
"mr_interleave_factor",
"2 - default, try to give at least N Gbytes spaces between mapped memheaps of other pes that are local to me",
false, false,
mca_memheap_base_mr_interleave_factor, &mca_memheap_base_mr_interleave_factor);
/* Open up all available components */
if (OSHMEM_SUCCESS !=
mca_base_components_open("memheap", mca_memheap_base_output, mca_memheap_base_static_components,
&mca_memheap_base_components_opened, true)) {
return OSHMEM_ERROR;
}
/* register parameters */
mca_base_param_reg_string_name("memheap", NULL,
"Specify a specific memheap implementation to use",
false, false, NULL, &mca_memheap_base_include);
if (NULL == mca_memheap_base_include) {
mca_memheap_base_include = getenv(SHMEM_HEAP_TYPE);
if (NULL == mca_memheap_base_include)
mca_memheap_base_include = strdup("");
else
mca_memheap_base_include = strdup(mca_memheap_base_include);
}
(void) mca_base_param_reg_string_name("memheap","base_exclude",NULL,false,false,NULL, &mca_memheap_base_exclude);
memset(&mca_memheap_base_map, 0, sizeof(mca_memheap_base_map));
mca_memheap_base_map.n_segments = 0;
mca_memheap_base_map.num_transports = 0;
/* All done */
return OSHMEM_SUCCESS;
}

140
oshmem/mca/memheap/base/memheap_base_register.c
Просмотреть файл

@ -1,140 +0,0 @@
/*
* Copyright (c) 2012 Mellanox Technologies, Inc.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "oshmem_config.h"
#include "oshmem/proc/proc.h"
#include "oshmem/mca/memheap/memheap.h"
#include "oshmem/mca/memheap/base/base.h"
#include <stdio.h>
static int __dereg_segment(map_segment_t *s);
static int __reg_segment(map_segment_t *s, int *num_btl);
extern int mca_memheap_base_register(mca_memheap_map_t *memheap_map)
{
int ret = OSHMEM_SUCCESS;
int i;
for (i = 0; i < memheap_map->n_segments; i++) {
map_segment_t *s = &memheap_map->mem_segs[i];
MEMHEAP_VERBOSE(5, "register seg#%02d: 0x%llX - 0x%llX %llu bytes type=0x%X id=0x%X",
i,
(long long)s->start,
(long long)s->end,
(long long)(s->end - s->start),
s->type,
s->shmid
);
ret = __reg_segment(s, &memheap_map->num_transports);
}
return ret;
}
extern int mca_memheap_base_deregister(mca_memheap_map_t *memheap_map)
{
int ret = OSHMEM_SUCCESS;
int i;
for (i = 0; i < memheap_map->n_segments; i++) {
map_segment_t *s = &memheap_map->mem_segs[i];
if (!s->is_active)
continue;
MEMHEAP_VERBOSE(5, "deregistering segment#%d: %llx - %llx %llu bytes",
i,
(long long)s->start,
(long long)s->end,
(long long)(s->end - s->start)
);
ret = __dereg_segment(s);
}
return ret;
}
static int __dereg_segment(map_segment_t *s)
{
int rc = OSHMEM_SUCCESS;
int j;
int nprocs, my_pe;
nprocs = oshmem_num_procs();
my_pe = oshmem_my_proc_id();
MCA_SPML_CALL(deregister(s->mkeys));
if (s->mkeys_cache) {
for (j = 0; j < nprocs; j++) {
if (j == my_pe)
continue;
if (s->mkeys_cache[j])
{
free(s->mkeys_cache[j]);
s->mkeys_cache[j] = NULL;
}
}
free(s->mkeys_cache);
s->mkeys_cache = NULL;
}
s->is_active = 0;
return rc;
}
static int __reg_segment(map_segment_t *s, int *num_btl)
{
int rc = OSHMEM_SUCCESS;
int my_pe;
int nprocs;
nprocs = oshmem_num_procs();
my_pe = oshmem_my_proc_id();
s->mkeys_cache = (mca_spml_mkey_t **)calloc(nprocs, sizeof(mca_spml_mkey_t *));
if (NULL == s->mkeys_cache)
{
MEMHEAP_ERROR("Failed to allocate memory for remote segments");
rc = OSHMEM_ERROR;
}
if (!rc)
{
s->mkeys = MCA_SPML_CALL(register((void *)(unsigned long)s->start,
s->end - s->start,
MEMHEAP_SHM_CODE(s->type, s->shmid),
num_btl));
if (NULL == s->mkeys)
{
free(s->mkeys_cache);
s->mkeys_cache = NULL;
MEMHEAP_ERROR("Failed to register segment");
rc = OSHMEM_ERROR;
}
}
if (OSHMEM_SUCCESS == rc)
{
s->mkeys_cache[my_pe] = s->mkeys;
s->is_active = 1;
}
return rc;
}

267
oshmem/mca/memheap/base/memheap_base_select.c
Просмотреть файл

@ -1,267 +0,0 @@
/*
* Copyright (c) 2012 Mellanox Technologies, Inc.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "oshmem_config.h"
#include "opal/util/argv.h"
#include "opal/util/output.h"
#include "orte/util/show_help.h"
#include "opal/mca/mca.h"
#include "opal/mca/base/base.h"
#include "opal/mca/base/mca_base_component_repository.h"
#include "oshmem/mca/memheap/memheap.h"
#include "oshmem/mca/memheap/base/base.h"
#include "orte/mca/errmgr/errmgr.h"
#include "opal/runtime/opal.h"
mca_memheap_base_module_t mca_memheap;
/**
* Function for weeding out memheap components that shouldn't be executed.
* Implementation inspired by btl/base.
*
* Call the init function on all available components to find out if
* they want to run. Select all components that don't fail. Failing
* components will be closed and unloaded. The selected modules will
* be pointed to by mca_memheap_base_module_t.
*/
static memheap_context_t* __memheap_create(void);
/**
* Choose to init one component with the highest priority.
* If the include list if it is not empty choose a component that appear in the list.
* O/W choose the highest priority component not in the exclude list.
* Include and exclude lists may be given in the shmem launcher command line.
*/
int mca_memheap_base_select()
{
int priority = 0;
int max_priority = 0;
opal_list_item_t *max_priority_item = NULL;
opal_list_item_t *item = NULL;
mca_base_component_list_item_t *cli = NULL;
mca_memheap_base_component_t *component = NULL;
mca_memheap_base_component_t *max_priority_component = NULL;
mca_memheap_base_module_t *module = NULL;
memheap_context_t *context = NULL;
char** include = opal_argv_split(mca_memheap_base_include, ',');
char** exclude = opal_argv_split(mca_memheap_base_exclude, ',');
context = __memheap_create();
if (!context)
{
return OSHMEM_ERROR;
}
item = opal_list_get_first(&mca_memheap_base_components_opened);
while (item != opal_list_get_end(&mca_memheap_base_components_opened)) {
opal_list_item_t *next = opal_list_get_next(item);
cli = (mca_base_component_list_item_t *) item;
component = (mca_memheap_base_component_t *) cli->cli_component;
/* Verify if the component is in the include or the exclude list. */
/* If there is an include list - item must be in the list to be included */
if ( NULL != include ) {
char** argv = include;
bool found = false;
while(argv && *argv) {
if(strcmp(component->memheap_version.mca_component_name,*argv) == 0) {
found = true;
break;
}
argv++;
}
/* If not in the list do not choose this component */
if(found == false) {
item = next;
continue;
}
/* Otherwise - check the exclude list to see if this item has been specifically excluded */
} else if ( NULL != exclude ) {
char** argv = exclude;
bool found = false;
while(argv && *argv) {
if(strcmp(component->memheap_version.mca_component_name,*argv) == 0) {
found = true;
break;
}
argv++;
}
if(found == true) {
item = next;
continue;
}
}
/* Verify that the component has an init function */
if (NULL == component->memheap_init) {
MEMHEAP_VERBOSE(10,"select: no init function; for component %s. No component selected",
component->memheap_version.mca_component_name);
} else {
MEMHEAP_VERBOSE(5,"select: component %s size : user %d private: %d",
component->memheap_version.mca_component_name,
(int)context->user_size, (int)context->private_size);
/* Init the component in order to get its priority */
module = component->memheap_init(context, &priority);
/* If the component didn't initialize, remove it from the opened list, remove it from the component repository and return an error */
if (NULL == module) {
MEMHEAP_VERBOSE(10,"select: init of component %s returned failure",
component->memheap_version.mca_component_name);
MEMHEAP_VERBOSE(10,"select: module %s unloaded",
component->memheap_version.mca_component_name);
mca_base_component_repository_release((mca_base_component_t *) component);
}
/* Calculate memheap size in case it was not set during component initialization */
module->memheap_size = context->user_size;
}
/* Init max priority component */
if(NULL == max_priority_component) {
max_priority_component = component;
max_priority_item = item;
mca_memheap_base_module_initialized = module;
max_priority = priority;
}
/* Update max priority component if current component has greater priority */
if(priority > max_priority) {
max_priority = priority;
max_priority_component = component;
max_priority_item = item;
mca_memheap_base_module_initialized = module;
}
item = next;
}
opal_argv_free(include);
opal_argv_free(exclude);
/* Verify that a component was selected */
if(NULL == max_priority_component) {
MEMHEAP_VERBOSE(10,"select: no component selected");
return OSHMEM_ERROR;
}
/* Verify that some module was initialized */
if(NULL == mca_memheap_base_module_initialized) {
orte_show_help("help-shmem-mca.txt", "find-available:none-found", true, "memheap");
orte_errmgr.abort(1, NULL);
}
MEMHEAP_VERBOSE(10,"SELECTED %s component %s",
max_priority_component->memheap_version.mca_type_name,
max_priority_component->memheap_version.mca_component_name);
setenv(SHMEM_HEAP_TYPE, max_priority_component->memheap_version.mca_component_name, 1);
/* Clear open list */
/* Close all remaining opened components except for the selected component */
if (0 != opal_list_get_size(&mca_memheap_base_components_opened)) {
mca_base_components_close(mca_memheap_base_output,
&mca_memheap_base_components_opened, (mca_base_component_t*)max_priority_component);
}
mca_memheap = *mca_memheap_base_module_initialized;
return OSHMEM_SUCCESS;
}
static size_t memheap_size(void)
{
char *p;
unsigned long long factor;
int idx;
unsigned long long size;
p = getenv(SHMEM_HEAP_SIZE);
if (!p)
return SIZE_IN_MEGA_BYTES(DEFAULT_SYMMETRIC_HEAP_SIZE);
idx = strlen(p)-1;
if (p[idx] == 'k' || p[idx] == 'K') {
factor = 1024;
}
else if (p[idx] == 'm' || p[idx] == 'M') {
factor = 1024 * 1024;
}
else if (p[idx] == 'g' || p[idx] == 'G') {
factor = 1024 * 1024 * 1024;
}
else if (p[idx] == 't' || p[idx] == 'T') {
factor = 1024UL * 1024UL * 1024UL * 1024UL;
}
else
factor = 1;
size = atoll(p);
if (size == 0) {
MEMHEAP_ERROR("Incorrect symmetric heap size %s. Using default heap size %d M\n", p, DEFAULT_SYMMETRIC_HEAP_SIZE);
return SIZE_IN_MEGA_BYTES(DEFAULT_SYMMETRIC_HEAP_SIZE);
}
return (size_t)memheap_align(size * factor);
}
static memheap_context_t* __memheap_create(void)
{
int rc = OSHMEM_SUCCESS;
static memheap_context_t context;
size_t user_size;
user_size = memheap_size();
if (user_size < MEMHEAP_BASE_MIN_SIZE) {
MEMHEAP_ERROR("Requested memheap size is less than minimal meamheap size (%llu < %llu)",
(unsigned long long)user_size, MEMHEAP_BASE_MIN_SIZE);
return NULL;
}
/* Inititialize symmetric area */
if (OSHMEM_SUCCESS == rc)
{
rc = mca_memheap_base_alloc_init(&mca_memheap_base_map,
user_size + MEMHEAP_BASE_PRIVATE_SIZE);
}
/* Inititialize static/global variables area */
if (OSHMEM_SUCCESS == rc)
{
rc = mca_memheap_base_static_init(&mca_memheap_base_map);
}
/* Memory Registration */
if (OSHMEM_SUCCESS == rc)
{
rc = mca_memheap_base_register(&mca_memheap_base_map);
}
/* Init OOB channel */
if (OSHMEM_SUCCESS == rc)
{
rc = memheap_oob_init(&mca_memheap_base_map);
}
if (OSHMEM_SUCCESS == rc)
{
context.user_size = user_size;
context.private_size = MEMHEAP_BASE_PRIVATE_SIZE;
context.user_base_addr = (void*)((unsigned char*)mca_memheap_base_map.mem_segs[HEAP_SEG_INDEX].start + 0);
context.private_base_addr = (void*)((unsigned char*)mca_memheap_base_map.mem_segs[HEAP_SEG_INDEX].start + context.user_size);
}
return ( (OSHMEM_SUCCESS == rc) ? &context : NULL );
}

224
oshmem/mca/memheap/base/memheap_base_static.c
Просмотреть файл

@ -1,224 +0,0 @@
/*
* Copyright (c) 2012 Mellanox Technologies, Inc.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "oshmem_config.h"
#include "oshmem/proc/proc.h"
#include "oshmem/mca/memheap/memheap.h"
#include "oshmem/mca/memheap/base/base.h"
#include <stdio.h>
struct map_segment_desc {
uint64_t start;
uint64_t end;
char perms[8];
uint64_t offset;
char dev[8];
uint64_t inode;
char pathname[MAXPATHLEN];
};
typedef struct memheap_static_context
{
struct
{
uint64_t start;
uint64_t end;
} mem_segs[MCA_MEMHEAP_MAX_SEGMENTS];
int n_segments;
} memheap_static_context_t;
static memheap_static_context_t memheap_context;
static int __load_segments(void);
static int __check_perms(struct map_segment_desc *seg);
static int __check_address(struct map_segment_desc *seg);
static int __check_pathname(struct map_segment_desc *seg);
int mca_memheap_base_static_init(mca_memheap_map_t *map)
{
/* read and parse segments from /proc/self/maps */
int ret = OSHMEM_SUCCESS;
assert(map);
assert(SYMB_SEG_INDEX <= map->n_segments);
ret = __load_segments();
if (OSHMEM_SUCCESS == ret)
{
int i;
size_t total_mem;
for (i = 0, total_mem = 0; i < memheap_context.n_segments; i++)
{
map_segment_t *s = &map->mem_segs[map->n_segments];
memset(s, 0, sizeof(*s));
s->is_active = 0;
s->shmid = MEMHEAP_SHM_INVALID;
s->start = memheap_context.mem_segs[i].start;
s->end = memheap_context.mem_segs[i].end;
s->size = s->end - s->start;
s->type = MAP_SEGMENT_STATIC;
s->context = NULL;
map->n_segments++;
total_mem += s->end - s->start;
}
MEMHEAP_VERBOSE(1, "Memheap static memory: %llu byte(s), %d segments",
(unsigned long long)total_mem, map->n_segments);
}
return ret;
}
void mca_memheap_base_static_exit(mca_memheap_map_t *map)
{
assert(map);
}
static int __check_perms(struct map_segment_desc *seg)
{
if (!strcmp(seg->perms, "rw-p") || !strcmp(seg->perms, "rwxp"))
return OSHMEM_SUCCESS;
return OSHMEM_ERROR;
}
static int __check_address(struct map_segment_desc *seg)
{
extern unsigned _end;
unsigned long data_end = (unsigned long)&_end;
/**
* Sasha:
* SGI shmem only supports globals&static in main program.
* It does not support them in shared objects or in dlopen()
* (Clarified on PGAS 2011 tutorial)
*
* So ignored any maps that start higher then process _end
* FIXME: make sure we do not register symmetric heap twice
* if we decide to allow shared objects
*/
if (seg->start > data_end) {
MEMHEAP_VERBOSE(100, "skip segment: data _end < segment start (%llx < %llx)",
(unsigned long long)data_end,
(unsigned long long)seg->start
);
return OSHMEM_ERROR;
}
return OSHMEM_SUCCESS;
}
static int __check_pathname(struct map_segment_desc *seg)
{
return OSHMEM_SUCCESS;
#if 0 /* To press check coverity issue */
char *p;
if ('\0' == seg->pathname[0])
return OSHMEM_SUCCESS;
if (0 == strncmp(seg->pathname, "/lib", 4))
return OSHMEM_ERROR;
if (0 == strncmp(seg->pathname, "/usr/lib", 8))
return OSHMEM_ERROR;
if (0 == strncmp(seg->pathname, "/dev", 4))
return OSHMEM_ERROR;
if (0 == strcmp(seg->pathname, "[stack]"))
return OSHMEM_ERROR;
if (0 == strcmp(seg->pathname, "[vdso]"))
return OSHMEM_ERROR;
if (0 == strcmp(seg->pathname, "[vsyscall]"))
return OSHMEM_ERROR;
p = rindex(seg->pathname, '/');
if (p) {
if (0 == strncmp(p+1, "libshmem.so", 11))
return OSHMEM_ERROR;
if (0 == strncmp(p+1, "libmpi.so", 9))
return OSHMEM_ERROR;
if (0 == strncmp(p+1, "libmca_common_sm.so", 19))
return OSHMEM_ERROR;
}
return OSHMEM_SUCCESS;
#endif
}
static int __load_segments(void)
{
FILE *fp;
char line[1024];
struct map_segment_desc seg;
memheap_context.n_segments = 0;
fp = fopen("/proc/self/maps", "r");
if (NULL == fp) {
MEMHEAP_ERROR("Failed to open /proc/self/maps");
return OSHMEM_ERROR;
}
while (NULL != fgets(line, sizeof(line), fp)) {
memset(&seg, 0, sizeof(seg));
sscanf(line, "%llx-%llx %s %llx %s %llx %s",
(long long *)&seg.start,
(long long *)&seg.end,
seg.perms,
(long long *)&seg.offset,
seg.dev,
(long long *)&seg.inode,
seg.pathname);
if (OSHMEM_ERROR == __check_address(&seg))
continue;
if (OSHMEM_ERROR == __check_pathname(&seg))
continue;
if (OSHMEM_ERROR == __check_perms(&seg))
continue;
MEMHEAP_VERBOSE(5, "add: %s", line);
if (MCA_MEMHEAP_MAX_SEGMENTS <= memheap_context.n_segments) {
MEMHEAP_ERROR("too many segments (max = %d): skip %s",
MCA_MEMHEAP_MAX_SEGMENTS, line);
continue;
}
if (memheap_context.n_segments > 0 &&
seg.start == memheap_context.mem_segs[memheap_context.n_segments-1].end) {
MEMHEAP_VERBOSE(5, "Coalescing segment");
memheap_context.mem_segs[memheap_context.n_segments-1].end = seg.end;
}
else {
memheap_context.mem_segs[memheap_context.n_segments].start = seg.start;
memheap_context.mem_segs[memheap_context.n_segments].end = seg.end;
memheap_context.n_segments ++;
}
}
fclose(fp);
return OSHMEM_SUCCESS;
}

12
oshmem/mca/memheap/buddy/.windows
Просмотреть файл

@ -1,12 +0,0 @@
#
# Copyright (c) 2012 Mellanox Technologies, Inc.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# Specific to this module
mca_dependencies=libshmem libopen-rte

41
oshmem/mca/memheap/buddy/Makefile.am
Просмотреть файл

@ -1,41 +0,0 @@
#
# Copyright (c) 2012 Mellanox Technologies, Inc.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
EXTRA_DIST =
AM_CFLAGS = $(OSHMEM_CFLAGS)
buddy_sources = \
memheap_buddy.c \
memheap_buddy.h \
memheap_buddy_component.c \
memheap_buddy_component.h
#if OMPI_BUILD_memheap_buddy_DSO
if MCA_BUILD_ompi_pml_ob1_DSO
component_noinst =
component_install = mca_memheap_buddy.la
else
component_noinst = libmca_memheap_buddy.la
component_install =
endif
mcacomponentdir = $(pkglibdir)
mcacomponent_LTLIBRARIES = $(component_install)
mca_memheap_buddy_la_SOURCES = $(buddy_sources)
mca_memheap_buddy_la_LDFLAGS = -module -avoid-version
#noinst_LTLIBRARIES = $(lib)
noinst_LTLIBRARIES = $(component_noinst)
libmca_memheap_buddy_la_SOURCES = $(buddy_sources)
libmca_memheap_buddy_la_LDFLAGS = -module -avoid-version

Некоторые файлы не были показаны из-за слишком большого количества измененных файлов Показать больше