1
1

build: Add third-party package infrastructure

With Open MPI 5.0, the decision was made to stop building 3rd-party
packages, such as Libevent, HWLOC, PMIx, and PRRTE as MCA components
and instead 1) start relying on external libraries whenever possible
and 2) Open MPI builds the 3rd party libraries (if needed) as
independent libraries, rather than linked into libopen-pal.

This patch is the first step in that process, providing foundational
changes required for supporting 3rd-party packages, such as changes
to autogen.pl, the top-level Makefile.am, and introducing two
Autoconf macros to support running sub-configure scripts; one
supporting source in tarball form and the other supporting
source in a sub-tree.

Signed-off-by: Brian Barrett <bbarrett@amazon.com>
Этот коммит содержится в:
Brian Barrett 2020-04-28 02:37:08 +00:00
родитель 675a899532
Коммит 389b4b3a78
7 изменённых файлов: 402 добавлений и 3 удалений

28
3rd-party/Makefile.am поставляемый Обычный файл
Просмотреть файл

@ -0,0 +1,28 @@
#
# Copyright (c) 2020 Amazon.com, Inc. or its affiliates.
# All Rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# This Makefile exists (as opposed to being part of the top level
# Makefile) solely to have a blank check-recursive rule. Otherwise, the
# stability of Open MPI's ability to run "make check" or "make
# distcheck" is dependent on the ability of our 3rd-party packages to
# do the same. Libevent's tests are not stable enough for that kind
# of choice.
SUBDIRS = $(OPAL_3RDPARTY_SUBDIRS)
DIST_SUBDIRS = $(OPAL_3RDPARTY_DIST_SUBDIRS)
EXTRA_DIST = $(OPAL_3RDPARTY_EXTRA_DIST)
distclean-local:
rm -rf $(OPAL_3RDPARTY_DISTCLEAN_DIRS)
check:
@echo "auto-recursing into 3rd-party packages for check disabled"
check-recursive:
@echo "auto-recursing into 3rd-party packages for check disabled"

Просмотреть файл

@ -22,12 +22,12 @@
# $HEADER$
#
SUBDIRS = config contrib $(MCA_PROJECT_SUBDIRS) test
SUBDIRS = config contrib 3rd-party $(MCA_PROJECT_SUBDIRS) test
if OMPI_WANT_PRRTE
SUBDIRS += prrte
endif
DIST_SUBDIRS = config contrib $(MCA_PROJECT_DIST_SUBDIRS) test
DIST_SUBDIRS = config contrib 3rd-party $(MCA_PROJECT_DIST_SUBDIRS) test
if OMPI_WANT_PRRTE
DIST_SUBDIRS += prrte
endif
@ -50,7 +50,8 @@ install-exec-hook:
--brief \
--top_builddir=$(top_builddir) \
--top_srcdir=$(top_srcdir) \
--objext=$(OBJEXT) ; \
--objext=$(OBJEXT) \
--skipdir=3rd-party ; \
fi
ACLOCAL_AMFLAGS = -I config

Просмотреть файл

@ -8,6 +8,8 @@
# Copyright (c) 2015-2019 Research Organization for Information Science
# and Technology (RIST). All rights reserved.
# Copyright (c) 2015-2020 IBM Corporation. All rights reserved.
# Copyright (c) 2020 Amazon.com, Inc. or its affiliates.
# All Rights reserved.
#
# $COPYRIGHT$
#
@ -49,6 +51,7 @@ my $no_ompi_arg = 0;
my $no_orte_arg = 0;
my $no_prrte_arg = 0;
my $no_oshmem_arg = 0;
my $no_3rdparty_arg = "";
my $quiet_arg = 0;
my $debug_arg = 0;
my $help_arg = 0;
@ -1125,6 +1128,7 @@ my $ok = Getopt::Long::GetOptions("no-ompi" => \$no_ompi_arg,
"no-orte" => \$no_orte_arg,
"no-prrte" => \$no_prrte_arg,
"no-oshmem" => \$no_oshmem_arg,
"no-3rdparty=s" => \$no_3rdparty_arg,
"quiet|q" => \$quiet_arg,
"debug|d" => \$debug_arg,
"help|h" => \$help_arg,
@ -1143,6 +1147,7 @@ if (!$ok || $help_arg) {
--no-orte | -no-orte Do not build Open MPI's runtime support (alias for --no-prrte)
--no-prrte | -no-prrte Do not build Open MPI's runtime support
--no-oshmem | -no-oshmem Do not build the OSHMEM layer
--no-3rdparty <package> Do not build the listed 3rd-party package (comma separtated list)
--quiet | -q Do not display normal verbose output
--debug | -d Output lots of debug information
--help | -h This help list
@ -1419,6 +1424,29 @@ mca_run_global($projects);
#---------------------------------------------------------------------------
# Handle 3rd-party packages
++$step;
verbose "\n$step. Setup for 3rd-party packages\n";
my @enabled_3rdparty_packages = ();
my @disabled_3rdparty_packages = split(/,/, $no_3rdparty_arg);
if ($no_prrte_arg) {
push(@disabled_3rdparty_packages, "prrte");
}
$m4 .= "\n$dnl_line
$dnl_line
$dnl_line
dnl 3rd-party package information\n";
# these are fairly one-off, so we did not try to do anything
# generic. Sorry :).
$m4 .= "\n";
#---------------------------------------------------------------------------
# Find MPI extensions and contribs
if (!$no_ompi_arg) {
++$step;

184
config/aclocal_subcfg.m4 Обычный файл
Просмотреть файл

@ -0,0 +1,184 @@
dnl Sandbox configure with additional arguments
dnl Usage: PAC_CONFIG_SUBDIR_ARGS(subdir,configure-args,configure-args-to-remove,action-if-success,action-if-failure)
dnl
dnl The subconfigure argument list is created based on "ac_precious_vars"
dnl instead of explicitly use of well-known Makefile variables, like
dnl CC/CFLAGS/CPPFLAGS..., this generalization is effective as long as
dnl calling configure.ac declares the needed variables to be passed down
dnl to subconfigure as "precious" appropriately. The precious variable
dnl can be created in the following ways:
dnl 1) implicit declaration through use of autoconf macros, like
dnl AC_PROG_CC (declares CC/CFLAGS/CPPFLAGS/LIBS/LDFLAGS), or
dnl AC_PROG_F77 (declares F77/FFLAGS/FLIBS) ...
dnl which are in turns invoked by other subconfigure.
dnl When in doubt, check "ac_precious_var" in the calling configure.
dnl 2) explicit "precious" declaration through AC_ARG_VAR.
dnl Without correct "precious" declaration in the calling configure.ac,
dnl there would be variables not being included in the subconfigure
dnl argument list.
dnl
dnl Note: I suspect this DEFUN body is underquoted in places, but it does not
dnl seem to cause problems in practice yet. [goodell@ 2010-05-18]
AC_DEFUN([PAC_CONFIG_SUBDIR_ARGS],[
dnl BWB: hack to make --help=recursive work with these
dnl configure options. Lifted from AC_CONFIG_SUBDIRS.
m4_append([_AC_LIST_SUBDIRS], [$1], [
])dnl
pac_dir="$1"
AC_MSG_NOTICE([===== configuring $1 =====])
pac_abs_srcdir=`(cd $srcdir && pwd)`
if test -f $pac_abs_srcdir/$1/setup ; then
AC_MSG_NOTICE([sourcing $pac_abs_srcdir/$1/setup])
. $pac_abs_srcdir/$1/setup
fi
# Adapted for MPICH from the autoconf-2.67 implementation of
# AC_CONFIG_SUBDIRS. Search for "MPICH note:" for relevant commentary and
# local modifications.
# Remove --cache-file, --srcdir, and --disable-option-checking arguments
# so they do not pile up. Otherwise relative paths (like --srcdir=.. from
# make distcheck) will be incorrect.
pac_sub_configure_args="$2"
pac_prev=
eval "set x $ac_configure_args"
shift
for pac_arg
do
if test -n "$pac_prev"; then
pac_prev=
continue
fi
case $pac_arg in
-cache-file | --cache-file | --cache-fil | --cache-fi \
| --cache-f | --cache- | --cache | --cach | --cac | --ca | --c)
pac_prev=cache_file ;;
-cache-file=* | --cache-file=* | --cache-fil=* | --cache-fi=* \
| --cache-f=* | --cache-=* | --cache=* | --cach=* | --cac=* | --ca=* \
| --c=*)
;;
--config-cache | -C)
;;
-srcdir | --srcdir | --srcdi | --srcd | --src | --sr)
pac_prev=srcdir ;;
-srcdir=* | --srcdir=* | --srcdi=* | --srcd=* | --src=* | --sr=*)
;;
-prefix | --prefix | --prefi | --pref | --pre | --pr | --p)
pac_prev=prefix ;;
-prefix=* | --prefix=* | --prefi=* | --pref=* | --pre=* | --pr=* | --p=*)
;;
--disable-option-checking)
;;
*)
# MPICH note: this is a more robust version of the "precious
# variable" propagation code that was present in the previous
# incarnation of this macro
for pac_pvar in $ac_precious_vars ; do
# check if configure argument token contains the
# precious variable, i.e. "name_of_prec_var=".
if ( echo $pac_arg | grep "^$pac_pvar=" >/dev/null 2>&1 ) ; then
# check if current precious variable is set in env
eval pvar_set=\${$pac_pvar+set}
if test "$pvar_set" = "set" ; then
# Append 'name_of_prec_var=value_of_prec_var'
# to the subconfigure arguments list, where
# value_of_prec_var is fetched from the env.
# this also overrides any value set on the command line
eval pac_pval=\${$pac_pvar}
pac_arg="$pac_pvar=$pac_pval"
break
fi
fi
done
case $pac_arg in
*\'*) pac_arg=`AS_ECHO(["$pac_arg"]) | sed "s/'/'\\\\\\\\''/g"` ;;
esac
AS_VAR_APPEND([pac_sub_configure_args], [" '$pac_arg'"]) ;;
esac
done
# Always prepend --prefix to ensure using the same prefix
# in subdir configurations.
# MPICH note: see tt#983 for an example of why this is necessary
pac_arg="--prefix=$prefix"
case $pac_arg in
*\'*) pac_arg=`AS_ECHO(["$pac_arg"]) | sed "s/'/'\\\\\\\\''/g"` ;;
esac
pac_sub_configure_args="'$pac_arg' $pac_sub_configure_args"
# Pass --silent
if test "$silent" = yes; then
pac_sub_configure_args="--silent $pac_sub_configure_args"
fi
# Always prepend --disable-option-checking to silence warnings, since
# different subdirs can have different --enable and --with options.
pac_sub_configure_args="--disable-option-checking $pac_sub_configure_args"
# remove arguments specified in third argument from the
# built-up list of arguments
m4_ifnblank([$3],
[m4_foreach(opt, [$3], [pac_sub_configure_args=$(echo $pac_sub_configure_args | sed "s,'opt',,")
])])
pac_popdir=`pwd`
AS_MKDIR_P(["$pac_dir"])
# MPICH note: we leave this internal macro reference for now. We can clone
# the macro locally if this turns out to be non-portable across several autoconf
# versions. It sets the following variables: ac_builddir,
# ac_top_builddir_sub, ac_top_build_prefix, ac_srcdir, ac_top_srcdir,
# ac_abs_top_builddir, ac_abs_builddir, ac_abs_top_srcdir, ac_abs_srcdir
_AC_SRCDIRS(["$pac_dir"])
cd "$pac_dir"
# Check for guested configure; otherwise get Cygnus style
# configure. Look for configure in source tree and then the
# build tree, as we sometimes configure from sub-tarballs
# expanded in the build tree.
if test -f "$ac_srcdir/configure.gnu"; then
pac_sub_configure=$ac_srcdir/configure.gnu
elif test -f "$ac_srcdir/configure"; then
pac_sub_configure=$ac_srcdir/configure
elif test -f "configure.gnu"; then
pac_sub_configure="configure.gnu"
ac_srcdir="."
elif test -f "configure"; then
pac_sub_configure="configure"
ac_srcdir="."
else
AC_MSG_WARN([no configuration information is in $pac_dir])
pac_sub_configure=
fi
# The recursion is here.
if test -n "$pac_sub_configure"; then
# MPICH note: overriding the cache file on purpose to prevent strange
# issues resulting from inter-dir caching
pac_sub_cache_file="/dev/null"
AC_MSG_NOTICE([running $SHELL $pac_sub_configure $pac_sub_configure_args --cache-file=$pac_sub_cache_file --srcdir=$ac_srcdir])
# The eval makes quoting arguments work.
# MPICH note: we want to execute the provided actions, not be silent
# or error out if the subconfigure succeeded/failed
if eval "\$SHELL \"\$pac_sub_configure\" $pac_sub_configure_args \
--cache-file=\"\$pac_sub_cache_file\" --srcdir=\"\$ac_srcdir\""
then
# restore the current dir for the provided actions
cd "$pac_popdir"
$4
else
# restore the current dir for the provided actions
cd "$pac_popdir"
$5
fi
fi
cd "$pac_popdir"
AC_MSG_NOTICE([===== done with $1 configure =====])
])

83
config/opal_config_3rdparty.m4 Обычный файл
Просмотреть файл

@ -0,0 +1,83 @@
dnl -*- autoconf -*-
dnl
dnl Copyright (c) 2009-2018 Cisco Systems, Inc. All rights reserved
dnl Copyright (c) 2013 Los Alamos National Security, LLC. All rights reserved.
dnl Copyright (c) 2015-2018 Research Organization for Information Science
dnl and Technology (RIST). All rights reserved.
dnl Copyright (c) 2020 Amazon.com, Inc. or its affiliates. All Rights
dnl reserved.
dnl $COPYRIGHT$
dnl
dnl Additional copyrights may follow
dnl
dnl $HEADER$
dnl
dnl OPAL_3RDPARTY_WITH(short package name, long package name, internal supported)
dnl
dnl Basic --with-pkg/--with-pkg-libdir handling for 3rd party
dnl packages, with the big long description of internal/external/path
dnl handling.
dnl
dnl At the end of this macro, with_pkg will contain an empty string or
dnl a path (implying external). Further, the shell variable opal_pkg_mode
dnl will be set to "internal", "external", or "unspecified". If a path is
dnl given to --with-pkg, then opal_pkg_mode will be set to external.
dnl
dnl If m4_ifdef(internal support) does not evaluate to true (ie, at
dnl autogen time), the references to internal in the help strings will
dnl be removed and internal will not be a supported option.
dnl
dnl $1: short package name
dnl $2: long pacakage name
AC_DEFUN([OPAL_3RDPARTY_WITH], [
m4_ifdef([$3],
[AC_ARG_WITH([$1],
[AC_HELP_STRING([--with-$1(=DIR)],
[Build $2 support. DIR can take one of three values: "internal", "external", or a valid directory name. "internal" forces Open MPI to use its internal copy of $2. "external" forces Open MPI to use an external installation of $2. Supplying a valid directory name also forces Open MPI to use an external installation of $2, and adds DIR/include, DIR/lib, and DIR/lib64 to the search path for headers and libraries. Note that Open MPI no longer supports --without-$1. If no argument is specified, Open MPI will search default locations for $2 and fall back to an internal version if one is not found.])])
AC_ARG_WITH([$1-libdir],
[AC_HELP_STRING([--with-$1-libdir=DIR],
[Search for $2 libraries in DIR. Should only be used if an external copy of $2 is being used.])])],
[AC_ARG_WITH([$1],
[AC_HELP_STRING([--with-$1(=DIR)],
[Build $2 support. DIR can take one of two values: "external" or a valid directory name. "external" forces Open MPI to use an external installation of $2. Supplying a valid directory name also forces Open MPI to use an external installation of $2, and adds DIR/include, DIR/lib, and DIR/lib64 to the search path for headers and libraries. Note that Open MPI no longer supports --without-$1. If no argument is specified, Open MPI will search default locations for $2 and error if one is not found.])])
AC_ARG_WITH([$1-libdir],
[AC_HELP_STRING([--with-$1-libdir=DIR],
[Search for $2 libraries in DIR. Should only be used if an external copy of $2 is being used.])])])
# Bozo check
AS_IF([test "$with_$1" = "no"],
[AC_MSG_WARN([It is not possible to configure Open MPI --without-$1])
AC_MSG_ERROR([Cannot continue])])
AS_IF([test "$with_$1_libdir" = "no" -o "$with_$1_libdir" = "yes"],
[AC_MSG_WARN([yes/no are invalid responses for --with-$1-libdir. Please specify a path.])
AC_MSG_ERROR([Cannot continue])])
# Make sure the user didn't specify --with-$1=internal and
# --with-$1-libdir=whatever (because you can only specify
# --with-$1-libdir when external $2 is being used).
AS_IF([test "$with_$1" = "internal" && test -n "$with_$1_libdir"],
[AC_MSG_WARN([Both --with-$1=internal and --with-$1-libdir=DIR])
AC_MSG_WARN([were specified, which does not make sense.])
AC_MSG_ERROR([Cannot continue])])
# clean up $with_$1 so that it contains only a path or empty
# string. To determine internal or external preferences, use
# $opal_$1_mode.
AS_IF([test "$with_$1" = "yes"], [with_$1=])
AS_CASE([$with_$1],
["internal"], [with_$1=""
opal_$1_mode="internal"],
["external"], [with_$1=""
opal_$1_mode="external"],
[""], [opal_$1_mode="unspecified"],
[opal_$1_mode="external"])
m4_ifdef([$3], [],
[AS_IF([test "$opal_$1_mode" = "internal"],
[AC_MSG_WARN([Invalid argument to --with-$1: internal.])
AC_MSG_ERROR([Cannot continue])])])
])

56
config/opal_expand_tarball.m4 Обычный файл
Просмотреть файл

@ -0,0 +1,56 @@
dnl -*- autoconf -*-
dnl
dnl Copyright (c) 2020 Amazon.com, Inc. or its affiliates. All Rights
dnl reserved.
dnl $COPYRIGHT$
dnl
dnl Additional copyrights may follow
dnl
dnl $HEADER$
dnl
dnl OPAL_EXPAND_TARBALL(tarball name, expansion path, test filename)
dnl
dnl <tarball name> path to the tarball to expand, relative to
dnl the source directory.
dnl <expansion path> path relative to the current directory, in
dnl which the tarball should be expanded.
dnl <test filename> a unique filename in the tarball to test for
dnl existance.
dnl
dnl If <build dir>/<expansion path>/<test filename> is not readable,
dnl then expand the tarball <src dir>/<tarball name> in the directory
dnl basename(<expansion path>).
AC_DEFUN([OPAL_EXPAND_TARBALL_PREREQ], [
AC_PATH_PROG(GZIP_BIN, [gzip], [])
AC_PATH_PROG(BZIP2_BIN, [bzip2], [])
])
AC_DEFUN([OPAL_EXPAND_TARBALL], [
AC_REQUIRE([OPAL_EXPAND_TARBALL_PREREQ])
OPAL_VAR_SCOPE_PUSH([expansion_dir expansion_tarball])
expansion_dir=`dirname "$2"`
AS_MKDIR_P([$expansion_dir])
expansion_tarball="$ac_abs_confdir/$1"
AS_IF([test ! -r $2/$3], [
AC_MSG_NOTICE([Expanding $srcdir/$1 in $expansion_dir])
case "$1" in
*.tar.gz|*.tgz)
AS_IF([test -z "$GZIP_BIN"], [AC_MSG_ERROR([gzip not found])])
(cd "$expansion_dir" ; umask 0022 ; "$GZIP_BIN" -d < "$expansion_tarball" | tar xf - || AC_MSG_ERROR([failed to extract $1]))
;;
*.tar.bz2|*.tbz2)
AS_IF([test -z "$BZIP2_BIN"], [AC_MSG_ERROR([bzip2 not found])])
(cd "$expansion_dir" ; umask 0022 ; "$BZIP_BIN" -d < "$expansion_tarball" | tar xf - || AC_MSG_ERROR([failed to extract $1]))
;;
*) # maybe tar is smart?
(cd "$expansion_dir" ; umask 0022 ; tar xf "$expansion_tarball" || AC_MSG_ERROR([failed to extract $1]))
;;
esac])
OPAL_VAR_SCOPE_POP
])

Просмотреть файл

@ -28,6 +28,8 @@
# Copyright (c) 2018 FUJITSU LIMITED. All rights reserved.
# Copyright (c) 2019 Triad National Security, LLC. All rights
# reserved.
# Copyright (c) 2020 Amazon.com, Inc. or its affiliates.
# All Rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
@ -1096,6 +1098,21 @@ dnl OPAL_SETUP_FT_OPTIONS
# they are controlled by the options
OPAL_SETUP_FT
##################################
# 3rd-party packages not called ROMIO
##################################
opal_show_title "3rd-party packages"
OPAL_3RDPARY_SUBDIRS=
OPAL_3RDPARY_DIST_SUBDIRS=
OPAL_3RDPARY_EXTRA_DIST=
OPAL_3RDPARY_DISTCLEAN_DIRS=
AC_SUBST(OPAL_3RDPARTY_SUBDIRS)
AC_SUBST(OPAL_3RDPARTY_DIST_SUBDIRS)
AC_SUBST(OPAL_3RDPARTY_EXTRA_DIST)
AC_SUBST(OPAL_3RDPARTY_DISTCLEAN_DIRS)
##################################
# MCA
##################################
@ -1443,6 +1460,8 @@ AC_CONFIG_FILES([
contrib/dist/mofed/debian/control
contrib/dist/mofed/debian/copyright:LICENSE
3rd-party/Makefile
test/Makefile
test/event/Makefile
test/asm/Makefile