diff --git a/3rd-party/Makefile.am b/3rd-party/Makefile.am new file mode 100644 index 0000000000..47e40990c6 --- /dev/null +++ b/3rd-party/Makefile.am @@ -0,0 +1,28 @@ +# +# Copyright (c) 2020 Amazon.com, Inc. or its affiliates. +# All Rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# +# This Makefile exists (as opposed to being part of the top level +# Makefile) solely to have a blank check-recursive rule. Otherwise, the +# stability of Open MPI's ability to run "make check" or "make +# distcheck" is dependent on the ability of our 3rd-party packages to +# do the same. Libevent's tests are not stable enough for that kind +# of choice. + +SUBDIRS = $(OPAL_3RDPARTY_SUBDIRS) +DIST_SUBDIRS = $(OPAL_3RDPARTY_DIST_SUBDIRS) +EXTRA_DIST = $(OPAL_3RDPARTY_EXTRA_DIST) + +distclean-local: + rm -rf $(OPAL_3RDPARTY_DISTCLEAN_DIRS) + +check: + @echo "auto-recursing into 3rd-party packages for check disabled" + +check-recursive: + @echo "auto-recursing into 3rd-party packages for check disabled" diff --git a/Makefile.am b/Makefile.am index 7586dc7a43..93f9916a5c 100644 --- a/Makefile.am +++ b/Makefile.am @@ -22,12 +22,12 @@ # $HEADER$ # -SUBDIRS = config contrib $(MCA_PROJECT_SUBDIRS) test +SUBDIRS = config contrib 3rd-party $(MCA_PROJECT_SUBDIRS) test if OMPI_WANT_PRRTE SUBDIRS += prrte endif -DIST_SUBDIRS = config contrib $(MCA_PROJECT_DIST_SUBDIRS) test +DIST_SUBDIRS = config contrib 3rd-party $(MCA_PROJECT_DIST_SUBDIRS) test if OMPI_WANT_PRRTE DIST_SUBDIRS += prrte endif @@ -50,7 +50,8 @@ install-exec-hook: --brief \ --top_builddir=$(top_builddir) \ --top_srcdir=$(top_srcdir) \ - --objext=$(OBJEXT) ; \ + --objext=$(OBJEXT) \ + --skipdir=3rd-party ; \ fi ACLOCAL_AMFLAGS = -I config diff --git a/autogen.pl b/autogen.pl index fb12fa2ed7..40232e43c2 100755 --- a/autogen.pl +++ b/autogen.pl @@ -8,6 +8,8 @@ # Copyright (c) 2015-2019 Research Organization for Information Science # and Technology (RIST). All rights reserved. # Copyright (c) 2015-2020 IBM Corporation. All rights reserved. +# Copyright (c) 2020 Amazon.com, Inc. or its affiliates. +# All Rights reserved. # # $COPYRIGHT$ # @@ -49,6 +51,7 @@ my $no_ompi_arg = 0; my $no_orte_arg = 0; my $no_prrte_arg = 0; my $no_oshmem_arg = 0; +my $no_3rdparty_arg = ""; my $quiet_arg = 0; my $debug_arg = 0; my $help_arg = 0; @@ -1125,6 +1128,7 @@ my $ok = Getopt::Long::GetOptions("no-ompi" => \$no_ompi_arg, "no-orte" => \$no_orte_arg, "no-prrte" => \$no_prrte_arg, "no-oshmem" => \$no_oshmem_arg, + "no-3rdparty=s" => \$no_3rdparty_arg, "quiet|q" => \$quiet_arg, "debug|d" => \$debug_arg, "help|h" => \$help_arg, @@ -1143,6 +1147,7 @@ if (!$ok || $help_arg) { --no-orte | -no-orte Do not build Open MPI's runtime support (alias for --no-prrte) --no-prrte | -no-prrte Do not build Open MPI's runtime support --no-oshmem | -no-oshmem Do not build the OSHMEM layer + --no-3rdparty Do not build the listed 3rd-party package (comma separtated list) --quiet | -q Do not display normal verbose output --debug | -d Output lots of debug information --help | -h This help list @@ -1419,6 +1424,29 @@ mca_run_global($projects); #--------------------------------------------------------------------------- +# Handle 3rd-party packages +++$step; +verbose "\n$step. Setup for 3rd-party packages\n"; + +my @enabled_3rdparty_packages = (); +my @disabled_3rdparty_packages = split(/,/, $no_3rdparty_arg); +if ($no_prrte_arg) { + push(@disabled_3rdparty_packages, "prrte"); +} + +$m4 .= "\n$dnl_line +$dnl_line +$dnl_line + +dnl 3rd-party package information\n"; + +# these are fairly one-off, so we did not try to do anything +# generic. Sorry :). + +$m4 .= "\n"; + +#--------------------------------------------------------------------------- + # Find MPI extensions and contribs if (!$no_ompi_arg) { ++$step; diff --git a/config/aclocal_subcfg.m4 b/config/aclocal_subcfg.m4 new file mode 100644 index 0000000000..a87fc086a0 --- /dev/null +++ b/config/aclocal_subcfg.m4 @@ -0,0 +1,184 @@ +dnl Sandbox configure with additional arguments +dnl Usage: PAC_CONFIG_SUBDIR_ARGS(subdir,configure-args,configure-args-to-remove,action-if-success,action-if-failure) +dnl +dnl The subconfigure argument list is created based on "ac_precious_vars" +dnl instead of explicitly use of well-known Makefile variables, like +dnl CC/CFLAGS/CPPFLAGS..., this generalization is effective as long as +dnl calling configure.ac declares the needed variables to be passed down +dnl to subconfigure as "precious" appropriately. The precious variable +dnl can be created in the following ways: +dnl 1) implicit declaration through use of autoconf macros, like +dnl AC_PROG_CC (declares CC/CFLAGS/CPPFLAGS/LIBS/LDFLAGS), or +dnl AC_PROG_F77 (declares F77/FFLAGS/FLIBS) ... +dnl which are in turns invoked by other subconfigure. +dnl When in doubt, check "ac_precious_var" in the calling configure. +dnl 2) explicit "precious" declaration through AC_ARG_VAR. +dnl Without correct "precious" declaration in the calling configure.ac, +dnl there would be variables not being included in the subconfigure +dnl argument list. +dnl +dnl Note: I suspect this DEFUN body is underquoted in places, but it does not +dnl seem to cause problems in practice yet. [goodell@ 2010-05-18] +AC_DEFUN([PAC_CONFIG_SUBDIR_ARGS],[ + dnl BWB: hack to make --help=recursive work with these + dnl configure options. Lifted from AC_CONFIG_SUBDIRS. + m4_append([_AC_LIST_SUBDIRS], [$1], [ +])dnl + + pac_dir="$1" + AC_MSG_NOTICE([===== configuring $1 =====]) + + pac_abs_srcdir=`(cd $srcdir && pwd)` + + if test -f $pac_abs_srcdir/$1/setup ; then + AC_MSG_NOTICE([sourcing $pac_abs_srcdir/$1/setup]) + . $pac_abs_srcdir/$1/setup + fi + + # Adapted for MPICH from the autoconf-2.67 implementation of + # AC_CONFIG_SUBDIRS. Search for "MPICH note:" for relevant commentary and + # local modifications. + + # Remove --cache-file, --srcdir, and --disable-option-checking arguments + # so they do not pile up. Otherwise relative paths (like --srcdir=.. from + # make distcheck) will be incorrect. + pac_sub_configure_args="$2" + pac_prev= + eval "set x $ac_configure_args" + shift + for pac_arg + do + if test -n "$pac_prev"; then + pac_prev= + continue + fi + case $pac_arg in + -cache-file | --cache-file | --cache-fil | --cache-fi \ + | --cache-f | --cache- | --cache | --cach | --cac | --ca | --c) + pac_prev=cache_file ;; + -cache-file=* | --cache-file=* | --cache-fil=* | --cache-fi=* \ + | --cache-f=* | --cache-=* | --cache=* | --cach=* | --cac=* | --ca=* \ + | --c=*) + ;; + --config-cache | -C) + ;; + -srcdir | --srcdir | --srcdi | --srcd | --src | --sr) + pac_prev=srcdir ;; + -srcdir=* | --srcdir=* | --srcdi=* | --srcd=* | --src=* | --sr=*) + ;; + -prefix | --prefix | --prefi | --pref | --pre | --pr | --p) + pac_prev=prefix ;; + -prefix=* | --prefix=* | --prefi=* | --pref=* | --pre=* | --pr=* | --p=*) + ;; + --disable-option-checking) + ;; + *) + # MPICH note: this is a more robust version of the "precious + # variable" propagation code that was present in the previous + # incarnation of this macro + for pac_pvar in $ac_precious_vars ; do + # check if configure argument token contains the + # precious variable, i.e. "name_of_prec_var=". + if ( echo $pac_arg | grep "^$pac_pvar=" >/dev/null 2>&1 ) ; then + # check if current precious variable is set in env + eval pvar_set=\${$pac_pvar+set} + if test "$pvar_set" = "set" ; then + # Append 'name_of_prec_var=value_of_prec_var' + # to the subconfigure arguments list, where + # value_of_prec_var is fetched from the env. + # this also overrides any value set on the command line + eval pac_pval=\${$pac_pvar} + pac_arg="$pac_pvar=$pac_pval" + break + fi + fi + done + case $pac_arg in + *\'*) pac_arg=`AS_ECHO(["$pac_arg"]) | sed "s/'/'\\\\\\\\''/g"` ;; + esac + AS_VAR_APPEND([pac_sub_configure_args], [" '$pac_arg'"]) ;; + esac + done + + # Always prepend --prefix to ensure using the same prefix + # in subdir configurations. + # MPICH note: see tt#983 for an example of why this is necessary + pac_arg="--prefix=$prefix" + case $pac_arg in + *\'*) pac_arg=`AS_ECHO(["$pac_arg"]) | sed "s/'/'\\\\\\\\''/g"` ;; + esac + pac_sub_configure_args="'$pac_arg' $pac_sub_configure_args" + + # Pass --silent + if test "$silent" = yes; then + pac_sub_configure_args="--silent $pac_sub_configure_args" + fi + + # Always prepend --disable-option-checking to silence warnings, since + # different subdirs can have different --enable and --with options. + pac_sub_configure_args="--disable-option-checking $pac_sub_configure_args" + + # remove arguments specified in third argument from the + # built-up list of arguments + m4_ifnblank([$3], + [m4_foreach(opt, [$3], [pac_sub_configure_args=$(echo $pac_sub_configure_args | sed "s,'opt',,") + ])]) + + pac_popdir=`pwd` + + AS_MKDIR_P(["$pac_dir"]) + # MPICH note: we leave this internal macro reference for now. We can clone + # the macro locally if this turns out to be non-portable across several autoconf + # versions. It sets the following variables: ac_builddir, + # ac_top_builddir_sub, ac_top_build_prefix, ac_srcdir, ac_top_srcdir, + # ac_abs_top_builddir, ac_abs_builddir, ac_abs_top_srcdir, ac_abs_srcdir + _AC_SRCDIRS(["$pac_dir"]) + + cd "$pac_dir" + + # Check for guested configure; otherwise get Cygnus style + # configure. Look for configure in source tree and then the + # build tree, as we sometimes configure from sub-tarballs + # expanded in the build tree. + if test -f "$ac_srcdir/configure.gnu"; then + pac_sub_configure=$ac_srcdir/configure.gnu + elif test -f "$ac_srcdir/configure"; then + pac_sub_configure=$ac_srcdir/configure + elif test -f "configure.gnu"; then + pac_sub_configure="configure.gnu" + ac_srcdir="." + elif test -f "configure"; then + pac_sub_configure="configure" + ac_srcdir="." + else + AC_MSG_WARN([no configuration information is in $pac_dir]) + pac_sub_configure= + fi + + # The recursion is here. + if test -n "$pac_sub_configure"; then + # MPICH note: overriding the cache file on purpose to prevent strange + # issues resulting from inter-dir caching + pac_sub_cache_file="/dev/null" + + AC_MSG_NOTICE([running $SHELL $pac_sub_configure $pac_sub_configure_args --cache-file=$pac_sub_cache_file --srcdir=$ac_srcdir]) + # The eval makes quoting arguments work. + # MPICH note: we want to execute the provided actions, not be silent + # or error out if the subconfigure succeeded/failed + if eval "\$SHELL \"\$pac_sub_configure\" $pac_sub_configure_args \ + --cache-file=\"\$pac_sub_cache_file\" --srcdir=\"\$ac_srcdir\"" + then + # restore the current dir for the provided actions + cd "$pac_popdir" + $4 + else + # restore the current dir for the provided actions + cd "$pac_popdir" + $5 + fi + fi + + cd "$pac_popdir" + + AC_MSG_NOTICE([===== done with $1 configure =====]) +]) diff --git a/config/opal_config_3rdparty.m4 b/config/opal_config_3rdparty.m4 new file mode 100644 index 0000000000..dac2e50cf3 --- /dev/null +++ b/config/opal_config_3rdparty.m4 @@ -0,0 +1,83 @@ +dnl -*- autoconf -*- +dnl +dnl Copyright (c) 2009-2018 Cisco Systems, Inc. All rights reserved +dnl Copyright (c) 2013 Los Alamos National Security, LLC. All rights reserved. +dnl Copyright (c) 2015-2018 Research Organization for Information Science +dnl and Technology (RIST). All rights reserved. +dnl Copyright (c) 2020 Amazon.com, Inc. or its affiliates. All Rights +dnl reserved. +dnl $COPYRIGHT$ +dnl +dnl Additional copyrights may follow +dnl +dnl $HEADER$ +dnl + +dnl OPAL_3RDPARTY_WITH(short package name, long package name, internal supported) +dnl +dnl Basic --with-pkg/--with-pkg-libdir handling for 3rd party +dnl packages, with the big long description of internal/external/path +dnl handling. +dnl +dnl At the end of this macro, with_pkg will contain an empty string or +dnl a path (implying external). Further, the shell variable opal_pkg_mode +dnl will be set to "internal", "external", or "unspecified". If a path is +dnl given to --with-pkg, then opal_pkg_mode will be set to external. +dnl +dnl If m4_ifdef(internal support) does not evaluate to true (ie, at +dnl autogen time), the references to internal in the help strings will +dnl be removed and internal will not be a supported option. +dnl +dnl $1: short package name +dnl $2: long pacakage name +AC_DEFUN([OPAL_3RDPARTY_WITH], [ + m4_ifdef([$3], + [AC_ARG_WITH([$1], + [AC_HELP_STRING([--with-$1(=DIR)], + [Build $2 support. DIR can take one of three values: "internal", "external", or a valid directory name. "internal" forces Open MPI to use its internal copy of $2. "external" forces Open MPI to use an external installation of $2. Supplying a valid directory name also forces Open MPI to use an external installation of $2, and adds DIR/include, DIR/lib, and DIR/lib64 to the search path for headers and libraries. Note that Open MPI no longer supports --without-$1. If no argument is specified, Open MPI will search default locations for $2 and fall back to an internal version if one is not found.])]) + + AC_ARG_WITH([$1-libdir], + [AC_HELP_STRING([--with-$1-libdir=DIR], + [Search for $2 libraries in DIR. Should only be used if an external copy of $2 is being used.])])], + [AC_ARG_WITH([$1], + [AC_HELP_STRING([--with-$1(=DIR)], + [Build $2 support. DIR can take one of two values: "external" or a valid directory name. "external" forces Open MPI to use an external installation of $2. Supplying a valid directory name also forces Open MPI to use an external installation of $2, and adds DIR/include, DIR/lib, and DIR/lib64 to the search path for headers and libraries. Note that Open MPI no longer supports --without-$1. If no argument is specified, Open MPI will search default locations for $2 and error if one is not found.])]) + + AC_ARG_WITH([$1-libdir], + [AC_HELP_STRING([--with-$1-libdir=DIR], + [Search for $2 libraries in DIR. Should only be used if an external copy of $2 is being used.])])]) + + # Bozo check + AS_IF([test "$with_$1" = "no"], + [AC_MSG_WARN([It is not possible to configure Open MPI --without-$1]) + AC_MSG_ERROR([Cannot continue])]) + + AS_IF([test "$with_$1_libdir" = "no" -o "$with_$1_libdir" = "yes"], + [AC_MSG_WARN([yes/no are invalid responses for --with-$1-libdir. Please specify a path.]) + AC_MSG_ERROR([Cannot continue])]) + + # Make sure the user didn't specify --with-$1=internal and + # --with-$1-libdir=whatever (because you can only specify + # --with-$1-libdir when external $2 is being used). + AS_IF([test "$with_$1" = "internal" && test -n "$with_$1_libdir"], + [AC_MSG_WARN([Both --with-$1=internal and --with-$1-libdir=DIR]) + AC_MSG_WARN([were specified, which does not make sense.]) + AC_MSG_ERROR([Cannot continue])]) + + # clean up $with_$1 so that it contains only a path or empty + # string. To determine internal or external preferences, use + # $opal_$1_mode. + AS_IF([test "$with_$1" = "yes"], [with_$1=]) + AS_CASE([$with_$1], + ["internal"], [with_$1="" + opal_$1_mode="internal"], + ["external"], [with_$1="" + opal_$1_mode="external"], + [""], [opal_$1_mode="unspecified"], + [opal_$1_mode="external"]) + + m4_ifdef([$3], [], + [AS_IF([test "$opal_$1_mode" = "internal"], + [AC_MSG_WARN([Invalid argument to --with-$1: internal.]) + AC_MSG_ERROR([Cannot continue])])]) +]) diff --git a/config/opal_expand_tarball.m4 b/config/opal_expand_tarball.m4 new file mode 100644 index 0000000000..ba230a74c5 --- /dev/null +++ b/config/opal_expand_tarball.m4 @@ -0,0 +1,56 @@ +dnl -*- autoconf -*- +dnl +dnl Copyright (c) 2020 Amazon.com, Inc. or its affiliates. All Rights +dnl reserved. +dnl $COPYRIGHT$ +dnl +dnl Additional copyrights may follow +dnl +dnl $HEADER$ +dnl + +dnl OPAL_EXPAND_TARBALL(tarball name, expansion path, test filename) +dnl +dnl path to the tarball to expand, relative to +dnl the source directory. +dnl path relative to the current directory, in +dnl which the tarball should be expanded. +dnl a unique filename in the tarball to test for +dnl existance. +dnl +dnl If // is not readable, +dnl then expand the tarball / in the directory +dnl basename(). + +AC_DEFUN([OPAL_EXPAND_TARBALL_PREREQ], [ + AC_PATH_PROG(GZIP_BIN, [gzip], []) + AC_PATH_PROG(BZIP2_BIN, [bzip2], []) +]) + +AC_DEFUN([OPAL_EXPAND_TARBALL], [ + AC_REQUIRE([OPAL_EXPAND_TARBALL_PREREQ]) + + OPAL_VAR_SCOPE_PUSH([expansion_dir expansion_tarball]) + + expansion_dir=`dirname "$2"` + AS_MKDIR_P([$expansion_dir]) + + expansion_tarball="$ac_abs_confdir/$1" + AS_IF([test ! -r $2/$3], [ + AC_MSG_NOTICE([Expanding $srcdir/$1 in $expansion_dir]) + case "$1" in + *.tar.gz|*.tgz) + AS_IF([test -z "$GZIP_BIN"], [AC_MSG_ERROR([gzip not found])]) + (cd "$expansion_dir" ; umask 0022 ; "$GZIP_BIN" -d < "$expansion_tarball" | tar xf - || AC_MSG_ERROR([failed to extract $1])) + ;; + *.tar.bz2|*.tbz2) + AS_IF([test -z "$BZIP2_BIN"], [AC_MSG_ERROR([bzip2 not found])]) + (cd "$expansion_dir" ; umask 0022 ; "$BZIP_BIN" -d < "$expansion_tarball" | tar xf - || AC_MSG_ERROR([failed to extract $1])) + ;; + *) # maybe tar is smart? + (cd "$expansion_dir" ; umask 0022 ; tar xf "$expansion_tarball" || AC_MSG_ERROR([failed to extract $1])) + ;; + esac]) + + OPAL_VAR_SCOPE_POP +]) diff --git a/configure.ac b/configure.ac index 45934c4136..b999c73596 100644 --- a/configure.ac +++ b/configure.ac @@ -28,6 +28,8 @@ # Copyright (c) 2018 FUJITSU LIMITED. All rights reserved. # Copyright (c) 2019 Triad National Security, LLC. All rights # reserved. +# Copyright (c) 2020 Amazon.com, Inc. or its affiliates. +# All Rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -1096,6 +1098,21 @@ dnl OPAL_SETUP_FT_OPTIONS # they are controlled by the options OPAL_SETUP_FT +################################## +# 3rd-party packages not called ROMIO +################################## +opal_show_title "3rd-party packages" + +OPAL_3RDPARY_SUBDIRS= +OPAL_3RDPARY_DIST_SUBDIRS= +OPAL_3RDPARY_EXTRA_DIST= +OPAL_3RDPARY_DISTCLEAN_DIRS= + +AC_SUBST(OPAL_3RDPARTY_SUBDIRS) +AC_SUBST(OPAL_3RDPARTY_DIST_SUBDIRS) +AC_SUBST(OPAL_3RDPARTY_EXTRA_DIST) +AC_SUBST(OPAL_3RDPARTY_DISTCLEAN_DIRS) + ################################## # MCA ################################## @@ -1443,6 +1460,8 @@ AC_CONFIG_FILES([ contrib/dist/mofed/debian/control contrib/dist/mofed/debian/copyright:LICENSE + 3rd-party/Makefile + test/Makefile test/event/Makefile test/asm/Makefile