From 389b4b3a788a851b26fa2943870294a3c9e771f9 Mon Sep 17 00:00:00 2001 From: Brian Barrett Date: Tue, 28 Apr 2020 02:37:08 +0000 Subject: [PATCH] build: Add third-party package infrastructure With Open MPI 5.0, the decision was made to stop building 3rd-party packages, such as Libevent, HWLOC, PMIx, and PRRTE as MCA components and instead 1) start relying on external libraries whenever possible and 2) Open MPI builds the 3rd party libraries (if needed) as independent libraries, rather than linked into libopen-pal. This patch is the first step in that process, providing foundational changes required for supporting 3rd-party packages, such as changes to autogen.pl, the top-level Makefile.am, and introducing two Autoconf macros to support running sub-configure scripts; one supporting source in tarball form and the other supporting source in a sub-tree. Signed-off-by: Brian Barrett --- 3rd-party/Makefile.am | 28 +++++ Makefile.am | 7 +- autogen.pl | 28 +++++ config/aclocal_subcfg.m4 | 184 +++++++++++++++++++++++++++++++++ config/opal_config_3rdparty.m4 | 83 +++++++++++++++ config/opal_expand_tarball.m4 | 56 ++++++++++ configure.ac | 19 ++++ 7 files changed, 402 insertions(+), 3 deletions(-) create mode 100644 3rd-party/Makefile.am create mode 100644 config/aclocal_subcfg.m4 create mode 100644 config/opal_config_3rdparty.m4 create mode 100644 config/opal_expand_tarball.m4 diff --git a/3rd-party/Makefile.am b/3rd-party/Makefile.am new file mode 100644 index 0000000000..47e40990c6 --- /dev/null +++ b/3rd-party/Makefile.am @@ -0,0 +1,28 @@ +# +# Copyright (c) 2020 Amazon.com, Inc. or its affiliates. +# All Rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# +# This Makefile exists (as opposed to being part of the top level +# Makefile) solely to have a blank check-recursive rule. Otherwise, the +# stability of Open MPI's ability to run "make check" or "make +# distcheck" is dependent on the ability of our 3rd-party packages to +# do the same. Libevent's tests are not stable enough for that kind +# of choice. + +SUBDIRS = $(OPAL_3RDPARTY_SUBDIRS) +DIST_SUBDIRS = $(OPAL_3RDPARTY_DIST_SUBDIRS) +EXTRA_DIST = $(OPAL_3RDPARTY_EXTRA_DIST) + +distclean-local: + rm -rf $(OPAL_3RDPARTY_DISTCLEAN_DIRS) + +check: + @echo "auto-recursing into 3rd-party packages for check disabled" + +check-recursive: + @echo "auto-recursing into 3rd-party packages for check disabled" diff --git a/Makefile.am b/Makefile.am index 7586dc7a43..93f9916a5c 100644 --- a/Makefile.am +++ b/Makefile.am @@ -22,12 +22,12 @@ # $HEADER$ # -SUBDIRS = config contrib $(MCA_PROJECT_SUBDIRS) test +SUBDIRS = config contrib 3rd-party $(MCA_PROJECT_SUBDIRS) test if OMPI_WANT_PRRTE SUBDIRS += prrte endif -DIST_SUBDIRS = config contrib $(MCA_PROJECT_DIST_SUBDIRS) test +DIST_SUBDIRS = config contrib 3rd-party $(MCA_PROJECT_DIST_SUBDIRS) test if OMPI_WANT_PRRTE DIST_SUBDIRS += prrte endif @@ -50,7 +50,8 @@ install-exec-hook: --brief \ --top_builddir=$(top_builddir) \ --top_srcdir=$(top_srcdir) \ - --objext=$(OBJEXT) ; \ + --objext=$(OBJEXT) \ + --skipdir=3rd-party ; \ fi ACLOCAL_AMFLAGS = -I config diff --git a/autogen.pl b/autogen.pl index fb12fa2ed7..40232e43c2 100755 --- a/autogen.pl +++ b/autogen.pl @@ -8,6 +8,8 @@ # Copyright (c) 2015-2019 Research Organization for Information Science # and Technology (RIST). All rights reserved. # Copyright (c) 2015-2020 IBM Corporation. All rights reserved. +# Copyright (c) 2020 Amazon.com, Inc. or its affiliates. +# All Rights reserved. # # $COPYRIGHT$ # @@ -49,6 +51,7 @@ my $no_ompi_arg = 0; my $no_orte_arg = 0; my $no_prrte_arg = 0; my $no_oshmem_arg = 0; +my $no_3rdparty_arg = ""; my $quiet_arg = 0; my $debug_arg = 0; my $help_arg = 0; @@ -1125,6 +1128,7 @@ my $ok = Getopt::Long::GetOptions("no-ompi" => \$no_ompi_arg, "no-orte" => \$no_orte_arg, "no-prrte" => \$no_prrte_arg, "no-oshmem" => \$no_oshmem_arg, + "no-3rdparty=s" => \$no_3rdparty_arg, "quiet|q" => \$quiet_arg, "debug|d" => \$debug_arg, "help|h" => \$help_arg, @@ -1143,6 +1147,7 @@ if (!$ok || $help_arg) { --no-orte | -no-orte Do not build Open MPI's runtime support (alias for --no-prrte) --no-prrte | -no-prrte Do not build Open MPI's runtime support --no-oshmem | -no-oshmem Do not build the OSHMEM layer + --no-3rdparty Do not build the listed 3rd-party package (comma separtated list) --quiet | -q Do not display normal verbose output --debug | -d Output lots of debug information --help | -h This help list @@ -1419,6 +1424,29 @@ mca_run_global($projects); #--------------------------------------------------------------------------- +# Handle 3rd-party packages +++$step; +verbose "\n$step. Setup for 3rd-party packages\n"; + +my @enabled_3rdparty_packages = (); +my @disabled_3rdparty_packages = split(/,/, $no_3rdparty_arg); +if ($no_prrte_arg) { + push(@disabled_3rdparty_packages, "prrte"); +} + +$m4 .= "\n$dnl_line +$dnl_line +$dnl_line + +dnl 3rd-party package information\n"; + +# these are fairly one-off, so we did not try to do anything +# generic. Sorry :). + +$m4 .= "\n"; + +#--------------------------------------------------------------------------- + # Find MPI extensions and contribs if (!$no_ompi_arg) { ++$step; diff --git a/config/aclocal_subcfg.m4 b/config/aclocal_subcfg.m4 new file mode 100644 index 0000000000..a87fc086a0 --- /dev/null +++ b/config/aclocal_subcfg.m4 @@ -0,0 +1,184 @@ +dnl Sandbox configure with additional arguments +dnl Usage: PAC_CONFIG_SUBDIR_ARGS(subdir,configure-args,configure-args-to-remove,action-if-success,action-if-failure) +dnl +dnl The subconfigure argument list is created based on "ac_precious_vars" +dnl instead of explicitly use of well-known Makefile variables, like +dnl CC/CFLAGS/CPPFLAGS..., this generalization is effective as long as +dnl calling configure.ac declares the needed variables to be passed down +dnl to subconfigure as "precious" appropriately. The precious variable +dnl can be created in the following ways: +dnl 1) implicit declaration through use of autoconf macros, like +dnl AC_PROG_CC (declares CC/CFLAGS/CPPFLAGS/LIBS/LDFLAGS), or +dnl AC_PROG_F77 (declares F77/FFLAGS/FLIBS) ... +dnl which are in turns invoked by other subconfigure. +dnl When in doubt, check "ac_precious_var" in the calling configure. +dnl 2) explicit "precious" declaration through AC_ARG_VAR. +dnl Without correct "precious" declaration in the calling configure.ac, +dnl there would be variables not being included in the subconfigure +dnl argument list. +dnl +dnl Note: I suspect this DEFUN body is underquoted in places, but it does not +dnl seem to cause problems in practice yet. [goodell@ 2010-05-18] +AC_DEFUN([PAC_CONFIG_SUBDIR_ARGS],[ + dnl BWB: hack to make --help=recursive work with these + dnl configure options. Lifted from AC_CONFIG_SUBDIRS. + m4_append([_AC_LIST_SUBDIRS], [$1], [ +])dnl + + pac_dir="$1" + AC_MSG_NOTICE([===== configuring $1 =====]) + + pac_abs_srcdir=`(cd $srcdir && pwd)` + + if test -f $pac_abs_srcdir/$1/setup ; then + AC_MSG_NOTICE([sourcing $pac_abs_srcdir/$1/setup]) + . $pac_abs_srcdir/$1/setup + fi + + # Adapted for MPICH from the autoconf-2.67 implementation of + # AC_CONFIG_SUBDIRS. Search for "MPICH note:" for relevant commentary and + # local modifications. + + # Remove --cache-file, --srcdir, and --disable-option-checking arguments + # so they do not pile up. Otherwise relative paths (like --srcdir=.. from + # make distcheck) will be incorrect. + pac_sub_configure_args="$2" + pac_prev= + eval "set x $ac_configure_args" + shift + for pac_arg + do + if test -n "$pac_prev"; then + pac_prev= + continue + fi + case $pac_arg in + -cache-file | --cache-file | --cache-fil | --cache-fi \ + | --cache-f | --cache- | --cache | --cach | --cac | --ca | --c) + pac_prev=cache_file ;; + -cache-file=* | --cache-file=* | --cache-fil=* | --cache-fi=* \ + | --cache-f=* | --cache-=* | --cache=* | --cach=* | --cac=* | --ca=* \ + | --c=*) + ;; + --config-cache | -C) + ;; + -srcdir | --srcdir | --srcdi | --srcd | --src | --sr) + pac_prev=srcdir ;; + -srcdir=* | --srcdir=* | --srcdi=* | --srcd=* | --src=* | --sr=*) + ;; + -prefix | --prefix | --prefi | --pref | --pre | --pr | --p) + pac_prev=prefix ;; + -prefix=* | --prefix=* | --prefi=* | --pref=* | --pre=* | --pr=* | --p=*) + ;; + --disable-option-checking) + ;; + *) + # MPICH note: this is a more robust version of the "precious + # variable" propagation code that was present in the previous + # incarnation of this macro + for pac_pvar in $ac_precious_vars ; do + # check if configure argument token contains the + # precious variable, i.e. "name_of_prec_var=". + if ( echo $pac_arg | grep "^$pac_pvar=" >/dev/null 2>&1 ) ; then + # check if current precious variable is set in env + eval pvar_set=\${$pac_pvar+set} + if test "$pvar_set" = "set" ; then + # Append 'name_of_prec_var=value_of_prec_var' + # to the subconfigure arguments list, where + # value_of_prec_var is fetched from the env. + # this also overrides any value set on the command line + eval pac_pval=\${$pac_pvar} + pac_arg="$pac_pvar=$pac_pval" + break + fi + fi + done + case $pac_arg in + *\'*) pac_arg=`AS_ECHO(["$pac_arg"]) | sed "s/'/'\\\\\\\\''/g"` ;; + esac + AS_VAR_APPEND([pac_sub_configure_args], [" '$pac_arg'"]) ;; + esac + done + + # Always prepend --prefix to ensure using the same prefix + # in subdir configurations. + # MPICH note: see tt#983 for an example of why this is necessary + pac_arg="--prefix=$prefix" + case $pac_arg in + *\'*) pac_arg=`AS_ECHO(["$pac_arg"]) | sed "s/'/'\\\\\\\\''/g"` ;; + esac + pac_sub_configure_args="'$pac_arg' $pac_sub_configure_args" + + # Pass --silent + if test "$silent" = yes; then + pac_sub_configure_args="--silent $pac_sub_configure_args" + fi + + # Always prepend --disable-option-checking to silence warnings, since + # different subdirs can have different --enable and --with options. + pac_sub_configure_args="--disable-option-checking $pac_sub_configure_args" + + # remove arguments specified in third argument from the + # built-up list of arguments + m4_ifnblank([$3], + [m4_foreach(opt, [$3], [pac_sub_configure_args=$(echo $pac_sub_configure_args | sed "s,'opt',,") + ])]) + + pac_popdir=`pwd` + + AS_MKDIR_P(["$pac_dir"]) + # MPICH note: we leave this internal macro reference for now. We can clone + # the macro locally if this turns out to be non-portable across several autoconf + # versions. It sets the following variables: ac_builddir, + # ac_top_builddir_sub, ac_top_build_prefix, ac_srcdir, ac_top_srcdir, + # ac_abs_top_builddir, ac_abs_builddir, ac_abs_top_srcdir, ac_abs_srcdir + _AC_SRCDIRS(["$pac_dir"]) + + cd "$pac_dir" + + # Check for guested configure; otherwise get Cygnus style + # configure. Look for configure in source tree and then the + # build tree, as we sometimes configure from sub-tarballs + # expanded in the build tree. + if test -f "$ac_srcdir/configure.gnu"; then + pac_sub_configure=$ac_srcdir/configure.gnu + elif test -f "$ac_srcdir/configure"; then + pac_sub_configure=$ac_srcdir/configure + elif test -f "configure.gnu"; then + pac_sub_configure="configure.gnu" + ac_srcdir="." + elif test -f "configure"; then + pac_sub_configure="configure" + ac_srcdir="." + else + AC_MSG_WARN([no configuration information is in $pac_dir]) + pac_sub_configure= + fi + + # The recursion is here. + if test -n "$pac_sub_configure"; then + # MPICH note: overriding the cache file on purpose to prevent strange + # issues resulting from inter-dir caching + pac_sub_cache_file="/dev/null" + + AC_MSG_NOTICE([running $SHELL $pac_sub_configure $pac_sub_configure_args --cache-file=$pac_sub_cache_file --srcdir=$ac_srcdir]) + # The eval makes quoting arguments work. + # MPICH note: we want to execute the provided actions, not be silent + # or error out if the subconfigure succeeded/failed + if eval "\$SHELL \"\$pac_sub_configure\" $pac_sub_configure_args \ + --cache-file=\"\$pac_sub_cache_file\" --srcdir=\"\$ac_srcdir\"" + then + # restore the current dir for the provided actions + cd "$pac_popdir" + $4 + else + # restore the current dir for the provided actions + cd "$pac_popdir" + $5 + fi + fi + + cd "$pac_popdir" + + AC_MSG_NOTICE([===== done with $1 configure =====]) +]) diff --git a/config/opal_config_3rdparty.m4 b/config/opal_config_3rdparty.m4 new file mode 100644 index 0000000000..dac2e50cf3 --- /dev/null +++ b/config/opal_config_3rdparty.m4 @@ -0,0 +1,83 @@ +dnl -*- autoconf -*- +dnl +dnl Copyright (c) 2009-2018 Cisco Systems, Inc. All rights reserved +dnl Copyright (c) 2013 Los Alamos National Security, LLC. All rights reserved. +dnl Copyright (c) 2015-2018 Research Organization for Information Science +dnl and Technology (RIST). All rights reserved. +dnl Copyright (c) 2020 Amazon.com, Inc. or its affiliates. All Rights +dnl reserved. +dnl $COPYRIGHT$ +dnl +dnl Additional copyrights may follow +dnl +dnl $HEADER$ +dnl + +dnl OPAL_3RDPARTY_WITH(short package name, long package name, internal supported) +dnl +dnl Basic --with-pkg/--with-pkg-libdir handling for 3rd party +dnl packages, with the big long description of internal/external/path +dnl handling. +dnl +dnl At the end of this macro, with_pkg will contain an empty string or +dnl a path (implying external). Further, the shell variable opal_pkg_mode +dnl will be set to "internal", "external", or "unspecified". If a path is +dnl given to --with-pkg, then opal_pkg_mode will be set to external. +dnl +dnl If m4_ifdef(internal support) does not evaluate to true (ie, at +dnl autogen time), the references to internal in the help strings will +dnl be removed and internal will not be a supported option. +dnl +dnl $1: short package name +dnl $2: long pacakage name +AC_DEFUN([OPAL_3RDPARTY_WITH], [ + m4_ifdef([$3], + [AC_ARG_WITH([$1], + [AC_HELP_STRING([--with-$1(=DIR)], + [Build $2 support. DIR can take one of three values: "internal", "external", or a valid directory name. "internal" forces Open MPI to use its internal copy of $2. "external" forces Open MPI to use an external installation of $2. Supplying a valid directory name also forces Open MPI to use an external installation of $2, and adds DIR/include, DIR/lib, and DIR/lib64 to the search path for headers and libraries. Note that Open MPI no longer supports --without-$1. If no argument is specified, Open MPI will search default locations for $2 and fall back to an internal version if one is not found.])]) + + AC_ARG_WITH([$1-libdir], + [AC_HELP_STRING([--with-$1-libdir=DIR], + [Search for $2 libraries in DIR. Should only be used if an external copy of $2 is being used.])])], + [AC_ARG_WITH([$1], + [AC_HELP_STRING([--with-$1(=DIR)], + [Build $2 support. DIR can take one of two values: "external" or a valid directory name. "external" forces Open MPI to use an external installation of $2. Supplying a valid directory name also forces Open MPI to use an external installation of $2, and adds DIR/include, DIR/lib, and DIR/lib64 to the search path for headers and libraries. Note that Open MPI no longer supports --without-$1. If no argument is specified, Open MPI will search default locations for $2 and error if one is not found.])]) + + AC_ARG_WITH([$1-libdir], + [AC_HELP_STRING([--with-$1-libdir=DIR], + [Search for $2 libraries in DIR. Should only be used if an external copy of $2 is being used.])])]) + + # Bozo check + AS_IF([test "$with_$1" = "no"], + [AC_MSG_WARN([It is not possible to configure Open MPI --without-$1]) + AC_MSG_ERROR([Cannot continue])]) + + AS_IF([test "$with_$1_libdir" = "no" -o "$with_$1_libdir" = "yes"], + [AC_MSG_WARN([yes/no are invalid responses for --with-$1-libdir. Please specify a path.]) + AC_MSG_ERROR([Cannot continue])]) + + # Make sure the user didn't specify --with-$1=internal and + # --with-$1-libdir=whatever (because you can only specify + # --with-$1-libdir when external $2 is being used). + AS_IF([test "$with_$1" = "internal" && test -n "$with_$1_libdir"], + [AC_MSG_WARN([Both --with-$1=internal and --with-$1-libdir=DIR]) + AC_MSG_WARN([were specified, which does not make sense.]) + AC_MSG_ERROR([Cannot continue])]) + + # clean up $with_$1 so that it contains only a path or empty + # string. To determine internal or external preferences, use + # $opal_$1_mode. + AS_IF([test "$with_$1" = "yes"], [with_$1=]) + AS_CASE([$with_$1], + ["internal"], [with_$1="" + opal_$1_mode="internal"], + ["external"], [with_$1="" + opal_$1_mode="external"], + [""], [opal_$1_mode="unspecified"], + [opal_$1_mode="external"]) + + m4_ifdef([$3], [], + [AS_IF([test "$opal_$1_mode" = "internal"], + [AC_MSG_WARN([Invalid argument to --with-$1: internal.]) + AC_MSG_ERROR([Cannot continue])])]) +]) diff --git a/config/opal_expand_tarball.m4 b/config/opal_expand_tarball.m4 new file mode 100644 index 0000000000..ba230a74c5 --- /dev/null +++ b/config/opal_expand_tarball.m4 @@ -0,0 +1,56 @@ +dnl -*- autoconf -*- +dnl +dnl Copyright (c) 2020 Amazon.com, Inc. or its affiliates. All Rights +dnl reserved. +dnl $COPYRIGHT$ +dnl +dnl Additional copyrights may follow +dnl +dnl $HEADER$ +dnl + +dnl OPAL_EXPAND_TARBALL(tarball name, expansion path, test filename) +dnl +dnl path to the tarball to expand, relative to +dnl the source directory. +dnl path relative to the current directory, in +dnl which the tarball should be expanded. +dnl a unique filename in the tarball to test for +dnl existance. +dnl +dnl If // is not readable, +dnl then expand the tarball / in the directory +dnl basename(). + +AC_DEFUN([OPAL_EXPAND_TARBALL_PREREQ], [ + AC_PATH_PROG(GZIP_BIN, [gzip], []) + AC_PATH_PROG(BZIP2_BIN, [bzip2], []) +]) + +AC_DEFUN([OPAL_EXPAND_TARBALL], [ + AC_REQUIRE([OPAL_EXPAND_TARBALL_PREREQ]) + + OPAL_VAR_SCOPE_PUSH([expansion_dir expansion_tarball]) + + expansion_dir=`dirname "$2"` + AS_MKDIR_P([$expansion_dir]) + + expansion_tarball="$ac_abs_confdir/$1" + AS_IF([test ! -r $2/$3], [ + AC_MSG_NOTICE([Expanding $srcdir/$1 in $expansion_dir]) + case "$1" in + *.tar.gz|*.tgz) + AS_IF([test -z "$GZIP_BIN"], [AC_MSG_ERROR([gzip not found])]) + (cd "$expansion_dir" ; umask 0022 ; "$GZIP_BIN" -d < "$expansion_tarball" | tar xf - || AC_MSG_ERROR([failed to extract $1])) + ;; + *.tar.bz2|*.tbz2) + AS_IF([test -z "$BZIP2_BIN"], [AC_MSG_ERROR([bzip2 not found])]) + (cd "$expansion_dir" ; umask 0022 ; "$BZIP_BIN" -d < "$expansion_tarball" | tar xf - || AC_MSG_ERROR([failed to extract $1])) + ;; + *) # maybe tar is smart? + (cd "$expansion_dir" ; umask 0022 ; tar xf "$expansion_tarball" || AC_MSG_ERROR([failed to extract $1])) + ;; + esac]) + + OPAL_VAR_SCOPE_POP +]) diff --git a/configure.ac b/configure.ac index 45934c4136..b999c73596 100644 --- a/configure.ac +++ b/configure.ac @@ -28,6 +28,8 @@ # Copyright (c) 2018 FUJITSU LIMITED. All rights reserved. # Copyright (c) 2019 Triad National Security, LLC. All rights # reserved. +# Copyright (c) 2020 Amazon.com, Inc. or its affiliates. +# All Rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -1096,6 +1098,21 @@ dnl OPAL_SETUP_FT_OPTIONS # they are controlled by the options OPAL_SETUP_FT +################################## +# 3rd-party packages not called ROMIO +################################## +opal_show_title "3rd-party packages" + +OPAL_3RDPARY_SUBDIRS= +OPAL_3RDPARY_DIST_SUBDIRS= +OPAL_3RDPARY_EXTRA_DIST= +OPAL_3RDPARY_DISTCLEAN_DIRS= + +AC_SUBST(OPAL_3RDPARTY_SUBDIRS) +AC_SUBST(OPAL_3RDPARTY_DIST_SUBDIRS) +AC_SUBST(OPAL_3RDPARTY_EXTRA_DIST) +AC_SUBST(OPAL_3RDPARTY_DISTCLEAN_DIRS) + ################################## # MCA ################################## @@ -1443,6 +1460,8 @@ AC_CONFIG_FILES([ contrib/dist/mofed/debian/control contrib/dist/mofed/debian/copyright:LICENSE + 3rd-party/Makefile + test/Makefile test/event/Makefile test/asm/Makefile