From 8cb780d1af50ee2c77dd4dd4ec8ab2a884830911 Mon Sep 17 00:00:00 2001 From: Jeff Squyres Date: Sat, 18 Sep 2010 09:07:56 +0000 Subject: [PATCH] Remove some really old stuff that isn't used anymore. This commit was SVN r23768. --- contrib/nightly/build_sample_config.txt | 54 -- contrib/nightly/build_tarball.pl | 1027 ----------------------- contrib/nightly/build_tests.pl | 962 --------------------- contrib/nightly/ompi_cronjob.sh | 167 ---- contrib/nightly/unimplemented_report.sh | 357 -------- 5 files changed, 2567 deletions(-) delete mode 100644 contrib/nightly/build_sample_config.txt delete mode 100755 contrib/nightly/build_tarball.pl delete mode 100755 contrib/nightly/build_tests.pl delete mode 100755 contrib/nightly/ompi_cronjob.sh delete mode 100755 contrib/nightly/unimplemented_report.sh diff --git a/contrib/nightly/build_sample_config.txt b/contrib/nightly/build_sample_config.txt deleted file mode 100644 index 8065b27336..0000000000 --- a/contrib/nightly/build_sample_config.txt +++ /dev/null @@ -1,54 +0,0 @@ -# -# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana -# University Research and Technology -# Corporation. All rights reserved. -# Copyright (c) 2004-2005 The University of Tennessee and The University -# of Tennessee Research Foundation. All rights -# reserved. -# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, -# University of Stuttgart. All rights reserved. -# Copyright (c) 2004-2005 The Regents of the University of California. -# All rights reserved. -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# -# Sample configuration file for the test_build.pl script. -# -# First argument: String name of the configuration, used for -# describing and labeling the result. Names should not be repeated in -# one configuration file (no error checking is performed; if multiple -# lines have the same name, the results of only one of them will be -# reported). -# -# Second argument: Whether to combine stdout and stderr (1), or leave -# them separate (blank). You should probably only use (1) for when -# you want to see the warnings from "make all", and leave it blank for -# all other cases. -# -# Third argument: VPATH mode. Must be the strings "absolute", -# "relative", or blank. If "absolute", then the configure script will -# be run as an absolute VPATH (i.e., top_builddir != top_srcdir, and -# configure's absolute path will be used for invocation). If -# "relative", the setup as the same except that a relative path to -# configure is used. If blank (or any other value), a non-VPATH build -# is used. -# -# Fourth argument: arbitrary strings to pass to configure. You can -# set environment flags here, specific options to configure, etc. You -# should *not* specify a --prefix argument; that is the only argument -# supplied by the testing script. -# -# Examples: -# -# warnings:1::--enable-picky -# default build::: -# absolute vpath:::absolute:CFLAGS=-g -# static build:::--disable-shared --enable-static -# -warnings:1::--enable-picky -default build::: -static build:::--disable-shared --enable-static -non-threaded build:::--without-threads diff --git a/contrib/nightly/build_tarball.pl b/contrib/nightly/build_tarball.pl deleted file mode 100755 index 33167d72fa..0000000000 --- a/contrib/nightly/build_tarball.pl +++ /dev/null @@ -1,1027 +0,0 @@ -#!/usr/bin/env perl -# -# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana -# University Research and Technology -# Corporation. All rights reserved. -# Copyright (c) 2004-2005 The University of Tennessee and The University -# of Tennessee Research Foundation. All rights -# reserved. -# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, -# University of Stuttgart. All rights reserved. -# Copyright (c) 2004-2005 The Regents of the University of California. -# All rights reserved. -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# -# This script is used to take a nightly snapshot tarball of Open MPI -# and try building it. -# -# $1: scratch root -# $2: e-mail address for destination -# $3: URL_ARG -# $4: config file -# - -use strict; -use Getopt::Long; -use File::Basename; - -# Set this to true for additional output; typically only when -# debugging -my $debug = 0; - -# -# Set this to true, if You want to disable checking -# Useful, if the check might fail on Your machine, but you -# nevertheless want to compile/debug other tests. -# -my $nocheck = 0; - -# -# Set this to the installation directory -# (possibly accesible by all nodes in a cluster). -# -my $install_dir; - -# download "latest" filename -my $latest_name = "latest_snapshot.txt"; - -# checksum filenames -my $md5_checksums = "md5sums.txt"; -my $sha1_checksums = "sha1sums.txt"; - -# max length of logfile to send in an e-mail -my $max_log_len = 100; - -# email subjects -my $success_subject = "Open MPI: Build success (\@version@)"; -my $fail_subject = "Open MPI: === BUILD FAILURE (\@version@) ==="; - -# max number of snapshots to keep downloaded -my $max_snapshots = 3; - -# max number of old build roots to maintain (note that build roots are -# only left if a) you use --leave-install or b) a build fails) -my $max_build_roots = 3; - -# make arguments -my $make_args = "-j 2"; - -############################################################################ -# Shouldn't need to change below this line -############################################################################ - -# Need to define some globals here -- can't use "our" because we have -# to run on machines with older versions of perl. #$%#@$%#... - -my $version; -my $mail; -my @email_output; -my $tarball_name; -my $ret; -my $last_test_version_name; -my $save_tmpdir; -my $save_tmpdir_exists; -my $scratch_root_arg; -my $email_arg; -my $url_arg; -my $config_arg; -my $debug_arg; -my $file_arg; -my $patches_arg; -my $leave_install_arg; -my $help_arg; -my $force_arg; -my $nocheck_arg; -my $install_dir_arg; -my $make_arg; - -#-------------------------------------------------------------------------- - -# send a mail -sub send_mail { - my ($subject, $to) = @_; - shift; - shift; - my $msg = \@_; - - # Restore tmpdir -- if we don't do this, for some reason (when - # TMPDIR has been changed), we get permission denied's from the - # mail command (something about a temporary file cannot be created - # -- shrug). - if ($save_tmpdir_exists) { - $ENV{TMPDIR} = $save_tmpdir; - } else { - delete $ENV{TMPDIR}; - } - - $subject =~ s/\@version\@/$version/; - - open MAIL, "|$mail -s \"$subject\" \"$to\"" || - die "Could not open pipe to output e-mail\n"; - my $i = 0; - while ($i <= $#$msg) { - print MAIL $$msg[$i]; - ++$i; - } - close MAIL; -} - -#-------------------------------------------------------------------------- - -# abort the script, and send an error e-mail as a result -sub test_abort { - my $msg = \@_; - - push(@email_output, "Building the nightly tarball ended in error:\n\n"); - push(@email_output, @$msg); - send_mail($fail_subject, $email_arg, @email_output); - exit(1); -} - -#-------------------------------------------------------------------------- - -# check to see if we've tested this version before -sub check_last_version { - my ($new_version) = @_; - my $old_version; - - print "This version: $new_version\n" if ($debug); - if (! -f $last_test_version_name) { - print "Never tested on this system before\n" if ($debug); - return; - } - - $old_version = `cat $last_test_version_name`; - chomp($old_version); - print "Last tested version: $old_version\n" if ($debug); - - # If the version hasn't changed since the last time we tested, - # then don't bother testing again. - - if ($new_version eq $old_version) { - if ($debug || $force_arg) { - print "Test tarball version is the same as the last version we tested\nOnly testing again because we're in --debug or --force mode\n" - if ($debug); - } else { - exit(0); - } - } -} - -#-------------------------------------------------------------------------- - -# run a command and save the stdout / stderr -sub do_command { - my ($merge_output, $cmd) = @_; - - print "*** Running command: $cmd\n" if ($debug); - pipe OUTread, OUTwrite; - pipe ERRread, ERRwrite - if (!$merge_output); - - # Child - - my $pid; - if (($pid = fork()) == 0) { - close OUTread; - close ERRread - if (!$merge_output); - - close(STDERR); - if ($merge_output) { - open STDERR, ">&OUTwrite" || - die "Can't redirect stderr\n"; - } else { - open STDERR, ">&ERRwrite" || - die "Can't redirect stderr\n"; - } - select STDERR; - $| = 1; - - close(STDOUT); - open STDOUT, ">&OUTwrite" || - die "Can't redirect stdout\n"; - select STDOUT; - $| = 1; - - # Turn shell-quoted words ("foo bar baz") into individual tokens - - my @tokens; - while ($cmd =~ /\".*\"/) { - my $prefix; - my $middle; - my $suffix; - - $cmd =~ /(.*?)\"(.*?)\"(.*)/; - $prefix = $1; - $middle = $2; - $suffix = $3; - - if ($prefix) { - foreach my $token (split(' ', $prefix)) { - push(@tokens, $token); - } - } - if ($middle) { - push(@tokens, $middle); - } else { - push(@tokens, ""); - } - $cmd = $suffix; - } - if ($cmd) { - push(@tokens, split(' ', $cmd)); - } - - # Run it! - - exec(@tokens) || - die "Can't execute command: $cmd\n"; - } - close OUTwrite; - close ERRwrite - if (!$merge_output); - - # Parent - - my (@out, @err); - my ($rin, $rout); - my $done = $merge_output ? 1 : 2; - - # Keep watching over the pipe(s) - - $rin = ''; - vec($rin, fileno(OUTread), 1) = 1; - vec($rin, fileno(ERRread), 1) = 1 - if (!$merge_output); - - while ($done > 0) { - my $nfound = select($rout = $rin, undef, undef, undef); - if (vec($rout, fileno(OUTread), 1) == 1) { - my $data = ; - if (!defined($data)) { - vec($rin, fileno(OUTread), 1) = 0; - --$done; - } else { - push(@out, $data); - print "OUT:$data" if ($debug); - } - } - - if (!$merge_output && vec($rout, fileno(ERRread), 1) == 1) { - my $data = ; - if (!defined($data)) { - vec($rin, fileno(ERRread), 1) = 0; - --$done; - } else { - push(@err, $data); - print "ERR:$data" if ($debug); - } - } - } - close OUTerr; - close OUTread - if (!$merge_output); - - # The pipes are closed, so the process should be dead. Reap it. - - waitpid($pid, 0); - my $status = $?; - print "*** Command complete, exit status: $status\n" if ($debug); - - # Return an anonymous hash containing the relevant data - - my $ret = { - stdout => \@out, - status => $status - }; - - # If we had stderr, return that, too - - $ret->{stderr} = \@err - if (!$merge_output); - return $ret; -} - -#-------------------------------------------------------------------------- - -# find a program from a list and load it into the target variable -sub find_program { - my @names = @_; - - # loop through the list and save the first one that we find - my $i = 0; - while ($i <= $#names) { - foreach my $dir (split(/:/, $ENV{PATH})) { - if (-x "$dir/$names[$i]") { - return $names[$i]; - } - } - ++$i; - } - return undef; -} - -#-------------------------------------------------------------------------- - -sub trim { - my ($max, $msg) = @_; - - my @out; - - my $i = 0; - if ($#$msg > $max) { - $i = $#$msg - $max + 1; - push(@out, "[...previous $i lines snipped; last $max lines shown...]\n"); - } - while ($i <= $#$msg) { - push(@out, $$msg[$i]); - ++$i; - } - if (! ($out[$#out] =~ /\n/)) { - push(@out, "\n"); - } - \@out; -} - -#-------------------------------------------------------------------------- - -# subroutine for building a single configuration -sub try_build { - my ($name, $merge_output, $tarball, $srcroot, $installdir, - $vpath_mode, $confargs) = @_; - my $ret; - my $startdir = `pwd`; - my $unpacker; - my $status; - - chomp($startdir); - - # make the source root - if (! -d $srcroot) { - mkdir($srcroot, 0777); - } - chdir($srcroot); - - # expand the tarball (do NOT assume GNU tar). don't use - # do_command here because we need to use a pipe. This isn't an - # "interesting" command, anyway -- we don't need any stdout or - # stderr. So if it fails, it fails -- no big deal. - - - if ($tarball =~ /.*\.bz2$/) { - $unpacker="bunzip2"; - } - elsif ($tarball =~ /.*\.gz$/) { - $unpacker="gunzip"; - } - else { - return { - status => $status, - message => "Failed to figure out compression of tarball", - } - } - - system("$unpacker -c $tarball | tar xf -"); - $status = $? >> 8; - if ($status != 0) { - return { - status => $status, - message => "Failed to unzip the tarball", - } - } - - test_abort ("Tarball does not contain expected openmpi-directory:openmpi-$version -- aborting") - if (! -d "openmpi-$version"); - - chdir("openmpi-$version"); - - # In case of patch-files passed to build_tarball, patch from the root-level source-dir - if ($patches_arg) { - my $dirname; - $dirname = dirname("$tarball"); - foreach my $patch_file (split(',', $patches_arg)) { - - if (! -e "$dirname/$patch_file") { - print "Patch-File $dirname/$patch_file not available -- skipping\n" if ($debug); - next; - } - - $ret = do_command($merge_output, "patch -p0 -i $dirname/$patch_file"); - if ($ret->{status} != 0) { - $ret->{message} = "Failed to \"patch with $dirname/$patch_file\""; - return $ret; - } - } - } - - # configure it - my $config_command = "./configure"; - if ($vpath_mode) { - mkdir("vpath_build", 0777); - chdir("vpath_build"); - if ($vpath_mode eq "relative") { - $config_command = "../configure"; - } else { - $config_command = "$srcroot/openmpi-$version/configure"; - } - } - $ret = do_command(1, "$config_command $confargs --prefix=$installdir"); - if ($ret->{status} != 0) { - $ret->{message} = "Failed to configure the tarball"; - return $ret; - } - - # build it - $ret = do_command($merge_output, "make $make_args all"); - if ($ret->{status} != 0) { - $ret->{message} = "Failed to \"make $make_args all\""; - return $ret; - } - - # save the compile warnings - my $make_all_stderr = $ret->{stderr}; - - # and check it, if the user does not disable it. Make sure to - # restore tmpdir. - if ( $nocheck == 0 ) { - $ENV{TMPDIR} = "$installdir/tmp"; - mkdir($ENV{TMPDIR}, 0777); - $ret = do_command($merge_output, "make check"); - if ($save_tmpdir_exists) { - $ENV{TMPDIR} = $save_tmpdir; - } else { - delete $ENV{TMPDIR}; - } - if ($ret->{status} != 0) { - $ret->{make_all_stderr} = $make_all_stderr; - $ret->{message} = "Failed to \"make check\""; - return $ret; - } - } - - # Ensure LD_LIBRARY_PATH points to our shared libraries - - if (exists($ENV{LD_LIBRARY_PATH})) { - $ENV{LD_LIBRARY_PATH} = "$installdir/lib:" . $ENV{LD_LIBRARY_PATH}; - } else { - $ENV{LD_LIBRARY_PATH}; - } - - # install it - $ret = do_command(1, "make install"); - if ($ret->{status} != 0) { - $ret->{make_all_stderr} = $make_all_stderr; - $ret->{message} = "Failed to \"make install\""; - return $ret; - } - - # try compiling and linking a simple C application - chdir(".."); - if (! -d test) { - mkdir("test-compile", 0777); - } - chdir("test-compile"); - open C, ">hello.c"; - print C "#include -int main(int argc, char* argv[]) { - MPI_Init(&argc, &argv); - MPI_Finalize(); - return 0; -}\n"; - close(C); - $ret = do_command(1, "$installdir/bin/mpicc hello.c -o hello"); - if ($ret->{status} != 0) { - $ret->{make_all_stderr} = $make_all_stderr; - $ret->{message} = "Failed to compile/link C \"hello world\" MPI app"; - return $ret; - } - unlink "hello.c", "hello"; - - # if we have a C++ compiler, try compiling and linking a simple - # C++ application - open INFO, "$installdir/bin/ompi_info --parsable|"; - my @have_cxx = grep { /^bindings:cxx:/ } ; - chomp @have_cxx; - close INFO; - if ($have_cxx[0] eq "bindings:cxx:yes") { - open CXX, ">hello.cc"; - print CXX "#include -int main(int argc, char* argv[]) { - MPI::Init(argc, argv); - MPI::Finalize(); - return 0; -}\n"; - close(CXX); - do_command(1, "$installdir/bin/mpic++ hello.cc -o hello"); - if ($ret->{status} != 0) { - $ret->{make_all_stderr} = $make_all_stderr; - $ret->{message} = - "Failed to compile/link C++ \"hello world\" MPI app"; - return $ret; - } - unlink "hello.cc", "hello"; - } - - # if we have a F77 compiler, try compiling and linking a simple - # F77 application - open INFO, "$installdir/bin/ompi_info --parsable|"; - my @have_f77 = grep { /^bindings:f77:/ } ; - chomp(@have_f77); - close INFO; - if ($have_f77[0] eq "bindings:f77:yes") { - open F77, ">hello.f"; - print F77 "C - program main - include 'mpif.h' - call MPI_INIT(ierr) - call MPI_FINALIZE(ierr) - stop - end\n"; - close(F77); - do_command(1, "$installdir/bin/mpif77 hello.f -o hello"); - if ($ret->{status} != 0) { - $ret->{make_all_stderr} = $make_all_stderr; - $ret->{message} = - "Failed to compile/link F77 \"hello world\" MPI app"; - return $ret; - } - unlink "hello.f", "hello"; - } - - # all done -- clean up (unless user specified --leave-install) - chdir($startdir); - if ($leave_install_arg) { - system("rm -rf $srcroot/openmpi* $srcroot/test-compile"); - } else { - system("rm -rf $srcroot"); - } - - return { - make_all_stderr => $make_all_stderr, - status => 0, - }; -} - -#-------------------------------------------------------------------------- - -# -# main -# - -$save_tmpdir = $ENV{TMPDIR}; -$save_tmpdir_exists = exists $ENV{TMPDIR}; - -# parse the command line -&Getopt::Long::Configure("bundling", "require_order"); -my $ok = Getopt::Long::GetOptions("url|u=s" => \$url_arg, - "scratch|s=s" => \$scratch_root_arg, - "email|e=s" => \$email_arg, - "config|c=s" => \$config_arg, - "file|f=s" => \$file_arg, - "patches|p=s" => \$patches_arg, - "debug|d" => \$debug_arg, - "leave-install|l=s" => \$leave_install_arg, - "install-dir=s" => \$install_dir_arg, - "help|h" => \$help_arg, - "force" => \$force_arg, - "nocheck" => \$nocheck_arg, - "make=s" => \$make_arg - ); - -if (!$ok || $help_arg) { - print("Command line error\n") - if (!$ok); - print "Usage: $0 [--scratch|-s scratch_directory_root] [--email|-e address]\n"; - print " [--config|-c config_file] [--help|-h] [--debug|-d]\n"; - print " [[--file|-f local_ompi_tarball] [--url|-u URL_base]]\n"; - print " [--patches|-p list_of_patch_files]\n"; - print " [--leave-install output_filename] [--install-dir install_dir]\n"; - print " [--nocheck]\n"; - exit(0); -} - -if ($file_arg && $url_arg) { - print("ERROR: Both --url and --file specified; which should I do?\n"); - die("Aborting in confusion!\n"); -} elsif (! $file_arg && ! $url_arg) { - print("ERROR: Neither --url nor --file specified; what should I do?\n"); - die("Aborting in confusion!\n"); -} elsif (! $email_arg) { - print("ERROR: Need e-mail address to mail results\n"); - die("Aborting in despair\n"); -} elsif (! $scratch_root_arg) { - my $tmp = "/tmp"; - $tmp = $ENV{TMPDIR} - if ($ENV{TMPDIR}); - $scratch_root_arg = "$tmp/open-mpi-test-build.$$"; -} - -# see if we want to override the make arguments -if ($make_arg) { - $make_args = $make_arg; -} - -# if the --file argument was a relative file, convert it to absolute -# because we're going to chdir before using it -if ($file_arg) { - if (! ($file_arg =~ /^\//)) { - my $tmp = `pwd`; - chomp($tmp); - $file_arg = "$tmp/$file_arg"; - } -} - -# ditto for --config arg -if ($config_arg) { - if (! ($config_arg =~ /^\//)) { - my $tmp = `pwd`; - chomp($tmp); - $config_arg = "$tmp/$config_arg"; - } -} - -# turn on debugging? -$debug = 1 - if ($debug_arg); - -# Disable checking? -$nocheck = 1 - if ($nocheck_arg); - -# prefix the e-mail -my $str = "Host: "; -my $first = 1; -foreach my $option (qw/n o r m/) { - my $out = `uname -$option 2>/dev/null`; - chomp($out); - if ($out) { - $str .= " / " - if (! $first); - $str .= $out; - $first = 0; - } -} -push(@email_output, $str . "\n"); - -# now that we have scratch root, fill in the filename for the last -# version that we tested -- separate it by hostname and config name so -# that multiple instances of this script can share a downloads -# directory -my $dir = "$scratch_root_arg/last-test-versions"; -if (! -d $dir) { - mkdir($dir); -} -my $hostname = `uname -n`; -chomp($hostname); -my $config = $config_arg ? `basename $config_arg` : "default"; -chomp($config); -$last_test_version_name = "$dir/$hostname-$config"; - -# Find a mail program -$mail = find_program(qw(Mail mailx mail)); -die "Could not find mail program; aborting in despair\n" - if (!defined($mail)); - -# figure out what download command to use -# if we need to download a version -- with file_arg, we do not -my $download = "true"; -if ($url_arg) { - $download = find_program(qw(wget lynx curl)); - test_abort("Cannot find downloading program -- aborting in despair\n") - if (!defined($download)); -} - -# move into the scratch directory, and ensure we have an absolute path -# for it -if (! -d $scratch_root_arg) { - mkdir($scratch_root_arg, 0777); -} -test_abort("Could not cd to scratch root: $scratch_root_arg\n") - if (! -d $scratch_root_arg); -chdir($scratch_root_arg); -$scratch_root_arg = `pwd`; -chomp($scratch_root_arg); - -# if we were given a URL base, get the latest snapshot version number -if ($url_arg) { - - # ensure some subdirs exist - foreach my $dir (qw(downloads)) { - if (! -d $dir) { - mkdir($dir, 0777); - } - } - - chdir("downloads"); - - unlink($latest_name); - do_command(1, "$download $url_arg/$latest_name"); - test_abort("Could not download latest snapshot number -- aborting") - if (! -f $latest_name); - $version = `cat $latest_name`; - chomp($version); - - check_last_version($version); - - push(@email_output, "Snapshot: $version\n\n"); - - # see if we need to download the tarball - $tarball_name = "openmpi-$version.tar.gz"; - if (! -f $tarball_name) { - do_command(1, "$download $url_arg/$tarball_name"); - test_abort ("Could not download tarball -- aborting") - if (! -f $tarball_name); - - # get the checksums - unlink($md5_checksums); - do_command(1, "$download $url_arg/$md5_checksums"); - - unlink($sha1_checksums); - do_command(1, "$download $url_arg/$sha1_checksums"); - } - - # compare the md5sum - my $md5_file = `grep $version.tar.gz $md5_checksums`; - chomp($md5_file); - my $md5sum = find_program(qw(md5sum gmd5sum)); - if (!defined($md5sum)) { - push(@email_output, -"WARNING: Could not find md5sum executable, so I will not be able to check -WARNING: the validity of downloaded executables against their known MD5 -WARNING: checksums. Proceeding anyway...\n\n"); - } elsif (! $md5_file) { - push(@email_output, -"WARNING: Could not find md5sum check file, so I will not be able to check -WARNING: the validity of downloaded executables against their known MD5 -WARNING: checksums. Proceeding anyway...\n\n"); - } else { - my $md5_actual = `$md5sum $tarball_name 2>&1`; - chomp($md5_actual); - test_abort("md5sum from checksum file does not match actual ($md5_file != $md5_actual)") - if ($md5_file ne $md5_actual); - } - - # compare the sha1sum - my $sha1_file = `grep $version.tar.gz $sha1_checksums`; - chomp($sha1_file); - my $sha1sum = find_program(qw(sha1sum gsha1sum)); - if (!defined($sha1sum)) { - push(@email_output, -"WARNING: Could not find sha1sum executable, so I will not be able to check -WARNING: the validity of downloaded executables against their known SHA1 -WARNING: checksums. Proceeding anyway...\n\n"); - } elsif (! $sha1_file) { - push(@email_output, -"WARNING: Could not find sha1sum check file, so I will not be able to check -WARNING: the validity of downloaded executables against their known SHA1 -WARNING: checksums. Proceeding anyway...\n\n"); - } else { - my $sha1_actual = `$sha1sum $tarball_name 2>&1`; - chomp($sha1_actual); - test_abort "sha1sum from checksum file does not match actual ($sha1_file != $sha1_actual)" - if ($sha1_file ne $sha1_actual); - } - - # now adjust the tarball name to be absolute - $tarball_name = "$scratch_root_arg/downloads/$tarball_name"; -} elsif ($file_arg) { - $tarball_name = $file_arg; - - die "ERROR: Cannot read file $tarball_name\n" - if (! -r $tarball_name); - $version = $tarball_name; - $version =~ s/.*openmpi-(.+)\.tar.*/$1/; - - check_last_version($version); - - push(@email_output, "Snapshot: $version\n\n"); -} - -# Make a root for this build to play in (scratch_root_arg is absolute, so -# root will be absolute) -my $root= "$scratch_root_arg/build-$version"; -system("rm -rf $root"); -mkdir($root, 0777); -chdir($root); - -# loop over all configurations -# be lazy: if no configurations supplied, do a default -# configure/build -my $results; -my $do_default = 0; -if (! $config_arg || ! -f $config_arg) { - my $dir = "$root/default"; - my $name = "[default]"; - my $config = "CFLAGS=-g --enable-debug"; - $ret = try_build($name, 0, $tarball_name, - $dir, "$dir/install", "", $config); - $results->{$name} = $ret; - $results->{$name}->{config} = $config; - $results->{$name}->{want_stderr} = 1; - $results->{$name}->{vpath_mode} = ""; - $results->{$name}->{installdir} = "$dir/install"; -} else { - open CONF, "$config_arg"; - my $i = 1; - while () { - my $line = $_; - chomp($line); - - # skip comments and blank lines - - if ($line && ! ($line =~ /[ \t]*\#/) && ! ($line =~ /^[ \t]+$/)) { - - # Parse out the parts - - my ($name, $want_stderr, $vpath_mode, $config) = split(/:/, $line); - if (! $name) { - $name = "[build-$i]"; - } - - # try to actually build it - - my $dir = "$root/$name"; - $dir =~ s/[\[\] \t"']/_/g; - # stupid emacs: '" - - $install_dir="$dir/install"; - if ($install_dir_arg) { - $install_dir="$install_dir_arg/$name"; - } - - my $merge_output = $want_stderr ? 0 : 1; - $ret = try_build($name, $merge_output, $tarball_name, - $dir, $install_dir, - $vpath_mode, $config); - $results->{$name} = $ret; - $results->{$name}->{config} = $config; - $results->{$name}->{want_stderr} = $want_stderr; - $results->{$name}->{vpath_mode} = $vpath_mode; - $results->{$name}->{installdir} = "$install_dir"; - ++$i; - } - } - close CONF; -} - -# remove this root if we're not leaving the install root -chdir($scratch_root_arg); -system("rm -rf build-$version") - if (!$leave_install_arg); - -my $dir = "$scratch_root_arg"; -opendir(ROOT, $dir); -my @roots = sort(grep { /^build.+[0-9]$/ && -d "$dir/$_" } readdir(ROOT)); - -my $i = $#roots; -my $j = 0; -while ($i >= $max_build_roots) { - print "Removing directory $roots[$j]\n" if ($debug); - system("rm -rf $roots[$j]"); - ++$j; - --$i; -} - -# trim the downloads dir to $max_snapshots -my $dir = "$scratch_root_arg/downloads"; -opendir(DOWNLOADS, $dir); -my @tarballs = sort(grep { /^openmpi.+gz$/ && -f "$dir/$_" } readdir(DOWNLOADS)); -closedir(DOWNLOADS); -$i = $#tarballs; -$j = 0; -while ($i >= $max_snapshots) { - print "Unlinking $tarballs[$j]\n" if ($debug); - unlink("$dir/$tarballs[$j]"); - ++$j; - --$i; -} - -# save the version that we just tested -open LAST, ">$last_test_version_name"; -print LAST "$version\n"; -close LAST; - -# if we're leaving the installdirs, output their names into the file -# indicated by the --leave-install arg -open LEAVE, ">$leave_install_arg" - if ($leave_install_arg); - -# send results mail -my $email_subject; -push(@email_output, -"SUMMARY OF RESULTS: - (Building tarball and compiling/linking MPI \"hello world\") ---------------------------------------------------------------------------\n"); -foreach my $config (keys(%$results)) { - my $str; - if ($results->{$config}->{status} == 0) { - $str = "Success "; - print LEAVE "$results->{$config}->{installdir}\n" - if ($leave_install_arg); - } else { - $str = "FAILURE "; - $email_subject = $fail_subject; - } - $str .= $config . "\n"; - push(@email_output, $str); -} -close(LEAVE) - if ($leave_install_arg); -$email_subject = $success_subject - if (!$email_subject); - -push(@email_output, -"-------------------------------------------------------------------------- -\n"); - - -# Include additional details if relevant -my $header = 0; -my $displayed = 0; -push(@email_output, "DETAILED RESULTS (as necessary):\n\n"); -foreach my $config (keys(%$results)) { - my $output = 0; - my $str; - - $str .= "==> Build: " . $config . " -==> Config options: " . $results->{$config}->{config} . " -==> Result: " . ($results->{$config}->{status} == 0 ? - "Success" : "FAILURE") . " -==> Output: " . ($results->{$config}->{want_stderr} ? - "stderr and stdout shown separately below" : - "stdout and stderr merged below") . "\n"; - - if ($results->{$config}->{status} != 0 && - $results->{$config}->{stdout} && - $#{$results->{$config}->{stdout}} >= 0) { - push(@email_output, $str); - $header = $displayed = $output = 1; - - if ($results->{$config}->{want_stderr}) { - push(@email_output, -"--Standard output---------------------------------------------------------\n"); - } else { - push(@email_output, -"--Standard output and standard error--------------------------------------\n"); - } - my $tmp = trim($max_log_len, $results->{$config}->{stdout}); - push(@email_output, @$tmp); - if ($results->{$config}->{want_stderr}) { - push(@email_output, -"--Standard output end-----------------------------------------------------\n"); - } else { - push(@email_output, -"--Standard output and standard error end----------------------------------\n"); - } - } - if ($results->{$config}->{stderr} && - $#{$results->{$config}->{stderr}} >= 0) { - push(@email_output, $str) - if ($output == 0); - $header = $displayed = $output = 1; - - push(@email_output, -"--Standard error----------------------------------------------------------\n"); - my $tmp = trim($max_log_len, $results->{$config}->{stderr}); - push(@email_output, @$tmp); - push(@email_output, -"--Standard error end------------------------------------------------------\n"); - } - if ($results->{$config}->{want_stderr} && - $results->{$config}->{make_all_stderr} && - $#{$results->{$config}->{make_all_stderr}} >= 0) { - push(@email_output, $str) - if ($output == 0); - $header = $displayed = $output = 1; - - push(@email_output, -"--\"make all\" standard error---------------------------------------------------\n"); - my $tmp = trim($max_log_len, $results->{$config}->{make_all_stderr}); - push(@email_output, @$tmp); - push(@email_output, -"--\"make all\" standard error end-----------------------------------------------\n"); - } - - if ($output == 1) { - push(@email_output, -"\n==========================================================================\n\n"); - } -} - -# if we didn't display anything extra, then say so - -if (!$displayed) { - push(@email_output, "No additional interesting intformation\n\n"); -} - -push(@email_output, "Your friendly server,\nCyrador\n"); -send_mail($email_subject, $email_arg, @email_output); - -# All done - -exit(0); diff --git a/contrib/nightly/build_tests.pl b/contrib/nightly/build_tests.pl deleted file mode 100755 index 7162ec2d96..0000000000 --- a/contrib/nightly/build_tests.pl +++ /dev/null @@ -1,962 +0,0 @@ -#!/usr/bin/env perl -# -# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana -# University Research and Technology -# Corporation. All rights reserved. -# Copyright (c) 2004-2005 The University of Tennessee and The University -# of Tennessee Research Foundation. All rights -# reserved. -# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, -# University of Stuttgart. All rights reserved. -# Copyright (c) 2004-2005 The Regents of the University of California. -# All rights reserved. -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# - -use strict; -use Getopt::Long; - -# Set this to true for additional output; typically only when -# debugging -my $debug = 0; - -# -# Set this to true, if You want to disable checking -# Useful, if the check might fail on Your machine, but you -# nevertheless want to compile/debug other tests. -# -my $nocheck = 0; - -# -# Set this to the installation directory -# (possibly accesible by all nodes in a cluster). -# -my $install_dir; - -# download "latest" filename -my $latest_name = "latest_snapshot.txt"; - -# checksum filenames -my $md5_checksums = "md5sums.txt"; -my $sha1_checksums = "sha1sums.txt"; - -# max length of logfile to send in an e-mail -my $max_log_len = 100; - -# email subjects -my $success_subject = "Open MPI-TESTS: Build success (\@version@)"; -my $fail_subject = "Open MPI-TESTS: === BUILD FAILURE (\@version@) ==="; - -# max number of snapshots to keep downloaded -my $max_snapshots = 3; - -# max number of old build roots to maintain (note that build roots are -# only left if a) you use --leave-install or b) a build fails) -my $max_build_roots = 3; - -my $file_base_name="openmpi-tests"; - -my $outfile; -my $outfile_arg; - -############################################################################ -# Shouldn't need to change below this line -############################################################################ - -# Need to define some globals here -- can't use "our" because we have -# to run on machines with older versions of perl. #$%#@$%#... - -my $version; -my $mail; -my @email_output; -my $tarball_name; -my $ret; -my $last_test_version_name; - -my $scratch_root_arg; -my $email_arg; -my $url_arg; -my $config_arg; -my $debug_arg; -my $file_arg; -my $leave_install_arg; -my $help_arg; -my $force_arg; -my $nocheck_arg; -my $install_dir_arg; - -#-------------------------------------------------------------------------- - -# send a mail -sub send_mail { - my ($subject, $to) = @_; - shift; - shift; - my $msg = \@_; - - $subject =~ s/\@version\@/$version/; - - open MAIL, "|$mail -s \"$subject\" \"$to\"" || - die "Could not open pipe to output e-mail\n"; - my $i = 0; - while ($i <= $#$msg) { - print MAIL $$msg[$i]; - ++$i; - } - close MAIL; -} - -#-------------------------------------------------------------------------- - -# abort the script, and send an error e-mail as a result -sub test_abort { - my $msg = \@_; - - push(@email_output, "Building the nightly test-tarball ended in error:\n\n"); - push(@email_output, @$msg); - send_mail($fail_subject, $email_arg, @email_output); - exit(1); -} - -#-------------------------------------------------------------------------- - -# check to see if we've tested this version before -sub check_last_version { - my ($new_version) = @_; - my $old_version; - - print "This version: $new_version\n" if ($debug); - if (! -f $last_test_version_name) { - print "Never tested on this system before\n" if ($debug); - return; - } - - $old_version = `cat $last_test_version_name`; - chomp($old_version); - print "Last tested version: $old_version\n" if ($debug); - - # If the version hasn't changed since the last time we tested, - # then don't bother testing again. - - if ($new_version eq $old_version) { - if ($debug || $force_arg) { - print "Test tarball version is the same as the last version we tested\nOnly testing again because we're in --debug or --force mode\n" - if ($debug); - } else { - exit(0); - } - } -} - -#-------------------------------------------------------------------------- - -# run a command and save the stdout / stderr -sub do_command { - my ($merge_output, $cmd) = @_; - - print "*** Running command: $cmd\n" if ($debug); - pipe OUTread, OUTwrite; - pipe ERRread, ERRwrite - if (!$merge_output); - - # Child - - my $pid; - if (($pid = fork()) == 0) { - close OUTread; - close ERRread - if (!$merge_output); - - close(STDERR); - if ($merge_output) { - open STDERR, ">&OUTwrite" || - die "Can't redirect stderr\n"; - } else { - open STDERR, ">&ERRwrite" || - die "Can't redirect stderr\n"; - } - select STDERR; - $| = 1; - - close(STDOUT); - open STDOUT, ">&OUTwrite" || - die "Can't redirect stdout\n"; - select STDOUT; - $| = 1; - - # Turn shell-quoted words ("foo bar baz") into individual tokens - - my @tokens; - while ($cmd =~ /\".*\"/) { - my $prefix; - my $middle; - my $suffix; - - $cmd =~ /(.*?)\"(.*?)\"(.*)/; - $prefix = $1; - $middle = $2; - $suffix = $3; - - if ($prefix) { - foreach my $token (split(' ', $prefix)) { - push(@tokens, $token); - } - } - if ($middle) { - push(@tokens, $middle); - } else { - push(@tokens, ""); - } - $cmd = $suffix; - } - if ($cmd) { - push(@tokens, split(' ', $cmd)); - } - - # Run it! - - exec(@tokens) || - die "Can't execute command: $cmd\n"; - } - close OUTwrite; - close ERRwrite - if (!$merge_output); - - # Parent - - my (@out, @err); - my ($rin, $rout); - my $done = $merge_output ? 1 : 2; - - # Keep watching over the pipe(s) - - $rin = ''; - vec($rin, fileno(OUTread), 1) = 1; - vec($rin, fileno(ERRread), 1) = 1 - if (!$merge_output); - - while ($done > 0) { - my $nfound = select($rout = $rin, undef, undef, undef); - if (vec($rout, fileno(OUTread), 1) == 1) { - my $data = ; - if (!defined($data)) { - vec($rin, fileno(OUTread), 1) = 0; - --$done; - } else { - push(@out, $data); - print "OUT:$data" if ($debug); - } - } - - if (!$merge_output && vec($rout, fileno(ERRread), 1) == 1) { - my $data = ; - if (!defined($data)) { - vec($rin, fileno(ERRread), 1) = 0; - --$done; - } else { - push(@err, $data); - print "ERR:$data" if ($debug); - } - } - } - close OUTerr; - close OUTread - if (!$merge_output); - - # The pipes are closed, so the process should be dead. Reap it. - - waitpid($pid, 0); - my $status = $?; - print "*** Command complete, exit status: $status\n" if ($debug); - - # Return an anonymous hash containing the relevant data - - my $ret = { - stdout => \@out, - status => $status - }; - - # If we had stderr, return that, too - - $ret->{stderr} = \@err - if (!$merge_output); - return $ret; -} - -#-------------------------------------------------------------------------- - -# find a program from a list and load it into the target variable -sub find_program { - my @names = @_; - - # loop through the list and save the first one that we find - my $i = 0; - while ($i <= $#names) { - my $ret = system("which $names[$i] 2>&1 >/dev/null"); - my $status = $ret >> 8; - if ($status == 0) { - return $names[$i]; - } - ++$i; - } - return undef; -} - -#-------------------------------------------------------------------------- - -sub trim { - my ($max, $msg) = @_; - - my @out; - - my $i = 0; - if ($#$msg > $max) { - $i = $#$msg - $max + 1; - push(@out, "[...previous $i lines snipped; last $max lines shown...]\n"); - } - while ($i <= $#$msg) { - push(@out, $$msg[$i]); - ++$i; - } - if (! ($out[$#out] =~ /\n/)) { - push(@out, "\n"); - } - \@out; -} - -#-------------------------------------------------------------------------- - -# subroutine for building a single configuration -sub try_untar { - my ($name, $merge_output, $tarball, $srcroot, $installdir, - $vpath_mode, $confargs) = @_; - my $ret; - my $startdir = `pwd`; - my $unpacker; - my $status; - - chomp($startdir); - - # make the source root - if (! -d $srcroot) { - mkdir($srcroot, 0777); - } - chdir($srcroot); - - # expand the tarball (do NOT assume GNU tar). don't use - # do_command here because we need to use a pipe. This isn't an - # "interesting" command, anyway -- we don't need any stdout or - # stderr. So if it fails, it fails -- no big deal. - - if ($tarball =~ /.*\.bz2$/) { - $unpacker="bunzip2"; - } - elsif ($tarball =~ /.*\.gz$/) { - $unpacker="gunzip"; - } - else { - return { - status => $status, - message => "Failed to figure out compression of tarball", - } - } - - system("$unpacker -c $tarball | tar xf -"); - $status = $? >> 8; - if ($status != 0) { - return { - status => $status, - message => "Failed to unzip the tarball", - } - } - - test_abort ("Tarball does not contain expected directory:$file_base_name-$version -- aborting") - if (! -d "$file_base_name-$version"); - chdir("$file_base_name-$version"); -} - -sub try_configure { - my ($name, $merge_output, $tarball, $srcroot, $package_dir, $installdir, - $vpath_mode, $confargs) = @_; - my $ret; - - # - # Attention -- package_dir should only be one level higher... - # - chdir ($package_dir); - - # configure it - my $config_command = "./configure"; - if ($vpath_mode) { - mkdir("vpath_build", 0777); - chdir("vpath_build"); - if ($vpath_mode eq "relative") { - $config_command = "../configure"; - } else { - $config_command = "$srcroot/$file_base_name-$version/$package_dir/configure"; - } - } - - $ret = do_command(1, "$config_command $confargs --prefix=$installdir"); - if ($ret->{status} != 0) { - $ret->{message} = "Failed to configure the tarball"; - return $ret; - } - - # build it - $ret = do_command($merge_output, "make all"); - if ($ret->{status} != 0) { - $ret->{message} = "Failed to \"make all\""; - return $ret; - } - - # save the compile warnings - my $make_all_stderr = $ret->{stderr}; - - # and check it, if the user does not disable it - if ( $nocheck == 0 ) { - $ret = do_command($merge_output, "make check"); - if ($ret->{status} != 0) { - $ret->{make_all_stderr} = $make_all_stderr; - $ret->{message} = "Failed to \"make check\""; - return $ret; - } - } - - # install it - $ret = do_command(1, "make install"); - if ($ret->{status} != 0) { - $ret->{make_all_stderr} = $make_all_stderr; - $ret->{message} = "Failed to \"make install\""; - return $ret; - } - - if ($vpath_mode) { - chdir (".."); - } - - chdir (".."); - - return { - make_all_stderr => $make_all_stderr, - status => 0, - }; -} - - -sub cleanup { - my ($name, $merge_output, $tarball, $srcroot, $installdir, - $vpath_mode, $confargs) = @_; - my $startdir=`pwd`; - - # all done -- clean up (unless user specified --leave-install) - chdir($startdir); - if ($leave_install_arg) { - system("rm -rf $srcroot/openmpi* $srcroot/test"); - } else { - system("rm -rf $srcroot"); - } -} - -#-------------------------------------------------------------------------- - -# -# main -# - -# parse the command line -&Getopt::Long::Configure("bundling", "require_order"); -my $ok = Getopt::Long::GetOptions("url|u=s" => \$url_arg, - "scratch|s=s" => \$scratch_root_arg, - "email|e=s" => \$email_arg, - "config|c=s" => \$config_arg, - "file|f=s" => \$file_arg, - "debug|d" => \$debug_arg, - "leave-install|l=s" => \$leave_install_arg, - "install-dir=s" => \$install_dir_arg, - "outfile=s" => \$outfile_arg, - "help|h" => \$help_arg, - "force" => \$force_arg, - "nocheck" => \$nocheck_arg - ); - -if (!$ok || $help_arg) { - print("Command line error\n") - if (!$ok); - print "Usage: $0 [--scratch|-s scratch_directory_root] [--email|-e address]\n"; - print " [--config|-c config_file] [--help|-h] [--debug|-d]\n"; - print " [[--file|-f local_ompi_tarball] [--url|-u URL_base]]\n"; - print " [--leave-install output_filename] [--outfile output_filename]\n"; - print " [--install-dir install_dir] [--nocheck]\n"; - exit(0); -} - -if ($file_arg && $url_arg) { - print("ERROR: Both --url and --file specified; which should I do?\n"); - die("Aborting in confusion!\n"); -} elsif (! $file_arg && ! $url_arg) { - print("ERROR: Neither --url nor --file specified; what should I do?\n"); - die("Aborting in confusion!\n"); -} elsif (! $email_arg) { - print("ERROR: Need e-mail address to mail results\n"); - die("Aborting in despair\n"); -} elsif (! $scratch_root_arg) { - my $tmp = "/tmp"; - $tmp = $ENV{TMPDIR} - if ($ENV{TMPDIR}); - $scratch_root_arg = "$tmp/open-mpi-test-build.$$"; -} - -# if the --file argument was a relative file, convert it to absolute -# because we're going to chdir before using it -if ($file_arg) { - if (! ($file_arg =~ /^\//)) { - my $tmp = `pwd`; - chomp($tmp); - $file_arg = "$tmp/$file_arg"; - } -} - -# ditto for --config arg -if ($config_arg) { - if (! ($config_arg =~ /^\//)) { - my $tmp = `pwd`; - chomp($tmp); - $config_arg = "$tmp/$config_arg"; - } -} - -# turn on debugging? -$debug = 1 - if ($debug_arg); - -# Disable checking? -$nocheck = 1 - if ($nocheck_arg); - -# prefix the e-mail -my $str = "Host: "; -my $first = 1; -foreach my $option (qw/n o r m/) { - my $out = `uname -$option 2>/dev/null`; - chomp($out); - if ($out) { - $str .= " / " - if (! $first); - $str .= $out; - $first = 0; - } -} -push(@email_output, $str . "\n"); - -# now that we have scratch root, fill in the filename for the last -# version that we tested -- separate it by hostname and config name so -# that multiple instances of this script can share a downloads -# directory -my $dir = "$scratch_root_arg/last-test-versions"; -if (! -d $dir) { - mkdir($dir); -} -my $hostname = `uname -n`; -chomp($hostname); -my $config = $config_arg ? `basename $config_arg` : "default"; -chomp($config); -$last_test_version_name = "$dir/$hostname-$config"; - -# Find a mail program -$mail = find_program(qw(Mail mailx mail)); -die "Could not find mail program; aborting in despair\n" - if (!defined($mail)); - -# figure out what download command to use -# if we need to download a version -- with file_arg, we do not -my $download = "true"; -if ($url_arg) { - $download = find_program(qw(wget lynx curl)); - test_abort("Cannot find downloading program -- aborting in despair\n") - if (!defined($download)); -} - -# move into the scratch directory, and ensure we have an absolute path -# for it -if (! -d $scratch_root_arg) { - mkdir($scratch_root_arg, 0777); -} -test_abort("Could not cd to scratch root: $scratch_root_arg\n") - if (! -d $scratch_root_arg); -chdir($scratch_root_arg); -$scratch_root_arg = `pwd`; -chomp($scratch_root_arg); - -# if we were given a URL base, get the latest snapshot version number -if ($url_arg) { - - # ensure some subdirs exist - foreach my $dir (qw(downloads)) { - if (! -d $dir) { - mkdir($dir, 0777); - } - } - - chdir("downloads"); - - unlink($latest_name); - do_command(1, "$download $url_arg/$latest_name"); - test_abort("Could not download latest snapshot number -- aborting") - if (! -f $latest_name); - $version = `cat $latest_name`; - chomp($version); - - check_last_version($version); - - push(@email_output, "Snapshot: $version\n\n"); - - # see if we need to download the tarball - $tarball_name = "$file_base_name-$version.tar.gz"; - if (! -f $tarball_name) { - do_command(1, "$download $url_arg/$tarball_name"); - test_abort ("Could not download tarball -- aborting") - if (! -f $tarball_name); - - # get the checksums - unlink($md5_checksums); - do_command(1, "$download $url_arg/$md5_checksums"); - - unlink($sha1_checksums); - do_command(1, "$download $url_arg/$sha1_checksums"); - } - - # compare the md5sum - my $md5_file = `grep $version.tar.gz $md5_checksums`; - chomp($md5_file); - my $md5sum = find_program(qw(md5sum)); - if (!defined($md5sum)) { - push(@email_output, -"WARNING: Could not find md5sum executable, so I will not be able to check -WARNING: the validity of downloaded executables against their known MD5 -WARNING: checksums. Proceeding anyway...\n\n"); - } elsif (! $md5_file) { - push(@email_output, -"WARNING: Could not find md5sum check file, so I will not be able to check -WARNING: the validity of downloaded executables against their known MD5 -WARNING: checksums. Proceeding anyway...\n\n"); - } else { - my $md5_actual = `$md5sum $tarball_name 2>&1`; - chomp($md5_actual); - test_abort("md5sum from checksum file does not match actual ($md5_file != $md5_actual)") - if ($md5_file ne $md5_actual); - } - - # compare the sha1sum - my $sha1_file = `grep $version.tar.gz $sha1_checksums`; - chomp($sha1_file); - my $sha1sum = find_program(qw(sha1sum)); - if (!defined($sha1sum)) { - push(@email_output, -"WARNING: Could not find sha1sum executable, so I will not be able to check -WARNING: the validity of downloaded executables against their known SHA1 -WARNING: checksums. Proceeding anyway...\n\n"); - } elsif (! $sha1_file) { - push(@email_output, -"WARNING: Could not find sha1sum check file, so I will not be able to check -WARNING: the validity of downloaded executables against their known SHA1 -WARNING: checksums. Proceeding anyway...\n\n"); - } else { - my $sha1_actual = `$sha1sum $tarball_name 2>&1`; - chomp($sha1_actual); - test_abort "sha1sum from checksum file does not match actual ($sha1_file != $sha1_actual)" - if ($sha1_file ne $sha1_actual); - } - - # now adjust the tarball name to be absolute - $tarball_name = "$scratch_root_arg/downloads/$tarball_name"; -} elsif ($file_arg) { - $tarball_name = $file_arg; - - die "ERROR: Cannot read file $tarball_name\n" - if (! -r $tarball_name); - $version = $tarball_name; - $version =~ s/.*$file_base_name-(.+)\.tar.*/$1/; - - check_last_version($version); - - push(@email_output, "Snapshot: $version\n\n"); -} - -# Make a root for this build to play in (scratch_root_arg is absolute, so -# root will be absolute) -my $root= "$scratch_root_arg/build-$version"; -system("rm -rf $root"); -mkdir($root, 0777); -chdir($root); - -$outfile="out-$version.txt"; - -if ($outfile_arg) { - $outfile="$outfile_arg"; -} - -test_abort("Could not find output file:$outfile -- aborting") - if ( ! -f $outfile ); - - -# loop over all configurations -# be lazy: if no configurations supplied, do a default -# configure/build -my $results; -my $do_default = 0; - -if (! $config_arg || ! -f $config_arg) { - my $dir = "$root/default"; - my $name = "[default]"; - my $config = "CFLAGS=-g --disable-f77 --enable-debug"; - - die "NOT SUPPORTED YET\n"; - - $install_dir="$dir/install"; - - if ($install_dir_arg) { - $install_dir="$install_dir_arg/$name"; - } - - $ret = try_untar($name, 0, $tarball_name, - $dir, "$install_dir", "", $config); - $ret = try_configure($name, 0, $tarball_name, - $dir, "$install_dir", "", $config); - $ret = cleanup($name, 0, $tarball_name, - $dir, "$install_dir", "", $config); - $results->{$name} = $ret; - $results->{$name}->{config} = $config; - $results->{$name}->{want_stderr} = 1; - $results->{$name}->{vpath_mode} = ""; - $results->{$name}->{installdir} = "$dir/install"; -} else { - open CONF, "$config_arg"; - my $i = 1; - while () { - my $line = $_; - chomp($line); - - # skip comments and blank lines - - if ($line && ! ($line =~ /[ \t]*\#/) && ! ($line =~ /^[ \t]+$/)) { - - # Parse out the parts - - my ($name, $want_stderr, $vpath_mode, $config) = split(/:/, $line); - if (! $name) { - $name = "[build-$i]"; - } - - # try to actually build it - - my $dir = "$root/$name"; - $dir =~ s/[\[\] \t"']/_/g; - # stupid emacs: '" - my $merge_output = $want_stderr ? 0 : 1; - $ret = try_untar($name, $merge_output, $tarball_name, - $dir, "$install_dir", - $vpath_mode, $config); - - open OUTFILE, "$outfile"; - while () { - my $mpi_dir=$_; - my $added_config="$config --with-mpi-dir=$mpi_dir"; - my @check_this = split (/\//, $mpi_dir); - my $mpi_installation = pop (@check_this); - chomp($mpi_installation); - # print "mpi_dir:$mpi_dir\n"; - - $install_dir="$dir/install"; - - if ($install_dir_arg) { - $install_dir="$install_dir_arg/mpi_$mpi_installation/config_$name"; - } - # printf "install_dir:$install_dir\n"; - - $ret = try_configure($name, $merge_output, $tarball_name, - $dir, "PMB-2.2", "$install_dir", - $vpath_mode, $added_config); - $ret = try_configure($name, $merge_output, $tarball_name, - $dir, "mpich_tester", "$install_dir", - $vpath_mode, $added_config); - - $results->{$name} = $ret; - $results->{$name} = $ret; - $results->{$name} = $ret; - $results->{$name}->{config} = $config; - $results->{$name}->{want_stderr} = $want_stderr; - $results->{$name}->{vpath_mode} = $vpath_mode; - $results->{$name}->{installdir} = "$install_dir"; - } - close OUTFILE; -# $ret = cleanup($name, $merge_output, $tarball_name, -# $dir, "$install_dir", -# $vpath_mode, $config); - ++$i; - } - } - close CONF; -} - -# remove this root if we're not leaving the install root -chdir($scratch_root_arg); -system("rm -rf build-$version") - if (!$leave_install_arg); - -my $dir = "$scratch_root_arg"; -opendir(ROOT, $dir); -my @roots = sort(grep { /^build.+[0-9]$/ && -d "$dir/$_" } readdir(ROOT)); - -my $i = $#roots; -my $j = 0; -while ($i >= $max_build_roots) { - print "Removing directory $roots[$j]\n" if ($debug); - system("rm -rf $roots[$j]"); - ++$j; - --$i; -} - -# trim the downloads dir to $max_snapshots -my $dir = "$scratch_root_arg/downloads"; -opendir(DOWNLOADS, $dir); -my @tarballs = sort(grep { /^openmpi.+gz$/ && -f "$dir/$_" } readdir(DOWNLOADS)); -closedir(DOWNLOADS); -$i = $#tarballs; -$j = 0; -while ($i >= $max_snapshots) { - print "Unlinking $tarballs[$j]\n" if ($debug); - unlink("$dir/$tarballs[$j]"); - ++$j; - --$i; -} - -# save the version that we just tested -open LAST, ">$last_test_version_name"; -print LAST "$version\n"; -close LAST; - -# if we're leaving the installdirs, output their names into the file -# indicated by the --leave-install arg -open LEAVE, ">$leave_install_arg" - if ($leave_install_arg); - -# send results mail -my $email_subject; -push(@email_output, -"SUMMARY OF RESULTS: - (Building and compiling/linking PMB-2.2 and MPIch-testsuite) ---------------------------------------------------------------------------\n"); - -foreach my $config (keys(%$results)) { - my $str; - if ($results->{$config}->{status} == 0) { - $str = "Success "; - print LEAVE "$results->{$config}->{installdir}\n" - if ($leave_install_arg); - } else { - $str = "FAILURE "; - $email_subject = $fail_subject; - } - $str .= $config . "\n"; - push(@email_output, $str); -} -close(LEAVE) - if ($leave_install_arg); -$email_subject = $success_subject - if (!$email_subject); - -push(@email_output, -"-------------------------------------------------------------------------- -\n"); - - -# Include additional details if relevant -my $header = 0; -my $displayed = 0; -push(@email_output, "DETAILED RESULTS (as necessary):\n\n"); -foreach my $config (keys(%$results)) { - my $output = 0; - my $str; - - $str .= "==> Build: " . $config . " -==> Config options: " . $results->{$config}->{config} . " -==> Result: " . ($results->{$config}->{status} == 0 ? - "Success" : "FAILURE") . " -==> Output: " . ($results->{$config}->{want_stderr} ? - "stderr and stdout shown separately below" : - "stdout and stderr merged below") . "\n"; - - if ($results->{$config}->{status} != 0 && - $results->{$config}->{stdout} && - $#{$results->{$config}->{stdout}} >= 0) { - push(@email_output, $str); - $header = $displayed = $output = 1; - - if ($results->{$config}->{want_stderr}) { - push(@email_output, -"--Standard output---------------------------------------------------------\n"); - } else { - push(@email_output, -"--Standard output and standard error--------------------------------------\n"); - } - my $tmp = trim($max_log_len, $results->{$config}->{stdout}); - push(@email_output, @$tmp); - if ($results->{$config}->{want_stderr}) { - push(@email_output, -"--Standard output end-----------------------------------------------------\n"); - } else { - push(@email_output, -"--Standard output and standard error end----------------------------------\n"); - } - } - if ($results->{$config}->{stderr} && - $#{$results->{$config}->{stderr}} >= 0) { - push(@email_output, $str) - if ($output == 0); - $header = $displayed = $output = 1; - - push(@email_output, -"--Standard error----------------------------------------------------------\n"); - my $tmp = trim($max_log_len, $results->{$config}->{stderr}); - push(@email_output, @$tmp); - push(@email_output, -"--Standard error end------------------------------------------------------\n"); - } - if ($results->{$config}->{want_stderr} && - $results->{$config}->{make_all_stderr} && - $#{$results->{$config}->{make_all_stderr}} >= 0) { - push(@email_output, $str) - if ($output == 0); - $header = $displayed = $output = 1; - - push(@email_output, -"--\"make all\" standard error---------------------------------------------------\n"); - my $tmp = trim($max_log_len, $results->{$config}->{make_all_stderr}); - push(@email_output, @$tmp); - push(@email_output, -"--\"make all\" standard error end-----------------------------------------------\n"); - } - - if ($output == 1) { - push(@email_output, -"\n==========================================================================\n\n"); - } -} - -# if we didn't display anything extra, then say so - -if (!$displayed) { - push(@email_output, "No additional interesting intformation\n\n"); -} - -push(@email_output, "Your friendly server,\nCyrador\n"); -send_mail($email_subject, $email_arg, @email_output); - -# All done - -exit(0); diff --git a/contrib/nightly/ompi_cronjob.sh b/contrib/nightly/ompi_cronjob.sh deleted file mode 100755 index 0863e43f6a..0000000000 --- a/contrib/nightly/ompi_cronjob.sh +++ /dev/null @@ -1,167 +0,0 @@ -#!/bin/sh -# -# Please adjust the below parameters for Your setup. -# -# -SRCDIR=$HOME/WORK/OPENMPI # Where the script and the tar-balls reside -DATE_STRING=`date +%Y.%m.%d` # Date string for tmp-/scratch-dir -TMPDIR=`ws_allocate OpenMPI-$DATE_STRING 3` # Where to build the OMPI -LAST_VERSION=1.0a1r6896 # Should be detected by download -TEST_VERSION=r362 -CONFIG_FILE=build-$HOSTNAME.txt -PATCHES="ompi_orte.diff" # Comma-separated list of patches to apply when building -#EMAIL="testing@open-mpi.org" -EMAIL="keller@hlrs.de" - -# download "latest" filename -latest_name="latest_snapshot.txt" - - -# -# This is required on some machines, that do not -# have autoconf-tools (cron passes only a short PATH) -# -export PATH=$HOME/bin:$PATH - - -if [ "x$HOSTNAME" = "xstrider" ] ; then - export PATH=$PATH:/opt/pathscale/bin -fi - -####################################################### -# -# Local functions -# -# find a program from a list and load it into the target variable -find_program() { - var=$1 - shift - - # first zero out the target variable - str="$var=" - eval $str - - # loop through the list and save the first one that we find - am_done= - while test -z "$am_done" -a -n "$1" ; do - prog=$1 - shift - - if test -z "$prog" ; then - am_done=1 - else - not_found="`which $prog 2>&1 | egrep '^no'`" - which $prog > /dev/null 2>&1 - if test "$?" = "0" -a -z "$not_found" ; then - str="$var=$prog" - eval $str - am_done=1 - fi - fi - done -} - - -dump () { - file=$1 - shift - echo "$1" >> $file -} - -############################################ -# Main part -############################################ - -if [ \! -d $SRCDIR ] ; then - echo "WARNING $SRCDIR does not exist" - exit -fi - -cd $SRCDIR - -# figure out what download command to use -find_program download wget lynx curl -if test -z "$download"; then - echo "cannot find downloading program -- aborting in despair" - exit 1 -fi - -# get the latest snapshot version number -# rm -f "$latest_name" -# $download "$url_arg/$latest_name" -# if test ! -f "$latest_name" ; then -# echo "Could not download latest snapshot number -- aborting" -# exit 1 -# fi -# version=$LAST_VERSION -version="`cat $latest_name`" -test_version=$TEST_VERSION - - -# -# If the config-file does not exist, create one standard test file -# -if [ \! -r $CONFIG_FILE ] ; then - echo "CONFIG_FILE:$CONFIG_FILE does not exist, create default file" - - CONFIG_FILE=build_sample_config.txt - rm -f $CONFIG_FILE - echo "gcc:1::--enable-picky" >> $CONFIG_FILE - # echo "default build:::" >> $CONFIG_FILE - # echo "static build:::--disable-shared --enable-static" >> $CONFIG_FILE - # echo "non-threaded build:::--without-threads" >> $CONFIG_FILE -fi - - -# -# Here comes the main part. -# -perl build_tarball.pl --email $EMAIL --config $CONFIG_FILE --scratch $TMPDIR --file openmpi-$version.tar.bz2 \ - --patches "$PATCHES" \ - --leave-install $SRCDIR/ompi-out-$version.txt --install-dir=$HOME/ompi-install --nocheck - -if [ \! -r ompi-out-$version.txt ] ; then - echo "No ompi-out-$version.txt file found; none of the configurations have been built?" - exit -fi - -# -# Now compile and install the tests for different combinations... -# -perl build_tests.pl --email $EMAIL --config $CONFIG_FILE --file openmpi-tests-$test_version.tar.bz2 --leave-install $SRCDIR/ompi-tests-out-$test_version.txt --outfile $SRCDIR/ompi-out-$version.txt --install-dir=$HOME/ompi-tests-install --nocheck - -f="ompi-script-$version" -rm -fr $f -dump $f "#!/bin/bash" -dump $f "#PBS -l nodes=4,walltime=0:10:00" # Four nodes, and 10 minutes are enough -dump $f "#PBS -k eo" # Combine stderr/stdout -dump $f "#PBS -m ae" # Send mail on abort/end of job -dump $f "#PBS -M $EMAIL" # Mail-adresses to send to -dump $f "" -dump $f "OLD_PATH=\$PATH" -dump $f "OLD_LD_LIBRARY_PATH=\$LD_LIBRARY_PATH" -dump $f "rm -fr $HOME/ompi-tmp/openmpi-sessions-*" -dump $f "killall -9 orted" - -for MPI_DIR in `cat $SRCDIR/ompi-out-$version.txt` ; do - dump $f " echo Starting to test MPI_DIR:$MPI_DIR" - dump $f " echo -------------------------------------------------" - dump $f " export PATH=$MPI_DIR/bin:\$OLD_PATH" - dump $f " export LD_LIBRARY_PATH=$MPI_DIR/lib:\$OLD_LD_LIBRARY_PATH" - dump $f " rm -fr $HOME/ompi-tmp/openmpi-sessions-*" - # Start the orted daemon in a commonly accessible directory (/tmp is local to each node) - dump $f " orted --tmpdir $HOME/ompi-tmp --seed --persistent --scope public" - for j in `cat $SRCDIR/ompi-tests-out-$test_version.txt` ; do - dump $f " cd $j/bin" - dump $f " mpirun -x LD_LIBRARY_PATH -hostfile \$PBS_NODEFILE -tmpdir $HOME/ompi-tmp -np 8 ./PMB-MPI1 PingPong PingPing Sendrecv Exchange Allreduce Reduce Allgather Allgatherv Alltoall Bcast Barrier" - dump $f " ./tester -m -x LD_LIBRARY_PATH -hostfile \$PBS_NODEFILE -tmpdir $HOME/ompi-tmp -- mpich_all_tests" - dump $f " cd -" - dump $f " echo -------------------------------------------------" - done - dump $f " killall -9 orted" - dump $f " sleep 10" - dump $f " echo -------------------------------------------------" -done - -# qsub $SRCDIR/ompi-script-$version - diff --git a/contrib/nightly/unimplemented_report.sh b/contrib/nightly/unimplemented_report.sh deleted file mode 100755 index a81c34f96e..0000000000 --- a/contrib/nightly/unimplemented_report.sh +++ /dev/null @@ -1,357 +0,0 @@ -#!/bin/sh -# -# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana -# University Research and Technology -# Corporation. All rights reserved. -# Copyright (c) 2004-2005 The University of Tennessee and The University -# of Tennessee Research Foundation. All rights -# reserved. -# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, -# University of Stuttgart. All rights reserved. -# Copyright (c) 2004-2005 The Regents of the University of California. -# All rights reserved. -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# -# This script is used to make a report of how may unimplemented -# top-level MPI functions exist. -# -# $1: scratch root -# $2: e-mail address for destination -# $3: URL_ARG -# - -scratch_root_arg="$1" -email_arg="$2" -url_arg="$3" - -# Set this to any value for additional output; typically only when -# debugging -debug= - -# do you want a success mail? -want_success_mail=1 - -# download "latest" filename -latest_name="latest_snapshot.txt" - -# checksum filenames -md5_checksums="md5sums.txt" -sha1_checksums="sha1sums.txt" - -# max length of logfile to send in an e-mail -max_log_len=250 - -# email subject -subject="Unimplemented function report" - -# max number of snapshots to keep downloaded -max_snapshots=3 - -# files to not report -skip='accumulate.c|^get.c$|put.c|^win_' - -############################################################################ -# Shouldn't need to change below this line -############################################################################ - -start_time="`date`" - -# This gets filled in later -config_guess= - -# Sanity checks -if test -z "$scratch_root_arg" -o -z "$email_arg" -o -z "$url_arg"; then - echo "Must specify scratch root directory, e-mail address, and URL_ARG" - exit 1 -fi - -# send a mail -# should only be called after logdir is set -send_error_mail() { - outfile="$scratch_root_arg/output.txt" - rm -f "$outfile" - touch "$outfile" - for file in `/bin/ls $logdir/* | sort`; do - len="`wc -l $file | awk '{ print $1}'`" - if test "`expr $len \> $max_log_len`" = "1"; then - tail -$max_log_len "$file" >> "$outfile" - else - cat "$file" >> "$outfile" - fi - done - $mail -s "Failed report" "$email_arg" < "$outfile" - rm -f "$outfile" -} - -# send output error message -die() { - msg="$*" - cat > "$logdir/00_announce.txt" < "$logfile" 2>&1 - st=$? - echo "*** Command complete: exit status: $st" - else - eval $cmd > "$logfile" 2>&1 - st=$? - fi - if test "$st" != "0"; then - cat > "$logdir/15-error.txt" < "$logdir/25-error.txt" < /dev/null 2>&1 - if test "$?" = "0" -a -z "$not_found"; then - str="$var=$prog" - eval $str - am_done=1 - fi - fi - done -} - -# Find a mail program -find_program mail Mail mailx mail -if test -z "$mail"; then - echo "Could not find mail program; aborting in despair" - exit 1 -fi - -# figure out what download command to use -find_program download wget lynx curl -if test -z "$download"; then - echo "cannot find downloading program -- aborting in despair" - exit 1 -fi - -# move into the scratch directory, and ensure we have an absolute path -# for it -if test ! -d "$scratch_root_arg"; then - mkdir -p "$scratch_root_arg" -fi -if test ! -d "$scratch_root_arg"; then - echo "Could not cd to scratch root: $scratch_root_arg" - exit 1 -fi -cd "$scratch_root_arg" -scratch_root_arg="`pwd`" -logdir="$scratch_root_arg/logs" - -# ensure some subdirs exist -for dir in downloads logs; do - if test ! -d $dir; then - mkdir $dir - fi -done - -# get the latest snapshot version number -cd downloads -rm -f "$latest_name" -do_command $download "$url_arg/$latest_name" -if test ! -f "$latest_name"; then - die "Could not download latest snapshot number -- aborting" -fi -version="`cat $latest_name`" - -# see if we need to download the tarball -tarball_name="openmpi-$version.tar.gz" -if test ! -f "$tarball_name"; then - do_command $download "$url_arg/$tarball_name" - if test ! -f "$tarball_name"; then - die "Could not download tarball -- aborting" - fi - - # get the checksums - rm -f "$md5_checksums" "$sha1_checksums" - do_command $download "$url_arg/$md5_checksums" - do_command $download "$url_arg/$sha1_checksums" -fi - -# verify the checksums -md5_file="`grep $version.tar.gz $md5_checksums`" -find_program md5sum md5sum -if test -z "$md5sum"; then - cat > $logdir/05_md5sum_warning.txt < $logdir/05_md5sum_warning.txt < $logdir/06_sha1sum_warning.txt < $logdir/06_sha1sum_warning.txt < "$report_c" -total_c="`ls -1 *.c | wc -l | awk '{ print $1 }'`" -num_c="`wc -l $report_c | awk '{ print $1 }'`" - -# count unimplemented F77 functions -cd ../f77 -grep -ls "not yet implemented" *.c > "$report_f77" -total_f77="`ls -1 *.c | wc -l | awk '{ print $1 }'`" -num_f77="`wc -l $report_f77 | awk '{ print $1 }'`" - -# trim the downloads dir to $max_snapshots -cd "$scratch_root_arg/downloads" -for ext in gz; do - count="`ls openmpi*.tar.$ext | wc -l | awk '{ print $1 }'`" - if test "`expr $count \> $max_snapshots`" = "1"; then - num_old="`expr $count - $max_snapshots`" - old="`ls -rt openmpi*.tar.$ext | head -n $num_old`" - rm -f $old - fi -done - -# send success mail -$mail -s "$subject" "$email_arg" <