1
1

Huge improvement over previous nightly test script; not quite finished

yet (still have stdout/stderr separated -- need to merge in some kind
of sane way).  Will send details of how to use this script to the
devel list soon.

This commit was SVN r2265.
Этот коммит содержится в:
Jeff Squyres 2004-08-23 11:34:20 +00:00
родитель ac381a79d5
Коммит c61d124c76

Просмотреть файл

@ -8,19 +8,14 @@
# $2: e-mail address for destination
# $3: URL_ARG
# $4: config file
# $5: vpath_arg
#
use strict;
use Data::Dumper;
use Getopt::Long;
# Set this to true for additional output; typically only when
# debugging
my $debug = 1;
# do you want a success mail?
my $want_success_mail = 1;
our $debug = 1;
# download "latest" filename
my $latest_name = "latest_snapshot.txt";
@ -33,8 +28,8 @@ my $sha1_checksums = "sha1sums.txt";
my $max_log_len = 100;
# email subjects
my $success_subject = "Success";
my $fail_subject = "=== TEST FAILURE ===";
my $success_subject = "Success (\@version@)";
my $fail_subject = "=== TEST FAILURE (\@version@) ===";
# max number of snapshots to keep downloaded
my $max_snapshots = 3;
@ -53,6 +48,9 @@ sub send_mail {
my $msg = \@_;
our $mail;
our $version;
$subject =~ s/\@version\@/$version/;
open MAIL, "|$mail -s \"$subject\" \"$to\"" ||
die "Could not open pipe to output e-mail\n";
@ -93,8 +91,8 @@ sub do_command {
my $pid;
if (($pid = fork()) == 0) {
# close OUTread;
# close ERRread;
close OUTread;
close ERRread;
close(STDERR);
open STDERR, ">&ERRwrite" ||
@ -192,152 +190,28 @@ sub trim {
my $i = 0;
if ($#$msg > $max) {
$i = $#$msg - $max + 1;
push(@out, "...previous $i lines snipped...\n");
push(@out, "[...previous $i lines snipped; last $max lines shown...]\n");
}
while ($i <= $#$msg) {
push(@out, $$msg[$i]);
++$i;
}
if (! ($out[$#out] =~ /\n/)) {
push(@out, "\n");
}
\@out;
}
#--------------------------------------------------------------------------
#
# main
#
# grab the args
my $scratch_root_arg = $ARGV[0];
my $email_arg = $ARGV[1];
my $url_arg = $ARGV[2];
my $config_arg = $ARGV[3];
my $vpath_arg = $ARGV[4];
# global vars
our @email_output;
my $ret;
# Sanity checks
if (-z $scratch_root_arg || -z $email_arg || -z $url_arg) {
print "Must specify scratch root directory, e-mail address, and URL_ARG
Can also optionally specify a config file and whether want VPATH building\n";
exit(1);
}
# prefix the e-mail
my $unamen = `uname -n`;
my $unameo = `uname -o`;
my $unamer = `uname -r`;
my $unamem = `uname -m`;
chomp($unamen);
chomp($unameo);
chomp($unamer);
chomp($unamem);
push(@email_output, "Host: $unamen / $unameo / $unamer / $unamem\n");
# Find a mail program
our $mail = find_program(qw(Mail mailx mail));
print "Found mail: $mail\n";
die "Could not find mail program; aborting in despair\n"
if (!defined($mail));
# figure out what download command to use
my $download = find_program(qw(wget lynx curl));
test_abort("Cannot find downloading program -- aborting in despair\n")
if (!defined($download));
# move into the scratch directory, and ensure we have an absolute path
# for it
if (! -d $scratch_root_arg) {
mkdir($scratch_root_arg);
}
test_abort("Could not cd to scratch root: $scratch_root_arg\n")
if (! -d $scratch_root_arg);
chdir($scratch_root_arg);
$scratch_root_arg = `pwd`;
chomp($scratch_root_arg);
# ensure some subdirs exist
foreach my $dir (qw(downloads)) {
if (! -d $dir) {
mkdir($dir);
}
}
# get the latest snapshot version number
chdir("downloads");
unlink($latest_name);
do_command("$download $url_arg/$latest_name");
test_abort("Could not download latest snapshot number -- aborting")
if (! -f $latest_name);
my $version = `cat $latest_name`;
chomp($version);
push(@email_output, "Snapshot: $version\n\n");
# see if we need to download the tarball
my $tarball_name = "openmpi-$version.tar.gz";
if (! -f $tarball_name) {
do_command("$download $url_arg/$tarball_name");
test_abort "Could not download tarball -- aborting"
if (! -f $tarball_name);
# get the checksums
unlink($md5_checksums);
do_command("$download $url_arg/$md5_checksums");
unlink($sha1_checksums);
do_command("$download $url_arg/$sha1_checksums");
}
# compare the md5sum
my $md5_file = `grep $version.tar.gz $md5_checksums`;
chomp($md5_file);
my $md5sum = find_program(qw(md5sum));
if (!defined($md5sum)) {
push(@email_output,
"WARNING: Could not find md5sum executable, so I will not be able to check
WARNING: the validity of downloaded executables against their known MD5
WARNING: checksums. Proceeding anyway...\n\n");
} elsif (! $md5_file) {
push(@email_output,
"WARNING: Could not find md5sum check file, so I will not be able to check
WARNING: the validity of downloaded executables against their known MD5
WARNING: checksums. Proceeding anyway...\n\n");
} else {
my $md5_actual = `$md5sum $tarball_name 2>&1`;
chomp($md5_actual);
test_abort("md5sum from checksum file does not match actual ($md5_file != $md5_actual)")
if ($md5_file ne $md5_actual);
}
# compare the sha1sum
my $sha1_file = `grep $version.tar.gz $sha1_checksums`;
chomp($sha1_file);
my $sha1sum = find_program(qw(sha1sum));
if (!defined($sha1sum)) {
push(@email_output,
"WARNING: Could not find sha1sum executable, so I will not be able to check
WARNING: the validity of downloaded executables against their known SHA1
WARNING: checksums. Proceeding anyway...\n\n");
} elsif (! $sha1_file) {
push(@email_output,
"WARNING: Could not find sha1sum check file, so I will not be able to check
WARNING: the validity of downloaded executables against their known SHA1
WARNING: checksums. Proceeding anyway...\n\n");
} else {
my $sha1_actual = `$sha1sum $tarball_name 2>&1`;
chomp($sha1_actual);
test_abort "sha1sum from checksum file does not match actual ($sha1_file != $sha1_actual)"
if ($sha1_file ne $sha1_actual);
}
# subroutine for building a single configuration
sub try_build {
my ($srcroot, $installdir, $confargs, $vpath_mode) = @_;
my ($tarball, $srcroot, $installdir, $vpath_mode, $confargs) = @_;
my $ret;
my $startdir = `pwd`;
our $version;
our $scratch_root_arg;
chomp($startdir);
# make the source root
@ -356,7 +230,7 @@ sub try_build {
# "interesting" command, anyway -- we don't need any stdout or
# stderr. So if it fails, it fails -- no big deal.
system("gunzip -c $scratch_root_arg/downloads/$tarball_name | tar xf -");
system("gunzip -c $tarball | tar xf -");
my $status = $? >> 8;
if ($status != 0) {
return {
@ -433,7 +307,7 @@ int main(int argc, char* argv[]) {
open CXX, ">hello.cc";
print CXX "#include <mpi.h>
int main(int argc, char* argv[]) {
MPI::nit(argc, argv);
MPI::Init(argc, argv);
MPI::Finalize();
return 0;
}\n";
@ -483,6 +357,205 @@ int main(int argc, char* argv[]) {
};
}
#--------------------------------------------------------------------------
#
# main
#
# parse the command line
our $scratch_root_arg;
our $email_arg;
our $url_arg;
our $config_arg;
our $debug_arg;
our $file_arg;
my $help_arg;
&Getopt::Long::Configure("bundling", "require_order");
my $ok = Getopt::Long::GetOptions("url|u=s" => \$url_arg,
"scratch|s=s" => \$scratch_root_arg,
"email|e=s" => \$email_arg,
"config|c=s" => \$config_arg,
"file|f=s" => \$file_arg,
"debug|d" => \$debug_arg,
"help|h" => \$help_arg,
);
if (!$ok || $help_arg) {
print("Command line error\n")
if (!$ok);
print "Usage: $0 [--scratch|-s scratch_directory_root] [--email|-e address]\n";
print "[--config|-c config_file] [--help|-h] [--debug|-d]\n";
print "[[--file|-f local_ompi_tarball] [--url|-u URL_base]]\n";
exit(0);
}
if ($file_arg && $url_arg) {
print("ERROR: Both --url and --file specified; which should I do?\n");
die("Aborting in confusion!\n");
} elsif (! $file_arg && ! $url_arg) {
print("ERROR: Neither --url nor --file specified; what should I do?\n");
die("Aborting in confusion!\n");
} elsif (! $email_arg) {
print("ERROR: Need e-mail address to mail results\n");
die("Aborting in despair\n");
} elsif (! $scratch_root_arg) {
my $tmp = "/tmp";
$tmp = $ENV{TMPDIR}
if ($ENV{TMPDIR});
$scratch_root_arg = "$tmp/open-mpi-test-build.$$";
}
# if the --file argument was a relative file, convert it to absolute
# because we're going to chdir before using it
if ($file_arg) {
my $foo = $file_arg =~ /^\//;
if (! ($file_arg =~ /^\//)) {
my $tmp = `pwd`;
chomp($tmp);
$file_arg = "$tmp/$file_arg";
}
}
# ditto for --config arg
if ($config_arg) {
my $foo = $config_arg =~ /^\//;
if (! ($config_arg =~ /^\//)) {
my $tmp = `pwd`;
chomp($tmp);
$config_arg = "$tmp/$config_arg";
}
}
# turn on debugging?
$debug = 1
if ($debug_arg);
# global vars
our @email_output;
my $ret;
# prefix the e-mail
my $unamen = `uname -n`;
my $unameo = `uname -o`;
my $unamer = `uname -r`;
my $unamem = `uname -m`;
chomp($unamen);
chomp($unameo);
chomp($unamer);
chomp($unamem);
push(@email_output, "Host: $unamen / $unameo / $unamer / $unamem\n");
# Find a mail program
our $mail = find_program(qw(Mail mailx mail));
die "Could not find mail program; aborting in despair\n"
if (!defined($mail));
# figure out what download command to use
my $download = find_program(qw(wget lynx curl));
test_abort("Cannot find downloading program -- aborting in despair\n")
if (!defined($download));
# move into the scratch directory, and ensure we have an absolute path
# for it
if (! -d $scratch_root_arg) {
mkdir($scratch_root_arg);
}
test_abort("Could not cd to scratch root: $scratch_root_arg\n")
if (! -d $scratch_root_arg);
chdir($scratch_root_arg);
$scratch_root_arg = `pwd`;
chomp($scratch_root_arg);
# ensure some subdirs exist
foreach my $dir (qw(downloads)) {
if (! -d $dir) {
mkdir($dir);
}
}
# if we were given a URL base, get the latest snapshot version number
our $tarball_name;
our $version;
if ($url_arg) {
chdir("downloads");
unlink($latest_name);
do_command("$download $url_arg/$latest_name");
test_abort("Could not download latest snapshot number -- aborting")
if (! -f $latest_name);
$version = `cat $latest_name`;
chomp($version);
push(@email_output, "Snapshot: $version\n\n");
# see if we need to download the tarball
$tarball_name = "openmpi-$version.tar.gz";
if (! -f $tarball_name) {
do_command("$download $url_arg/$tarball_name");
test_abort "Could not download tarball -- aborting"
if (! -f $tarball_name);
# get the checksums
unlink($md5_checksums);
do_command("$download $url_arg/$md5_checksums");
unlink($sha1_checksums);
do_command("$download $url_arg/$sha1_checksums");
}
# compare the md5sum
my $md5_file = `grep $version.tar.gz $md5_checksums`;
chomp($md5_file);
my $md5sum = find_program(qw(md5sum));
if (!defined($md5sum)) {
push(@email_output,
"WARNING: Could not find md5sum executable, so I will not be able to check
WARNING: the validity of downloaded executables against their known MD5
WARNING: checksums. Proceeding anyway...\n\n");
} elsif (! $md5_file) {
push(@email_output,
"WARNING: Could not find md5sum check file, so I will not be able to check
WARNING: the validity of downloaded executables against their known MD5
WARNING: checksums. Proceeding anyway...\n\n");
} else {
my $md5_actual = `$md5sum $tarball_name 2>&1`;
chomp($md5_actual);
test_abort("md5sum from checksum file does not match actual ($md5_file != $md5_actual)")
if ($md5_file ne $md5_actual);
}
# compare the sha1sum
my $sha1_file = `grep $version.tar.gz $sha1_checksums`;
chomp($sha1_file);
my $sha1sum = find_program(qw(sha1sum));
if (!defined($sha1sum)) {
push(@email_output,
"WARNING: Could not find sha1sum executable, so I will not be able to check
WARNING: the validity of downloaded executables against their known SHA1
WARNING: checksums. Proceeding anyway...\n\n");
} elsif (! $sha1_file) {
push(@email_output,
"WARNING: Could not find sha1sum check file, so I will not be able to check
WARNING: the validity of downloaded executables against their known SHA1
WARNING: checksums. Proceeding anyway...\n\n");
} else {
my $sha1_actual = `$sha1sum $tarball_name 2>&1`;
chomp($sha1_actual);
test_abort "sha1sum from checksum file does not match actual ($sha1_file != $sha1_actual)"
if ($sha1_file ne $sha1_actual);
}
# now adjust the tarball name to be absolute
$tarball_name = "$scratch_root_arg/downloads/$tarball_name";
} elsif ($file_arg) {
$tarball_name = $file_arg;
system("pwd");
die "ERROR: Cannot read file $tarball_name\n"
if (! -r $tarball_name);
$version = $tarball_name;
$version =~ s/.*openmpi-(.+).tar.gz/$1/;
}
# Make a root for this build to play in (scratch_root_arg is absolute, so
# root will be absolute)
my $root= "$scratch_root_arg/build-$version";
@ -491,29 +564,26 @@ mkdir($root);
chdir($root);
# loop over all configurations
# be lazy: if no configurations supplied and no vpath, do a default
# be lazy: if no configurations supplied, do a default
# configure/build
my $results;
my $do_default = 0;
if (! $config_arg || ! -f $config_arg) {
if (! $vpath_arg) {
$do_default = 1;
}
}
if ($do_default) {
my $dir = "$root/default";
my $name = "[default]";
my $config = "CFLAGS=-g --disable-f77 --enable-debug";
$ret = try_build($dir, "$dir/install", $config, "");
$ret = try_build($tarball_name, $dir, "$dir/install", "", $config);
$results->{$name} = $ret;
$results->{$name}->{config} = $config;
} elsif (-f $config_arg) {
$results->{$name}->{want_stderr} = 1;
$results->{$name}->{vpath_mode} = "";
} else {
open CONF, "$config_arg";
my $i = 1;
while (<CONF>) {
my $line = $_;
chomp($line);
my ($name, $config) = split(/:/, $line);
my ($name, $want_stderr, $vpath_mode, $config) = split(/:/, $line);
if (! $config) {
$config = $name;
}
@ -523,32 +593,17 @@ if ($do_default) {
}
my $dir = "$root/config-$i";
$ret = try_build($dir, "$dir/install", "$config", "");
$ret = try_build($tarball_name, $dir, "$dir/install",
$vpath_mode, $config);
$results->{$name} = $ret;
$results->{$name}->{config} = $config;
$results->{$name}->{want_stderr} = $want_stderr;
$results->{$name}->{vpath_mode} = $vpath_mode;
++$i;
}
close CONF;
}
# did we want vpath builds?
if ($vpath_arg) {
my $config = "CFLAGS=-g";
my ($dir, $name);
$dir = "$root/vpath-relative";
$name = "vpath-relative";
$ret = try_build($dir, "$dir/install", $config, "relative");
$results->{$name} = $ret;
$results->{$name}->{config} = $config;
$dir = "$root/vpath-absolute";
$name = "vpath-absolute";
$ret = try_build($dir, "$dir/install", $config, "absolute");
$results->{$name} = $ret;
$results->{$name}->{config} = $config;
}
# trim the downloads dir to $max_snapshots
my $dir = "$scratch_root_arg/downloads";
opendir(DOWNLOADS, $dir);
@ -556,11 +611,13 @@ my @tarballs = sort(grep { /^openmpi.+gz\$/ && -f "$dir/$_" } readdir(DOWNLOADS)
closedir(DOWNLOADS);
my $i = $#tarballs;
while ($i > $max_snapshots) {
print "UNLINKING: $tarballs[$i]\n";
unlink($tarballs[$i]);
++$i;
}
# send success mail
my $email_subject;
push(@email_output,
"Summary of results:
Building tarball and compiling/linking MPI \"hello world\"
@ -571,10 +628,13 @@ foreach my $config (keys(%$results)) {
$str = "Success ";
} else {
$str = "FAILURE ";
$email_subject = $fail_subject;
}
$str .= $config . "\n";
push(@email_output, $str);
}
$email_subject = $success_subject
if (!$email_subject);
push(@email_output,
"--------------------------------------------------------------------------
@ -582,8 +642,6 @@ push(@email_output,
# Include additional details if relevant
print Dumper($results);
my $displayed = 0;
foreach my $config (keys(%$results)) {
my $output = 0;
@ -618,7 +676,8 @@ Result: " . ($results->{$config}->{status} == 0 ?
push(@email_output,
"--Standard error end------------------------------------------------------\n");
}
if ($results->{$config}->{make_all_stderr} &&
if ($results->{$config}->{want_stderr} &&
$results->{$config}->{make_all_stderr} &&
$#{$results->{$config}->{make_all_stderr}} >= 0) {
push(@email_output, $str)
if ($output == 0);
@ -645,6 +704,6 @@ if (!$displayed) {
}
push(@email_output, "Your friendly server,\nCyrador\n");
send_mail($success_subject, $email_arg, @email_output);
send_mail($email_subject, $email_arg, @email_output);
exit(0);