Huge improvement over previous nightly test script; not quite finished
yet (still have stdout/stderr separated -- need to merge in some kind of sane way). Will send details of how to use this script to the devel list soon. This commit was SVN r2265.
Этот коммит содержится в:
родитель
ac381a79d5
Коммит
c61d124c76
@ -8,19 +8,14 @@
|
||||
# $2: e-mail address for destination
|
||||
# $3: URL_ARG
|
||||
# $4: config file
|
||||
# $5: vpath_arg
|
||||
#
|
||||
|
||||
use strict;
|
||||
use Data::Dumper;
|
||||
use Getopt::Long;
|
||||
|
||||
# Set this to true for additional output; typically only when
|
||||
# debugging
|
||||
|
||||
my $debug = 1;
|
||||
|
||||
# do you want a success mail?
|
||||
my $want_success_mail = 1;
|
||||
our $debug = 1;
|
||||
|
||||
# download "latest" filename
|
||||
my $latest_name = "latest_snapshot.txt";
|
||||
@ -33,8 +28,8 @@ my $sha1_checksums = "sha1sums.txt";
|
||||
my $max_log_len = 100;
|
||||
|
||||
# email subjects
|
||||
my $success_subject = "Success";
|
||||
my $fail_subject = "=== TEST FAILURE ===";
|
||||
my $success_subject = "Success (\@version@)";
|
||||
my $fail_subject = "=== TEST FAILURE (\@version@) ===";
|
||||
|
||||
# max number of snapshots to keep downloaded
|
||||
my $max_snapshots = 3;
|
||||
@ -53,6 +48,9 @@ sub send_mail {
|
||||
my $msg = \@_;
|
||||
|
||||
our $mail;
|
||||
our $version;
|
||||
|
||||
$subject =~ s/\@version\@/$version/;
|
||||
|
||||
open MAIL, "|$mail -s \"$subject\" \"$to\"" ||
|
||||
die "Could not open pipe to output e-mail\n";
|
||||
@ -93,8 +91,8 @@ sub do_command {
|
||||
|
||||
my $pid;
|
||||
if (($pid = fork()) == 0) {
|
||||
# close OUTread;
|
||||
# close ERRread;
|
||||
close OUTread;
|
||||
close ERRread;
|
||||
|
||||
close(STDERR);
|
||||
open STDERR, ">&ERRwrite" ||
|
||||
@ -192,152 +190,28 @@ sub trim {
|
||||
my $i = 0;
|
||||
if ($#$msg > $max) {
|
||||
$i = $#$msg - $max + 1;
|
||||
push(@out, "...previous $i lines snipped...\n");
|
||||
push(@out, "[...previous $i lines snipped; last $max lines shown...]\n");
|
||||
}
|
||||
while ($i <= $#$msg) {
|
||||
push(@out, $$msg[$i]);
|
||||
++$i;
|
||||
}
|
||||
if (! ($out[$#out] =~ /\n/)) {
|
||||
push(@out, "\n");
|
||||
}
|
||||
\@out;
|
||||
}
|
||||
|
||||
#--------------------------------------------------------------------------
|
||||
|
||||
#
|
||||
# main
|
||||
#
|
||||
|
||||
# grab the args
|
||||
my $scratch_root_arg = $ARGV[0];
|
||||
my $email_arg = $ARGV[1];
|
||||
my $url_arg = $ARGV[2];
|
||||
my $config_arg = $ARGV[3];
|
||||
my $vpath_arg = $ARGV[4];
|
||||
|
||||
# global vars
|
||||
our @email_output;
|
||||
my $ret;
|
||||
|
||||
# Sanity checks
|
||||
if (-z $scratch_root_arg || -z $email_arg || -z $url_arg) {
|
||||
print "Must specify scratch root directory, e-mail address, and URL_ARG
|
||||
Can also optionally specify a config file and whether want VPATH building\n";
|
||||
exit(1);
|
||||
}
|
||||
|
||||
# prefix the e-mail
|
||||
my $unamen = `uname -n`;
|
||||
my $unameo = `uname -o`;
|
||||
my $unamer = `uname -r`;
|
||||
my $unamem = `uname -m`;
|
||||
chomp($unamen);
|
||||
chomp($unameo);
|
||||
chomp($unamer);
|
||||
chomp($unamem);
|
||||
push(@email_output, "Host: $unamen / $unameo / $unamer / $unamem\n");
|
||||
|
||||
# Find a mail program
|
||||
our $mail = find_program(qw(Mail mailx mail));
|
||||
print "Found mail: $mail\n";
|
||||
die "Could not find mail program; aborting in despair\n"
|
||||
if (!defined($mail));
|
||||
|
||||
# figure out what download command to use
|
||||
my $download = find_program(qw(wget lynx curl));
|
||||
test_abort("Cannot find downloading program -- aborting in despair\n")
|
||||
if (!defined($download));
|
||||
|
||||
# move into the scratch directory, and ensure we have an absolute path
|
||||
# for it
|
||||
if (! -d $scratch_root_arg) {
|
||||
mkdir($scratch_root_arg);
|
||||
}
|
||||
test_abort("Could not cd to scratch root: $scratch_root_arg\n")
|
||||
if (! -d $scratch_root_arg);
|
||||
chdir($scratch_root_arg);
|
||||
$scratch_root_arg = `pwd`;
|
||||
chomp($scratch_root_arg);
|
||||
|
||||
# ensure some subdirs exist
|
||||
foreach my $dir (qw(downloads)) {
|
||||
if (! -d $dir) {
|
||||
mkdir($dir);
|
||||
}
|
||||
}
|
||||
|
||||
# get the latest snapshot version number
|
||||
chdir("downloads");
|
||||
unlink($latest_name);
|
||||
do_command("$download $url_arg/$latest_name");
|
||||
test_abort("Could not download latest snapshot number -- aborting")
|
||||
if (! -f $latest_name);
|
||||
my $version = `cat $latest_name`;
|
||||
chomp($version);
|
||||
push(@email_output, "Snapshot: $version\n\n");
|
||||
|
||||
# see if we need to download the tarball
|
||||
my $tarball_name = "openmpi-$version.tar.gz";
|
||||
if (! -f $tarball_name) {
|
||||
do_command("$download $url_arg/$tarball_name");
|
||||
test_abort "Could not download tarball -- aborting"
|
||||
if (! -f $tarball_name);
|
||||
|
||||
# get the checksums
|
||||
unlink($md5_checksums);
|
||||
do_command("$download $url_arg/$md5_checksums");
|
||||
|
||||
unlink($sha1_checksums);
|
||||
do_command("$download $url_arg/$sha1_checksums");
|
||||
}
|
||||
|
||||
# compare the md5sum
|
||||
my $md5_file = `grep $version.tar.gz $md5_checksums`;
|
||||
chomp($md5_file);
|
||||
my $md5sum = find_program(qw(md5sum));
|
||||
if (!defined($md5sum)) {
|
||||
push(@email_output,
|
||||
"WARNING: Could not find md5sum executable, so I will not be able to check
|
||||
WARNING: the validity of downloaded executables against their known MD5
|
||||
WARNING: checksums. Proceeding anyway...\n\n");
|
||||
} elsif (! $md5_file) {
|
||||
push(@email_output,
|
||||
"WARNING: Could not find md5sum check file, so I will not be able to check
|
||||
WARNING: the validity of downloaded executables against their known MD5
|
||||
WARNING: checksums. Proceeding anyway...\n\n");
|
||||
} else {
|
||||
my $md5_actual = `$md5sum $tarball_name 2>&1`;
|
||||
chomp($md5_actual);
|
||||
test_abort("md5sum from checksum file does not match actual ($md5_file != $md5_actual)")
|
||||
if ($md5_file ne $md5_actual);
|
||||
}
|
||||
|
||||
# compare the sha1sum
|
||||
my $sha1_file = `grep $version.tar.gz $sha1_checksums`;
|
||||
chomp($sha1_file);
|
||||
my $sha1sum = find_program(qw(sha1sum));
|
||||
if (!defined($sha1sum)) {
|
||||
push(@email_output,
|
||||
"WARNING: Could not find sha1sum executable, so I will not be able to check
|
||||
WARNING: the validity of downloaded executables against their known SHA1
|
||||
WARNING: checksums. Proceeding anyway...\n\n");
|
||||
} elsif (! $sha1_file) {
|
||||
push(@email_output,
|
||||
"WARNING: Could not find sha1sum check file, so I will not be able to check
|
||||
WARNING: the validity of downloaded executables against their known SHA1
|
||||
WARNING: checksums. Proceeding anyway...\n\n");
|
||||
} else {
|
||||
my $sha1_actual = `$sha1sum $tarball_name 2>&1`;
|
||||
chomp($sha1_actual);
|
||||
test_abort "sha1sum from checksum file does not match actual ($sha1_file != $sha1_actual)"
|
||||
if ($sha1_file ne $sha1_actual);
|
||||
}
|
||||
|
||||
|
||||
# subroutine for building a single configuration
|
||||
sub try_build {
|
||||
my ($srcroot, $installdir, $confargs, $vpath_mode) = @_;
|
||||
my ($tarball, $srcroot, $installdir, $vpath_mode, $confargs) = @_;
|
||||
my $ret;
|
||||
my $startdir = `pwd`;
|
||||
our $version;
|
||||
our $scratch_root_arg;
|
||||
|
||||
chomp($startdir);
|
||||
|
||||
# make the source root
|
||||
@ -356,7 +230,7 @@ sub try_build {
|
||||
# "interesting" command, anyway -- we don't need any stdout or
|
||||
# stderr. So if it fails, it fails -- no big deal.
|
||||
|
||||
system("gunzip -c $scratch_root_arg/downloads/$tarball_name | tar xf -");
|
||||
system("gunzip -c $tarball | tar xf -");
|
||||
my $status = $? >> 8;
|
||||
if ($status != 0) {
|
||||
return {
|
||||
@ -433,7 +307,7 @@ int main(int argc, char* argv[]) {
|
||||
open CXX, ">hello.cc";
|
||||
print CXX "#include <mpi.h>
|
||||
int main(int argc, char* argv[]) {
|
||||
MPI::nit(argc, argv);
|
||||
MPI::Init(argc, argv);
|
||||
MPI::Finalize();
|
||||
return 0;
|
||||
}\n";
|
||||
@ -483,6 +357,205 @@ int main(int argc, char* argv[]) {
|
||||
};
|
||||
}
|
||||
|
||||
#--------------------------------------------------------------------------
|
||||
|
||||
#
|
||||
# main
|
||||
#
|
||||
|
||||
# parse the command line
|
||||
our $scratch_root_arg;
|
||||
our $email_arg;
|
||||
our $url_arg;
|
||||
our $config_arg;
|
||||
our $debug_arg;
|
||||
our $file_arg;
|
||||
my $help_arg;
|
||||
|
||||
&Getopt::Long::Configure("bundling", "require_order");
|
||||
my $ok = Getopt::Long::GetOptions("url|u=s" => \$url_arg,
|
||||
"scratch|s=s" => \$scratch_root_arg,
|
||||
"email|e=s" => \$email_arg,
|
||||
"config|c=s" => \$config_arg,
|
||||
"file|f=s" => \$file_arg,
|
||||
"debug|d" => \$debug_arg,
|
||||
"help|h" => \$help_arg,
|
||||
);
|
||||
|
||||
if (!$ok || $help_arg) {
|
||||
print("Command line error\n")
|
||||
if (!$ok);
|
||||
print "Usage: $0 [--scratch|-s scratch_directory_root] [--email|-e address]\n";
|
||||
print "[--config|-c config_file] [--help|-h] [--debug|-d]\n";
|
||||
print "[[--file|-f local_ompi_tarball] [--url|-u URL_base]]\n";
|
||||
exit(0);
|
||||
}
|
||||
|
||||
if ($file_arg && $url_arg) {
|
||||
print("ERROR: Both --url and --file specified; which should I do?\n");
|
||||
die("Aborting in confusion!\n");
|
||||
} elsif (! $file_arg && ! $url_arg) {
|
||||
print("ERROR: Neither --url nor --file specified; what should I do?\n");
|
||||
die("Aborting in confusion!\n");
|
||||
} elsif (! $email_arg) {
|
||||
print("ERROR: Need e-mail address to mail results\n");
|
||||
die("Aborting in despair\n");
|
||||
} elsif (! $scratch_root_arg) {
|
||||
my $tmp = "/tmp";
|
||||
$tmp = $ENV{TMPDIR}
|
||||
if ($ENV{TMPDIR});
|
||||
$scratch_root_arg = "$tmp/open-mpi-test-build.$$";
|
||||
}
|
||||
|
||||
# if the --file argument was a relative file, convert it to absolute
|
||||
# because we're going to chdir before using it
|
||||
if ($file_arg) {
|
||||
my $foo = $file_arg =~ /^\//;
|
||||
if (! ($file_arg =~ /^\//)) {
|
||||
my $tmp = `pwd`;
|
||||
chomp($tmp);
|
||||
$file_arg = "$tmp/$file_arg";
|
||||
}
|
||||
}
|
||||
|
||||
# ditto for --config arg
|
||||
if ($config_arg) {
|
||||
my $foo = $config_arg =~ /^\//;
|
||||
if (! ($config_arg =~ /^\//)) {
|
||||
my $tmp = `pwd`;
|
||||
chomp($tmp);
|
||||
$config_arg = "$tmp/$config_arg";
|
||||
}
|
||||
}
|
||||
|
||||
# turn on debugging?
|
||||
$debug = 1
|
||||
if ($debug_arg);
|
||||
|
||||
# global vars
|
||||
our @email_output;
|
||||
my $ret;
|
||||
|
||||
# prefix the e-mail
|
||||
my $unamen = `uname -n`;
|
||||
my $unameo = `uname -o`;
|
||||
my $unamer = `uname -r`;
|
||||
my $unamem = `uname -m`;
|
||||
chomp($unamen);
|
||||
chomp($unameo);
|
||||
chomp($unamer);
|
||||
chomp($unamem);
|
||||
push(@email_output, "Host: $unamen / $unameo / $unamer / $unamem\n");
|
||||
|
||||
# Find a mail program
|
||||
our $mail = find_program(qw(Mail mailx mail));
|
||||
die "Could not find mail program; aborting in despair\n"
|
||||
if (!defined($mail));
|
||||
|
||||
# figure out what download command to use
|
||||
my $download = find_program(qw(wget lynx curl));
|
||||
test_abort("Cannot find downloading program -- aborting in despair\n")
|
||||
if (!defined($download));
|
||||
|
||||
# move into the scratch directory, and ensure we have an absolute path
|
||||
# for it
|
||||
if (! -d $scratch_root_arg) {
|
||||
mkdir($scratch_root_arg);
|
||||
}
|
||||
test_abort("Could not cd to scratch root: $scratch_root_arg\n")
|
||||
if (! -d $scratch_root_arg);
|
||||
chdir($scratch_root_arg);
|
||||
$scratch_root_arg = `pwd`;
|
||||
chomp($scratch_root_arg);
|
||||
|
||||
# ensure some subdirs exist
|
||||
foreach my $dir (qw(downloads)) {
|
||||
if (! -d $dir) {
|
||||
mkdir($dir);
|
||||
}
|
||||
}
|
||||
|
||||
# if we were given a URL base, get the latest snapshot version number
|
||||
our $tarball_name;
|
||||
our $version;
|
||||
if ($url_arg) {
|
||||
chdir("downloads");
|
||||
unlink($latest_name);
|
||||
do_command("$download $url_arg/$latest_name");
|
||||
test_abort("Could not download latest snapshot number -- aborting")
|
||||
if (! -f $latest_name);
|
||||
$version = `cat $latest_name`;
|
||||
chomp($version);
|
||||
push(@email_output, "Snapshot: $version\n\n");
|
||||
|
||||
# see if we need to download the tarball
|
||||
$tarball_name = "openmpi-$version.tar.gz";
|
||||
if (! -f $tarball_name) {
|
||||
do_command("$download $url_arg/$tarball_name");
|
||||
test_abort "Could not download tarball -- aborting"
|
||||
if (! -f $tarball_name);
|
||||
|
||||
# get the checksums
|
||||
unlink($md5_checksums);
|
||||
do_command("$download $url_arg/$md5_checksums");
|
||||
|
||||
unlink($sha1_checksums);
|
||||
do_command("$download $url_arg/$sha1_checksums");
|
||||
}
|
||||
|
||||
# compare the md5sum
|
||||
my $md5_file = `grep $version.tar.gz $md5_checksums`;
|
||||
chomp($md5_file);
|
||||
my $md5sum = find_program(qw(md5sum));
|
||||
if (!defined($md5sum)) {
|
||||
push(@email_output,
|
||||
"WARNING: Could not find md5sum executable, so I will not be able to check
|
||||
WARNING: the validity of downloaded executables against their known MD5
|
||||
WARNING: checksums. Proceeding anyway...\n\n");
|
||||
} elsif (! $md5_file) {
|
||||
push(@email_output,
|
||||
"WARNING: Could not find md5sum check file, so I will not be able to check
|
||||
WARNING: the validity of downloaded executables against their known MD5
|
||||
WARNING: checksums. Proceeding anyway...\n\n");
|
||||
} else {
|
||||
my $md5_actual = `$md5sum $tarball_name 2>&1`;
|
||||
chomp($md5_actual);
|
||||
test_abort("md5sum from checksum file does not match actual ($md5_file != $md5_actual)")
|
||||
if ($md5_file ne $md5_actual);
|
||||
}
|
||||
|
||||
# compare the sha1sum
|
||||
my $sha1_file = `grep $version.tar.gz $sha1_checksums`;
|
||||
chomp($sha1_file);
|
||||
my $sha1sum = find_program(qw(sha1sum));
|
||||
if (!defined($sha1sum)) {
|
||||
push(@email_output,
|
||||
"WARNING: Could not find sha1sum executable, so I will not be able to check
|
||||
WARNING: the validity of downloaded executables against their known SHA1
|
||||
WARNING: checksums. Proceeding anyway...\n\n");
|
||||
} elsif (! $sha1_file) {
|
||||
push(@email_output,
|
||||
"WARNING: Could not find sha1sum check file, so I will not be able to check
|
||||
WARNING: the validity of downloaded executables against their known SHA1
|
||||
WARNING: checksums. Proceeding anyway...\n\n");
|
||||
} else {
|
||||
my $sha1_actual = `$sha1sum $tarball_name 2>&1`;
|
||||
chomp($sha1_actual);
|
||||
test_abort "sha1sum from checksum file does not match actual ($sha1_file != $sha1_actual)"
|
||||
if ($sha1_file ne $sha1_actual);
|
||||
}
|
||||
|
||||
# now adjust the tarball name to be absolute
|
||||
$tarball_name = "$scratch_root_arg/downloads/$tarball_name";
|
||||
} elsif ($file_arg) {
|
||||
$tarball_name = $file_arg;
|
||||
system("pwd");
|
||||
die "ERROR: Cannot read file $tarball_name\n"
|
||||
if (! -r $tarball_name);
|
||||
$version = $tarball_name;
|
||||
$version =~ s/.*openmpi-(.+).tar.gz/$1/;
|
||||
}
|
||||
|
||||
# Make a root for this build to play in (scratch_root_arg is absolute, so
|
||||
# root will be absolute)
|
||||
my $root= "$scratch_root_arg/build-$version";
|
||||
@ -491,29 +564,26 @@ mkdir($root);
|
||||
chdir($root);
|
||||
|
||||
# loop over all configurations
|
||||
# be lazy: if no configurations supplied and no vpath, do a default
|
||||
# be lazy: if no configurations supplied, do a default
|
||||
# configure/build
|
||||
my $results;
|
||||
my $do_default = 0;
|
||||
if (! $config_arg || ! -f $config_arg) {
|
||||
if (! $vpath_arg) {
|
||||
$do_default = 1;
|
||||
}
|
||||
}
|
||||
if ($do_default) {
|
||||
my $dir = "$root/default";
|
||||
my $name = "[default]";
|
||||
my $config = "CFLAGS=-g --disable-f77 --enable-debug";
|
||||
$ret = try_build($dir, "$dir/install", $config, "");
|
||||
$ret = try_build($tarball_name, $dir, "$dir/install", "", $config);
|
||||
$results->{$name} = $ret;
|
||||
$results->{$name}->{config} = $config;
|
||||
} elsif (-f $config_arg) {
|
||||
$results->{$name}->{want_stderr} = 1;
|
||||
$results->{$name}->{vpath_mode} = "";
|
||||
} else {
|
||||
open CONF, "$config_arg";
|
||||
my $i = 1;
|
||||
while (<CONF>) {
|
||||
my $line = $_;
|
||||
chomp($line);
|
||||
my ($name, $config) = split(/:/, $line);
|
||||
my ($name, $want_stderr, $vpath_mode, $config) = split(/:/, $line);
|
||||
if (! $config) {
|
||||
$config = $name;
|
||||
}
|
||||
@ -523,32 +593,17 @@ if ($do_default) {
|
||||
}
|
||||
|
||||
my $dir = "$root/config-$i";
|
||||
$ret = try_build($dir, "$dir/install", "$config", "");
|
||||
$ret = try_build($tarball_name, $dir, "$dir/install",
|
||||
$vpath_mode, $config);
|
||||
$results->{$name} = $ret;
|
||||
$results->{$name}->{config} = $config;
|
||||
$results->{$name}->{want_stderr} = $want_stderr;
|
||||
$results->{$name}->{vpath_mode} = $vpath_mode;
|
||||
++$i;
|
||||
}
|
||||
close CONF;
|
||||
}
|
||||
|
||||
# did we want vpath builds?
|
||||
if ($vpath_arg) {
|
||||
my $config = "CFLAGS=-g";
|
||||
my ($dir, $name);
|
||||
|
||||
$dir = "$root/vpath-relative";
|
||||
$name = "vpath-relative";
|
||||
$ret = try_build($dir, "$dir/install", $config, "relative");
|
||||
$results->{$name} = $ret;
|
||||
$results->{$name}->{config} = $config;
|
||||
|
||||
$dir = "$root/vpath-absolute";
|
||||
$name = "vpath-absolute";
|
||||
$ret = try_build($dir, "$dir/install", $config, "absolute");
|
||||
$results->{$name} = $ret;
|
||||
$results->{$name}->{config} = $config;
|
||||
}
|
||||
|
||||
# trim the downloads dir to $max_snapshots
|
||||
my $dir = "$scratch_root_arg/downloads";
|
||||
opendir(DOWNLOADS, $dir);
|
||||
@ -556,11 +611,13 @@ my @tarballs = sort(grep { /^openmpi.+gz\$/ && -f "$dir/$_" } readdir(DOWNLOADS)
|
||||
closedir(DOWNLOADS);
|
||||
my $i = $#tarballs;
|
||||
while ($i > $max_snapshots) {
|
||||
print "UNLINKING: $tarballs[$i]\n";
|
||||
unlink($tarballs[$i]);
|
||||
++$i;
|
||||
}
|
||||
|
||||
# send success mail
|
||||
my $email_subject;
|
||||
push(@email_output,
|
||||
"Summary of results:
|
||||
Building tarball and compiling/linking MPI \"hello world\"
|
||||
@ -571,10 +628,13 @@ foreach my $config (keys(%$results)) {
|
||||
$str = "Success ";
|
||||
} else {
|
||||
$str = "FAILURE ";
|
||||
$email_subject = $fail_subject;
|
||||
}
|
||||
$str .= $config . "\n";
|
||||
push(@email_output, $str);
|
||||
}
|
||||
$email_subject = $success_subject
|
||||
if (!$email_subject);
|
||||
|
||||
push(@email_output,
|
||||
"--------------------------------------------------------------------------
|
||||
@ -582,8 +642,6 @@ push(@email_output,
|
||||
|
||||
|
||||
# Include additional details if relevant
|
||||
print Dumper($results);
|
||||
|
||||
my $displayed = 0;
|
||||
foreach my $config (keys(%$results)) {
|
||||
my $output = 0;
|
||||
@ -618,7 +676,8 @@ Result: " . ($results->{$config}->{status} == 0 ?
|
||||
push(@email_output,
|
||||
"--Standard error end------------------------------------------------------\n");
|
||||
}
|
||||
if ($results->{$config}->{make_all_stderr} &&
|
||||
if ($results->{$config}->{want_stderr} &&
|
||||
$results->{$config}->{make_all_stderr} &&
|
||||
$#{$results->{$config}->{make_all_stderr}} >= 0) {
|
||||
push(@email_output, $str)
|
||||
if ($output == 0);
|
||||
@ -645,6 +704,6 @@ if (!$displayed) {
|
||||
}
|
||||
|
||||
push(@email_output, "Your friendly server,\nCyrador\n");
|
||||
send_mail($success_subject, $email_arg, @email_output);
|
||||
send_mail($email_subject, $email_arg, @email_output);
|
||||
|
||||
exit(0);
|
||||
|
Загрузка…
x
Ссылка в новой задаче
Block a user