ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

Update the nightly tarball scripts to support the new web site

Просмотр исходного кода
Этот коммит содержится в:
Ralph Castain 2016-07-29 15:48:28 -07:00
родитель 50952c3a31
Коммит 11d4002954
5 изменённых файлов: 455 добавлений и 42 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

100
contrib/build-server/hwloc-nightly-tarball.sh
Просмотреть файл

@ -7,30 +7,36 @@
#####
# e-mail address to send results to
results_addr=hwloc-devel@open-mpi.org
#results_addr=hwloc-devel@open-mpi.org
results_addr=testing@open-mpi.org
# svn repository uri
code_uri=http://svn.open-mpi.org/svn/hwloc
# git repository URL
code_uri=https://github.com/open-mpi/hwloc.git
raw_uri=https://raw.github.com/open-mpi/hwloc
# where to put built tarballs
outputroot=/l/osl/www/www.open-mpi.org/software/hwloc/nightly
outputroot=$HOME/hwloc/nightly
# where to find the build script
script_uri=${code_uri}/trunk/contrib/nightly/create_tarball.sh
script_uri=contrib/nightly/create_tarball.sh
script_uri=contrib/nightly/make_snapshot_tarball
# The tarballs to make
if [ $# -eq 0 ] ; then
dirs="/trunk /branches/v1.5 /branches/v1.4 /branches/v1.3 /branches/v1.2 /branches/v1.1 /branches/v1.0"
# Branches v1.6 and earlier were not updated to build nightly
# snapshots from git, so only check v1.7 and later
branches="master v1.11"
else
dirs=$@
branches=$@
fi
# Build root - scratch space
build_root=/home/mpiteam/hwloc/nightly-tarball-build-root
build_root=$HOME/hwloc/nightly-tarball-build-root
export PATH=$HOME/local/bin:$PATH
export LD_LIBRARY_PATH=$HOME/local/lib:$LD_LIBRARY_PATH
# Coverity stuff
coverity_token=`cat $HOME/coverity/hwloc-token.txt`
export PATH=$HOME_PREFIX/bin:$PATH
export LD_LIBRARY_PATH=$HOME_PREFIX/lib:$LD_LIBRARY_PATH
#####
#
@ -39,23 +45,27 @@ export LD_LIBRARY_PATH=$HOME/local/lib:$LD_LIBRARY_PATH
#####
# load the modules configuration
. /etc/profile.d/modules.sh
module use ~/modules
. $MODULE_INIT
module use $AUTOTOOL_MODULE
# get our nightly build script
mkdir -p $build_root
cd $build_root
# Loop making them
for dir in $dirs; do
# Remove leading /
safe_dirname=`echo $dir | sed -e 's/^\///g'`
# Convert remaining /'s to -'s
safe_dirname=`echo $safe_dirname | sed -e 's/\//-/g'`
# Now form a URL-specific script name
script=$safe_dirname-`basename $script_uri`
pending_coverity=$build_root/tarballs-to-run-through-coverity.txt
rm -f $pending_coverity
touch $pending_coverity
debug=100
wget --quiet --no-check-certificate --tries=10 $code_uri/$dir/$script_uri -O $script
# Loop making them
for branch in $branches; do
# Get the last tarball version that was made
prev_snapshot=`cat $outputroot/$branch/latest_snapshot.txt`
# Form a URL-specific script name
script=$branch-`basename $script_uri`
wget --quiet --no-check-certificate --tries=10 $raw_uri/$branch/$script_uri -O $script
if test ! $? -eq 0 ; then
echo "wget of hwloc nightly tarball create script failed."
if test -f $script ; then
@ -67,16 +77,46 @@ for dir in $dirs; do
fi
chmod +x $script
ver=`basename $dir`
module load "autotools/hwloc-$ver"
module load "tex-live/hwloc-$ver"
module load "autotools/hwloc-$branch"
# module load "tex-live/hwloc-$branch"
./$script \
$build_root/$ver \
$build_root/$branch \
$results_addr \
$code_uri/$dir \
$outputroot/$ver >/dev/null 2>&1
$outputroot/$branch \
$code_uri \
$branch \
>/dev/null 2>&1
module unload autotools tex-live
# Did the script generate a new tarball? If so, save it so that we can
# spawn the coverity checker on it afterwards. Only for this for the
# master (for now).
latest_snapshot=`cat $outputroot/$branch/latest_snapshot.txt`
if test "$prev_snapshot" != "$latest_snapshot" && \
test "$branch" = "master"; then
echo "$outputroot/$branch/hwloc-$latest_snapshot.tar.bz2" >> $pending_coverity
echo "=== Posting tarball to open-mpi.org"
# tell the web server to cleanup old nightly tarballs
ssh -p 2222 ompiteam@192.185.39.252 "git/ompi/contrib/build-server/remove-old.pl 7 public_html/software/hwloc/nightly/$branch"
# upload the new ones
scp -P 2222 $outputroot/$branch/hwloc-$latest_snapshot.tar.* ompiteam@192.185.39.252:public_html/software/hwloc/nightly/$branch/
scp -P 2222 $outputroot/$branch/$latest_snapshot.txt ompiteam@192.185.39.252:public_html/software/hwloc/nightly/$branch/
# direct the web server to regenerate the checksums
ssh -p 2222 ompiteam@192.185.39.252 "cd public_html/software/hwloc/nightly/$branch && md5sum hwloc* > md5sums.txt"
ssh -p 2222 ompiteam@192.185.39.252 "cd public_html/software/hwloc/nightly/$branch && sha1sum hwloc* > sha1sums.txt"
fi
module unload autotools
done
# If we had any new snapshots to send to coverity, process them now
#for tarball in `cat $pending_coverity`; do
# /home/common/mpiteam/scripts/hwloc-nightly-coverity.pl \
# --filename=$tarball \
# --coverity-token=$coverity_token \
# --verbose \
# --logfile-dir=$HOME/coverity \
# --make-args="-j8"
#done
rm -f $pending_coverity

33
contrib/build-server/openmpi-nightly-tarball.sh
Просмотреть файл

@ -15,32 +15,33 @@ master_raw_uri=https://raw.github.com/open-mpi/ompi
release_code_uri=https://github.com/open-mpi/ompi-release.git
release_raw_uri=https://raw.github.com/open-mpi/ompi-release
# where to put built tarballs
outputroot=/l/osl/www/www.open-mpi.org/nightly
# where to put built tarballs - needs to be
# adjusted to match your site!
outputroot=$HOME/openmpi/nightly
# where to find the build script
script_uri=contrib/nightly/create_tarball.sh
# helper scripts dir
script_dir=/u/mpiteam/scripts
script_dir=$HOME/scripts
# The tarballs to make
if [ $# -eq 0 ] ; then
# We're no longer ever checking the 1.0 - 1.6 branches anymore
branches="master v1.8 v1.10 v2.x"
# We're no longer ever checking the 1.0 - 1.8 branches anymore
branches="master v1.10 v2.x"
else
branches=$@
fi
# Build root - scratch space
build_root=/home/mpiteam/openmpi/nightly-tarball-build-root
build_root=$HOME/openmpi/nightly-tarball-build-root
# Coverity stuff
coverity_token=`cat $HOME/coverity/openmpi-token.txt`
coverity_configure_args="--enable-debug --enable-mpi-fortran --enable-mpi-java --enable-oshmem --enable-oshmem-fortran --enable-oshmem-java --with-mxm=/opt/mellanox/mxm --with-psm --with-usnic --with-libfabric=/u/mpiteam/libfabric-current/install"
export PATH=$HOME/local/bin:$PATH
export LD_LIBRARY_PATH=$HOME/local/lib:$LD_LIBRARY_PATH
export PATH=$HOME_PREFIX/bin:$PATH
export LD_LIBRARY_PATH=$HOME_PREFIX/lib:$LD_LIBRARY_PATH
#####
#
@ -49,8 +50,8 @@ export LD_LIBRARY_PATH=$HOME/local/lib:$LD_LIBRARY_PATH
#####
# load the modules configuration
. /etc/profile.d/modules.sh
module use ~/modules
. $MODULE_INIT
module use $AUTOTOOL_MODULE
# get our nightly build script
mkdir -p $build_root
@ -115,17 +116,27 @@ for branch in $branches; do
test "$branch" = "master"; then
echo "=== Saving output for a Coverity run"
echo "$outputroot/$branch/openmpi-$latest_snapshot.tar.bz2" >> $pending_coverity
# tell the web server to cleanup old nightly tarballs
ssh -p 2222 ompiteam@192.185.39.252 "git/ompi/contrib/build-server/remove-old.pl 7 public_html/nightly/$branch"
# upload the new ones
scp -P 2222 $outputroot/$branch/openmpi-$latest_snapshot.tar.* ompiteam@192.185.39.252:public_html/nightly/$branch/
scp -P 2222 $outputroot/$branch/latest_snapshot.txt ompiteam@192.185.39.252:public_html/nightly/$branch/
# direct the web server to regenerate the checksums
ssh -p 2222 ompiteam@192.185.39.252 "cd public_html/nightly/$branch && md5sum openmpi* > md5sums.txt"
ssh -p 2222 ompiteam@192.185.39.252 "cd public_html/nightly/$branch && sha1sum openmpi* > sha1sums.txt"
else
echo "=== NOT saving output for a Coverity run"
fi
# Failed builds are not removed. But if a human forgets to come
# in here and clean up the old failed builds, we can accumulate
# many over time. So remove any old failed bbuilds that are over
# many over time. So remove any old failed builds that are over
# 4 weeks old.
${script_dir}/remove-old.pl 7 $build_root/$branch
done
# If we had any new snapshots to send to coverity, process them now
for tarball in `cat $pending_coverity`; do

160
contrib/build-server/pmix-nightly-coverity.pl Исполняемый файл
Просмотреть файл

@ -0,0 +1,160 @@
#!/usr/bin/env perl
use warnings;
use strict;
use Getopt::Long;
use File::Temp qw/ tempfile tempdir /;
use File::Basename;
my $coverity_project = "open-mpi%2Fpmix";
my $filename_arg;
my $coverity_token_arg;
my $dry_run_arg = 0;
my $verbose_arg = 0;
my $debug_arg = 0;
my $logfile_dir_arg = "/tmp";
my $configure_args = "";
my $make_args = "-j 32";
my $help_arg = 0;
&Getopt::Long::Configure("bundling");
my $ok = Getopt::Long::GetOptions("filename=s" => \$filename_arg,
"coverity-token=s" => \$coverity_token_arg,
"logfile-dir=s" => \$logfile_dir_arg,
"configure-args=s" => \$configure_args,
"make-args=s" => \$make_args,
"dry-run!" => \$dry_run_arg,
"verbose!" => \$verbose_arg,
"debug!" => \$debug_arg,
"help|h" => \$help_arg);
$ok = 0
if (!defined($filename_arg));
$ok = 0
if (!defined($coverity_token_arg));
if (!$ok || $help_arg) {
print "Usage: $0 --filename=FILENAME --coverity-token=TOKEN [--dry-run] [--verbose] [--help]\n";
exit($ok);
}
die "Cannot read $filename_arg"
if (! -r $filename_arg);
$verbose_arg = 1
if ($debug_arg);
######################################################################
sub verbose {
print @_
if ($verbose_arg);
}
# run a command and save the stdout / stderr
sub safe_system {
my $allowed_to_fail = shift;
my $cmd = shift;
my $stdout_file = shift;
# Redirect stdout if requested or not verbose
if (defined($stdout_file)) {
$stdout_file = "$logfile_dir_arg/$stdout_file";
unlink($stdout_file);
$cmd .= " >$stdout_file";
} elsif (!$debug_arg) {
$cmd .= " >/dev/null";
}
$cmd .= " 2>&1";
my $rc = system($cmd);
if (0 != $rc && !$allowed_to_fail) {
# If we die/fail, ensure to change out of the temp tree so
# that it can be removed upon exit.
chdir("/");
die "Command $cmd failed: exit status $rc";
}
system("cat $stdout_file")
if ($debug_arg && defined($stdout_file) && -f $stdout_file);
}
######################################################################
# Make an area to work
my $dir = tempdir(CLEANUP => 1);
chdir($dir);
verbose "*** Working in $dir\n";
######################################################################
# Get the coverity tool, put it in our path.
my $cdir = "/home/common/mpiteam/coverity";
safe_system(0, "mkdir $cdir")
if (! -d $cdir);
# Optimization: the tool is pretty large. If our local copy is less
# than a day old, just use that without re-downloading.
my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,
$atime,$mtime,$ctime,$blksize,$blocks) =
stat("$cdir/coverity_tool.tgz");
my $now = time();
if (!defined($mtime) || $mtime < $now - 24*60*60) {
verbose "*** Downloading new copy of the coverity tool\n";
safe_system(0, "wget https://scan.coverity.com/download/linux-64 --post-data \"token=$coverity_token_arg\&project=$coverity_project\" -O coverity_tool.tgz");
safe_system(0, "cp coverity_tool.tgz $cdir");
}
verbose "*** Expanding coverity tool tarball\n";
safe_system(0, "tar xf $cdir/coverity_tool.tgz");
opendir(my $dh, ".") ||
die "Can't opendir .";
my @files = grep { /^cov/ && -d "./$_" } readdir($dh);
closedir($dh);
my $cov_dir = "$dir/$files[0]/bin";
$ENV{PATH} = "$cov_dir:$ENV{PATH}";
######################################################################
# Expand the PMIX tarball, build it
verbose "*** Extracting PMIX tarball\n";
safe_system(0, "tar xf $filename_arg");
my $tarball_filename = basename($filename_arg);
$tarball_filename =~ m/^pmix-(.+)\.tar.+$/;
my $pmix_ver = $1;
chdir("pmix-$pmix_ver");
verbose "*** Configuring PMIX tarball\n";
safe_system(0, "./configure $configure_args", "configure");
verbose "*** Building PMIX tarball\n";
safe_system(0, "cov-build --dir cov-int make $make_args", "cov-build");
# Tar up the Coverity results
verbose "*** Tarring up results\n";
safe_system(0, "tar jcf $pmix_ver-analyzed.tar.bz2 cov-int");
# If not dry-run, submit to Coverity
if ($dry_run_arg) {
verbose "*** Would have submitted, but this is a dry run\n";
} else {
verbose "*** Submitting results\n";
safe_system(0, "curl --form token=$coverity_token_arg " .
"--form email=rhc\@open-mpi.org " .
"--form file=\@$pmix_ver-analyzed.tar.bz2 " .
"--form version=$pmix_ver " .
"--form description=nightly-master " .
"https://scan.coverity.com/builds?project=$coverity_project",
"coverity-submit");
}
verbose("*** All done\n");
# Chdir out of the tempdir so that it can be removed
chdir("/");
exit(0);

150
contrib/build-server/pmix-nightly-tarball.sh Исполняемый файл
Просмотреть файл

@ -0,0 +1,150 @@
#!/bin/sh
#####
#
# Configuration options
#
#####
# e-mail address to send results to
results_addr=testing@open-mpi.org
# svn repository uri
master_code_uri=https://github.com/pmix/master.git
master_raw_uri=https://raw.github.com/pmix/master
release_code_uri=https://github.com/pmix/releases.git
release_raw_uri=https://raw.github.com/pmix/releases
# where to put built tarballs
outputroot=$HOME/pmix/nightly
# where to find the build script
script_uri=contrib/nightly/create_tarball.sh
# helper scripts dir
script_dir=$HOME/scripts
# The tarballs to make
if [ $# -eq 0 ] ; then
branches="master"
else
branches=$@
fi
# Build root - scratch space
build_root=$HOME/pmix/nightly-tarball-build-root
# Coverity stuff
coverity_token=`cat /home/common/mpiteam/coverity/pmix-token.txt`
coverity_configure_args="--with-libevent=/home/common/local"
export PATH=$HOME_PREFIX/bin:$PATH
export LD_LIBRARY_PATH=$HOME_PREFIX/lib:$LD_LIBRARY_PATH
#####
#
# Actually do stuff
#
#####
# load the modules configuration
. $MODULE_INIT
module use $AUTOTOOL_MODULE
# get our nightly build script
mkdir -p $build_root
cd $build_root
pending_coverity=$build_root/tarballs-to-run-through-coverity.txt
rm -f $pending_coverity
touch $pending_coverity
# Loop making the tarballs
module unload autotools
for branch in $branches; do
echo "=== Branch: $branch"
# Get the last tarball version that was made
prev_snapshot=`cat $outputroot/$branch/latest_snapshot.txt`
echo "=== Previous snapshot: $prev_snapshot"
if test "$branch" = "master"; then
code_uri=$master_code_uri
raw_uri=$master_raw_uri
else
code_uri=$release_code_uri
raw_uri=$release_raw_uri
fi
# Form a URL-specific script name
script=$branch-`basename $script_uri`
echo "=== Getting script from: $raw_uri"
wget --quiet --no-check-certificate --tries=10 $raw_uri/$branch/$script_uri -O $script
if test ! $? -eq 0 ; then
echo "wget of PMIX nightly tarball create script failed."
if test -f $script ; then
echo "Using older version of $script for this run."
else
echo "No build script available. Aborting."
exit 1
fi
fi
chmod +x $script
module load "autotools/pmix-$branch"
# module load "libevent/pmix-$branch"
echo "=== Running script..."
./$script \
$build_root/$branch \
$results_addr \
$outputroot/$branch \
$code_uri \
$branch \
>/dev/null 2>&1
module unload autotools
echo "=== Done running script"
# Did the script generate a new tarball? If so, save it so that we can
# spawn the coverity checker on it afterwards. Only for this for the
# master (for now).
latest_snapshot=`cat $outputroot/$branch/latest_snapshot.txt`
echo "=== Latest snapshot: $latest_snapshot"
if test "$prev_snapshot" != "$latest_snapshot" && \
test "$branch" = "master"; then
echo "=== Saving output for a Coverity run"
echo "$outputroot/$branch/pmix-$latest_snapshot.tar.bz2" >> $pending_coverity
echo "=== Posting tarball to open-mpi.org"
# tell the web server to cleanup old nightly tarballs
ssh -p 2222 ompiteam@192.185.39.252 "git/ompi/contrib/build-server/remove-old.pl 7 public_html/software/pmix/nightly/$branch"
# upload the new ones
scp -P 2222 $outputroot/$branch/pmix-$latest_snapshot.tar.* ompiteam@192.185.39.252:public_html/software/pmix/nightly/$branch/
scp -P 2222 $outputroot/$branch/latest_snapshot.txt ompiteam@192.185.39.252:public_html/software/pmix/nightly/$branch/
# direct the web server to regenerate the checksums
ssh -p 2222 ompiteam@192.185.39.252 "cd public_html/software/pmix/nightly/$branch && md5sum pmix* > md5sums.txt"
ssh -p 2222 ompiteam@192.185.39.252 "cd public_html/software/pmix/nightly/$branch && sha1sum pmix* > sha1sums.txt"
else
echo "=== NOT saving output for a Coverity run"
fi
# Failed builds are not removed. But if a human forgets to come
# in here and clean up the old failed builds, we can accumulate
# many over time. So remove any old failed bbuilds that are over
# 4 weeks old.
${script_dir}/remove-old.pl 28 $build_root/$branch
done
# If we had any new snapshots to send to coverity, process them now
#for tarball in `cat $pending_coverity`; do
# echo "=== Submitting $tarball to Coverity..."
# /home/common/mpiteam/scripts/pmix-nightly-coverity.pl \
# --filename=$tarball \
# --coverity-token=$coverity_token \
# --verbose \
# --logfile-dir=$HOME/coverity \
# --make-args=-j8 \
# --configure-args="$coverity_configure_args"
#done
rm -f $pending_coverity

52
contrib/build-server/pmix-release.sh Исполняемый файл
Просмотреть файл

@ -0,0 +1,52 @@
#!/bin/sh -x
# The tarballs to make
if [ $# -eq 0 ] ; then
branches="v1.0"
else
branches=$1
shift
fi
# Build root - scratch space
build_root=/home/mpiteam/pmix/release
# Script to execute
script=contrib/make_dist_tarball
export PATH=$HOME/local/bin:$PATH
export LD_LIBRARY_PATH=$HOME/local/lib:$LD_LIBRARY_PATH
#####
#
# Actually do stuff
#
#####
# load the modules configuration
. /etc/profile.d/modules.sh
module use ~/modules
# move to the directory
# Loop making them
for branch in $branches; do
cd $build_root/$branch
module load "autotools/pmix-$branch"
module load libevent/pmix-$branch
./$script $@ >dist.out 2>&1
if test "$?" != "0"; then
cat <<EOF
=============================================================================
== Dist failure
== Last few lines of output (full results in dist.out file):
=============================================================================
EOF
tail -n 20 dist.out
exit 1
fi
module unload libevent
module unload autotools
done