diff --git a/contrib/build-server/hwloc-nightly-coverity.pl b/contrib/build-server/hwloc-nightly-coverity.pl deleted file mode 100755 index f2e1f1c4b8..0000000000 --- a/contrib/build-server/hwloc-nightly-coverity.pl +++ /dev/null @@ -1,167 +0,0 @@ -#!/usr/bin/env perl - -use warnings; -use strict; - -use Getopt::Long; -use File::Temp qw/ tempfile tempdir /; -use File::Basename; - -my $coverity_project = "hwloc"; -# Coverity changes this URL periodically -my $coverity_tool_url = "https://scan.coverity.com/download/cxx/linux64"; - -my $filename_arg; -my $coverity_token_arg; -my $dry_run_arg = 0; -my $verbose_arg = 0; -my $debug_arg = 0; -my $logfile_dir_arg; -my $configure_args = ""; -my $make_args = "-j 32"; -my $help_arg = 0; - -&Getopt::Long::Configure("bundling"); -my $ok = Getopt::Long::GetOptions("filename=s" => \$filename_arg, - "coverity-token=s" => \$coverity_token_arg, - "logfile-dir=s" => \$logfile_dir_arg, - "configure-args=s" => \$configure_args, - "make-args=s" => \$make_args, - "dry-run!" => \$dry_run_arg, - "verbose!" => \$verbose_arg, - "debug!" => \$debug_arg, - "help|h" => \$help_arg); - -$ok = 0 - if (!defined($filename_arg)); -$ok = 0 - if (!defined($coverity_token_arg)); -if (!$ok || $help_arg) { - print "Usage: $0 --filename=FILENAME --coverity-token=TOKEN [--dry-run] [--verbose] [--help]\n"; - exit($ok); -} - -die "Cannot read $filename_arg" - if (! -r $filename_arg); - -$verbose_arg = 1 - if ($debug_arg); - -###################################################################### - -sub verbose { - print @_ - if ($verbose_arg); -} - -# run a command and save the stdout / stderr -sub safe_system { - my $allowed_to_fail = shift; - my $cmd = shift; - my $stdout_file = shift; - - # Redirect stdout if requested or not verbose - if (defined($stdout_file)) { - $stdout_file = "$logfile_dir_arg/$stdout_file"; - unlink($stdout_file); - $cmd .= " >$stdout_file"; - } elsif (!$debug_arg) { - $cmd .= " >/dev/null"; - } - $cmd .= " 2>&1"; - - my $rc = system($cmd); - if (0 != $rc && !$allowed_to_fail) { - # If we die/fail, ensure to change out of the temp tree so - # that it can be removed upon exit. - chdir("/"); - print "Command $cmd failed: exit status $rc\n"; - if (defined($stdout_file) && -f $stdout_file) { - print "Last command output:\n"; - system("cat $stdout_file"); - } - die "Cannot continue"; - } - system("cat $stdout_file") - if ($debug_arg && defined($stdout_file) && -f $stdout_file); -} - -###################################################################### - -# Make an area to work - -my $dir = tempdir(CLEANUP => 0); -chdir($dir); -verbose "*** Working in $dir\n"; - -###################################################################### - -# Get the coverity tool, put it in our path - -my $cdir = "$ENV{HOME}/coverity"; -safe_system(0, "mkdir $cdir") - if (! -d $cdir); - -# Optimization: the tool is pretty large. If our local copy is less -# than a day old, just use that without re-downloading. -my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size, - $atime,$mtime,$ctime,$blksize,$blocks) = - stat("$cdir/coverity_tool.tgz"); -my $now = time(); -if (!defined($mtime) || $mtime < $now - 24*60*60) { - verbose "*** Downloading new copy of the coverity tool\n"; - safe_system(0, "wget $coverity_tool_url --post-data \"token=$coverity_token_arg&project=$coverity_project\" -O coverity_tool.tgz"); - safe_system(0, "cp coverity_tool.tgz $cdir"); -} - -verbose "*** Expanding coverity tool tarball\n"; -safe_system(0, "tar xf $cdir/coverity_tool.tgz"); -opendir(my $dh, ".") || - die "Can't opendir ."; -my @files = grep { /^cov/ && -d "./$_" } readdir($dh); -closedir($dh); - -my $cov_dir = "$dir/$files[0]/bin"; -$ENV{PATH} = "$cov_dir:$ENV{PATH}"; - -###################################################################### - -# Expand the HWLOC tarball, build it - -verbose "*** Extracting HWLOC tarball\n"; -safe_system(0, "tar xf $filename_arg"); -my $tarball_filename = basename($filename_arg); -$tarball_filename =~ m/^hwloc-(.+)\.tar.+$/; -my $hwloc_ver = $1; -chdir("hwloc-$hwloc_ver"); - -verbose "*** Configuring HWLOC tarball\n"; -safe_system(0, "./configure $configure_args", "configure"); - -verbose "*** Building HWLOC tarball\n"; -safe_system(0, "cov-build --dir cov-int make $make_args", "cov-build"); - -# Tar up the Coverity results -verbose "*** Tarring up results\n"; -safe_system(0, "tar jcf $hwloc_ver-analyzed.tar.bz2 cov-int"); - -# If not dry-run, submit to Coverity -if ($dry_run_arg) { - verbose "*** Would have submitted, but this is a dry run\n"; -} else { - verbose "*** Submitting results\n"; - safe_system(0, "curl --form token=$coverity_token_arg " . - "--form email=brice.goglin\@labri.fr " . - "--form file=\@$hwloc_ver-analyzed.tar.bz2 " . - "--form version=$hwloc_ver " . - "--form description=nightly-master " . - "https://scan.coverity.com/builds?project=hwloc", - "coverity-submit"); -} - -verbose("*** All done\n"); - -# Chdir out of the tempdir so that it can be removed -chdir("/"); - -exit(0); diff --git a/contrib/build-server/hwloc-nightly-tarball.sh b/contrib/build-server/hwloc-nightly-tarball.sh deleted file mode 100755 index 2d3d7891ad..0000000000 --- a/contrib/build-server/hwloc-nightly-tarball.sh +++ /dev/null @@ -1,186 +0,0 @@ -#!/bin/sh - -##### -# -# Configuration options -# -##### - -# e-mail address to send results to -results_addr=hwloc-devel@lists.open-mpi.org -#results_addr=rhc@open-mpi.org - -# git repository URL -code_uri=https://github.com/open-mpi/hwloc.git -raw_uri=https://raw.github.com/open-mpi/hwloc - -# where to put built tarballs -outputroot=$HOME/hwloc/nightly - -# Target where to scp the final tarballs -output_ssh_target=ompiteam@192.185.39.252 - -# where to find the build script -script_uri=contrib/nightly/make_snapshot_tarball - -# helper scripts dir -script_dir=$HOME/ompi/contrib/build-server - -# Set this to any value for additional output; typically only when -# debugging -: ${debug:=} - -# The tarballs to make -if [ $# -eq 0 ] ; then - # Branches v1.6 and earlier were not updated to build nightly - # snapshots from git, so only check v1.7 and later - branches="master v1.11" -else - branches=$@ -fi - -# Build root - scratch space -build_root=$HOME/hwloc/nightly-tarball-build-root - -# Coverity stuff -coverity_token=`cat $HOME/coverity/hwloc-token.txt` - -export PATH=$HOME_PREFIX/bin:$PATH -export LD_LIBRARY_PATH=$HOME_PREFIX/lib:$LD_LIBRARY_PATH - -##### -# -# Actually do stuff -# -##### - -debug() { - if test -n "$debug"; then - echo "=== DEBUG: $*" - fi -} - -run_command() { - debug "Running command: $*" - debug "Running in pwd: `pwd`" - if test -n "$debug"; then - eval $* - else - eval $* > /dev/null 2>&1 - fi - - if test $? -ne 0; then - echo "=== Command failed: $*" - fi -} - -# load the modules configuration -. $MODULE_INIT -module use $AUTOTOOL_MODULE - -# get our nightly build script -mkdir -p $build_root -cd $build_root - -pending_coverity=$build_root/tarballs-to-run-through-coverity.txt -rm -f $pending_coverity -touch $pending_coverity - -# Loop making them -module unload autotools -for branch in $branches; do - echo "=== Branch: $branch" - # Get the last tarball version that was made - prev_snapshot=`cat $outputroot/$branch/latest_snapshot.txt` - prev_snapshot_hash=`echo $prev_snapshot | cut -d- -f3` - echo "=== Previous snapshot: $prev_snapshot (hash: $prev_snapshot_hash)" - - # Form a URL-specific script name - script=$branch-`basename $script_uri` - - echo "=== Getting script from: $raw_uri" - run_command wget --quiet --no-check-certificate --tries=10 $raw_uri/$branch/$script_uri -O $script - if test ! $? -eq 0 ; then - echo "wget of hwloc nightly tarball create script failed." - if test -f $script ; then - echo "Using older version of $script for this run." - else - echo "No build script available. Aborting." - exit 1 - fi - fi - chmod +x $script - - module load "autotools/hwloc-$branch" - # module load "tex-live/hwloc-$branch" - - echo "=== Running script..." - run_command ./$script \ - $build_root/$branch \ - $results_addr \ - $outputroot/$branch \ - $code_uri \ - $branch - - module unload autotools - echo "=== Done running script" - - # Did the script generate a new tarball? Ensure to compare the - # only the hash of the previous tarball and the hash of the new - # tarball (the filename also contains the date/timestamp, which - # will always be different). - - # If so, save it so that we can spawn the coverity checker on it - # afterwards. Only for this for the master (for now). - latest_snapshot=`cat $outputroot/$branch/latest_snapshot.txt` - latest_snapshot_hash=`echo $latest_snapshot | cut -d- -f3` - echo "=== Latest snapshot: $latest_snapshot (hash: $latest_snapshot_hash)" - if test "$prev_snapshot_hash" = "$latest_snapshot_hash"; then - echo "=== Hash has not changed; no need to upload/save the new tarball" - else - if test "$branch" = "master"; then - echo "=== Saving output for a Coverity run" - echo "$outputroot/$branch/hwloc-$latest_snapshot.tar.bz2" >> $pending_coverity - else - echo "=== NOT saving output for a Coverity run" - fi - echo "=== Posting tarball to open-mpi.org" - # tell the web server to cleanup old nightly tarballs - run_command ssh -p 2222 \ - $output_ssh_target \ - \"git/ompi/contrib/build-server/remove-old.pl 7 public_html/software/hwloc/nightly/$branch\" - # upload the new ones - run_command scp -P 2222 \ - $outputroot/$branch/hwloc-$latest_snapshot.tar.* \ - $output_ssh_target:public_html/software/hwloc/nightly/$branch/ - run_command scp -P 2222 \ - $outputroot/$branch/latest_snapshot.txt \ - $output_ssh_target:public_html/software/hwloc/nightly/$branch/ - # direct the web server to regenerate the checksums - run_command ssh -p 2222 \ - $output_ssh_target \ - \"cd public_html/software/hwloc/nightly/$branch \&\& md5sum hwloc\* \> md5sums.txt\" - run_command ssh -p 2222 \ - $output_ssh_target \ - \"cd public_html/software/hwloc/nightly/$branch \&\& sha1sum hwloc\* \> sha1sums.txt\" - fi - - # Failed builds are not removed. But if a human forgets to come - # in here and clean up the old failed builds, we can accumulate - # many over time. So remove any old failed builds that are over - # 4 weeks old. - run_command ${script_dir}/remove-old.pl 7 $build_root/$branch - -done - -# If we had any new snapshots to send to coverity, process them now - -for tarball in `cat $pending_coverity`; do - run_command ${script_dir}/hwloc-nightly-coverity.pl \ - --filename=$tarball \ - --coverity-token=$coverity_token \ - --verbose \ - --logfile-dir=$HOME/coverity \ - --make-args=\"-j8\" -done -rm -f $pending_coverity diff --git a/contrib/build-server/openmpi-nightly-coverity.pl b/contrib/build-server/openmpi-nightly-coverity.pl deleted file mode 100755 index 12390ab22d..0000000000 --- a/contrib/build-server/openmpi-nightly-coverity.pl +++ /dev/null @@ -1,162 +0,0 @@ -#!/usr/bin/env perl - -use warnings; -use strict; - -use Getopt::Long; -use File::Temp qw/ tempfile tempdir /; -use File::Basename; - -my $coverity_project = "Open+MPI"; -# Coverity changes this URL periodically -my $coverity_tool_url = "https://scan.coverity.com/download/cxx/linux64"; - -my $filename_arg; -my $coverity_token_arg; -my $dry_run_arg = 0; -my $verbose_arg = 0; -my $debug_arg = 0; -my $logfile_dir_arg = "/tmp"; -my $configure_args = ""; -my $make_args = "-j 32"; -my $help_arg = 0; - -&Getopt::Long::Configure("bundling"); -my $ok = Getopt::Long::GetOptions("filename=s" => \$filename_arg, - "coverity-token=s" => \$coverity_token_arg, - "logfile-dir=s" => \$logfile_dir_arg, - "configure-args=s" => \$configure_args, - "make-args=s" => \$make_args, - "dry-run!" => \$dry_run_arg, - "verbose!" => \$verbose_arg, - "debug!" => \$debug_arg, - "help|h" => \$help_arg); - -$ok = 0 - if (!defined($filename_arg)); -$ok = 0 - if (!defined($coverity_token_arg)); -if (!$ok || $help_arg) { - print "Usage: $0 --filename=FILENAME --coverity-token=TOKEN [--dry-run] [--verbose] [--help]\n"; - exit($ok); -} - -die "Cannot read $filename_arg" - if (! -r $filename_arg); - -$verbose_arg = 1 - if ($debug_arg); - -###################################################################### - -sub verbose { - print @_ - if ($verbose_arg); -} - -# run a command and save the stdout / stderr -sub safe_system { - my $allowed_to_fail = shift; - my $cmd = shift; - my $stdout_file = shift; - - # Redirect stdout if requested or not verbose - if (defined($stdout_file)) { - $stdout_file = "$logfile_dir_arg/$stdout_file"; - unlink($stdout_file); - $cmd .= " >$stdout_file"; - } elsif (!$debug_arg) { - $cmd .= " >/dev/null"; - } - $cmd .= " 2>&1"; - - my $rc = system($cmd); - if (0 != $rc && !$allowed_to_fail) { - # If we die/fail, ensure to change out of the temp tree so - # that it can be removed upon exit. - chdir("/"); - die "Command $cmd failed: exit status $rc"; - } - system("cat $stdout_file") - if ($debug_arg && defined($stdout_file) && -f $stdout_file); -} - -###################################################################### - -# Make an area to work - -my $dir = tempdir(CLEANUP => 1); -chdir($dir); -verbose "*** Working in $dir\n"; - -###################################################################### - -# Get the coverity tool, put it in our path - -my $cdir = "$ENV{HOME}/coverity"; -safe_system(0, "mkdir $cdir") - if (! -d $cdir); - -# Optimization: the tool is pretty large. If our local copy is less -# than a day old, just use that without re-downloading. -my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size, - $atime,$mtime,$ctime,$blksize,$blocks) = - stat("$cdir/coverity_tool.tgz"); -my $now = time(); -if (!defined($mtime) || $mtime < $now - 24*60*60) { - verbose "*** Downloading new copy of the coverity tool\n"; - safe_system(0, "wget $coverity_tool_url --post-data \"token=$coverity_token_arg&project=$coverity_project\" -O coverity_tool.tgz"); - safe_system(0, "cp coverity_tool.tgz $cdir"); -} - -verbose "*** Expanding coverity tool tarball\n"; -safe_system(0, "tar xf $cdir/coverity_tool.tgz"); -opendir(my $dh, ".") || - die "Can't opendir ."; -my @files = grep { /^cov/ && -d "./$_" } readdir($dh); -closedir($dh); - -my $cov_dir = "$dir/$files[0]/bin"; -$ENV{PATH} = "$cov_dir:$ENV{PATH}"; - -###################################################################### - -# Expand the OMPI tarball, build it - -verbose "*** Extracting OMPI tarball\n"; -safe_system(0, "tar xf $filename_arg"); -my $tarball_filename = basename($filename_arg); -$tarball_filename =~ m/^openmpi-(.+)\.tar.+$/; -my $ompi_ver = $1; -chdir("openmpi-$ompi_ver"); - -verbose "*** Configuring OMPI tarball\n"; -safe_system(0, "./configure $configure_args", "configure"); - -verbose "*** Building OMPI tarball\n"; -safe_system(0, "cov-build --dir cov-int make $make_args", "cov-build"); - -# Tar up the Coverity results -verbose "*** Tarring up results\n"; -safe_system(0, "tar jcf $ompi_ver-analyzed.tar.bz2 cov-int"); - -# If not dry-run, submit to Coverity -if ($dry_run_arg) { - verbose "*** Would have submitted, but this is a dry run\n"; -} else { - verbose "*** Submitting results\n"; - safe_system(0, "curl --form token=$coverity_token_arg " . - "--form email=jsquyres\@cisco.com " . - "--form file=\@$ompi_ver-analyzed.tar.bz2 " . - "--form version=$ompi_ver " . - "--form description=nightly-master " . - "https://scan.coverity.com/builds?project=$coverity_project", - "coverity-submit"); -} - -verbose("*** All done\n"); - -# Chdir out of the tempdir so that it can be removed -chdir("/"); - -exit(0); diff --git a/contrib/build-server/openmpi-nightly-tarball.sh b/contrib/build-server/openmpi-nightly-tarball.sh deleted file mode 100755 index 52ca79bb2d..0000000000 --- a/contrib/build-server/openmpi-nightly-tarball.sh +++ /dev/null @@ -1,192 +0,0 @@ -#!/bin/sh - -##### -# -# Configuration options -# -##### - -# e-mail address to send results to -results_addr=testing@lists.open-mpi.org -#results_addr=rhc@open-mpi.org - -# Set this to any value for additional output; typically only when -# debugging -: ${debug:=} - -# svn repository uri -master_code_uri=https://github.com/open-mpi/ompi.git -master_raw_uri=https://raw.github.com/open-mpi/ompi - -# where to put built tarballs - needs to be -# adjusted to match your site! -outputroot=$HOME/openmpi/nightly - -# Target where to scp the final tarballs -output_ssh_target=ompiteam@192.185.39.252 - -# where to find the build script -script_uri=contrib/nightly/create_tarball.sh - -# helper scripts dir -script_dir=$HOME/ompi/contrib/build-server - -# The tarballs to make -if [ $# -eq 0 ] ; then - # We're no longer ever checking the 1.0 - 1.8 branches anymore - branches="master v1.10 v2.x v2.0.x v3.x" -else - branches=$@ -fi - -# Build root - scratch space -build_root=$HOME/openmpi/nightly-tarball-build-root - -# Coverity stuff -coverity_token=`cat $HOME/coverity/openmpi-token.txt` -coverity_configure_args="--enable-debug --enable-mpi-fortran --enable-mpi-cxx --enable-mpi-java --enable-oshmem --enable-oshmem-fortran --with-usnic --with-libfabric=/mnt/data/local-installs" - -export PATH=$HOME_PREFIX/bin:$PATH -export LD_LIBRARY_PATH=$HOME_PREFIX/lib:$LD_LIBRARY_PATH - -##### -# -# Actually do stuff -# -##### - -debug() { - if test -n "$debug"; then - echo "=== DEBUG: $*" - fi -} - -run_command() { - debug "Running command: $*" - debug "Running in pwd: `pwd`" - if test -n "$debug"; then - eval $* - else - eval $* > /dev/null 2>&1 - fi - - if test $? -ne 0; then - echo "=== Command failed: $*" - fi -} - -# load the modules configuration -. $MODULE_INIT -module use $AUTOTOOL_MODULE - -# get our nightly build script -mkdir -p $build_root -cd $build_root - -pending_coverity=$build_root/tarballs-to-run-through-coverity.txt -rm -f $pending_coverity -touch $pending_coverity - -# Loop making the tarballs -module unload autotools -for branch in $branches; do - echo "=== Branch: $branch" - # Get the last tarball version that was made - prev_snapshot=`cat $outputroot/$branch/latest_snapshot.txt` - prev_snapshot_hash=`echo $prev_snapshot | cut -d- -f3` - echo "=== Previous snapshot: $prev_snapshot (hash: $prev_snapshot_hash)" - - code_uri=$master_code_uri - raw_uri=$master_raw_uri - - # Form a URL-specific script name - script=$branch-`basename $script_uri` - - echo "=== Getting script from: $raw_uri" - run_command wget --quiet --no-check-certificate --tries=10 $raw_uri/$branch/$script_uri -O $script - if test ! $? -eq 0 ; then - echo "wget of OMPI nightly tarball create script failed." - if test -f $script ; then - echo "Using older version of $script for this run." - else - echo "No build script available. Aborting." - exit 1 - fi - fi - chmod +x $script - - module load "autotools/ompi-$branch" - - echo "=== Running script..." - run_command eval ./$script \ - $build_root/$branch \ - $results_addr \ - $outputroot/$branch \ - $code_uri \ - $branch - - module unload autotools - echo "=== Done running script" - - # Did the script generate a new tarball? Ensure to compare the - # only the hash of the previous tarball and the hash of the new - # tarball (the filename also contains the date/timestamp, which - # will always be different). - - # If so, save it so that we can spawn the coverity checker on it - # afterwards. Only for this for the master (for now). - latest_snapshot=`cat $outputroot/$branch/latest_snapshot.txt` - latest_snapshot_hash=`echo $latest_snapshot | cut -d- -f3` - echo "=== Latest snapshot: $latest_snapshot (hash: $latest_snapshot_hash)" - if test "$prev_snapshot_hash" = "$latest_snapshot_hash"; then - echo "=== Hash has not changed; no need to upload/save the new tarball" - else - if test "$branch" = "master"; then - echo "=== Saving output for a Coverity run" - echo "$outputroot/$branch/openmpi-$latest_snapshot.tar.bz2" >> $pending_coverity - else - echo "=== NOT saving output for a Coverity run" - fi - echo "=== Posting tarball to open-mpi.org" - # tell the web server to cleanup old nightly tarballs - run_command ssh -p 2222 \ - $output_ssh_target \ - \"git/ompi/contrib/build-server/remove-old.pl 7 public_html/nightly/$branch\" - # upload the new ones - run_command scp -P 2222 \ - $outputroot/$branch/openmpi-$latest_snapshot.tar.* \ - $output_ssh_target:public_html/nightly/$branch/ - run_command scp -P 2222 \ - $outputroot/$branch/latest_snapshot.txt \ - $output_ssh_target:public_html/nightly/$branch/ - # direct the web server to regenerate the checksums - run_command ssh -p 2222 \ - $output_ssh_target \ - \"cd public_html/nightly/$branch \&\& md5sum openmpi\* \> md5sums.txt\" - run_command ssh -p 2222 \ - $output_ssh_target \ - \"cd public_html/nightly/$branch \&\& sha1sum openmpi\* \> sha1sums.txt\" - fi - - # Failed builds are not removed. But if a human forgets to come - # in here and clean up the old failed builds, we can accumulate - # many over time. So remove any old failed builds that are over - # 4 weeks old. - run_command ${script_dir}/remove-old.pl 7 $build_root/$branch - -done - - -# If we had any new snapshots to send to coverity, process them now - -for tarball in `cat $pending_coverity`; do - echo "=== Submitting $tarball to Coverity..." - run_command ${script_dir}/openmpi-nightly-coverity.pl \ - --filename=$tarball \ - --coverity-token=$coverity_token \ - --verbose \ - --logfile-dir=$HOME/coverity \ - --make-args=-j4 \ - --configure-args=\"$coverity_configure_args\" -done -rm -f $pending_coverity diff --git a/contrib/build-server/pmix-nightly-coverity.pl b/contrib/build-server/pmix-nightly-coverity.pl deleted file mode 100755 index 329393fd3b..0000000000 --- a/contrib/build-server/pmix-nightly-coverity.pl +++ /dev/null @@ -1,162 +0,0 @@ -#!/usr/bin/env perl - -use warnings; -use strict; - -use Getopt::Long; -use File::Temp qw/ tempfile tempdir /; -use File::Basename; - -my $coverity_project = "open-mpi%2Fpmix"; -# Coverity changes this URL periodically -my $coverity_tool_url = "https://scan.coverity.com/download/cxx/linux64"; - -my $filename_arg; -my $coverity_token_arg; -my $dry_run_arg = 0; -my $verbose_arg = 0; -my $debug_arg = 0; -my $logfile_dir_arg = "/tmp"; -my $configure_args = ""; -my $make_args = "-j 32"; -my $help_arg = 0; - -&Getopt::Long::Configure("bundling"); -my $ok = Getopt::Long::GetOptions("filename=s" => \$filename_arg, - "coverity-token=s" => \$coverity_token_arg, - "logfile-dir=s" => \$logfile_dir_arg, - "configure-args=s" => \$configure_args, - "make-args=s" => \$make_args, - "dry-run!" => \$dry_run_arg, - "verbose!" => \$verbose_arg, - "debug!" => \$debug_arg, - "help|h" => \$help_arg); - -$ok = 0 - if (!defined($filename_arg)); -$ok = 0 - if (!defined($coverity_token_arg)); -if (!$ok || $help_arg) { - print "Usage: $0 --filename=FILENAME --coverity-token=TOKEN [--dry-run] [--verbose] [--help]\n"; - exit($ok); -} - -die "Cannot read $filename_arg" - if (! -r $filename_arg); - -$verbose_arg = 1 - if ($debug_arg); - -###################################################################### - -sub verbose { - print @_ - if ($verbose_arg); -} - -# run a command and save the stdout / stderr -sub safe_system { - my $allowed_to_fail = shift; - my $cmd = shift; - my $stdout_file = shift; - - # Redirect stdout if requested or not verbose - if (defined($stdout_file)) { - $stdout_file = "$logfile_dir_arg/$stdout_file"; - unlink($stdout_file); - $cmd .= " >$stdout_file"; - } elsif (!$debug_arg) { - $cmd .= " >/dev/null"; - } - $cmd .= " 2>&1"; - - my $rc = system($cmd); - if (0 != $rc && !$allowed_to_fail) { - # If we die/fail, ensure to change out of the temp tree so - # that it can be removed upon exit. - chdir("/"); - die "Command $cmd failed: exit status $rc"; - } - system("cat $stdout_file") - if ($debug_arg && defined($stdout_file) && -f $stdout_file); -} - -###################################################################### - -# Make an area to work - -my $dir = tempdir(CLEANUP => 1); -chdir($dir); -verbose "*** Working in $dir\n"; - -###################################################################### - -# Get the coverity tool, put it in our path. - -my $cdir = "$ENV{HOME}/coverity"; -safe_system(0, "mkdir $cdir") - if (! -d $cdir); - -# Optimization: the tool is pretty large. If our local copy is less -# than a day old, just use that without re-downloading. -my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size, - $atime,$mtime,$ctime,$blksize,$blocks) = - stat("$cdir/coverity_tool.tgz"); -my $now = time(); -if (!defined($mtime) || $mtime < $now - 24*60*60) { - verbose "*** Downloading new copy of the coverity tool\n"; - safe_system(0, "wget $coverity_tool_url --post-data \"token=$coverity_token_arg&project=$coverity_project\" -O coverity_tool.tgz"); - safe_system(0, "cp coverity_tool.tgz $cdir"); -} - -verbose "*** Expanding coverity tool tarball\n"; -safe_system(0, "tar xf $cdir/coverity_tool.tgz"); -opendir(my $dh, ".") || - die "Can't opendir ."; -my @files = grep { /^cov/ && -d "./$_" } readdir($dh); -closedir($dh); - -my $cov_dir = "$dir/$files[0]/bin"; -$ENV{PATH} = "$cov_dir:$ENV{PATH}"; - -###################################################################### - -# Expand the PMIX tarball, build it - -verbose "*** Extracting PMIX tarball\n"; -safe_system(0, "tar xf $filename_arg"); -my $tarball_filename = basename($filename_arg); -$tarball_filename =~ m/^pmix-(.+)\.tar.+$/; -my $pmix_ver = $1; -chdir("pmix-$pmix_ver"); - -verbose "*** Configuring PMIX tarball\n"; -safe_system(0, "./configure $configure_args", "configure"); - -verbose "*** Building PMIX tarball\n"; -safe_system(0, "cov-build --dir cov-int make $make_args", "cov-build"); - -# Tar up the Coverity results -verbose "*** Tarring up results\n"; -safe_system(0, "tar jcf $pmix_ver-analyzed.tar.bz2 cov-int"); - -# If not dry-run, submit to Coverity -if ($dry_run_arg) { - verbose "*** Would have submitted, but this is a dry run\n"; -} else { - verbose "*** Submitting results\n"; - safe_system(0, "curl --form token=$coverity_token_arg " . - "--form email=rhc\@open-mpi.org " . - "--form file=\@$pmix_ver-analyzed.tar.bz2 " . - "--form version=$pmix_ver " . - "--form description=nightly-master " . - "https://scan.coverity.com/builds?project=$coverity_project", - "coverity-submit"); -} - -verbose("*** All done\n"); - -# Chdir out of the tempdir so that it can be removed -chdir("/"); - -exit(0); diff --git a/contrib/build-server/pmix-nightly-tarball.sh b/contrib/build-server/pmix-nightly-tarball.sh deleted file mode 100755 index 6a248ef1e4..0000000000 --- a/contrib/build-server/pmix-nightly-tarball.sh +++ /dev/null @@ -1,196 +0,0 @@ -#!/bin/sh - -##### -# -# Configuration options -# -##### - -# e-mail address to send results to -#results_addr=testing@lists.open-mpi.org -results_addr=rhc@open-mpi.org - -# Set this to any value for additional output; typically only when -# debugging -: ${debug:=} - -# svn repository uri -master_code_uri=https://github.com/pmix/master.git -master_raw_uri=https://raw.github.com/pmix/master -release_code_uri=https://github.com/pmix/releases.git -release_raw_uri=https://raw.github.com/pmix/releases - -# where to put built tarballs -outputroot=$HOME/pmix/nightly - -# Target where to scp the final tarballs -output_ssh_target=ompiteam@192.185.39.252 - -# where to find the build script -script_uri=contrib/nightly/create_tarball.sh - -# helper scripts dir -script_dir=$HOME/ompi/contrib/build-server - -# The tarballs to make -if [ $# -eq 0 ] ; then - branches="master" -else - branches=$@ -fi - -# Build root - scratch space -build_root=$HOME/pmix/nightly-tarball-build-root - -# Coverity stuff -coverity_token=`cat $HOME/coverity/pmix-token.txt` -coverity_configure_args="--with-libevent=$HOME_PREFIX" - -export PATH=$HOME_PREFIX/bin:$PATH -export LD_LIBRARY_PATH=$HOME_PREFIX/lib:$LD_LIBRARY_PATH - -##### -# -# Actually do stuff -# -##### - -debug() { - if test -n "$debug"; then - echo "=== DEBUG: $*" - fi -} - -run_command() { - debug "Running command: $*" - debug "Running in pwd: `pwd`" - if test -n "$debug"; then - eval $* - else - eval $* > /dev/null 2>&1 - fi - - if test $? -ne 0; then - echo "=== Command failed: $*" - fi -} - -# load the modules configuration -. $MODULE_INIT -module use $AUTOTOOL_MODULE - -# get our nightly build script -mkdir -p $build_root -cd $build_root - -pending_coverity=$build_root/tarballs-to-run-through-coverity.txt -rm -f $pending_coverity -touch $pending_coverity - -# Loop making the tarballs -module unload autotools -for branch in $branches; do - echo "=== Branch: $branch" - # Get the last tarball version that was made - prev_snapshot=`cat $outputroot/$branch/latest_snapshot.txt` - prev_snapshot_hash=`echo $prev_snapshot | cut -d- -f3` - echo "=== Previous snapshot: $prev_snapshot (hash: $prev_snapshot_hash)" - - if test "$branch" = "master"; then - code_uri=$master_code_uri - raw_uri=$master_raw_uri - else - code_uri=$release_code_uri - raw_uri=$release_raw_uri - fi - - # Form a URL-specific script name - script=$branch-`basename $script_uri` - - echo "=== Getting script from: $raw_uri" - run_command wget --quiet --no-check-certificate --tries=10 $raw_uri/$branch/$script_uri -O $script - if test ! $? -eq 0 ; then - echo "wget of PMIX nightly tarball create script failed." - if test -f $script ; then - echo "Using older version of $script for this run." - else - echo "No build script available. Aborting." - exit 1 - fi - fi - chmod +x $script - - module load "autotools/pmix-$branch" - # module load "libevent/pmix-$branch" - - echo "=== Running script..." - run_command ./$script \ - $build_root/$branch \ - $results_addr \ - $outputroot/$branch \ - $code_uri \ - $branch - - module unload autotools - echo "=== Done running script" - - # Did the script generate a new tarball? Ensure to compare the - # only the hash of the previous tarball and the hash of the new - # tarball (the filename also contains the date/timestamp, which - # will always be different). - - # If so, save it so that we can spawn the coverity checker on it - # afterwards. Only for this for the master (for now). - latest_snapshot=`cat $outputroot/$branch/latest_snapshot.txt` - latest_snapshot_hash=`echo $latest_snapshot | cut -d- -f3` - echo "=== Latest snapshot: $latest_snapshot (hash: $latest_snapshot_hash)" - if test "$prev_snapshot_hash" = "$latest_snapshot_hash"; then - echo "=== Hash has not changed; no need to upload/save the new tarball" - else - if test "$branch" = "master"; then - echo "=== Saving output for a Coverity run" - echo "$outputroot/$branch/pmix-$latest_snapshot.tar.bz2" >> $pending_coverity - else - echo "=== NOT saving output for a Coverity run" - fi - echo "=== Posting tarball to open-mpi.org" - # tell the web server to cleanup old nightly tarballs - run_command ssh -p 2222 \ - $output_ssh_target \ - \"git/ompi/contrib/build-server/remove-old.pl 7 public_html/software/pmix/nightly/$branch\" - # upload the new ones - run_command scp -P 2222 \ - $outputroot/$branch/pmix-$latest_snapshot.tar.* \ - $output_ssh_target:public_html/software/pmix/nightly/$branch/ - run_command scp -P 2222 \ - $outputroot/$branch/latest_snapshot.txt \ - $output_ssh_target:public_html/software/pmix/nightly/$branch/ - # direct the web server to regenerate the checksums - run_command ssh -p 2222 \ - $output_ssh_target \ - \"cd public_html/software/pmix/nightly/$branch \&\& md5sum pmix\* \> md5sums.txt\" - run_command ssh -p 2222 \ - $output_ssh_target \ - \"cd public_html/software/pmix/nightly/$branch \&\& sha1sum pmix\* \> sha1sums.txt\" - fi - - # Failed builds are not removed. But if a human forgets to come - # in here and clean up the old failed builds, we can accumulate - # many over time. So remove any old failed bbuilds that are over - # 4 weeks old. - run_command ${script_dir}/remove-old.pl 7 $build_root/$branch -done - -# If we had any new snapshots to send to coverity, process them now - -for tarball in `cat $pending_coverity`; do - echo "=== Submitting $tarball to Coverity..." - run_command ${script_dir}/pmix-nightly-coverity.pl \ - --filename=$tarball \ - --coverity-token=$coverity_token \ - --verbose \ - --logfile-dir=$HOME/coverity \ - --make-args=-j8 \ - --configure-args=\"$coverity_configure_args\" -done -rm -f $pending_coverity diff --git a/contrib/build-server/remove-old.pl b/contrib/build-server/remove-old.pl deleted file mode 100755 index b16533eda2..0000000000 --- a/contrib/build-server/remove-old.pl +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env perl - -use strict; -use warnings; - -use POSIX qw(strftime); - -my $happy = 1; - -my $savedays = $ARGV[0]; -my $dir = $ARGV[1]; - -$happy = 0 - if ($savedays <= 0 || ! -d $dir); -die "Must specify number of days and a directory" - if (!$happy); - -#------------------------------------------------------------------ - -# Read in all the dir entries -opendir(DIR, $dir) || die "Cannot open $dir"; -my @files = readdir(DIR); -closedir(DIR); - -# How many days to keep? -my $t = time() - ($savedays * 60 * 60 * 24); -print "Deleting anything in $dir before: " . strftime("%D", localtime($t)) . "\n"; -my $to_delete = ""; - -# Check everything in the dir; if is a dir, is not . or .., and is -# older than the save date, keep it for deleting later. -foreach my $file (sort(@files)) { - if ($file ne "index.php" && - $file ne "md5sums.txt" && - $file ne "sha1sums.txt" && - $file ne "latest_snapshot.txt" && - $file ne "." && - $file ne "..") { - my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size, - $atime,$mtime,$ctime,$blksize,$blocks) = stat("$dir/$file"); - my $str = "SAVE"; - if ($mtime < $t) { - $to_delete = "$to_delete $dir/$file"; - $str = "DELETE"; - } - print "Found $file: $str (mtime: " . strftime("%D", localtime($mtime)) . ")\n"; - } -} - -# If we found anything to delete, do so. -if ($to_delete ne "") { - print "Deleting: $to_delete\n"; - system("chmod -R u=rwx $to_delete"); - system("rm -rf $to_delete"); - } else { - print "Nothing to delete!\n"; -} - -exit(0); diff --git a/contrib/nightly/create_tarball.sh b/contrib/nightly/create_tarball.sh deleted file mode 100644 index 3b595b4c0b..0000000000 --- a/contrib/nightly/create_tarball.sh +++ /dev/null @@ -1,311 +0,0 @@ -#!/bin/sh -# -# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana -# University Research and Technology -# Corporation. All rights reserved. -# Copyright (c) 2004-2005 The University of Tennessee and The University -# of Tennessee Research Foundation. All rights -# reserved. -# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, -# University of Stuttgart. All rights reserved. -# Copyright (c) 2004-2005 The Regents of the University of California. -# All rights reserved. -# Copyright (c) 2006-2017 Cisco Systems, Inc. All rights reserved. -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# -# This script is used to create a nightly snapshot tarball of Open MPI. -# -# $1: scratch root -# $2: e-mail address for destination -# $3: dest dir -# $4: git URL -# $5: git branch -# - -scratch_root=$1 -email=$2 -destdir=$3 -giturl=$4 -gitbranch=$5 - -# Set this to any value for additional output; typically only when -# debugging -: ${debug:=} - -# do you want a success mail? -want_success_mail=1 - -# max length of logfile to send in an e-mail -max_log_len=50 - -# how many snapshots to keep in the destdir? -max_snapshots=5 - -############################################################################ -# Shouldn't need to change below this line -############################################################################ - -start_time="`date`" - -# Sanity checks -if test -z "$scratch_root" -o -z "$email" -o -z "$giturl" -o -z "$gitbranch" \ - -o -z "$destdir"; then - echo "$0 scratch_root email_addr dest_dir git_url git_branch" - exit 1 -fi - -# Use the branch name as the "version" string (for if there is an -# error). This version string will be replaced upon successful "make -# distcheck" with the real version. -version=$gitbranch - -# send a mail -# should only be called after logdir is set -send_error_mail() { - outfile="$scratch_root/output.txt" - rm -f "$outfile" - touch "$outfile" - for file in `/bin/ls $logdir/* | sort`; do - len="`wc -l $file | awk '{ print $1}'`" - if test "`expr $len \> $max_log_len`" = "1"; then - echo "[... previous lines snipped ...]" >> "$outfile" - tail -n $max_log_len "$file" >> "$outfile" - else - cat "$file" >> "$outfile" - fi - done - Mail -s "=== CREATE FAILURE ($version) ===" "$email" < "$outfile" - rm -f "$outfile" -} - -# send output error message -die() { - msg="$*" - cat > "$logdir/00_announce.txt" < "$logfile" 2>&1 - st=$? - echo "*** Command complete: exit status: $st" - else - eval $cmd > "$logfile" 2>&1 - st=$? - fi - if test "$st" != "0"; then - cat > "$logdir/15-error.txt" < "$logdir/25-error.txt" < VERSION.new -cp -f VERSION.new VERSION -rm -f VERSION.new - -# lie about our username in $USER so that autogen will skip all -# .ompi_ignore'ed directories (i.e., so that we won't get -# .ompi_unignore'ed) -USER="ompibuilder" -export USER - -# autogen is our friend -do_command "./autogen.pl --force" - -# do config -do_command "./configure" - -# Do make distcheck (which will invoke config/distscript.csh to set -# the right values in VERSION). distcheck does many things; we need -# to ensure it doesn't pick up any other installs via LD_LIBRARY_PATH. -# It may be a bit Draconian to totally clean LD_LIBRARY_PATH (i.e., we -# may need something in there), but at least in the current building -# setup, we don't. But be advised that this may need to change in the -# future... -save=$LD_LIBRARY_PATH -LD_LIBRARY_PATH= -do_command "make -j 8 distcheck" -LD_LIBRARY_PATH=$save -save= - -# chmod the whole directory, so that core files are accessible by others -chmod a+rX -R . - -# move the resulting tarballs to the destdir -gz="`/bin/ls openmpi*tar.gz`" -bz2="`/bin/ls openmpi*tar.bz2`" -mv $gz $bz2 $destdir -if test "$?" != "0"; then - cat < latest_snapshot.txt - -# trim the destdir to $max_snapshots -for ext in gz bz2; do - count="`ls openmpi*.tar.$ext | wc -l | awk '{ print $1 }'`" - if test "`expr $count \> $max_snapshots`" = "1"; then - num_old="`expr $count - $max_snapshots`" - old="`ls -rt openmpi*.tar.$ext | head -n $num_old`" - rm -f $old - fi -done - -# generate md5 and sha1 sums -rm -f md5sums.txt sha1sums.txt -touch md5sums.txt sha1sums.txt -for file in `/bin/ls *gz *bz2 | grep -v latest`; do - md5sum $file >> md5sums.txt - sha1sum $file >> sha1sums.txt -done - -# remove temp dirs -cd "$scratch_root" -rm -rf "$root" - -# send success mail -if test "$want_success_mail" = "1"; then - Mail -s "Create success ($version)" "$email" <