ompi_cronjob.sh

Nightly build of trunk and test-suite (PMB and mpich-testsuite)
  as defined in build-HOSTNAME.txt
  Uses build_tarball.pl (which creates ompi-out-VERSION.txt)
  Uses build_tests.pl (which uses ompi-out-VERSION.txt and creates
ompi-tests-out-TEST_VERSION.txt)
  Then should qsub the all created combinations of MPI-installations

build_tests.pl:
 - Updated to take package_dir; the test-suites subdirectory to 
   configure -- currently PMB and mpich-testsuite

This commit was SVN r6604.

contrib/nightly/build_tests.pl

@@ -378,10 +378,15 @@ sub try_untar {
 }
     
 sub try_configure {
-    my ($name, $merge_output, $tarball, $srcroot, $installdir,
+    my ($name, $merge_output, $tarball, $srcroot, $package_dir, $installdir,
         $vpath_mode, $confargs) = @_;
     my $ret;
 
+    #
+    # Attention -- package_dir should only be one level higher...
+    #
+    chdir ($package_dir);
+
     # configure it
     my $config_command = "./configure";
     if ($vpath_mode) {
@@ -389,12 +394,9 @@ sub try_configure {
         chdir("vpath_build");
         if ($vpath_mode eq "relative") {
             $config_command = "../configure";
+        } else {
+            $config_command = "$srcroot/$file_base_name-$version/$package_dir/configure";
         }
-#
-# Disable absolute for now
-#         else {
-#            $config_command = "$srcroot/$file_base_name-$version/configure";
-#        }
     }
 
     $ret = do_command(1, "$config_command $confargs --prefix=$installdir");
@@ -435,6 +437,8 @@ sub try_configure {
       chdir ("..");
     }
 
+    chdir ("..");
+
     return {
         make_all_stderr => $make_all_stderr,
         status => 0,
@@ -697,7 +701,6 @@ if ($outfile_arg) {
 test_abort("Could not find output file:$outfile -- aborting")
   if ( ! -f $outfile );
 
-open OUTFILE, "$outfile";
 
 # loop over all configurations
 # be lazy: if no configurations supplied, do a default
@@ -712,12 +715,18 @@ if (! $config_arg || ! -f $config_arg) {
 
     die "NOT SUPPORTED YET\n";
 
+    $install_dir="$dir/install";
+
+    if ($install_dir_arg) {
+        $install_dir="$install_dir_arg/$name";
+    }
+
     $ret = try_untar($name, 0, $tarball_name,
-                     $dir, "$dir/install", "", $config);
+                     $dir, "$install_dir", "", $config);
     $ret = try_configure($name, 0, $tarball_name,
-                         $dir, "$dir/install", "", $config);
+                         $dir, "$install_dir", "", $config);
     $ret = cleanup($name, 0, $tarball_name,
-                   $dir, "$dir/install", "", $config);
+                   $dir, "$install_dir", "", $config);
     $results->{$name} = $ret;
     $results->{$name}->{config} = $config;
     $results->{$name}->{want_stderr} = 1;
@@ -748,30 +757,32 @@ if (! $config_arg || ! -f $config_arg) {
             # stupid emacs: '"
             my $merge_output = $want_stderr ? 0 : 1;
             $ret = try_untar($name, $merge_output, $tarball_name,
-                             $dir, "$dir/install",
+                             $dir, "$install_dir",
                              $vpath_mode, $config);
 
+            open OUTFILE, "$outfile";
             while (<OUTFILE>) {
                 my $mpi_dir=$_;
                 my $added_config="$config --with-mpi-dir=$mpi_dir";
+                my @check_this = split (/\//, $mpi_dir);
+                my $mpi_installation = pop (@check_this);
+                chomp($mpi_installation);
+                # print "mpi_dir:$mpi_dir\n";
 
                 $install_dir="$dir/install";
 
                 if ($install_dir_arg) {
-                    $install_dir="$install_dir_arg/$name";
+                    $install_dir="$install_dir_arg/mpi_$mpi_installation/config_$name";
                 }
+                # printf "install_dir:$install_dir\n";
  
-                chdir ("PMB-2.2");
                 $ret = try_configure($name, $merge_output, $tarball_name,
-                                     $dir, "$install_dir",
+                                     $dir, "PMB-2.2", "$install_dir",
+                                     $vpath_mode, $added_config);
+                $ret = try_configure($name, $merge_output, $tarball_name,
+                                     $dir, "mpich_tester", "$install_dir",
                                      $vpath_mode, $added_config);
-                chdir ("..");
 
-                chdir ("mpich_tester");
-                $ret = try_configure($name, $merge_output, $tarball_name,
-                                     $dir, "$install_dir",
-                                     $vpath_mode, $added_config);
-                chdir ("..");
                 $results->{$name} = $ret;
                 $results->{$name} = $ret;
                 $results->{$name} = $ret;
@@ -779,11 +790,11 @@ if (! $config_arg || ! -f $config_arg) {
                 $results->{$name}->{want_stderr} = $want_stderr;
                 $results->{$name}->{vpath_mode} = $vpath_mode;
                 $results->{$name}->{installdir} = "$install_dir";
-
             }
-            $ret = cleanup($name, $merge_output, $tarball_name,
+            close OUTFILE;
-                           $dir, "$install_dir",
+#            $ret = cleanup($name, $merge_output, $tarball_name,
-                           $vpath_mode, $config);
+#                           $dir, "$install_dir",
+#                           $vpath_mode, $config);
             ++$i;
         }
     }

contrib/nightly/ompi_cronjob.sh

@@ -0,0 +1,162 @@
+#!/bin/sh
+#
+# Please adjust the below parameters for Your setup.
+#
+#
+SRCDIR=$HOME/WORK/OPENMPI              # Where the script and the tar-balls reside
+LAST_VERSION=1.0a1r6211                # Should be detected by download
+TEST_VERSION=r360
+CONFIG_FILE=build-$HOSTNAME.txt
+#EMAIL="testing@open-mpi.org"
+EMAIL="keller@hlrs.de"
+
+# download "latest" filename
+latest_name="latest_snapshot.txt"
+
+
+#
+# This is required on some machines, that do not 
+# have autoconf-tools (cron passes only a short PATH)
+#
+export PATH=$HOME/bin:$PATH
+
+
+if [ "x$HOSTNAME" = "xstrider" ] ; then
+  export PATH=$PATH:/opt/pathscale/bin
+fi
+
+#######################################################
+#
+# Local functions
+#
+# find a program from a list and load it into the target variable
+find_program() {
+    var=$1
+    shift
+
+    # first zero out the target variable
+    str="$var="
+    eval $str
+
+    # loop through the list and save the first one that we find
+    am_done=
+    while test -z "$am_done" -a -n "$1" ; do
+        prog=$1
+        shift
+
+        if test -z "$prog" ; then
+            am_done=1
+        else
+            not_found="`which $prog 2>&1 | egrep '^no'`"
+            which $prog > /dev/null 2>&1
+            if test "$?" = "0" -a -z "$not_found" ; then
+                str="$var=$prog"
+                eval $str
+                am_done=1
+            fi
+        fi
+    done
+}
+
+
+dump () {
+  file=$1
+  shift
+  echo "$1" >> $file
+}
+
+############################################
+# Main part
+############################################
+
+if [ \! -d $SRCDIR ] ; then
+  echo "WARNING $SRCDIR does not exist"
+  exit
+fi
+
+cd $SRCDIR
+
+# figure out what download command to use
+find_program download wget lynx curl
+if test -z "$download"; then
+    echo "cannot find downloading program -- aborting in despair"
+    exit 1
+fi
+
+# get the latest snapshot version number
+# rm -f "$latest_name"
+# $download "$url_arg/$latest_name"
+# if test ! -f "$latest_name" ; then
+#     echo "Could not download latest snapshot number -- aborting"
+#     exit 1
+# fi
+# version=$LAST_VERSION
+version="`cat $latest_name`"
+test_version=$TEST_VERSION
+
+
+#
+# If the config-file does not exist, create one standard test file
+#
+if [ \! -r $CONFIG_FILE ] ; then
+  echo "CONFIG_FILE:$CONFIG_FILE does not exist, create default file"
+
+  CONFIG_FILE=build_sample_config.txt
+  rm -f $CONFIG_FILE
+  echo "gcc:1::--enable-picky" >> $CONFIG_FILE
+  # echo "default build:::" >> $CONFIG_FILE
+  # echo "static build:::--disable-shared --enable-static" >> $CONFIG_FILE
+  # echo "non-threaded build:::--without-threads"  >> $CONFIG_FILE
+fi
+
+
+#
+# Here comes the main part.
+#
+perl build_tarball.pl --email $EMAIL --config $CONFIG_FILE --file openmpi-$version.tar.bz2 --leave-install $SRCDIR/ompi-out-$version.txt --install-dir=$HOME/ompi-install --nocheck
+
+if [ \! -r ompi-out-$version.txt ] ; then
+  echo "No ompi-out-$version.txt file found; none of the configurations have been built?"
+  exit
+fi
+
+#
+# Now compile and install the tests for different combinations...
+#
+perl build_tests.pl --email $EMAIL --config $CONFIG_FILE --file openmpi-tests-$test_version.tar.bz2 --leave-install $SRCDIR/ompi-tests-out-$test_version.txt --outfile $SRCDIR/ompi-out-$version.txt --install-dir=$HOME/ompi-tests-install --nocheck
+
+f="ompi-script-$version"
+rm -fr $f
+dump $f "#!/bin/bash"
+dump $f "#PBS -l nodes=4,walltime=0:10:00" # Four nodes, and 10 minutes are enough
+dump $f "#PBS -k eo"                   # Combine stderr/stdout
+dump $f "#PBS -m ae"                   # Send mail on abort/end of job
+dump $f "#PBS -M $EMAIL"               # Mail-adresses to send to
+dump $f ""
+dump $f "OLD_PATH=\$PATH"
+dump $f "OLD_LD_LIBRARY_PATH=\$LD_LIBRARY_PATH"
+dump $f "rm -fr $HOME/ompi-tmp/openmpi-sessions-*"
+dump $f "killall -9 orted"
+
+for MPI_DIR in `cat $SRCDIR/ompi-out-$version.txt` ; do
+  dump $f "  echo Starting to test MPI_DIR:$MPI_DIR"
+  dump $f "  echo -------------------------------------------------"
+  dump $f "  export PATH=$MPI_DIR/bin:\$OLD_PATH"
+  dump $f "  export LD_LIBRARY_PATH=$MPI_DIR/lib:\$OLD_LD_LIBRARY_PATH"
+  dump $f "  rm -fr $HOME/ompi-tmp/openmpi-sessions-*"
+  # Start the orted deamon in a commonly accessible directory (/tmp is local to each node)
+  dump $f "  orted --tmpdir $HOME/ompi-tmp --seed --persistent --scope public"
+  for j in `cat $SRCDIR/ompi-tests-out-$test_version.txt` ; do
+    dump $f "    cd $j/bin"
+    dump $f "    mpirun -x LD_LIBRARY_PATH -hostfile \$PBS_NODEFILE -tmpdir $HOME/ompi-tmp -np 8 ./PMB-MPI1 PingPong PingPing Sendrecv Exchange Allreduce Reduce Allgather Allgatherv Alltoall Bcast Barrier"
+    dump $f "    ./tester -m -x LD_LIBRARY_PATH -hostfile \$PBS_NODEFILE -tmpdir $HOME/ompi-tmp -- mpich_all_tests"
+    dump $f "    cd -"
+    dump $f "    echo -------------------------------------------------"
+  done
+  dump $f "  killall -9 orted"
+  dump $f "  sleep 10"
+  dump $f "  echo -------------------------------------------------"
+done
+
+# qsub $SRCDIR/ompi-script-$version
+