1
1
openmpi/contrib/ompi-time.sh
Igor Ivanov 36d3a7aa6c contrib: Add bash script to measure performance
This script is useful to measure times from launching ompi
application to different internal points. A user can easy add
it`s test basing on existing tests.
See readme information inside the script.
2015-12-14 17:42:19 +02:00

1245 строки
32 KiB
Bash
Исполняемый файл

#!/bin/sh
#
# Copyright (c) 2015 Mellanox Technologies, Inc.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# This script is used to measure PMIx performance.
#
# --exec: scenario to run as sync or "test1 test2"
# --mpidir: path to mpi installation (/usr default)
# --parse: path to collected results
# HOWTO:
# 1 .Set test matrix using variables $node_list, ppn_list, test_list
# 2. Allocate nodes:
# $salloc --nodelist=node[1-4]
# or
# $salloc -N4
# 3. Launch script:
# $./opmi-time.sh --exec="test1 test2" --mpidir=<open>
# $./opmi-time.sh --mpidir=<open>
#
# Output location is test name folder
# Output file formats
# ()_base.log
# timestamp (usec) hostnode label
# 1441715028369350 mir14 start
# 1441715030540656 mir14 end
#
# ()_out.log
# timestamp (usec) rank node
# 1441715030460727 0 mir9
# 1441715030460628 1 mir10
#
# ()_result.log
# time rank node
# 2.089 3 mir12
# 2.093 2 mir11
#
# report.log
# nodes ppn mintime maxtime
# 4 1 2.089 2.093
# Settings
###############################################################################
node_list=(2 4)
ppn_list=(1 2)
test_list="test1 test2 test3 test4 test5 test6 test7 test8 test9 test10 test11 test12 test13"
# Declarations
###############################################################################
prefix=pmix
module=${BASH_SOURCE[0]}
# Command line parsing
###############################################################################
opt=""
while [ "$#" -gt 0 ]; do
case "$1" in
--parse=*) parse="${1#*=}"; shift 1;;
--exec=*) exec="${1#*=}"; shift 1;;
--mpidir=*) mpidir="${1#*=}"; shift 1;;
--parse|--exec|--mpidir) echo "$1 requires an argument" >&2; exit 1;;
-*) echo "unknown option: $1" >&2; exit 1;;
*) shift 1;;
esac
done
# The scenario of measurement
if [ -n "$exec" ]; then
test_list="$exec"
fi
# The mpi path
mpidir=${mpidir:=/usr}
# Functions
###############################################################################
# format text
function do_format() {
local is_format=true
if [[ $is_format == true ]] ; then
res=""
for ((i=2; i<=$#; i++)) ; do
case "${!i}" in
"bold" ) res="$res\e[1m" ;;
"underline" ) res="$res\e[4m" ;;
"reverse" ) res="$res\e[7m" ;;
"red" ) res="$res\e[91m" ;;
"green" ) res="$res\e[92m" ;;
"yellow" ) res="$res\e[93m" ;;
esac
done
echo -e "$res$1\e[0m"
else
echo "$1"
fi
}
# print message
function do_msg() {
echo -e "$*" 2>&1 | tee -a $logfile
}
# print error message and exit script
function do_err() {
echo -e $(do_format "$module failed. aborting. $*" "red" "bold") 2>&1 | tee -a $logfile
exit 1
}
# print the seconds and current microseconds.
function do_timestamp() {
do_msg "$(($(date +%s%N)/1000))\t$(hostname -s)" "$1"
}
# swap two files
function do_fswap() {
if (( $# == 2 )); then
mv "$1" /tmp/
mv "$2" "`dirname $1`"
mv "/tmp/`basename $1`" "`dirname $2`"
else
echo "Usage: swap <file1> <file2>"
return 1
fi
}
function do_cmd() {
cmd="$*"
do_msg "Doing:"
do_msg "=================================================="
do_msg "$*"
eval $cmd >> $logfile 2>&1
local status=$?
if test "$status" != "0"; then
echo "$module failed. Log:"
tail -20 $logfile
cat $logfile
exit $status
fi
do_msg "DONE"
do_msg ""
}
function do_export() {
do_msg "Exporting PATHs:"
do_msg "=================================================="
do_msg "$1"
export PATH="$1/bin:${PATH}"
export LD_LIBRARY_PATH="$1/lib:${LD_LIBRARY_PATH}"
export MANPATH="$1/share/man:${MANPATH}"
do_msg "DONE"
do_msg ""
}
function do_nodeinfo() {
do_msg "Node information:"
do_msg "=================================================="
do_msg $(hostname)
do_msg $(cat /etc/issue | grep We)
do_msg $(cat /proc/cpuinfo | grep 'model name' | sort -u | awk '{print $4, $5, $6, $7, $9}')
do_msg $(cat /proc/cpuinfo | grep proce | wc | awk '{print $1}')
do_msg $(uname -a | awk '{print $12}')
do_msg $(cat /proc/meminfo | grep [M,m]em)
do_msg $(uname -a | awk '{print $3}')
do_msg $(ibstat | grep -e "CA type" -e "Firmware version")
do_msg $(ibstatus | grep -e rate -e state | grep -v 'phys state')
do_msg $(ofed_info | head -6 | grep OFED)
do_msg "DONE"
do_msg ""
}
function do_validate() {
command -v mpiexec >/dev/null 2>&1 || { do_err "mpiexec is not found."; }
command -v srun >/dev/null 2>&1 || { do_err "srun is not found."; }
command -v salloc >/dev/null 2>&1 || { do_err "salloc is not found."; }
}
function do_check_pmix() {
eval "srun --mpi=list 2>&1 | grep pmix"
}
function do_checksync_mpisync() {
local status
local tooldir=${tempdir}/mpisync
local verbose=$1
local option=$*
do_msg "Checking synchronization using mpisync:"
if [ ! -e ${tooldir} ]; then
mkdir -p ${tooldir}
cd ${tooldir}
wget --no-check-certificate https://github.com/open-mpi/ompi/raw/master/ompi/tools/mpisync/mpigclock.c >> $logfile 2>&1
wget --no-check-certificate https://github.com/open-mpi/ompi/raw/master/ompi/tools/mpisync/mpigclock.h >> $logfile 2>&1
wget --no-check-certificate https://github.com/open-mpi/ompi/raw/master/ompi/tools/mpisync/hpctimer.c >> $logfile 2>&1
wget --no-check-certificate https://github.com/open-mpi/ompi/raw/master/ompi/tools/mpisync/hpctimer.h >> $logfile 2>&1
wget --no-check-certificate https://github.com/open-mpi/ompi/raw/master/ompi/tools/mpisync/sync.c >> $logfile 2>&1
mpicc hpctimer.c mpigclock.c sync.c -o mpisync >> $logfile 2>&1
fi
if [ ! -e "$tooldir" ] || [ ! -f "$tooldir/mpisync" ]; then
do_err "can not find $tooldir/mpisync"
fi
mpiexec -n $(($nodes)) -npernode 1 $mpioptions $tooldir/mpisync -o ${syncfile} ${option} 2>&1
do_msg "Analysing ${syncfile}"
cat ${syncfile} >> $logfile 2>&1
diff=$(grep -v '^#' ${syncfile} | cut -f3 -d' ' | sort -n | awk 'BEGIN {min=1000000; max=0;}; { if($1<min && $1 != "") min = $1; if($1>max && $1 != "") max = $1; } END { printf("%0.06f %0.06f %0.06f", min, max, max-min) }') >> $logfile 2>&1
do_msg "sync drift is equal: $diff"
diff=`echo $diff | cut -f3 -d' '`
status=$(if (( `bc <<< "$diff >= 0.001"` == 1 )); then echo "value $diff >= 0.001"; fi)
if [ -n "$status" ] && [ -n $verbose -a "$verbose" == "on" ]; then
do_err "mpisync reports issue with synchronization as $status"
else
do_msg "Warning: mpiperf reports issue with synchronization as $status"
fi
do_msg "DONE"
do_msg ""
}
function do_checksync_mpiperf() {
local status
local tooldir=${tempdir}/mpiperf-0.0.3
local verbose=$1
do_msg "Checking synchronization using mpiperf:"
if [ ! -f ${tempdir}/mpiperf-0.0.3.tar.gz ]; then
wget http://mpiperf.cpct.sibsutis.ru/uploads/Main/mpiperf-0.0.3.tar.gz >> $logfile 2>&1
tar zxvf mpiperf-0.0.3.tar.gz >> $logfile 2>&1
cd $tooldir
make >> $logfile 2>&1
fi
if [ ! -e "$tooldir" ] || [ ! -f "$tooldir/src/mpiperf" ]; then
do_err "can not find $tooldir/src/mpiperf"
fi
mpiexec -n 1 $mpioptions $tooldir/src/mpiperf -T >> $logfile 2>&1
if [ -z "$(mpiexec -n 1 $mpioptions $tooldir/src/mpiperf -j -t gettimeofday 2>&1 | tee -a $logfile | sed -n '/PASSED/p')" ]; then
do_err "mpiperf does not support gettimeofday"
fi
mpiexec -n $(($nodes)) -npernode 1 $mpioptions $tooldir/src/mpiperf -t gettimeofday WaitPatternNull >> ${syncfile} 2>&1
do_msg "Analysing ${syncfile}"
cat ${syncfile} >> $logfile 2>&1
status=$(grep -v '^#' ${syncfile} | awk -F ' ' '{ print $6 }' | while read i; do if (( `bc <<< "$i >= 1"` == 1 )); then echo "value $i >= 1.00"; break; fi; done)
if [ -n "$status" ] && [ -n $verbose -a "$verbose" == "on" ]; then
do_err "mpiperf reports issue with synchronization as $status"
else
do_msg "Warning: mpiperf reports issue with synchronization as $status"
fi
do_msg "DONE"
do_msg ""
}
# $1 - sync filename
# $2 - verbose mode: on - exit in case synchronization values exceed a treshold and off - silent mode (default: off)
# $3+ - application additional options
function do_checksync() {
if [ -z "$1" ]; then
syncfile=${tempdir}/mpisync.log
else
syncfile=$1
fi
do_checksync_mpisync $2 "-a 0"
# do_checksync_mpisync $2 "-a 1"
# do_checksync_mpiperf
do_msg "syncfile: $syncfile"
}
function do_analysis() {
local testdir=$1
local basefile=$2
local outfile=$3
local outfile1="${3}.1"
local resultfile=${testdir}/${nodes}x${ppn}_result.log
if [ ! -e $tesdir ]; then
do_err "can not find testdir: $testdir"
fi
if [ -z $basefile -o ! -f $basefile ]; then
do_err "can not find basefile: $basefile"
fi
if [ -z $outfile -o ! -f $outfile ]; then
do_err "can not find outfile: $outfile"
fi
if [ "$(cat $outfile | wc -l)" != "$(($nodes * $ppn))" ]; then
do_msg "Warning: number of lines in $outfile ($(cat $outfile | wc -l)) is not equal ($nodes * $ppn)."
fi
start_t=`awk -F $'\t' '{ if (NR == 1) print $1 }' $basefile`
# Add sync value in output file
while read line; do
if [[ ! $line =~ ^[0-9] ]]; then
do_msg "Warning: ignoring line: $line."
continue
fi
local n=$(echo $line | cut -f3 -d' ')
local v1=$(echo $line | cut -f1 -d' ')
local v2=0
if [ ! -z $syncfile -o -f $syncfile ]; then
v2=$(echo "scale=2; ($(grep $n $syncfile | cut -f3 -d' ') * 1000000)" | bc -l)
# Round float value to int
v2=$(echo ${v2%%.*})
v2=${v2:=0}
fi
echo -e "$(($v1 + $v2))\t${v2}\t${line}" >> $outfile1
done < $outfile
# Find maximum and minimum lines
min_line=`sort -n $outfile1 | head -n1`
max_line=`sort -n $outfile1 | tail -n1`
if [ -z "$min_line" -o -z "$max_line" ]; then
do_err "can not find max/min lines in : $outfile1"
fi
min_t=$( echo "$min_line" | cut -f1 -d$'\t')
max_t=$( echo "$max_line" | cut -f1 -d$'\t')
echo -e "`bc -l <<< "scale=3; (($min_t - $start_t) / 1000000)"`\t`echo "$min_line" | cut -f4 -d$'\t'`\t`echo "$min_line" | cut -f5 -d$'\t'`" >> $resultfile 2>&1
echo -e "`bc -l <<< "scale=3; (($max_t - $start_t) / 1000000)"`\t`echo "$max_line" | cut -f4 -d$'\t'`\t`echo "$max_line" | cut -f5 -d$'\t'`" >> $resultfile 2>&1
echo -e "\n# Used synchronization file: $syncfile" >> $outfile1
do_report $testdir $resultfile
}
function do_report() {
local testdir=$1
local resultfile=$2
local reportfile=${testdir}/report.log
if [ -z $resultfile -o ! -f $resultfile ]; then
do_err "can not find resultfile: $resultfile"
fi
min_t=`awk -F $'\t' '{ if (NR == 1) print $1 }' $resultfile`
max_t=`awk -F $'\t' '{ if (NR == 2) print $1 }' $resultfile`
echo -e "${nodes}\t${ppn}\t${min_t}\t${max_t}" >> $reportfile 2>&1
}
function do_postresult() {
cd $tempdir/..
tar -zcvf $PWD/pmix.$$.tar.gz $tempdir > /dev/null 2>&1
}
include_timestamp_func=$(cat <<END_MSG
#include <dlfcn.h>
#include <stddef.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <fcntl.h>
#include <unistd.h>
#include <stdint.h>
#include <sys/stat.h>
#include <time.h>
#include <sys/time.h>
static inline void timestamp(FILE *file)
{
struct timeval tv;
char name[256];
char *host_name = NULL;
char *domain = NULL;
int procid = -1;
char *str = NULL;
gettimeofday(&tv, NULL);
if (gethostname(name, sizeof(name)) != 0)
strcpy(name, "localhost");
host_name = strdup(name);
domain = strchr(host_name, '.');
if (domain)
*domain = '\0';
str = getenv("SLURM_PROCID");
procid = ( str ? strtol(str, NULL, 10) : -1);
fprintf(file, "%lld\t%d\t%s\n", tv.tv_sec * 1000000LL + tv.tv_usec, procid, host_name);
fflush(file);
}
END_MSG
)
function do_exec() {
# The number of nodes (see SLURM_NNODES)
nodes=${SLURM_NNODES}
nodes=${nodes:=2}
# The number of tasks per node (see SLURM_NTASKS_PER_NODE or SLURM_TASKS_PER_NODE)
ppn=${SLURM_NTASKS_PER_NODE}
ppn=${ppn:=1}
mpioptions=' -novm -mca btl_openib_warn_default_gid_prefix 0 -mca mpi_add_procs_cutoff 100000 '
slurmoptions=' OMPI_MCA_btl_openib_warn_default_gid_prefix=0 OMPI_MCA_mpi_add_procs_cutoff=100000 '
if [ -z "$(env | grep SLURM)" ]; then
do_err "Do not see allocated nodes by SLURM. Probably salloc -N option is not set"
fi
#if [ "${SLURM_NPROCS}" != "$(($nodes * $ppn))" ]; then
# do_err "SLURM_NPROCS=${SLURM_NPROCS} is not equal ($nodes * $ppn). Probably salloc -N option is not set"
#fi
do_msg ""
do_msg "Configuration:"
do_msg "=================================================="
do_msg "tempdir: $tempdir"
do_msg "logfile: $logfile"
do_msg "mpi: $mpidir"
do_msg "exec: $exec"
do_msg "nodes: $nodes"
do_msg "ppn: $ppn"
do_msg "mpioptions: $mpioptions"
do_msg "slurmoptions: $slurmoptions"
do_msg "node list: $node_list"
do_msg "ppn list: $ppn_list"
do_msg "test list: $test_list"
do_msg ""
do_export $mpidir
do_nodeinfo
do_validate
if [ -f "${tempdir}/mpisync.log" ]; then
syncfile=${tempdir}/mpisync.log
do_msg "found sync data at ${syncfile}"
elif [ -f "${tempdir}/mpiperf.log" ]; then
syncfile=${tempdir}/mpiperf.log
do_msg "found sync data at ${syncfile}"
else
do_msg "sync data is not found"
fi
# Launch scenario
node_list_len=${#node_list[*]}
ppn_list_len=${#ppn_list[*]}
for ((i=0; $i < $node_list_len; i=$((i=$i+1)))); do
for ((j=0; $j < $ppn_list_len; j=$((j=$j+1)))); do
for test in $test_list; do
nodes=${node_list[$i]}
ppn=${ppn_list[$j]}
if [ "$test" = "sync" ]; then
do_checksync "${tempdir}/${nodes}x${ppn}_mpisync00.log" "off"
else
do_checksync "${tempdir}/${nodes}x${ppn}_mpisync_before.log" "off"
eval "do_${test}"
do_checksync "${tempdir}/${nodes}x${ppn}_mpisyn_after.log" "off"
fi
done
done
done
do_postresult
}
# $1 - result location
function do_parse() {
local parsedir=$1
local result_list
local test_list
local parsefile
for result in `ls -1 $workdir`; do
if [ ! -d "${parsedir}/${result}" ]; then
continue
fi
for test in `ls -1 "${parsedir}/${result}" | grep -v mpisync`; do
if [ ! -d "${parsedir}/${result}/${test}" ]; then
continue
fi
result_list="${result_list} ${result}"
test_list="${test_list} ${test}"
done
done
result_list=`echo $result_list | tr " " "\n" | sort | uniq | tr "\n" " "`
test_list=`echo $test_list | tr " " "\n" | sort | uniq | tr "\n" " "`
do_msg "results: $result_list"
do_msg "tests: $test_list"
for test in $test_list; do
parsefile="${parsedir}/parse_${test}.log"
for result in $result_list; do
echo -e "\n${result}:" >> $parsefile 2>&1
echo -e "nodes\tppn\tmin\tmax" >> $parsefile 2>&1
cat "${parsedir}/${result}/${test}/report.log" >> $parsefile 2>&1
done
done
}
# Pure application srun launch
#####################################################
function do_test1
{
local status
local scenario=test1
local testdir=${tempdir}/$scenario
local outfile=${testdir}/${nodes}x${ppn}_out.log
local basefile=${testdir}/${nodes}x${ppn}_base.log
do_msg "Running $scenario ${nodes}x${ppn} :"
mkdir -p $testdir
cd $testdir
cat > $scenario.c <<END_MSG
$include_timestamp_func
int main()
{
timestamp(stdout);
return 0;
}
END_MSG
gcc $scenario.c -o $scenario.out >> $logfile 2>&1
# Do test
do_timestamp "start" 2>&1 | tee -a $basefile
srun -n$(($nodes * $ppn)) -N$nodes --ntasks-per-node=$ppn ./$scenario.out >> $outfile 2>&1
test $? -eq 0 && status=OK || status=FAIL
do_timestamp "end" 2>&1 | tee -a $basefile
if [ "$status" == "FAIL" ]; then
do_err "can not launch a test"
fi
echo -e "srun pure overhead" > ${testdir}/info.log 2>&1
do_analysis $testdir $basefile $outfile
do_msg "DONE"
}
# Pure application mpiexec launch
#####################################################
function do_test2
{
local status
local scenario=test2
local testdir=${tempdir}/$scenario
local outfile=${testdir}/${nodes}x${ppn}_out.log
local basefile=${testdir}/${nodes}x${ppn}_base.log
do_msg "Running $scenario ${nodes}x${ppn} :"
mkdir -p $testdir
cd $testdir
cat > $scenario.c <<END_MSG
$include_timestamp_func
int main()
{
timestamp(stdout);
return 0;
}
END_MSG
gcc $scenario.c -o $scenario.out >> $logfile 2>&1
# Do test
do_timestamp "start" 2>&1 | tee -a $basefile
mpiexec -n $(($nodes * $ppn)) -npernode $ppn $mpioptions ./$scenario.out >> $outfile 2>&1
test $? -eq 0 && status=OK || status=FAIL
do_timestamp "end" 2>&1 | tee -a $basefile
if [ "$status" == "FAIL" ]; then
do_err "can not launch a test"
fi
echo -e "mpiexec pure overhead" > ${testdir}/info.log 2>&1
do_analysis $testdir $basefile $outfile
do_msg "DONE"
}
# Pure application oshrun launch
#####################################################
function do_test3
{
local status
local scenario=test3
local testdir=${tempdir}/$scenario
local outfile=${testdir}/${nodes}x${ppn}_out.log
local basefile=${testdir}/${nodes}x${ppn}_base.log
do_msg "Running $scenario ${nodes}x${ppn} :"
mkdir -p $testdir
cd $testdir
cat > $scenario.c <<END_MSG
$include_timestamp_func
int main()
{
timestamp(stdout);
return 0;
}
END_MSG
gcc $scenario.c -o $scenario.out >> $logfile 2>&1
# Do test
do_timestamp "start" 2>&1 | tee -a $basefile
oshrun -n $(($nodes * $ppn)) -npernode $ppn $mpioptions ./$scenario.out >> $outfile 2>&1
test $? -eq 0 && status=OK || status=FAIL
do_timestamp "end" 2>&1 | tee -a $basefile
if [ "$status" == "FAIL" ]; then
do_err "can not launch a test"
fi
echo -e "osrun pure overhead" > ${testdir}/info.log 2>&1
do_analysis $testdir $basefile $outfile
do_msg "DONE"
}
# MPI_init application srun/pmi2 launch
#####################################################
function do_test4
{
local status
local scenario=test4
local testdir=${tempdir}/$scenario
local outfile=${testdir}/${nodes}x${ppn}_out.log
local basefile=${testdir}/${nodes}x${ppn}_base.log
do_msg "Running $scenario ${nodes}x${ppn} :"
mkdir -p $testdir
cd $testdir
cat > $scenario.c <<END_MSG
$include_timestamp_func
#include "mpi.h"
int main(int argc, char* argv[])
{
MPI_Init(&argc, &argv);
timestamp(stdout);
MPI_Finalize();
return 0;
}
END_MSG
mpicc $scenario.c -o $scenario.out >> $logfile 2>&1
# Do test
do_timestamp "start" 2>&1 | tee -a $basefile
env $slurmoptions srun -n$(($nodes * $ppn)) -N$nodes --ntasks-per-node=$ppn --mpi=pmi2 ./$scenario.out >> $outfile 2>&1
test $? -eq 0 && status=OK || status=FAIL
do_timestamp "end" 2>&1 | tee -a $basefile
if [ "$status" == "FAIL" ]; then
do_err "can not launch a test"
fi
echo -e "srun --mpi=pmi2:MPI_Init" > ${testdir}/info.log 2>&1
do_analysis $testdir $basefile $outfile
do_msg "DONE"
}
# start_pes application srun/pmi2 launch
#####################################################
function do_test5
{
local status
local scenario=test5
local testdir=${tempdir}/$scenario
local outfile=${testdir}/${nodes}x${ppn}_out.log
local basefile=${testdir}/${nodes}x${ppn}_base.log
do_msg "Running $scenario ${nodes}x${ppn} :"
mkdir -p $testdir
cd $testdir
cat > $scenario.c <<END_MSG
$include_timestamp_func
#include "shmem.h"
int main(int argc, char* argv[])
{
start_pes(0);
timestamp(stdout);
return 0;
}
END_MSG
oshcc $scenario.c -o $scenario.out >> $logfile 2>&1
# Do test
do_timestamp "start" 2>&1 | tee -a $basefile
env $slurmoptions srun -n$(($nodes * $ppn)) -N$nodes --ntasks-per-node=$ppn --mpi=pmi2 ./$scenario.out >> $outfile 2>&1
test $? -eq 0 && status=OK || status=FAIL
do_timestamp "end" 2>&1 | tee -a $basefile
if [ "$status" == "FAIL" ]; then
do_err "can not launch a test"
fi
echo -e "srun --mpi=pmi2:start_pes" > ${testdir}/info.log 2>&1
do_analysis $testdir $basefile $outfile
do_msg "DONE"
}
# MPI_Init application mpiexec launch
#####################################################
function do_test6
{
local status
local scenario=test6
local testdir=${tempdir}/$scenario
local outfile=${testdir}/${nodes}x${ppn}_out.log
local basefile=${testdir}/${nodes}x${ppn}_base.log
do_msg "Running $scenario ${nodes}x${ppn} :"
mkdir -p $testdir
cd $testdir
cat > $scenario.c <<END_MSG
$include_timestamp_func
#include "mpi.h"
int main(int argc, char* argv[])
{
MPI_Init(&argc, &argv);
timestamp(stdout);
MPI_Finalize();
return 0;
}
END_MSG
mpicc $scenario.c -o $scenario.out >> $logfile 2>&1
# Do test
do_timestamp "start" 2>&1 | tee -a $basefile
mpiexec -n $(($nodes * $ppn)) -npernode $ppn $mpioptions ./$scenario.out >> $outfile 2>&1
test $? -eq 0 && status=OK || status=FAIL
do_timestamp "end" 2>&1 | tee -a $basefile
if [ "$status" == "FAIL" ]; then
do_err "can not launch a test"
fi
echo -e "mpiexec:MPI_Init" > ${testdir}/info.log 2>&1
do_analysis $testdir $basefile $outfile
do_msg "DONE"
}
# start_pes application oshrun launch
#####################################################
function do_test7
{
local status
local scenario=test7
local testdir=${tempdir}/$scenario
local outfile=${testdir}/${nodes}x${ppn}_out.log
local basefile=${testdir}/${nodes}x${ppn}_base.log
do_msg "Running $scenario ${nodes}x${ppn} :"
mkdir -p $testdir
cd $testdir
cat > $scenario.c <<END_MSG
$include_timestamp_func
#include "shmem.h"
int main(int argc, char* argv[])
{
start_pes(0);
timestamp(stdout);
return 0;
}
END_MSG
oshcc $scenario.c -o $scenario.out >> $logfile 2>&1
# Do test
do_timestamp "start" 2>&1 | tee -a $basefile
oshrun -n $(($nodes * $ppn)) -npernode $ppn $mpioptions ./$scenario.out >> $outfile 2>&1
test $? -eq 0 && status=OK || status=FAIL
do_timestamp "end" 2>&1 | tee -a $basefile
if [ "$status" == "FAIL" ]; then
do_err "can not launch a test"
fi
echo -e "osrun:start_pes" > ${testdir}/info.log 2>&1
do_analysis $testdir $basefile $outfile
do_msg "DONE"
}
# pure application mpiexec:orte_daemon
#####################################################
function do_test8
{
local status
local scenario=test8
local testdir=${tempdir}/$scenario
local outfile=${testdir}/${nodes}x${ppn}_out.log
local basefile=${testdir}/${nodes}x${ppn}_base.log
do_msg "Running $scenario ${nodes}x${ppn} :"
mkdir -p $testdir
cd $testdir
cat > $scenario.c <<END_MSG
#include "mpi.h"
int main(int argc, char* argv[])
{
return 0;
}
END_MSG
cat > lib_$scenario.c <<END_MSG
#define _GNU_SOURCE
$include_timestamp_func
int orte_daemon(int argc, char *argv[])
{
static int (*_orte_daemon)(int argc, char *argv[]) = NULL;
if (!_orte_daemon) {
_orte_daemon=dlsym(RTLD_NEXT,"orte_daemon");
if (!_orte_daemon) {
fprintf(stderr, "Error in 'dlsym': %s\n", dlerror());
exit(1);
} else {
FILE *fd = NULL;
char filename[1024];
char *str = getenv("SLURM_PROCID");
if (str) {
sprintf(filename, "%s.%s", "$outfile", str);
fd = fopen(filename, "a");
if (fd) {
timestamp(fd);
fclose(fd);
}
}
}
}
return _orte_daemon(argc, argv);
}
END_MSG
mpicc $scenario.c -o $scenario.out >> $logfile 2>&1
gcc lib_$scenario.c -o $scenario.so -shared -fPIC -ldl >> $logfile 2>&1
# Do test
do_timestamp "start" 2>&1 | tee -a $basefile
LD_PRELOAD=$PWD/$scenario.so mpiexec -n $(($nodes * $ppn)) -npernode $ppn $mpioptions ./$scenario.out
test $? -eq 0 && status=OK || status=FAIL
do_timestamp "end" 2>&1 | tee -a $basefile
if [ "$status" == "FAIL" ]; then
do_err "can not launch a test"
fi
eval cat $outfile.* >> $outfile
rm $outfile.*
echo -e "mpiexec:orte_daemon" > ${testdir}/info.log 2>&1
do_analysis $testdir $basefile $outfile
do_msg "DONE"
}
# pure application oshrun:orte_daemon
#####################################################
function do_test9
{
local status
local scenario=test9
local testdir=${tempdir}/$scenario
local outfile=${testdir}/${nodes}x${ppn}_out.log
local basefile=${testdir}/${nodes}x${ppn}_base.log
do_msg "Running $scenario ${nodes}x${ppn} :"
mkdir -p $testdir
cd $testdir
cat > $scenario.c <<END_MSG
#include "mpi.h"
int main(int argc, char* argv[])
{
return 0;
}
END_MSG
cat > lib_$scenario.c <<END_MSG
#define _GNU_SOURCE
$include_timestamp_func
int orte_daemon(int argc, char *argv[])
{
static int (*_orte_daemon)(int argc, char *argv[]) = NULL;
if (!_orte_daemon) {
_orte_daemon=dlsym(RTLD_NEXT,"orte_daemon");
if (!_orte_daemon) {
fprintf(stderr, "Error in 'dlsym': %s\n", dlerror());
exit(1);
} else {
FILE *fd = NULL;
char filename[1024];
char *str = getenv("SLURM_PROCID");
if (str) {
sprintf(filename, "%s.%s", "$outfile", str);
fd = fopen(filename, "a");
if (fd) {
timestamp(fd);
fclose(fd);
}
}
}
}
return _orte_daemon(argc, argv);
}
END_MSG
mpicc $scenario.c -o $scenario.out >> $logfile 2>&1
gcc lib_$scenario.c -o $scenario.so -shared -fPIC -ldl >> $logfile 2>&1
# Do test
do_timestamp "start" 2>&1 | tee -a $basefile
LD_PRELOAD=$PWD/$scenario.so oshrun -n $(($nodes * $ppn)) -npernode $ppn $mpioptions ./$scenario.out
test $? -eq 0 && status=OK || status=FAIL
do_timestamp "end" 2>&1 | tee -a $basefile
if [ "$status" == "FAIL" ]; then
do_err "can not launch a test"
fi
eval cat $outfile.* >> $outfile
rm $outfile.*
echo -e "oshrun:orte_daemon" > ${testdir}/info.log 2>&1
do_analysis $testdir $basefile $outfile
do_msg "DONE"
}
# pure application mpiexec:orte_rml_base_update_contact_info
#####################################################
function do_test10
{
local status
local scenario=test10
local testdir=${tempdir}/$scenario
local outfile=${testdir}/${nodes}x${ppn}_out.log
local basefile=${testdir}/${nodes}x${ppn}_base.log
do_msg "Running $scenario ${nodes}x${ppn} :"
mkdir -p $testdir
cd $testdir
cat > $scenario.c <<END_MSG
#include "mpi.h"
int main(int argc, char* argv[])
{
return 0;
}
END_MSG
cat > lib_$scenario.c <<END_MSG
#define _GNU_SOURCE
$include_timestamp_func
int orte_rml_base_update_contact_info(void * data)
{
static int (*_real_func)(void* data) = NULL;
if (!_real_func) {
_real_func=dlsym(RTLD_NEXT,"orte_rml_base_update_contact_info");
if (!_real_func) {
fprintf(stderr, "Error in 'dlsym': %s\n", dlerror());
exit(1);
} else {
FILE *fd = NULL;
char filename[1024];
char *str = getenv("SLURM_PROCID");
if (str) {
sprintf(filename, "%s.%s", "$outfile", str);
fd = fopen(filename, "a");
if (fd) {
timestamp(fd);
fclose(fd);
}
}
}
}
return _real_func(data);
}
END_MSG
mpicc $scenario.c -o $scenario.out >> $logfile 2>&1
mpicc lib_$scenario.c -o $scenario.so -shared -fPIC -ldl >> $logfile 2>&1
# Do test
do_timestamp "start" 2>&1 | tee -a $basefile
LD_PRELOAD=$PWD/$scenario.so mpiexec -n $(($nodes * $ppn)) -npernode $ppn $mpioptions ./$scenario.out
test $? -eq 0 && status=OK || status=FAIL
do_timestamp "end" 2>&1 | tee -a $basefile
if [ "$status" == "FAIL" ]; then
do_err "can not launch a test"
fi
eval "cat $outfile.* >> $outfile" >> $logfile 2>&1
rm $outfile.* >> $logfile 2>&1
echo -e "mpiexec:orte_rml_base_update_contact_info" > ${testdir}/info.log 2>&1
do_analysis $testdir $basefile $outfile
do_msg "DONE"
}
# pure application oshrun:orte_rml_base_update_contact_info
#####################################################
function do_test11
{
local status
local scenario=test11
local testdir=${tempdir}/$scenario
local outfile=${testdir}/${nodes}x${ppn}_out.log
local basefile=${testdir}/${nodes}x${ppn}_base.log
do_msg "Running $scenario ${nodes}x${ppn} :"
mkdir -p $testdir
cd $testdir
cat > $scenario.c <<END_MSG
#include "shmem.h"
int main(int argc, char* argv[])
{
return 0;
}
END_MSG
cat > lib_$scenario.c <<END_MSG
#define _GNU_SOURCE
$include_timestamp_func
int orte_rml_base_update_contact_info(void * data)
{
static int (*_real_func)(void* data) = NULL;
if (!_real_func) {
_real_func=dlsym(RTLD_NEXT,"orte_rml_base_update_contact_info");
if (!_real_func) {
fprintf(stderr, "Error in 'dlsym': %s\n", dlerror());
exit(1);
} else {
FILE *fd = NULL;
char filename[1024];
char *str = getenv("SLURM_PROCID");
if (str) {
sprintf(filename, "%s.%s", "$outfile", str);
fd = fopen(filename, "a");
if (fd) {
timestamp(fd);
fclose(fd);
}
}
}
}
return _real_func(data);
}
END_MSG
oshcc $scenario.c -o $scenario.out >> $logfile 2>&1
oshcc lib_$scenario.c -o $scenario.so -shared -fPIC -ldl >> $logfile 2>&1
# Do test
do_timestamp "start" 2>&1 | tee -a $basefile
LD_PRELOAD=$PWD/$scenario.so oshrun -n $(($nodes * $ppn)) -npernode $ppn $mpioptions ./$scenario.out
test $? -eq 0 && status=OK || status=FAIL
do_timestamp "end" 2>&1 | tee -a $basefile
if [ "$status" == "FAIL" ]; then
do_err "can not launch a test"
fi
eval "cat $outfile.* >> $outfile" >> $logfile 2>&1
rm $outfile.* >> $logfile 2>&1
echo -e "oshrun:orte_rml_base_update_contact_info" > ${testdir}/info.log 2>&1
do_analysis $testdir $basefile $outfile
do_msg "DONE"
}
# MPI_Init application mpiexec:srun/pmix
#####################################################
function do_test12
{
local status
local scenario=test12
local testdir=${tempdir}/$scenario
local outfile=${testdir}/${nodes}x${ppn}_out.log
local basefile=${testdir}/${nodes}x${ppn}_base.log
do_msg "Running $scenario ${nodes}x${ppn} :"
do_check_pmix
if [ $? -eq 0 ]; then
do_msg "slurm has pmix plugin"
else
do_msg "skipping this test : slurm does not have pmix plugin"
return 1
fi
mkdir -p $testdir
cd $testdir
cat > $scenario.c <<END_MSG
$include_timestamp_func
#include "mpi.h"
int main(int argc, char* argv[])
{
MPI_Init(&argc, &argv);
timestamp(stdout);
MPI_Finalize();
return 0;
}
END_MSG
mpicc $scenario.c -o $scenario.out >> $logfile 2>&1
# Do test
do_timestamp "start" 2>&1 | tee -a $basefile
env $slurmoptions srun -n$(($nodes * $ppn)) -N$nodes --ntasks-per-node=$ppn --mpi=pmix ./$scenario.out >> $outfile 2>&1
test $? -eq 0 && status=OK || status=FAIL
do_timestamp "end" 2>&1 | tee -a $basefile
if [ "$status" == "FAIL" ]; then
do_err "can not launch a test"
fi
echo -e "srun --mpi=pmix:MPI_Init" > ${testdir}/info.log 2>&1
do_analysis $testdir $basefile $outfile
do_msg "DONE"
}
# start_pes application oshrun:srun/pmix
#####################################################
function do_test13
{
local status
local scenario=test13
local testdir=${tempdir}/$scenario
local outfile=${testdir}/${nodes}x${ppn}_out.log
local basefile=${testdir}/${nodes}x${ppn}_base.log
do_msg "Running $scenario ${nodes}x${ppn} :"
do_check_pmix
if [ $? -eq 0 ]; then
do_msg "slurm has pmix plugin"
else
do_msg "skipping this test : slurm does not have pmix plugin"
return 1
fi
mkdir -p $testdir
cd $testdir
cat > $scenario.c <<END_MSG
$include_timestamp_func
#include "shmem.h"
int main(int argc, char* argv[])
{
start_pes(0);
timestamp(stdout);
return 0;
}
END_MSG
oshcc $scenario.c -o $scenario.out >> $logfile 2>&1
# Do test
do_timestamp "start" 2>&1 | tee -a $basefile
env $slurmoptions srun -n$(($nodes * $ppn)) -N$nodes --ntasks-per-node=$ppn --mpi=pmix ./$scenario.out >> $outfile 2>&1
test $? -eq 0 && status=OK || status=FAIL
do_timestamp "end" 2>&1 | tee -a $basefile
if [ "$status" == "FAIL" ]; then
do_err "can not launch a test"
fi
echo -e "srun --mpi=pmix:start_pes" > ${testdir}/info.log 2>&1
do_analysis $testdir $basefile $outfile
do_msg "DONE"
}
# Main
###############################################################################
# Check if --exec option is passed ($exec is defined)
if test ${exec+defined}; then
tempdir=$PWD/tmp/${prefix}.$$
logfile=${tempdir}/${prefix}-time.log
mkdir -p $tempdir
rm -f $logfile
cd $tempdir
do_exec
fi
# Check if --parse option is passed ($parse is defined)
if test ${parse+defined}; then
if [ -z "$parse" ]; then
tempdir=$PWD/tmp
else
tempdir=$parse
fi
logfile=${tempdir}/${prefix}-parse.log
mkdir -p $tempdir
rm -f $logfile
cd $tempdir
do_parse "$tempdir"
fi
exit 0