1
1

- Convert shell script to perl ;-) Only (XML) svn log, instead of

one PER eligible revision speeds this up 24x...
   Now should be worthwhile to run for the v1.4 branch as well.

   As before creates HTML output by default:
     perl ompi_branch_check_revisions.pl > revisions-v1.5.html

   By default now does the parsing of the branch's svn log -- should
   then work hand in hand with Jeff's gkcommit.pl

This commit was SVN r22917.
Этот коммит содержится в:
Rainer Keller 2010-03-31 03:00:58 +00:00
родитель 62f8d3c471
Коммит 66ce72201f
2 изменённых файлов: 420 добавлений и 198 удалений

420
contrib/ompi_branch_check_revisions.pl Исполняемый файл
Просмотреть файл

@ -0,0 +1,420 @@
#!/usr/bin/env perl
#
# Copyright (c) 2010 Oak Ridge National Labs. All rights reserved.
#
# Script to check which revisions on Trunk have not yet been merged into
# a branch.
#
use strict;
### use warnings;
use Getopt::Long;
use XML::Parser;
use SVN::Client;
use Cwd;
use LWP;
use File::Temp qw/ :POSIX /;
my $base_trac_url = "https://svn.open-mpi.org/trac/ompi/";
my $base_svn_url = "https://svn.open-mpi.org/svn/ompi/";
###########################################################################
# Command line parsing
my $branch_arg = "v1.5";
my $url_arg;
my $output_arg = "html";
my $html=1;
my $width_arg = 90;
my $start_rev_arg = 0;
my $notes_file_arg = 0;
my $debug_arg = 0;
my $help_arg = 0;
my $trunk_url;
my $branch_url;
my $svn_ctx_trunk;
my $svn_ctx_branch;
my $notes_file;
my $notes_file_fh;
my $branch_rev;
my $pool = SVN::Pool->new_default;
my $cmd;
my $cmd_output;
my $xml;
my %logentries;
my %logentries_branch;
my %logentries_trunk;
my $logentry;
my $chars;
my $i;
my $x;
my @revs;
my @revs_svn;
my @revs_eligible;
my @revs_merged;
my $rev;
my $line;
my %count;
&Getopt::Long::Configure("bundling");
my $ok = Getopt::Long::GetOptions("branch|b=s" => \$branch_arg,
"url|u=s" => \$url_arg,
"output|o=s" => \$output_arg,
"width|w=s" => \$width_arg,
"start-rev|s=s" => \$start_rev_arg,
"notes-file|n=s" => \$notes_file_arg,
"debug|d!" => \$debug_arg,
"help|h!" => \$help_arg);
if (!$ok || $help_arg) {
print "
Usage: $0 [--help|-h] [--branch|-b=BRANCH] [--url|-u=URL]
[--output|-o=OUTPUT TYPE] [--width|-w=WIDTH] [--start-rev|-s=REV]
[--merge-file|-m=FILE] [--revision-file|-r=FILE]
Outputs a HTML or a comma-separated file, listing outstanding r-numbers.
May use an input file per branch, describing notes such as pending CMRs,
dependencies or reasons, why a revision should not be moved.
-b Branch version, which is appended to default URL [default:$branch_arg]
-u URL to compare to [default: https://svn..../ompi/branches/$branch_arg]
-o Output Type, possible: txt, html [default:$output_arg]
-w Width of commit msg. that will be cut at [default:$width_arg]
-s Start with revision number [default:Creation of branch]
-n Notes file, comma separated revision, notes [FUTURE: default:svn update of $0-$branch_arg.txt]
-d Debug output
-h This help
When parsing the revision file notes (see option -n), the following marks will be done
MAILED Author has been made aware of filing a CMR
CMR:BRANCH[:NUM] There exists a CMR (with link to TRACS)
MERGED The Revision will not show up in the output
The markings are in increasing order, i.e. if MERGED is part of notes the line is not shown.
\n";
exit(0);
}
###########################################################################
# Make sure we're in an SVN tree
die "Not in a SVN tree"
if (! -d ".svn");
$trunk_url = $base_svn_url . "/trunk";
$svn_ctx_trunk = new SVN::Client( url => $trunk_url );
die "SVN URL $trunk_url is wrong"
if (! $svn_ctx_trunk );
if ($url_arg) {
$branch_url = $url_arg;
} else {
$branch_url = $base_svn_url . "/branches/" . $branch_arg;
}
$svn_ctx_branch = new SVN::Client( url => $branch_url );
die "SVN URL $branch_url is wrong, specify with [--url|-u=URL]"
if (! $svn_ctx_branch );
print "branch_url: $branch_url\n"
if ($debug_arg);
# XXX For now (as long as ompi_branch_check_revisions-v1.5 is not on the branch, hard-code the file:
$notes_file = "ompi_branch_check_revisions-v1.5.txt";
# if ($notes_file_arg) {
# $notes_file = $notes_file_arg;
# } else {
# $notes_file = $branch_url . "/contrib/ompi_branch_check_revisions-" . $branch_arg . ".txt";
# $branch_rev = $svn_ctx_branch->update ( $notes_file, "HEAD", 0, $pool );
# }
open $notes_file_fh, '<', $notes_file or die "Cannot not open notes-file $notes_file [--notes-file|-n]";
chomp ($output_arg);
if (lc($output_arg) eq "text") {
$html = 0;
} elsif (lc($output_arg) eq "html") {
$html = 1;
} else {
die "Output type wrong [--output|-o=OUTPUT TYPE]"
}
###########################################################################
# First grep the eligible revisions
$cmd = "svn mergeinfo --show-revs eligible $trunk_url $branch_url";
print "Running: $cmd\n"
if ($debug_arg);
open (CMD, "$cmd|");
$cmd_output .= $_
while (<CMD>);
close(CMD);
@revs = split('\n', $cmd_output);
foreach $rev (@revs) {
$rev =~ s/^r?([0-9]+)/$1/;
push @revs_svn, $rev;
}
if ($debug_arg) {
print "Number of svn-eligible revs: $#revs_svn\n";
$i = 0;
foreach $rev (@revs_svn) {
print "i:$i rev:$rev\n";
$i += 1;
}
}
###########################################################################
# Then check, whether these revisions are already merged (parse log messages of branch!)
$cmd = "svn log --stop-on-copy --xml $branch_url";
open (CMD, "$cmd|");
$xml .= $_
while (<CMD>);
close(CMD);
# Somehow, I cannot set another Start/End-handler, that would assign
# to my specific %hash (e.g. one end-handler for %logentries_branch another one for trunk)
# So just copy the hashes after the parse.
$x = new XML::Parser ( Style => 'Subs',
Handlers => {
Char => \&my_char_handler
});
%logentries = ();
$x->parse ($xml);
%logentries_branch = %logentries;
# Search all revisions stored in the hash and grok their msg for From patterns.
# Delete those revisions form the above eligible revisions.
foreach $rev (keys %logentries_branch) {
# Scan each line in the commit msg.
foreach $line (split('\n', $logentries_branch{$rev}->{msg})) {
# Lines matching a specific merge "> From rXXXX:" or "(Import rXXX)"?
if ($line =~ /^\>? *From *r?([0-9]+):?/) {
$line =~ s/^\>? *From *r?([0-9]+):?/$1/g;
push @revs_merged, $line;
}
}
}
if ($debug_arg) {
print "Number of parsed merged revs: $#revs_merged\n";
$i = 0;
foreach $rev (@revs_merged) {
print "i:$i parsed merged rev:$rev\n";
$i += 1;
}
}
# Count each revision from BOTH arrays (the svn-eligible and already merged) into a hash
foreach $rev (@revs_svn, @revs_merged) { $count{$rev}++ }
# Copy all revisions from the svn-eligible, whose count is only ONE
foreach $rev (@revs_svn) {
push @revs_eligible, $rev
if ($count{$rev} == 1);
}
if ($debug_arg) {
print "Number of eligible revs: $#revs_eligible\n";
$i = 0;
foreach $rev (@revs_eligible) {
print "i:$i eligible rev:$rev\n";
$i += 1;
}
}
###########################################################################
# svn log all the required revisions -- we collect ALL -- it is quicker and
# actually saver, as we don't fill the cmd-line up.
$cmd = "svn log --xml --revision $revs_eligible[0]:HEAD $base_svn_url ";
$xml = ();
open (CMD, "$cmd|");
$xml .= $_
while (<CMD>);
close(CMD);
$x = new XML::Parser ( Style => 'Subs',
Handlers => {
Char => \&my_char_handler
});
%logentries = ();
$x->parse ($xml);
%logentries_trunk = %logentries;
###########################################################################
# Read in the notes_file and put the notes into the HASH!!!
while (<$notes_file_fh>) {
chomp;
my ($rev, $note) = split /,/;
if ($rev =~ /^#/) {
next;
}
$rev =~ s/^r?([0-9]+)/$1/;
# Replace the #CMR with a proper link.
if ($html) {
$note =~ s%#([0-9]+)%\<A HREF=\"$base_trac_url/ticket/$1\"\>#$1\<\/A\>%g;
}
print "rev:$rev After CMR replacement: note:$note\n"
if ($debug_arg);
$logentries_trunk{$rev}->{note} = $note;
}
close $notes_file_fh;
###########################################################################
# Output the HTML (or text)
if ($html) {
my $sec; my $min; my $hour; my $mday; my $mon; my $year; my $wday; my $yday; my $isdst;
($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst)=localtime(time);
my $string = sprintf("Revisions eligible to merge into branch $branch_arg as of %4d-%02d-%02d %02d:%02d:%02d",
$year+1900, $mon+1, $mday, $hour, $min, $sec);
print "<HTML><HEAD><TITLE>$string</TITLE></HEAD>\n";
print "<BODY><H2>$string</H2>\n";
print "<TABLE BORDER=1 BGCOLOR=#FFFFFF><TR BGCOLOR=#CCCCCC><TH>Revision</TH><TH>Author</TH><TH>Commit Msg.</TH><TH>Notes</TH></TR>\n";
}
# Now loop over the available revisions and output
$i = 0;
foreach $rev (@revs_eligible) {
my @lines = split('\n', $logentries_trunk{$rev}->{msg});
my $bgcolor;
my $msg = substr ($lines[0], 0, $width_arg);
$bgcolor = ();
if ($logentries_trunk{$rev}->{note}) {
my $note = lc($logentries_trunk{$rev}->{note});
### print "Note:$note";
# Set color accordingly, possibly sed the TRAC-CMR number by a URL to trac
# Mark MAILED as Yellow
if (-1 != index $note, "mailed") {
$bgcolor="#FFFF00";
}
# Mark FIX as Red
elsif (-1 != index $note, "fix") {
$bgcolor="#FF0000";
}
# Mark CMR as green -- but only if for this branch
elsif (-1 != index $note, "cmr:$branch_arg") {
$bgcolor="#00FF00";
}
# Continue upon MERGED!!! This means this was not detected!
# (investigate why svn merge / svn mergeinfo does not work)
elsif (-1 != index $note, "merged") {
print "\nWARNING: rev:$rev is marked as merged but shows up in `svn mergeinfo`, please check note (for discard) or whether this script is in error.\n\n"
if ($debug_arg);
next;
}
}
# If color has not been set, yet...
if (! $bgcolor) {
if ($i % 2 == 0) {
$bgcolor="#EEEEEE";
} else {
$bgcolor="#FFFFFF";
}
}
if ($html) {
print "<TR BGCOLOR=$bgcolor><TD><A HREF=\"$base_trac_url/changeset/$rev\">r$rev</A></TD><TD>$logentries_trunk{$rev}->{author}</TD><TD>$msg</TD><TD>$logentries_trunk{$rev}->{note}</TD></TR>\n";
} else {
printf "%d. REV:%6d AUTHOR:%10s MSG:%s",
$i, $rev, $logentries_trunk{$rev}->{author}, $msg;
if ($logentries_trunk{$rev}->{note}) {
printf " NOTE:%s\n",
$logentries_trunk{$rev}->{note};
} else {
printf "\n";
}
}
$i += 1;
}
if ($html) {
print "</TABLE></BODY></HTML>\n";
}
###########################################################################
# All the Handlers functions
###########################################################################
# Called for the first logentry tag in the XML parsing
sub logentry {
# The beginning logentry tag has arguments of the form:
# ($expat, 'logentry', attr1, val1, attr2, val2, ...);
# print "CALL logentry\n";
shift(@_);
shift(@_);
while (@_) {
my $attr = shift(@_);
my $val = shift(@_);
$logentry->{$attr} = $val;
}
}
# Called for the last logentry tag in the XML parsing
sub logentry_ {
# print "CALL logentry_\n";
# print "logentry:$logentry->{revision}\n";
%logentries->{$logentry->{revision}} = $logentry;
$logentry = undef;
}
# Called for the last anchor tag in the XML parsing
sub author_ {
# print "CALL author_\n";
chomp($chars);
$chars =~ s/^\n*//;
$logentry->{author} = $chars;
$chars = '';
}
# Called for the last date tag in the XML parsing
sub date_ {
# print "CALL date_\n";
chomp($chars);
$chars =~ s/^\n*//;
$logentry->{date} = $chars;
$chars = '';
}
# Called for the last revision tag in the XML parsing
sub revision_ {
# print "CALL revision_\n";
chomp($chars);
$chars =~ s/^\n*//;
$logentry->{revision} = $chars;
$chars = '';
}
# Called for the last msg tag in the XML parsing
sub msg_ {
# print "CALL msg_\n";
chomp($chars);
$chars =~ s/^\n*//;
$logentry->{msg} = $chars;
$chars = '';
}
# Called for general character data in XML parsing
sub my_char_handler {
my ($expat, $tmp) = @_;
$chars .= $tmp;
}

Просмотреть файл

@ -1,198 +0,0 @@
#!/bin/sh
#
# Some initialization
#
WANT_HTML=1
BRANCH_VERSION="v1.5"
TRAC_URL="https://svn.open-mpi.org/trac/ompi"
TRUNK_URL="https://svn.open-mpi.org/svn/ompi/trunk/"
RELATIVE_URL="https://svn.open-mpi.org/svn/ompi/branches/$BRANCH_VERSION"
# XXX might be really tmp by spec $$
TMP_REVS="$TMPDIR/ompi_revs.txt"
REVS_NOTES="ompi_branch_check_revisions-$BRANCH_VERSION.txt"
MERGE_LOG="$TMPDIR/ompi_merge_log.txt"
MERGE_FILE="$TMPDIR/ompi_merge.txt"
MSG_WIDTH=90
PARSE_REVS=0
function usage()
{
echo "Outputs a HTML or a comma-separated file, listing outstanding r-numbers."
echo "May use an input file per branch, describing notes such as pending CMRs,"
echo "dependencies or reasons, why a revision should not be moved."
echo ""
echo "Usage: $0 [-h] [-b Branch] [-u URL] [-o Output Type]"
echo ""
echo "-h This help"
echo "-b Branch version, which is appended to default URL [default:v1.5]"
echo "-u URL to compare to [default: https://svn..../ompi/branches/BRANCH_VERSION]"
echo "-o Output Type, possible: txt, html [default:html]"
echo "-w Width of commit msg. that will be cut at [default:90]"
echo "-m Merge file [default: extract from svn log of BRANCH_VERSION]"
echo "-r Revision file with notes [default:svn update of `basename $0 .sh`-BRANCH_VERSION.txt]"
echo "-p Parse revision file notes (see notes below) [default:off]"
echo ""
echo "When parsing the revision file notes (see option -r), the following marks will be done"
echo " MAILED Author has been made aware of filing a CMR"
echo " CMR:BRANCH[:NUM] There exists a CMR (with link to TRACS)"
echo " MERGED The Revision will not show up in the output"
echo "The markings are in increasing order, i.e. if MERGED is part of notes the line is not shown."
exit 1
}
function check_svn_url()
{
URL=$1
svn list $URL > /dev/null 2>&1
if test $? != 0 ; then
echo "ERROR: The provided URL:$URL is not correct or cannot svn list"
exit 1
fi
}
#
# Check the options
#
while getopts "b:o:u:w:m:r:ph" opt; do
case $opt in
b)
BRANCH_VERSION=$OPTARG
RELATIVE_URL="https://svn.open-mpi.org/svn/ompi/branches/$BRANCH_VERSION"
;;
o)
if test "$OPTARG" = "html" ; then
WANT_HTML=1
elif test "$OPTARG" = "txt" ; then
WANT_HTML=0
else
usage
fi
;;
u)
RELATIVE_URL=$OPTARG
;;
w)
MSG_WIDTH=$OPTARG
;;
m)
MERGE_FILE=$OPTARG
;;
r)
REVS_NOTES=$OPTARG
;;
p)
PARSE_REVS=1
;;
h)
usage
;;
:)
echo "Option -$OPTARG requires an argument."
usage
;;
\?)
echo "Invalid option: -$OPTARG"
usage
;;
esac
done
# Check the URLs
check_svn_url $TRUNK_URL
check_svn_url $RELATIVE_URL
svn mergeinfo --show-revs eligible $TRUNK_URL $RELATIVE_URL > $TMP_REVS
if test \! -f $TMP_REVS ; then
echo "ERROR: TMP_REVS:$TMP_REVS file is not available"
exit 1
fi
svn update $TRUNK_URL/contrib/$REVS_NOTES > /dev/null 2>&1
# If there is no notes file (nothing checked out, nothing local) reset NOTES
if test \! -f $REVS_NOTES ; then
unset REVS_NOTES
NOTES=""
fi
# The ALREADY merged (but by means of "svn mergeinfo" not recognised)
# revisions can be "savely" parsed:
if test $PARSE_REVS = 1 ; then
svn log --incremental --stop-on-copy $RELATIVE_URL > $MERGE_LOG
grep '^From r' $MERGE_LOG | sed -e 's/From \(r[0-9]*\):/\1/' > $MERGE_FILE
fi
DATE=`date +%Y.%m.%d`
if test "x$WANT_HTML" = "x1" ; then
echo "<HTML><HEAD><TITLE>Revisions eligible to merge into $BRANCH_VERSION as of $DATE</TITLE></HEAD>"
echo "<TABLE BORDER=1 BGCOLOR=#FFFFFF><TR BGCOLOR=#CCCCCC><TH>Revision</TH><TH>Author</TH><TH>Commit Msg.</TH><TH>Notes</TH></TR>"
else
echo "# Revisions eligible to merge from trunk into $BRANCH_VERSION as of $DATE"
echo "# Revision, Author, Commit Msg, Notes"
fi
OLD_IFS=$IFS
IFS='^'
LINE_ODD=0;
while read REV ; do
LOG=`svn log -r$REV $TRUNK_URL`
REV_NUMBER=${REV##*r}
AUTHOR=`echo $LOG | head -n2 | tail -n1 | cut -f2 -d'|'`
MSG=`echo $LOG | head -n4 | tail -n1 | cut -c-$MSG_WIDTH`
if test "x$REVS_NOTES" != "x" ; then
NOTES=`grep "^$REV" $REVS_NOTES | cut -f2 -d','`
fi
unset BGCOLOR
if test $PARSE_REVS = 1 ; then
# Mark MAILED as Yellow
echo $NOTES | grep -iq "MAILED" && BGCOLOR=#FFFF00
# Mark FIX as Red
echo $NOTES | grep -iq "FIX" && BGCOLOR=#FF0000
# Mark CMR as green -- but only if for this branch
echo $NOTES | grep -iq "CMR:$BRANCH_VERSION" && BGCOLOR=#00FF00 && NOTES=`echo $NOTES | sed -e "s%#\([0-9]*\)%\<A HREF=\"$TRAC_URL/ticket/\1\"\>\0\<\/A\>%"`
# Skip MERGED (investigate why svn merge / svn mergeinfo does not work)
echo $NOTES | grep -iq "MERGED" && continue
# At last, if we found the revision in the MERGE_FILE, skip as well
grep -q $REV $MERGE_FILE && continue
fi
if test "x$WANT_HTML" = "x1" ; then
# Only overwrite the color if it has not yet been set
if test "x$BGCOLOR" = "x" ; then
if test $LINE_ODD = 1 ; then
BGCOLOR=#EEEEEEE
else
BGCOLOR=#FFFFFFF
fi
fi
echo "<TR BGCOLOR="$BGCOLOR"><TD><A HREF=\""$TRAC_URL/changeset/$REV_NUMBER"\">$REV</A></TD><TD>$AUTHOR</TD><TD>$MSG</TD><TD>$NOTES</TD></TR>"
else
echo "$REV, $AUTHOR, \"$MSG\", $NOTES"
fi
if test $LINE_ODD = 0 ; then
LINE_ODD=1
else
LINE_ODD=0
fi
done < $TMP_REVS
IFS=$OLD_IFS
if test "x$WANT_HTML" = "x1" ; then
echo "</TABLE></HTML>"
else
echo "#"
fi
# rm $TMP_REVS $MERGE_LOG $MERGE_FILE