Hash :
4014b502
Author :
Date :
2011-07-26T14:51:20
test harness: allow more metadata in log files This change reworks and improves the parallel test harness to use more specialized reStructuredText fields in the log files (instead of relying on specially-placed of "magic lines" and more indirect semantical formatting); the new fields are the following: - ":recheck:": tell whether the associated test will have to be re-run by "make recheck"; - ":copy-in-global-log:": tell whether the content of the log file should be copied in the "global log" `test-suite.log'; - ":end-metadata:", which inhibits the scanning of the rest of the log file (for what concerns test metadata). Also, the special `:test-result:' value "END" has been removed, superseded by the new `:end-metadata:' field. * doc/automake.texi (Log files generation and test results recording): Document the new API and semantics. Remove or fix some obsolete comments. * lib/am/check.am ($(TEST_SUITE_LOG), recheck, recheck-html): Adjust comments and code. * lib/tap-driver (finish): Adjust, with the help of ... (must_recheck, copy_in_global_log): ... these new functions. * lib/test-driver (Main code): Adjust, with the help of ... ($recheck, $gcopy): ... these new variables. * tests/trivial-test-driver: Update to obey the new APIs. * tests/test-driver-recheck.test: Likewise. * tests/test-driver-global-log.test: Likewise. * tests/tap-passthrough.test: Relax the test, avoiding to check what is written in `test-suite.log'; such check has been moved ... * tests/tap-global-log.test: ... in this new test, and extended. * tests/test-driver-metadata-no-leading-space.test: New test. * tests/test-driver-end-test-results.test: Removed, it checked the old APIs; superseded by ... * tests/test-driver-end-metadata.test: ... this new test. * tests/tap-log.test: Improve syncing with ... * tests/test-log.test: ... this new test. * tests/parallel-tests.test: Remove some duplication w.r.t. this last new test. Updated heading comments. * tests/Makefile.am (TESTS): Update.
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168
#! /bin/sh
# Copyright (C) 2011 Free Software Foundation, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# Check parallel-tests features:
# - log file creation
# - log file removal
# - stdout and stderr of a test script go in its log file
# - TEST_SUITE_LOG redefinition, at either automake or make time
# - VERBOSE environment variable support
# Keep in sync with 'tap-log.test'.
parallel_tests=yes
. ./defs || Exit 1
cat >> configure.in <<END
AC_OUTPUT
END
cat > Makefile.am << 'END'
TESTS = pass.test skip.test xfail.test fail.test xpass.test error.test
XFAIL_TESTS = xpass.test xfail.test
TEST_SUITE_LOG = global.log
END
# Custom markers, for use in grepping checks.
cmarker=::: # comment marker
pmarker=%%% # plain maker
cat > pass.test <<END
#! /bin/sh
echo "$pmarker pass $pmarker" >&2
echo "# $cmarker pass $cmarker" >&2
exit 0
END
cat > skip.test <<END
#! /bin/sh
echo "$pmarker skip $pmarker"
echo "# $cmarker skip $cmarker"
exit 1
END
cat > xfail.test <<END
#! /bin/sh
echo "$pmarker xfail $pmarker" >&2
echo "# $cmarker xfail $cmarker" >&2
exit 1
END
cat > fail.test <<END
#! /bin/sh
echo 1..1
echo "$pmarker fail $pmarker"
echo "# $cmarker fail $cmarker"
exit 1
END
cat > xpass.test <<END
#! /bin/sh
echo "$pmarker xpass $pmarker" >&2
echo "# $cmarker xpass $cmarker" >&2
exit 0
END
cat > error.test <<END
#! /bin/sh
echo "$pmarker error $pmarker"
echo "# $cmarker error $cmarker"
exit 99
END
chmod a+x *.test
$ACLOCAL
$AUTOCONF
$AUTOMAKE -a
./configure
TEST_SUITE_LOG=my.log $MAKE -e check && Exit 1
ls -l # For debugging.
test ! -f test-suite.log
test ! -f global.log
test -f my.log
st=0
for result in pass fail xfail xpass skip error; do
cat $result.log # For debugging.
$FGREP "$pmarker $result $pmarker" $result.log || st=1
$FGREP "$cmarker $result $cmarker" $result.log || st=1
done
test $st -eq 0 || Exit 1
cat my.log # For debugging.
for result in xfail fail xpass skip error; do
cat $result.log # For debugging.
$FGREP "$pmarker $result $pmarker" my.log || st=1
$FGREP "$cmarker $result $cmarker" my.log || st=1
done
test `$FGREP -c "$pmarker" my.log` -eq 5
test `$FGREP -c "$cmarker" my.log` -eq 5
# Passed test scripts shouldn't be mentioned in the global log.
$EGREP '(^pass|[^x]pass)\.test' my.log && Exit 1
# But failing (expectedly or not) and skipped ones should.
$FGREP 'xfail.test' my.log
$FGREP 'skip.test' my.log
$FGREP 'fail.test' my.log
$FGREP 'xpass.test' my.log
$FGREP 'error.test' my.log
touch error2.log test-suite.log global.log
TEST_SUITE_LOG=my.log $MAKE -e mostlyclean
ls -l # For debugging.
test ! -f my.log
test ! -f pass.log
test ! -f fail.log
test ! -f xfail.log
test ! -f xpass.log
test ! -f skip.log
test ! -f error.log
# "make mostlyclean" shouldn't remove unrelated log files.
test -f error2.log
test -f test-suite.log
test -f global.log
rm -f *.log
VERBOSE=yes $MAKE check >stdout && { cat stdout; Exit 1; }
cat stdout
cat global.log
test ! -f my.log
test ! -f test-suite.log
# Check that VERBOSE causes the global testsuite log to be
# emitted on stdout.
out=`cat stdout`
log=`cat global.log`
case $out in *"$log"*) ;; *) Exit 1;; esac
touch error2.log test-suite.log my.log
$MAKE clean
ls -l # For debugging.
test ! -f global.log
test ! -f pass.log
test ! -f fail.log
test ! -f xfail.log
test ! -f xpass.log
test ! -f skip.log
test ! -f error.log
# "make clean" shouldn't remove unrelated log files.
test -f error2.log
test -f test-suite.log
test -f my.log
rm -f *.log
: