Hash :
15400bca
Author :
Date :
2011-06-30T14:55:43
parallel-tests: new recognized test result 'ERROR' * lib/am/check.am ($(TEST_SUITE_LOG)): Recognize a new test result `ERROR'. Use it when encountering unreadable test logs (previously a simple `FAIL' was used in this situations). * lib/test-driver: Set the global test result to `ERROR' when the test exit status is 99. When doing colorized output, color `ERROR' results in magenta. * doc/automake.texi (Log files generation and test results recording): Update by also listing `ERROR' among the list of valid `:test-results:' arguments. * NEWS: Update. * tests/trivial-test-driver: Update. * tests/parallel-tests.test: Likewise. * tests/parallel-tests-harderror.test: Likewise. * tests/parallel-tests-no-spurious-summary.test: Likewise. * tests/test-driver-global-log.test: Likewise. * tests/test-driver-recheck.test: Likewise. * tests/test-driver-custom-multitest-recheck.test: Likewise. * tests/test-driver-custom-multitest-recheck2.test: Likewise. * tests/test-driver-custom-multitest.test: Likewise. * tests/test-driver-custom-no-html.test: Likewise. * tests/test-driver-end-test-results.test: Likewise. * tests/color.test: Likewise. Also, make stricter, and also test from VPATH. * tests/color2.test: Likewise, and improve syncing with color.test. * tests/parallel-tests-exit-statuses.test: New test. * tests/parallel-tests-console-output.test: Likewise. * tests/Makefile.am (TESTS): Update.
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108
#! /bin/sh
# Copyright (C) 2011 Free Software Foundation, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# Check parallel-tests features: normal and special exit statuses
# in the test scripts.
parallel_tests=yes
. ./defs || Exit 1
cat >> configure.in << 'END'
AC_OUTPUT
END
# $failure_statuses should be defined to the list of all integers between
# 1 and 255 (inclusive), excluded 77 and 99.
# Let's use `seq' if available, it's faster than the loop.
failure_statuses=`seq 1 255 \
|| { i=1; while test $i -le 255; do echo $i; i=\`expr $i + 1\`; done; }`
failure_statuses=`
for i in $failure_statuses; do
test $i -eq 77 || test $i -eq 99 || echo $i
done | tr "$nl" ' '`
# For debugging.
echo "failure_statuses: $failure_statuses"
# Sanity check.
test `for st in $failure_statuses; do echo $st; done | wc -l` -eq 253 \
|| fatal_ "initializing list of exit statuses for simple failures"
cat > Makefile.am <<END
LOG_COMPILER = ./do-exit
fail_tests = $failure_statuses
TESTS = 0 77 99 $failure_statuses
\$(TESTS):
END
cat > do-exit <<'END'
#!/bin/sh
echo "$0: $1"
case $1 in
[0-9]|[0-9][0-9]|[0-9][0-9][0-9]) st=$1;;
*/[0-9]|*/[0-9][0-9]|*/[0-9][0-9][0-9]) st=`echo x"$1" | sed 's|.*/||'`;;
*) st=99;;
esac
exit $st
END
chmod a+x do-exit
$ACLOCAL
$AUTOCONF
$AUTOMAKE -a
{
echo PASS: 0
echo SKIP: 77
echo ERROR: 99
for st in $failure_statuses; do
echo "FAIL: $st"
done
} | LC_ALL=C sort > exp-fail
sed 's/^FAIL:/XFAIL:/' exp-fail | LC_ALL=C sort > exp-xfail-1
sed '/^ERROR:/d' exp-xfail-1 > exp-xfail-2
sort exp-fail
sort exp-xfail-1
sort exp-xfail-2
./configure
st=1
$MAKE check >stdout && st=0
cat stdout
cat test-suite.log
test $st -gt 0 || Exit 1
LC_ALL=C grep '^[A-Z][A-Z]*:' stdout | LC_ALL=C sort > got-fail
diff exp-fail got-fail
st=1
XFAIL_TESTS="$failure_statuses 99" $MAKE -e check >stdout && st=0
cat stdout
cat test-suite.log
test $st -gt 0 || Exit 1
LC_ALL=C grep '^[A-Z][A-Z]*:' stdout | LC_ALL=C sort > got-xfail-1
diff exp-xfail-1 got-xfail-1
st=0
XFAIL_TESTS="$failure_statuses" TESTS="0 77 $failure_statuses" \
$MAKE -e check >stdout || st=$?
cat stdout
cat test-suite.log
test $st -eq 0 || Exit 1
LC_ALL=C grep '^[A-Z][A-Z]*:' stdout | LC_ALL=C sort > got-xfail-2
diff exp-xfail-2 got-xfail-2
: