Hash :
ffd4ecc9
Author :
Date :
2013-05-20T11:23:01
tests: better idiom to override make macro defs on the cmdline We now do so with a new wrapper shell function 'run_make()', which relies on the use of AM_MAKEFLAGS, avoiding the use of the '-e' make option. The use of that option (that causes the environment variables to take precedence over the macro definitions in the Makefile) has proved to be quite brittle in the past, causing annoying and sometimes problematic spurious failures. This has worsened in some still unpublished work going on in development branches. It's time to solve the issue once and for all. Note that we do not convert all uses of $MAKE in the testsuite right away; we might do so in follow-up changes, with leisure, to avoid a "patch bomb" effect (this commit is already too much of a bomb itself). What we do in this commit is to get rid of all "$MAKE -e" invocations. We admit that the implementation and feature-set of 'run_make()' are far from perfect, but good enough for our current purposes. We'll improve 'run_make()' if and when the need arises. * syntax-checks.mk (sc_tests_no_make_e): New syntax check, guard against the use of "$MAKE -e". (syntax_check_rules): Add it. (sc_tests_overriding_macros_on_cmdline): Adjust. (lint): New, alias for 'maintainer-check', for lazy typists. Idea backported from the 'maint' branch (Automake 1.13a). * t/ax/am-test-lib.sh (run_make): New function. Run $MAKE with the given command-line arguments, handling command-line override of variable definitions in a smart way (using AM_MAKEFLAGS if a non-GNU make implementation is detected to be in use). (useless_vpath_rebuild): Adjust to use 'run_make', to avoid a spurious maintainer check failure. (yl_distcheck): Use 'run_make' rather than bare '$MAKE'. (single_quote, append_single_quoted, is_valid_varname): New auxiliary function, used, directly or indirectly, by it. * Many tests: Adjust to avoid the use of "$MAKE -e", and prefer the use of 'run_make' in few other contexts as well, where it makes sense. Other minor fixlets while at it. Signed-off-by: Stefano Lattarini <stefano.lattarini@gmail.com>
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152
#! /bin/sh
# Copyright (C) 2007-2013 Free Software Foundation, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# Test Automake TESTS color output, by forcing it.
# Keep this in sync with the sister test 'color-tests2.sh'.
required='grep-nonprint'
# For gen-testsuite-part: ==> try-with-serial-tests <==
. test-init.sh
# Escape '[' for grep, below.
red="$esc\[0;31m"
grn="$esc\[0;32m"
lgn="$esc\[1;32m"
blu="$esc\[1;34m"
mgn="$esc\[0;35m"
std="$esc\[m"
cat >>configure.ac <<END
AC_OUTPUT
END
cat >Makefile.am <<'END'
TESTS = $(check_SCRIPTS)
check_SCRIPTS = pass fail skip xpass xfail error
XFAIL_TESTS = xpass xfail
END
cat >pass <<END
#! /bin/sh
exit 0
END
cat >fail <<END
#! /bin/sh
exit 1
END
cat >skip <<END
#! /bin/sh
exit 77
END
cat >error <<END
#! /bin/sh
exit 99
END
cp fail xfail
cp pass xpass
chmod +x pass fail skip xpass xfail error
$ACLOCAL
$AUTOCONF
$AUTOMAKE --add-missing
test_color ()
{
# Not a useless use of cat; see above comments "grep-nonprinting"
# requirement in 'test-init.sh'.
cat stdout | grep "^${grn}PASS${std}: .*pass"
cat stdout | grep "^${red}FAIL${std}: .*fail"
cat stdout | grep "^${blu}SKIP${std}: .*skip"
cat stdout | grep "^${lgn}XFAIL${std}: .*xfail"
cat stdout | grep "^${red}XPASS${std}: .*xpass"
# The old serial testsuite driver doesn't distinguish between failures
# and hard errors.
if test x"$am_serial_tests" = x"yes"; then
cat stdout | grep "^${red}FAIL${std}: .*error"
else
cat stdout | grep "^${mgn}ERROR${std}: .*error"
fi
:
}
test_no_color ()
{
# With make implementations that, like Solaris make, in case of errors
# print the whole failing recipe on standard output, we should content
# ourselves with a laxer check, to avoid false positives.
# Keep this in sync with lib/am/check.am:$(am__color_tests).
if $FGREP '= Xalways; then' stdout; then
# Extra verbose make, resort to laxer checks.
# Note that we also want to check that the testsuite summary is
# not unduly colorized.
(
set +e # In case some grepped regex below isn't matched.
# Not a useless use of cat; see above comments "grep-nonprinting"
# requirement in 'test-init.sh'.
cat stdout | grep "TOTAL.*:"
cat stdout | grep "PASS.*:"
cat stdout | grep "FAIL.*:"
cat stdout | grep "SKIP.*:"
cat stdout | grep "XFAIL.*:"
cat stdout | grep "XPASS.*:"
cat stdout | grep "ERROR.*:"
cat stdout | grep 'test.*expected'
cat stdout | grep 'test.*not run'
cat stdout | grep '===='
cat stdout | grep '[Ss]ee .*test-suite\.log'
cat stdout | grep '[Tt]estsuite summary'
) | grep "$esc" && exit 1
: For shells with broken 'set -e'
else
cat stdout | grep "$esc" && exit 1
: For shells with broken 'set -e'
fi
}
for vpath in false :; do
if $vpath; then
mkdir build
cd build
srcdir=..
else
srcdir=.
fi
$srcdir/configure
# Forced colorization should take place also with non-ANSI terminals;
# hence the "TERM=dumb" definition.
AM_COLOR_TESTS=always; export AM_COLOR_TESTS
run_make TERM=dumb check >stdout && { cat stdout; exit 1; }
cat stdout
test_color
unset AM_COLOR_TESTS
run_make TERM=ansi check >stdout && { cat stdout; exit 1; }
cat stdout
test_no_color
$MAKE distclean
cd $srcdir
done
: