Hash :
8f33eec4
Author :
Date :
2009-03-28T22:58:34
parallel-tests: redo lazy checking: recheck and RECHECK_LOGS. Replace the LAZY_TEST_SUITE API with a simpler yet more powerful one: RECHECK_LOGS specifies those tests which are to be removed in any case before testing. Provide a `recheck' convenience target to set RECHECK_LOGS to all failed and unexpectedly passed tests. Document several ways to limit the set of tests run. * lib/am/check.am [PARALLEL_TESTS] (RECHECK_LOGS): New variable, default to $(TESTS_LOGS). (check-TESTS): Remove $(RECHECK_LOGS) not $(TEST_LOGS). Drop use of LAZY_TEST_SUITE. ($(TEST_SUITE_LOG)): Do not output note about lazy rerun, as LAZY_TEST_SUITE is gone. (recheck): New target. (recheck-am, recheck-TESTS): New internal targets. * doc/automake.texi (Tests): Update @vindex for TESTS and TEST_LOGS. Replace description of LAZY_TEST_SUITE with a list of ways the set of tests to be run can be modified. Document RECHECK_LOGS and the recheck target. * tests/defs.in: Unset RECHECK_LOGS not LAZY_TEST_SUITE. * tests/parallel-tests.test: Adjust, replacing LAZY_TEST_SUITE with corresponding RECHECK_LOGS settings, and add another RECHECK_LOGS test. * tests/parallel-tests9.test: New test, test `recheck'. * tests/Makefile.am: Update. Suggestion and different implementation by Akim Demaille. Signed-off-by: Ralf Wildenhues <Ralf.Wildenhues@gmx.de>
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390
## automake - create Makefile.in from Makefile.am
## Copyright (C) 2001, 2003, 2006, 2007, 2008 Free Software Foundation,
## Inc.
## This program is free software; you can redistribute it and/or modify
## it under the terms of the GNU General Public License as published by
## the Free Software Foundation; either version 3, or (at your option)
## any later version.
## This program is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
## GNU General Public License for more details.
## You should have received a copy of the GNU General Public License
## along with this program. If not, see <http://www.gnu.org/licenses/>.
if %?COLOR%
# If stdout is a non-dumb tty, use colors. If test -t is not supported,
# then this fails; a conservative approach. Of course do not redirect
# stdout here, just stderr.
am__tty_colors = \
red=; grn=; lgn=; blu=; std=; \
test "X$(AM_COLOR_TESTS)" != Xno \
&& test "X$$TERM" != Xdumb \
&& { test "X$(AM_COLOR_TESTS)" = Xalways || test -t 1 2>/dev/null; } \
&& { \
red='[0;31m'; \
grn='[0;32m'; \
lgn='[1;32m'; \
blu='[1;34m'; \
std='[m'; \
}
else !%?COLOR%
am__tty_colors = \
red=; grn=; lgn=; blu=; std=
endif !%?COLOR%
.PHONY: check-TESTS
if %?PARALLEL_TESTS%
include inst-vars.am
## New parallel test driver.
##
## This code is adapted from check.mk which was originally
## written at EPITA/LRDE, further developed at Gostai, then made
## its way from GNU coreutils to end up, largely rewritten, in
## Automake.
##
## It provides special support for "unit tests", that is to say,
## tests that (once run) no longer need to be re-compiled and
## re-run at each "make check", unless their sources changed. To
## enable unit-test supports, set RECHECK_LOGS to empty. In such a
## setting, that heavily relies on correct dependencies, its users may
## prefer to define EXTRA_PROGRAMS instead of check_PROGRAMS, because
## it allows intertwined compilation and execution of the tests.
## Sometimes this helps catching errors earlier (you don't have to
## wait for all the tests to be compiled).
##
## Define TEST_SUITE_LOG to be the name of the global log to create.
## Define TEST_LOGS to the set of logs to include in it. It defaults
## to $(TESTS), with `.test' and `@EXEEXT@' removed, and `'.log'
## appended.
##
## In addition to the magic "exit 77 means SKIP" feature (which was
## imported from automake), there is a magic "exit 99 means FAIL" feature
## which is useful if you need to issue a hard error no matter whether the
## test is XFAIL or not. You can disable this feature by setting the
## variable DISABLE_HARD_ERRORS to a nonempty value.
# Restructured Text title and section.
am__rst_title = sed 's/.*/ & /;h;s/./=/g;p;x;p;g;p;s/.*//'
am__rst_section = sed 'p;s/./=/g;p;g'
# Put stdin (possibly several lines separated by ". ") in a box.
am__text_box = $(AWK) '{ \
n = split($$0, lines, "\\. "); max = 0; \
for (i = 1; i <= n; ++i) \
if (max < length(lines[i])) \
max = length(lines[i]); \
for (i = 0; i < max; ++i) line = line "="; \
print line; \
for (i = 1; i <= n; ++i) if (lines[i]) print lines[i];\
print line; \
}'
# Solaris 10 'make', and several other traditional 'make' implementations,
# pass "-e" to $(SHELL). This contradicts POSIX. Work around the problem
# by disabling -e (using the XSI extension "set +e") if it's set.
am__sh_e_setup = case $$- in *e*) set +e;; esac
# To be inserted before the command running the test. Creates the
# directory for the log if needed. Stores in $dir the directory
# containing $f, in $tst the test, in $log the log, and passes
# TESTS_ENVIRONMENT. Save and restore TERM around use of
# TESTS_ENVIRONMENT, in case that unsets it.
am__check_pre = \
$(am__sh_e_setup); \
$(am__vpath_adj_setup) $(am__vpath_adj) \
srcdir=$(srcdir); export srcdir; \
rm -f $@-t; \
trap 'st=$$?; rm -f '\''$(abs_builddir)/$@-t'\''; (exit $$st); exit $$st' \
1 2 13 15; \
am__odir=`echo "./$@" | sed 's|/[^/]*$$||'`; \
test "x$$am__odir" = x. || $(MKDIR_P) "$$am__odir" || exit $$?; \
if test -f "./$$f"; then dir=./; \
elif test -f "$$f"; then dir=; \
else dir="$(srcdir)/"; fi; \
tst=$$dir$$f; log='$@'; __SAVED_TERM=$$TERM; \
$(TESTS_ENVIRONMENT)
# To be appended to the command running the test. Handle the stdout
# and stderr redirection, and catch the exit status.
am__check_post = \
>$@-t 2>&1; \
estatus=$$?; \
if test -n '$(DISABLE_HARD_ERRORS)' \
&& test $$estatus -eq 99; then \
estatus=1; \
fi; \
TERM=$$__SAVED_TERM; export TERM; \
$(am__tty_colors); \
xfailed=PASS; \
case " $(XFAIL_TESTS) " in \
*[\ \ ]$$f[\ \ ]* | *[\ \ ]$$dir$$f[\ \ ]*) \
xfailed=XFAIL;; \
esac; \
case $$estatus:$$xfailed in \
0:XFAIL) col=$$red; res=XPASS;; \
0:*) col=$$grn; res=PASS ;; \
77:*) col=$$blu; res=SKIP ;; \
99:*) col=$$red; res=FAIL ;; \
*:XFAIL) col=$$lgn; res=XFAIL;; \
*:*) col=$$red; res=FAIL ;; \
esac; \
echo "$${col}$$res$${std}: $$f"; \
echo "$$res: $$f (exit: $$estatus)" | \
$(am__rst_section) >$@; \
cat $@-t >>$@; \
rm -f $@-t
$(TEST_SUITE_LOG): $(TEST_LOGS)
@$(am__sh_e_setup); \
list='$(TEST_LOGS)'; \
results=`for f in $$list; do \
read line < $$f && echo "$$line" || echo FAIL; \
done`; \
all=`echo "$$results" | sed '/^$$/d' | wc -l | sed -e 's/^[ ]*//'`; \
fail=`echo "$$results" | grep -c '^FAIL'`; \
pass=`echo "$$results" | grep -c '^PASS'`; \
skip=`echo "$$results" | grep -c '^SKIP'`; \
xfail=`echo "$$results" | grep -c '^XFAIL'`; \
xpass=`echo "$$results" | grep -c '^XPASS'`; \
failures=`expr $$fail + $$xpass`; \
all=`expr $$all - $$skip`; \
if test "$$all" -eq 1; then tests=test; All=; \
else tests=tests; All="All "; fi; \
case fail=$$fail:xpass=$$xpass:xfail=$$xfail in \
fail=0:xpass=0:xfail=0) \
msg="$$All$$all $$tests passed. "; \
exit=true;; \
fail=0:xpass=0:xfail=*) \
msg="$$All$$all $$tests behaved as expected"; \
if test "$$xfail" -eq 1; then xfailures=failure; \
else xfailures=failures; fi; \
msg="$$msg ($$xfail expected $$xfailures). "; \
exit=true;; \
fail=*:xpass=0:xfail=*) \
msg="$$fail of $$all $$tests failed. "; \
exit=false;; \
fail=*:xpass=*:xfail=*) \
msg="$$failures of $$all $$tests did not behave as expected"; \
if test "$$xpass" -eq 1; then xpasses=pass; \
else xpasses=passes; fi; \
msg="$$msg ($$xpass unexpected $$xpasses). "; \
exit=false;; \
*) \
echo >&2 "incorrect case"; exit 4;; \
esac; \
if test "$$skip" -ne 0; then \
if test "$$skip" -eq 1; then \
msg="$$msg($$skip test was not run). "; \
else \
msg="$$msg($$skip tests were not run). "; \
fi; \
fi; \
{ \
echo "$(PACKAGE_STRING): $(subdir)/$(TEST_SUITE_LOG)" | \
$(am__rst_title); \
echo "$$msg"; \
echo; \
echo ".. contents:: :depth: 2"; \
echo; \
for f in $$list; do \
read line < $$f; \
case $$line in \
PASS:*|XFAIL:*);; \
*) echo; cat $$f;; \
esac; \
done; \
} >$(TEST_SUITE_LOG).tmp; \
mv $(TEST_SUITE_LOG).tmp $(TEST_SUITE_LOG); \
if test "$$failures" -ne 0; then \
msg="$${msg}See $(subdir)/$(TEST_SUITE_LOG). "; \
if test -n "$(PACKAGE_BUGREPORT)"; then \
msg="$${msg}Please report to $(PACKAGE_BUGREPORT). "; \
fi; \
fi; \
$(am__tty_colors); \
if $$exit; then \
echo $(ECHO_N) "$$grn$(ECHO_C)"; \
else \
echo $(ECHO_N) "$$red$(ECHO_C)"; \
fi; \
echo "$$msg" | $(am__text_box); \
echo $(ECHO_N) "$$std$(ECHO_C)"; \
test x"$$VERBOSE" = x || $$exit || cat $(TEST_SUITE_LOG); \
$$exit
RECHECK_LOGS = $(TEST_LOGS)
# Run all the tests.
check-TESTS:
## Expand $(RECHECK_LOGS) only once, to avoid exceeding line length limits.
@list='$(RECHECK_LOGS)'; test -z "$$list" || rm -f $$list
## We always have to remove TEST_SUITE_LOG, to ensure its rule is run
## in any case even in lazy mode: otherwise, if no test needs rerunning,
## or a prior run plus reruns all happen within the same timestamp
## (can happen with a prior `make TESTS=<subset>'),
## then we get no log output.
## OTOH, this means that, in the rule for `$(TEST_SUITE_LOG)', we
## cannot use `$?' to compute the set of lazily rerun tests, lest
## we rely on .PHONY to work portably.
@test -z "$(TEST_SUITE_LOG)" || rm -f $(TEST_SUITE_LOG)
@set_logs=; if test "X$(TEST_LOGS)" = X.log; then \
set_logs=TEST_LOGS=; \
fi; \
$(MAKE) $(AM_MAKEFLAGS) $(TEST_SUITE_LOG) $$set_logs
## Rerun all FAILed or XPASSed tests (as well as all whose logs are out
## of date or do not exist).
recheck-TESTS:
@list='$(TEST_LOGS)'; \
logs=`for f in $$list; do \
if read line < $$f; then \
case $$line in FAIL*|XPASS*) echo $$f;; esac; \
else echo $$f; fi; \
done | tr '\012\015' ' '`; \
$(MAKE) $(AM_MAKEFLAGS) check-TESTS RECHECK_LOGS="$$logs"
recheck-am: recheck-TESTS
recheck: recheck-am
.PHONY: recheck recheck-am recheck-TESTS
.MAKE: recheck-am
## -------------- ##
## Produce HTML. ##
## -------------- ##
.log.html:
@list='$(RST2HTML) $$RST2HTML rst2html rst2html.py'; \
for r2h in $$list; do \
if ($$r2h --version) >/dev/null 2>&1; then \
R2H=$$r2h; \
fi; \
done; \
if test -z "$$R2H"; then \
echo >&2 "cannot find rst2html, cannot create $@"; \
exit 2; \
fi; \
$$R2H $< >$@.tmp
@mv $@.tmp $@
# Be sure to run check-TESTS first, and then to convert the result.
# Beware of concurrent executions. And expect check-TESTS to fail.
check-html:
@if $(MAKE) $(AM_MAKEFLAGS) check-TESTS; then \
rv=0; else rv=$$?; \
fi; \
$(MAKE) $(AM_MAKEFLAGS) $(TEST_SUITE_HTML) || exit 4; \
exit $$rv
.PHONY: check-html
.MAKE: check-html
else !%?PARALLEL_TESTS%
check-TESTS: $(TESTS)
@failed=0; all=0; xfail=0; xpass=0; skip=0; \
srcdir=$(srcdir); export srcdir; \
## Make sure Solaris VPATH-expands all members of this list, even
## the first and the last one; thus the spaces around $(TESTS)
list=' $(TESTS) '; \
$(am__tty_colors); \
if test -n "$$list"; then \
for tst in $$list; do \
if test -f ./$$tst; then dir=./; \
## Note: Solaris 2.7 seems to expand TESTS using VPATH. That's
## why we also try `dir='
elif test -f $$tst; then dir=; \
else dir="$(srcdir)/"; fi; \
if $(TESTS_ENVIRONMENT) $${dir}$$tst; then \
## Success
all=`expr $$all + 1`; \
case " $(XFAIL_TESTS) " in \
*[\ \ ]$$tst[\ \ ]*) \
xpass=`expr $$xpass + 1`; \
failed=`expr $$failed + 1`; \
col=$$red; res=XPASS; \
;; \
*) \
col=$$grn; res=PASS; \
;; \
esac; \
elif test $$? -ne 77; then \
## Failure
all=`expr $$all + 1`; \
case " $(XFAIL_TESTS) " in \
*[\ \ ]$$tst[\ \ ]*) \
xfail=`expr $$xfail + 1`; \
col=$$lgn; res=XFAIL; \
;; \
*) \
failed=`expr $$failed + 1`; \
col=$$red; res=FAIL; \
;; \
esac; \
else \
## Skipped
skip=`expr $$skip + 1`; \
col=$$blu; res=SKIP; \
fi; \
echo "$${col}$$res$${std}: $$tst"; \
done; \
## Prepare the banner
if test "$$all" -eq 1; then \
tests="test"; \
All=""; \
else \
tests="tests"; \
All="All "; \
fi; \
if test "$$failed" -eq 0; then \
if test "$$xfail" -eq 0; then \
banner="$$All$$all $$tests passed"; \
else \
if test "$$xfail" -eq 1; then failures=failure; else failures=failures; fi; \
banner="$$All$$all $$tests behaved as expected ($$xfail expected $$failures)"; \
fi; \
else \
if test "$$xpass" -eq 0; then \
banner="$$failed of $$all $$tests failed"; \
else \
if test "$$xpass" -eq 1; then passes=pass; else passes=passes; fi; \
banner="$$failed of $$all $$tests did not behave as expected ($$xpass unexpected $$passes)"; \
fi; \
fi; \
## DASHES should contain the largest line of the banner.
dashes="$$banner"; \
skipped=""; \
if test "$$skip" -ne 0; then \
if test "$$skip" -eq 1; then \
skipped="($$skip test was not run)"; \
else \
skipped="($$skip tests were not run)"; \
fi; \
test `echo "$$skipped" | wc -c` -le `echo "$$banner" | wc -c` || \
dashes="$$skipped"; \
fi; \
report=""; \
if test "$$failed" -ne 0 && test -n "$(PACKAGE_BUGREPORT)"; then \
report="Please report to $(PACKAGE_BUGREPORT)"; \
test `echo "$$report" | wc -c` -le `echo "$$banner" | wc -c` || \
dashes="$$report"; \
fi; \
dashes=`echo "$$dashes" | sed s/./=/g`; \
if test "$$failed" -eq 0; then \
echo "$$grn$$dashes"; \
else \
echo "$$red$$dashes"; \
fi; \
echo "$$banner"; \
test -z "$$skipped" || echo "$$skipped"; \
test -z "$$report" || echo "$$report"; \
echo "$$dashes$$std"; \
test "$$failed" -eq 0; \
else :; fi
endif !%?PARALLEL_TESTS%