kmx git

#! /bin/sh
# Copyright (C) 2011 Free Software Foundation, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

# TAP support:
#  - testsuite summary
#  - colorized testsuite summary
# TODO: splitting this into two distinct test scripts might be worthwhile

parallel_tests=yes
. ./defs || Exit 1

cp "$top_testsrcdir"/lib/tap-driver . \
  || fatal_ "failed to fetch auxiliary script tap-driver"

cat > configure.in <<END
AC_INIT([GNU AutoTAP], [5.12], [bug-automake@gnu.org])
AM_INIT_AUTOMAKE([parallel-tests])
AC_SUBST([PERL], ['$PERL'])
AC_CONFIG_FILES([Makefile])
AC_OUTPUT
END

cat > Makefile.stub << 'END'
TEST_LOG_DRIVER = $(PERL) $(srcdir)/tap-driver
TEST_LOG_COMPILER = cat
TESTS = all.test
END

$ACLOCAL
$AUTOCONF

br='============================================================================'

# The following shell variables are influential for this function:
#  - expect_failure
#  - use_colors
do_check ()
{
  case $#,$1 in
    1,--pass) expect_failure=no;;
    1,--fail) expect_failure=yes;;
           *) fatal_ "invalid usage of 'do_check'";;
  esac
  shift
  cat > summary.exp
  cat all.test
  st=0
  if test $use_colors = yes; then
    make_cmd="env AM_COLOR_TESTS=always $MAKE -e"
  else
    make_cmd=$MAKE
  fi
  $make_cmd check > stdout || st=$?
  cat stdout
  if test $expect_failure = yes; then
    test $st -gt 0 || Exit 1
  else
    test $st -eq 0 || Exit 1
  fi
  $PERL -w "$testsrcdir"/extract-testsuite-summary stdout > summary.got \
    || fatal_ "cannot extract testsuite summary"
  cat summary.exp
  cat summary.got
  if test $use_colors = yes; then
    # Use cmp, not diff, because the files might contain binary data.
    compare=cmp
  else
    compare=diff
  fi
  $compare summary.exp summary.got || Exit 1
}

for use_colors in "no" "yes"; do

  cp Makefile.stub Makefile.am

  if test $use_colors = yes; then
    red='[0;31m'
    grn='[0;32m'
    lgn='[1;32m'
    blu='[1;34m'
    mgn='[0;35m'
    brg='[1m'
    std='[m'
    echo AUTOMAKE_OPTIONS = color-tests >> Makefile.am
  else
    red= grn= lgn= blu= mgn= brg= std=
  fi

  success_header="\
${grn}${br}${std}
${grn}Testsuite summary for GNU AutoTAP 5.12${std}
${grn}${br}${std}"

  success_footer=${grn}${br}${std}

  failure_header="\
${red}${br}${std}
${red}Testsuite summary for GNU AutoTAP 5.12${std}
${red}${br}${std}"

  failure_footer="\
${red}${br}${std}
${red}See ./test-suite.log${std}
${red}Please report to bug-automake@gnu.org${std}
${red}${br}${std}"

  $AUTOMAKE --add-missing

  ./configure

  # 1 pass.
  { echo 1..1 && echo ok; } > all.test
  do_check --pass <<END
$success_header
${brg}# TOTAL: 1${std}
${grn}# PASS:  1${std}
# SKIP:  0
# XFAIL: 0
# FAIL:  0
# XPASS: 0
# ERROR: 0
$success_footer
END

  # 1 skip.
  { echo 1..1 && echo 'ok # SKIP'; } > all.test
  do_check --pass <<END
$success_header
${brg}# TOTAL: 1${std}
# PASS:  0
${blu}# SKIP:  1${std}
# XFAIL: 0
# FAIL:  0
# XPASS: 0
# ERROR: 0
$success_footer
END

  # 1 xfail.
  { echo 1..1 && echo 'not ok # TODO'; } > all.test
  do_check --pass <<END
$success_header
${brg}# TOTAL: 1${std}
# PASS:  0
# SKIP:  0
${lgn}# XFAIL: 1${std}
# FAIL:  0
# XPASS: 0
# ERROR: 0
$success_footer
END

  # 1 fail.
  { echo 1..1 && echo not ok; } > all.test
  do_check --fail <<END
$failure_header
${brg}# TOTAL: 1${std}
# PASS:  0
# SKIP:  0
# XFAIL: 0
${red}# FAIL:  1${std}
# XPASS: 0
# ERROR: 0
$failure_footer
END

  # 1 xpass.
  { echo 1..1 && echo 'ok # TODO'; } > all.test
  do_check --fail <<END
$failure_header
${brg}# TOTAL: 1${std}
# PASS:  0
# SKIP:  0
# XFAIL: 0
# FAIL:  0
${red}# XPASS: 1${std}
# ERROR: 0
$failure_footer
END

  # 1 hard error.
  { echo 1..1 && echo 'Bail out!'; } > all.test
  do_check --fail <<END
$failure_header
${brg}# TOTAL: 1${std}
# PASS:  0
# SKIP:  0
# XFAIL: 0
# FAIL:  0
# XPASS: 0
${mgn}# ERROR: 1${std}
$failure_footer
END

  # 3 non-failing results.
  cat > all.test <<END
1..3
ok
not ok # TODO
ok # SKIP
END
  do_check --pass <<END
$success_header
${brg}# TOTAL: 3${std}
${grn}# PASS:  1${std}
${blu}# SKIP:  1${std}
${lgn}# XFAIL: 1${std}
# FAIL:  0
# XPASS: 0
# ERROR: 0
$success_footer
END

  # 1 pass, 1 skip, 1 fail.
  cat > all.test <<END
1..3
ok
ok # SKIP
not ok
END
  do_check --fail <<END
$failure_header
${brg}# TOTAL: 3${std}
${grn}# PASS:  1${std}
${blu}# SKIP:  1${std}
# XFAIL: 0
${red}# FAIL:  1${std}
# XPASS: 0
# ERROR: 0
$failure_footer
END

  # 1 pass, 1 xfail, 1 xpass.
  cat > all.test <<END
1..3
ok
ok # TODO
not ok # TODO
END
  do_check --fail <<END
$failure_header
${brg}# TOTAL: 3${std}
${grn}# PASS:  1${std}
# SKIP:  0
${lgn}# XFAIL: 1${std}
# FAIL:  0
${red}# XPASS: 1${std}
# ERROR: 0
$failure_footer
END

  # 1 skip, 1 xfail, 1 error.
  cat > all.test <<END
1..3
ok # SKIP
not ok # TODO
Bail out!
END
  do_check --fail <<END
$failure_header
${brg}# TOTAL: 3${std}
# PASS:  0
${blu}# SKIP:  1${std}
${lgn}# XFAIL: 1${std}
# FAIL:  0
# XPASS: 0
${mgn}# ERROR: 1${std}
$failure_footer
END

  # 1 of each kind
  cat > all.test <<END
1..6
ok
not ok
ok # TODO
not ok # TODO
ok # SKIP
Bail out!
END
  do_check --fail <<END
$failure_header
${brg}# TOTAL: 6${std}
${grn}# PASS:  1${std}
${blu}# SKIP:  1${std}
${lgn}# XFAIL: 1${std}
${red}# FAIL:  1${std}
${red}# XPASS: 1${std}
${mgn}# ERROR: 1${std}
$failure_footer
END

  # Prepare some common data for later.
  for i in 0 1 2 3 4 5 6 7 8 9; do
    for j in 0 1 2 3 4 5 6 7 8 9; do
      echo "ok"
      echo "not ok # TODO"
      echo "ok # SKIP"
    done
  done > tap

  # Lots of non-failures (300 per kind).
  (cat tap && cat tap && cat tap) > all.test
  test `wc -l <all.test` -eq 900 || Exit 99 # Sanity check.
  echo 1..900 >> all.test # Test plan.
  do_check --pass <<END
$success_header
${brg}# TOTAL: 900${std}
${grn}# PASS:  300${std}
${blu}# SKIP:  300${std}
${lgn}# XFAIL: 300${std}
# FAIL:  0
# XPASS: 0
# ERROR: 0
$success_footer
END

  # 1 failure and lots of non-failures means failure.
  (cat tap && echo "not ok" && cat tap) > all.test
  test `wc -l <all.test` -eq 601 || Exit 99 # Sanity check.
  echo 1..601 >> all.test # Test plan.
  do_check --fail <<END
$failure_header
${brg}# TOTAL: 601${std}
${grn}# PASS:  200${std}
${blu}# SKIP:  200${std}
${lgn}# XFAIL: 200${std}
${red}# FAIL:  1${std}
# XPASS: 0
# ERROR: 0
$failure_footer
END

  # 1 error and lots of non-failures means failure.
  (cat tap && sed 30q tap && echo 'Bail out!') > all.test
  test `wc -l <all.test` -eq 331 || Exit 99 # Sanity check.
  echo 1..331 >> all.test # Test plan.
  do_check --fail <<END
$failure_header
${brg}# TOTAL: 331${std}
${grn}# PASS:  110${std}
${blu}# SKIP:  110${std}
${lgn}# XFAIL: 110${std}
# FAIL:  0
# XPASS: 0
${mgn}# ERROR: 1${std}
$failure_footer
END

done # use_colors in "yes" "no"

:
kc3-lang/automake/tests/tap-summary.test

Commit

tests/tap-summary.test
