commit b5e0e7b5987cd6b79ae876b70bf8608929e5630b
Author: Stefan Liebler <stli@linux.ibm.com>
Date: Wed Sep 4 16:07:19 2019 +0200
Show details about test fails in output of make check.
Currently there is no indication which tests fail in an
invocation of "make check".
This patch creates one <test-name>.test-result file per test
with an indication like PASS or FAIL. Furthermore the file tests.sum
is created with one line per test in order to indicate the test result.
At the end of "make check" the failing tests are listed and the
corresponding out files are dumped.
This way you'll get an indication of failing tests even if you can
just view the log file instead of having access to the build-directory.
The scripts and some parts of the Makefile comes from glibc in order to
be more equal.
This change also allows to mark an test as:
-XFAIL or XPASS: Add a Makefile variable like test-xfail-<testname> = yes.
-UNSUPPORTED: The testcase has to return with exit-code 77.
Signed-off-by: Stefan Liebler <stli@linux.ibm.com>
diff --git a/Makefile.in b/Makefile.in
index 7bf3030..757615b 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -379,7 +379,7 @@ ifeq ($(cxxdecimal), yes)
libdfp_cxx_tests = test-ostream test-ostream-g-spec test-istream
endif
-libdfp_tests = $(libdfp_c_tests) $(libdfp_cxx_tests)
+libdfp_tests = $(libdfp_c_tests) $(libdfp_cxx_tests) check-abi-libdfp check-localplt
test-printf.os: $(top_builddir)/printf_dfp.os
test-strtod.os: $(top_builddir)/strtod32.os $(top_builddir)/strtod64.os $(top_builddir)/strtod128.os
@@ -422,9 +422,21 @@ test-%-d64.c: %.input
test-%-d128.c: %.input
$(top_srcdir)/tests/gen-libdfp-tests.py -t decimal128 $^ > $(top_builddir)/$@
+# The name to give to a test in test results summaries.
+test-name = $(strip $(patsubst %.out, %, $@))
+
+# Likewise, in XFAIL variable names.
+test-xfail-name = $(strip $(patsubst %.out, %, $@))
+
+evaluate-test = $(top_srcdir)/scripts/evaluate-test.sh $(test-name) $$? \
+ $(if $(test-xfail-$(test-xfail-name)),true,false) \
+ false \
+ > $(top_builddir)/$(test-name).test-result
+
# Make check main rule
-$(addsuffix .out,$(libdfp_tests)): %.out:%
- LD_LIBRARY_PATH=$(top_builddir)/:$$LD_LIBRARY_PATH $(top_builddir)/$(patsubst %.out,%,$@) 2> $(top_builddir)/$@ 1> /dev/null
+%.out: %
+ LD_LIBRARY_PATH=$(top_builddir)/:$$LD_LIBRARY_PATH $(top_builddir)/$(test-name) 2> $(top_builddir)/$@ 1> /dev/null; \
+ $(evaluate-test)
# Check ABI rules
@@ -433,11 +445,9 @@ generated += $(extra-libs:=.symlist)
vpath %.abilist $(+sysdeps_dirs)
-check-abi: check-abi-libdfp.out
- $(warning $@)
-
check-abi-libdfp.out: libdfp$(dfp_abi_extn).abilist libdfp.symlist
- diff -p -U 0 $(filter %.abilist,$^) $(filter %.symlist,$^) > $@
+ diff -p -U 0 $(filter %.abilist,$^) $(filter %.symlist,$^) > $@; \
+ $(evaluate-test)
%.symlist: $(top_srcdir)/scripts/abilist.awk %.dynsym
LC_ALL=C $(AWK) -f $^ > $@T
@@ -448,8 +458,6 @@ check-abi-libdfp.out: libdfp$(dfp_abi_extn).abilist libdfp.symlist
mv -f $@T $@
# Check local PLT calls
-check-localplt: check-localplt.out
-
vpath localplt.data $(+sysdep_dirs)
check-localplt.out: $(top_srcdir)/scripts/check-localplt.awk \
@@ -458,7 +466,8 @@ check-localplt.out: $(top_srcdir)/scripts/check-localplt.awk \
localplt.data
LC_ALL=C $(AWK) -f $(filter-out $< %localplt.data,$^) | \
LC_ALL=C $(AWK) -f $< $(filter %localplt.data,$^) - \
- > $@
+ > $@; \
+ $(evaluate-test)
libdfp.so.jmprel: libdfp.so
@rm -f $@T
@@ -466,10 +475,26 @@ libdfp.so.jmprel: libdfp.so
test -s $@T
mv -f $@T $@
+define summarize-tests
+@egrep -v '^(PASS|XFAIL):' $(top_builddir)/$1 || true
+@echo "Summary of test results$2:"
+@sed 's/:.*//' < $(top_builddir)/$1 | sort | uniq -c
+@! egrep -q -v '^(X?PASS|XFAIL|UNSUPPORTED):' $(top_builddir)/$1
+endef
+
+tests: $(libdfp_tests:%=%.out)
+ $(top_srcdir)/scripts/merge-test-results.sh -s $(top_builddir)/ "" \
+ $(sort $(libdfp_tests)) \
+ > tests.sum
+ @echo Details of failed tests:
+ awk -v builddir=$(top_builddir) \
+ '{ if ($$1 != "PASS:") printf "%s/%s.out ", builddir, $$2; }' \
+ tests.sum | xargs -n 1 -t cat
+ $(call summarize-tests,tests.sum)
+
# The .out files are predicated in another rule on the actual test executables, so
# those are built from those rules.
-check: $(addsuffix .out,$(libdfp_tests)) check-abi check-localplt
- @echo +Completed make check
+check: tests
.PHONY: check
@@ -485,8 +510,8 @@ clean:
rm -f libdfp.map $(SHARED_LINKERNAME_LIB).jmprel libdfp.symlist
rm -f Versions.tmp
rm -f sysd-versions
- rm -f check-abi-libdfp.out check-localplt.out
rm -f $(addsuffix .out,$(libdfp_tests))
+ rm -f $(addsuffix .test-result,$(libdfp_tests))
rm -f $(addsuffix .conf,$(libdfp_tests))
rm -f $(libdfp_tests)
rm -f libdfp-test-ulps.h
diff --git a/scripts/evaluate-test.sh b/scripts/evaluate-test.sh
new file mode 100755
index 0000000..7bb8642
--- /dev/null
+++ b/scripts/evaluate-test.sh
@@ -0,0 +1,50 @@
+#!/bin/sh
+# Output a test status line.
+# Copyright (C) 2012-2019 Free Software Foundation, Inc.
+# This file is part of the GNU C Library.
+
+# The GNU C Library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+
+# The GNU C Library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+
+# You should have received a copy of the GNU Lesser General Public
+# License along with the GNU C Library; if not, see
+# <http://www.gnu.org/licenses/>.
+
+# usage: evaluate-test.sh test_name rc xfail stop_on_failure
+
+test_name=$1
+rc=$2
+orig_rc=$rc
+xfail=$3
+stop_on_failure=$4
+
+if [ $rc -eq 77 ]; then
+ result="UNSUPPORTED"
+ rc=0
+else
+ if [ $rc -eq 0 ]; then
+ result="PASS"
+ else
+ result="FAIL"
+ fi
+
+ if $xfail; then
+ result="X$result"
+ rc=0
+ fi
+fi
+
+echo "$result: $test_name"
+echo "original exit status $orig_rc"
+if $stop_on_failure; then
+ exit $rc
+else
+ exit 0
+fi
diff --git a/scripts/merge-test-results.sh b/scripts/merge-test-results.sh
new file mode 100755
index 0000000..919bbae
--- /dev/null
+++ b/scripts/merge-test-results.sh
@@ -0,0 +1,61 @@
+#!/bin/sh
+# Merge test results of individual tests or subdirectories.
+# Copyright (C) 2014-2019 Free Software Foundation, Inc.
+# This file is part of the GNU C Library.
+
+# The GNU C Library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+
+# The GNU C Library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+
+# You should have received a copy of the GNU Lesser General Public
+# License along with the GNU C Library; if not, see
+# <http://www.gnu.org/licenses/>.
+
+# usage: merge-test-results.sh -s objpfx subdir test-name...
+# (subdirectory tests; empty subdir at top level), or
+# merge-test-results.sh -t objpfx subdir-file-name subdir...
+# (top-level merge)
+
+set -e
+
+type=$1
+objpfx=$2
+shift 2
+
+case $type in
+ -s)
+ subdir=$1
+ shift
+ subdir=${subdir:+$subdir/}
+ for t in "$@"; do
+ if [ -s "$objpfx$t.test-result" ]; then
+ head -n1 "$objpfx$t.test-result"
+ else
+ echo "UNRESOLVED: $subdir$t"
+ fi
+ done
+ ;;
+
+ -t)
+ subdir_file_name=$1
+ shift
+ for d in "$@"; do
+ if [ -f "$objpfx$d/$subdir_file_name" ]; then
+ cat "$objpfx$d/$subdir_file_name"
+ else
+ echo "ERROR: test results for $d directory missing"
+ fi
+ done
+ ;;
+
+ *)
+ echo "unknown type $type" >&2
+ exit 1
+ ;;
+esac