From 5abb71025f96b9f567859db0ec23d0fda7994ed9 Mon Sep 17 00:00:00 2001
From: Theodore Ts'o <tytso@mit.edu>
Date: Wed, 27 Jul 2022 15:57:11 -0400
Subject: test-appliance: take advantage of xfstests' xunit-quiet and fail loop
 retry

The latest xfstests upstream supports a new report type, xunit-quiet,
which doesn't include the NNN.out.bad and NNN.full in the results.xml
file.  Since we save the all of the test artifacts, there's no point
in keeping a copy those files in the junit XML file.

The latest xfstests also supports the -L <n> option, which retries a
failed test <n> times so we can easily determine whether a test
failure is a hard or flaky failure.  By default, we retry each failed
test 4 additional times; this can be changed by the
command-line-option --fail-loop-count <n>.

Signed-off-by: Theodore Ts'o <tytso@mit.edu>
---
 run-fstests/util/parse_cli            |  4 ++++
 test-appliance/files/root/runtests.sh | 22 ++++++++++++++++++++--
 2 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/run-fstests/util/parse_cli b/run-fstests/util/parse_cli
index 5848bf43..7acdde82 100644
--- a/run-fstests/util/parse_cli
+++ b/run-fstests/util/parse_cli
@@ -238,6 +238,7 @@ cpu-type:
 disable-serial
 email:
 enable-serial
+fail-loop-count:
 gce-disk-spec:
 gce-network:
 gce-zone:
@@ -398,6 +399,9 @@ while (( $# >= 1 )); do
 	    ROOT_FS="$1"
 	    IMAGE_FLAG="--image-family"
 	    ;;
+	--fail-loop-count) shift
+	    FSTESTOPT="$FSTESTOPT,fail_loop_count,$1"
+	    ;;
 	--update-xfstests-tar)
 	    supported_flavors kvm gce
 	    UPDATE_XFSTESTS_TAR=yes
diff --git a/test-appliance/files/root/runtests.sh b/test-appliance/files/root/runtests.sh
index 66dd99a7..9958a35c 100755
--- a/test-appliance/files/root/runtests.sh
+++ b/test-appliance/files/root/runtests.sh
@@ -125,6 +125,7 @@ else
 fi
 
 RPT_COUNT=1
+FAIL_LOOP_COUNT=4
 
 while [ "$1" != "" ]; do
   case $1 in
@@ -134,6 +135,9 @@ while [ "$1" != "" ]; do
     count) shift
 	RPT_COUNT=$1
 	;;
+    fail_loop_count) shift
+	FAIL_LOOP_COUNT=$1
+	;;
     no_punch)
 	ALL_FSSTRESS_AVOID="$ALL_FSSTRESS_AVOID -f punch=0"
 	ALL_FSX_AVOID="$ALL_FSX_AVOID -H"
@@ -278,6 +282,17 @@ fi
 
 touch "$RESULTS/fstest-completed"
 
+./check --help > /tmp/check-help
+report_fmt=xunit
+if grep -q xunit-quiet /tmp/check-help ; then
+    report_fmt=xunit-quiet
+fi
+fail_test_loop=
+if test $RPT_COUNT -eq 1 && test $FAIL_LOOP_COUNT -gt 0 && \
+	grep -q -- "-L <n>" /tmp/check-help ; then
+    fail_test_loop="-L $FAIL_LOOP_COUNT"
+fi
+
 [ -e /proc/slabinfo ] && cp /proc/slabinfo "$RESULTS/slabinfo.before"
 cp /proc/meminfo "$RESULTS/meminfo.before"
 
@@ -577,8 +592,11 @@ do
 	    fi
 	    if test -s /tmp/tests-to-run
 	    then
-		bash ./check -R xunit -T $EXTRA_OPT $AEX $TEST_SET_EXCLUDE \
-		     $(cat /tmp/tests-to-run)
+		echo ./check -R $report_fmt $fail_test_loop -T $EXTRA_OPT \
+		     $AEX $TEST_SET_EXCLUDE $(cat /tmp/tests-to-run) \
+		     > "$RESULT_BASE/check-cmd"
+		bash ./check -R $report_fmt $fail_test_loop -T $EXTRA_OPT \
+		     $AEX $TEST_SET_EXCLUDE $(cat /tmp/tests-to-run)
 		copy_xunit_results
 	    else
 		echo "No tests to run"
-- 
cgit 1.2.3-korg