[m-rev.] diff: bootcheck and expected failures

[Zoltan Somogyi]

tools/bootcheck:
	Add a mechanism for expecting the failure of some test cases.
	The mechanism is a file called EXPECT_FAIL_TESTS.all_grades
	in the test directory, possibly augmented by files named
	EXPECT_FAIL_TESTS.$grade for any grade-specific expected test
	failures. The new mechanism makes it easier to see whether the
	change the bootcheck is testing works or not, since the expected
	failure of unrelated test cases doesn't influence the exit status.

	Add an option --expect-no-failures, which disables this mechanism.

Zoltan.

cvs diff: Diffing .
Index: bootcheck
===================================================================
RCS file: /home/mercury/mercury1/repository/mercury/tools/bootcheck,v
retrieving revision 1.172
diff -u -b -r1.172 bootcheck
--- bootcheck	20 May 2005 06:15:26 -0000	1.172
+++ bootcheck	3 Jun 2005 06:52:09 -0000
@@ -27,6 +27,9 @@
         Run only the tests which failed on the last run, as recorded in the
         tests/runtests.errs file. Meaningful only in the absence of -s and -f
         options.
+    --expect-no-failures
+        Exit with a nonzero exit status if any test fails, even if the failure
+        of that test is otherwise expected.
     -e, --extras
         Test the programs in the extras directory.
     -h, --help
@@ -154,6 +157,7 @@
 specified_tests_only=""
 failing_tests_only=false
 error_file_only=false
+expect_listed_failures=true
 extras=false
 jfactor=""
 keep_objs=false
@@ -248,6 +252,9 @@
     --error-file)
         error_file_only=true ;;
 
+    --expect-no-failures)
+        expect_listed_failures=false ;;
+
     -h|--help)
         echo "$usage"
         exit 0 ;;
@@ -1644,6 +1651,47 @@
 
 exitstatus=0
 
+tests_dir="$root/${tests_prefix}tests"
+
+cat /dev/null > $tests_dir/EXPECT_FAIL_TESTS
+
+if test -s $tests_dir/FAILED_TESTS_SUMMARY
+then
+    sed -e 's/ .*//' < $tests_dir/FAILED_TESTS_SUMMARY > $tests_dir/FAIL_TESTS
+
+    if test -f $tests_dir/EXPECT_FAIL_TESTS.all_grades
+    then
+        cat $tests_dir/EXPECT_FAIL_TESTS.all_grades >> \
+            $tests_dir/EXPECT_FAIL_TESTS
+    fi
+    if test -f $tests_dir/EXPECT_FAIL_TESTS.$test_grade
+    then
+        cat $tests_dir/EXPECT_FAIL_TESTS.$test_grade >> \
+            $tests_dir/EXPECT_FAIL_TESTS
+    fi
+
+    sort $tests_dir/EXPECT_FAIL_TESTS > $tests_dir/EXPECT_FAIL_TESTS.sort
+    mv $tests_dir/EXPECT_FAIL_TESTS.sort $tests_dir/EXPECT_FAIL_TESTS
+
+    sort $tests_dir/FAIL_TESTS > $tests_dir/FAIL_TESTS.sort
+    mv $tests_dir/FAIL_TESTS.sort $tests_dir/FAIL_TESTS
+
+    comm -23 $tests_dir/FAIL_TESTS \
+        $tests_dir/EXPECT_FAIL_TESTS \
+        > $tests_dir/UNEXPECTED_FAILED_TESTS
+    comm -12 $tests_dir/FAIL_TESTS \
+        $tests_dir/EXPECT_FAIL_TESTS \
+        > $tests_dir/EXPECTED_FAILED_TESTS
+
+    if $expect_listed_failures
+    then
+        if test ! -s $tests_dir/UNEXPECTED_FAILED_TESTS
+        then
+            test_status=0
+        fi
+    fi
+fi
+
 if test "$diff_status" != 0
 then
     echo "error exit: stages 2 and 3 differ"
@@ -1668,9 +1716,19 @@
     exitstatus=1
 fi
 
-if test -s $root/${tests_prefix}tests/FAILED_TESTS_SUMMARY
+if test -s $tests_dir/FAILED_TESTS_SUMMARY
 then
-    cat $root/${tests_prefix}tests/FAILED_TESTS_SUMMARY
+    if test -s $tests_dir/EXPECTED_FAILED_TESTS
+    then
+        echo expected test case failures:
+        cat $tests_dir/EXPECTED_FAILED_TESTS
+    fi
+
+    if test -s $tests_dir/UNEXPECTED_FAILED_TESTS
+    then
+        echo unexpected test case failures:
+        cat $tests_dir/UNEXPECTED_FAILED_TESTS
+    fi
 fi
 
 echo "finishing at `date`"
--------------------------------------------------------------------------
mercury-reviews mailing list
post:  mercury-reviews at cs.mu.oz.au
administrative address: owner-mercury-reviews at cs.mu.oz.au
unsubscribe: Address: mercury-reviews-request at cs.mu.oz.au Message: unsubscribe
subscribe:   Address: mercury-reviews-request at cs.mu.oz.au Message: subscribe
--------------------------------------------------------------------------