Print failed test results at the end for the spec test suite

check.py

@@ -21,6 +21,7 @@
 import sys
 import unittest
 from collections import OrderedDict
+from contextlib import contextmanager
 from multiprocessing.pool import ThreadPool
 from pathlib import Path
 
@@ -219,6 +220,7 @@ def run_one_spec_test(wast: Path, stdout=None):
             return  # don't try all the binary format stuff TODO
         else:
             shared.fail_with_error(str(e))
+            raise
 
     check_expected(actual, expected, stdout=stdout)
 
@@ -248,32 +250,63 @@ def run_one_spec_test(wast: Path, stdout=None):
 
 
 def run_spec_test_with_wrapped_stdout(wast: Path):
+    '''
+    Returns (bool, str) where the first element is whether the test was
+    successful and the second is the combined stdout and stderr of the test.
+    '''
     out = io.StringIO()
     try:
         run_one_spec_test(wast, stdout=out)
     except Exception as e:
         # Serialize exceptions into the output string buffer
         # so they can be reported on the main thread.
         print(e, file=out)
-        raise
-    return out.getvalue()
+        return False, out.getvalue()
+    return True, out.getvalue()
+
+
+@contextmanager
+def red_stdout():
+    try:
+        print("\033[31m", end="")
+        yield
+    finally:
+        print("\033[0m", end="")
 
 
 def run_spec_tests():
     print('\n[ checking wasm-shell spec testcases... ]\n')
 
     worker_count = os.cpu_count()
     print("Running with", worker_count, "workers")
-    test_paths = [Path(x) for x in shared.options.spec_tests]
+    test_paths = (Path(x) for x in shared.options.spec_tests)
+
+    failed_stdouts = []
     with ThreadPool(processes=worker_count) as pool:
         try:
-            for result in pool.imap_unordered(run_spec_test_with_wrapped_stdout, test_paths):
-                print(result, end="")
+            for success, stdout in pool.imap_unordered(run_spec_test_with_wrapped_stdout, test_paths):
+                if success:
+                    print(stdout, end="")
+                    continue
+
+                failed_stdouts.append(stdout)
+                if shared.options.abort_on_first_failure:
+                    with red_stdout():
+                        print("Aborted tests execution after first failure. Set --no-fail-fast to disable this.")
+                    break
         except KeyboardInterrupt:
             # Hard exit to avoid threads continuing to run after Ctrl-C.
             # There's no concern of deadlocking during shutdown here.
             os._exit(1)
 
+    if not failed_stdouts:
+        return
+
+    with red_stdout():
+        print("Failed tests:")
+        for failed in failed_stdouts:
+            print(failed, end="")
+
 
 def run_validator_tests():
     print('\n[ running validation tests... ]\n')

scripts/test/shared.py

@@ -40,12 +40,12 @@ def parse_args(args):
         '--no-torture', dest='torture', action='store_false',
         help='Disables running the torture testcases.')
     parser.add_argument(
-        '--abort-on-first-failure', dest='abort_on_first_failure',
+        '--abort-on-first-failure', "--fail-fast", dest='abort_on_first_failure',
         action='store_true', default=True,
         help=('Specifies whether to halt test suite execution on first test error.'
               ' Default: true.'))
     parser.add_argument(
-        '--no-abort-on-first-failure', dest='abort_on_first_failure',
+        '--no-abort-on-first-failure', "--no-fail-fast", dest='abort_on_first_failure',
         action='store_false',
         help=('If set, the whole test suite will run to completion independent of'
               ' earlier errors.'))