Merge pull request #3255 from verilog-to-routing/reg_test_false_negative

Fix regression testing false negative

Author: AlexandreSinger

run_reg_test.py

@@ -200,7 +200,10 @@ def vtr_command_main(arg_list, prog=None):
     elif tests_run and total_num_func_failures != 0 or total_num_qor_failures != 0:
         print("Error: {} tests failed".format(total_num_func_failures + total_num_qor_failures))
 
-    sys.exit(total_num_func_failures + total_num_qor_failures)
+    # If the QoR parsing script throws an exception, it returns -1.
+    # This could potentially cancel a run failure and result in a false negative.
+    # Absolute value is taken to avoid that.
+    sys.exit(abs(total_num_func_failures) + abs(total_num_qor_failures))
 
 
 def display_qor(reg_test):

vtr_flow/scripts/python_libs/vtr/parse_vtr_task.py

@@ -422,14 +422,6 @@ def check_two_files(
     for (arch, circuit, script_params), _ in first_results.all_metrics().items():
         first_primary_keys.append((arch, circuit, script_params))
 
-    # Ensure that first result file  has all the second result file cases
-    for arch, circuit, script_params in second_primary_keys:
-        if first_results.metrics(arch, circuit, script_params) is None:
-            raise InspectError(
-                "Required case {}/{} missing from {} results: {}".format(
-                    arch, circuit, first_name, first_results_filepath
-                )
-            )
 
     # Warn about any elements in first result file that are not found in second result file
     for arch, circuit, script_params in first_primary_keys:
@@ -444,6 +436,19 @@ def check_two_files(
     for arch, circuit, script_params in second_primary_keys:
         second_metrics = second_results.metrics(arch, circuit, script_params)
         first_metrics = first_results.metrics(arch, circuit, script_params)
+
+        if first_metrics is None:
+            print(
+                "\n{}...[Fail]".format(
+                    "/".join(str((Path(config.config_dir).parent)).split("/")[-3:])
+                )
+            )
+            print("Required case {}/{} missing from {} results: {}".format(
+                    arch, circuit, first_name, first_results_filepath
+                ))
+            num_qor_failures += 1
+            continue
+
         first_fail = True
         for metric in pass_requirements.keys():