diff --git a/.github/scripts/tests/test_validate_unity_test_results.py b/.github/scripts/tests/test_validate_unity_test_results.py new file mode 100644 index 00000000..8e81aba9 --- /dev/null +++ b/.github/scripts/tests/test_validate_unity_test_results.py @@ -0,0 +1,259 @@ +import contextlib +import io +import sys +import tempfile +import unittest +import xml.etree.ElementTree as ElementTree +from pathlib import Path + + +SCRIPT_DIRECTORY = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(SCRIPT_DIRECTORY)) + +import validate_unity_test_results as validator + + +REQUIRED_TESTS = ( + "Backtrace.Unity.Tests.Runtime.AndroidNativeInitializationTests." + "RejectedBeforeNativeBridgeDoesNotRollback", + "Backtrace.Unity.Tests.Runtime.AndroidNativeInitializationTests." + "NativeBridgeFalseResultRollsBackPartialState", +) + + +class UnityTestResultValidatorTests(unittest.TestCase): + def setUp(self): + self._temporary_directory = tempfile.TemporaryDirectory() + self.results_root = Path(self._temporary_directory.name) + + def tearDown(self): + self._temporary_directory.cleanup() + + def _write_result( + self, + relative_path="playmode-results.xml", + total=None, + failed=0, + tests=REQUIRED_TESTS, + counter_overrides=None, + root_result="Passed", + ): + if total is None: + total = len(tests) + + counters = { + "testcasecount": str(total), + "total": str(total), + "passed": str(max(total - failed, 0)), + "failed": str(failed), + "inconclusive": "0", + "skipped": "0", + "result": root_result, + } + if counter_overrides: + for counter, value in counter_overrides.items(): + if value is None: + counters.pop(counter, None) + else: + counters[counter] = value + + root = ElementTree.Element("test-run", counters) + suite = ElementTree.SubElement(root, "test-suite") + for test in tests: + if isinstance(test, tuple): + full_name, result = test + else: + full_name, result = test, "Passed" + ElementTree.SubElement( + suite, + "test-case", + {"fullname": full_name, "result": result}, + ) + + result_file = self.results_root / relative_path + result_file.parent.mkdir(parents=True, exist_ok=True) + ElementTree.ElementTree(root).write( + str(result_file), encoding="utf-8", xml_declaration=True + ) + return result_file + + def _run(self, mode="playmode", required_tests=REQUIRED_TESTS): + arguments = [ + "--results-root", + str(self.results_root), + "--mode", + mode, + ] + for required_test in required_tests: + arguments.extend(("--require-test", required_test)) + + stdout = io.StringIO() + stderr = io.StringIO() + with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): + result = validator.main(arguments) + return result, stdout.getvalue(), stderr.getvalue() + + def test_valid_result_passes(self): + self._write_result() + + result, stdout, stderr = self._run() + + self.assertEqual(0, result) + self.assertIn("2 test(s), 0 failed", stdout) + self.assertEqual("", stderr) + + def test_no_result_files_fails(self): + result, _, stderr = self._run() + + self.assertEqual(1, result) + self.assertIn("no playmode-results.xml files", stderr) + + def test_malformed_xml_fails(self): + result_file = self.results_root / "playmode-results.xml" + result_file.write_text("", encoding="utf-8") + + result, _, stderr = self._run() + + self.assertEqual(1, result) + self.assertIn("could not parse Unity test results", stderr) + + def test_wrong_root_element_fails(self): + result_file = self.results_root / "playmode-results.xml" + result_file.write_text("", encoding="utf-8") + + result, _, stderr = self._run() + + self.assertEqual(1, result) + self.assertIn("expected a root element", stderr) + + def test_missing_or_malformed_counters_fail(self): + invalid_values = (None, "", "1.5", "+1", "-1", " 1") + for counter in validator.COUNTER_ATTRIBUTES: + for invalid_value in invalid_values: + with self.subTest(counter=counter, invalid_value=invalid_value): + overrides = {counter: invalid_value} + self._write_result(counter_overrides=overrides) + + result, _, stderr = self._run() + + self.assertEqual(1, result) + self.assertIn("{0!r} counter".format(counter), stderr) + + def test_zero_total_is_not_masked_by_another_file(self): + self._write_result("valid/playmode-results.xml") + self._write_result("empty/playmode-results.xml", total=0, tests=()) + + result, _, stderr = self._run() + + self.assertEqual(1, result) + self.assertIn("without discovering any tests", stderr) + + def test_failed_result_is_not_masked_by_another_file(self): + self._write_result("valid/playmode-results.xml") + self._write_result( + "failed/playmode-results.xml", + total=3, + failed=1, + tests=REQUIRED_TESTS + (("Example.FailedTest", "Failed"),), + ) + + result, _, stderr = self._run() + + self.assertEqual(1, result) + self.assertIn("reported 1 failed test", stderr) + + def test_nonpassed_root_result_fails(self): + self._write_result(root_result="Failed") + + result, _, stderr = self._run() + + self.assertEqual(1, result) + self.assertIn("test-run result was not Passed", stderr) + + def test_zero_testcasecount_fails(self): + self._write_result(counter_overrides={"testcasecount": "0"}) + + result, _, stderr = self._run() + + self.assertEqual(1, result) + self.assertIn("reported no test cases", stderr) + + def test_zero_passed_count_fails(self): + self._write_result(counter_overrides={"passed": "0"}) + + result, _, stderr = self._run() + + self.assertEqual(1, result) + self.assertIn("reported no passed tests", stderr) + + def test_missing_required_test_fails(self): + self._write_result(tests=REQUIRED_TESTS[:1]) + + result, _, stderr = self._run() + + self.assertEqual(1, result) + self.assertIn("required test was not discovered", stderr) + self.assertIn(REQUIRED_TESTS[1], stderr) + + def test_skipped_required_test_fails(self): + self._write_result( + tests=(REQUIRED_TESTS[0], (REQUIRED_TESTS[1], "Skipped")) + ) + + result, _, stderr = self._run() + + self.assertEqual(1, result) + self.assertIn("required test did not pass", stderr) + + def test_nonpassed_duplicate_required_test_fails(self): + self._write_result( + total=3, + tests=( + REQUIRED_TESTS[0], + REQUIRED_TESTS[1], + (REQUIRED_TESTS[1], "Skipped"), + ), + ) + + result, _, stderr = self._run() + + self.assertEqual(1, result) + self.assertIn("required test did not pass", stderr) + self.assertIn("Passed, Skipped", stderr) + + def test_required_tests_cannot_be_combined_across_result_files(self): + self._write_result("first/playmode-results.xml", tests=REQUIRED_TESTS[:1]) + self._write_result("second/playmode-results.xml", tests=REQUIRED_TESTS[1:]) + + result, _, stderr = self._run() + + self.assertEqual(1, result) + self.assertEqual(2, stderr.count("required test was not discovered")) + + def test_multiple_complete_result_files_pass(self): + self._write_result("first/playmode-results.xml") + self._write_result("second/playmode-results.xml") + + result, stdout, stderr = self._run() + + self.assertEqual(0, result) + self.assertEqual(2, stdout.count("Validated")) + self.assertEqual("", stderr) + + def test_modes_are_validated_independently(self): + self._write_result("playmode-results.xml") + self._write_result("editmode-results.xml", total=0, tests=()) + + playmode_result, _, playmode_stderr = self._run("playmode") + editmode_result, _, editmode_stderr = self._run( + "editmode", required_tests=() + ) + + self.assertEqual(0, playmode_result) + self.assertEqual("", playmode_stderr) + self.assertEqual(1, editmode_result) + self.assertIn("without discovering any tests", editmode_stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/scripts/validate_unity_test_results.py b/.github/scripts/validate_unity_test_results.py new file mode 100644 index 00000000..dfffb4ca --- /dev/null +++ b/.github/scripts/validate_unity_test_results.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 +"""Fail closed when Unity's NUnit result artifacts do not prove tests ran.""" + +import argparse +import re +import sys +import xml.etree.ElementTree as ElementTree +from pathlib import Path + + +COUNTER_ATTRIBUTES = ( + "testcasecount", + "total", + "passed", + "failed", + "inconclusive", + "skipped", +) +INTEGER_PATTERN = re.compile(r"[0-9]+\Z") + + +class ValidationError(Exception): + """Raised when a Unity test-result artifact violates the CI contract.""" + + +def _parse_counter(root, attribute, result_file): + raw_value = root.get(attribute) + if raw_value is None: + raise ValidationError( + "{0}: is missing the {1!r} counter".format( + result_file, attribute + ) + ) + + if INTEGER_PATTERN.fullmatch(raw_value) is None: + raise ValidationError( + "{0}: has a malformed {1!r} counter: {2!r}".format( + result_file, attribute, raw_value + ) + ) + + return int(raw_value) + + +def validate_result_file(result_file, required_tests): + """Validate one NUnit XML file without relying on any other result file.""" + try: + root = ElementTree.parse(str(result_file)).getroot() + except (OSError, ElementTree.ParseError) as error: + raise ValidationError( + "{0}: could not parse Unity test results ({1})".format( + result_file, type(error).__name__ + ) + ) + + if root.tag != "test-run": + raise ValidationError( + "{0}: expected a root element, found <{1}>".format( + result_file, root.tag + ) + ) + + counters = { + attribute: _parse_counter(root, attribute, result_file) + for attribute in COUNTER_ATTRIBUTES + } + + if counters["total"] <= 0: + raise ValidationError( + "{0}: Unity completed without discovering any tests".format(result_file) + ) + + if counters["testcasecount"] <= 0: + raise ValidationError( + "{0}: Unity reported no test cases".format(result_file) + ) + + if counters["passed"] <= 0: + raise ValidationError( + "{0}: Unity reported no passed tests".format(result_file) + ) + + if counters["failed"] > 0: + raise ValidationError( + "{0}: Unity reported {1} failed test(s)".format( + result_file, counters["failed"] + ) + ) + + if root.get("result") != "Passed": + raise ValidationError( + "{0}: Unity test-run result was not Passed: {1!r}".format( + result_file, root.get("result") + ) + ) + + matching_results = {required_test: [] for required_test in required_tests} + for test_case in root.iter("test-case"): + full_name = test_case.get("fullname") + if full_name in matching_results: + matching_results[full_name].append(test_case.get("result")) + + for required_test, results in matching_results.items(): + if not results: + raise ValidationError( + "{0}: required test was not discovered: {1}".format( + result_file, required_test + ) + ) + if any(result != "Passed" for result in results): + raise ValidationError( + "{0}: required test did not pass: {1} (results: {2})".format( + result_file, + required_test, + ", ".join(str(result) for result in results), + ) + ) + + return counters["total"] + + +def discover_result_files(results_root, mode): + """Return only result files for the requested Unity test mode.""" + pattern = "{0}-results.xml".format(mode) + return sorted(path for path in results_root.rglob(pattern) if path.is_file()) + + +def build_argument_parser(): + parser = argparse.ArgumentParser( + description="Validate Unity NUnit XML artifacts without aggregating away failures." + ) + parser.add_argument( + "--results-root", + required=True, + type=Path, + help="Directory containing Unity test-result artifacts.", + ) + parser.add_argument( + "--mode", + required=True, + choices=("playmode", "editmode"), + help="Unity test mode whose result files must be validated.", + ) + parser.add_argument( + "--require-test", + action="append", + default=[], + help="Fully qualified test name that must be present and passed in every result file.", + ) + return parser + + +def main(arguments=None): + options = build_argument_parser().parse_args(arguments) + result_files = discover_result_files(options.results_root, options.mode) + + if not result_files: + print( + "Error: no {0}-results.xml files found under {1}.".format( + options.mode, options.results_root + ), + file=sys.stderr, + ) + return 1 + + errors = [] + validated = [] + for result_file in result_files: + try: + total = validate_result_file(result_file, options.require_test) + validated.append((result_file, total)) + except ValidationError as error: + errors.append(str(error)) + + if errors: + for error in errors: + print("Error: {0}".format(error), file=sys.stderr) + return 1 + + for result_file, total in validated: + print("Validated {0}: {1} test(s), 0 failed.".format(result_file, total)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b7709e4d..beeb18e6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6,7 +6,21 @@ on: pull_request: jobs: + validate-result-checker: + name: Validate Unity test-result checker + runs-on: ubuntu-latest + steps: + - name: Checkout Repo + uses: actions/checkout@v4 + + - name: Run validator unit tests + run: >- + python3 -m unittest discover + --start-directory .github/scripts/tests + --pattern 'test_*.py' + test: + needs: validate-result-checker name: Run Tests in ${{ matrix.testMode }} ${{ matrix.unityVersion }} runs-on: ubuntu-latest strategy: @@ -14,7 +28,6 @@ jobs: matrix: testMode: - playmode - - editmode projectPath: - test-package unityVersion: @@ -43,4 +56,17 @@ jobs: projectPath: ${{ matrix.projectPath }}/ testMode: ${{ matrix.testMode }} unityVersion: ${{ matrix.unityVersion }} + artifactsPath: artifacts coverageOptions: "generateAdditionalMetrics;generateHtmlReport;generateBadgeReport;assemblyFilters:+my.assembly.*" + + - name: Require Unity PlayMode tests + if: ${{ always() }} + run: | + python3 .github/scripts/validate_unity_test_results.py \ + --results-root artifacts \ + --mode "${{ matrix.testMode }}" \ + --require-test 'Backtrace.Unity.Tests.Runtime.AndroidNativeInitializationTests.RejectedBeforeNativeBridgeDoesNotRollback' \ + --require-test 'Backtrace.Unity.Tests.Runtime.AndroidNativeInitializationTests.NativeBridgeFalseResultRollsBackPartialState' \ + --require-test 'Backtrace.Unity.Tests.Runtime.AndroidNativeInitializationTests.NativeBridgeFalseResultRollbackFailureIsContained' \ + --require-test 'Backtrace.Unity.Tests.Runtime.AndroidNativeClientOomTests.OnOomAttemptsTimestampAfterFirstAttributeWriteFails' \ + --require-test 'Backtrace.Unity.Tests.Runtime.BacktraceAttributeTests.SetAttributes_NativeClientFailure_DoesNotInterruptManagedAttributes'