python(feat): TestResults: Have pytest assertion errors be marked as …

python/lib/sift_client/_tests/util/test_test_results_utils.py

@@ -368,26 +368,38 @@ def test_bad_assert(self, report_context, step):
         parent_step = None
         substep = None
         nested_substep = None
+        nested_substep_2 = None
         sibling_substep = None
         with step.substep("Top Level Step", "Should fail") as parent_step_context:
             parent_step = parent_step_context.current_step
             with parent_step_context.substep("Parent Step", "Should fail") as substep_context:
                 substep = substep_context.current_step
                 with substep_context.substep(
-                    "Nested Substep", "Has a bad assert"
+                    "Nested Substep",
+                    "Has a bad assert. Pytest util should nominally mark this as fail instead of error.",
                 ) as nested_substep_context:
                     nested_substep = nested_substep_context.current_step
                     nested_substep_context.force_result = True
                     assert False == True
+                with substep_context.substep(
+                    "Nested Substep 2",
+                    "Has a bad assert and shows assertion errors. Pytest util should mark this as error.",
+                ) as nested_substep_2_context:
+                    nested_substep_2 = nested_substep_2_context.current_step
+                    nested_substep_2_context.assertion_as_fail_not_error = True
+                    nested_substep_2_context.force_result = True
+                    assert False == True
                 with substep_context.substep(
                     "Sibling Substep", "Should pass"
                 ) as sibling_substep_context:
                     sibling_substep = sibling_substep_context.current_step
 
         assert parent_step.status == TestStatus.FAILED
         assert substep.status == TestStatus.FAILED
-        assert nested_substep.status == TestStatus.ERROR
-        assert "AssertionError" in nested_substep.error_info.error_message
+        assert nested_substep.status == TestStatus.FAILED
+        assert nested_substep.error_info is None
+        assert nested_substep_2.status == TestStatus.ERROR
+        assert "AssertionError" in nested_substep_2.error_info.error_message
         assert sibling_substep.status == TestStatus.PASSED
 
         # If this test was successful, mark that at a high level.

python/lib/sift_client/resources/__init__.py

@@ -176,6 +176,14 @@ async def main():
     FileAttachmentsAPI,
 )
 
+import sys
+
+if "pytest" in sys.modules:
+    # These are not test classes, so we need to set __test__ to False to avoid pytest warnings.
+    # Do this here because for some reason our docs generation doesn't like it when done in the classes themselves.
+    TestResultsAPI.__test__ = False  # type: ignore
+    TestResultsAPIAsync.__test__ = False  # type: ignore
+
 __all__ = [
     "AssetsAPI",
     "AssetsAPIAsync",

python/lib/sift_client/sift_types/__init__.py

@@ -129,6 +129,8 @@
 ```
 """
 
+import sys
+
 from sift_client.sift_types.asset import Asset, AssetUpdate
 from sift_client.sift_types.calculated_channel import (
     CalculatedChannel,
@@ -164,6 +166,7 @@
     TestMeasurement,
     TestMeasurementCreate,
     TestMeasurementType,
+    TestMeasurementUpdate,
     TestReport,
     TestReportCreate,
     TestReportUpdate,
@@ -173,6 +176,21 @@
     TestStepType,
 )
 
+if "pytest" in sys.modules:
+    # These are not test classes, so we need to set __test__ to False to avoid pytest warnings.
+    # Do this here because for some reason our docs generation doesn't like it when done in the classes themselves.
+    TestStepType.__test__ = False  # type: ignore
+    TestMeasurementType.__test__ = False  # type: ignore
+    TestMeasurement.__test__ = False  # type: ignore
+    TestMeasurementCreate.__test__ = False  # type: ignore
+    TestMeasurementUpdate.__test__ = False  # type: ignore
+    TestStatus.__test__ = False  # type: ignore
+    TestStep.__test__ = False  # type: ignore
+    TestStepCreate.__test__ = False  # type: ignore
+    TestReport.__test__ = False  # type: ignore
+    TestReportCreate.__test__ = False  # type: ignore
+    TestReportUpdate.__test__ = False  # type: ignore
+
 __all__ = [
     "Asset",
     "AssetUpdate",

python/lib/sift_client/sift_types/_base.py

@@ -57,7 +57,7 @@ def _update(self, other: BaseType[ProtoT, SelfT]) -> BaseType[ProtoT, SelfT]:
         """Update this instance with the values from another instance."""
         # This bypasses the frozen status of the model
         for key in other.__class__.model_fields.keys():
-            if key in self.model_fields:
+            if key in self.__class__.model_fields:
                 self.__dict__.update({key: getattr(other, key)})
 
         # Make sure we also update the proto since it is excluded
@@ -68,7 +68,7 @@ def _update(self, other: BaseType[ProtoT, SelfT]) -> BaseType[ProtoT, SelfT]:
     @model_validator(mode="after")
     def _validate_timezones(self):
         """Validate datetime fiels have timezone information."""
-        for field_name in self.model_fields.keys():
+        for field_name in self.__class__.model_fields.keys():
             val = getattr(self, field_name)
             if isinstance(val, datetime) and val.tzinfo is None:
                 raise ValueError(f"{field_name} must have timezone information")

python/lib/sift_client/util/test_results/context_manager.py

@@ -94,9 +94,16 @@ def __exit__(self, exc_type, exc_value, traceback):
         self.report.update(update)
         return True
 
-    def new_step(self, name: str, description: str | None = None) -> NewStep:
+    def new_step(
+        self, name: str, description: str | None = None, assertion_as_fail_not_error: bool = True
+    ) -> NewStep:
         """Alias to return a new step context manager from this report context. Use create_step for actually creating a TestStep in the current context."""
-        return NewStep(self, name=name, description=description)
+        return NewStep(
+            self,
+            name=name,
+            description=description,
+            assertion_as_fail_not_error=assertion_as_fail_not_error,
+        )
 
     def get_next_step_path(self) -> str:
         """Get the next step path for the current depth."""
@@ -191,24 +198,28 @@ class NewStep(AbstractContextManager):
 
     report_context: ReportContext
     client: SiftClient
+    assertion_as_fail_not_error: bool = True
     current_step: TestStep | None = None
 
     def __init__(
         self,
         report_context: ReportContext,
         name: str,
         description: str | None = None,
+        assertion_as_fail_not_error: bool = True,
     ):
         """Initialize a new step context.
 
         Args:
             report_context: The report context to create the step in.
             name: The name of the step.
             description: The description of the step.
+            assertion_as_fail_not_error: Mark steps with assertion errors as failed instead of error+traceback (some users want assertions to work as simple failures especially when using pytest).
         """
         self.report_context = report_context
         self.client = report_context.report.client
         self.current_step = self.report_context.create_step(name, description)
+        self.assertion_as_fail_not_error = assertion_as_fail_not_error
 
     def __enter__(self):
         """Enter the context manager to create a new step.
@@ -233,15 +244,19 @@ def update_step_from_result(
         returns: The false if step failed or errored, true otherwise.
         """
         error_info = None
-        if exc:
-            stack = traceback.format_exception(exc, exc_value, tb)  # type: ignore
-            stack = [stack[0], *stack[-10:]] if len(stack) > 10 else stack
-            trace = "".join(stack)
-            error_info = ErrorInfo(
-                error_code=1,
-                error_message=trace,
-            )
         assert self.current_step is not None
+        if exc:
+            if isinstance(exc_value, AssertionError) and not self.assertion_as_fail_not_error:
+                # If we're not showing assertion errors (i.e. pytest), mark step as failed but don't set error info.
+                self.report_context.record_step_outcome(False, self.current_step)
+            else:
+                stack = traceback.format_exception(exc, exc_value, tb)  # type: ignore
+                stack = [stack[0], *stack[-10:]] if len(stack) > 10 else stack
+                trace = "".join(stack)
+                error_info = ErrorInfo(
+                    error_code=1,
+                    error_message=trace,
+                )
 
         # Resolve the status of this step (i.e. fail if children failed) and propagate the result to the parent step.
         result = self.report_context.resolve_and_propagate_step_result(
@@ -272,6 +287,7 @@ def __exit__(self, exc, exc_value, tb):
         self.report_context.exit_step(self.current_step)
 
         # Test only attribute (hence not public class variable)
+        # This changes the result after the status and error info are set.
         if hasattr(self, "force_result"):
             result = self.force_result
 
@@ -421,4 +437,8 @@ def report_outcome(self, name: str, result: bool, reason: str | None = None) ->
 
     def substep(self, name: str, description: str | None = None) -> NewStep:
         """Alias to return a new step context manager from the current step. The ReportContext will manage nesting of steps."""
-        return self.report_context.new_step(name=name, description=description)
+        return self.report_context.new_step(
+            name=name,
+            description=description,
+            assertion_as_fail_not_error=self.assertion_as_fail_not_error,
+        )

python/lib/sift_client/util/test_results/pytest_util.py

@@ -63,7 +63,9 @@ def _step_impl(
 ) -> Generator[NewStep | None, None, None]:
     name = str(request.node.name)
     existing_docstring = request.node.obj.__doc__ or None
-    with report_context.new_step(name=name, description=existing_docstring) as new_step:
+    with report_context.new_step(
+        name=name, description=existing_docstring, assertion_as_fail_not_error=False
+    ) as new_step:
         yield new_step
         if hasattr(request.node, "rep_call") and request.node.rep_call.excinfo:
             new_step.update_step_from_result(