diff --git a/python/lib/sift_client/_tests/util/test_test_results_utils.py b/python/lib/sift_client/_tests/util/test_test_results_utils.py index 82bea7c0c..79e0a690a 100644 --- a/python/lib/sift_client/_tests/util/test_test_results_utils.py +++ b/python/lib/sift_client/_tests/util/test_test_results_utils.py @@ -368,17 +368,27 @@ def test_bad_assert(self, report_context, step): parent_step = None substep = None nested_substep = None + nested_substep_2 = None sibling_substep = None with step.substep("Top Level Step", "Should fail") as parent_step_context: parent_step = parent_step_context.current_step with parent_step_context.substep("Parent Step", "Should fail") as substep_context: substep = substep_context.current_step with substep_context.substep( - "Nested Substep", "Has a bad assert" + "Nested Substep", + "Has a bad assert. Pytest util should nominally mark this as fail instead of error.", ) as nested_substep_context: nested_substep = nested_substep_context.current_step nested_substep_context.force_result = True assert False == True + with substep_context.substep( + "Nested Substep 2", + "Has a bad assert and shows assertion errors. Pytest util should mark this as error.", + ) as nested_substep_2_context: + nested_substep_2 = nested_substep_2_context.current_step + nested_substep_2_context.assertion_as_fail_not_error = True + nested_substep_2_context.force_result = True + assert False == True with substep_context.substep( "Sibling Substep", "Should pass" ) as sibling_substep_context: @@ -386,8 +396,10 @@ def test_bad_assert(self, report_context, step): assert parent_step.status == TestStatus.FAILED assert substep.status == TestStatus.FAILED - assert nested_substep.status == TestStatus.ERROR - assert "AssertionError" in nested_substep.error_info.error_message + assert nested_substep.status == TestStatus.FAILED + assert nested_substep.error_info is None + assert nested_substep_2.status == TestStatus.ERROR + assert "AssertionError" in nested_substep_2.error_info.error_message assert sibling_substep.status == TestStatus.PASSED # If this test was successful, mark that at a high level. diff --git a/python/lib/sift_client/resources/__init__.py b/python/lib/sift_client/resources/__init__.py index 5058ac366..cdc651485 100644 --- a/python/lib/sift_client/resources/__init__.py +++ b/python/lib/sift_client/resources/__init__.py @@ -176,6 +176,14 @@ async def main(): FileAttachmentsAPI, ) +import sys + +if "pytest" in sys.modules: + # These are not test classes, so we need to set __test__ to False to avoid pytest warnings. + # Do this here because for some reason our docs generation doesn't like it when done in the classes themselves. + TestResultsAPI.__test__ = False # type: ignore + TestResultsAPIAsync.__test__ = False # type: ignore + __all__ = [ "AssetsAPI", "AssetsAPIAsync", diff --git a/python/lib/sift_client/sift_types/__init__.py b/python/lib/sift_client/sift_types/__init__.py index b55717c60..73d219704 100644 --- a/python/lib/sift_client/sift_types/__init__.py +++ b/python/lib/sift_client/sift_types/__init__.py @@ -129,6 +129,8 @@ ``` """ +import sys + from sift_client.sift_types.asset import Asset, AssetUpdate from sift_client.sift_types.calculated_channel import ( CalculatedChannel, @@ -164,6 +166,7 @@ TestMeasurement, TestMeasurementCreate, TestMeasurementType, + TestMeasurementUpdate, TestReport, TestReportCreate, TestReportUpdate, @@ -173,6 +176,21 @@ TestStepType, ) +if "pytest" in sys.modules: + # These are not test classes, so we need to set __test__ to False to avoid pytest warnings. + # Do this here because for some reason our docs generation doesn't like it when done in the classes themselves. + TestStepType.__test__ = False # type: ignore + TestMeasurementType.__test__ = False # type: ignore + TestMeasurement.__test__ = False # type: ignore + TestMeasurementCreate.__test__ = False # type: ignore + TestMeasurementUpdate.__test__ = False # type: ignore + TestStatus.__test__ = False # type: ignore + TestStep.__test__ = False # type: ignore + TestStepCreate.__test__ = False # type: ignore + TestReport.__test__ = False # type: ignore + TestReportCreate.__test__ = False # type: ignore + TestReportUpdate.__test__ = False # type: ignore + __all__ = [ "Asset", "AssetUpdate", diff --git a/python/lib/sift_client/sift_types/_base.py b/python/lib/sift_client/sift_types/_base.py index c8c0f8fae..3e9a214e9 100644 --- a/python/lib/sift_client/sift_types/_base.py +++ b/python/lib/sift_client/sift_types/_base.py @@ -57,7 +57,7 @@ def _update(self, other: BaseType[ProtoT, SelfT]) -> BaseType[ProtoT, SelfT]: """Update this instance with the values from another instance.""" # This bypasses the frozen status of the model for key in other.__class__.model_fields.keys(): - if key in self.model_fields: + if key in self.__class__.model_fields: self.__dict__.update({key: getattr(other, key)}) # Make sure we also update the proto since it is excluded @@ -68,7 +68,7 @@ def _update(self, other: BaseType[ProtoT, SelfT]) -> BaseType[ProtoT, SelfT]: @model_validator(mode="after") def _validate_timezones(self): """Validate datetime fiels have timezone information.""" - for field_name in self.model_fields.keys(): + for field_name in self.__class__.model_fields.keys(): val = getattr(self, field_name) if isinstance(val, datetime) and val.tzinfo is None: raise ValueError(f"{field_name} must have timezone information") diff --git a/python/lib/sift_client/util/test_results/context_manager.py b/python/lib/sift_client/util/test_results/context_manager.py index da7de8c65..937f21971 100644 --- a/python/lib/sift_client/util/test_results/context_manager.py +++ b/python/lib/sift_client/util/test_results/context_manager.py @@ -94,9 +94,16 @@ def __exit__(self, exc_type, exc_value, traceback): self.report.update(update) return True - def new_step(self, name: str, description: str | None = None) -> NewStep: + def new_step( + self, name: str, description: str | None = None, assertion_as_fail_not_error: bool = True + ) -> NewStep: """Alias to return a new step context manager from this report context. Use create_step for actually creating a TestStep in the current context.""" - return NewStep(self, name=name, description=description) + return NewStep( + self, + name=name, + description=description, + assertion_as_fail_not_error=assertion_as_fail_not_error, + ) def get_next_step_path(self) -> str: """Get the next step path for the current depth.""" @@ -191,6 +198,7 @@ class NewStep(AbstractContextManager): report_context: ReportContext client: SiftClient + assertion_as_fail_not_error: bool = True current_step: TestStep | None = None def __init__( @@ -198,6 +206,7 @@ def __init__( report_context: ReportContext, name: str, description: str | None = None, + assertion_as_fail_not_error: bool = True, ): """Initialize a new step context. @@ -205,10 +214,12 @@ def __init__( report_context: The report context to create the step in. name: The name of the step. description: The description of the step. + assertion_as_fail_not_error: Mark steps with assertion errors as failed instead of error+traceback (some users want assertions to work as simple failures especially when using pytest). """ self.report_context = report_context self.client = report_context.report.client self.current_step = self.report_context.create_step(name, description) + self.assertion_as_fail_not_error = assertion_as_fail_not_error def __enter__(self): """Enter the context manager to create a new step. @@ -233,15 +244,19 @@ def update_step_from_result( returns: The false if step failed or errored, true otherwise. """ error_info = None - if exc: - stack = traceback.format_exception(exc, exc_value, tb) # type: ignore - stack = [stack[0], *stack[-10:]] if len(stack) > 10 else stack - trace = "".join(stack) - error_info = ErrorInfo( - error_code=1, - error_message=trace, - ) assert self.current_step is not None + if exc: + if isinstance(exc_value, AssertionError) and not self.assertion_as_fail_not_error: + # If we're not showing assertion errors (i.e. pytest), mark step as failed but don't set error info. + self.report_context.record_step_outcome(False, self.current_step) + else: + stack = traceback.format_exception(exc, exc_value, tb) # type: ignore + stack = [stack[0], *stack[-10:]] if len(stack) > 10 else stack + trace = "".join(stack) + error_info = ErrorInfo( + error_code=1, + error_message=trace, + ) # Resolve the status of this step (i.e. fail if children failed) and propagate the result to the parent step. result = self.report_context.resolve_and_propagate_step_result( @@ -272,6 +287,7 @@ def __exit__(self, exc, exc_value, tb): self.report_context.exit_step(self.current_step) # Test only attribute (hence not public class variable) + # This changes the result after the status and error info are set. if hasattr(self, "force_result"): result = self.force_result @@ -421,4 +437,8 @@ def report_outcome(self, name: str, result: bool, reason: str | None = None) -> def substep(self, name: str, description: str | None = None) -> NewStep: """Alias to return a new step context manager from the current step. The ReportContext will manage nesting of steps.""" - return self.report_context.new_step(name=name, description=description) + return self.report_context.new_step( + name=name, + description=description, + assertion_as_fail_not_error=self.assertion_as_fail_not_error, + ) diff --git a/python/lib/sift_client/util/test_results/pytest_util.py b/python/lib/sift_client/util/test_results/pytest_util.py index 90a30ced3..c2bd3f9bc 100644 --- a/python/lib/sift_client/util/test_results/pytest_util.py +++ b/python/lib/sift_client/util/test_results/pytest_util.py @@ -63,7 +63,9 @@ def _step_impl( ) -> Generator[NewStep | None, None, None]: name = str(request.node.name) existing_docstring = request.node.obj.__doc__ or None - with report_context.new_step(name=name, description=existing_docstring) as new_step: + with report_context.new_step( + name=name, description=existing_docstring, assertion_as_fail_not_error=False + ) as new_step: yield new_step if hasattr(request.node, "rep_call") and request.node.rep_call.excinfo: new_step.update_step_from_result(