ref(grouping): Label stacktrace rule hints with rule origin (#103266)

In grouping info, we tell people what rule has been used to mark a frame in- or out-of-app, or to ignore or unignore it. However, we give no indication if the rule is ours or one of theirs. (This is useful information when trying to debug grouping.)

This PR fixes that, by keeping a set of the string representations of any custom rules the user has created, comparing that set to the rules mentioned in grouping info hints, and modifying those hints accordingly. Thus `marked in-app by stacktrace rule (...)` becomes either `marked in-app by custom stacktrace rule (...)` (if the rule is in the set) or `marked in-app by built-in stacktrace rule (...)` (if it's not).

Author: lobsterkatie

src/sentry/grouping/enhancer/__init__.py

@@ -3,6 +3,7 @@
 import base64
 import logging
 import os
+import re
 import zlib
 from collections.abc import Sequence
 from dataclasses import dataclass
@@ -43,6 +44,8 @@
 # base64 strings never contain '#')
 BASE64_ENHANCEMENTS_DELIMITER = b"#"
 
+HINT_STACKTRACE_RULE_REGEX = re.compile(r"stack trace rule \((.+)\)$")
+
 VALID_PROFILING_MATCHER_PREFIXES = (
     "stack.abs_path",
     "path",  # stack.abs_path alias
@@ -61,6 +64,7 @@ class EnhancementsConfigData:
     # Note: This is not an actual config, just a container to make it easier to pass data between
     # functions while loading a config.
     rules: list[EnhancementRule]
+    rule_strings: list[str]
     rust_enhancements: RustEnhancements
     version: int | None = None
     bases: list[str] | None = None
@@ -170,12 +174,26 @@ def _can_use_hint(
     return True
 
 
+def _add_rule_source_to_hint(hint: str | None, custom_rules: set[str]) -> str | None:
+    """Add 'custom' or 'built-in' to the rule description in the given hint (if any)."""
+    if not hint:
+        return None
+
+    def _add_type_to_rule(rule_regex_match: re.Match) -> str:
+        rule_str = rule_regex_match.group(1)
+        rule_type = "custom" if rule_str in custom_rules else "built-in"
+        return f"{rule_type} stack trace rule ({rule_str})"
+
+    return HINT_STACKTRACE_RULE_REGEX.sub(_add_type_to_rule, hint)
+
+
 def _get_hint_for_frame(
     variant_name: str,
     frame: dict[str, Any],
     frame_component: FrameGroupingComponent,
     rust_frame: RustFrame,
     desired_hint_type: Literal["in-app", "contributes"],
+    custom_rules: set[str],
 ) -> str | None:
     """
     Determine a hint to use for the frame, handling special-casing and precedence.
@@ -210,7 +228,10 @@ def _get_hint_for_frame(
     if variant_name == "app" and rust_hint_type == "in-app" and rust_in_app == client_in_app:
         can_use_rust_hint = False
 
-    return rust_hint if can_use_rust_hint else incoming_hint if can_use_incoming_hint else None
+    raw_hint = rust_hint if can_use_rust_hint else incoming_hint if can_use_incoming_hint else None
+
+    # Add 'custom' or 'built-in' to any stacktrace rule description as appropriate
+    return _add_rule_source_to_hint(raw_hint, custom_rules)
 
 
 def _split_rules(
@@ -243,15 +264,22 @@ def _split_rules(
         if rule is not None  # mypy appeasment
     ]
 
-    classifier_rules_text = "\n".join(rule.text for rule in classifier_rules)
-    contributes_rules_text = "\n".join(rule.text for rule in contributes_rules)
+    classifier_rule_strings = [rule.text for rule in classifier_rules]
+    contributes_rule_strings = [rule.text for rule in contributes_rules]
+
+    classifier_rules_text = "\n".join(classifier_rule_strings)
+    contributes_rules_text = "\n".join(contributes_rule_strings)
 
     classifier_rust_enhancements = _get_rust_enhancements("config_string", classifier_rules_text)
     contributes_rust_enhancements = _get_rust_enhancements("config_string", contributes_rules_text)
 
     return (
-        EnhancementsConfigData(classifier_rules, classifier_rust_enhancements),
-        EnhancementsConfigData(contributes_rules, contributes_rust_enhancements),
+        EnhancementsConfigData(
+            classifier_rules, classifier_rule_strings, classifier_rust_enhancements
+        ),
+        EnhancementsConfigData(
+            contributes_rules, contributes_rule_strings, contributes_rust_enhancements
+        ),
     )
 
 
@@ -363,6 +391,12 @@ def __init__(
             self.bases, contributes_config.rust_enhancements, type="contributes"
         )
 
+        # We store the rule strings individually in a set so it's quick to test if a given rule
+        # mentioned in a hint is custom or built-in
+        self.custom_rule_strings = set(
+            classifier_config.rule_strings + contributes_config.rule_strings
+        )
+
     def apply_category_and_updated_in_app_to_frames(
         self,
         frames: Sequence[dict[str, Any]],
@@ -458,13 +492,23 @@ def assemble_stacktrace_component(
 
             in_app_hint = (
                 _get_hint_for_frame(
-                    variant_name, frame, frame_component, in_app_rust_frame, "in-app"
+                    variant_name,
+                    frame,
+                    frame_component,
+                    in_app_rust_frame,
+                    "in-app",
+                    self.custom_rule_strings,
                 )
                 if variant_name == "app"
                 else None  # In-app hints don't apply to the system stacktrace
             )
             contributes_hint = _get_hint_for_frame(
-                variant_name, frame, frame_component, contributes_rust_frame, "contributes"
+                variant_name,
+                frame,
+                frame_component,
+                contributes_rust_frame,
+                "contributes",
+                self.custom_rule_strings,
             )
             hint = _combine_hints(variant_name, frame_component, in_app_hint, contributes_hint)
 
@@ -524,7 +568,9 @@ def _get_config_from_base64_bytes(cls, bytes_str: bytes) -> EnhancementsConfigDa
         except (LookupError, AttributeError, TypeError, ValueError) as e:
             raise ValueError("invalid stack trace rule config: %s" % e)
 
-        return EnhancementsConfigData(rules, rust_enhancements, version, bases)
+        return EnhancementsConfigData(
+            rules, [rule.text for rule in rules], rust_enhancements, version, bases
+        )
 
     @classmethod
     def from_base64_string(

tests/sentry/grouping/enhancements/test_hints.py

@@ -308,7 +308,14 @@ def test_get_hint_for_frame(
     rust_frame = DummyRustFrame(hint=rust_hint)
 
     assert (
-        _get_hint_for_frame(variant_name, frame, frame_component, rust_frame, desired_hint_type)  # type: ignore[arg-type]
+        _get_hint_for_frame(
+            variant_name,
+            frame,
+            frame_component,
+            rust_frame,  # type: ignore[arg-type] # rust frame mock fails typecheck
+            desired_hint_type,
+            set(),
+        )
         == expected_result
     )
 
@@ -343,3 +350,39 @@ def test_combining_hints(
         _combine_hints(variant_name, frame_component, in_app_hint, contributes_hint)
         == expected_result
     )
+
+
+def test_adds_rule_source_to_stacktrace_rule_hints() -> None:
+    frame = {"in_app": True}
+    frame_component = FrameGroupingComponent(in_app=True, values=[])
+    custom_rules = {"function:roll_over +app"}
+
+    built_in_rule_rust_frame = DummyRustFrame(
+        hint="marked in-app by stack trace rule (function:shake +app)"
+    )
+    custom_rule_rust_frame = DummyRustFrame(
+        hint="marked in-app by stack trace rule (function:roll_over +app)"
+    )
+
+    assert (
+        _get_hint_for_frame(
+            "app",
+            frame,
+            frame_component,
+            built_in_rule_rust_frame,  # type: ignore[arg-type] # rust frame mock fails typecheck
+            "in-app",
+            custom_rules,
+        )
+        == "marked in-app by built-in stack trace rule (function:shake +app)"
+    )
+    assert (
+        _get_hint_for_frame(
+            "app",
+            frame,
+            frame_component,
+            custom_rule_rust_frame,  # type: ignore[arg-type] # rust frame mock fails typecheck
+            "in-app",
+            custom_rules,
+        )
+        == "marked in-app by custom stack trace rule (function:roll_over +app)"
+    )

tests/sentry/grouping/snapshots/grouphash_metadata/test_metadata_from_variants/newstyle@2023_01_11/block_invoke.pysnap

@@ -32,6 +32,6 @@ contributing variants:
             frame* (marked in-app by the client)
               function*
                 "__46+[FudgeGlobalHandler setupGlobalHandlersIfNeeded]_block_invoke_2"
-            frame* (marked in-app by stack trace rule (family:native package:**/Containers/Bundle/Application/** +app))
+            frame* (marked in-app by built-in stack trace rule (family:native package:**/Containers/Bundle/Application/** +app))
               function*
                 "__99+[Something else]_block_invoke_2"

tests/sentry/grouping/snapshots/grouphash_metadata/test_metadata_from_variants/newstyle@2023_01_11/cocoa_dispatch_client_callout.pysnap

@@ -29,16 +29,16 @@ contributing variants:
       app*
         threads*
           stacktrace*
-            frame* (marked in-app by stack trace rule (family:native package:**/Containers/Bundle/Application/** +app))
+            frame* (marked in-app by built-in stack trace rule (family:native package:**/Containers/Bundle/Application/** +app))
               function*
                 "unicorn"
             frame* (marked in-app by the client)
               function*
                 "__46+[FudgeGlobalHandler setupGlobalHandlersIfNeeded]_block_invoke_2"
-            frame* (marked in-app by stack trace rule (family:native package:**/Containers/Bundle/Application/** +app))
+            frame* (marked in-app by built-in stack trace rule (family:native package:**/Containers/Bundle/Application/** +app))
               function*
                 "FudgeLogTaggedError"
-            frame* (marked in-app by stack trace rule (family:native package:**/Containers/Bundle/Application/** +app))
+            frame* (marked in-app by built-in stack trace rule (family:native package:**/Containers/Bundle/Application/** +app))
               function*
                 "SentrySetupInteractor.setupSentry"
   system*

tests/sentry/grouping/snapshots/grouphash_metadata/test_metadata_from_variants/newstyle@2023_01_11/contributing_system_and_app_frames.pysnap

@@ -29,7 +29,7 @@ contributing variants:
       app*
         exception*
           stacktrace*
-            frame* (marked in-app by stack trace rule (function:playFetch +app))
+            frame* (marked in-app by custom stack trace rule (function:playFetch +app))
               filename*
                 "dogpark.js"
               function*

tests/sentry/grouping/snapshots/grouphash_metadata/test_metadata_from_variants/newstyle@2023_01_11/stacktrace_rust.pysnap

@@ -29,7 +29,7 @@ contributing variants:
       app*
         exception*
           stacktrace*
-            frame* (marked in-app by stack trace rule (family:native function:log_demo::* +app))
+            frame* (marked in-app by custom stack trace rule (family:native function:log_demo::* +app))
               function*
                 "log_demo::main"
           type*

tests/sentry/grouping/snapshots/grouphash_metadata/test_metadata_from_variants/newstyle@2023_01_11/stacktrace_rust2.pysnap

@@ -29,7 +29,7 @@ contributing variants:
       app*
         exception*
           stacktrace*
-            frame* (marked in-app by stack trace rule (family:native function:log_demo::* +app))
+            frame* (marked in-app by custom stack trace rule (family:native function:log_demo::* +app))
               function*
                 "log_demo::main"
           type*

tests/sentry/grouping/snapshots/grouphash_metadata/test_metadata_from_variants/newstyle@2023_01_11/unreal_assert_mac.pysnap

@@ -29,79 +29,79 @@ contributing variants:
       app*
         exception*
           stacktrace*
-            frame* (marked in-app by stack trace rule (family:native function:FDebug::CheckVerifyFailedImpl* v+app -app ^-app))
+            frame* (marked in-app by built-in stack trace rule (family:native function:FDebug::CheckVerifyFailedImpl* v+app -app ^-app))
               function*
                 "-[FCocoaGameThread main]"
-            frame* (marked in-app by stack trace rule (family:native function:FDebug::CheckVerifyFailedImpl* v+app -app ^-app))
+            frame* (marked in-app by built-in stack trace rule (family:native function:FDebug::CheckVerifyFailedImpl* v+app -app ^-app))
               function*
                 "-[UEAppDelegate runGameThread:]"
-            frame* (marked in-app by stack trace rule (family:native function:FDebug::CheckVerifyFailedImpl* v+app -app ^-app))
+            frame* (marked in-app by built-in stack trace rule (family:native function:FDebug::CheckVerifyFailedImpl* v+app -app ^-app))
               function*
                 "GuardedMain"
-            frame* (marked in-app by stack trace rule (family:native function:FDebug::CheckVerifyFailedImpl* v+app -app ^-app))
+            frame* (marked in-app by built-in stack trace rule (family:native function:FDebug::CheckVerifyFailedImpl* v+app -app ^-app))
               function*
                 "FEngineLoop::Tick"
-            frame* (marked in-app by stack trace rule (family:native function:FDebug::CheckVerifyFailedImpl* v+app -app ^-app))
+            frame* (marked in-app by built-in stack trace rule (family:native function:FDebug::CheckVerifyFailedImpl* v+app -app ^-app))
               function*
                 "FSlateApplication::Tick"
-            frame* (marked in-app by stack trace rule (family:native function:FDebug::CheckVerifyFailedImpl* v+app -app ^-app))
+            frame* (marked in-app by built-in stack trace rule (family:native function:FDebug::CheckVerifyFailedImpl* v+app -app ^-app))
               function*
                 "FSlateApplication::TickPlatform"
-            frame* (marked in-app by stack trace rule (family:native function:FDebug::CheckVerifyFailedImpl* v+app -app ^-app))
+            frame* (marked in-app by built-in stack trace rule (family:native function:FDebug::CheckVerifyFailedImpl* v+app -app ^-app))
               function*
                 "FMacApplication::ProcessDeferredEvents"
-            frame* (marked in-app by stack trace rule (family:native function:FDebug::CheckVerifyFailedImpl* v+app -app ^-app))
+            frame* (marked in-app by built-in stack trace rule (family:native function:FDebug::CheckVerifyFailedImpl* v+app -app ^-app))
               function*
                 "FMacApplication::ProcessEvent"
-            frame* (marked in-app by stack trace rule (family:native function:FDebug::CheckVerifyFailedImpl* v+app -app ^-app))
+            frame* (marked in-app by built-in stack trace rule (family:native function:FDebug::CheckVerifyFailedImpl* v+app -app ^-app))
               function*
                 "FMacApplication::ProcessMouseUpEvent"
-            frame* (marked in-app by stack trace rule (family:native function:FDebug::CheckVerifyFailedImpl* v+app -app ^-app))
+            frame* (marked in-app by built-in stack trace rule (family:native function:FDebug::CheckVerifyFailedImpl* v+app -app ^-app))
               function*
                 "FSlateApplication::OnMouseUp"
-            frame* (marked in-app by stack trace rule (family:native function:FDebug::CheckVerifyFailedImpl* v+app -app ^-app))
+            frame* (marked in-app by built-in stack trace rule (family:native function:FDebug::CheckVerifyFailedImpl* v+app -app ^-app))
               function*
                 "FSlateApplication::ProcessMouseButtonUpEvent"
-            frame* (marked in-app by stack trace rule (family:native function:FDebug::CheckVerifyFailedImpl* v+app -app ^-app))
+            frame* (marked in-app by built-in stack trace rule (family:native function:FDebug::CheckVerifyFailedImpl* v+app -app ^-app))
               function*
                 "FSlateApplication::RoutePointerUpEvent"
-            frame* (marked in-app by stack trace rule (family:native function:FDebug::CheckVerifyFailedImpl* v+app -app ^-app))
+            frame* (marked in-app by built-in stack trace rule (family:native function:FDebug::CheckVerifyFailedImpl* v+app -app ^-app))
               function*
                 "SButton::OnMouseButtonUp"
-            frame* (marked in-app by stack trace rule (family:native function:FDebug::CheckVerifyFailedImpl* v+app -app ^-app))
+            frame* (marked in-app by built-in stack trace rule (family:native function:FDebug::CheckVerifyFailedImpl* v+app -app ^-app))
               function*
                 "SButton::ExecuteOnClick"
-            frame* (marked in-app by stack trace rule (family:native function:FDebug::CheckVerifyFailedImpl* v+app -app ^-app))
+            frame* (marked in-app by built-in stack trace rule (family:native function:FDebug::CheckVerifyFailedImpl* v+app -app ^-app))
               function*
                 "UButton::SlateHandleClicked"
-            frame* (marked in-app by stack trace rule (family:native function:FDebug::CheckVerifyFailedImpl* v+app -app ^-app))
+            frame* (marked in-app by built-in stack trace rule (family:native function:FDebug::CheckVerifyFailedImpl* v+app -app ^-app))
               function*
                 "TMulticastScriptDelegate<T>::ProcessMulticastDelegate<T>"
-            frame* (marked in-app by stack trace rule (family:native function:FDebug::CheckVerifyFailedImpl* v+app -app ^-app))
+            frame* (marked in-app by built-in stack trace rule (family:native function:FDebug::CheckVerifyFailedImpl* v+app -app ^-app))
               function*
                 "UObject::ProcessEvent"
-            frame* (marked in-app by stack trace rule (family:native function:FDebug::CheckVerifyFailedImpl* v+app -app ^-app))
+            frame* (marked in-app by built-in stack trace rule (family:native function:FDebug::CheckVerifyFailedImpl* v+app -app ^-app))
               function*
                 "UFunction::Invoke"
-            frame* (marked in-app by stack trace rule (family:native function:FDebug::CheckVerifyFailedImpl* v+app -app ^-app))
+            frame* (marked in-app by built-in stack trace rule (family:native function:FDebug::CheckVerifyFailedImpl* v+app -app ^-app))
               function*
                 "ProcessLocalScriptFunction"
-            frame* (marked in-app by stack trace rule (family:native function:FDebug::CheckVerifyFailedImpl* v+app -app ^-app))
+            frame* (marked in-app by built-in stack trace rule (family:native function:FDebug::CheckVerifyFailedImpl* v+app -app ^-app))
               function*
                 "ProcessLocalFunction"
-            frame* (marked in-app by stack trace rule (family:native function:FDebug::CheckVerifyFailedImpl* v+app -app ^-app))
+            frame* (marked in-app by built-in stack trace rule (family:native function:FDebug::CheckVerifyFailedImpl* v+app -app ^-app))
               function*
                 "ProcessScriptFunction<T>"
-            frame* (marked in-app by stack trace rule (family:native function:FDebug::CheckVerifyFailedImpl* v+app -app ^-app))
+            frame* (marked in-app by built-in stack trace rule (family:native function:FDebug::CheckVerifyFailedImpl* v+app -app ^-app))
               function*
                 "ProcessLocalScriptFunction"
-            frame* (marked in-app by stack trace rule (family:native function:FDebug::CheckVerifyFailedImpl* v+app -app ^-app))
+            frame* (marked in-app by built-in stack trace rule (family:native function:FDebug::CheckVerifyFailedImpl* v+app -app ^-app))
               function*
                 "UObject::execCallMathFunction"
-            frame* (marked in-app by stack trace rule (family:native function:FDebug::CheckVerifyFailedImpl* v+app -app ^-app))
+            frame* (marked in-app by built-in stack trace rule (family:native function:FDebug::CheckVerifyFailedImpl* v+app -app ^-app))
               function*
                 "USentryPlaygroundUtils::execTerminate"
-            frame* (marked in-app by stack trace rule (family:native function:FDebug::CheckVerifyFailedImpl* v+app -app ^-app))
+            frame* (marked in-app by built-in stack trace rule (family:native function:FDebug::CheckVerifyFailedImpl* v+app -app ^-app))
               function*
                 "USentryPlaygroundUtils::Terminate"
           type*