fix: mask authorization credentials and signed urls in code variables (#1015)

Author: ablaszkiewicz

.sampo/changesets/mask-authorization-credentials.md

@@ -0,0 +1,5 @@
+---
+pypi/posthog: patch
+---
+
+Mask `Bearer` and `Basic` credentials in exception code variables, including values held apart from their header name, such as in header lists and ASGI scopes. Also mask signed URLs that carry a `sig` query parameter.

posthog/exception_utils.py

@@ -5,6 +5,8 @@
 # 💖open source (under MIT License)
 # We want to keep payloads as similar to Sentry as possible for easy interoperability
 
+import base64
+import binascii
 import dataclasses
 import functools
 import json
@@ -67,6 +69,9 @@
     r"(?i)conn_str",
     r"(?i)connstr",
     r"(?i)dsn",
+    # The signature parameter of a signed URL, such as an Azure SAS URL. It is matched
+    # as a query parameter, because the bare word `sig` occurs in common names.
+    r"(?i)[?&]sig=",
 ]
 
 DEFAULT_CODE_VARIABLES_IGNORE_PATTERNS = [r"^__.*"]
@@ -124,6 +129,63 @@ def _redact_url_credentials(value):
     )
 
 
+# Matches the credential of an HTTP `Authorization` value, e.g. `Bearer <token>` or
+# `Basic <base64 user:pass>`. A header pair list or an ASGI scope holds this value apart
+# from its header name, so the name patterns never see it. The separator also accepts a
+# colon or an opening quote, as in `Bearer: <token>`, but it must not be empty, so that
+# names such as `basicConfig` stay untouched.
+_AUTH_HEADER_CREDENTIALS_RE = re.compile(
+    r"\b(bearer|basic)((?:\s*:\s*|\s+)['\"]?)([A-Za-z0-9._~+/-]+=*)", re.IGNORECASE
+)
+
+# Shorter values are prose, such as "the bearer of", and so is a lowercase word of up to
+# 15 letters, such as "bearer transportation". A random lowercase token is longer than
+# that. A `Basic` credential of any length is still redacted when it decodes to
+# `user:password`, e.g. `YTpi` for `a:b`.
+_AUTH_HEADER_CREDENTIAL_MIN_LENGTH = 8
+_AUTH_HEADER_PROSE_WORD_MAX_LENGTH = 15
+
+
+def _is_basic_credential(credential):
+    try:
+        decoded = base64.b64decode(credential, validate=True).decode("utf-8")
+    except (binascii.Error, ValueError):
+        return False
+    return ":" in decoded
+
+
+def _is_prose_word(credential):
+    return (
+        len(credential) <= _AUTH_HEADER_PROSE_WORD_MAX_LENGTH
+        and credential.isalpha()
+        and credential.islower()
+    )
+
+
+def _redact_auth_header_match(match):
+    scheme, credential = match.group(1), match.group(3)
+    is_basic_pair = scheme.lower() == "basic" and _is_basic_credential(credential)
+    if not is_basic_pair and (
+        len(credential) < _AUTH_HEADER_CREDENTIAL_MIN_LENGTH
+        or _is_prose_word(credential)
+    ):
+        return match.group(0)
+    return scheme + match.group(2) + CODE_VARIABLES_REDACTED_VALUE
+
+
+def _redact_auth_header_credentials(value):
+    return _AUTH_HEADER_CREDENTIALS_RE.sub(_redact_auth_header_match, value)
+
+
+def _redact_embedded_credentials(value, config):
+    """Scrub credentials embedded in otherwise safe text: URL credentials when that toggle
+    is on, then `Authorization` values always. URLs go first, because the `Authorization`
+    pass can consume a URL scheme, as in `Bearer postgresql://user:pass@host`."""
+    if config.mask_url_credentials:
+        value = _redact_url_credentials(value)
+    return _redact_auth_header_credentials(value)
+
+
 DEFAULT_TOTAL_VARIABLES_SIZE_LIMIT = 10 * 1024
 
 
@@ -1270,16 +1332,14 @@ def _looks_like_secret(value):
 
 def _mask_string(value, config):
     """Apply the string masking policy: over-length cap, name/value patterns,
-    entropy-based secret detection, then embedded URL credentials."""
+    entropy-based secret detection, then embedded `Authorization` and URL credentials."""
     if len(value) > _MAX_VALUE_LENGTH_FOR_PATTERN_MATCH:
         return CODE_VARIABLES_TOO_LONG_VALUE
     if _matcher_matches(value, config.mask):
         return CODE_VARIABLES_REDACTED_VALUE
     if config.detect_secrets and _looks_like_secret(value):
         return CODE_VARIABLES_REDACTED_VALUE
-    if config.mask_url_credentials:
-        return _redact_url_credentials(value)
-    return value
+    return _redact_embedded_credentials(value, config)
 
 
 def _safe_type_name(value):
@@ -1306,9 +1366,7 @@ def _safe_repr(value, config):
     # A __repr__ that is itself a bare secret would otherwise bypass detection.
     if config.detect_secrets and _looks_like_secret(rendered):
         return CODE_VARIABLES_REDACTED_VALUE
-    if config.mask_url_credentials:
-        return _redact_url_credentials(rendered)
-    return rendered
+    return _redact_embedded_credentials(rendered, config)
 
 
 def _extract_object_attrs(value):
@@ -1485,8 +1543,8 @@ def _mask_mapping(items, config, seen, depth):
             out_key = _redacted_key(result)
         elif not key_is_json_safe and _key_parts_fail_masking(key, config, seen, depth):
             out_key = _redacted_key(result)
-        elif config.mask_url_credentials and isinstance(out_key, str):
-            out_key = _redact_url_credentials(out_key)
+        elif isinstance(out_key, str):
+            out_key = _redact_embedded_credentials(out_key, config)
         if out_key in result:
             # Two keys can end up with the same text, for example URLs that differ only in
             # their credentials. A placeholder keeps the later entry from overwriting.

posthog/test/test_code_variables.py

@@ -238,6 +238,70 @@ def test_strings_without_credentials_are_left_untouched(self, value):
         assert _redact_url_credentials(value) == value
 
 
+# --- 2b. Authorization credential scrubbing ------------------------------------------
+
+_BEARER_TOKEN = _key("tok_", "Zx81Qm7Lp2Vb9Nc4")
+_BASIC_CREDENTIAL = _key("c3ZjOmZha2Ut", "cGFzcy0xMjM=")  # svc:fake-pass-123
+
+
+class TestAuthorizationCredentialScrubbing:
+    """A `Bearer` or `Basic` credential is removed even when no header name sits next
+    to it, so name patterns can't catch it. The scheme stays for context."""
+
+    @pytest.mark.parametrize(
+        "value, credential",
+        [
+            # header pair list: the name is redacted, the value used to survive
+            ([("authorization", "Bearer " + _BEARER_TOKEN)], _BEARER_TOKEN),
+            # ASGI scope: raw byte headers
+            (
+                {
+                    "headers": [
+                        (b"authorization", b"Basic " + _BASIC_CREDENTIAL.encode())
+                    ]
+                },
+                _BASIC_CREDENTIAL,
+            ),
+            # a local with a neutral name
+            ("Basic " + _BASIC_CREDENTIAL, _BASIC_CREDENTIAL),
+            # a short Basic credential: `YTpi` is `a:b`
+            ("Basic YTpi", "YTpi"),
+            # a credential of lowercase letters only
+            ("Bearer " + _key("zqwklmno", "pxyzrstu"), _key("zqwklmno", "pxyzrstu")),
+            # a DSN after the scheme keeps its own credential redacted
+            (
+                "Bearer postgresql://alice:" + _key("sunfl", "ower99") + "@db/app",
+                _key("sunfl", "ower99"),
+            ),
+            # a colon or a quote between the scheme and the credential
+            ("Bearer: " + _BEARER_TOKEN, _BEARER_TOKEN),
+            ("Bearer '" + _BEARER_TOKEN + "'", _BEARER_TOKEN),
+        ],
+    )
+    def test_credential_is_removed(self, value, credential):
+        result = json.dumps(mask(value))
+        assert credential not in result
+        assert REDACTED in result
+
+    def test_scheme_is_kept(self):
+        assert mask("Bearer " + _BEARER_TOKEN) == "Bearer " + REDACTED
+
+    @pytest.mark.parametrize(
+        "value",
+        [
+            "basicConfig(level=10)",
+            "basic auth",
+            "basic: configuration",
+            "Bearer token",
+            "bearer transportation",
+            "the bearer of bad news",
+        ],
+    )
+    def test_prose_is_left_untouched(self, value):
+        # name patterns off, so only the Authorization check can change the text
+        assert mask(value, patterns=[]) == value
+
+
 # --- 3. scalar masking ---------------------------------------------------------------
 
 
@@ -288,6 +352,17 @@ def test_url_scrubbing_can_be_turned_off(self):
         result = mask("postgresql://user:p4ss@host/db", patterns=[], mask_urls=False)
         assert result == "postgresql://user:p4ss@host/db"
 
+    def test_signed_url_is_redacted(self):
+        # an Azure SAS URL carries its signature in the `sig` query parameter
+        url = "https://acct.blob.core.windows.net/c/f?sv=2022-11-02&sp=r&sig=" + _key(
+            "q2VxT8", "fKz1aB3dE%3D"
+        )
+        assert mask(url) == REDACTED
+
+    @pytest.mark.parametrize("value", ["design", "signal_handler", "/signup?step=2"])
+    def test_sig_inside_other_words_is_left_untouched(self, value):
+        assert mask(value) == value
+
 
 # --- 5. collection masking -----------------------------------------------------------
 

references/public_api_snapshot.txt

@@ -825,7 +825,7 @@ attribute posthog.exception_utils.CODE_VARIABLES_REDACTED_VALUE = '$$_posthog_re
 attribute posthog.exception_utils.CODE_VARIABLES_TOO_LONG_VALUE = '$$_posthog_value_too_long_$$'
 attribute posthog.exception_utils.DEFAULT_CODE_VARIABLES_DETECT_SECRETS = True
 attribute posthog.exception_utils.DEFAULT_CODE_VARIABLES_IGNORE_PATTERNS = ['^__.*']
-attribute posthog.exception_utils.DEFAULT_CODE_VARIABLES_MASK_PATTERNS = ['(?i)password', '(?i)secret', '(?i)passwd', '(?i)pwd', '(?i)api_key', '(?i)apikey', '(?i)auth', '(?i)credentials', '(?i)privatekey', '(?i)private_key', '(?i)token', '(?i)aws_access_key_id', '(?i)_pass', '(?i)sk_', '(?i)jwt', '(?i)connection_string', '(?i)connectionstring', '(?i)conn_str', '(?i)connstr', '(?i)dsn']
+attribute posthog.exception_utils.DEFAULT_CODE_VARIABLES_MASK_PATTERNS = ['(?i)password', '(?i)secret', '(?i)passwd', '(?i)pwd', '(?i)api_key', '(?i)apikey', '(?i)auth', '(?i)credentials', '(?i)privatekey', '(?i)private_key', '(?i)token', '(?i)aws_access_key_id', '(?i)_pass', '(?i)sk_', '(?i)jwt', '(?i)connection_string', '(?i)connectionstring', '(?i)conn_str', '(?i)connstr', '(?i)dsn', '(?i)[?&]sig=']
 attribute posthog.exception_utils.DEFAULT_CODE_VARIABLES_MASK_URL_CREDENTIALS = True
 attribute posthog.exception_utils.DEFAULT_MAX_VALUE_LENGTH = 1024
 attribute posthog.exception_utils.DEFAULT_TOTAL_VARIABLES_SIZE_LIMIT = 10 * 1024
@@ -1408,6 +1408,7 @@ function posthog.load_feature_flags()
 function posthog.mcp.asgi.autowire_stateless_mint(server: Any) -> None
 function posthog.mcp.asgi.get_mcp_session(request_or_scope: Any) -> Optional[SessionTokenPayload]
 function posthog.mcp.feedback.send_feedback_result() -> Dict[str, Any]
+function posthog.mcp.get_tool_input_properties(input_value: Any, input_schema: Any = None, options: Optional[ToolInputOptions] = None)
 function posthog.mcp.instrument(server: Any, posthog_client: Optional[Client] = None, options: Optional[MCPAnalyticsOptions] = None) -> McpAnalytics
 function posthog.mcp.logger.set_logger(logger: Optional[LoggerFn]) -> None
 function posthog.mcp.request_headers.get_request_headers(extra: Any) -> Optional[RequestHeaderBag]