chore(seer grouping): Add debug logging for old excess frames check (#…

…83082)

As of #82414, we should no longer be using `TooManyOnlySystemFramesException`, but the fact that we're still getting `did_call_seer` metrics tagged with `"over-threshold-frames"` says that we still are, somehow. This is currently a blocker on merging #82434, which removes `TooManyOnlySystemFramesException` and the code associated with it, so it'd be nice to figure out why it's happening so we can fix it.

Try as I might, I can't replicate it, nor can I find the flaw in the logic, so this adds a temporary log to try to help figure out how in the heck it's happening. It also adds a `record_metrics` parameter to the `has_too_many_contributing_frames` helper, so that I can call it for purposes of the logging without messing up the metrics it collects. Once I figure it out what's happening, I'll back out all of these changes.

src/sentry/grouping/ingest/seer.py

@@ -200,8 +200,21 @@ def _circuit_breaker_broken(event: Event, project: Project) -> bool:
 
 
 def _has_empty_stacktrace_string(event: Event, variants: dict[str, BaseVariant]) -> bool:
+    # For purposes of a temporary debug log - will be removed as soon as the mysterious behavior
+    # is sorted
+    logger_extra = {
+        "event_id": event.event_id,
+        "project_id": event.project_id,
+        "platform": event.platform,
+        "has_too_many_frames_result": has_too_many_contributing_frames(
+            event, variants, ReferrerOptions.INGEST, record_metrics=False
+        ),
+    }
     stacktrace_string = get_stacktrace_string_with_metrics(
-        get_grouping_info_from_variants(variants), event.platform, ReferrerOptions.INGEST
+        get_grouping_info_from_variants(variants),
+        event.platform,
+        ReferrerOptions.INGEST,
+        logger_extra=logger_extra,
     )
     if not stacktrace_string:
         if stacktrace_string == "":

src/sentry/seer/similarity/utils.py

@@ -1,4 +1,5 @@
 import logging
+import traceback
 from collections.abc import Mapping, Sequence
 from enum import StrEnum
 from typing import Any, TypedDict, TypeVar
@@ -269,7 +270,10 @@ def is_base64_encoded_frame(frame_dict: Mapping[str, Any]) -> bool:
 
 
 def get_stacktrace_string_with_metrics(
-    data: dict[str, Any], platform: str | None, referrer: ReferrerOptions
+    data: dict[str, Any],
+    platform: str | None,
+    referrer: ReferrerOptions,
+    logger_extra: dict[str, Any] | None = None,
 ) -> str | None:
     stacktrace_string = None
     sample_rate = options.get("seer.similarity.metrics_sample_rate")
@@ -283,6 +287,14 @@ def get_stacktrace_string_with_metrics(
             tags={"platform": platform, "referrer": referrer},
         )
         if referrer == ReferrerOptions.INGEST:
+            # Temporary log to debug how we're still landing here, which we shouldn't be anymore
+            logger_extra = logger_extra or {}
+            logger_extra.update({"current_stacktrace": "".join(traceback.format_stack())})
+            logger.info(
+                "record_did_call_seer_metric.over-threshold-frames",
+                extra=logger_extra,
+            )
+
             record_did_call_seer_metric(call_made=False, blocker="over-threshold-frames")
     except Exception:
         logger.exception("Unexpected exception in stacktrace string formatting")
@@ -312,6 +324,7 @@ def has_too_many_contributing_frames(
     event: Event | GroupEvent,
     variants: dict[str, BaseVariant],
     referrer: ReferrerOptions,
+    record_metrics: bool = True,
 ) -> bool:
     platform = event.platform
     shared_tags = {"referrer": referrer.value, "platform": platform}
@@ -339,30 +352,33 @@ def has_too_many_contributing_frames(
     # with the existing data, we turn off the filter for them (instead their stacktraces will be
     # truncated)
     if platform in EVENT_PLATFORMS_BYPASSING_FRAME_COUNT_CHECK:
-        metrics.incr(
-            "grouping.similarity.frame_count_filter",
-            sample_rate=options.get("seer.similarity.metrics_sample_rate"),
-            tags={**shared_tags, "outcome": "bypass"},
-        )
+        if record_metrics:
+            metrics.incr(
+                "grouping.similarity.frame_count_filter",
+                sample_rate=options.get("seer.similarity.metrics_sample_rate"),
+                tags={**shared_tags, "outcome": "bypass"},
+            )
         return False
 
     stacktrace_type = "in_app" if contributing_variant.variant_name == "app" else "system"
     key = f"{stacktrace_type}_contributing_frames"
     shared_tags["stacktrace_type"] = stacktrace_type
 
     if contributing_component.frame_counts[key] > MAX_FRAME_COUNT:
+        if record_metrics:
+            metrics.incr(
+                "grouping.similarity.frame_count_filter",
+                sample_rate=options.get("seer.similarity.metrics_sample_rate"),
+                tags={**shared_tags, "outcome": "block"},
+            )
+        return True
+
+    if record_metrics:
         metrics.incr(
             "grouping.similarity.frame_count_filter",
             sample_rate=options.get("seer.similarity.metrics_sample_rate"),
-            tags={**shared_tags, "outcome": "block"},
+            tags={**shared_tags, "outcome": "pass"},
         )
-        return True
-
-    metrics.incr(
-        "grouping.similarity.frame_count_filter",
-        sample_rate=options.get("seer.similarity.metrics_sample_rate"),
-        tags={**shared_tags, "outcome": "pass"},
-    )
     return False