From 5084264e3eeb08177ccdf66ee79d9730ee668f79 Mon Sep 17 00:00:00 2001 From: Ivana Kellyer Date: Thu, 24 Sep 2026 10:03:14 +0200 Subject: [PATCH 1/5] fix: Isolate user callbacks, emit client reports --- sentry_sdk/client.py | 61 ++++++++--- sentry_sdk/profiler/transaction_profiler.py | 11 +- sentry_sdk/scope.py | 6 +- sentry_sdk/tracing.py | 4 + sentry_sdk/tracing_utils.py | 4 + tests/profiler/test_transaction_profiler.py | 31 ++++++ tests/test_basics.py | 109 ++++++++++++++++++++ tests/test_logs.py | 39 +++++++ tests/test_metrics.py | 37 +++++++ tests/tracing/test_span_streaming.py | 27 +++++ 10 files changed, 314 insertions(+), 15 deletions(-) diff --git a/sentry_sdk/client.py b/sentry_sdk/client.py index e3c6370c9c..16c17813c2 100644 --- a/sentry_sdk/client.py +++ b/sentry_sdk/client.py @@ -930,14 +930,22 @@ def _prepare_event( and event.get("type") != "transaction" ): new_event = None + exception_raised_in_before_send = False with capture_internal_exceptions(): - new_event = before_send(event, hint or {}) + try: + new_event = before_send(event, hint or {}) + except Exception: + exception_raised_in_before_send = True + raise if new_event is None: logger.info("before send dropped event") if self.transport: - self.transport.record_lost_event( - "before_send", data_category="error" + reason = ( + "callback_error" + if exception_raised_in_before_send + else "before_send" ) + self.transport.record_lost_event(reason, data_category="error") # If this is an exception, reset the DedupeIntegration. It still # remembers the dropped exception as the last exception, meaning @@ -955,17 +963,29 @@ def _prepare_event( and event.get("type") == "transaction" ): new_event = None + exception_raised_in_before_send_transaction = False spans_before = len(cast(List[Dict[str, object]], event.get("spans", []))) + with capture_internal_exceptions(): - new_event = before_send_transaction(event, hint or {}) + try: + new_event = before_send_transaction(event, hint or {}) + except Exception: + exception_raised_in_before_send_transaction = True + raise + if new_event is None: logger.info("before send transaction dropped event") if self.transport: + reason = ( + "callback_error" + if exception_raised_in_before_send_transaction + else "before_send" + ) self.transport.record_lost_event( - reason="before_send", data_category="transaction" + reason=reason, data_category="transaction" ) self.transport.record_lost_event( - reason="before_send", + reason=reason, data_category="span", quantity=spans_before + 1, # +1 for the transaction itself ) @@ -1265,15 +1285,32 @@ def _capture_telemetry( exception_raised_in_before_send_func = True raise - if ty in ("log", "metric"): - # We are ok with dropping metrics and logs when an exception is raised - # because we allow users to drop them in their respect before_send_* - # functions. + if ty == "log": if exception_raised_in_before_send_func: + if self.transport: + self.transport.record_lost_event( + "callback_error", data_category="log_item" + ) return - # Logs and metrics can be dropped in their respective - # before_send, so if we get None, don't queue them for sending. if serialized is None: + if self.transport: + self.transport.record_lost_event( + "before_send", data_category="log_item" + ) + return + + elif ty == "metric": + if exception_raised_in_before_send_func: + if self.transport: + self.transport.record_lost_event( + "callback_error", data_category="trace_metric" + ) + return + if serialized is None: + if self.transport: + self.transport.record_lost_event( + "before_send", data_category="trace_metric" + ) return elif ty == "span" and isinstance(telemetry, StreamedSpan): diff --git a/sentry_sdk/profiler/transaction_profiler.py b/sentry_sdk/profiler/transaction_profiler.py index fe65774c0b..3ed1da8429 100644 --- a/sentry_sdk/profiler/transaction_profiler.py +++ b/sentry_sdk/profiler/transaction_profiler.py @@ -286,7 +286,16 @@ def _set_initial_sampling_decision( options = client.options if callable(options.get("profiles_sampler")): - sample_rate = options["profiles_sampler"](sampling_context) + try: + sample_rate = options["profiles_sampler"](sampling_context) + except Exception: + logger.warning( + "[Profiling] profiles_sampler raised; falling back to profiles_sample_rate", + exc_info=True, + ) + sample_rate = options["profiles_sample_rate"] + if sample_rate is None: + sample_rate = options["_experiments"].get("profiles_sample_rate") elif options["profiles_sample_rate"] is not None: sample_rate = options["profiles_sample_rate"] else: diff --git a/sentry_sdk/scope.py b/sentry_sdk/scope.py index 4d547a3f48..dab8decb75 100644 --- a/sentry_sdk/scope.py +++ b/sentry_sdk/scope.py @@ -1828,7 +1828,9 @@ def run_error_processors(self, event: "Event", hint: "Hint") -> "Optional[Event] ) for error_processor in error_processors: - new_event = error_processor(event, exc_info) + new_event = None + with capture_internal_exceptions(): + new_event = error_processor(event, exc_info) if new_event is None: return self._drop(error_processor, "error processor") @@ -1856,7 +1858,7 @@ def run_event_processors(self, event: "Event", hint: "Hint") -> "Optional[Event] ) for event_processor in event_processors: - new_event = event + new_event = None with capture_internal_exceptions(): new_event = event_processor(event, hint) if new_event is None: diff --git a/sentry_sdk/tracing.py b/sentry_sdk/tracing.py index aab2794621..34e07f4c62 100644 --- a/sentry_sdk/tracing.py +++ b/sentry_sdk/tracing.py @@ -1220,6 +1220,10 @@ def _set_initial_sampling_decision( "[Tracing] traces_sampler raised; falling back to parent sample rate or traces_sample_rate", exc_info=True, ) + if client.transport: + client.transport.record_lost_event( + "callback_error", data_category="transaction" + ) sample_rate = ( sampling_context["parent_sampled"] if sampling_context["parent_sampled"] is not None diff --git a/sentry_sdk/tracing_utils.py b/sentry_sdk/tracing_utils.py index 42ebc7f1b3..6ab4b13551 100644 --- a/sentry_sdk/tracing_utils.py +++ b/sentry_sdk/tracing_utils.py @@ -1705,6 +1705,10 @@ def _make_sampling_decision( "[Tracing] traces_sampler raised; falling back to parent sample rate or traces_sample_rate", exc_info=True, ) + if client.transport: + client.transport.record_lost_event( + "callback_error", data_category="span" + ) if propagation_context.parent_sampled is not None: sample_rate = propagation_context.parent_sampled else: diff --git a/tests/profiler/test_transaction_profiler.py b/tests/profiler/test_transaction_profiler.py index 749e91add0..0d185a47af 100644 --- a/tests/profiler/test_transaction_profiler.py +++ b/tests/profiler/test_transaction_profiler.py @@ -236,6 +236,37 @@ def test_profiles_sampler( assert record_lost_event_calls == [("sample_rate", "profile", None, 1)] +@pytest.mark.tests_internal_exceptions +@mock.patch("sentry_sdk.profiler.transaction_profiler.PROFILE_MINIMUM_SAMPLES", 0) +def test_profiles_sampler_exception_falls_back_to_profiles_sample_rate( + sentry_init, + capture_envelopes, + teardown_profiling, +): + def bad_sampler(sampling_context): + raise ValueError("profiles_sampler error") + + sentry_init( + traces_sample_rate=1.0, + profiles_sampler=bad_sampler, + profiles_sample_rate=1.0, + ) + + envelopes = capture_envelopes() + + with start_transaction(name="profiling"): + pass + + items = defaultdict(list) + for envelope in envelopes: + for item in envelope.items: + items[item.type].append(item) + + assert len(items["transaction"]) == 1 + # Falls back to profiles_sample_rate=1.0, so profile should be sent + assert len(items["profile"]) == 1 + + def test_minimum_unique_samples_required( sentry_init, capture_envelopes, diff --git a/tests/test_basics.py b/tests/test_basics.py index cb3dffd2fe..30277e54c3 100644 --- a/tests/test_basics.py +++ b/tests/test_basics.py @@ -768,6 +768,115 @@ def foo(event, hint): sentry_sdk.scope.global_event_processors = old_processors +@pytest.mark.tests_internal_exceptions +def test_event_processor_exception_drops_event_and_records_client_report( + sentry_init, capture_events, capture_record_lost_event_calls +): + sentry_init(default_integrations=False) + events = capture_events() + record_lost_event_calls = capture_record_lost_event_calls() + + old_processors = sentry_sdk.scope.global_event_processors + + try: + sentry_sdk.scope.global_event_processors = ( + sentry_sdk.scope.global_event_processors.copy() + ) + + @add_global_event_processor + def bad_processor(event, hint): + raise ValueError("processor error") + + capture_message("should be dropped") + + assert len(events) == 0 + assert ("event_processor", "error", None, 1) in record_lost_event_calls + + finally: + sentry_sdk.scope.global_event_processors = old_processors + + +@pytest.mark.tests_internal_exceptions +def test_error_processor_exception_drops_event( + sentry_init, capture_events, capture_record_lost_event_calls +): + sentry_init(default_integrations=False) + events = capture_events() + record_lost_event_calls = capture_record_lost_event_calls() + + def bad_error_processor(event, exc_info): + raise ValueError("error processor error") + + sentry_sdk.get_isolation_scope().add_error_processor(bad_error_processor) + + try: + raise ValueError("original error") + except Exception: + capture_exception() + + assert len(events) == 0 + assert ("event_processor", "error", None, 1) in record_lost_event_calls + + +@pytest.mark.tests_internal_exceptions +def test_before_send_exception_records_callback_error( + sentry_init, capture_events, capture_record_lost_event_calls +): + def bad_before_send(event, hint): + raise ValueError("before_send error") + + sentry_init(before_send=bad_before_send, default_integrations=False) + events = capture_events() + record_lost_event_calls = capture_record_lost_event_calls() + + capture_message("should be dropped") + + assert len(events) == 0 + assert ("callback_error", "error", None, 1) in record_lost_event_calls + + +def test_before_send_returning_none_records_before_send( + sentry_init, capture_events, capture_record_lost_event_calls +): + def dropping_before_send(event, hint): + return None + + sentry_init(before_send=dropping_before_send) + events = capture_events() + record_lost_event_calls = capture_record_lost_event_calls() + + capture_message("should be dropped") + + assert len(events) == 0 + assert ("before_send", "error", None, 1) in record_lost_event_calls + + +@pytest.mark.tests_internal_exceptions +def test_before_send_transaction_exception_records_callback_error( + sentry_init, capture_events, capture_record_lost_event_calls +): + def bad_before_send_transaction(event, hint): + raise ValueError("before_send_transaction error") + + sentry_init( + traces_sample_rate=1.0, + before_send_transaction=bad_before_send_transaction, + default_integrations=False, + ) + events = capture_events() + record_lost_event_calls = capture_record_lost_event_calls() + + with start_transaction(name="test"): + pass + + assert not any(e.get("type") == "transaction" for e in events) + assert ("callback_error", "transaction", None, 1) in record_lost_event_calls + assert any( + reason == "callback_error" and category == "span" + for reason, category, _, _ in record_lost_event_calls + ) + + @pytest.mark.parametrize( "installed_integrations, expected_name", [ diff --git a/tests/test_logs.py b/tests/test_logs.py index e8a098ff44..103490fde1 100644 --- a/tests/test_logs.py +++ b/tests/test_logs.py @@ -182,6 +182,45 @@ def _before_log(record, hint): assert not logs +@minimum_python_37 +@pytest.mark.tests_internal_exceptions +def test_logs_before_send_log_raises_records_callback_error( + sentry_init, capture_items, capture_record_lost_event_calls +): + def _before_log(record, hint): + raise ValueError("before_send_log error") + + sentry_init(before_send_log=_before_log) + items = capture_items("log") + record_lost_event_calls = capture_record_lost_event_calls() + + sentry_sdk.logger.error("This is an error log...") + + get_client().flush() + + assert not [item.payload for item in items] + assert ("callback_error", "log_item", None, 1) in record_lost_event_calls + + +@minimum_python_37 +def test_logs_before_send_log_returns_none_records_before_send( + sentry_init, capture_items, capture_record_lost_event_calls +): + def _before_log(record, hint): + return None + + sentry_init(before_send_log=_before_log) + items = capture_items("log") + record_lost_event_calls = capture_record_lost_event_calls() + + sentry_sdk.logger.error("This is an error log...") + + get_client().flush() + + assert not [item.payload for item in items] + assert ("before_send", "log_item", None, 1) in record_lost_event_calls + + @minimum_python_37 def test_logs_attributes(sentry_init, capture_items): """ diff --git a/tests/test_metrics.py b/tests/test_metrics.py index e07e507ba7..979ef6e1bc 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -289,6 +289,43 @@ def _before_metric(record, hint): assert not metrics +@pytest.mark.tests_internal_exceptions +def test_metrics_before_send_raises_records_callback_error( + sentry_init, capture_items, capture_record_lost_event_calls +): + def _before_metric(record, hint): + raise ValueError("before_send_metric error") + + sentry_init(before_send_metric=_before_metric) + items = capture_items("trace_metric") + record_lost_event_calls = capture_record_lost_event_calls() + + sentry_sdk.metrics.count("test.keep", 1) + + get_client().flush() + + assert not [item.payload for item in items] + assert ("callback_error", "trace_metric", None, 1) in record_lost_event_calls + + +def test_metrics_before_send_returns_none_records_before_send( + sentry_init, capture_items, capture_record_lost_event_calls +): + def _before_metric(record, hint): + return None + + sentry_init(before_send_metric=_before_metric) + items = capture_items("trace_metric") + record_lost_event_calls = capture_record_lost_event_calls() + + sentry_sdk.metrics.count("test.drop", 1) + + get_client().flush() + + assert not [item.payload for item in items] + assert ("before_send", "trace_metric", None, 1) in record_lost_event_calls + + def test_transport_format(sentry_init, capture_envelopes): sentry_init(server_name="test-server", release="1.0.0") diff --git a/tests/tracing/test_span_streaming.py b/tests/tracing/test_span_streaming.py index 8aef2f7d17..2a056d2633 100644 --- a/tests/tracing/test_span_streaming.py +++ b/tests/tracing/test_span_streaming.py @@ -564,6 +564,33 @@ def traces_sampler(sampling_context): assert span["attributes"]["drop"] is False +@pytest.mark.tests_internal_exceptions +def test_traces_sampler_exception_falls_back_and_records_client_report( + sentry_init, capture_items, capture_record_lost_event_calls +): + def traces_sampler(sampling_context): + raise ValueError("traces_sampler error") + + sentry_init( + traces_sampler=traces_sampler, + traces_sample_rate=1.0, + trace_lifecycle="stream", + ) + + items = capture_items("span") + record_lost_event_calls = capture_record_lost_event_calls() + + with sentry_sdk.traces.start_span(name="test"): + ... + + sentry_sdk.get_client().flush() + spans = [item.payload for item in items] + + # Falls back to traces_sample_rate=1.0, so span should still be sent + assert len(spans) == 1 + assert ("callback_error", "span", None, 1) in record_lost_event_calls + + def test_traces_sampler_called_once_per_segment(sentry_init): traces_sampler_called = 0 span_name_in_traces_sampler = None From c8c168ab935de0407b55c1101299341cdefd039e Mon Sep 17 00:00:00 2001 From: Ivana Kellyer Date: Thu, 24 Sep 2026 10:29:09 +0200 Subject: [PATCH 2/5] . --- tests/test_basics.py | 27 ++++++++++----------------- tests/test_logs.py | 4 ++-- tests/test_metrics.py | 4 ++-- 3 files changed, 14 insertions(+), 21 deletions(-) diff --git a/tests/test_basics.py b/tests/test_basics.py index 30277e54c3..218b703bb8 100644 --- a/tests/test_basics.py +++ b/tests/test_basics.py @@ -776,24 +776,16 @@ def test_event_processor_exception_drops_event_and_records_client_report( events = capture_events() record_lost_event_calls = capture_record_lost_event_calls() - old_processors = sentry_sdk.scope.global_event_processors - - try: - sentry_sdk.scope.global_event_processors = ( - sentry_sdk.scope.global_event_processors.copy() - ) - - @add_global_event_processor - def bad_processor(event, hint): - raise ValueError("processor error") + scope = sentry_sdk.get_isolation_scope() - capture_message("should be dropped") + @scope.add_event_processor + def bad_processor(event, hint): + raise ValueError("processor error") - assert len(events) == 0 - assert ("event_processor", "error", None, 1) in record_lost_event_calls + capture_message("should be dropped") - finally: - sentry_sdk.scope.global_event_processors = old_processors + assert len(events) == 0 + assert ("event_processor", "error", None, 1) in record_lost_event_calls @pytest.mark.tests_internal_exceptions @@ -804,11 +796,12 @@ def test_error_processor_exception_drops_event( events = capture_events() record_lost_event_calls = capture_record_lost_event_calls() + scope = sentry_sdk.get_isolation_scope() + + @scope.add_error_processor def bad_error_processor(event, exc_info): raise ValueError("error processor error") - sentry_sdk.get_isolation_scope().add_error_processor(bad_error_processor) - try: raise ValueError("original error") except Exception: diff --git a/tests/test_logs.py b/tests/test_logs.py index 103490fde1..15f75456be 100644 --- a/tests/test_logs.py +++ b/tests/test_logs.py @@ -198,7 +198,7 @@ def _before_log(record, hint): get_client().flush() - assert not [item.payload for item in items] + assert not items assert ("callback_error", "log_item", None, 1) in record_lost_event_calls @@ -217,7 +217,7 @@ def _before_log(record, hint): get_client().flush() - assert not [item.payload for item in items] + assert not items assert ("before_send", "log_item", None, 1) in record_lost_event_calls diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 979ef6e1bc..5583a22cc8 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -304,7 +304,7 @@ def _before_metric(record, hint): get_client().flush() - assert not [item.payload for item in items] + assert not items assert ("callback_error", "trace_metric", None, 1) in record_lost_event_calls @@ -322,7 +322,7 @@ def _before_metric(record, hint): get_client().flush() - assert not [item.payload for item in items] + assert not items assert ("before_send", "trace_metric", None, 1) in record_lost_event_calls From 9be4b22c4b9caf858ca9232d32741212e727429e Mon Sep 17 00:00:00 2001 From: Ivana Kellyer Date: Thu, 24 Sep 2026 10:32:18 +0200 Subject: [PATCH 3/5] . --- sentry_sdk/tracing.py | 4 ---- sentry_sdk/tracing_utils.py | 4 ---- tests/tracing/test_span_streaming.py | 6 +----- 3 files changed, 1 insertion(+), 13 deletions(-) diff --git a/sentry_sdk/tracing.py b/sentry_sdk/tracing.py index 34e07f4c62..aab2794621 100644 --- a/sentry_sdk/tracing.py +++ b/sentry_sdk/tracing.py @@ -1220,10 +1220,6 @@ def _set_initial_sampling_decision( "[Tracing] traces_sampler raised; falling back to parent sample rate or traces_sample_rate", exc_info=True, ) - if client.transport: - client.transport.record_lost_event( - "callback_error", data_category="transaction" - ) sample_rate = ( sampling_context["parent_sampled"] if sampling_context["parent_sampled"] is not None diff --git a/sentry_sdk/tracing_utils.py b/sentry_sdk/tracing_utils.py index 6ab4b13551..42ebc7f1b3 100644 --- a/sentry_sdk/tracing_utils.py +++ b/sentry_sdk/tracing_utils.py @@ -1705,10 +1705,6 @@ def _make_sampling_decision( "[Tracing] traces_sampler raised; falling back to parent sample rate or traces_sample_rate", exc_info=True, ) - if client.transport: - client.transport.record_lost_event( - "callback_error", data_category="span" - ) if propagation_context.parent_sampled is not None: sample_rate = propagation_context.parent_sampled else: diff --git a/tests/tracing/test_span_streaming.py b/tests/tracing/test_span_streaming.py index 2a056d2633..5de65e4802 100644 --- a/tests/tracing/test_span_streaming.py +++ b/tests/tracing/test_span_streaming.py @@ -565,9 +565,7 @@ def traces_sampler(sampling_context): @pytest.mark.tests_internal_exceptions -def test_traces_sampler_exception_falls_back_and_records_client_report( - sentry_init, capture_items, capture_record_lost_event_calls -): +def test_traces_sampler_exception_falls_back(sentry_init, capture_items): def traces_sampler(sampling_context): raise ValueError("traces_sampler error") @@ -578,7 +576,6 @@ def traces_sampler(sampling_context): ) items = capture_items("span") - record_lost_event_calls = capture_record_lost_event_calls() with sentry_sdk.traces.start_span(name="test"): ... @@ -588,7 +585,6 @@ def traces_sampler(sampling_context): # Falls back to traces_sample_rate=1.0, so span should still be sent assert len(spans) == 1 - assert ("callback_error", "span", None, 1) in record_lost_event_calls def test_traces_sampler_called_once_per_segment(sentry_init): From 7c784b4fe713f8b167617a3f1613ca3b1dfabe37 Mon Sep 17 00:00:00 2001 From: Ivana Kellyer Date: Thu, 24 Sep 2026 10:38:26 +0200 Subject: [PATCH 4/5] . --- sentry_sdk/integrations/aws_lambda.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentry_sdk/integrations/aws_lambda.py b/sentry_sdk/integrations/aws_lambda.py index d8bf35ef7e..6a88ad0f07 100644 --- a/sentry_sdk/integrations/aws_lambda.py +++ b/sentry_sdk/integrations/aws_lambda.py @@ -466,7 +466,7 @@ def event_processor( else: request["query_string"] = query_string - if "headers" in aws_event: + if "headers" in aws_event and isinstance(aws_event["headers"], dict): request["headers"] = _filter_headers(aws_event["headers"]) client_options = sentry_sdk.get_client().options From d22684fbb3311fdd7196d567e86121db6f12c0fa Mon Sep 17 00:00:00 2001 From: Ivana Kellyer Date: Thu, 24 Sep 2026 10:45:45 +0200 Subject: [PATCH 5/5] . --- sentry_sdk/integrations/aws_lambda.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentry_sdk/integrations/aws_lambda.py b/sentry_sdk/integrations/aws_lambda.py index 6a88ad0f07..009d7328b2 100644 --- a/sentry_sdk/integrations/aws_lambda.py +++ b/sentry_sdk/integrations/aws_lambda.py @@ -508,7 +508,7 @@ def _get_url(aws_event: "Any", aws_context: "Any") -> str: path = aws_event.get("path", None) headers = aws_event.get("headers") - if headers is None: + if not isinstance(headers, dict): headers = {} host = headers.get("Host", None)