feat(aws): Apply data_collection filtering to URL query strings

sentry_sdk/integrations/aws_lambda.py

@@ -11,6 +11,7 @@
 import sentry_sdk
 from sentry_sdk.api import continue_trace
 from sentry_sdk.consts import OP
+from sentry_sdk.data_collection import _apply_key_value_collection_filtering
 from sentry_sdk.integrations import Integration
 from sentry_sdk.integrations._wsgi_common import _filter_headers
 from sentry_sdk.integrations.cloud_resource_context import (
@@ -27,6 +28,7 @@
     capture_internal_exceptions,
     ensure_integration_enabled,
     event_from_exception,
+    has_data_collection_enabled,
     logger,
     reraise,
 )
@@ -164,10 +166,20 @@ def sentry_handler(
                     "httpMethod"
                 ]
 
-            if should_send_default_pii() and "queryStringParameters" in request_data:
+            if "queryStringParameters" in request_data:
                 qs = request_data["queryStringParameters"]
                 if qs:
-                    additional_attributes["url.query"] = urlencode(qs)
+                    if has_data_collection_enabled(client.options):
+                        filtered_qs = _apply_key_value_collection_filtering(
+                            items=qs,
+                            behaviour=client.options["data_collection"][
+                                "url_query_params"
+                            ],
+                        )
+                        if filtered_qs:
+                            additional_attributes["url.query"] = urlencode(filtered_qs)
+                    elif should_send_default_pii():
+                        additional_attributes["url.query"] = urlencode(qs)
 
             sampling_context = {
                 "aws_event": aws_event,
@@ -409,7 +421,18 @@ def event_processor(
         request["url"] = _get_url(aws_event, aws_context)
 
         if "queryStringParameters" in aws_event:
-            request["query_string"] = aws_event["queryStringParameters"]
+            query_string = aws_event["queryStringParameters"]
+            client_options = sentry_sdk.get_client().options
+            if has_data_collection_enabled(client_options):
+                if query_string:
+                    filtered_qs = _apply_key_value_collection_filtering(
+                        items=query_string,
+                        behaviour=client_options["data_collection"]["url_query_params"],
+                    )
+                    if filtered_qs:
+                        request["query_string"] = filtered_qs
+            else:
+                request["query_string"] = query_string
 
         if "headers" in aws_event:
             request["headers"] = _filter_headers(aws_event["headers"])

sentry_sdk/integrations/gcp.py

@@ -8,6 +8,7 @@
 import sentry_sdk
 from sentry_sdk.api import continue_trace
 from sentry_sdk.consts import OP
+from sentry_sdk.data_collection import _apply_data_collection_filtering_to_query_string
 from sentry_sdk.integrations import Integration
 from sentry_sdk.integrations._wsgi_common import _filter_headers
 from sentry_sdk.integrations.cloud_resource_context import CLOUD_PROVIDER
@@ -20,6 +21,7 @@
     TimeoutThread,
     capture_internal_exceptions,
     event_from_exception,
+    has_data_collection_enabled,
     logger,
     reraise,
 )
@@ -100,10 +102,20 @@ def sentry_func(
             if hasattr(gcp_event, "method"):
                 additional_attributes["http.request.method"] = gcp_event.method
 
-            if should_send_default_pii() and hasattr(gcp_event, "query_string"):
-                additional_attributes["url.query"] = gcp_event.query_string.decode(
-                    "utf-8", errors="replace"
-                )
+            if hasattr(gcp_event, "query_string"):
+                query_string = gcp_event.query_string.decode("utf-8", errors="replace")
+                if query_string:
+                    if has_data_collection_enabled(client.options):
+                        filtered_qs = _apply_data_collection_filtering_to_query_string(
+                            query_string=query_string,
+                            behaviour=client.options["data_collection"][
+                                "url_query_params"
+                            ],
+                        )
+                        if filtered_qs:
+                            additional_attributes["url.query"] = filtered_qs
+                    elif should_send_default_pii():
+                        additional_attributes["url.query"] = query_string
 
             sampling_context = {
                 "gcp_env": {
@@ -235,9 +247,18 @@ def event_processor(event: "Event", hint: "Hint") -> "Optional[Event]":
             request["method"] = gcp_event.method
 
         if hasattr(gcp_event, "query_string"):
-            request["query_string"] = gcp_event.query_string.decode(
-                "utf-8", errors="replace"
-            )
+            query_string = gcp_event.query_string.decode("utf-8", errors="replace")
+            client_options = sentry_sdk.get_client().options
+            if has_data_collection_enabled(client_options):
+                if query_string:
+                    filtered_qs = _apply_data_collection_filtering_to_query_string(
+                        query_string=query_string,
+                        behaviour=client_options["data_collection"]["url_query_params"],
+                    )
+                    if filtered_qs:
+                        request["query_string"] = filtered_qs
+            else:
+                request["query_string"] = query_string
 
         if hasattr(gcp_event, "headers"):
             request["headers"] = _filter_headers(gcp_event.headers)

sentry_sdk/integrations/quart.py

@@ -5,6 +5,7 @@
 from typing import TYPE_CHECKING
 
 import sentry_sdk
+from sentry_sdk.data_collection import _apply_data_collection_filtering_to_query_string
 from sentry_sdk.integrations import DidNotEnable, Integration
 from sentry_sdk.integrations._wsgi_common import _filter_headers
 from sentry_sdk.integrations.asgi import SentryAsgiMiddleware
@@ -17,6 +18,8 @@
     capture_internal_exceptions,
     ensure_integration_enabled,
     event_from_exception,
+    has_data_collection_enabled,
+    parse_url,
 )
 
 if TYPE_CHECKING:
@@ -203,7 +206,39 @@ async def _request_websocket_started(app: "Quart", **kwargs: "Any") -> None:
 
             segment.set_attributes(header_attributes)
 
-            if should_send_default_pii():
+            client_options = sentry_sdk.get_client().options
+            filtered_query_string = None
+            if has_data_collection_enabled(client_options):
+                query_string = request_websocket.query_string.decode(
+                    "utf-8", errors="replace"
+                )
+                if query_string:
+                    filtered_query_string = (
+                        _apply_data_collection_filtering_to_query_string(
+                            query_string=query_string,
+                            behaviour=client_options["data_collection"][
+                                "url_query_params"
+                            ],
+                        )
+                    )
+                    if filtered_query_string:
+                        segment.set_attribute(
+                            "url.query",
+                            filtered_query_string,
+                        )
+
+                parsed_url = parse_url(request_websocket.url)
+                segment.set_attribute(
+                    "url.full",
+                    f"{parsed_url.url}?{filtered_query_string}"
+                    if filtered_query_string
+                    else parsed_url.url,
+                )
+
+                # TODO: Add the user properties that are seen in the branch below here once
+                # code is added to respect the `user_info` settings within the data collection
+                # configuration
+            elif should_send_default_pii():
                 segment.set_attribute("url.full", request_websocket.url)
                 segment.set_attribute(
                     "url.query",

tests/integrations/aws_lambda/lambda_functions_with_embedded_sdk/BasicOkDataCollectionUrlQueryAllowlist/.gitignore

@@ -0,0 +1,11 @@
+# Need to add some ignore rules in this directory, because the unit tests will add the Sentry SDK and its dependencies
+# into this directory to create a Lambda function package that contains everything needed to instrument a Lambda function using Sentry.
+
+# Ignore everything
+*
+
+# But not index.py
+!index.py
+
+# And not .gitignore itself
+!.gitignore

tests/integrations/aws_lambda/lambda_functions_with_embedded_sdk/BasicOkDataCollectionUrlQueryAllowlist/index.py

@@ -0,0 +1,24 @@
+import os
+
+import sentry_sdk
+from sentry_sdk.integrations.aws_lambda import AwsLambdaIntegration
+
+sentry_sdk.init(
+    dsn=os.environ.get("SENTRY_DSN"),
+    traces_sample_rate=1.0,
+    integrations=[AwsLambdaIntegration()],
+    _experiments={
+        "data_collection": {
+            "url_query_params": {
+                "mode": "allowlist",
+                # "token" is allowlisted on purpose to show that an allowlist
+                # entry cannot override the built-in sensitive denylist.
+                "terms": ["page", "token"],
+            }
+        }
+    },
+)
+
+
+def handler(event, context):
+    return {"event": event}

tests/integrations/aws_lambda/lambda_functions_with_embedded_sdk/BasicOkDataCollectionUrlQueryDenylist/.gitignore

@@ -0,0 +1,11 @@
+# Need to add some ignore rules in this directory, because the unit tests will add the Sentry SDK and its dependencies
+# into this directory to create a Lambda function package that contains everything needed to instrument a Lambda function using Sentry.
+
+# Ignore everything
+*
+
+# But not index.py
+!index.py
+
+# And not .gitignore itself
+!.gitignore

tests/integrations/aws_lambda/lambda_functions_with_embedded_sdk/BasicOkDataCollectionUrlQueryDenylist/index.py

@@ -0,0 +1,24 @@
+import os
+
+import sentry_sdk
+from sentry_sdk.integrations.aws_lambda import AwsLambdaIntegration
+
+sentry_sdk.init(
+    dsn=os.environ.get("SENTRY_DSN"),
+    traces_sample_rate=1.0,
+    integrations=[AwsLambdaIntegration()],
+    _experiments={
+        "data_collection": {
+            "url_query_params": {
+                "mode": "denylist",
+                # Custom terms deny otherwise non-sensitive query params on top
+                # of the built-in sensitive denylist.
+                "terms": ["tracking"],
+            }
+        }
+    },
+)
+
+
+def handler(event, context):
+    return {"event": event}

tests/integrations/aws_lambda/lambda_functions_with_embedded_sdk/BasicOkDataCollectionUrlQueryOff/.gitignore

@@ -0,0 +1,11 @@
+# Need to add some ignore rules in this directory, because the unit tests will add the Sentry SDK and its dependencies
+# into this directory to create a Lambda function package that contains everything needed to instrument a Lambda function using Sentry.
+
+# Ignore everything
+*
+
+# But not index.py
+!index.py
+
+# And not .gitignore itself
+!.gitignore

tests/integrations/aws_lambda/lambda_functions_with_embedded_sdk/BasicOkDataCollectionUrlQueryOff/index.py

@@ -0,0 +1,19 @@
+import os
+
+import sentry_sdk
+from sentry_sdk.integrations.aws_lambda import AwsLambdaIntegration
+
+sentry_sdk.init(
+    dsn=os.environ.get("SENTRY_DSN"),
+    traces_sample_rate=1.0,
+    integrations=[AwsLambdaIntegration()],
+    _experiments={
+        "data_collection": {
+            "url_query_params": {"mode": "off"},
+        }
+    },
+)
+
+
+def handler(event, context):
+    return {"event": event}

tests/integrations/aws_lambda/lambda_functions_with_embedded_sdk/BasicOkSpanStreamingDataCollection/.gitignore

@@ -0,0 +1,11 @@
+# Need to add some ignore rules in this directory, because the unit tests will add the Sentry SDK and its dependencies
+# into this directory to create a Lambda function package that contains everything needed to instrument a Lambda function using Sentry.
+
+# Ignore everything
+*
+
+# But not index.py
+!index.py
+
+# And not .gitignore itself
+!.gitignore

tests/integrations/aws_lambda/lambda_functions_with_embedded_sdk/BasicOkSpanStreamingDataCollection/index.py

@@ -0,0 +1,23 @@
+import os
+
+import sentry_sdk
+from sentry_sdk.integrations.aws_lambda import AwsLambdaIntegration
+
+sentry_sdk.init(
+    dsn=os.environ.get("SENTRY_DSN"),
+    traces_sample_rate=1.0,
+    integrations=[AwsLambdaIntegration()],
+    trace_lifecycle="stream",
+    _experiments={
+        "data_collection": {
+            "url_query_params": {
+                "mode": "denylist",
+                "terms": ["tracking"],
+            }
+        },
+    },
+)
+
+
+def handler(event, context):
+    return {"event": event}