From ee8ff094026783d0d7af7ec909277dbf5741ae70 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Wed, 28 Oct 2026 12:16:32 -0400 Subject: [PATCH 1/7] feat(api_core): centralize telemetry error attribute and trace context builders in _observability - Move error attribute extraction from gapic_v1.method into _observability.build_telemetry_error_attributes with single-pass exception unwrapping, rpc.response.status_code resolution, native GoogleAPICallError property access, raw gRPC RpcError trailing_metadata parsing, and optional exception.* fields. - Update gapic_v1.method and tests to call _observability.build_telemetry_error_attributes directly without separate _extract_status_code calls. - Add has_opentelemetry() (with module-level caching) and extract_trace_context() in _observability. - Add unit tests achieving 100% statement and branch coverage. --- .../google/api_core/_observability.py | 231 +++++++- .../google/api_core/gapic_v1/method.py | 103 +--- packages/google-api-core/tests/conftest.py | 1 + .../tests/unit/gapic/test_method.py | 203 +------ .../tests/unit/test_observability.py | 537 +++++++++++++++++- 5 files changed, 773 insertions(+), 302 deletions(-) diff --git a/packages/google-api-core/google/api_core/_observability.py b/packages/google-api-core/google/api_core/_observability.py index 34e1ca152c37..0442e07fb5e5 100644 --- a/packages/google-api-core/google/api_core/_observability.py +++ b/packages/google-api-core/google/api_core/_observability.py @@ -19,10 +19,11 @@ from __future__ import annotations import contextlib +import traceback import urllib.parse from typing import TYPE_CHECKING, Any, Callable, Sequence -from google.api_core import _feature_gating_helpers +from google.api_core import _feature_gating_helpers, exceptions from google.api_core.client_options import ClientOptions try: @@ -41,6 +42,22 @@ from google.api_core.grpc_helpers import ClientInterceptor # noqa: F401 _TRACER_PROVIDER = "tracer_provider" +_HAS_OPENTELEMETRY: bool | None = None + + +def has_opentelemetry() -> bool: + """Checks once per process if opentelemetry-api is installed.""" + global _HAS_OPENTELEMETRY + if _HAS_OPENTELEMETRY is None: + try: + # Ignored by mypy: Optional dependency only checked at runtime + # flake8: Imported only to verify package installation + import opentelemetry.trace # type: ignore[import-not-found] # noqa: F401 + + _HAS_OPENTELEMETRY = True + except ImportError: + _HAS_OPENTELEMETRY = False + return _HAS_OPENTELEMETRY def is_otel_capabilities_enabled( @@ -74,6 +91,218 @@ def is_otel_capabilities_enabled( return False +def extract_trace_context(span: Any = None) -> dict[str, Any]: + """Extracts trace_id, span_id, and Cloud Logging trace correlation fields. + + If `span` is None, falls back to `opentelemetry.trace.get_current_span()` + when OpenTelemetry is installed. Returns an empty dict if no valid recording + span context exists or if OpenTelemetry is not installed. + + Args: + span (Optional[Any]): The active OpenTelemetry span, if any. + + Returns: + dict[str, Any]: Trace and span correlation attributes. + """ + if span is None and has_opentelemetry(): + try: + # Ignored by mypy: Optional dependency only loaded after + # has_opentelemetry() check + from opentelemetry import trace # type: ignore[import-not-found] + + span = trace.get_current_span() + # Defensive fail-open: telemetry correlation must never crash an RPC call + # if a custom or broken OpenTelemetry TracerProvider raises at runtime. + except Exception: + span = None + + if span is None or not hasattr(span, "get_span_context"): + return {} + + try: + ctx = span.get_span_context() + # Defensive fail-open: guard against broken third-party or mock span objects. + except Exception: + return {} + + if ctx is None or not getattr(ctx, "is_valid", False): + return {} + + try: + trace_id_hex = f"{ctx.trace_id:032x}" + span_id_hex = f"{ctx.span_id:016x}" + trace_flags = getattr(ctx, "trace_flags", None) + if trace_flags is not None and hasattr(trace_flags, "sampled"): + sampled = bool(trace_flags.sampled) + elif isinstance(trace_flags, int): + sampled = bool(trace_flags & 0x01) + else: + sampled = False + except (TypeError, ValueError, AttributeError): + return {} + + return { + "trace_id": trace_id_hex, + "span_id": span_id_hex, + "logging.googleapis.com/trace": trace_id_hex, + "logging.googleapis.com/spanId": span_id_hex, + "logging.googleapis.com/trace_sampled": sampled, + } + + +def _extract_error_info_fields(target_exc: Any) -> tuple[Any, Any, Any]: + """Resolves (domain, reason, metadata) from an exception, ErrorInfo, or dict.""" + # Prefer native GoogleAPICallError properties (.domain, .reason, .metadata) + # when present; fall back to .error_info or raw gRPC trailing_metadata. + error_info = getattr(target_exc, "error_info", None) + if error_info is None and not ( + getattr(target_exc, "reason", None) + or getattr(target_exc, "domain", None) + or getattr(target_exc, "metadata", None) + ): + rpc_call = ( + target_exc + if hasattr(target_exc, "trailing_metadata") + else getattr(target_exc, "response", None) + ) + if rpc_call is not None and hasattr(rpc_call, "trailing_metadata"): + try: + _, error_info = exceptions._parse_grpc_error_details(rpc_call) + # Defensive fail-open: malformed trailing metadata must not abort + # error attribute extraction. + except Exception: + pass + + source = error_info if error_info is not None else target_exc + if isinstance(source, dict): + return ( + source.get("domain"), + source.get("reason"), + source.get("metadata"), + ) + return ( + getattr(source, "domain", None), + getattr(source, "reason", None), + getattr(source, "metadata", None), + ) + + +def _extract_error_message( + target_exc: Any, http_response: Any = None, fallback: str | None = None +) -> str | None: + """Resolves the human-readable error message across HTTP, gRPC, and Python errors.""" + http_msg = ( + _TraceContext._extract_http_error_message(http_response) + if http_response is not None + else None + ) + details = getattr(target_exc, "details", None) + details_msg = details() if callable(details) else details + exc_msg = str(target_exc) if isinstance(target_exc, BaseException) else None + raw_message = ( + http_msg + or getattr(target_exc, "message", None) + or details_msg + or fallback + or exc_msg + ) + return str(raw_message) if raw_message else None + + +def build_telemetry_error_attributes( + exc: BaseException | None, + status_code: str | None = None, + status_message: str | None = None, + include_exception: bool = False, +) -> dict[str, Any]: + """Extracts rpc.response.status_code, gcp.errors.*, error.type, status.message, and exception.* fields. + + Error details and ErrorInfo structures are resolved by inspecting: + * Chained exceptions: Unwraps RetryError or __cause__ to the root error. + * Canonical status code: Resolves rpc.response.status_code via + exceptions._extract_status_code. + * GoogleAPICallError & ErrorInfo attributes: Reads .domain, .reason, and + .metadata directly (or from .error_info / gRPC trailing_metadata). + * Unified attribute extraction: Populates gcp.errors.domain, error.type, + gcp.errors.metadata.*, status.message, and optional exception.* fields. + + Args: + exc (Optional[BaseException]): An exception (e.g. GoogleAPICallError + or grpc.RpcError) or ErrorInfo object. + status_code (Optional[str]): Optional pre-extracted or HTTP status code + fallback when exc is None or lacks a status code. + status_message (Optional[str]): Optional status message fallback when + exc is None or lacks a message. + include_exception (bool): When True, also populates exception.type, + exception.message, and exception.stacktrace. + + Returns: + dict[str, Any]: Extracted error and exception attributes. + """ + if exc is None: + if status_code is None: + return {} + code_str = str(status_code) + return { + "rpc.response.status_code": code_str, + "error.type": code_str, + "status.message": status_message or f"HTTP {code_str}", + } + + # Unwrap chained exceptions once and resolve the canonical status code + target_exc = getattr(exc, "cause", None) or getattr(exc, "__cause__", None) or exc + resolved_code = exceptions._extract_status_code(target_exc, default=status_code) + domain, reason, metadata = _extract_error_info_fields(target_exc) + + # Identify HTTP transport responses (excluding gRPC Call objects) + http_response = getattr(target_exc, "response", None) + http_status = ( + getattr(http_response, "status_code", getattr(http_response, "status", None)) + if http_response is not None and not hasattr(http_response, "trailing_metadata") + else None + ) + + # Populate OpenTelemetry semantic error attributes and ErrorInfo metadata + attrs: dict[str, Any] = {"rpc.response.status_code": resolved_code} + if domain: + attrs["gcp.errors.domain"] = domain + if reason: + attrs["error.type"] = reason + elif ( + http_status is not None + and not isinstance(http_status, bool) + and str(http_status).isdigit() + ): + attrs["error.type"] = str(http_status) + else: + # Every failed span or log records a low-cardinality error.type; reuse + # the resolved canonical status code name or exception class name. + attrs["error.type"] = resolved_code + + if metadata and hasattr(metadata, "items"): + for k, v in metadata.items(): + attrs[f"gcp.errors.metadata.{k}"] = str(v) + + # Extract human-readable error description for cross-language parity + message = _extract_error_message( + target_exc, + http_response if http_status is not None else None, + status_message, + ) + if message: + attrs["status.message"] = message + + # Populate exception.* diagnostic fields for structured logs when requested + if include_exception and isinstance(exc, BaseException): + attrs["exception.type"] = exc.__class__.__name__ + attrs["exception.message"] = str(exc) + attrs["exception.stacktrace"] = "".join( + traceback.format_exception(type(exc), exc, exc.__traceback__) + ) + + return attrs + + def _extract_endpoint_attributes( client_options: ClientOptions | dict[str, Any] | None = None, default_host: str | None = None, diff --git a/packages/google-api-core/google/api_core/gapic_v1/method.py b/packages/google-api-core/google/api_core/gapic_v1/method.py index 698a4fee9d4b..2dd3073da70d 100644 --- a/packages/google-api-core/google/api_core/gapic_v1/method.py +++ b/packages/google-api-core/google/api_core/gapic_v1/method.py @@ -21,9 +21,9 @@ import contextlib import enum import functools -from typing import Any, List, Optional, Tuple +from typing import List, Tuple -from google.api_core import _observability, exceptions, grpc_helpers +from google.api_core import _observability, grpc_helpers from google.api_core.gapic_v1 import client_info from google.api_core.gapic_v1.client_info import METRICS_METADATA_KEY from google.api_core.timeout import TimeToDeadlineTimeout @@ -126,98 +126,6 @@ def _extract_rpc_identity( return method_str, service, method -def _extract_error_attributes(exc: Optional[BaseException]) -> dict[str, Any]: - """Extract gcp.errors.* and error.type attributes from an exception. - - Error details and ErrorInfo structures are resolved by inspecting the following locations: - * Chained exceptions: Unwraps RetryError or __cause__ to the root exception. - * GoogleAPICallError attributes: Reads ErrorInfo from ._error_info or .error_info. - * Native gRPC trailing metadata: Parses google.rpc.Status binary details from trailing_metadata. - * Unified attribute extraction: Extracts domain, reason, and metadata from ErrorInfo or exception attributes. - - Args: - exc (Optional[BaseException]): An exception (such as GoogleAPICallError or grpc.RpcError) or ErrorInfo object. - - Returns: - dict[str, Any]: Extracted error attributes (e.g. gcp.errors.domain, error.type, gcp.errors.metadata.*). - """ - attrs: dict[str, Any] = {} - if exc is None: - return attrs - - # 1. Unwrap chained exceptions: unwrap RetryError or __cause__ to the root failure - target_exc = getattr(exc, "cause", None) or getattr(exc, "__cause__", None) or exc - - # 2. Check GoogleAPICallError ErrorInfo attributes - error_info = getattr(target_exc, "_error_info", None) or getattr( - target_exc, "error_info", None - ) - - # 3. Check native gRPC trailing metadata for binary google.rpc.Status details - if error_info is None: - rpc_call = ( - target_exc - if hasattr(target_exc, "trailing_metadata") - else getattr(target_exc, "response", None) - ) - if rpc_call is not None and hasattr(rpc_call, "trailing_metadata"): - try: - _, error_info = exceptions._parse_grpc_error_details(rpc_call) - except Exception: - pass - - # 4. Unified attribute extraction: extract domain, reason, and metadata from ErrorInfo or exception attributes - http_response = getattr(target_exc, "response", None) - http_status = ( - getattr(http_response, "status_code", getattr(http_response, "status", None)) - if http_response is not None and not hasattr(http_response, "trailing_metadata") - else None - ) - - source = error_info or target_exc - if isinstance(source, dict): - domain = source.get("domain") - reason = source.get("reason") - metadata = source.get("metadata") - else: - domain = getattr(source, "domain", None) - reason = getattr(source, "reason", None) - metadata = getattr(source, "metadata", None) - - if domain: - attrs["gcp.errors.domain"] = domain - if reason: - attrs["error.type"] = reason - elif isinstance(http_status, int): - attrs["error.type"] = str(http_status) - else: - # Fallback per OpenTelemetry Semantic Conventions: every failed span should record - # a low-cardinality error.type. Use canonical status code name or exception class name. - status_code = exceptions._extract_status_code(target_exc) - attrs["error.type"] = status_code or target_exc.__class__.__name__ - if metadata: - for k, v in metadata.items(): - attrs[f"gcp.errors.metadata.{k}"] = str(v) - - # 5. Extract human-readable error description for cross-language PRD parity - message = ( - _observability._TraceContext._extract_http_error_message(http_response) - if http_status is not None - else None - ) - if not message: - message = getattr(target_exc, "message", None) - if not message and hasattr(target_exc, "details"): - details = target_exc.details - message = details() if callable(details) else details - if not message and isinstance(target_exc, BaseException): - message = str(target_exc) - if message: - attrs["status.message"] = str(message) - - return attrs - - class _GapicCallable(object): """Callable that applies retry, timeout, and metadata logic. @@ -381,10 +289,9 @@ def _trace_span(self): raise except BaseException as exc: if span is not None and hasattr(span, "set_attribute"): - span.set_attribute( - "rpc.response.status_code", exceptions._extract_status_code(exc) - ) - for k, v in _extract_error_attributes(exc).items(): + for k, v in _observability.build_telemetry_error_attributes( + exc + ).items(): span.set_attribute(k, v) raise diff --git a/packages/google-api-core/tests/conftest.py b/packages/google-api-core/tests/conftest.py index d8ef2918c9f2..ad186435fe17 100644 --- a/packages/google-api-core/tests/conftest.py +++ b/packages/google-api-core/tests/conftest.py @@ -43,6 +43,7 @@ def mock_otel(monkeypatch): mock_trace = mock.Mock() mock_trace.get_tracer.return_value = mock_tracer + mock_trace.get_current_span.return_value = mock_span mock_trace.SpanKind.CLIENT = "CLIENT" mock_trace.SpanKind.INTERNAL = "INTERNAL" mock_trace.StatusCode.ERROR = "ERROR" diff --git a/packages/google-api-core/tests/unit/gapic/test_method.py b/packages/google-api-core/tests/unit/gapic/test_method.py index ad094fbe46dd..292f05a73b50 100644 --- a/packages/google-api-core/tests/unit/gapic/test_method.py +++ b/packages/google-api-core/tests/unit/gapic/test_method.py @@ -14,6 +14,7 @@ import datetime import sys +import types from unittest import mock import pytest @@ -780,43 +781,8 @@ def test_wrap_method_otel_tracing_endpoint_attributes_from_default_host( ) -def test_extract_error_attributes_standard_exception(): - """Proves that _extract_error_attributes returns fallback error.type for exceptions without ErrorInfo.""" - assert google.api_core.gapic_v1.method._extract_error_attributes( - ValueError("fail") - ) == {"error.type": "ValueError", "status.message": "fail"} - assert google.api_core.gapic_v1.method._extract_error_attributes( - exceptions.InvalidArgument("invalid argument") - ) == {"error.type": "INVALID_ARGUMENT", "status.message": "invalid argument"} - assert google.api_core.gapic_v1.method._extract_error_attributes(None) == {} - - -def test_extract_error_attributes_with_error_info(): - """Proves that _extract_error_attributes extracts domain, error.type, and metadata from ErrorInfo.""" - import types - - error_info = types.SimpleNamespace( - domain="googleapis.com", - reason="SERVICE_DISABLED", - metadata={ - "service": "secretmanager.googleapis.com", - "consumer": "projects/123", - }, - ) - exc = types.SimpleNamespace(error_info=error_info) - attrs = google.api_core.gapic_v1.method._extract_error_attributes(exc) - assert attrs == { - "gcp.errors.domain": "googleapis.com", - "error.type": "SERVICE_DISABLED", - "gcp.errors.metadata.service": "secretmanager.googleapis.com", - "gcp.errors.metadata.consumer": "projects/123", - } - - def test_wrap_method_otel_tracing_records_gcp_error_attributes(mock_otel): """Proves that method spans record gcp.errors.* attributes when ErrorInfo is present.""" - import types - error_info = types.SimpleNamespace( domain="googleapis.com", reason="RESOURCE_EXHAUSTED", @@ -844,173 +810,6 @@ def test_wrap_method_otel_tracing_records_gcp_error_attributes(mock_otel): mock_otel.span.set_attribute.assert_any_call("status.message", "quota exceeded") -def test_extract_error_attributes_variations(): - """Proves that _extract_error_attributes handles __cause__, gRPC error details parsing, and direct fallbacks.""" - import types - - from google.api_core.gapic_v1.method import _extract_error_attributes - - # 1. __cause__ attribute fallback - inner_err = types.SimpleNamespace( - error_info=types.SimpleNamespace(domain="d", reason="r", metadata={"k": "v"}) - ) - outer_err = types.SimpleNamespace(__cause__=inner_err) - assert _extract_error_attributes(outer_err) == { - "gcp.errors.domain": "d", - "error.type": "r", - "gcp.errors.metadata.k": "v", - } - - # 2. rpc_call with trailing_metadata parsed via _parse_grpc_error_details - rpc_call = types.SimpleNamespace(trailing_metadata=[("meta", "val")]) - exc_with_call = types.SimpleNamespace(trailing_metadata=rpc_call.trailing_metadata) - error_info = types.SimpleNamespace( - domain="parse_d", reason="parse_r", metadata={"foo": "bar"} - ) - with mock.patch( - "google.api_core.exceptions._parse_grpc_error_details", - return_value=(None, error_info), - ): - assert _extract_error_attributes(exc_with_call) == { - "gcp.errors.domain": "parse_d", - "error.type": "parse_r", - "gcp.errors.metadata.foo": "bar", - } - - # 3. rpc_call with response attribute holding trailing_metadata and _parse_grpc_error_details raising Exception - exc_with_resp = types.SimpleNamespace( - response=types.SimpleNamespace(trailing_metadata=[]) - ) - with mock.patch( - "google.api_core.exceptions._parse_grpc_error_details", - side_effect=ValueError("bad proto"), - ): - assert _extract_error_attributes(exc_with_resp) == { - "error.type": "SimpleNamespace" - } - - # 4. error_info with empty domain, empty reason, empty metadata - error_info_empty = types.SimpleNamespace(domain="", reason="", metadata=None) - exc_empty = types.SimpleNamespace(error_info=error_info_empty) - assert _extract_error_attributes(exc_empty) == {"error.type": "SimpleNamespace"} - - # 5. else fallback where target_exc directly has domain, reason, and metadata - exc_fallback = types.SimpleNamespace( - domain="fallback_d", - reason="fallback_r", - metadata={"f_key": 42}, - ) - assert _extract_error_attributes(exc_fallback) == { - "gcp.errors.domain": "fallback_d", - "error.type": "fallback_r", - "gcp.errors.metadata.f_key": "42", - } - - # 6. else fallback with empty attributes (e.g. domain="", reason="", metadata={}) - exc_fallback_empty = types.SimpleNamespace( - domain="", - reason="", - metadata={}, - ) - assert _extract_error_attributes(exc_fallback_empty) == { - "error.type": "SimpleNamespace" - } - - # 7. status.message extraction from .message attribute - exc_with_msg = types.SimpleNamespace(message="api call failed") - assert _extract_error_attributes(exc_with_msg) == { - "error.type": "SimpleNamespace", - "status.message": "api call failed", - } - - # 8. status.message extraction from .details() callable (e.g. gRPC RpcError) - exc_with_details = types.SimpleNamespace(details=lambda: "rpc deadline exceeded") - assert _extract_error_attributes(exc_with_details) == { - "error.type": "SimpleNamespace", - "status.message": "rpc deadline exceeded", - } - - # 9. status.message extraction from Exception string representation - exc_standard = ValueError("invalid argument passed") - assert _extract_error_attributes(exc_standard) == { - "error.type": "ValueError", - "status.message": "invalid argument passed", - } - - # 10. Exception with empty message string does not populate status.message - exc_empty_msg = ValueError("") - assert _extract_error_attributes(exc_empty_msg) == { - "error.type": "ValueError", - } - - # 11. status.message extraction from BaseException subclasses (e.g. asyncio.CancelledError) - import asyncio - - exc_cancelled_with_msg = asyncio.CancelledError("async task cancelled") - assert _extract_error_attributes(exc_cancelled_with_msg) == { - "error.type": "CancelledError", - "status.message": "async task cancelled", - } - - exc_cancelled_empty = asyncio.CancelledError() - assert _extract_error_attributes(exc_cancelled_empty) == { - "error.type": "CancelledError", - } - - # 12. HTTP response attached to GoogleAPICallError (direct and wrapped in RetryError) - http_resp_503 = types.SimpleNamespace( - status_code=503, - json=lambda: {"error": {"message": "Persistent outage"}}, - ) - http_exc = exceptions.ServiceUnavailable( - "POST http://localhost:7469/v1beta1/sequences/23: Persistent outage", - response=http_resp_503, - ) - assert _extract_error_attributes(http_exc) == { - "error.type": "503", - "status.message": "Persistent outage", - } - retry_http_exc = exceptions.RetryError("Deadline of 0.1s exceeded", cause=http_exc) - assert _extract_error_attributes(retry_http_exc) == { - "error.type": "503", - "status.message": "Persistent outage", - } - - # 13. HTTP response with dict ErrorInfo (from JSON format_http_response_error) - http_exc_dict_info = exceptions.BadRequest( - "POST http://localhost:7469/v1beta1/echo:echo: Quota exceeded", - response=types.SimpleNamespace( - status_code=400, - json=lambda: {"error": {"message": "Quota exceeded"}}, - ), - error_info={ - "domain": "googleapis.com", - "reason": "RATE_LIMIT_EXCEEDED", - "metadata": {"service": "showcase.googleapis.com"}, - }, - ) - assert _extract_error_attributes(http_exc_dict_info) == { - "gcp.errors.domain": "googleapis.com", - "error.type": "RATE_LIMIT_EXCEEDED", - "gcp.errors.metadata.service": "showcase.googleapis.com", - "status.message": "Quota exceeded", - } - - # 14. gRPC Call-like response (has trailing_metadata) is NOT treated as an HTTP response - grpc_call_like = types.SimpleNamespace( - status_code=503, - trailing_metadata=(), - ) - grpc_exc_with_resp = exceptions.ServiceUnavailable( - "gRPC transport error", - response=grpc_call_like, - ) - assert _extract_error_attributes(grpc_exc_with_resp) == { - "error.type": "UNAVAILABLE", - "status.message": "gRPC transport error", - } - - def test_wrap_method_otel_tracing_partial_span_capabilities(mock_otel): """Proves handling when span has or lacks set_attribute.""" # Test span with set_attribute diff --git a/packages/google-api-core/tests/unit/test_observability.py b/packages/google-api-core/tests/unit/test_observability.py index 7cd31e95070f..86d6470802f4 100644 --- a/packages/google-api-core/tests/unit/test_observability.py +++ b/packages/google-api-core/tests/unit/test_observability.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import asyncio import sys import types import urllib.parse @@ -19,7 +20,7 @@ import pytest -from google.api_core import _observability +from google.api_core import _observability, exceptions from google.api_core._feature_gating_helpers import FeatureGatingError from google.api_core.client_options import ClientOptions @@ -1633,3 +1634,537 @@ def test_suppressing_interceptors_define_slots(): assert _observability._AsyncSuppressingClientInterceptor.__slots__ == () assert "__dict__" not in _observability._SuppressingClientInterceptor.__dict__ assert "__dict__" not in _observability._AsyncSuppressingClientInterceptor.__dict__ + + +def test_has_opentelemetry_caching(monkeypatch): + """Proves has_opentelemetry caches True and False states across calls.""" + monkeypatch.setattr(_observability, "_HAS_OPENTELEMETRY", None) + monkeypatch.setitem(sys.modules, "opentelemetry", types.ModuleType("opentelemetry")) + monkeypatch.setitem( + sys.modules, + "opentelemetry.trace", + types.ModuleType("opentelemetry.trace"), + ) + + assert _observability.has_opentelemetry() is True + assert _observability._HAS_OPENTELEMETRY is True + # Cached call returns True even if sys.modules changes + monkeypatch.setitem(sys.modules, "opentelemetry.trace", None) + assert _observability.has_opentelemetry() is True + + # Reset sentinel and test when ImportError occurs + monkeypatch.setattr(_observability, "_HAS_OPENTELEMETRY", None) + assert _observability.has_opentelemetry() is False + assert _observability._HAS_OPENTELEMETRY is False + + # Cached call returns False even if opentelemetry.trace becomes available + monkeypatch.setitem( + sys.modules, + "opentelemetry.trace", + types.ModuleType("opentelemetry.trace"), + ) + assert _observability.has_opentelemetry() is False + + +@pytest.mark.parametrize( + "trace_flags,expected_sampled", + [ + pytest.param( + types.SimpleNamespace(sampled=True), True, id="flag_obj_sampled_true" + ), + pytest.param( + types.SimpleNamespace(sampled=False), + False, + id="flag_obj_sampled_false", + ), + pytest.param(1, True, id="int_bitmask_sampled_true"), + pytest.param(0, False, id="int_bitmask_sampled_false"), + pytest.param(2, False, id="int_bitmask_other_bit_unsampled"), + pytest.param(None, False, id="none_trace_flags"), + ], +) +def test_extract_trace_context_explicit_span_flags(trace_flags, expected_sampled): + """Proves extract_trace_context formats hex IDs and resolves trace_flags.""" + span = mock.Mock(spec_set=["get_span_context"]) + span.get_span_context.return_value = types.SimpleNamespace( + is_valid=True, + trace_id=0x0123456789ABCDEF0123456789ABCDEF, + span_id=0x0123456789ABCDEF, + trace_flags=trace_flags, + ) + + expected = { + "trace_id": "0123456789abcdef0123456789abcdef", + "span_id": "0123456789abcdef", + "logging.googleapis.com/trace": "0123456789abcdef0123456789abcdef", + "logging.googleapis.com/spanId": "0123456789abcdef", + "logging.googleapis.com/trace_sampled": expected_sampled, + } + assert _observability.extract_trace_context(span) == expected + + +def test_extract_trace_context_invalid_or_broken_span(): + """Proves extract_trace_context returns {} on broken spans or non-int IDs.""" + bad_span = mock.Mock(spec_set=["get_span_context"]) + bad_span.get_span_context.side_effect = RuntimeError("broken span") + assert _observability.extract_trace_context(bad_span) == {} + + invalid_ctx_span = mock.Mock(spec_set=["get_span_context"]) + invalid_ctx_span.get_span_context.return_value = types.SimpleNamespace( + is_valid=False + ) + assert _observability.extract_trace_context(invalid_ctx_span) == {} + + bad_ctx_span = mock.Mock(spec_set=["get_span_context"]) + bad_ctx_span.get_span_context.return_value = types.SimpleNamespace( + is_valid=True, trace_id="not_an_int", span_id="not_an_int" + ) + assert _observability.extract_trace_context(bad_ctx_span) == {} + + +def test_extract_trace_context_ambient_fallback(monkeypatch, mock_otel): + """Proves extract_trace_context falls back to get_current_span().""" + monkeypatch.setattr(_observability, "_HAS_OPENTELEMETRY", True) + mock_otel.span.get_span_context.return_value = types.SimpleNamespace( + is_valid=True, + trace_id=0x0123456789ABCDEF0123456789ABCDEF, + span_id=0x0123456789ABCDEF, + trace_flags=types.SimpleNamespace(sampled=True), + ) + + res = _observability.extract_trace_context(None) + assert res["trace_id"] == "0123456789abcdef0123456789abcdef" + assert res["logging.googleapis.com/trace_sampled"] is True + + # Ambient get_current_span raising an exception returns {} + mock_otel.trace.get_current_span.side_effect = RuntimeError("ambient error") + assert _observability.extract_trace_context(None) == {} + + # OTel not installed returns {} + monkeypatch.setattr(_observability, "_HAS_OPENTELEMETRY", False) + assert _observability.extract_trace_context(None) == {} + + +@pytest.mark.parametrize( + "target_exc,expected", + [ + pytest.param( + exceptions.BadRequest( + "bad request", + error_info={ + "domain": "googleapis.com", + "reason": "RATE_LIMIT_EXCEEDED", + "metadata": {"service": "showcase.googleapis.com"}, + }, + ), + ( + "googleapis.com", + "RATE_LIMIT_EXCEEDED", + {"service": "showcase.googleapis.com"}, + ), + id="google_api_call_error_with_dict_error_info", + ), + pytest.param( + types.SimpleNamespace( + error_info=types.SimpleNamespace( + domain="googleapis.com", + reason="SERVICE_DISABLED", + metadata={ + "service": "secretmanager.googleapis.com", + "consumer": "projects/123", + }, + ) + ), + ( + "googleapis.com", + "SERVICE_DISABLED", + { + "service": "secretmanager.googleapis.com", + "consumer": "projects/123", + }, + ), + id="error_info_namespace", + ), + pytest.param( + types.SimpleNamespace( + domain="fallback_d", + reason="fallback_r", + metadata={"f_key": 42}, + ), + ("fallback_d", "fallback_r", {"f_key": 42}), + id="direct_attributes_fallback", + ), + pytest.param( + { + "domain": "googleapis.com", + "reason": "RATE_LIMIT_EXCEEDED", + "metadata": {"quota": "100"}, + }, + ("googleapis.com", "RATE_LIMIT_EXCEEDED", {"quota": "100"}), + id="raw_dict_input", + ), + pytest.param( + types.SimpleNamespace( + error_info=types.SimpleNamespace(domain="", reason="", metadata=None) + ), + ("", "", None), + id="empty_error_info_attributes", + ), + pytest.param( + ValueError("invalid argument"), + (None, None, None), + id="standard_exception_without_error_info", + ), + ], +) +def test_extract_error_info_fields(target_exc, expected): + """Proves _extract_error_info_fields resolves (domain, reason, metadata) across exception and dict shapes.""" + assert _observability._extract_error_info_fields(target_exc) == expected + + +@pytest.mark.parametrize( + "target_exc,return_value,side_effect,expected", + [ + pytest.param( + types.SimpleNamespace(trailing_metadata=[("meta", "val")]), + ( + None, + types.SimpleNamespace( + domain="parse_d", + reason="parse_r", + metadata={"foo": "bar"}, + ), + ), + None, + ("parse_d", "parse_r", {"foo": "bar"}), + id="direct_trailing_metadata_parsed", + ), + pytest.param( + types.SimpleNamespace(response=types.SimpleNamespace(trailing_metadata=[])), + None, + ValueError("bad proto"), + (None, None, None), + id="response_trailing_metadata_parse_error_fails_open", + ), + ], +) +def test_extract_error_info_fields_grpc_trailing_metadata( + target_exc, return_value, side_effect, expected +): + """Proves _extract_error_info_fields parses gRPC trailing metadata and fails open on malformed protos.""" + with mock.patch( + "google.api_core.exceptions._parse_grpc_error_details", + return_value=return_value, + side_effect=side_effect, + ): + assert _observability._extract_error_info_fields(target_exc) == expected + + +@pytest.mark.parametrize( + "target_exc,http_response,fallback,expected", + [ + pytest.param( + exceptions.ServiceUnavailable( + "POST http://localhost:7469/v1beta1/sequences/23: Persistent outage" + ), + types.SimpleNamespace( + status_code=503, + json=lambda: {"error": {"message": "Persistent outage"}}, + ), + None, + "Persistent outage", + id="http_response_json_takes_precedence", + ), + pytest.param( + types.SimpleNamespace(message="api call failed"), + None, + None, + "api call failed", + id="message_attribute", + ), + pytest.param( + types.SimpleNamespace(details=lambda: "rpc deadline exceeded"), + None, + None, + "rpc deadline exceeded", + id="details_callable", + ), + pytest.param( + types.SimpleNamespace(details="static details string"), + None, + None, + "static details string", + id="details_string_attribute", + ), + pytest.param( + types.SimpleNamespace(), + None, + "explicit fallback", + "explicit fallback", + id="fallback_argument", + ), + pytest.param( + ValueError("invalid argument passed"), + None, + None, + "invalid argument passed", + id="standard_exception_str", + ), + pytest.param( + ValueError(""), + None, + None, + None, + id="standard_exception_empty_str", + ), + pytest.param( + asyncio.CancelledError("async task cancelled"), + None, + None, + "async task cancelled", + id="base_exception_subclass_with_msg", + ), + pytest.param( + asyncio.CancelledError(), + None, + None, + None, + id="base_exception_subclass_empty", + ), + ], +) +def test_extract_error_message(target_exc, http_response, fallback, expected): + """Proves _extract_error_message resolves messages across HTTP responses, .message, .details(), fallback, and str(exc).""" + assert ( + _observability._extract_error_message(target_exc, http_response, fallback) + == expected + ) + + +_INVALID_ARGUMENT_CODE = ( + "INVALID_ARGUMENT" if exceptions.grpc is not None else "BAD_REQUEST" +) +_UNAVAILABLE_CODE = ( + "UNAVAILABLE" if exceptions.grpc is not None else "SERVICE_UNAVAILABLE" +) + + +@pytest.mark.parametrize( + "exc,kwargs,expected", + [ + pytest.param(None, {}, {}, id="none_exc_no_kwargs"), + pytest.param( + None, + {"status_code": "503"}, + { + "rpc.response.status_code": "503", + "error.type": "503", + "status.message": "HTTP 503", + }, + id="none_exc_with_status_code", + ), + pytest.param( + None, + {"status_code": "503", "status_message": "Service Unavailable"}, + { + "rpc.response.status_code": "503", + "error.type": "503", + "status.message": "Service Unavailable", + }, + id="none_exc_with_status_code_and_message", + ), + pytest.param( + ValueError("fail"), + {}, + { + "rpc.response.status_code": "ValueError", + "error.type": "ValueError", + "status.message": "fail", + }, + id="standard_exception", + ), + pytest.param( + ValueError(""), + {}, + { + "rpc.response.status_code": "ValueError", + "error.type": "ValueError", + }, + id="standard_exception_empty_message", + ), + pytest.param( + exceptions.InvalidArgument("invalid argument"), + {}, + { + "rpc.response.status_code": _INVALID_ARGUMENT_CODE, + "error.type": _INVALID_ARGUMENT_CODE, + "status.message": "invalid argument", + }, + id="google_api_call_error_without_error_info", + ), + pytest.param( + types.SimpleNamespace( + error_info=types.SimpleNamespace( + domain="googleapis.com", + reason="SERVICE_DISABLED", + metadata={ + "service": "secretmanager.googleapis.com", + "consumer": "projects/123", + "count": 42, + }, + ) + ), + {}, + { + "rpc.response.status_code": "SimpleNamespace", + "gcp.errors.domain": "googleapis.com", + "error.type": "SERVICE_DISABLED", + "gcp.errors.metadata.service": "secretmanager.googleapis.com", + "gcp.errors.metadata.consumer": "projects/123", + "gcp.errors.metadata.count": "42", + }, + id="error_info_with_domain_reason_and_metadata", + ), + pytest.param( + types.SimpleNamespace(domain="", reason="", metadata={}), + {}, + { + "rpc.response.status_code": "SimpleNamespace", + "error.type": "SimpleNamespace", + }, + id="empty_error_info_falls_back_to_resolved_code", + ), + pytest.param( + { + "domain": "googleapis.com", + "reason": "RATE_LIMIT_EXCEEDED", + "metadata": {"quota": "100"}, + }, + {"status_message": "Quota exceeded"}, + { + "rpc.response.status_code": "dict", + "gcp.errors.domain": "googleapis.com", + "error.type": "RATE_LIMIT_EXCEEDED", + "gcp.errors.metadata.quota": "100", + "status.message": "Quota exceeded", + }, + id="dict_error_info", + ), + pytest.param( + types.SimpleNamespace( + __cause__=types.SimpleNamespace( + error_info=types.SimpleNamespace( + domain="d", reason="r", metadata={"k": "v"} + ) + ) + ), + {}, + { + "rpc.response.status_code": "SimpleNamespace", + "gcp.errors.domain": "d", + "error.type": "r", + "gcp.errors.metadata.k": "v", + }, + id="dunder_cause_unwrapping", + ), + pytest.param( + exceptions.ServiceUnavailable( + "POST http://localhost:7469/v1beta1/sequences/23: Persistent outage", + response=types.SimpleNamespace( + status_code=503, + json=lambda: {"error": {"message": "Persistent outage"}}, + ), + ), + {}, + { + "rpc.response.status_code": _UNAVAILABLE_CODE, + "error.type": "503", + "status.message": "Persistent outage", + }, + id="http_response_error_without_reason", + ), + pytest.param( + exceptions.RetryError( + "Deadline of 0.1s exceeded", + cause=exceptions.ServiceUnavailable( + "POST http://localhost:7469/v1beta1/sequences/23: Persistent outage", + response=types.SimpleNamespace( + status_code=503, + json=lambda: {"error": {"message": "Persistent outage"}}, + ), + ), + ), + {}, + { + "rpc.response.status_code": _UNAVAILABLE_CODE, + "error.type": "503", + "status.message": "Persistent outage", + }, + id="retry_error_wrapping_http_response_error", + ), + pytest.param( + exceptions.BadRequest( + "POST http://localhost:7469/v1beta1/echo:echo: Quota exceeded", + response=types.SimpleNamespace( + status_code=400, + json=lambda: {"error": {"message": "Quota exceeded"}}, + ), + error_info={ + "domain": "googleapis.com", + "reason": "RATE_LIMIT_EXCEEDED", + "metadata": {"service": "showcase.googleapis.com"}, + }, + ), + {}, + { + "rpc.response.status_code": "BAD_REQUEST", + "gcp.errors.domain": "googleapis.com", + "error.type": "RATE_LIMIT_EXCEEDED", + "gcp.errors.metadata.service": "showcase.googleapis.com", + "status.message": "Quota exceeded", + }, + id="http_response_error_with_dict_error_info", + ), + pytest.param( + exceptions.ServiceUnavailable( + "gRPC transport error", + response=types.SimpleNamespace( + status_code=503, + trailing_metadata=(), + ), + ), + {}, + { + "rpc.response.status_code": _UNAVAILABLE_CODE, + "error.type": _UNAVAILABLE_CODE, + "status.message": "gRPC transport error", + }, + id="grpc_call_like_response_excluded_from_http_handling", + ), + ], +) +def test_build_telemetry_error_attributes(exc, kwargs, expected): + """Proves build_telemetry_error_attributes populates status code, error.type, domain, metadata, and message.""" + assert _observability.build_telemetry_error_attributes(exc, **kwargs) == expected + + +def test_build_telemetry_error_attributes_include_exception(): + """Proves include_exception=True formats real exception stacktraces.""" + try: + raise RuntimeError("") + except RuntimeError as caught_exc: + exc_empty = caught_exc + + attrs_inc = _observability.build_telemetry_error_attributes( + exc_empty, + status_code="UNAVAILABLE", + status_message="backend down", + include_exception=True, + ) + assert attrs_inc["rpc.response.status_code"] == "UNAVAILABLE" + assert attrs_inc["error.type"] == "UNAVAILABLE" + assert attrs_inc["status.message"] == "backend down" + assert attrs_inc["exception.type"] == "RuntimeError" + assert attrs_inc["exception.message"] == "" + assert "Traceback (most recent call last):" in attrs_inc["exception.stacktrace"] + assert "raise RuntimeError" in attrs_inc["exception.stacktrace"] From 8a5a6742683a786db2cc78978c0e5dcb8fcf873b Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Thu, 5 Nov 2026 05:01:39 -0500 Subject: [PATCH 2/7] refactor(api_core): simplify extract_trace_context with single fail-open try block and bitmask check --- .../google/api_core/_observability.py | 55 ++++++++----------- .../tests/unit/test_observability.py | 18 ++---- 2 files changed, 27 insertions(+), 46 deletions(-) diff --git a/packages/google-api-core/google/api_core/_observability.py b/packages/google-api-core/google/api_core/_observability.py index 0442e07fb5e5..899d7dc346f0 100644 --- a/packages/google-api-core/google/api_core/_observability.py +++ b/packages/google-api-core/google/api_core/_observability.py @@ -104,50 +104,39 @@ def extract_trace_context(span: Any = None) -> dict[str, Any]: Returns: dict[str, Any]: Trace and span correlation attributes. """ - if span is None and has_opentelemetry(): - try: + try: + if span is None: + if not has_opentelemetry(): + return {} # Ignored by mypy: Optional dependency only loaded after # has_opentelemetry() check from opentelemetry import trace # type: ignore[import-not-found] span = trace.get_current_span() - # Defensive fail-open: telemetry correlation must never crash an RPC call - # if a custom or broken OpenTelemetry TracerProvider raises at runtime. - except Exception: - span = None - - if span is None or not hasattr(span, "get_span_context"): - return {} - try: ctx = span.get_span_context() - # Defensive fail-open: guard against broken third-party or mock span objects. - except Exception: - return {} - - if ctx is None or not getattr(ctx, "is_valid", False): - return {} + if not ctx or not ctx.is_valid: + return {} - try: trace_id_hex = f"{ctx.trace_id:032x}" span_id_hex = f"{ctx.span_id:016x}" - trace_flags = getattr(ctx, "trace_flags", None) - if trace_flags is not None and hasattr(trace_flags, "sampled"): - sampled = bool(trace_flags.sampled) - elif isinstance(trace_flags, int): - sampled = bool(trace_flags & 0x01) - else: - sampled = False - except (TypeError, ValueError, AttributeError): - return {} + # Per W3C Trace Context, trace_flags is an 8-bit integer (TraceFlags(int)) + # that uses bits as configuration flags; the least-significant (last) bit + # (0x01) indicates whether the span is sampled (True: recorded/exported, + # allowing Cloud Logging to link the log entry to Cloud Trace) or not + # (False: not recorded). + sampled = bool(getattr(ctx, "trace_flags", 0) & 0x01) - return { - "trace_id": trace_id_hex, - "span_id": span_id_hex, - "logging.googleapis.com/trace": trace_id_hex, - "logging.googleapis.com/spanId": span_id_hex, - "logging.googleapis.com/trace_sampled": sampled, - } + return { + "trace_id": trace_id_hex, + "span_id": span_id_hex, + "logging.googleapis.com/trace": trace_id_hex, + "logging.googleapis.com/spanId": span_id_hex, + "logging.googleapis.com/trace_sampled": sampled, + } + except Exception: + # Defensive fail-open: telemetry correlation must never crash an RPC call. + return {} def _extract_error_info_fields(target_exc: Any) -> tuple[Any, Any, Any]: diff --git a/packages/google-api-core/tests/unit/test_observability.py b/packages/google-api-core/tests/unit/test_observability.py index 86d6470802f4..012418b3c242 100644 --- a/packages/google-api-core/tests/unit/test_observability.py +++ b/packages/google-api-core/tests/unit/test_observability.py @@ -1669,18 +1669,10 @@ def test_has_opentelemetry_caching(monkeypatch): @pytest.mark.parametrize( "trace_flags,expected_sampled", [ - pytest.param( - types.SimpleNamespace(sampled=True), True, id="flag_obj_sampled_true" - ), - pytest.param( - types.SimpleNamespace(sampled=False), - False, - id="flag_obj_sampled_false", - ), - pytest.param(1, True, id="int_bitmask_sampled_true"), - pytest.param(0, False, id="int_bitmask_sampled_false"), - pytest.param(2, False, id="int_bitmask_other_bit_unsampled"), - pytest.param(None, False, id="none_trace_flags"), + pytest.param(0x01, True, id="int_bitmask_sampled_true"), + pytest.param(0x00, False, id="int_bitmask_sampled_false"), + pytest.param(0x02, False, id="int_bitmask_other_bit_unsampled"), + pytest.param(0x03, True, id="int_bitmask_multiple_bits_sampled"), ], ) def test_extract_trace_context_explicit_span_flags(trace_flags, expected_sampled): @@ -1729,7 +1721,7 @@ def test_extract_trace_context_ambient_fallback(monkeypatch, mock_otel): is_valid=True, trace_id=0x0123456789ABCDEF0123456789ABCDEF, span_id=0x0123456789ABCDEF, - trace_flags=types.SimpleNamespace(sampled=True), + trace_flags=0x01, ) res = _observability.extract_trace_context(None) From 653d02d7b711804e155449d23dec0d725fe5b15f Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Thu, 5 Nov 2026 05:52:37 -0500 Subject: [PATCH 3/7] refactor(api_core): simplify HTTP response validation and attribute updates in build_telemetry_error_attributes --- .../google/api_core/_observability.py | 37 +++++++++---------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/packages/google-api-core/google/api_core/_observability.py b/packages/google-api-core/google/api_core/_observability.py index 899d7dc346f0..e92c462a3013 100644 --- a/packages/google-api-core/google/api_core/_observability.py +++ b/packages/google-api-core/google/api_core/_observability.py @@ -238,18 +238,25 @@ def build_telemetry_error_attributes( "status.message": status_message or f"HTTP {code_str}", } - # Unwrap chained exceptions once and resolve the canonical status code + # Unwrap chained wrapper exceptions (e.g. RetryError) by one level to reach + # the enriched GoogleAPICallError without unwrapping past it to the raw + # transport error. target_exc = getattr(exc, "cause", None) or getattr(exc, "__cause__", None) or exc resolved_code = exceptions._extract_status_code(target_exc, default=status_code) domain, reason, metadata = _extract_error_info_fields(target_exc) - # Identify HTTP transport responses (excluding gRPC Call objects) + # GoogleAPICallError.response holds either an HTTP Response or a gRPC Call + # (which has trailing_metadata). Validate once up front so http_response is + # only retained when it is an actual HTTP response with an integer status code. http_response = getattr(target_exc, "response", None) http_status = ( getattr(http_response, "status_code", getattr(http_response, "status", None)) if http_response is not None and not hasattr(http_response, "trailing_metadata") else None ) + if not isinstance(http_status, int) or isinstance(http_status, bool): + http_status = None + http_response = None # Populate OpenTelemetry semantic error attributes and ErrorInfo metadata attrs: dict[str, Any] = {"rpc.response.status_code": resolved_code} @@ -257,37 +264,27 @@ def build_telemetry_error_attributes( attrs["gcp.errors.domain"] = domain if reason: attrs["error.type"] = reason - elif ( - http_status is not None - and not isinstance(http_status, bool) - and str(http_status).isdigit() - ): + elif http_status is not None: attrs["error.type"] = str(http_status) else: # Every failed span or log records a low-cardinality error.type; reuse # the resolved canonical status code name or exception class name. attrs["error.type"] = resolved_code - if metadata and hasattr(metadata, "items"): - for k, v in metadata.items(): - attrs[f"gcp.errors.metadata.{k}"] = str(v) + if metadata: + attrs.update({f"gcp.errors.metadata.{k}": str(v) for k, v in metadata.items()}) # Extract human-readable error description for cross-language parity - message = _extract_error_message( - target_exc, - http_response if http_status is not None else None, - status_message, - ) + message = _extract_error_message(target_exc, http_response, status_message) if message: attrs["status.message"] = message - # Populate exception.* diagnostic fields for structured logs when requested - if include_exception and isinstance(exc, BaseException): + # Populate exception.* diagnostic fields for the terminal exception raised to + # the caller when structured logging is enabled + if include_exception: attrs["exception.type"] = exc.__class__.__name__ attrs["exception.message"] = str(exc) - attrs["exception.stacktrace"] = "".join( - traceback.format_exception(type(exc), exc, exc.__traceback__) - ) + attrs["exception.stacktrace"] = "".join(traceback.format_exception(exc)) return attrs From f2c01880a08c20ea9bf9617bcb5fbe4140ee11b5 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Thu, 5 Nov 2026 06:48:18 -0500 Subject: [PATCH 4/7] refactor(api_core): cache has_opentelemetry with functools.cache --- .../google/api_core/_observability.py | 20 +++--- .../tests/unit/test_observability.py | 65 +++++++++++-------- 2 files changed, 47 insertions(+), 38 deletions(-) diff --git a/packages/google-api-core/google/api_core/_observability.py b/packages/google-api-core/google/api_core/_observability.py index e92c462a3013..7e16b574580f 100644 --- a/packages/google-api-core/google/api_core/_observability.py +++ b/packages/google-api-core/google/api_core/_observability.py @@ -19,6 +19,7 @@ from __future__ import annotations import contextlib +import functools import traceback import urllib.parse from typing import TYPE_CHECKING, Any, Callable, Sequence @@ -42,22 +43,19 @@ from google.api_core.grpc_helpers import ClientInterceptor # noqa: F401 _TRACER_PROVIDER = "tracer_provider" -_HAS_OPENTELEMETRY: bool | None = None +@functools.cache def has_opentelemetry() -> bool: """Checks once per process if opentelemetry-api is installed.""" - global _HAS_OPENTELEMETRY - if _HAS_OPENTELEMETRY is None: - try: - # Ignored by mypy: Optional dependency only checked at runtime - # flake8: Imported only to verify package installation - import opentelemetry.trace # type: ignore[import-not-found] # noqa: F401 + try: + # Ignored by mypy: Optional dependency only checked at runtime + # flake8: Imported only to verify package installation + import opentelemetry.trace # type: ignore[import-not-found] # noqa: F401 - _HAS_OPENTELEMETRY = True - except ImportError: - _HAS_OPENTELEMETRY = False - return _HAS_OPENTELEMETRY + return True + except ImportError: + return False def is_otel_capabilities_enabled( diff --git a/packages/google-api-core/tests/unit/test_observability.py b/packages/google-api-core/tests/unit/test_observability.py index 012418b3c242..dd831607c272 100644 --- a/packages/google-api-core/tests/unit/test_observability.py +++ b/packages/google-api-core/tests/unit/test_observability.py @@ -1636,34 +1636,45 @@ def test_suppressing_interceptors_define_slots(): assert "__dict__" not in _observability._AsyncSuppressingClientInterceptor.__dict__ -def test_has_opentelemetry_caching(monkeypatch): - """Proves has_opentelemetry caches True and False states across calls.""" - monkeypatch.setattr(_observability, "_HAS_OPENTELEMETRY", None) - monkeypatch.setitem(sys.modules, "opentelemetry", types.ModuleType("opentelemetry")) - monkeypatch.setitem( - sys.modules, - "opentelemetry.trace", - types.ModuleType("opentelemetry.trace"), - ) +def test_has_opentelemetry_installed_and_cached(monkeypatch): + """When installed, returns True and caches the result across calls.""" + _observability.has_opentelemetry.cache_clear() + try: + monkeypatch.setitem( + sys.modules, "opentelemetry", types.ModuleType("opentelemetry") + ) + monkeypatch.setitem( + sys.modules, + "opentelemetry.trace", + types.ModuleType("opentelemetry.trace"), + ) - assert _observability.has_opentelemetry() is True - assert _observability._HAS_OPENTELEMETRY is True - # Cached call returns True even if sys.modules changes - monkeypatch.setitem(sys.modules, "opentelemetry.trace", None) - assert _observability.has_opentelemetry() is True + assert _observability.has_opentelemetry() is True + # Cached call returns True even if sys.modules changes + monkeypatch.setitem(sys.modules, "opentelemetry.trace", None) + assert _observability.has_opentelemetry() is True + assert _observability.has_opentelemetry.cache_info().hits == 1 + finally: + _observability.has_opentelemetry.cache_clear() - # Reset sentinel and test when ImportError occurs - monkeypatch.setattr(_observability, "_HAS_OPENTELEMETRY", None) - assert _observability.has_opentelemetry() is False - assert _observability._HAS_OPENTELEMETRY is False - # Cached call returns False even if opentelemetry.trace becomes available - monkeypatch.setitem( - sys.modules, - "opentelemetry.trace", - types.ModuleType("opentelemetry.trace"), - ) - assert _observability.has_opentelemetry() is False +def test_has_opentelemetry_missing_and_cached(monkeypatch): + """When missing, returns False and caches the negative result across calls.""" + _observability.has_opentelemetry.cache_clear() + try: + monkeypatch.setitem(sys.modules, "opentelemetry.trace", None) + + assert _observability.has_opentelemetry() is False + # Cached call returns False even if opentelemetry.trace becomes available + monkeypatch.setitem( + sys.modules, + "opentelemetry.trace", + types.ModuleType("opentelemetry.trace"), + ) + assert _observability.has_opentelemetry() is False + assert _observability.has_opentelemetry.cache_info().hits == 1 + finally: + _observability.has_opentelemetry.cache_clear() @pytest.mark.parametrize( @@ -1716,7 +1727,7 @@ def test_extract_trace_context_invalid_or_broken_span(): def test_extract_trace_context_ambient_fallback(monkeypatch, mock_otel): """Proves extract_trace_context falls back to get_current_span().""" - monkeypatch.setattr(_observability, "_HAS_OPENTELEMETRY", True) + monkeypatch.setattr(_observability, "has_opentelemetry", lambda: True) mock_otel.span.get_span_context.return_value = types.SimpleNamespace( is_valid=True, trace_id=0x0123456789ABCDEF0123456789ABCDEF, @@ -1733,7 +1744,7 @@ def test_extract_trace_context_ambient_fallback(monkeypatch, mock_otel): assert _observability.extract_trace_context(None) == {} # OTel not installed returns {} - monkeypatch.setattr(_observability, "_HAS_OPENTELEMETRY", False) + monkeypatch.setattr(_observability, "has_opentelemetry", lambda: False) assert _observability.extract_trace_context(None) == {} From 52748e9ae2f1b1a91df6b5dd0d83fd26c6d8f193 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Thu, 5 Nov 2026 07:13:57 -0500 Subject: [PATCH 5/7] test(api_core): use real OpenTelemetry SpanContext and TraceFlags in extract_trace_context tests --- .../google/api_core/_observability.py | 5 ++- .../tests/unit/test_observability.py | 44 +++++++++---------- 2 files changed, 23 insertions(+), 26 deletions(-) diff --git a/packages/google-api-core/google/api_core/_observability.py b/packages/google-api-core/google/api_core/_observability.py index 7e16b574580f..e487687f5673 100644 --- a/packages/google-api-core/google/api_core/_observability.py +++ b/packages/google-api-core/google/api_core/_observability.py @@ -122,8 +122,9 @@ def extract_trace_context(span: Any = None) -> dict[str, Any]: # that uses bits as configuration flags; the least-significant (last) bit # (0x01) indicates whether the span is sampled (True: recorded/exported, # allowing Cloud Logging to link the log entry to Cloud Trace) or not - # (False: not recorded). - sampled = bool(getattr(ctx, "trace_flags", 0) & 0x01) + # (False: not recorded). Using bitmask (& 0x01) handles both TraceFlags(0x01) + # instances and raw TraceFlags.SAMPLED (int) constants passed to SpanContext. + sampled = bool(ctx.trace_flags & 0x01) return { "trace_id": trace_id_hex, diff --git a/packages/google-api-core/tests/unit/test_observability.py b/packages/google-api-core/tests/unit/test_observability.py index dd831607c272..702b243f0656 100644 --- a/packages/google-api-core/tests/unit/test_observability.py +++ b/packages/google-api-core/tests/unit/test_observability.py @@ -19,6 +19,7 @@ from unittest import mock import pytest +from opentelemetry.trace import INVALID_SPAN_CONTEXT, SpanContext, TraceFlags from google.api_core import _observability, exceptions from google.api_core._feature_gating_helpers import FeatureGatingError @@ -1678,24 +1679,27 @@ def test_has_opentelemetry_missing_and_cached(monkeypatch): @pytest.mark.parametrize( - "trace_flags,expected_sampled", + "flags,expected_sampled", [ - pytest.param(0x01, True, id="int_bitmask_sampled_true"), - pytest.param(0x00, False, id="int_bitmask_sampled_false"), - pytest.param(0x02, False, id="int_bitmask_other_bit_unsampled"), - pytest.param(0x03, True, id="int_bitmask_multiple_bits_sampled"), + pytest.param(TraceFlags(TraceFlags.SAMPLED), True, id="trace_flags_sampled"), + pytest.param(TraceFlags(TraceFlags.DEFAULT), False, id="trace_flags_default"), + pytest.param(TraceFlags.SAMPLED, True, id="int_constant_sampled"), + pytest.param(TraceFlags.DEFAULT, False, id="int_constant_default"), + pytest.param(None, False, id="none_flags"), ], ) -def test_extract_trace_context_explicit_span_flags(trace_flags, expected_sampled): - """Proves extract_trace_context formats hex IDs and resolves trace_flags.""" - span = mock.Mock(spec_set=["get_span_context"]) - span.get_span_context.return_value = types.SimpleNamespace( - is_valid=True, +def test_extract_trace_context_with_real_span_context(flags, expected_sampled): + """Proves extract_trace_context formats real OpenTelemetry SpanContext objects.""" + context = SpanContext( trace_id=0x0123456789ABCDEF0123456789ABCDEF, span_id=0x0123456789ABCDEF, - trace_flags=trace_flags, + is_remote=False, + trace_flags=flags, ) + span = mock.Mock(spec_set=["get_span_context"]) + span.get_span_context.return_value = context + expected = { "trace_id": "0123456789abcdef0123456789abcdef", "span_id": "0123456789abcdef", @@ -1707,32 +1711,24 @@ def test_extract_trace_context_explicit_span_flags(trace_flags, expected_sampled def test_extract_trace_context_invalid_or_broken_span(): - """Proves extract_trace_context returns {} on broken spans or non-int IDs.""" + """Proves extract_trace_context returns {} on broken or invalid spans.""" bad_span = mock.Mock(spec_set=["get_span_context"]) bad_span.get_span_context.side_effect = RuntimeError("broken span") assert _observability.extract_trace_context(bad_span) == {} invalid_ctx_span = mock.Mock(spec_set=["get_span_context"]) - invalid_ctx_span.get_span_context.return_value = types.SimpleNamespace( - is_valid=False - ) + invalid_ctx_span.get_span_context.return_value = INVALID_SPAN_CONTEXT assert _observability.extract_trace_context(invalid_ctx_span) == {} - bad_ctx_span = mock.Mock(spec_set=["get_span_context"]) - bad_ctx_span.get_span_context.return_value = types.SimpleNamespace( - is_valid=True, trace_id="not_an_int", span_id="not_an_int" - ) - assert _observability.extract_trace_context(bad_ctx_span) == {} - def test_extract_trace_context_ambient_fallback(monkeypatch, mock_otel): """Proves extract_trace_context falls back to get_current_span().""" monkeypatch.setattr(_observability, "has_opentelemetry", lambda: True) - mock_otel.span.get_span_context.return_value = types.SimpleNamespace( - is_valid=True, + mock_otel.span.get_span_context.return_value = SpanContext( trace_id=0x0123456789ABCDEF0123456789ABCDEF, span_id=0x0123456789ABCDEF, - trace_flags=0x01, + is_remote=False, + trace_flags=TraceFlags.SAMPLED, ) res = _observability.extract_trace_context(None) From 6e116c82fa48358772fc38512a73d73df705448a Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Thu, 5 Nov 2026 07:33:32 -0500 Subject: [PATCH 6/7] test(api_core): parametrize extract_trace_context broken and invalid span cases --- .../tests/unit/test_observability.py | 41 ++++++++++++++----- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/packages/google-api-core/tests/unit/test_observability.py b/packages/google-api-core/tests/unit/test_observability.py index 702b243f0656..65e284f97744 100644 --- a/packages/google-api-core/tests/unit/test_observability.py +++ b/packages/google-api-core/tests/unit/test_observability.py @@ -19,7 +19,7 @@ from unittest import mock import pytest -from opentelemetry.trace import INVALID_SPAN_CONTEXT, SpanContext, TraceFlags +from opentelemetry.trace import INVALID_SPAN, SpanContext, TraceFlags from google.api_core import _observability, exceptions from google.api_core._feature_gating_helpers import FeatureGatingError @@ -1710,15 +1710,36 @@ def test_extract_trace_context_with_real_span_context(flags, expected_sampled): assert _observability.extract_trace_context(span) == expected -def test_extract_trace_context_invalid_or_broken_span(): - """Proves extract_trace_context returns {} on broken or invalid spans.""" - bad_span = mock.Mock(spec_set=["get_span_context"]) - bad_span.get_span_context.side_effect = RuntimeError("broken span") - assert _observability.extract_trace_context(bad_span) == {} - - invalid_ctx_span = mock.Mock(spec_set=["get_span_context"]) - invalid_ctx_span.get_span_context.return_value = INVALID_SPAN_CONTEXT - assert _observability.extract_trace_context(invalid_ctx_span) == {} +@pytest.mark.parametrize( + "span", + [ + pytest.param( + INVALID_SPAN, + id="invalid_span_context", + ), + pytest.param( + mock.Mock( + get_span_context=mock.Mock(side_effect=RuntimeError("broken span")) + ), + id="get_span_context_raises_runtime_error", + ), + pytest.param( + mock.Mock( + get_span_context=mock.Mock( + return_value=mock.Mock( + is_valid=True, + trace_id="not_an_int", + span_id="not_an_int", + ) + ) + ), + id="corrupt_non_int_ids", + ), + ], +) +def test_extract_trace_context_fail_open_on_broken_spans(span): + """Proves extract_trace_context cleanly returns {} when span or context is invalid.""" + assert _observability.extract_trace_context(span) == {} def test_extract_trace_context_ambient_fallback(monkeypatch, mock_otel): From e6f531c170c5e10c73a2e671aefbea5d628859c6 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Thu, 5 Nov 2026 08:08:37 -0500 Subject: [PATCH 7/7] test(api_core): replace SimpleNamespace with real ErrorInfo protos and exceptions in error attribute tests --- .../tests/unit/test_observability.py | 292 +++++++++++++----- 1 file changed, 221 insertions(+), 71 deletions(-) diff --git a/packages/google-api-core/tests/unit/test_observability.py b/packages/google-api-core/tests/unit/test_observability.py index 65e284f97744..ff0e71627aba 100644 --- a/packages/google-api-core/tests/unit/test_observability.py +++ b/packages/google-api-core/tests/unit/test_observability.py @@ -19,6 +19,7 @@ from unittest import mock import pytest +from google.rpc import error_details_pb2 from opentelemetry.trace import INVALID_SPAN, SpanContext, TraceFlags from google.api_core import _observability, exceptions @@ -1743,7 +1744,7 @@ def test_extract_trace_context_fail_open_on_broken_spans(span): def test_extract_trace_context_ambient_fallback(monkeypatch, mock_otel): - """Proves extract_trace_context falls back to get_current_span().""" + """Proves extract_trace_context falls back to get_current_span() when span is None.""" monkeypatch.setattr(_observability, "has_opentelemetry", lambda: True) mock_otel.span.get_span_context.return_value = SpanContext( trace_id=0x0123456789ABCDEF0123456789ABCDEF, @@ -1752,19 +1753,42 @@ def test_extract_trace_context_ambient_fallback(monkeypatch, mock_otel): trace_flags=TraceFlags.SAMPLED, ) - res = _observability.extract_trace_context(None) - assert res["trace_id"] == "0123456789abcdef0123456789abcdef" - assert res["logging.googleapis.com/trace_sampled"] is True + assert _observability.extract_trace_context(None) == { + "trace_id": "0123456789abcdef0123456789abcdef", + "span_id": "0123456789abcdef", + "logging.googleapis.com/trace": "0123456789abcdef0123456789abcdef", + "logging.googleapis.com/spanId": "0123456789abcdef", + "logging.googleapis.com/trace_sampled": True, + } + - # Ambient get_current_span raising an exception returns {} +def test_extract_trace_context_ambient_get_current_span_error(monkeypatch, mock_otel): + """Proves extract_trace_context fails open to {} if get_current_span() raises.""" + monkeypatch.setattr(_observability, "has_opentelemetry", lambda: True) mock_otel.trace.get_current_span.side_effect = RuntimeError("ambient error") assert _observability.extract_trace_context(None) == {} - # OTel not installed returns {} + +def test_extract_trace_context_ambient_otel_missing(monkeypatch): + """Proves extract_trace_context returns {} when span is None and OTel is not installed.""" monkeypatch.setattr(_observability, "has_opentelemetry", lambda: False) assert _observability.extract_trace_context(None) == {} +def _with_cause(exc: BaseException, cause: BaseException) -> BaseException: + """Attaches __cause__ to an exception instance for parametrized test construction.""" + exc.__cause__ = cause + return exc + + +def _with_error_info_attr(exc: BaseException, error_info: object) -> BaseException: + """Attaches a public .error_info attribute to an exception instance.""" + # Ignored by mypy: BaseException does not declare a public .error_info + # attribute, which we attach dynamically here to test the .error_info fallback. + exc.error_info = error_info # type: ignore[attr-defined] + return exc + + @pytest.mark.parametrize( "target_exc,expected", [ @@ -1785,15 +1809,16 @@ def test_extract_trace_context_ambient_fallback(monkeypatch, mock_otel): id="google_api_call_error_with_dict_error_info", ), pytest.param( - types.SimpleNamespace( - error_info=types.SimpleNamespace( + exceptions.BadRequest( + "Service disabled", + error_info=error_details_pb2.ErrorInfo( domain="googleapis.com", reason="SERVICE_DISABLED", metadata={ "service": "secretmanager.googleapis.com", "consumer": "projects/123", }, - ) + ), ), ( "googleapis.com", @@ -1803,16 +1828,38 @@ def test_extract_trace_context_ambient_fallback(monkeypatch, mock_otel): "consumer": "projects/123", }, ), - id="error_info_namespace", + id="google_api_call_error_with_proto_error_info", ), pytest.param( - types.SimpleNamespace( + _with_error_info_attr( + exceptions.BadRequest("Service disabled"), + error_details_pb2.ErrorInfo( + domain="googleapis.com", + reason="SERVICE_DISABLED", + metadata={ + "service": "secretmanager.googleapis.com", + "consumer": "projects/123", + }, + ), + ), + ( + "googleapis.com", + "SERVICE_DISABLED", + { + "service": "secretmanager.googleapis.com", + "consumer": "projects/123", + }, + ), + id="exception_with_public_error_info_attr", + ), + pytest.param( + error_details_pb2.ErrorInfo( domain="fallback_d", reason="fallback_r", - metadata={"f_key": 42}, + metadata={"f_key": "42"}, ), - ("fallback_d", "fallback_r", {"f_key": 42}), - id="direct_attributes_fallback", + ("fallback_d", "fallback_r", {"f_key": "42"}), + id="direct_error_info_proto_input", ), pytest.param( { @@ -1824,11 +1871,12 @@ def test_extract_trace_context_ambient_fallback(monkeypatch, mock_otel): id="raw_dict_input", ), pytest.param( - types.SimpleNamespace( - error_info=types.SimpleNamespace(domain="", reason="", metadata=None) + exceptions.BadRequest( + "bad request", + error_info=error_details_pb2.ErrorInfo(), ), - ("", "", None), - id="empty_error_info_attributes", + ("", "", {}), + id="empty_error_info_proto_attributes", ), pytest.param( ValueError("invalid argument"), @@ -1838,7 +1886,7 @@ def test_extract_trace_context_ambient_fallback(monkeypatch, mock_otel): ], ) def test_extract_error_info_fields(target_exc, expected): - """Proves _extract_error_info_fields resolves (domain, reason, metadata) across exception and dict shapes.""" + """Proves _extract_error_info_fields resolves (domain, reason, metadata) across exception, proto, and dict shapes.""" assert _observability._extract_error_info_fields(target_exc) == expected @@ -1846,10 +1894,13 @@ def test_extract_error_info_fields(target_exc, expected): "target_exc,return_value,side_effect,expected", [ pytest.param( - types.SimpleNamespace(trailing_metadata=[("meta", "val")]), + mock.Mock( + spec=["trailing_metadata"], + trailing_metadata=[("meta", "val")], + ), ( - None, - types.SimpleNamespace( + [], + error_details_pb2.ErrorInfo( domain="parse_d", reason="parse_r", metadata={"foo": "bar"}, @@ -1860,7 +1911,10 @@ def test_extract_error_info_fields(target_exc, expected): id="direct_trailing_metadata_parsed", ), pytest.param( - types.SimpleNamespace(response=types.SimpleNamespace(trailing_metadata=[])), + exceptions.BadRequest( + "bad proto", + response=mock.Mock(spec=["trailing_metadata"], trailing_metadata=()), + ), None, ValueError("bad proto"), (None, None, None), @@ -1887,41 +1941,46 @@ def test_extract_error_info_fields_grpc_trailing_metadata( exceptions.ServiceUnavailable( "POST http://localhost:7469/v1beta1/sequences/23: Persistent outage" ), - types.SimpleNamespace( + mock.Mock( status_code=503, - json=lambda: {"error": {"message": "Persistent outage"}}, + json=mock.Mock( + return_value={"error": {"message": "Persistent outage"}} + ), ), None, "Persistent outage", id="http_response_json_takes_precedence", ), pytest.param( - types.SimpleNamespace(message="api call failed"), + exceptions.GoogleAPICallError("api call failed"), None, None, "api call failed", - id="message_attribute", + id="google_api_call_error_message_attribute", ), pytest.param( - types.SimpleNamespace(details=lambda: "rpc deadline exceeded"), + mock.Mock( + spec=["details"], + details=mock.Mock(return_value="rpc deadline exceeded"), + ), None, None, "rpc deadline exceeded", - id="details_callable", + id="grpc_details_callable", ), pytest.param( - types.SimpleNamespace(details="static details string"), + mock.Mock(spec=["details"], details="static details string"), None, None, "static details string", id="details_string_attribute", ), pytest.param( - types.SimpleNamespace(), + ValueError(""), None, "explicit fallback", "explicit fallback", - id="fallback_argument", + id="fallback_argument_when_exception_str_is_empty", ), pytest.param( ValueError("invalid argument passed"), @@ -2023,34 +2082,38 @@ def test_extract_error_message(target_exc, http_response, fallback, expected): id="google_api_call_error_without_error_info", ), pytest.param( - types.SimpleNamespace( - error_info=types.SimpleNamespace( + exceptions.BadRequest( + "Service disabled", + error_info=error_details_pb2.ErrorInfo( domain="googleapis.com", reason="SERVICE_DISABLED", metadata={ "service": "secretmanager.googleapis.com", "consumer": "projects/123", - "count": 42, }, - ) + ), ), {}, { - "rpc.response.status_code": "SimpleNamespace", + "rpc.response.status_code": "BAD_REQUEST", "gcp.errors.domain": "googleapis.com", "error.type": "SERVICE_DISABLED", "gcp.errors.metadata.service": "secretmanager.googleapis.com", "gcp.errors.metadata.consumer": "projects/123", - "gcp.errors.metadata.count": "42", + "status.message": "Service disabled", }, - id="error_info_with_domain_reason_and_metadata", + id="error_info_proto_with_domain_reason_and_metadata", ), pytest.param( - types.SimpleNamespace(domain="", reason="", metadata={}), + exceptions.BadRequest( + "bad request", + error_info=error_details_pb2.ErrorInfo(), + ), {}, { - "rpc.response.status_code": "SimpleNamespace", - "error.type": "SimpleNamespace", + "rpc.response.status_code": "BAD_REQUEST", + "error.type": "BAD_REQUEST", + "status.message": "bad request", }, id="empty_error_info_falls_back_to_resolved_code", ), @@ -2058,7 +2121,7 @@ def test_extract_error_message(target_exc, http_response, fallback, expected): { "domain": "googleapis.com", "reason": "RATE_LIMIT_EXCEEDED", - "metadata": {"quota": "100"}, + "metadata": {"quota": "100", "count": 42}, }, {"status_message": "Quota exceeded"}, { @@ -2066,33 +2129,40 @@ def test_extract_error_message(target_exc, http_response, fallback, expected): "gcp.errors.domain": "googleapis.com", "error.type": "RATE_LIMIT_EXCEEDED", "gcp.errors.metadata.quota": "100", + "gcp.errors.metadata.count": "42", "status.message": "Quota exceeded", }, - id="dict_error_info", + id="dict_error_info_with_non_str_metadata_value", ), pytest.param( - types.SimpleNamespace( - __cause__=types.SimpleNamespace( - error_info=types.SimpleNamespace( + _with_cause( + RuntimeError("wrapper error"), + exceptions.BadRequest( + "Service disabled", + error_info=error_details_pb2.ErrorInfo( domain="d", reason="r", metadata={"k": "v"} - ) - ) + ), + ), ), {}, { - "rpc.response.status_code": "SimpleNamespace", + "rpc.response.status_code": "BAD_REQUEST", "gcp.errors.domain": "d", "error.type": "r", "gcp.errors.metadata.k": "v", + "status.message": "Service disabled", }, id="dunder_cause_unwrapping", ), pytest.param( exceptions.ServiceUnavailable( "POST http://localhost:7469/v1beta1/sequences/23: Persistent outage", - response=types.SimpleNamespace( + response=mock.Mock( + spec=["status_code", "json"], status_code=503, - json=lambda: {"error": {"message": "Persistent outage"}}, + json=mock.Mock( + return_value={"error": {"message": "Persistent outage"}} + ), ), ), {}, @@ -2103,14 +2173,55 @@ def test_extract_error_message(target_exc, http_response, fallback, expected): }, id="http_response_error_without_reason", ), + pytest.param( + exceptions.ServiceUnavailable( + "POST http://localhost:7469/v1beta1/sequences/23: Persistent outage", + response=mock.Mock( + spec=["status", "json"], + status=503, + json=mock.Mock( + return_value={"error": {"message": "Persistent outage"}} + ), + ), + ), + {}, + { + "rpc.response.status_code": _UNAVAILABLE_CODE, + "error.type": "503", + "status.message": "Persistent outage", + }, + id="http_response_using_status_attribute", + ), + pytest.param( + exceptions.ServiceUnavailable( + "Service unavailable fallback message", + response=mock.Mock( + spec=["status_code", "json"], + status_code=True, + json=mock.Mock( + return_value={"error": {"message": "Should be ignored"}} + ), + ), + ), + {}, + { + "rpc.response.status_code": _UNAVAILABLE_CODE, + "error.type": _UNAVAILABLE_CODE, + "status.message": "Service unavailable fallback message", + }, + id="http_response_with_boolean_status_code_ignored", + ), pytest.param( exceptions.RetryError( "Deadline of 0.1s exceeded", cause=exceptions.ServiceUnavailable( "POST http://localhost:7469/v1beta1/sequences/23: Persistent outage", - response=types.SimpleNamespace( + response=mock.Mock( + spec=["status_code", "json"], status_code=503, - json=lambda: {"error": {"message": "Persistent outage"}}, + json=mock.Mock( + return_value={"error": {"message": "Persistent outage"}} + ), ), ), ), @@ -2125,9 +2236,12 @@ def test_extract_error_message(target_exc, http_response, fallback, expected): pytest.param( exceptions.BadRequest( "POST http://localhost:7469/v1beta1/echo:echo: Quota exceeded", - response=types.SimpleNamespace( + response=mock.Mock( + spec=["status_code", "json"], status_code=400, - json=lambda: {"error": {"message": "Quota exceeded"}}, + json=mock.Mock( + return_value={"error": {"message": "Quota exceeded"}} + ), ), error_info={ "domain": "googleapis.com", @@ -2148,7 +2262,8 @@ def test_extract_error_message(target_exc, http_response, fallback, expected): pytest.param( exceptions.ServiceUnavailable( "gRPC transport error", - response=types.SimpleNamespace( + response=mock.Mock( + spec=["status_code", "trailing_metadata"], status_code=503, trailing_metadata=(), ), @@ -2168,23 +2283,58 @@ def test_build_telemetry_error_attributes(exc, kwargs, expected): assert _observability.build_telemetry_error_attributes(exc, **kwargs) == expected -def test_build_telemetry_error_attributes_include_exception(): - """Proves include_exception=True formats real exception stacktraces.""" +@pytest.mark.parametrize( + "exc_factory,kwargs,expected_code,expected_error_type,expected_status_msg,expected_exc_type,expected_exc_msg_substr", + [ + pytest.param( + lambda: RuntimeError(""), + {"status_code": "UNAVAILABLE", "status_message": "backend down"}, + "UNAVAILABLE", + "UNAVAILABLE", + "backend down", + "RuntimeError", + "", + id="standalone_runtime_error_with_empty_message", + ), + pytest.param( + lambda: exceptions.RetryError( + "Deadline of 0.1s exceeded", + cause=exceptions.ServiceUnavailable("Persistent outage"), + ), + {}, + _UNAVAILABLE_CODE, + _UNAVAILABLE_CODE, + "Persistent outage", + "RetryError", + "Deadline of 0.1s exceeded", + id="chained_retry_error_records_outer_exception_and_inner_status", + ), + ], +) +def test_build_telemetry_error_attributes_include_exception( + exc_factory, + kwargs, + expected_code, + expected_error_type, + expected_status_msg, + expected_exc_type, + expected_exc_msg_substr, +): + """Proves include_exception=True formats stacktraces and records outer exception.* alongside unwrapped status.""" try: - raise RuntimeError("") - except RuntimeError as caught_exc: - exc_empty = caught_exc + raise exc_factory() + except BaseException as caught_exc: + exc = caught_exc attrs_inc = _observability.build_telemetry_error_attributes( - exc_empty, - status_code="UNAVAILABLE", - status_message="backend down", + exc, include_exception=True, + **kwargs, ) - assert attrs_inc["rpc.response.status_code"] == "UNAVAILABLE" - assert attrs_inc["error.type"] == "UNAVAILABLE" - assert attrs_inc["status.message"] == "backend down" - assert attrs_inc["exception.type"] == "RuntimeError" - assert attrs_inc["exception.message"] == "" + assert attrs_inc["rpc.response.status_code"] == expected_code + assert attrs_inc["error.type"] == expected_error_type + assert attrs_inc["status.message"] == expected_status_msg + assert attrs_inc["exception.type"] == expected_exc_type + assert expected_exc_msg_substr in attrs_inc["exception.message"] assert "Traceback (most recent call last):" in attrs_inc["exception.stacktrace"] - assert "raise RuntimeError" in attrs_inc["exception.stacktrace"] + assert "raise exc_factory()" in attrs_inc["exception.stacktrace"]