Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@
from typing import Any, Dict, Iterator, List, Optional, Tuple

import grpc
from fi_instrumentation import REDACTED_VALUE, TraceConfig
from google.cloud.discoveryengine_v1.types import (
Answer,
AnswerQueryRequest,
Expand All @@ -67,6 +68,13 @@
from opentelemetry.sdk.trace import ReadableSpan, TracerProvider
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
from opentelemetry.trace import StatusCode

# Published core 1.1.0 retains the instrumentor's redacted error text. Newer
# core derives error-detail hiding from content-hide flags and normalizes
# redundant exception-type prefixes. Detect the API, not a version string:
# source installs may still carry the published package's version number.
NATIVE_ERROR_POLICY = hasattr(TraceConfig(), "hide_error_details")

SEARCH_SERVICE = "google.cloud.discoveryengine.v1.SearchService"
CONVERSATION_SERVICE = "google.cloud.discoveryengine.v1.ConversationalSearchService"
Expand Down Expand Up @@ -599,3 +607,24 @@ def event(span: ReadableSpan, name: str) -> Optional[Dict[str, Any]]:
if item.name == name:
return dict(item.attributes or {})
return None


def assert_hidden_query_error(span: ReadableSpan, exception_type: str) -> None:
"""Pin both core contracts: absent details, or nonempty redacted text.

Never accept an arbitrary empty message as evidence of masking on legacy
core, or a placeholder where native core must remove the field entirely.
"""
assert span.status.status_code is StatusCode.ERROR
assert [item.name for item in span.events] == ["exception"]
exception = event(span, "exception")
assert exception is not None
assert exception["exception.type"] == exception_type
if NATIVE_ERROR_POLICY:
assert span.status.description is None
assert exception == {"exception.type": exception_type}
else:
assert sorted(exception) == ["exception.message", "exception.type"]
message = exception["exception.message"]
assert REDACTED_VALUE in message
assert span.status.description == exception_type.rsplit(".", 1)[-1] + ": " + message
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,15 @@

import pytest
from google.api_core import exceptions as core_exceptions
from opentelemetry.trace import StatusCode

from _discoveryengine_support import (
FAIL_DENIED,
FAIL_DETAILS,
NATIVE_ERROR_POLICY,
SERVING_CONFIG,
FakeDiscoveryEngine,
assert_hidden_query_error,
answer_client,
answer_request,
async_search_client,
Expand All @@ -31,6 +34,13 @@
# status detail the fake adds quotes it escaped (protobuf text format).
QUOTED_QUERY = FAIL_DETAILS + " Zelda's \"quoted\" a\\b caf\xe9"
QUOTED_DETAIL = r'Zelda\'s \"quoted\" a\\b caf' + "\xe9"
OTHER_CONTENT_HIDE_FLAGS = (
"hide_outputs",
"hide_input_messages",
"hide_output_messages",
"hide_input_text",
"hide_output_text",
)


@pytest.fixture()
Expand All @@ -41,8 +51,9 @@ def fake():

@pytest.fixture(autouse=True)
def clean_env(monkeypatch):
for name in ("FI_HIDE_INPUTS", "FI_HIDE_OUTPUTS", "FI_PII_REDACTION"):
monkeypatch.delenv(name, raising=False)
flags = ("hide_inputs", "hide_error_details", "pii_redaction", *OTHER_CONTENT_HIDE_FLAGS)
for flag in flags:
monkeypatch.delenv("FI_" + flag.upper(), raising=False)


def _error_texts(span):
Expand Down Expand Up @@ -88,11 +99,10 @@ def test_hide_inputs_removes_the_query_everywhere_including_error_text(fake, mon
values = attrs(span)
assert values["input.value"] == REDACTED_VALUE
assert "gen_ai.retrieval.query" not in values
assert_hidden_query_error(span, "google.api_core.exceptions.PermissionDenied")
for text in _error_texts(span):
assert text
assert DENIED_QUERY not in text
assert EMAIL not in text
assert REDACTED_VALUE in span.status.description
assert DENIED_QUERY not in span.to_json()


Expand All @@ -118,11 +128,10 @@ def test_the_server_echo_of_the_query_includes_an_escaped_copy(fake):
def test_escaped_copies_of_the_query_are_removed_from_error_text(fake, monkeypatch, source):
options = _hidden_options(monkeypatch, source)
span = _quoted(fake, options)
assert_hidden_query_error(span, "google.api_core.exceptions.InvalidArgument")
for text in _error_texts(span):
assert text
assert "Zelda" not in text
assert "quoted" not in text
assert REDACTED_VALUE in text
assert "Zelda" not in span.to_json()


Expand All @@ -149,10 +158,9 @@ def test_octal_escaped_copies_of_the_query_are_removed_from_error_text(fake, mon
search_client(fake).search(request=search_request(CONTROL_QUERY))
assert CONTROL_DETAIL in str(raised.value)
span = traced.one()
assert_hidden_query_error(span, "google.api_core.exceptions.InvalidArgument")
for text in _error_texts(span):
assert text
assert "Zelda" not in text and "control" not in text, text
assert REDACTED_VALUE in text
assert "Zelda" not in span.to_json()


Expand Down Expand Up @@ -199,6 +207,35 @@ def test_hide_outputs_changes_nothing_because_no_output_text_is_recorded(fake):
assert attrs(hidden.one()) == attrs(plain.one())


@pytest.mark.parametrize("source", ["config", "env"])
@pytest.mark.parametrize("flag", OTHER_CONTENT_HIDE_FLAGS)
def test_other_content_hide_flags_follow_core_error_policy(fake, monkeypatch, flag, source):
# These flags do not hide a retrieval query, but native core derives error
# privacy from all content-hide flags, not only hide_inputs. No opt-out.
plain = _denied(fake, {})
options = {}
if source == "config":
options["config"] = _config(**{flag: True})
else:
monkeypatch.setenv("FI_" + flag.upper(), "true")
span = _denied(fake, options)
assert attrs(span)["input.value"] == DENIED_QUERY
assert attrs(span)["discoveryengine.error.code"] == 403
assert attrs(span)["discoveryengine.error.status"] == "PERMISSION_DENIED"
assert span.status.status_code is StatusCode.ERROR
assert [item.name for item in span.events] == ["exception"]
if NATIVE_ERROR_POLICY:
assert span.status.description is None
assert event(span, "exception") == {
"exception.type": "google.api_core.exceptions.PermissionDenied"
}
else:
assert span.status.description == plain.status.description
assert event(span, "exception") == event(plain, "exception")
for text in _error_texts(span):
assert DENIED_QUERY in text


@pytest.mark.parametrize("source", ["config", "env"])
def test_pii_redaction_covers_the_query_and_the_error_text(fake, monkeypatch, source):
options = {}
Expand Down Expand Up @@ -269,6 +306,7 @@ async def call():

span = traced.one()
assert attrs(span)["input.value"] == REDACTED_VALUE
assert_hidden_query_error(span, "google.api_core.exceptions.PermissionDenied")
assert DENIED_QUERY not in span.to_json()


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
FAIL_UNSEEN,
METADATA_TOKEN,
MINTED_TOKEN,
NATIVE_ERROR_POLICY,
UNSEEN_ACCESS_TOKEN,
UNSEEN_API_KEY,
UNSEEN_REFRESH_TOKEN,
Expand Down Expand Up @@ -273,7 +274,11 @@ def test_retry_exhaustion_is_one_error_span_with_the_last_status(fake):
# RetryError has no status of its own; the last attempt's is recorded.
assert values["discoveryengine.error.status"] == "UNAVAILABLE"
assert values["discoveryengine.error.code"] == 503
assert span.status.description.startswith("RetryError: Timeout of 0.3s exceeded")
exception = event(span, "exception")
assert exception["exception.type"] == "google.api_core.exceptions.RetryError"
assert exception["exception.message"].startswith("Timeout of 0.3s exceeded")
prefix = "" if NATIVE_ERROR_POLICY else "RetryError: "
assert span.status.description == prefix + exception["exception.message"]
assert [item.name for item in span.events] == ["exception"]


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,10 @@
ANSWER_REFERENCES,
FAIL_DENIED,
METADATA_TOKEN,
NATIVE_ERROR_POLICY,
SEARCH_RESULTS,
FakeDiscoveryEngine,
assert_hidden_query_error,
answer_client,
answer_request,
async_answer_client,
Expand Down Expand Up @@ -95,10 +97,10 @@ def test_a_query_that_cannot_be_read_keeps_no_server_text(fake, monkeypatch):
search_client(fake).search(request=search_request(FAIL_DENIED))

span = traced.one()
assert span.status.description == "PermissionDenied: " + REDACTED_VALUE
exception = [item for item in span.events if item.name == "exception"][0].attributes
assert exception["exception.message"] == REDACTED_VALUE
assert "exception.stacktrace" not in exception
assert_hidden_query_error(span, "google.api_core.exceptions.PermissionDenied")
if not NATIVE_ERROR_POLICY:
assert event(span, "exception")["exception.message"] == REDACTED_VALUE
assert FAIL_DENIED not in span.to_json()
assert attrs(span)["discoveryengine.error.status"] == "PERMISSION_DENIED"


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,11 @@
ANSWER_TEXT,
CONTENT_MARKERS,
DOCUMENT_TITLE,
NATIVE_ERROR_POLICY,
SERVING_CONFIG,
SESSION_NAME,
SNIPPET,
assert_hidden_query_error,
attrs,
event,
instrumented,
Expand Down Expand Up @@ -240,8 +242,12 @@ def test_a_rest_error_records_the_status_from_the_json_body(endpoint):
assert values["discoveryengine.error.status"] == "PERMISSION_DENIED"
assert values["discoveryengine.error.code"] == 403
assert span.status.status_code is StatusCode.ERROR
assert span.status.description.startswith("Forbidden: 403 POST ")
assert sorted(event(span, "exception")) == ["exception.message", "exception.type"]
exception = event(span, "exception")
assert exception["exception.type"] == "google.api_core.exceptions.Forbidden"
assert exception["exception.message"].startswith("403 POST ")
prefix = "" if NATIVE_ERROR_POLICY else "Forbidden: "
assert span.status.description == prefix + exception["exception.message"]
assert sorted(exception) == ["exception.message", "exception.type"]
# Default: the query is recorded, so the server's echo of it stays.
assert QUERY in span.status.description

Expand All @@ -256,7 +262,7 @@ def test_with_hide_inputs_the_query_is_removed_from_rest_error_text(endpoint, mo
span = traced.one()
assert attrs(span)["input.value"] == REDACTED_VALUE
assert "gen_ai.retrieval.query" not in attrs(span)
assert REDACTED_VALUE in span.status.description
assert_hidden_query_error(span, "google.api_core.exceptions.Forbidden")
assert QUERY not in traced.wire()


Expand Down Expand Up @@ -317,6 +323,10 @@ def test_live_rest_error_bodies_record_status_and_code(endpoint, key, operation,
assert values["gen_ai.retrieval.query"] == query
assert "discoveryengine.result_count" not in values
assert span.status.status_code is StatusCode.ERROR
assert span.status.description.startswith("{0}: {1} POST ".format(error_class.__name__, code))
exception = event(span, "exception")
assert exception["exception.type"] == "google.api_core.exceptions." + error_class.__name__
assert exception["exception.message"].startswith("{0} POST ".format(code))
prefix = "" if NATIVE_ERROR_POLICY else error_class.__name__ + ": "
assert span.status.description == prefix + exception["exception.message"]
assert LIVE_ERRORS[key][1]["error"]["message"] in span.status.description
assert sorted(event(span, "exception")) == ["exception.message", "exception.type"]
assert sorted(exception) == ["exception.message", "exception.type"]
Loading