diff --git a/python/frameworks/discoveryengine/CHANGELOG.md b/python/frameworks/discoveryengine/CHANGELOG.md new file mode 100644 index 000000000..1eebfb1ce --- /dev/null +++ b/python/frameworks/discoveryengine/CHANGELOG.md @@ -0,0 +1,36 @@ +# Changelog + +## 0.1.0 + +First release. + +- Trace `google-cloud-discoveryengine` v1 `SearchServiceClient.search`, + `SearchServiceClient.search_lite` and + `ConversationalSearchServiceClient.answer_query`, and their async twins: + one span per call with `gen_ai.span.kind` = `RETRIEVER`, `discoveryengine.serving_config` and + `discoveryengine.result_count`; for answers also + `discoveryengine.answer.length`, `discoveryengine.answer.state` and + `discoveryengine.session`. No model name, tokens or cost. +- The query follows `TraceConfig`: recorded by default + (`gen_ai.retrieval.query`, `input.value`), hidden with `FI_HIDE_INPUTS` / + `hide_inputs`, and then also removed from error text, verbatim and in the + escaped copies the client libraries write. No package-specific switch (a + pre-release `capture_query` option is ignored with a warning). Results and + answer text are never recorded. +- The `exception` event carries the type and the cleaned message only; no + `exception.stacktrace` (traceback frames hold local file paths). +- REST transport: `discoveryengine.error.status` comes from the canonical + `status` of the JSON error body (`google-api-core` raises REST errors by + HTTP code, for example `Forbidden`, which has no gRPC status). +- Google credentials (transport credentials, read again when an error is + recorded; `client_options.api_key`; per-call auth metadata; and Google + token shapes, including JWTs) are removed from every recorded value. + Errors record the gRPC status and code, then re-raise. +- `TraceConfig` `hide_inputs` and `pii_redaction` apply, including to error + text. Spans come from `FITracer`, so `using_*` context attributes apply. +- Accepts `google-cloud-discoveryengine>=0.20.5,<1`; tested with 0.20.5 on + Python 3.10 to 3.13. +- Checked against the live v1 API (2026-10-18): `search` on the gRPC, REST + and gRPC asyncio transports, and real `NOT_FOUND` and + `FAILED_PRECONDITION` errors from `search`, `search_lite` and + `answer_query`. The REST tests replay those error bodies. diff --git a/python/frameworks/discoveryengine/README.md b/python/frameworks/discoveryengine/README.md new file mode 100644 index 000000000..653ff7779 --- /dev/null +++ b/python/frameworks/discoveryengine/README.md @@ -0,0 +1,387 @@ +# traceAI-discoveryengine + +OpenTelemetry instrumentation for the search and answer methods of the +[`google-cloud-discoveryengine`](https://pypi.org/project/google-cloud-discoveryengine/) +client (Google Discovery Engine, also sold as Vertex AI Search). + +It traces the v1 `search`, `search_lite` and `answer_query` calls, sync and +async. Conversation and session management (`create_conversation`, +`converse_conversation` and the rest), `stream_answer_query`, document +ingestion, data store and engine administration produce no spans from this +package. It does not instrument `google.genai`; for the Gemini client use +`traceAI-google-genai`. + +Status: alpha. Verified against the live `discoveryengine.googleapis.com` +on 2026-10-18 (see [Live vendor verification](#live-vendor-verification-2026-10-18)): +`search` succeeded on the gRPC, REST and gRPC asyncio transports, and the +error paths of `search`, `search_lite` and `answer_query` were recorded from +real server errors. The success paths of `search_lite` and `answer_query` +were not available on that setup and are covered by the loopback tests only. + +## Why a wrapper + +The client does not trace itself. With `google-cloud-discoveryengine` 0.20.5 +an uninstrumented `search`, `search_lite` or `answer_query` emits no span, +including with `GOOGLE_SDK_EXPERIMENTAL_PYTHON_TRACING_ENABLED=true` +(`google-api-core` traces only methods whose transport passes `method_name` +to `wrap_method`, and this client does not). +`tests/test_discoveryengine_stock_spans.py` measures this before any wrapper +is installed. A gRPC client instrumentation you add yourself creates +transport spans, which carry no query, result count or retriever kind. + +## Installation + +```bash +pip install traceAI-discoveryengine +``` + +To install from a checkout of this repository instead: + +```bash +pip install "fi-instrumentation-otel>=1.1.0" "google-cloud-discoveryengine>=0.20.5,<1" \ + ./python/frameworks/discoveryengine +``` + +It accepts `google-cloud-discoveryengine>=0.20.5,<1` and +`fi-instrumentation-otel>=1.1.0`. The suite runs with +`google-cloud-discoveryengine` 0.20.5 on Python 3.10, 3.11, 3.12 and 3.13. + +## Usage + +```python +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType +from google.cloud import discoveryengine_v1 as discoveryengine +from traceai_discoveryengine import DiscoveryEngineInstrumentor + +tracer_provider = register(project_type=ProjectType.OBSERVE, project_name="my-search-app") +DiscoveryEngineInstrumentor().instrument(tracer_provider=tracer_provider) + +serving_config = ( + "projects/my-gcp-project/locations/global/collections/default_collection/" + "engines/my-engine/servingConfigs/default_search" +) +client = discoveryengine.SearchServiceClient() # Application Default Credentials +pager = client.search(request={"serving_config": serving_config, "query": "open telemetry"}) + +answers = discoveryengine.ConversationalSearchServiceClient() +answers.answer_query(request={"serving_config": serving_config, "query": {"text": "What is OTLP?"}}) +``` + +Import the clients from `google.cloud.discoveryengine_v1`, as above. +**`from google.cloud import discoveryengine` produces no spans**: at 0.20.5 +that unversioned module re-exports the v1beta clients, which are not traced +(nor are v1alpha clients). + +The methods are wrapped on the client classes, so clients built before or +after `instrument()` are both traced. See +[`examples/search_and_answer.py`](examples/search_and_answer.py). + +### Google credentials and projects + +The Google clients keep reading Application Default Credentials (ADC) or the +credentials you pass them, exactly as without this package; it never reads +ADC or adds a credential of its own. `register()` reads `FI_API_KEY` and +`FI_SECRET_KEY` for Future AGI. + +The Future AGI project (`project_name` above) is not your GCP project. The +GCP project appears only inside resource names such as the serving config. + +An engine in a multi-region location needs its regional endpoint: + +```python +client = discoveryengine.SearchServiceClient( + client_options={"api_endpoint": "eu-discoveryengine.googleapis.com"} +) +``` + +## Instrumented calls + +| Call | Span name | +|---|---| +| `SearchServiceClient.search`, `SearchServiceAsyncClient.search` | `discoveryengine.search` | +| `SearchServiceClient.search_lite`, `SearchServiceAsyncClient.search_lite` | `discoveryengine.search_lite` | +| `ConversationalSearchServiceClient.answer_query`, `ConversationalSearchServiceAsyncClient.answer_query` | `discoveryengine.answer_query` | + +What the server allows depends on your app, not on this package: + +- `search_lite` is meant for public website search with an API key + (`client_options={"api_key": ...}`). Against any other data store the + server answers `FAILED_PRECONDITION` "API key is only supported for public + website search." (seen live with OAuth credentials and an unstructured + data store). +- `answer_query` needs an app with the Large Language Model add-on + (Enterprise edition). On a Standard edition app the server answers + `FAILED_PRECONDITION` "This feature is only available when Large Language + Model add-on is enabled." (seen live). + +Both errors are traced like any other error (status ERROR, +`discoveryengine.error.status` = `FAILED_PRECONDITION`, code `400`). + +Each call produces one span with `gen_ai.span.kind` = `RETRIEVER`. Retries made +by `google-api-core` inside one call (when you pass `retry=`; at 0.20.5 these +methods have no default retry) stay in that one span. The span is a child of the active span +when there is one, and a root span otherwise. It is current while the +transport sends the request, so a span from a gRPC client instrumentation +nests under it. + +`search` and `search_lite` return a pager. Iterating past the first page +makes one more RPC of the same method per page, directly through the +transport and not through `search()` or `search_lite()`, so later pages add +no span: the span covers the call that created the pager, and +`discoveryengine.result_count` counts the first page only. + +## Span attributes + +| Attribute | Calls | Value | +|---|---|---| +| `gen_ai.span.kind` | all | `RETRIEVER` | +| `discoveryengine.serving_config` | requests with `serving_config` | The serving config resource name from the request, cut to 1 KB. | +| `discoveryengine.result_count` | successful calls | `search`, `search_lite`: number of results in the returned (first) page. `answer_query`: number of `references` on the answer (the grounding sources). Omitted when the response has no answer. | +| `discoveryengine.answer.length` | successful `answer_query` with an answer | Length of `answer_text` in characters. The text itself is never recorded. | +| `discoveryengine.answer.state` | successful `answer_query` with an answer | The answer state name, for example `SUCCEEDED` or `FAILED`. | +| `discoveryengine.session` | `answer_query` with a session | The session resource name from the request and then from the response. A request for a new session (`.../sessions/-`) records the name the response returns. | +| `gen_ai.retrieval.query`, `input.value` | requests with a query | `SearchRequest.query` or `AnswerQueryRequest.query.text`, recorded by default. See [Privacy](#privacy). | +| `discoveryengine.error.status`, `discoveryengine.error.code` | failed calls | The gRPC status name (for example `PERMISSION_DENIED`) and the HTTP-style code `google-api-core` maps it to (for example `403`). On the REST transport, where `google-api-core` raises by HTTP code (403 is `Forbidden`, without a gRPC status), the status name is the canonical `status` of the JSON error body. For a `RetryError`, the last attempt's. | +| `discoveryengine.cancelled` | cancelled async calls | `true`; see [Errors and cancellation](#errors-and-cancellation). | + +No model name, token count or cost is recorded. At 0.20.5 +`AnswerQueryResponse` and `Answer` carry no model field, and the model +version a request asks for (`answer_generation_spec.model_spec`) is not the +model that answered, so none is reported. Discovery Engine is billed in GCP, +not per span. + +Spans come from an `fi_instrumentation.FITracer`, so the context attributes +of `using_session`, `using_user`, `using_metadata`, `using_tags` and +`using_attributes` are stamped on every Discovery Engine span started inside +them (`session.id`, `user.id`, `metadata`, `tag.tags`). +`discoveryengine.session` is Discovery Engine's own session and is a +separate attribute. + +## Privacy + +The query follows `TraceConfig`, like the input of every traceAI +instrumentor: it is recorded by default (`gen_ai.retrieval.query` and +`input.value`, with credentials removed and cut to 1 KB of UTF-8 on a +character boundary), and `FI_HIDE_INPUTS=true` or +`TraceConfig(hide_inputs=True)` hides it everywhere, including server error +text. There is no package-specific switch. + +Also recorded: counts, the answer state, the serving config and session +resource names, and error text (status name, code and message). + +Never recorded: Google credentials, search results (titles, snippets, +documents, summaries), answer text, citations and reference content, +request metadata, stack traces and local file paths. There is no setting +that adds them. + +To keep the query off the trace: + +```python +from fi_instrumentation import TraceConfig + +DiscoveryEngineInstrumentor().instrument( + tracer_provider=tracer_provider, + config=TraceConfig(hide_inputs=True), # or FI_HIDE_INPUTS=true +) +``` + +While the query is hidden, each occurrence of it in server-written text +(the error status description and the `exception` event's message) becomes +`__REDACTED__`. That +covers the verbatim text and the escaped copies the client libraries write: +protobuf text format (a status detail that quotes the query, which +`google-api-core` appends to the error message with `'`, `"` and `\` +backslash-escaped, `\n`, `\r` and `\t` as short escapes, and other ASCII +control characters and DEL as 3-digit octal such as `\013`), the hex +escapes of older gRPC cores (the chained gRPC +error's `debug_error_string`, with non-ASCII bytes as `\xHH`; seen with +grpcio 1.62, not with 1.82 or 1.84), and the Python repr and JSON string +forms. A short query is removed wherever it occurs, including inside other +words. A copy escaped or reformatted any other way is not matched. +If the query cannot be read, the error message is recorded as +`__REDACTED__`. Other request fields that a server error may quote (a +`filter`, `user_pseudo_id`) are not removed. + +Credentials are removed from every recorded value, including error +messages, and replaced with `[redacted]`. The package looks for them +where the client keeps them: the transport's google.auth credentials +(`token`, `refresh_token`, `client_secret`; an API key credential's +`token`), `client_options.api_key`, and per-call `metadata` entries named +`authorization`, `proxy-authorization`, `x-goog-api-key` or +`x-goog-iam-authorization-token` (with and without the `Bearer` prefix). +It reads them before the call and again when it records an error, so a +token set on those credentials during the call (on first use or by a +refresh) is removed too. +The package reads `metadata` once and passes the client a tuple of the same +pairs, so metadata given as a generator still reaches the server; if +reading it raises, the client raises the same error, as without the package. +Text shaped like a Google credential is replaced too, even if the client +never held it: `ya29.` access tokens, `AIza` API keys, `1//` refresh tokens, +JWTs (`eyJ...`: three base64url parts, such as a service account's +self-signed token), and, in server-written text only, the value after +`Bearer`. If the place the client keeps credentials (or the call's +`metadata`) cannot be read, before or after the call, the call is traced +without the query and with +`[not recorded: the client credentials could not be read]` in place of the +error message. + +`TraceConfig` settings apply. Pass `config=TraceConfig(...)` or set the +environment variables before calling `instrument()`: + +| Setting | Effect | +|---|---| +| `hide_inputs` / `FI_HIDE_INPUTS=true` | Records `input.value` as `__REDACTED__` and drops `gen_ai.retrieval.query`. The query is also removed from error text as described above. Counts and resource names stay. | +| `hide_outputs` / `FI_HIDE_OUTPUTS=true` | No effect: no output text is recorded. | +| `pii_redaction` / `FI_PII_REDACTION=true` | Replaces emails, phone numbers, SSNs, card numbers, IPv4 addresses and `sk-`/`pk-` style keys with tokens such as `` in every recorded text: attributes, the error status and the `exception` event. It runs after credentials (and the hidden query) are removed and before the size caps, so a cut cannot leave part of an email. The patterns also match ids: in a resource name with a GCP project number, the last ten digits are replaced (`projects/123456789012/...` is recorded as `projects/12/...`). | + +`config` must be a `fi_instrumentation.TraceConfig`; anything else raises +`TypeError` and nothing is wrapped. + +## Errors and cancellation + +An error raised by the client (a `google.api_core.exceptions` subclass such +as `PermissionDenied`, `InvalidArgument` or `ServiceUnavailable`, or a +`RetryError` when retries run out) sets the span status to ERROR, records +`discoveryengine.error.status` and `discoveryengine.error.code`, and records +one `exception` event. Credentials are removed from both, and the query when +it is not recorded (see [Privacy](#privacy)). The exception is re-raised +unchanged, so your code still sees the server's original message. + +The error text is cleaned in this order: credentials, then the hidden query, +then PII with `pii_redaction`. It is then cut on a UTF-8 character boundary: +the status description's message and `exception.message` to 1 KB. The +`exception` event carries `exception.type` and `exception.message` only: +`exception.stacktrace` is never written, because traceback frames contain +local file paths (home directory, user name, virtualenv). + +An `answer_query` that returns an answer with state `FAILED` is not an +exception: the span gets status ERROR with description `answer state FAILED` +and no `exception` event. + +Cancelling an async call (`asyncio.CancelledError`) ends the span with +status ERROR, description `cancelled`, and `discoveryengine.cancelled` = +`true`, without an exception event. + +If the instrumentation itself fails (reading a request, a response or the +credentials, starting a span, recording an error), it logs at debug level and +your call proceeds with its own result or exception. +`fi_instrumentation.suppress_tracing()` skips the span. + +## Limits + +- Recorded text is cut on a UTF-8 character boundary after it is cleaned: + 1 KB for the query and each resource name, 1 KB for `exception.message` + and the message in the error status. No stack trace is recorded. + With `pii_redaction`, `FITracer` runs its own PII pass on attributes after + the cut, and a new match can make a value a few bytes longer; that pass + only redacts more. +- A token carried by a gRPC channel you build yourself (call credentials on + a `channel=` you pass to the transport) is not visible to the package; + only the credential shapes above are removed from text. +- When it builds the gRPC channel, `google-api-core` gives it a scoped copy + of credentials that require scopes, such as service-account credentials + without scopes. A token minted on that copy during the call (a self-signed + JWT or an access token) is never on the credentials the client holds, so + only its shape (`eyJ...` or `ya29.`) removes it. +- Only v1 is traced. `google.cloud.discoveryengine` (v1beta at 0.20.5), + `discoveryengine_v1beta` and `discoveryengine_v1alpha` clients produce no + spans. +- `stream_answer_query` and `converse_conversation` are not traced. +- The tests cover the gRPC, gRPC asyncio and REST (`transport="rest"`) + transports. The async REST transport is not exercised. +- `instrument()` checks the installed `google-cloud-discoveryengine` + version. Outside `>=0.20.5,<1` it logs an error and wraps nothing. +- Do not combine this package with another instrumentor that wraps the same + client methods, or calls will produce duplicate spans. A gRPC + instrumentation adds transport spans under these spans; it does not + duplicate them. + +## Live vendor verification (2026-10-18) + +Run against the real `discoveryengine.googleapis.com` (v1, `global`) in a +staging GCP project, with the smallest app: one unstructured data store with +three inline `text/plain` documents and one Standard edition generic search +engine (no LLM add-on). The engine and data store were deleted after the run. +The code followed [Usage](#usage); the only change was +`credentials=google.oauth2.credentials.Credentials(token=..., quota_project_id=...)` +on each client, because the machine had no ADC. + +Versions: `google-cloud-discoveryengine` 0.20.5 (latest on PyPI), +`fi-instrumentation-otel` 1.1.0, google-api-core 2.42.0, google-auth +2.62.0, grpcio 1.84.0, protobuf 7.36.2, Python 3.12. Spans were exported +over OTLP/HTTP to a local Future AGI and checked in its store. + +| Call | Transport | Live result | Span | +|---|---|---|---| +| `search` | gRPC, REST, gRPC asyncio | 2 results | OK, `discoveryengine.result_count` = 2, query, serving config | +| `search` on a serving config that does not exist | gRPC, REST | `NotFound` 404 | ERROR, `NOT_FOUND` / 404 | +| `search_lite` (OAuth, not a website data store) | gRPC | `FailedPrecondition` 400 | ERROR, `FAILED_PRECONDITION` / 400 | +| `answer_query` (Standard edition app) | gRPC | `FailedPrecondition` 400 | ERROR, `FAILED_PRECONDITION` / 400 | +| `answer_query` (Standard edition app) | REST | `BadRequest` 400 | ERROR, `FAILED_PRECONDITION` (from the JSON error body) / 400 | + +With the defaults the query was stored on every Discovery Engine span; with +`FI_HIDE_INPUTS=true` it was in none of the stored columns (`input.value` = +`__REDACTED__`, no `gen_ai.retrieval.query`). In both modes the Discovery +Engine spans held no result content (titles, snippets), and no stored column +held a credential, the access token, a local path, the user name, the host +name, the account's e-mail address or a stack trace. The real error messages +seen did not quote the query. + +Not verified live: a successful `search_lite` (needs a public website data +store and an API key), a successful `answer_query` and its answer +attributes (`discoveryengine.answer.*`, `discoveryengine.session`; needs the +LLM add-on), pagination, a regional endpoint, the async REST transport, and +removal of a hidden query from a server error that quotes it. The loopback +tests cover these. + +## Tests + +The three error bodies the live API returned on 2026-10-18 are replayed in +`tests/test_discoveryengine_rest.py` (`test_live_rest_error_bodies_record_status_and_code`). + +The tests drive the real `google-cloud-discoveryengine` clients over their +own gRPC transports against a gRPC fake on 127.0.0.1 +(`tests/_discoveryengine_support.py`), and over the REST transport against a +loopback HTTP server that answers the documented v1 REST routes +(`tests/test_discoveryengine_rest.py`). They use no Google credentials, no +ADC and no GCP project: credentials are placeholders, and the serving config +is a placeholder resource name. The contract tests export through the real +`register()` OTLP exporter into the shared `harness.Receiver`, and post +captured spans with `harness.post_otlp()`. + +From the repository root: + +```bash +env -u PYTHONPATH PYTHONPATH="python/frameworks/discoveryengine:python:python/tests" \ + uv run --no-project --python 3.11 \ + --with pytest --with pytest-asyncio --with opentelemetry-api --with opentelemetry-sdk \ + --with opentelemetry-instrumentation --with opentelemetry-exporter-otlp-proto-http \ + --with wrapt --with requests --with jsonschema --with protobuf --with opentelemetry-proto \ + --with 'google-cloud-discoveryengine==0.20.5' \ + pytest python/frameworks/discoveryengine/tests -q -p no:cacheprovider --noconftest -o addopts= -rfEs +``` + +To run against the published `fi-instrumentation-otel` wheel instead of the +in-repo source, drop `python:` from `PYTHONPATH` and add +`--with 'fi-instrumentation-otel==1.1.0'`. + +Replace `3.11` with `3.10`, `3.12` or `3.13` for the other supported +Pythons. + +The stock-span measurement (`tests/test_discoveryengine_stock_spans.py`) +also runs with `opentelemetry-instrumentation-grpc` installed. Then +`GOOGLE_SDK_EXPERIMENTAL_PYTHON_TRACING_ENABLED=true` turns +`google-api-core`'s experimental tracing on, and the file checks that the +unwrapped client still emits no span: + +```bash +env -u PYTHONPATH PYTHONPATH="python/frameworks/discoveryengine:python:python/tests" \ + uv run --no-project --python 3.11 \ + --with pytest --with pytest-asyncio --with opentelemetry-api --with opentelemetry-sdk \ + --with opentelemetry-instrumentation --with opentelemetry-exporter-otlp-proto-http \ + --with wrapt --with requests --with jsonschema --with protobuf --with opentelemetry-proto \ + --with 'google-cloud-discoveryengine==0.20.5' --with opentelemetry-instrumentation-grpc \ + pytest python/frameworks/discoveryengine/tests/test_discoveryengine_stock_spans.py \ + -q -p no:cacheprovider --noconftest -o addopts= -rfEs +``` diff --git a/python/frameworks/discoveryengine/examples/search_and_answer.py b/python/frameworks/discoveryengine/examples/search_and_answer.py new file mode 100644 index 000000000..3941e5d11 --- /dev/null +++ b/python/frameworks/discoveryengine/examples/search_and_answer.py @@ -0,0 +1,63 @@ +"""Trace one Discovery Engine search and one grounded answer. + +Set DISCOVERY_ENGINE_SERVING_CONFIG to a serving config resource name: + + projects/PROJECT/locations/LOCATION/collections/default_collection/engines/ENGINE/servingConfigs/default_search + +The Google clients read Application Default Credentials (ADC) themselves: +run ``gcloud auth application-default login`` or set +GOOGLE_APPLICATION_CREDENTIALS. register() reads FI_API_KEY, FI_SECRET_KEY +and FI_BASE_URL. The Future AGI project is not your GCP project. +""" + +import os +from typing import Any, Dict, Optional + +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType +from google.cloud import discoveryengine_v1 as discoveryengine + +from traceai_discoveryengine import DiscoveryEngineInstrumentor + +QUERY = "open telemetry retrieval" +QUESTION = "What does OpenTelemetry trace?" + + +def client_options(serving_config: str) -> Dict[str, Any]: + """A regional engine (``locations/eu`` or ``locations/us``) needs its regional endpoint.""" + location = serving_config.split("/locations/", 1)[-1].split("/", 1)[0] + if location and location != "global" and "/locations/" in serving_config: + return {"api_endpoint": "{0}-discoveryengine.googleapis.com".format(location)} + return {} + + +def main(search_client: Optional[Any] = None, answer_client: Optional[Any] = None) -> None: + tracer_provider = register( + project_name="discoveryengine-example", + project_type=ProjectType.OBSERVE, + batch=False, + verbose=False, + ) + DiscoveryEngineInstrumentor().instrument(tracer_provider=tracer_provider) + + serving_config = os.environ["DISCOVERY_ENGINE_SERVING_CONFIG"] + options = client_options(serving_config) + search_client = search_client or discoveryengine.SearchServiceClient(client_options=options) + answer_client = answer_client or discoveryengine.ConversationalSearchServiceClient( + client_options=options + ) + + pager = search_client.search( + request={"serving_config": serving_config, "query": QUERY, "page_size": 5} + ) + print("{0} results on the first page".format(len(pager.results))) + + response = answer_client.answer_query( + request={"serving_config": serving_config, "query": {"text": QUESTION}} + ) + print("answer state: {0}".format(response.answer.state.name)) + tracer_provider.force_flush() + + +if __name__ == "__main__": + main() diff --git a/python/frameworks/discoveryengine/pyproject.toml b/python/frameworks/discoveryengine/pyproject.toml new file mode 100644 index 000000000..6d37678cc --- /dev/null +++ b/python/frameworks/discoveryengine/pyproject.toml @@ -0,0 +1,33 @@ +[tool.poetry] +name = "traceAI-discoveryengine" +version = "0.1.0" +description = "OpenTelemetry instrumentation for Google Discovery Engine (Vertex AI Search) search and answer calls" +authors = ["Future AGI "] +readme = "README.md" +packages = [ + { include = "traceai_discoveryengine" } +] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Software Development :: Libraries :: Python Modules", +] + +[tool.poetry.dependencies] +python = ">=3.10,<3.14" +fi-instrumentation-otel = ">=1.1.0" +google-cloud-discoveryengine = ">=0.20.5,<1" +wrapt = ">=1.0.0" + +[tool.poetry.group.dev.dependencies] +pytest = ">=7.0.0" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/python/frameworks/discoveryengine/tests/_discoveryengine_support.py b/python/frameworks/discoveryengine/tests/_discoveryengine_support.py new file mode 100644 index 000000000..b85bbb505 --- /dev/null +++ b/python/frameworks/discoveryengine/tests/_discoveryengine_support.py @@ -0,0 +1,601 @@ +"""Shared test support: a loopback Discovery Engine gRPC fake and an in-memory tracer. + +``FakeDiscoveryEngine`` is a real gRPC server on 127.0.0.1 that implements +the four v1 RPCs the tests need (``SearchService/Search``, +``SearchService/SearchLite``, ``ConversationalSearchService/AnswerQuery`` +and ``ConversationalSearchService/CreateConversation``) plus +``StreamAnswerQuery``. The real ``google-cloud-discoveryengine`` clients talk +to it through their own gRPC transports, so request serialization, metadata, +retries, paging and ``google.api_core`` error mapping are the library's own. + +It listens twice: an insecure port (no credentials, as with a transport built +on ``grpc.insecure_channel``) and a local-TCP port that accepts +``grpc.local_channel_credentials()``, so a client built with real +``google.auth`` credentials (an OAuth access token or an API key) sends them +as gRPC metadata exactly as it would to Google. Nothing leaves 127.0.0.1; no +ADC, no GCP project and no live endpoint are used, and every token is a +placeholder. + +The query text (``SearchRequest.query`` or ``AnswerQueryRequest.query.text``) +selects the behaviour: + +- ``fail-denied``: PERMISSION_DENIED whose message echoes the query and the + ``authorization`` / ``x-goog-api-key`` metadata the client sent +- ``fail-huge``: INVALID_ARGUMENT whose ~24 KB message echoes the credential + (percent-encoded to ~72 KB of trailer, so clients built with credentials + raise gRPC's 8 KB / 16 KB received-metadata limits) +- ``fail-unseen``: INVALID_ARGUMENT quoting token-shaped strings the client + never held (a ``ya29.`` access token, an ``AIza`` key, a ``1//`` refresh + token, a bearer value) +- ``fail-token``: UNAUTHENTICATED quoting the access token the client sent, + without its ``Bearer`` prefix +- ``fail-details``: INVALID_ARGUMENT quoting the query in its message and + again in a ``BadRequest`` status detail, which ``str(error)`` appends in + protobuf text format (escaped) +- ``fail-unavailable``: UNAVAILABLE on every call +- ``unavailable-once``: UNAVAILABLE on the first call, then success +- ``pages``: a first page with ``next_page_token`` and a second page +- ``slow``: held until the fake closes (for cancellation) +- ``answer-failed``: an answer whose state is FAILED +- ``answer-missing``: a response without an ``answer`` +""" + +from __future__ import annotations + +import contextlib +import datetime +import threading +from concurrent import futures +from dataclasses import dataclass +from typing import Any, Dict, Iterator, List, Optional, Tuple + +import grpc +from google.cloud.discoveryengine_v1.types import ( + Answer, + AnswerQueryRequest, + AnswerQueryResponse, + Conversation, + CreateConversationRequest, + SearchRequest, + SearchResponse, + Session, +) +from google.protobuf import any_pb2, struct_pb2 +from google.rpc import code_pb2, error_details_pb2, status_pb2 +from grpc_status import rpc_status +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import ReadableSpan, TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +SEARCH_SERVICE = "google.cloud.discoveryengine.v1.SearchService" +CONVERSATION_SERVICE = "google.cloud.discoveryengine.v1.ConversationalSearchService" + +SERVING_CONFIG = ( + "projects/test-project/locations/global/collections/default_collection/" + "engines/test-engine/servingConfigs/default_search" +) +SESSION_NAME = ( + "projects/test-project/locations/global/collections/default_collection/" + "engines/test-engine/sessions/session-abc" +) +NEW_SESSION = SESSION_NAME.rsplit("/", 1)[0] + "/-" +CONVERSATION_PARENT = ( + "projects/test-project/locations/global/collections/default_collection/" + "dataStores/test-store" +) + +# Placeholder credentials. The fake echoes them into error messages; none may +# reach a span. They deliberately do not look like Google tokens, so only the +# package's lookup of where the client keeps them can redact them. +ACCESS_TOKEN = "placeholder-access-token-must-not-be-exported" +API_KEY = "placeholder-api-key-must-not-be-exported" +METADATA_TOKEN = "placeholder-metadata-token-must-not-be-exported" +# Token-shaped strings the package never saw (an access token minted by a +# refresh, an API key quoted by the server): caught by their shape. +UNSEEN_ACCESS_TOKEN = "ya29.a0AfB_unseenTokenValue-123" +UNSEEN_API_KEY = "AIza" + "S" * 35 +UNSEEN_REFRESH_TOKEN = "1//0g" + "R" * 30 +# Set on the credentials during the call (first use or refresh), after the +# wrapper first read them. +MINTED_TOKEN = "placeholder-minted-token-must-not-be-exported" + +FAIL_DENIED = "fail-denied" +FAIL_HUGE = "fail-huge" +HUGE_ERROR_CHARS = 8000 +FAIL_UNSEEN = "fail-unseen" +BEARER_VALUE = "opaque-bearer-value-must-not-be-exported" +FAIL_TOKEN = "fail-token" +FAIL_DETAILS = "fail-details" +FAIL_UNAVAILABLE = "fail-unavailable" +UNAVAILABLE_ONCE = "unavailable-once" +PAGES = "pages" +SLOW = "slow" +ANSWER_FAILED = "answer-failed" +ANSWER_MISSING = "answer-missing" + +SEARCH_RESULTS = 2 +SECOND_PAGE_RESULTS = 1 +ANSWER_REFERENCES = 2 + +# Response content the fake returns. None of it may reach a span. +DOCUMENT_TITLE = "DOCUMENT-TITLE-MUST-NOT-BE-EXPORTED" +SNIPPET = "SNIPPET-TEXT-MUST-NOT-BE-EXPORTED" +SUMMARY = "SUMMARY-TEXT-MUST-NOT-BE-EXPORTED" +ANSWER_TEXT = "ANSWER-TEXT-MUST-NOT-BE-EXPORTED" +CHUNK_CONTENT = "CHUNK-CONTENT-MUST-NOT-BE-EXPORTED" +CONTENT_MARKERS = (DOCUMENT_TITLE, SNIPPET, SUMMARY, ANSWER_TEXT, CHUNK_CONTENT) + + +def _struct(**values: Any) -> struct_pb2.Struct: + struct = struct_pb2.Struct() + struct.update(values) + return struct + + +def _search_result(index: int) -> SearchResponse.SearchResult: + return SearchResponse.SearchResult( + id="doc-{0}".format(index), + document={ + "id": "doc-{0}".format(index), + "name": "{0}/branches/0/documents/doc-{1}".format(CONVERSATION_PARENT, index), + "derived_struct_data": _struct( + title=DOCUMENT_TITLE, snippets=[{"snippet": SNIPPET}] + ), + }, + ) + + +def _answer(state: Answer.State) -> Answer: + return Answer( + name=SESSION_NAME + "/answers/answer-1", + state=state, + answer_text=ANSWER_TEXT, + citations=[Answer.Citation(start_index=0, end_index=5, sources=[{"reference_id": "0"}])], + references=[ + Answer.Reference( + chunk_info=Answer.Reference.ChunkInfo( + chunk="chunk-{0}".format(index), content=CHUNK_CONTENT + ) + ) + for index in range(ANSWER_REFERENCES) + ], + ) + + +@dataclass +class Call: + method: str + request: Any + metadata: Dict[str, Any] + + +class FakeDiscoveryEngine: + """Loopback stand-in for discoveryengine.googleapis.com (gRPC, v1).""" + + def __init__(self) -> None: + self.calls: List[Call] = [] + self._lock = threading.Lock() + self._release = threading.Event() + self.received = threading.Event() + self._unavailable_sent: set = set() + self._server = grpc.server(futures.ThreadPoolExecutor(max_workers=8)) + unary = grpc.unary_unary_rpc_method_handler + self._server.add_generic_rpc_handlers( + ( + grpc.method_handlers_generic_handler( + SEARCH_SERVICE, + { + "Search": unary( + self._search("Search"), + request_deserializer=SearchRequest.deserialize, + response_serializer=SearchResponse.serialize, + ), + "SearchLite": unary( + self._search("SearchLite"), + request_deserializer=SearchRequest.deserialize, + response_serializer=SearchResponse.serialize, + ), + }, + ), + grpc.method_handlers_generic_handler( + CONVERSATION_SERVICE, + { + "AnswerQuery": unary( + self._answer_query, + request_deserializer=AnswerQueryRequest.deserialize, + response_serializer=AnswerQueryResponse.serialize, + ), + "StreamAnswerQuery": grpc.unary_stream_rpc_method_handler( + self._stream_answer_query, + request_deserializer=AnswerQueryRequest.deserialize, + response_serializer=AnswerQueryResponse.serialize, + ), + "CreateConversation": unary( + self._create_conversation, + request_deserializer=CreateConversationRequest.deserialize, + response_serializer=Conversation.serialize, + ), + }, + ), + ) + ) + self.port = self._server.add_insecure_port("127.0.0.1:0") + self.secure_port = self._server.add_secure_port( + "127.0.0.1:0", grpc.local_server_credentials(grpc.LocalConnectionType.LOCAL_TCP) + ) + self.target = "127.0.0.1:{0}".format(self.port) + self.secure_target = "127.0.0.1:{0}".format(self.secure_port) + self._server.start() + + # -- recording ----------------------------------------------------------- + + def _record(self, method: str, request: Any, context: grpc.ServicerContext) -> Dict[str, Any]: + metadata = {key.lower(): value for key, value in context.invocation_metadata()} + with self._lock: + self.calls.append(Call(method, request, metadata)) + self.received.set() + return metadata + + def methods(self) -> List[str]: + with self._lock: + return [call.method for call in self.calls] + + # -- behaviours ---------------------------------------------------------- + + def _fail(self, trigger: str, query: str, metadata: Dict[str, Any], context: Any) -> None: + credential = "authorization={0} x-goog-api-key={1}".format( + metadata.get("authorization", ""), metadata.get("x-goog-api-key", "") + ) + if FAIL_DENIED in trigger: + context.abort( + grpc.StatusCode.PERMISSION_DENIED, + "Permission denied on serving config for query '{0}' ({1})".format(query, credential), + ) + if FAIL_HUGE in trigger: + context.abort( + grpc.StatusCode.INVALID_ARGUMENT, + "echo {0} {1}".format(credential, "\u20ac" * HUGE_ERROR_CHARS), + ) + if FAIL_UNSEEN in trigger: + context.abort( + grpc.StatusCode.INVALID_ARGUMENT, + "token {0} key {1} refresh {2} header Authorization: Bearer {3}".format( + UNSEEN_ACCESS_TOKEN, UNSEEN_API_KEY, UNSEEN_REFRESH_TOKEN, BEARER_VALUE + ), + ) + if FAIL_TOKEN in trigger: + token = metadata.get("authorization", "").split(" ", 1)[-1] + context.abort( + grpc.StatusCode.UNAUTHENTICATED, + "Request had invalid authentication credentials: {0}".format(token), + ) + if FAIL_DETAILS in trigger: + detail = any_pb2.Any() + detail.Pack( + error_details_pb2.BadRequest( + field_violations=[ + error_details_pb2.BadRequest.FieldViolation( + field="query", description="Query '{0}' is not supported".format(query) + ) + ] + ) + ) + status = status_pb2.Status( + code=code_pb2.INVALID_ARGUMENT, + message="Invalid query '{0}'".format(query), + details=[detail], + ) + context.abort_with_status(rpc_status.to_status(status)) + if FAIL_UNAVAILABLE in trigger: + context.abort(grpc.StatusCode.UNAVAILABLE, "backend unavailable") + if UNAVAILABLE_ONCE in trigger: + with self._lock: + first = query not in self._unavailable_sent + self._unavailable_sent.add(query) + if first: + context.abort(grpc.StatusCode.UNAVAILABLE, "try again") + if SLOW in trigger: + self._release.wait(30) + context.abort(grpc.StatusCode.UNAVAILABLE, "released") + + def _search(self, method: str): + def handler(request: SearchRequest, context: Any) -> SearchResponse: + metadata = self._record(method, request, context) + self._fail(request.query, request.query, metadata, context) + if PAGES in request.query and request.page_token == "page-2": + return SearchResponse( + results=[_search_result(SEARCH_RESULTS + index) for index in range(SECOND_PAGE_RESULTS)], + total_size=SEARCH_RESULTS + SECOND_PAGE_RESULTS, + ) + return SearchResponse( + results=[_search_result(index) for index in range(SEARCH_RESULTS)], + total_size=SEARCH_RESULTS, + attribution_token="attribution-token", + next_page_token="page-2" if PAGES in request.query else "", + summary={"summary_text": SUMMARY}, + ) + + return handler + + def _answer_query(self, request: AnswerQueryRequest, context: Any) -> AnswerQueryResponse: + metadata = self._record("AnswerQuery", request, context) + query = request.query.text + self._fail(query, query, metadata, context) + if ANSWER_MISSING in query: + return AnswerQueryResponse(answer_query_token="token-1") + state = Answer.State.FAILED if ANSWER_FAILED in query else Answer.State.SUCCEEDED + response = AnswerQueryResponse(answer=_answer(state), answer_query_token="token-1") + if request.session: + # A new session ("sessions/-") comes back with its real name. + response.session = Session(name=SESSION_NAME) + return response + + def _stream_answer_query(self, request: AnswerQueryRequest, context: Any) -> Iterator[AnswerQueryResponse]: + self._record("StreamAnswerQuery", request, context) + yield AnswerQueryResponse(answer=_answer(Answer.State.STREAMING)) + yield AnswerQueryResponse(answer=_answer(Answer.State.SUCCEEDED)) + + def _create_conversation(self, request: CreateConversationRequest, context: Any) -> Conversation: + self._record("CreateConversation", request, context) + return Conversation(name=request.parent + "/conversations/conversation-1") + + # -- lifecycle ----------------------------------------------------------- + + def close(self) -> None: + self._release.set() + self._server.stop(grace=None).wait(5) + + def __enter__(self) -> "FakeDiscoveryEngine": + return self + + def __exit__(self, *_: Any) -> None: + self.close() + + +# -- clients ------------------------------------------------------------------ + + +def _local_credentials() -> grpc.ChannelCredentials: + return grpc.local_channel_credentials(grpc.LocalConnectionType.LOCAL_TCP) + + +# Room for the fail-huge trailer (the defaults are 8 KB soft, 16 KB hard). +_METADATA_LIMITS = [ + ("grpc.max_metadata_size", 256 * 1024), + ("grpc.absolute_max_metadata_size", 256 * 1024), +] + + +def _channel(transport_class: Any) -> Any: + """A channel factory for the transport's ``channel`` argument. + + Unlike a channel instance, a factory keeps the transport's credentials: + the transport calls it with them, exactly as it calls ``create_channel``. + """ + + def create(host: str, **kwargs: Any) -> Any: + kwargs["options"] = list(kwargs.get("options") or []) + _METADATA_LIMITS + return transport_class.create_channel(host, **kwargs) + + return create + + +def search_client(fake: FakeDiscoveryEngine, credentials: Any = None) -> Any: + """A real SearchServiceClient on the gRPC transport, talking to the fake.""" + from google.cloud.discoveryengine_v1 import SearchServiceClient + from google.cloud.discoveryengine_v1.services.search_service.transports.grpc import ( + SearchServiceGrpcTransport, + ) + + if credentials is None: + transport = SearchServiceGrpcTransport(channel=grpc.insecure_channel(fake.target)) + else: + transport = SearchServiceGrpcTransport( + host=fake.secure_target, + credentials=credentials, + ssl_channel_credentials=_local_credentials(), + channel=_channel(SearchServiceGrpcTransport), + ) + return SearchServiceClient(transport=transport) + + +def search_client_with_api_key(fake: FakeDiscoveryEngine, api_key: str) -> Any: + """The search_lite set-up: ``client_options.api_key``, the client builds the credentials.""" + from google.cloud.discoveryengine_v1 import SearchServiceClient + from google.cloud.discoveryengine_v1.services.search_service.transports.grpc import ( + SearchServiceGrpcTransport, + ) + + def transport(**kwargs: Any) -> Any: + kwargs["host"] = fake.secure_target + return SearchServiceGrpcTransport( + ssl_channel_credentials=_local_credentials(), + channel=_channel(SearchServiceGrpcTransport), + **kwargs, + ) + + return SearchServiceClient(client_options={"api_key": api_key}, transport=transport) + + +def async_search_client(fake: FakeDiscoveryEngine, credentials: Any = None) -> Any: + """A real SearchServiceAsyncClient; build it inside the running event loop.""" + from google.cloud.discoveryengine_v1 import SearchServiceAsyncClient + from google.cloud.discoveryengine_v1.services.search_service.transports.grpc_asyncio import ( + SearchServiceGrpcAsyncIOTransport, + ) + + if credentials is None: + transport = SearchServiceGrpcAsyncIOTransport(channel=grpc.aio.insecure_channel(fake.target)) + else: + transport = SearchServiceGrpcAsyncIOTransport( + host=fake.secure_target, + credentials=credentials, + ssl_channel_credentials=_local_credentials(), + channel=_channel(SearchServiceGrpcAsyncIOTransport), + ) + return SearchServiceAsyncClient(transport=transport) + + +def answer_client(fake: FakeDiscoveryEngine, credentials: Any = None) -> Any: + from google.cloud.discoveryengine_v1 import ConversationalSearchServiceClient + from google.cloud.discoveryengine_v1.services.conversational_search_service.transports.grpc import ( + ConversationalSearchServiceGrpcTransport, + ) + + if credentials is None: + transport = ConversationalSearchServiceGrpcTransport( + channel=grpc.insecure_channel(fake.target) + ) + else: + transport = ConversationalSearchServiceGrpcTransport( + host=fake.secure_target, + credentials=credentials, + ssl_channel_credentials=_local_credentials(), + channel=_channel(ConversationalSearchServiceGrpcTransport), + ) + return ConversationalSearchServiceClient(transport=transport) + + +def async_answer_client(fake: FakeDiscoveryEngine, credentials: Any = None) -> Any: + from google.cloud.discoveryengine_v1 import ConversationalSearchServiceAsyncClient + from google.cloud.discoveryengine_v1.services.conversational_search_service.transports.grpc_asyncio import ( + ConversationalSearchServiceGrpcAsyncIOTransport, + ) + + if credentials is None: + transport = ConversationalSearchServiceGrpcAsyncIOTransport( + channel=grpc.aio.insecure_channel(fake.target) + ) + else: + transport = ConversationalSearchServiceGrpcAsyncIOTransport( + host=fake.secure_target, + credentials=credentials, + ssl_channel_credentials=_local_credentials(), + channel=_channel(ConversationalSearchServiceGrpcAsyncIOTransport), + ) + return ConversationalSearchServiceAsyncClient(transport=transport) + + +def oauth_credentials(token: str = ACCESS_TOKEN) -> Any: + """User credentials holding a placeholder access token (never refreshed).""" + from google.oauth2.credentials import Credentials + + return Credentials(token=token) + + +def minting_credentials(token: Optional[str] = None) -> Any: + """User credentials that set ``MINTED_TOKEN`` when the gRPC auth plugin asks for a token. + + With no ``token`` (first use) or an expired one (refresh), the plugin + calls ``refresh`` during the call, after the wrapper first read the + credentials. Nothing is fetched. + """ + from google.oauth2.credentials import Credentials + + class MintingCredentials(Credentials): + def refresh(self, request: Any) -> None: + self.token = MINTED_TOKEN + self.expiry = None + + expiry = None if token is None else datetime.datetime(2000, 1, 1) + return MintingCredentials(token=token, expiry=expiry) + + +def self_signed_jwt_credentials() -> Any: + """Service-account credentials that sign their own JWT, with a placeholder signer. + + google-auth mints the JWT (``eyJ...``) when the call first needs a + token, on the scoped copy google-api-core gives the channel. The token + endpoint is a closed loopback port: a token request (none is expected) + would fail locally. + """ + from google.auth import crypt + from google.oauth2 import service_account + + class PlaceholderSigner(crypt.Signer): + @property + def key_id(self) -> str: + return "placeholder-key-id" + + def sign(self, message: Any) -> bytes: + return b"placeholder-signature" + + credentials = service_account.Credentials( + PlaceholderSigner(), "placeholder@example.invalid", "http://127.0.0.1:9/token" + ) + return credentials.with_always_use_jwt_access(True) + + +def api_key_credentials(key: str = API_KEY) -> Any: + from google.auth import api_key + + return api_key.Credentials(key) + + +def search_request(query: str = "open telemetry retrieval", **fields: Any) -> Dict[str, Any]: + request: Dict[str, Any] = {"serving_config": SERVING_CONFIG, "query": query} + request.update(fields) + return request + + +def answer_request(query: str = "what is open telemetry", **fields: Any) -> Dict[str, Any]: + request: Dict[str, Any] = {"serving_config": SERVING_CONFIG, "query": {"text": query}} + request.update(fields) + return request + + +# -- tracing ------------------------------------------------------------------ + + +@dataclass +class Traced: + exporter: InMemorySpanExporter + provider: TracerProvider + + def spans(self) -> List[ReadableSpan]: + return list(self.exporter.get_finished_spans()) + + def names(self) -> List[str]: + return [span.name for span in self.spans()] + + def one(self) -> ReadableSpan: + spans = self.spans() + assert len(spans) == 1, [span.name for span in spans] + return spans[0] + + def wire(self) -> str: + """Everything an exporter could send: attributes, events, status.""" + return "".join(span.to_json() for span in self.spans()) + + +def new_provider() -> Tuple[InMemorySpanExporter, TracerProvider]: + exporter = InMemorySpanExporter() + provider = TracerProvider(resource=Resource.create({"service.name": "test-discoveryengine"})) + provider.add_span_processor(SimpleSpanProcessor(exporter)) + return exporter, provider + + +@contextlib.contextmanager +def instrumented(**options: Any) -> Iterator[Traced]: + """Instrument Discovery Engine against a fresh in-memory provider; uninstrument after.""" + from traceai_discoveryengine import DiscoveryEngineInstrumentor + + exporter, provider = new_provider() + instrumentor = DiscoveryEngineInstrumentor() + instrumentor.instrument(tracer_provider=provider, **options) + try: + yield Traced(exporter, provider) + finally: + instrumentor.uninstrument() + + +def attrs(span: ReadableSpan) -> Dict[str, Any]: + return dict(span.attributes or {}) + + +def event(span: ReadableSpan, name: str) -> Optional[Dict[str, Any]]: + for item in span.events: + if item.name == name: + return dict(item.attributes or {}) + return None diff --git a/python/frameworks/discoveryengine/tests/test_discoveryengine_answer.py b/python/frameworks/discoveryengine/tests/test_discoveryengine_answer.py new file mode 100644 index 000000000..1a2e306bd --- /dev/null +++ b/python/frameworks/discoveryengine/tests/test_discoveryengine_answer.py @@ -0,0 +1,124 @@ +"""J3, AC-04: ConversationalSearchServiceClient.answer_query is one RETRIEVER span.""" + +from __future__ import annotations + +import pytest +from google.cloud.discoveryengine_v1 import AnswerQueryRequest, Query +from opentelemetry.trace import StatusCode + +from _discoveryengine_support import ( + ANSWER_FAILED, + ANSWER_MISSING, + ANSWER_REFERENCES, + ANSWER_TEXT, + CONTENT_MARKERS, + NEW_SESSION, + SERVING_CONFIG, + SESSION_NAME, + FakeDiscoveryEngine, + answer_client, + answer_request, + attrs, + instrumented, +) + +QUERY = "ANSWER-QUERY-MARKER" + + +@pytest.fixture() +def fake(): + with FakeDiscoveryEngine() as server: + yield server + + +def test_answer_query_is_one_retriever_span_with_counts_not_the_answer(fake): + with instrumented() as traced: + response = answer_client(fake).answer_query(request=answer_request(QUERY)) + + assert response.answer.answer_text == ANSWER_TEXT + span = traced.one() + assert span.name == "discoveryengine.answer_query" + assert attrs(span) == { + "gen_ai.span.kind": "RETRIEVER", + "discoveryengine.serving_config": SERVING_CONFIG, + "discoveryengine.result_count": ANSWER_REFERENCES, + "discoveryengine.answer.length": len(ANSWER_TEXT), + "discoveryengine.answer.state": "SUCCEEDED", + # TraceConfig default: the query (the span's input) is recorded. + "gen_ai.retrieval.query": QUERY, + "input.value": QUERY, + } + assert span.status.status_code is StatusCode.OK + # The answer text and reference content are never recorded. + wire = traced.wire() + for marker in CONTENT_MARKERS: + assert marker not in wire, marker + + +def test_no_model_name_is_invented(fake): + # AC-04: the 0.20.5 AnswerQueryResponse carries no model name, so the + # span has none, even when the request names a model version. + request = answer_request(answer_generation_spec={"model_spec": {"model_version": "MODEL-VERSION-MARKER"}}) + with instrumented() as traced: + answer_client(fake).answer_query(request=request) + + for key in attrs(traced.one()): + assert "model" not in key and "token" not in key and "cost" not in key, key + assert "MODEL-VERSION-MARKER" not in traced.wire() + + +def test_the_session_comes_from_the_request_then_the_response(fake): + with instrumented() as traced: + client = answer_client(fake) + client.answer_query(request=answer_request(session=SESSION_NAME)) + # "sessions/-" starts a session; the response carries its real name. + client.answer_query(request=answer_request(session=NEW_SESSION)) + client.answer_query(request=answer_request()) + + named, created, none = traced.spans() + assert attrs(named)["discoveryengine.session"] == SESSION_NAME + assert attrs(created)["discoveryengine.session"] == SESSION_NAME + assert "discoveryengine.session" not in attrs(none) + + +def test_a_failed_answer_state_is_an_error_span(fake): + with instrumented() as traced: + response = answer_client(fake).answer_query(request=answer_request(ANSWER_FAILED)) + + assert response.answer.state.name == "FAILED" + span = traced.one() + assert attrs(span)["discoveryengine.answer.state"] == "FAILED" + assert span.status.status_code is StatusCode.ERROR + assert span.status.description == "answer state FAILED" + # Nothing was raised, so there is no exception event. + assert span.events == () + + +def test_a_response_without_an_answer_omits_the_counts(fake): + with instrumented() as traced: + answer_client(fake).answer_query(request=answer_request(ANSWER_MISSING)) + + values = attrs(traced.one()) + for key in ("discoveryengine.result_count", "discoveryengine.answer.length", "discoveryengine.answer.state"): + assert key not in values, key + assert traced.one().status.status_code is StatusCode.OK + + +@pytest.mark.parametrize( + "request_factory", + [ + pytest.param(lambda q: answer_request(q), id="dict"), + pytest.param(lambda q: AnswerQueryRequest(answer_request(q)), id="message"), + pytest.param( + lambda q: {"serving_config": SERVING_CONFIG, "query": Query(text=q)}, id="dict-with-query-message" + ), + ], +) +def test_the_query_text_is_recorded_by_default(fake, request_factory): + with instrumented() as traced: + answer_client(fake).answer_query(request_factory("what is a retriever span")) + + values = attrs(traced.one()) + assert values["input.value"] == "what is a retriever span" + assert values["gen_ai.retrieval.query"] == "what is a retriever span" + assert ANSWER_TEXT not in traced.wire() diff --git a/python/frameworks/discoveryengine/tests/test_discoveryengine_async.py b/python/frameworks/discoveryengine/tests/test_discoveryengine_async.py new file mode 100644 index 000000000..69cbf0355 --- /dev/null +++ b/python/frameworks/discoveryengine/tests/test_discoveryengine_async.py @@ -0,0 +1,151 @@ +"""J6, AC-07: the async clients produce the same spans as the sync clients.""" + +from __future__ import annotations + +import asyncio + +import pytest +from google.api_core import exceptions as core_exceptions +from opentelemetry.trace import StatusCode + +from _discoveryengine_support import ( + FAIL_DENIED, + PAGES, + SEARCH_RESULTS, + SECOND_PAGE_RESULTS, + SLOW, + FakeDiscoveryEngine, + answer_client, + answer_request, + async_answer_client, + async_search_client, + attrs, + event, + instrumented, + search_client, + search_request, +) + + +@pytest.fixture() +def fake(): + with FakeDiscoveryEngine() as server: + yield server + + +def _sync_journey(fake): + search = search_client(fake) + search.search(request=search_request("parity")) + search.search_lite(request=search_request("parity")) + answer_client(fake).answer_query(request=answer_request("parity")) + + +async def _async_journey(fake): + search = async_search_client(fake) + answer = async_answer_client(fake) + try: + await search.search(request=search_request("parity")) + await search.search_lite(request=search_request("parity")) + await answer.answer_query(request=answer_request("parity")) + finally: + await search.transport.close() + await answer.transport.close() + + +@pytest.mark.parametrize("hide_inputs", [False, True], ids=["default", "hide_inputs"]) +def test_async_spans_match_sync_spans(fake, hide_inputs): + from fi_instrumentation import TraceConfig + + with instrumented(config=TraceConfig(hide_inputs=hide_inputs)) as traced: + _sync_journey(fake) + asyncio.run(_async_journey(fake)) + + spans = traced.spans() + assert [span.name for span in spans] == [ + "discoveryengine.search", + "discoveryengine.search_lite", + "discoveryengine.answer_query", + ] * 2 + for sync_span, async_span in zip(spans[:3], spans[3:]): + assert attrs(async_span) == attrs(sync_span), sync_span.name + assert async_span.status.status_code is StatusCode.OK + assert fake.methods() == ["Search", "SearchLite", "AnswerQuery"] * 2 + + +def test_async_pager_iteration_keeps_one_span(fake): + async def call(): + client = async_search_client(fake) + try: + pager = await client.search(request=search_request(PAGES)) + return [result async for result in pager] + finally: + await client.transport.close() + + with instrumented() as traced: + results = asyncio.run(call()) + + assert len(results) == SEARCH_RESULTS + SECOND_PAGE_RESULTS + assert fake.methods() == ["Search", "Search"] + assert attrs(traced.one())["discoveryengine.result_count"] == SEARCH_RESULTS + + +def test_async_error_matches_sync_error(fake): + async def call(): + client = async_search_client(fake) + try: + await client.search(request=search_request(FAIL_DENIED)) + finally: + await client.transport.close() + + with instrumented() as traced: + with pytest.raises(core_exceptions.PermissionDenied): + search_client(fake).search(request=search_request(FAIL_DENIED)) + with pytest.raises(core_exceptions.PermissionDenied): + asyncio.run(call()) + + sync_span, async_span = traced.spans() + assert attrs(async_span) == attrs(sync_span) + assert async_span.status.status_code is StatusCode.ERROR + assert async_span.status.description == sync_span.status.description + assert event(async_span, "exception")["exception.message"] == event(sync_span, "exception")["exception.message"] + + +def test_cancelling_an_async_call_ends_the_span_as_cancelled(fake): + async def call(): + client = async_answer_client(fake) + try: + task = asyncio.ensure_future(client.answer_query(request=answer_request(SLOW))) + while not fake.received.is_set(): + await asyncio.sleep(0.01) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + finally: + await client.transport.close() + + with instrumented() as traced: + asyncio.run(call()) + + span = traced.one() + assert span.name == "discoveryengine.answer_query" + assert attrs(span)["discoveryengine.cancelled"] is True + assert span.status.status_code is StatusCode.ERROR + assert span.status.description == "cancelled" + assert span.events == () + assert "discoveryengine.result_count" not in attrs(span) + + +def test_concurrent_async_calls_get_one_span_each(fake): + async def call(): + client = async_search_client(fake) + try: + await asyncio.gather(*(client.search(request=search_request("q{0}".format(i))) for i in range(5))) + finally: + await client.transport.close() + + with instrumented() as traced: + asyncio.run(call()) + + assert traced.names() == ["discoveryengine.search"] * 5 + assert len({span.context.span_id for span in traced.spans()}) == 5 + assert all(span.parent is None for span in traced.spans()) diff --git a/python/frameworks/discoveryengine/tests/test_discoveryengine_config.py b/python/frameworks/discoveryengine/tests/test_discoveryengine_config.py new file mode 100644 index 000000000..a0b94090f --- /dev/null +++ b/python/frameworks/discoveryengine/tests/test_discoveryengine_config.py @@ -0,0 +1,294 @@ +"""Query capture follows TraceConfig (on by default, FI_HIDE_INPUTS hides it); PII redaction; error text.""" + +from __future__ import annotations + +import asyncio + +import pytest +from google.api_core import exceptions as core_exceptions + +from _discoveryengine_support import ( + FAIL_DENIED, + FAIL_DETAILS, + SERVING_CONFIG, + FakeDiscoveryEngine, + answer_client, + answer_request, + async_search_client, + attrs, + event, + instrumented, + search_client, + search_request, +) +from traceai_discoveryengine import DiscoveryEngineInstrumentor, _wrappers + +EMAIL = "jane.doe@example.com" +EMAIL_TOKEN = "" +REDACTED_VALUE = "__REDACTED__" +DENIED_QUERY = FAIL_DENIED + " ask about " + EMAIL +# An apostrophe, a double quote, a backslash and a non-ASCII letter: the +# status detail the fake adds quotes it escaped (protobuf text format). +QUOTED_QUERY = FAIL_DETAILS + " Zelda's \"quoted\" a\\b caf\xe9" +QUOTED_DETAIL = r'Zelda\'s \"quoted\" a\\b caf' + "\xe9" + + +@pytest.fixture() +def fake(): + with FakeDiscoveryEngine() as server: + yield server + + +@pytest.fixture(autouse=True) +def clean_env(monkeypatch): + for name in ("FI_HIDE_INPUTS", "FI_HIDE_OUTPUTS", "FI_PII_REDACTION"): + monkeypatch.delenv(name, raising=False) + + +def _error_texts(span): + exception = event(span, "exception") or {} + # No stack trace is ever recorded: frames carry local file paths. + assert "exception.stacktrace" not in exception + return ( + span.status.description or "", + exception.get("exception.message", ""), + ) + + +def _denied(fake, traced_options): + with instrumented(**traced_options) as traced: + with pytest.raises(core_exceptions.PermissionDenied) as raised: + search_client(fake).search(request=search_request(DENIED_QUERY)) + # The server echoed the query; the caller still sees it. + assert DENIED_QUERY in str(raised.value) + return traced.one() + + +def test_the_query_is_recorded_by_default_and_kept_in_error_text(fake): + # TraceConfig default (DEFAULT_HIDE_INPUTS = False), as in every traceAI + # instrumentor: the query reaches the span and the server's echo of it + # stays in the error text. + span = _denied(fake, {}) + values = attrs(span) + assert values["input.value"] == DENIED_QUERY + assert values["gen_ai.retrieval.query"] == DENIED_QUERY + for text in _error_texts(span): + assert DENIED_QUERY in text + + +@pytest.mark.parametrize("source", ["config", "env"]) +def test_hide_inputs_removes_the_query_everywhere_including_error_text(fake, monkeypatch, source): + # Control for the test above: FI_HIDE_INPUTS (or TraceConfig) hides it. + options = {} + if source == "config": + options["config"] = _config(hide_inputs=True) + else: + monkeypatch.setenv("FI_HIDE_INPUTS", "true") + span = _denied(fake, options) + values = attrs(span) + assert values["input.value"] == REDACTED_VALUE + assert "gen_ai.retrieval.query" not in values + for text in _error_texts(span): + assert text + assert DENIED_QUERY not in text + assert EMAIL not in text + assert REDACTED_VALUE in span.status.description + assert DENIED_QUERY not in span.to_json() + + +def _quoted(fake, traced_options): + with instrumented(**traced_options) as traced: + with pytest.raises(core_exceptions.InvalidArgument) as raised: + search_client(fake).search(request=search_request(QUOTED_QUERY)) + # google-api-core appends the BadRequest detail, escaped, to the message. + assert QUOTED_QUERY in str(raised.value) + assert QUOTED_DETAIL in str(raised.value) + return traced.one() + + +def test_the_server_echo_of_the_query_includes_an_escaped_copy(fake): + # Control for the test below: by default (query recorded) both copies are kept. + span = _quoted(fake, {}) + for text in _error_texts(span): + assert QUOTED_QUERY in text + assert QUOTED_DETAIL in text + + +@pytest.mark.parametrize("source", ["config", "env"]) +def test_escaped_copies_of_the_query_are_removed_from_error_text(fake, monkeypatch, source): + options = _hidden_options(monkeypatch, source) + span = _quoted(fake, options) + for text in _error_texts(span): + assert text + assert "Zelda" not in text + assert "quoted" not in text + assert REDACTED_VALUE in text + assert "Zelda" not in span.to_json() + + +# ASCII control characters other than \t, \n and \r, and DEL: protobuf text +# format writes them as 3-digit octal (\013, \033, \177), which none of the +# repr, JSON or CHexEscape copies matches. +CONTROL_QUERY = FAIL_DETAILS + " Zelda\x0bcontrol\x1bcopy\x7fend" +CONTROL_DETAIL = r"Zelda\013control\033copy\177end" + + +def test_the_status_detail_writes_control_characters_as_octal(fake): + # Control: the escaped copy really is in the error the caller sees. + with instrumented(): + with pytest.raises(core_exceptions.InvalidArgument) as raised: + search_client(fake).search(request=search_request(CONTROL_QUERY)) + assert CONTROL_DETAIL in str(raised.value) + + +@pytest.mark.parametrize("source", ["config", "env"]) +def test_octal_escaped_copies_of_the_query_are_removed_from_error_text(fake, monkeypatch, source): + options = _hidden_options(monkeypatch, source) + with instrumented(**options) as traced: + with pytest.raises(core_exceptions.InvalidArgument) as raised: + search_client(fake).search(request=search_request(CONTROL_QUERY)) + assert CONTROL_DETAIL in str(raised.value) + span = traced.one() + for text in _error_texts(span): + assert text + assert "Zelda" not in text and "control" not in text, text + assert REDACTED_VALUE in text + assert "Zelda" not in span.to_json() + + +def _config(**fields): + from fi_instrumentation import TraceConfig + + return TraceConfig(**fields) + + +def _hidden_options(monkeypatch, source): + if source == "config": + return {"config": _config(hide_inputs=True)} + monkeypatch.setenv("FI_HIDE_INPUTS", "true") + return {} + + +def test_hide_inputs_records_only_the_placeholder(fake): + with instrumented(config=_config(hide_inputs=True)) as traced: + search_client(fake).search(request=search_request("plain")) + + values = attrs(traced.one()) + assert values["input.value"] == REDACTED_VALUE + assert "gen_ai.retrieval.query" not in values + assert "plain" not in traced.wire() + + +def test_capture_query_is_ignored_with_a_warning(fake, caplog): + # The pre-release package-specific switch is gone (one privacy switch: + # TraceConfig). Passing it changes nothing and logs why. + with caplog.at_level("WARNING", logger="traceai_discoveryengine"): + with instrumented(capture_query=False) as traced: + search_client(fake).search(request=search_request("still recorded")) + + assert attrs(traced.one())["input.value"] == "still recorded" + assert "capture_query is ignored" in caplog.text + + +def test_hide_outputs_changes_nothing_because_no_output_text_is_recorded(fake): + with instrumented() as plain: + answer_client(fake).answer_query(request=answer_request("same")) + with instrumented(config=_config(hide_outputs=True)) as hidden: + answer_client(fake).answer_query(request=answer_request("same")) + + assert attrs(hidden.one()) == attrs(plain.one()) + + +@pytest.mark.parametrize("source", ["config", "env"]) +def test_pii_redaction_covers_the_query_and_the_error_text(fake, monkeypatch, source): + options = {} + if source == "config": + options["config"] = _config(pii_redaction=True) + else: + monkeypatch.setenv("FI_PII_REDACTION", "true") + span = _denied(fake, options) + + assert attrs(span)["input.value"] == FAIL_DENIED + " ask about " + EMAIL_TOKEN + for text in _error_texts(span): + assert EMAIL not in text + assert EMAIL_TOKEN in text + assert EMAIL not in span.to_json() + + +def test_pii_straddling_the_cap_is_replaced_before_the_cut(fake): + query = "x" * (_wrappers.MAX_VALUE_BYTES - 8) + " " + EMAIL + with instrumented(config=_config(pii_redaction=True)) as traced: + search_client(fake).search(request=search_request(query)) + + value = attrs(traced.one())["input.value"] + assert "jane" not in value + assert len(value.encode("utf-8")) <= _wrappers.MAX_VALUE_BYTES + + +def test_pii_redaction_also_matches_a_project_number_in_resource_names(fake): + # Documented: the phone pattern matches the last ten digits of a GCP + # project number, so resource names lose part of it with pii_redaction. + serving_config = SERVING_CONFIG.replace("projects/test-project/", "projects/123456789012/") + with instrumented(config=_config(pii_redaction=True)) as traced: + search_client(fake).search(request=search_request(serving_config=serving_config)) + + value = attrs(traced.one())["discoveryengine.serving_config"] + assert value == serving_config.replace("123456789012", "12") + + +def test_the_query_is_capped_on_a_character_boundary(fake): + query = "\u20ac" * 600 # 1800 bytes of UTF-8 + with instrumented() as traced: + search_client(fake).search(request=search_request(query)) + + value = attrs(traced.one())["input.value"] + assert value == "\u20ac" * (_wrappers.MAX_VALUE_BYTES // 3) + assert len(value.encode("utf-8")) == 1023 + + +def test_using_the_default_config_matches_an_explicit_default(fake): + with instrumented() as implicit: + search_client(fake).search(request=search_request()) + with instrumented(config=_config()) as explicit: + search_client(fake).search(request=search_request()) + + assert attrs(implicit.one()) == attrs(explicit.one()) + + +def test_async_calls_honour_the_same_settings(fake): + async def call(): + client = async_search_client(fake) + try: + with pytest.raises(core_exceptions.PermissionDenied): + await client.search(request=search_request(DENIED_QUERY)) + finally: + await client.transport.close() + + with instrumented(config=_config(hide_inputs=True)) as traced: + asyncio.run(call()) + + span = traced.one() + assert attrs(span)["input.value"] == REDACTED_VALUE + assert DENIED_QUERY not in span.to_json() + + +@pytest.mark.parametrize( + "options, message", + [ + ({"config": {"hide_inputs": True}}, "config must be a fi_instrumentation.TraceConfig"), + ({"config": "hide"}, "config must be a fi_instrumentation.TraceConfig"), + ], +) +def test_bad_options_raise_type_error_and_wrap_nothing(options, message): + from google.cloud.discoveryengine_v1 import SearchServiceClient + + before = vars(SearchServiceClient)["search"] + instrumentor = DiscoveryEngineInstrumentor() + with pytest.raises(TypeError, match=message): + instrumentor.instrument(**options) + try: + assert vars(SearchServiceClient)["search"] is before + assert not instrumentor.is_instrumented_by_opentelemetry + finally: + instrumentor.uninstrument() + assert vars(SearchServiceClient)["search"] is before diff --git a/python/frameworks/discoveryengine/tests/test_discoveryengine_context.py b/python/frameworks/discoveryengine/tests/test_discoveryengine_context.py new file mode 100644 index 000000000..10eecdc69 --- /dev/null +++ b/python/frameworks/discoveryengine/tests/test_discoveryengine_context.py @@ -0,0 +1,200 @@ +"""Trace context: the span is current while the transport sends, nests, and carries using_* attributes.""" + +from __future__ import annotations + +import asyncio +import json + +import grpc +import pytest +from opentelemetry import trace as trace_api + +from _discoveryengine_support import ( + FakeDiscoveryEngine, + answer_client, + answer_request, + async_answer_client, + async_search_client, + attrs, + instrumented, + search_client, + search_request, +) + + +@pytest.fixture() +def fake(): + with FakeDiscoveryEngine() as server: + yield server + + +def _current_span_id() -> int: + return trace_api.get_current_span().get_span_context().span_id + + +class _RecordCurrentSpan(grpc.UnaryUnaryClientInterceptor): + """Runs where a gRPC client instrumentation would start its span.""" + + def __init__(self, seen): + self.seen = seen + + def intercept_unary_unary(self, continuation, client_call_details, request): + self.seen.append(_current_span_id()) + return continuation(client_call_details, request) + + +class _AsyncRecordCurrentSpan(grpc.aio.UnaryUnaryClientInterceptor): + def __init__(self, seen): + self.seen = seen + + async def intercept_unary_unary(self, continuation, client_call_details, request): + self.seen.append(_current_span_id()) + return await continuation(client_call_details, request) + + +def test_sync_rpc_runs_inside_the_discoveryengine_span(fake): + from google.cloud.discoveryengine_v1 import SearchServiceClient + from google.cloud.discoveryengine_v1.services.search_service.transports.grpc import ( + SearchServiceGrpcTransport, + ) + + seen = [] + channel = grpc.intercept_channel(grpc.insecure_channel(fake.target), _RecordCurrentSpan(seen)) + client = SearchServiceClient(transport=SearchServiceGrpcTransport(channel=channel)) + with instrumented() as traced: + client.search(request=search_request()) + client.search_lite(request=search_request()) + + assert seen == [span.context.span_id for span in traced.spans()] + assert len(seen) == 2 + assert _current_span_id() == 0 + + +def test_async_rpc_runs_inside_the_discoveryengine_span(fake): + from google.cloud.discoveryengine_v1 import ConversationalSearchServiceAsyncClient + from google.cloud.discoveryengine_v1.services.conversational_search_service.transports.grpc_asyncio import ( + ConversationalSearchServiceGrpcAsyncIOTransport, + ) + + seen = [] + + async def call(): + channel = grpc.aio.insecure_channel(fake.target, interceptors=[_AsyncRecordCurrentSpan(seen)]) + client = ConversationalSearchServiceAsyncClient( + transport=ConversationalSearchServiceGrpcAsyncIOTransport(channel=channel) + ) + try: + await client.answer_query(request=answer_request()) + finally: + await client.transport.close() + + with instrumented() as traced: + asyncio.run(call()) + + assert seen == [traced.one().context.span_id] + + +def test_spans_are_children_of_the_active_span(fake): + async def call(): + client = async_search_client(fake) + try: + await client.search(request=search_request()) + finally: + await client.transport.close() + + with instrumented() as traced: + tracer = traced.provider.get_tracer("agent") + with tracer.start_as_current_span("agent.step") as parent: + search_client(fake).search(request=search_request()) + answer_client(fake).answer_query(request=answer_request()) + asyncio.run(call()) + + spans = traced.spans() + agent = [span for span in spans if span.name == "agent.step"][0] + assert agent.context.span_id == parent.get_span_context().span_id + children = [span for span in spans if span.name != "agent.step"] + assert [span.name for span in children] == [ + "discoveryengine.search", + "discoveryengine.answer_query", + "discoveryengine.search", + ] + for span in children: + assert span.parent.span_id == agent.context.span_id + assert span.context.trace_id == agent.context.trace_id + + +def test_a_call_without_an_active_span_is_a_root_span(fake): + with instrumented() as traced: + search_client(fake).search(request=search_request()) + + assert traced.one().parent is None + + +def test_using_session_and_using_user_stamp_sync_and_async_spans(fake): + from fi_instrumentation import using_session, using_user + + async def call(): + search = async_search_client(fake) + answer = async_answer_client(fake) + try: + await search.search(request=search_request()) + await answer.answer_query(request=answer_request()) + finally: + await search.transport.close() + await answer.transport.close() + + with instrumented() as traced: + with using_session("s1"), using_user("u1"): + search_client(fake).search_lite(request=search_request()) + answer_client(fake).answer_query(request=answer_request()) + asyncio.run(call()) + search_client(fake).search(request=search_request()) + + spans = traced.spans() + assert len(spans) == 5 + for span in spans[:4]: + values = attrs(span) + assert values["session.id"] == "s1", span.name + assert values["user.id"] == "u1", span.name + assert "session.id" not in attrs(spans[4]) + assert "user.id" not in attrs(spans[4]) + + +def test_using_attributes_stamps_metadata_and_tags(fake): + from fi_instrumentation import using_attributes + + metadata = {"team": "search", "run": 3} + with instrumented() as traced: + with using_attributes(session_id="s2", user_id="u2", metadata=metadata, tags=["t1", "t2"]): + answer_client(fake).answer_query(request=answer_request()) + + values = attrs(traced.one()) + assert values["session.id"] == "s2" + assert values["user.id"] == "u2" + assert json.loads(values["metadata"]) == metadata + assert list(values["tag.tags"]) == ["t1", "t2"] + # The Discovery Engine session is a separate attribute and is not set here. + assert "discoveryengine.session" not in values + + +def test_spans_come_from_fitracer_not_a_plain_tracer(fake, monkeypatch): + # Mutation guard: if the wrapper used the plain OTel tracer, the using_* + # stamps and the PII pass on attributes would silently disappear. + from fi_instrumentation import FITracer + from fi_instrumentation import using_session + + created = [] + original = FITracer.start_span + + def spy(self, *args, **kwargs): + span = original(self, *args, **kwargs) + created.append(span) + return span + + monkeypatch.setattr(FITracer, "start_span", spy) + with instrumented() as traced: + with using_session("s3"): + search_client(fake).search(request=search_request()) + + assert len(created) == 1 + assert attrs(traced.one())["session.id"] == "s3" diff --git a/python/frameworks/discoveryengine/tests/test_discoveryengine_contract.py b/python/frameworks/discoveryengine/tests/test_discoveryengine_contract.py new file mode 100644 index 000000000..bdedcfc65 --- /dev/null +++ b/python/frameworks/discoveryengine/tests/test_discoveryengine_contract.py @@ -0,0 +1,220 @@ +"""Shared-harness contract tests for traceAI-discoveryengine (TH-8332). + +The real ``google-cloud-discoveryengine`` clients make real gRPC calls to the +loopback fake. Spans leave through the real ``fi_instrumentation.register()`` +OTLP exporter into ``harness.Receiver``. The architecture's round-trip check, +``harness.post_otlp()`` of spans the unit tests captured in memory, is the +last test. Nothing calls discoveryengine.googleapis.com, no ADC is read and +every credential is a placeholder. +""" + +from __future__ import annotations + +import asyncio +import json +import sys +from pathlib import Path +from typing import Dict, List + +import pytest + +pytest.importorskip("opentelemetry.proto", reason="the harness decodes OTLP protobuf") + +sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "tests")) + +from harness import Receiver, _flatten_attributes, post_otlp # noqa: E402 + +from _discoveryengine_support import ( # noqa: E402 + ACCESS_TOKEN, + ANSWER_REFERENCES, + ANSWER_TEXT, + CONTENT_MARKERS, + FAIL_DENIED, + SEARCH_RESULTS, + SERVING_CONFIG, + SESSION_NAME, + FakeDiscoveryEngine, + answer_client, + answer_request, + async_search_client, + instrumented, + oauth_credentials, + search_client, + search_request, +) + +FI_API_KEY = "placeholder-fi-api-key" +FI_SECRET_KEY = "placeholder-fi-secret-key" +PROJECT = "discoveryengine-contract" +QUERY = "CONTRACT-QUERY-MARKER" + + +def _journey(receiver, fake, monkeypatch, **options) -> List[dict]: + from fi_instrumentation import register + from fi_instrumentation.fi_types import ProjectType + + from traceai_discoveryengine import DiscoveryEngineInstrumentor + + monkeypatch.setenv("FI_BASE_URL", receiver.origin) + monkeypatch.setenv("FI_API_KEY", FI_API_KEY) + monkeypatch.setenv("FI_SECRET_KEY", FI_SECRET_KEY) + provider = register(project_type=ProjectType.OBSERVE, project_name=PROJECT, verbose=False) + instrumentor = DiscoveryEngineInstrumentor() + instrumentor.instrument(tracer_provider=provider, **options) + try: + client = search_client(fake, credentials=oauth_credentials()) + with provider.get_tracer("app").start_as_current_span("app.request"): + pager = client.search(request=search_request(QUERY)) + # The content flowed into the client, so its absence below is meaningful. + assert len(pager.results) == SEARCH_RESULTS + client.search_lite(request=search_request(QUERY)) + response = answer_client(fake).answer_query( + request=answer_request(QUERY, session=SESSION_NAME) + ) + assert response.answer.answer_text == ANSWER_TEXT + with pytest.raises(Exception): + client.search(request=search_request(FAIL_DENIED + " " + QUERY)) + + async def call() -> None: + async_client = async_search_client(fake) + try: + await async_client.search(request=search_request(QUERY)) + finally: + await async_client.transport.close() + + asyncio.run(call()) + assert provider.force_flush(timeout_millis=10_000) + finally: + instrumentor.uninstrument() + provider.shutdown() + return receiver.spans() + + +def _by_name(spans) -> Dict[str, List[dict]]: + grouped: Dict[str, List[dict]] = {} + for span in spans: + grouped.setdefault(span["name"], []).append(span) + return grouped + + +def test_real_client_calls_reach_the_collector_contract(monkeypatch): + with FakeDiscoveryEngine() as fake, Receiver() as receiver: + spans = _journey(receiver, fake, monkeypatch) + exports = receiver.requests() + + # The real clients made every RPC (no monkeypatched client methods). + assert fake.methods() == ["Search", "SearchLite", "AnswerQuery", "Search", "Search"] + + # Collector contract on every export. + assert exports + for export in exports: + assert export["path"] == "/tracer/v1/traces" + assert export["headers"]["x-api-key"] == FI_API_KEY + assert export["headers"]["x-secret-key"] == FI_SECRET_KEY + assert "authorization" not in export["headers"] + for resource in export["resource_attributes"]: + assert resource["project_name"] == PROJECT + assert resource["project_type"] == "observe" + + by_name = _by_name(spans) + assert sorted((name, len(group)) for name, group in by_name.items()) == [ + ("app.request", 1), + ("discoveryengine.answer_query", 1), + ("discoveryengine.search", 3), + ("discoveryengine.search_lite", 1), + ] + (parent,) = by_name["app.request"] + ok_search, failed_search, async_search = by_name["discoveryengine.search"] + (lite,) = by_name["discoveryengine.search_lite"] + (answer,) = by_name["discoveryengine.answer_query"] + + # Children of the active span, in its trace; the later calls are roots. + for child in (ok_search, lite, answer): + assert child["parentSpanId"] == parent["spanId"] + assert child["traceId"] == parent["traceId"] + assert not failed_search.get("parentSpanId") + assert not async_search.get("parentSpanId") + + for span in (ok_search, lite, async_search): + values = _flatten_attributes(span["attributes"]) + assert values["gen_ai.span.kind"] == "RETRIEVER" + assert values["discoveryengine.serving_config"] == SERVING_CONFIG + assert int(values["discoveryengine.result_count"]) == SEARCH_RESULTS + # TraceConfig default: the query is recorded. + assert values["input.value"] == QUERY + # register()'s processor also promotes UNSET to OK on export + # (fi_instrumentation/otel.py _auto_set_ok_status), so the wrapper's + # own OK is pinned by the in-memory tests. + assert span["status"]["code"] == "STATUS_CODE_OK" + + values = _flatten_attributes(answer["attributes"]) + assert values["gen_ai.span.kind"] == "RETRIEVER" + assert int(values["discoveryengine.result_count"]) == ANSWER_REFERENCES + assert int(values["discoveryengine.answer.length"]) == len(ANSWER_TEXT) + assert values["discoveryengine.answer.state"] == "SUCCEEDED" + assert values["discoveryengine.session"] == SESSION_NAME + assert not any("model" in key or "token" in key or "cost" in key for key in values) + + failed = _flatten_attributes(failed_search["attributes"]) + assert failed["discoveryengine.error.status"] == "PERMISSION_DENIED" + assert int(failed["discoveryengine.error.code"]) == 403 + assert "discoveryengine.result_count" not in failed + assert failed_search["status"]["code"] == "STATUS_CODE_ERROR" + assert [event["name"] for event in failed_search.get("events", [])] == ["exception"] + + +def test_with_fi_hide_inputs_no_credential_query_or_content_is_exported(monkeypatch): + monkeypatch.setenv("FI_HIDE_INPUTS", "true") + with FakeDiscoveryEngine() as fake, Receiver() as receiver: + spans = _journey(receiver, fake, monkeypatch) + exports = receiver.requests() + + # The fake received the token, and its PERMISSION_DENIED message echoed + # both the token and the query into the vendor's exception. + assert fake.calls[0].metadata["authorization"] == "Bearer " + ACCESS_TOKEN + wire = json.dumps(spans) + json.dumps(exports) + for secret in (ACCESS_TOKEN, QUERY) + CONTENT_MARKERS: + assert secret not in wire, secret + + +def test_the_query_reaches_the_collector_by_default(monkeypatch): + # Control for the test above: by default the query is on the wire, + # still without the token, any response content or a stack trace. + monkeypatch.delenv("FI_HIDE_INPUTS", raising=False) + with FakeDiscoveryEngine() as fake, Receiver() as receiver: + spans = _journey(receiver, fake, monkeypatch) + + search = _flatten_attributes(_by_name(spans)["discoveryengine.search"][0]["attributes"]) + assert search["input.value"] == QUERY + assert search["gen_ai.retrieval.query"] == QUERY + wire = json.dumps(spans) + assert ACCESS_TOKEN not in wire + for marker in CONTENT_MARKERS: + assert marker not in wire, marker + for local in ("exception.stacktrace", "Traceback", 'File "', "site-packages", str(Path.home())): + assert local not in wire, local + + +def test_post_otlp_round_trip_of_captured_spans(): + # The architecture's harness use: post_otlp() of spans the unit test + # captured with InMemorySpanExporter, no child process. + from google.protobuf.json_format import MessageToDict + from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans + + with FakeDiscoveryEngine() as fake: + with instrumented() as traced: + search_client(fake).search(request=search_request()) + answer_client(fake).answer_query(request=answer_request()) + captured = traced.spans() + payload = MessageToDict(encode_spans(captured)) + + with Receiver() as receiver: + assert post_otlp(payload, receiver.endpoint) == 200 + received = receiver.spans() + + assert [span["name"] for span in received] == [span.name for span in captured] + for sent, got in zip(captured, received): + values = _flatten_attributes(got["attributes"]) + assert set(values) == set(sent.attributes) + for key, value in sent.attributes.items(): + assert str(values[key]) == str(value), key diff --git a/python/frameworks/discoveryengine/tests/test_discoveryengine_errors.py b/python/frameworks/discoveryengine/tests/test_discoveryengine_errors.py new file mode 100644 index 000000000..29fb7e39e --- /dev/null +++ b/python/frameworks/discoveryengine/tests/test_discoveryengine_errors.py @@ -0,0 +1,537 @@ +"""J5, AC-06, AC-08: errors set ERROR with the status code; no credential is exported.""" + +from __future__ import annotations + +import asyncio + +import pytest +from google.api_core import exceptions as core_exceptions +from opentelemetry.trace import StatusCode + +from _discoveryengine_support import ( + ACCESS_TOKEN, + API_KEY, + BEARER_VALUE, + FAIL_DENIED, + FAIL_HUGE, + FAIL_TOKEN, + FAIL_UNAVAILABLE, + FAIL_UNSEEN, + METADATA_TOKEN, + MINTED_TOKEN, + UNSEEN_ACCESS_TOKEN, + UNSEEN_API_KEY, + UNSEEN_REFRESH_TOKEN, + FakeDiscoveryEngine, + answer_client, + async_search_client, + attrs, + event, + instrumented, + minting_credentials, + new_provider, + oauth_credentials, + api_key_credentials, + search_client, + search_client_with_api_key, + search_request, + answer_request, + self_signed_jwt_credentials, +) +from traceai_discoveryengine import _wrappers + +SECRETS = (ACCESS_TOKEN, API_KEY, METADATA_TOKEN) + + +@pytest.fixture() +def fake(): + with FakeDiscoveryEngine() as server: + yield server + + +def _error_texts(span): + exception = event(span, "exception") or {} + # No stack trace is ever recorded: frames carry local file paths. + assert "exception.stacktrace" not in exception + return ( + span.status.description or "", + exception.get("exception.message", ""), + ) + + +def test_permission_denied_sets_error_with_the_status_code_and_reraises(fake): + client = search_client(fake, credentials=oauth_credentials()) + with instrumented() as traced: + with pytest.raises(core_exceptions.PermissionDenied) as raised: + client.search(request=search_request(FAIL_DENIED)) + + # The caller's exception is the vendor's own, message untouched. + assert ACCESS_TOKEN in str(raised.value) + assert fake.calls[0].metadata["authorization"] == "Bearer " + ACCESS_TOKEN + + span = traced.one() + values = attrs(span) + assert values["discoveryengine.error.status"] == "PERMISSION_DENIED" + assert values["discoveryengine.error.code"] == 403 + assert "discoveryengine.result_count" not in values + assert span.status.status_code is StatusCode.ERROR + assert span.status.description.startswith("PermissionDenied: 403 Permission denied") + exception = event(span, "exception") + assert exception["exception.type"] == "google.api_core.exceptions.PermissionDenied" + assert [item.name for item in span.events] == ["exception"] + for text in _error_texts(span): + assert ACCESS_TOKEN not in text + assert "[redacted]" in exception["exception.message"] + assert ACCESS_TOKEN not in traced.wire() + + +def test_an_api_key_from_client_options_is_never_exported(fake): + # search_lite is the method Google documents for API-key auth. + client = search_client_with_api_key(fake, API_KEY) + with instrumented() as traced: + client.search_lite(request=search_request("ok")) + with pytest.raises(core_exceptions.PermissionDenied): + client.search_lite(request=search_request(FAIL_DENIED)) + + assert [call.metadata.get("x-goog-api-key") for call in fake.calls] == [API_KEY, API_KEY] + assert traced.names() == ["discoveryengine.search_lite"] * 2 + assert API_KEY not in traced.wire() + assert "[redacted]" in event(traced.spans()[1], "exception")["exception.message"] + + +def test_api_key_credentials_on_the_transport_are_never_exported(fake): + client = search_client(fake, credentials=api_key_credentials()) + with instrumented() as traced: + with pytest.raises(core_exceptions.PermissionDenied): + client.search(request=search_request(FAIL_DENIED)) + + assert fake.calls[0].metadata["x-goog-api-key"] == API_KEY + assert API_KEY not in traced.wire() + + +@pytest.mark.parametrize( + "key, value", + [("authorization", "Bearer " + METADATA_TOKEN), ("x-goog-api-key", METADATA_TOKEN)], + ids=["authorization", "x-goog-api-key"], +) +def test_auth_metadata_passed_per_call_is_never_exported(fake, key, value): + metadata = [(key, value)] + with instrumented() as traced: + with pytest.raises(core_exceptions.PermissionDenied): + search_client(fake).search(request=search_request(FAIL_DENIED), metadata=metadata) + answer_client(fake).answer_query(request=answer_request(), metadata=metadata) + + assert fake.calls[0].metadata[key.lower()] == value + assert METADATA_TOKEN in str(fake.calls[0].metadata) + assert METADATA_TOKEN not in traced.wire() + # No request metadata is recorded as an attribute. + for span in traced.spans(): + for key in attrs(span): + assert "authorization" not in key and "metadata" not in key, key + + +def test_a_query_that_contains_the_token_is_redacted_when_recorded(fake): + client = search_client(fake, credentials=oauth_credentials()) + with instrumented() as traced: + client.search(request=search_request("find " + ACCESS_TOKEN)) + + values = attrs(traced.one()) + assert values["input.value"] == "find [redacted]" + assert values["gen_ai.retrieval.query"] == "find [redacted]" + + +def test_token_shaped_strings_the_client_never_held_are_redacted(fake): + with instrumented() as traced: + with pytest.raises(core_exceptions.InvalidArgument) as raised: + search_client(fake).search(request=search_request(FAIL_UNSEEN)) + + assert UNSEEN_ACCESS_TOKEN in str(raised.value) + span = traced.one() + for text in _error_texts(span): + assert text + for secret in ( + UNSEEN_ACCESS_TOKEN, + UNSEEN_API_KEY, + UNSEEN_REFRESH_TOKEN, + BEARER_VALUE, + "ya29.", + "AIza", + "1//", + ): + assert secret not in text, secret + assert "Bearer [redacted]" in text + + +@pytest.mark.parametrize( + "token", [None, "placeholder-stale-token-value"], ids=["first-use", "refresh"] +) +def test_a_token_minted_during_the_call_is_redacted(fake, token): + # The wrapper reads the credentials before the call; the token is set + # during it. The server quotes it without "Bearer" and in no Google shape. + client = search_client(fake, credentials=minting_credentials(token)) + with instrumented() as traced: + with pytest.raises(core_exceptions.Unauthenticated) as raised: + client.search(request=search_request(FAIL_TOKEN)) + + assert fake.calls[0].metadata["authorization"] == "Bearer " + MINTED_TOKEN + assert MINTED_TOKEN in str(raised.value) + span = traced.one() + for text in _error_texts(span): + assert text + assert MINTED_TOKEN not in text + assert "[redacted]" in event(span, "exception")["exception.message"] + assert MINTED_TOKEN not in traced.wire() + + +def test_a_self_signed_jwt_minted_for_a_service_account_is_redacted(fake): + # google-api-core gives the channel a scoped copy of service-account + # credentials, so the JWT google-auth mints during the call is never on + # the credentials the client holds: only its shape can remove it. + credentials = self_signed_jwt_credentials() + client = search_client(fake, credentials=credentials) + with instrumented() as traced: + with pytest.raises(core_exceptions.Unauthenticated) as raised: + client.search(request=search_request(FAIL_TOKEN)) + + scheme, jwt = fake.calls[0].metadata["authorization"].split(" ", 1) + assert scheme == "Bearer" + assert jwt.startswith("eyJ") and jwt.count(".") == 2 + assert client._transport._credentials.token is None + assert jwt in str(raised.value) + span = traced.one() + for text in _error_texts(span): + assert text + assert jwt not in text + assert "eyJ" not in text + assert "credentials: [redacted]" in event(span, "exception")["exception.message"] + assert jwt not in traced.wire() + + +def test_the_bearer_pass_runs_on_server_text_only(fake): + # A captured query is the user's words: "bearer " stays. Token + # shapes and held credentials are still removed from it. + query = "bearer responsibility " + UNSEEN_ACCESS_TOKEN + with instrumented() as traced: + search_client(fake).search(request=search_request(query)) + + assert attrs(traced.one())["input.value"] == "bearer responsibility [redacted]" + + +def test_refresh_token_and_client_secret_are_never_exported(): + from google.oauth2.credentials import Credentials + + refresh_token = "placeholder-refresh-token-value" + client_secret = "placeholder-client-secret-value" + + class Transport: + _credentials = Credentials( + token=None, refresh_token=refresh_token, client_id="client-id", client_secret=client_secret + ) + + class Client: + _transport = Transport() + _client_options = None + + error = core_exceptions.Unauthenticated( + "refresh failed for {0} with {1}".format(refresh_token, client_secret) + ) + span = _call_wrapper( + _wrappers.OperationWrapper, "search", Client(), error, search_request("q") + ) + + wire = span.to_json() + assert refresh_token not in wire and client_secret not in wire + assert event(span, "exception")["exception.message"] == "401 refresh failed for [redacted] with [redacted]" + + +def test_server_unavailable_records_its_code(fake): + with instrumented() as traced: + with pytest.raises(core_exceptions.ServiceUnavailable): + search_client(fake).search(request=search_request(FAIL_UNAVAILABLE)) + + values = attrs(traced.one()) + assert values["discoveryengine.error.status"] == "UNAVAILABLE" + assert values["discoveryengine.error.code"] == 503 + + +def test_retry_exhaustion_is_one_error_span_with_the_last_status(fake): + from google.api_core import retry as retries + + retry = retries.Retry( + predicate=retries.if_exception_type(core_exceptions.ServiceUnavailable), + initial=0.01, + maximum=0.02, + timeout=0.3, + ) + with instrumented() as traced: + with pytest.raises(core_exceptions.RetryError): + search_client(fake).search(request=search_request(FAIL_UNAVAILABLE), retry=retry) + + assert len(fake.calls) >= 2 + span = traced.one() + values = attrs(span) + # RetryError has no status of its own; the last attempt's is recorded. + assert values["discoveryengine.error.status"] == "UNAVAILABLE" + assert values["discoveryengine.error.code"] == 503 + assert span.status.description.startswith("RetryError: Timeout of 0.3s exceeded") + assert [item.name for item in span.events] == ["exception"] + + +def test_error_text_is_capped_after_redaction(fake): + client = search_client(fake, credentials=oauth_credentials()) + with instrumented() as traced: + with pytest.raises(core_exceptions.InvalidArgument): + client.search(request=search_request(FAIL_HUGE)) + + span = traced.one() + description, message = _error_texts(span) + prefix = "InvalidArgument: " + assert description.startswith(prefix) + assert len(description[len(prefix):].encode("utf-8")) <= _wrappers.MAX_VALUE_BYTES + assert _wrappers.MAX_VALUE_BYTES - 3 <= len(message.encode("utf-8")) <= _wrappers.MAX_VALUE_BYTES + for text in (description, message): + assert ACCESS_TOKEN not in text + # Whole characters only: the cut never leaves a broken euro sign. + text.encode("utf-8").decode("utf-8") + + +class _Transport: + def __init__(self, token): + self._credentials = type("Creds", (), {"token": token})() + + +class _Client: + def __init__(self, token): + self._transport = _Transport(token) + self._client_options = None + + +def _call_wrapper(wrapper_type, operation, instance, error, request, **options): + exporter, provider = new_provider() + wrapper = wrapper_type(provider.get_tracer("t"), operation, _wrappers.Options(**options)) + + def wrapped(*_args, **_kwargs): + raise error + + with pytest.raises(type(error)) as raised: + wrapper(wrapped, instance, (), {"request": request}) + assert raised.value is error + (span,) = exporter.get_finished_spans() + return span + + +def test_a_token_straddling_the_cap_is_redacted_before_the_cut(): + token = "straddling-token-value-0123456789" + filler = "x" * (_wrappers.MAX_VALUE_BYTES - 10) + error = core_exceptions.PermissionDenied(filler + token + " tail") + span = _call_wrapper(_wrappers.OperationWrapper, "search", _Client(token), error, search_request("q")) + + message = event(span, "exception")["exception.message"] + assert "straddl" not in message and token[:8] not in message + assert message.startswith("403 " + filler[:20]) + assert len(message.encode("utf-8")) <= _wrappers.MAX_VALUE_BYTES + + +ESCAPED_QUERY = "Zelda's \"quoted\" a\\b caf\xe9 2024" + + +@pytest.mark.parametrize( + "copy", + [ + # Python repr (single quotes, as for a text with both quote marks). + r"""Zelda\'s "quoted" a\\b caf""" + "\xe9 2024", + # JSON string, non-ASCII as \u. + r"""Zelda's \"quoted\" a\\b caf""" + "\\u00e9 2024", + # Protobuf text format: a status detail google-api-core appends. + r"""Zelda\'s \"quoted\" a\\b caf""" + "\xe9 2024", + # absl CHexEscape (debug_error_string of older gRPC cores): bytes + # outside printable ASCII as \xHH. + r"""Zelda\'s \"quoted\" a\\b caf\xc3\xa9 2024""", + ], + ids=["repr", "json", "protobuf-text", "grpc-chexescape"], +) +def test_an_escaped_copy_of_a_hidden_query_is_removed(copy): + cause = RuntimeError('debug_error_string = "{grpc_message:\\"' + copy + '\\"}"') + error = core_exceptions.InvalidArgument("bad query '{0}'".format(copy)) + error.__cause__ = cause + span = _call_wrapper( + _wrappers.OperationWrapper, + "search", + _Client(None), + error, + search_request(ESCAPED_QUERY), + hide_inputs=True, + ) + + for text in _error_texts(span): + assert "Zelda" not in text and "quoted" not in text, text + assert "__REDACTED__" in text + # The chained gRPC cause (debug_error_string) is never exported: no stack trace. + assert "debug_error_string" not in span.to_json() + + +def _protobuf_text(text): + from google.rpc import error_details_pb2 + + violation = error_details_pb2.BadRequest.FieldViolation(description=text) + written = str(violation) + prefix = 'description: "' + assert written.startswith(prefix) and written.rstrip("\n").endswith('"'), written + return written.rstrip("\n")[len(prefix) : -1] + + +@pytest.mark.parametrize( + "text", + [ + "".join(chr(code) for code in range(0x00, 0x20)) + "\x7f", + "Zelda's \"quoted\" a\\b caf\xe9 2024", + "tab\tnew\nline\rret vt\x0b esc\x1b del\x7f \u00e9\u4e2d\U0001f600", + ], + ids=["controls-and-del", "quotes-backslash-nonascii", "mixed"], +) +def test_the_protobuf_text_copy_matches_what_protobuf_writes(text): + # Pin _c_escaped to the installed protobuf's own text format (the copy a + # status detail appends to the error), not to a hand-written table. + assert _wrappers._c_escaped(text) == _protobuf_text(text) + + +def test_a_hex_digit_after_a_hex_escape_is_matched_as_grpc_escapes_it(): + # absl escapes a hex digit that follows \xHH too: U+00E9 then 1 is \xc3\xa9\x31. + error = core_exceptions.InvalidArgument(r"bad query 'caf\xc3\xa9\x31\x32 x'") + span = _call_wrapper( + _wrappers.OperationWrapper, + "search", + _Client(None), + error, + search_request("caf\xe912 x"), + hide_inputs=True, + ) + + assert event(span, "exception")["exception.message"] == "400 bad query '__REDACTED__'" + + +def test_an_exception_whose_str_raises_is_recorded_without_breaking_the_call(): + class Unprintable(Exception): + def __str__(self): + raise ValueError("no str for you") + + span = _call_wrapper(_wrappers.OperationWrapper, "search", _Client(None), Unprintable(), search_request("q")) + assert span.status.status_code is StatusCode.ERROR + assert span.status.description.startswith("Unprintable") + + +def test_credentials_that_cannot_be_read_fail_closed(): + # If any place the client keeps a credential cannot be read, no free + # text (query, error message, stack trace) is recorded for the call. + class Unreadable: + @property + def _transport(self): + raise RuntimeError("credential lookup failed") + + error = core_exceptions.PermissionDenied("denied for secret-query and secret-credential") + span = _call_wrapper( + _wrappers.OperationWrapper, + "search", + Unreadable(), + error, + search_request("secret-query"), + ) + values = attrs(span) + assert "input.value" not in values + assert "gen_ai.retrieval.query" not in values + assert values["discoveryengine.error.status"] == "PERMISSION_DENIED" + wire = span.to_json() + assert "secret-query" not in wire and "secret-credential" not in wire + assert span.status.description == "PermissionDenied: " + _wrappers.UNREADABLE + exception = event(span, "exception") + assert exception["exception.type"] == "google.api_core.exceptions.PermissionDenied" + assert exception["exception.message"] == _wrappers.UNREADABLE + assert "exception.stacktrace" not in exception + + +def test_credentials_that_cannot_be_read_again_after_the_call_fail_closed(): + # They are read again when an error is recorded; if that read fails, + # no free text is recorded, as when the first read fails. + class Transport: + def __init__(self): + self.reads = 0 + + @property + def _credentials(self): + self.reads += 1 + if self.reads > 1: + raise RuntimeError("credential lookup failed") + return None + + client = _Client(None) + client._transport = Transport() + error = core_exceptions.PermissionDenied("denied for secret-credential") + span = _call_wrapper(_wrappers.OperationWrapper, "search", client, error, search_request("q")) + + assert client._transport.reads == 2 + assert span.status.description == "PermissionDenied: " + _wrappers.UNREADABLE + exception = event(span, "exception") + assert exception["exception.message"] == _wrappers.UNREADABLE + assert "exception.stacktrace" not in exception + assert "secret-credential" not in span.to_json() + + +def test_async_errors_are_redacted_the_same_way(fake): + async def call(): + client = async_search_client(fake, credentials=oauth_credentials()) + try: + await client.search(request=search_request(FAIL_DENIED)) + finally: + await client.transport.close() + + with instrumented() as traced: + with pytest.raises(core_exceptions.PermissionDenied) as raised: + asyncio.run(call()) + + assert ACCESS_TOKEN in str(raised.value) + span = traced.one() + assert attrs(span)["discoveryengine.error.code"] == 403 + assert ACCESS_TOKEN not in traced.wire() + + +def test_no_stack_trace_or_local_path_is_recorded(fake): + # traceback frames carry absolute file paths (home directory, user name, + # virtualenv). The exception event has the type and the cleaned message + # only, so none of that leaves the process. + import getpass + import os + import socket + from pathlib import Path + + with instrumented() as traced: + with pytest.raises(core_exceptions.PermissionDenied): + search_client(fake).search(request=search_request(FAIL_DENIED)) + with pytest.raises(core_exceptions.RetryError): + from google.api_core import retry as retries + + search_client(fake).search( + request=search_request(FAIL_UNAVAILABLE), + retry=retries.Retry( + predicate=retries.if_exception_type(core_exceptions.ServiceUnavailable), + initial=0.01, + maximum=0.02, + timeout=0.1, + ), + ) + + for span in traced.spans(): + assert sorted(event(span, "exception")) == ["exception.message", "exception.type"] + wire = traced.wire() + for local in ( + "exception.stacktrace", + "Traceback", + 'File "', + "site-packages", + str(Path.home()), + os.getcwd(), + getpass.getuser(), + socket.gethostname(), + ): + # Very short user or host names (CI runners) could match by chance. + if len(local) >= 4: + assert local not in wire, local diff --git a/python/frameworks/discoveryengine/tests/test_discoveryengine_example.py b/python/frameworks/discoveryengine/tests/test_discoveryengine_example.py new file mode 100644 index 000000000..0936aa26b --- /dev/null +++ b/python/frameworks/discoveryengine/tests/test_discoveryengine_example.py @@ -0,0 +1,106 @@ +"""examples/search_and_answer.py runs end to end into harness.Receiver. + +The example's ``main()`` takes optional clients so the test can hand it real +clients on the loopback fake's channel; everything else (register(), +instrument(), the calls and the flush) is the example's own code. Without +clients it builds them with Application Default Credentials, which this test +never does. The architecture asks for no child process here. +""" + +from __future__ import annotations + +import importlib.util +import json +import sys +from pathlib import Path + +import pytest + +pytest.importorskip("opentelemetry.proto", reason="the harness decodes OTLP protobuf") + +sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "tests")) + +from harness import Receiver, _flatten_attributes # noqa: E402 + +from _discoveryengine_support import ( # noqa: E402 + CONTENT_MARKERS, + SEARCH_RESULTS, + SERVING_CONFIG, + FakeDiscoveryEngine, + answer_client, + search_client, +) + +EXAMPLE = Path(__file__).resolve().parents[1] / "examples" / "search_and_answer.py" +FI_API_KEY = "placeholder-fi-api-key" +FI_SECRET_KEY = "placeholder-fi-secret-key" + + +def _load_example(): + spec = importlib.util.spec_from_file_location("discoveryengine_example", EXAMPLE) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_example_exports_one_search_and_one_answer_span(monkeypatch, capsys): + from traceai_discoveryengine import DiscoveryEngineInstrumentor + + example = _load_example() + with FakeDiscoveryEngine() as fake, Receiver() as receiver: + monkeypatch.setenv("FI_BASE_URL", receiver.origin) + monkeypatch.setenv("FI_API_KEY", FI_API_KEY) + monkeypatch.setenv("FI_SECRET_KEY", FI_SECRET_KEY) + monkeypatch.setenv("DISCOVERY_ENGINE_SERVING_CONFIG", SERVING_CONFIG) + try: + example.main(search_client=search_client(fake), answer_client=answer_client(fake)) + finally: + # instrument() is a process-wide singleton; leave nothing wrapped. + DiscoveryEngineInstrumentor().uninstrument() + spans = receiver.spans() + exports = receiver.requests() + + assert fake.methods() == ["Search", "AnswerQuery"] + assert capsys.readouterr().out.splitlines() == [ + "{0} results on the first page".format(SEARCH_RESULTS), + "answer state: SUCCEEDED", + ] + assert [span["name"] for span in spans] == [ + "discoveryengine.search", + "discoveryengine.answer_query", + ] + search, answer = (_flatten_attributes(span["attributes"]) for span in spans) + assert search["gen_ai.span.kind"] == answer["gen_ai.span.kind"] == "RETRIEVER" + assert search["discoveryengine.serving_config"] == SERVING_CONFIG + assert int(search["discoveryengine.result_count"]) == SEARCH_RESULTS + assert answer["discoveryengine.answer.state"] == "SUCCEEDED" + assert all(span["status"]["code"] == "STATUS_CODE_OK" for span in spans) + + assert exports + for export in exports: + assert export["path"] == "/tracer/v1/traces" + for resource in export["resource_attributes"]: + assert resource["project_name"] == "discoveryengine-example" + assert resource["project_type"] == "observe" + + # TraceConfig default: the query is the span's input; results and the + # answer text are never recorded. + assert search["input.value"] == example.QUERY + assert answer["input.value"] == example.QUESTION + wire = json.dumps(spans) + for marker in CONTENT_MARKERS: + assert marker not in wire, marker + + +@pytest.mark.parametrize( + "serving_config, endpoint", + [ + (SERVING_CONFIG, None), + (SERVING_CONFIG.replace("/locations/global/", "/locations/eu/"), "eu-discoveryengine.googleapis.com"), + (SERVING_CONFIG.replace("/locations/global/", "/locations/us/"), "us-discoveryengine.googleapis.com"), + ], + ids=["global", "eu", "us"], +) +def test_example_picks_the_regional_endpoint_from_the_serving_config(serving_config, endpoint): + options = _load_example().client_options(serving_config) + assert options.get("api_endpoint") == endpoint diff --git a/python/frameworks/discoveryengine/tests/test_discoveryengine_isolation.py b/python/frameworks/discoveryengine/tests/test_discoveryengine_isolation.py new file mode 100644 index 000000000..2f5ebaf90 --- /dev/null +++ b/python/frameworks/discoveryengine/tests/test_discoveryengine_isolation.py @@ -0,0 +1,231 @@ +"""Instrumentation failures never reach the caller; suppression and nesting guards hold.""" + +from __future__ import annotations + +import asyncio +import contextlib + +import pytest +from google.api_core import exceptions as core_exceptions +from opentelemetry.trace import StatusCode + +from _discoveryengine_support import ( + ANSWER_REFERENCES, + FAIL_DENIED, + METADATA_TOKEN, + SEARCH_RESULTS, + FakeDiscoveryEngine, + answer_client, + answer_request, + async_answer_client, + async_search_client, + attrs, + event, + instrumented, + search_client, + search_request, +) +from traceai_discoveryengine import DiscoveryEngineInstrumentor, _wrappers + + +@pytest.fixture() +def fake(): + with FakeDiscoveryEngine() as server: + yield server + + +def _boom(*_args, **_kwargs): + raise RuntimeError("instrumentation bug") + + +def test_a_failing_request_reader_still_returns_the_result(fake, monkeypatch): + monkeypatch.setattr(_wrappers, "_request_attributes", _boom) + with instrumented() as traced: + pager = search_client(fake).search(request=search_request("q")) + + assert len(pager.results) == SEARCH_RESULTS + span = traced.one() + assert attrs(span)["gen_ai.span.kind"] == "RETRIEVER" + assert span.status.status_code is StatusCode.OK + + +def test_a_failing_response_reader_still_returns_the_result_and_ends_the_span(fake, monkeypatch): + monkeypatch.setattr(_wrappers, "_response_attributes", _boom) + + async def call(): + client = async_answer_client(fake) + try: + return await client.answer_query(request=answer_request()) + finally: + await client.transport.close() + + with instrumented() as traced: + pager = search_client(fake).search(request=search_request()) + response = asyncio.run(call()) + + assert len(pager.results) == SEARCH_RESULTS + assert len(response.answer.references) == ANSWER_REFERENCES + spans = traced.spans() + assert [span.name for span in spans] == ["discoveryengine.search", "discoveryengine.answer_query"] + assert all(span.end_time is not None for span in spans) + assert all(span.status.status_code is StatusCode.OK for span in spans) + + +def test_a_failing_error_recorder_still_raises_the_vendor_error(fake, monkeypatch): + monkeypatch.setattr(_wrappers, "_exception_attributes", _boom) + monkeypatch.setattr(_wrappers, "_error_attributes", _boom) + with instrumented() as traced: + with pytest.raises(core_exceptions.PermissionDenied): + search_client(fake).search(request=search_request(FAIL_DENIED)) + + span = traced.one() + assert span.end_time is not None + assert span.status.status_code is StatusCode.ERROR + assert span.events == () + + +def test_a_query_that_cannot_be_read_keeps_no_server_text(fake, monkeypatch): + # With hide_inputs, if the query cannot be read, the server's error + # text (which may echo it) is replaced as a whole. + from fi_instrumentation import REDACTED_VALUE, TraceConfig + + monkeypatch.setattr(_wrappers, "_hidden_inputs", _boom) + with instrumented(config=TraceConfig(hide_inputs=True)) as traced: + with pytest.raises(core_exceptions.PermissionDenied): + search_client(fake).search(request=search_request(FAIL_DENIED)) + + span = traced.one() + assert span.status.description == "PermissionDenied: " + REDACTED_VALUE + exception = [item for item in span.events if item.name == "exception"][0].attributes + assert exception["exception.message"] == REDACTED_VALUE + assert "exception.stacktrace" not in exception + assert attrs(span)["discoveryengine.error.status"] == "PERMISSION_DENIED" + + +def test_a_failing_credential_lookup_still_runs_the_call(fake, monkeypatch): + monkeypatch.setattr(_wrappers, "_credential_values", _boom) + with instrumented() as traced: + pager = search_client(fake).search(request=search_request("q")) + + assert len(pager.results) == SEARCH_RESULTS + # Fail closed: the query is not recorded when credentials could not be read. + values = attrs(traced.one()) + assert "input.value" not in values + assert values["discoveryengine.result_count"] == SEARCH_RESULTS + + +def _metadata_pairs(): + # A one-shot iterable: a generator can be read once only. + yield ("x-caller-header", "caller-value") + yield ("x-goog-api-key", METADATA_TOKEN) + + +@pytest.mark.parametrize("traced", [False, True], ids=["uninstrumented", "instrumented"]) +def test_metadata_from_a_generator_reaches_the_server(fake, traced): + async def call(): + client = async_search_client(fake) + try: + await client.search_lite(request=search_request(), metadata=_metadata_pairs()) + finally: + await client.transport.close() + + with instrumented() if traced else contextlib.nullcontext() as tracing: + with pytest.raises(core_exceptions.PermissionDenied): + search_client(fake).search(request=search_request(FAIL_DENIED), metadata=_metadata_pairs()) + answer_client(fake).answer_query(request=answer_request(), metadata=_metadata_pairs()) + asyncio.run(call()) + + assert fake.methods() == ["Search", "AnswerQuery", "SearchLite"] + for received in fake.calls: + assert received.metadata["x-caller-header"] == "caller-value" + assert received.metadata["x-goog-api-key"] == METADATA_TOKEN + if traced: + assert len(tracing.spans()) == 3 + # The pairs read once also feed the credential lookup. + assert METADATA_TOKEN not in tracing.wire() + + +def _failing_metadata_pairs(): + yield ("x-caller-header", "caller-value") + raise RuntimeError("caller metadata failed") + + +@pytest.mark.parametrize("traced", [False, True], ids=["uninstrumented", "instrumented"]) +def test_metadata_that_fails_while_read_raises_as_without_instrumentation(fake, traced): + with instrumented() if traced else contextlib.nullcontext() as tracing: + with pytest.raises(RuntimeError, match="caller metadata failed"): + search_client(fake).search(request=search_request(), metadata=_failing_metadata_pairs()) + + # The client raised before sending anything, as it does uninstrumented. + assert fake.calls == [] + if traced: + span = tracing.one() + # Nothing from metadata that could not be read: no free text at all. + assert span.status.description == "RuntimeError: " + _wrappers.UNREADABLE + assert event(span, "exception")["exception.message"] == _wrappers.UNREADABLE + + +class _BrokenTracer: + def start_span(self, *_args, **_kwargs): + raise RuntimeError("tracer bug") + + +class _BrokenProvider: + def get_tracer(self, *_args, **_kwargs): + return _BrokenTracer() + + +def test_a_tracer_that_cannot_start_spans_leaves_calls_untraced(fake): + instrumentor = DiscoveryEngineInstrumentor() + instrumentor.instrument(tracer_provider=_BrokenProvider()) + try: + pager = search_client(fake).search(request=search_request()) + + async def call(): + client = async_answer_client(fake) + try: + return await client.answer_query(request=answer_request()) + finally: + await client.transport.close() + + response = asyncio.run(call()) + finally: + instrumentor.uninstrument() + + assert len(pager.results) == SEARCH_RESULTS + assert len(response.answer.references) == ANSWER_REFERENCES + + +def test_suppress_tracing_skips_the_span(fake): + from fi_instrumentation import suppress_tracing + + with instrumented() as traced: + with suppress_tracing(): + search_client(fake).search(request=search_request()) + answer_client(fake).answer_query(request=answer_request()) + + assert fake.methods() == ["Search", "AnswerQuery"] + assert traced.names() == ["discoveryengine.answer_query"] + + +def test_a_wrapped_call_made_inside_a_traced_call_is_not_a_second_span(fake, monkeypatch): + # No 0.20.5 method calls another wrapped one, but a later release could + # (an async client delegating to its sync client). The guard keeps one + # span per user call. + from google.cloud.discoveryengine_v1 import SearchServiceClient + + with instrumented() as traced: + client = search_client(fake) + inner = client.search_lite + original = client._transport._wrapped_methods[client._transport.search] + + def nested(request, **kwargs): + inner(request=search_request()) + return original(request, **kwargs) + + monkeypatch.setitem(client._transport._wrapped_methods, client._transport.search, nested) + client.search(request=search_request()) + assert isinstance(client, SearchServiceClient) + + assert fake.methods() == ["SearchLite", "Search"] + assert traced.names() == ["discoveryengine.search"] diff --git a/python/frameworks/discoveryengine/tests/test_discoveryengine_packaging.py b/python/frameworks/discoveryengine/tests/test_discoveryengine_packaging.py new file mode 100644 index 000000000..a6736c0e9 --- /dev/null +++ b/python/frameworks/discoveryengine/tests/test_discoveryengine_packaging.py @@ -0,0 +1,134 @@ +"""Packaging contract: dependency ranges, the runtime version check, Pythons, README.""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest +from packaging.requirements import Requirement +from packaging.specifiers import SpecifierSet + +from traceai_discoveryengine import DiscoveryEngineInstrumentor, _wrappers +from traceai_discoveryengine.package import _instruments +from traceai_discoveryengine.version import __version__ + +PACKAGE = Path(__file__).resolve().parents[1] +PYPROJECT = PACKAGE / "pyproject.toml" +README = PACKAGE / "README.md" +CHANGELOG = PACKAGE / "CHANGELOG.md" + + +def _dependency(name: str) -> SpecifierSet: + match = re.search( + r'^{0}\s*=\s*"([^"]+)"'.format(re.escape(name)), PYPROJECT.read_text(), re.M + ) + assert match, name + return SpecifierSet(match.group(1)) + + +def test_discoveryengine_is_a_range_from_the_measured_pin_not_an_exact_pin(): + (requirement,) = DiscoveryEngineInstrumentor().instrumentation_dependencies() + instruments = Requirement(requirement) + assert instruments.name == "google-cloud-discoveryengine" + # pyproject and the runtime dependency check agree. + assert instruments.specifier == _dependency("google-cloud-discoveryengine") + for version in ("0.20.5", "0.20.9", "0.21.0", "0.99.0"): + assert instruments.specifier.contains(version), version + for version in ("0.19.0", "0.20.4", "1.0.0"): + assert not instruments.specifier.contains(version), version + assert tuple(_instruments) == tuple(DiscoveryEngineInstrumentor().instrumentation_dependencies()) + + +def _fake_installed(monkeypatch, installed_version): + dependencies = pytest.importorskip("opentelemetry.instrumentation.dependencies") + real = dependencies.version + + def version(name): + if name.lower().replace("_", "-") == "google-cloud-discoveryengine": + return installed_version + return real(name) + + monkeypatch.setattr(dependencies, "version", version) + + +@pytest.mark.parametrize( + "installed, wrapped", + [("0.20.5", True), ("0.21.3", True), ("0.20.4", False), ("1.0.0", False)], +) +def test_instrument_honours_the_range_at_runtime(monkeypatch, installed, wrapped): + from google.cloud.discoveryengine_v1 import SearchServiceClient + + from _discoveryengine_support import new_provider + + _fake_installed(monkeypatch, installed) + before = vars(SearchServiceClient)["search"] + _, provider = new_provider() + instrumentor = DiscoveryEngineInstrumentor() + instrumentor.instrument(tracer_provider=provider) + try: + assert (vars(SearchServiceClient)["search"] is not before) is wrapped + finally: + instrumentor.uninstrument() + assert vars(SearchServiceClient)["search"] is before + + +def test_fi_instrumentation_floor_is_1_1_0(): + # 0.1.11/0.1.12 export to the old create_otel_span path and the published + # 1.0.0 fails while building its headers; 1.1.0 is the first good release. + spec = _dependency("fi-instrumentation-otel") + assert spec.contains("1.1.0") + assert spec.contains("1.2.0") + for version in ("0.1.11", "0.1.12", "0.1.16", "1.0.0"): + assert not spec.contains(version), version + + +def test_classifiers_and_python_range_match_the_tested_versions(): + # google-cloud-discoveryengine 0.20.5 requires Python >=3.10 and + # fi-instrumentation-otel 1.1.0 stops before 3.14. The suite runs on + # every minor in between. + text = PYPROJECT.read_text() + versions = re.findall(r'"Programming Language :: Python :: (3\.\d+)"', text) + assert versions == ["3.10", "3.11", "3.12", "3.13"] + python = _dependency("python") + for version in ("3.10.0", "3.11.9", "3.12.10", "3.13.1"): + assert python.contains(version), version + for version in ("3.9.18", "3.14.0"): + assert not python.contains(version), version + readme = README.read_text() + assert "Python 3.10, 3.11, 3.12 and 3.13" in readme + assert "`google-cloud-discoveryengine` 0.20.5" in readme + + +def test_versions_agree(): + assert re.search(r'^version = "{0}"$'.format(re.escape(__version__)), PYPROJECT.read_text(), re.M) + assert CHANGELOG.read_text().splitlines()[0] == "# Changelog" + assert "## {0}".format(__version__) in CHANGELOG.read_text() + + +def _attribute_constants(): + return { + value + for name, value in vars(_wrappers).items() + if name.isupper() and isinstance(value, str) and value.startswith("discoveryengine.") + } + + +def test_the_readme_documents_every_attribute_the_code_can_write(): + span_names = {"discoveryengine." + name for name in ("search", "search_lite", "answer_query")} + documented = set( + re.findall(r"`(discoveryengine\.(?!googleapis)[a-z_.]+)`", README.read_text()) + ) - span_names + assert _attribute_constants() <= documented + # And nothing in the README is an attribute the code never writes. + assert documented - _attribute_constants() == set() + + +def test_the_readme_states_the_caps_the_code_applies(): + readme = README.read_text() + assert _wrappers.MAX_VALUE_BYTES == 1024 + assert "1 KB" in readme + # No stack trace is recorded, so there is no stack trace cap. + assert not hasattr(_wrappers, "MAX_STACKTRACE_BYTES") + assert "16 KB" not in readme + assert "`exception.stacktrace` is never written" in readme diff --git a/python/frameworks/discoveryengine/tests/test_discoveryengine_rest.py b/python/frameworks/discoveryengine/tests/test_discoveryengine_rest.py new file mode 100644 index 000000000..dff2de938 --- /dev/null +++ b/python/frameworks/discoveryengine/tests/test_discoveryengine_rest.py @@ -0,0 +1,322 @@ +"""The REST transport (``transport="rest"``): same spans as gRPC, and REST error statuses. + +A loopback HTTP server answers the documented v1 REST routes +(``POST /v1/{serving_config}:search``, ``:searchLite`` and ``:answer``) with +JSON written by the library's own response types, and a Google JSON error +(``{"error": {"code", "message", "status"}}``) for failures. The real +clients build the requests, send them over HTTP and parse the responses. +""" + +from __future__ import annotations + +import json +import re +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any, Dict, List + +import pytest +from google.api_core import exceptions as core_exceptions +from google.auth.credentials import AnonymousCredentials +from opentelemetry.trace import StatusCode + +from _discoveryengine_support import ( + ANSWER_REFERENCES, + ANSWER_TEXT, + CONTENT_MARKERS, + DOCUMENT_TITLE, + SERVING_CONFIG, + SESSION_NAME, + SNIPPET, + attrs, + event, + instrumented, +) + +REDACTED_VALUE = "__REDACTED__" +QUERY = "REST-QUERY-MARKER" +API_KEY = "placeholder-rest-api-key-must-not-be-exported" +ROUTE = re.compile(r"^/v1/(projects/.+/servingConfigs/[^/:]+):(search|searchLite|answer)$") + +# Error bodies that discoveryengine.googleapis.com returned on 2026-10-18 (v1, +# a global Standard-tier generic search engine, OAuth credentials), verbatim. +LIVE_ERRORS = { + "live-answer-standard-tier": ( + 400, + { + "error": { + "code": 400, + "message": "This feature is only available when Large Language Model add-on is enabled.", + "status": "FAILED_PRECONDITION", + } + }, + ), + "live-search-lite-not-a-website": ( + 400, + { + "error": { + "code": 400, + "message": "API key is only supported for public website search.", + "status": "FAILED_PRECONDITION", + } + }, + ), + "live-missing-engine": ( + 404, + { + "error": { + "code": 404, + "message": "We were unable to find an engine to service the request. This may be an " + "intermittent issue -- please try again in 3-5 minutes. If the issue persists, please " + "contact support.", + "status": "NOT_FOUND", + } + }, + ), +} + + +def _search_body() -> str: + from google.cloud.discoveryengine_v1.types import SearchResponse + + results = [ + { + "id": "doc-{0}".format(index), + "document": { + "id": "doc-{0}".format(index), + "derived_struct_data": {"title": DOCUMENT_TITLE, "snippets": [{"snippet": SNIPPET}]}, + }, + } + for index in range(3) + ] + return SearchResponse.to_json(SearchResponse(results=results, total_size=3)) + + +def _answer_body() -> str: + from google.cloud.discoveryengine_v1.types import Answer, AnswerQueryResponse, Session + + response = AnswerQueryResponse( + answer=Answer( + state=Answer.State.SUCCEEDED, + answer_text=ANSWER_TEXT, + references=[{"chunk_info": {"content": "chunk"}}] * ANSWER_REFERENCES, + ), + session=Session(name=SESSION_NAME), + ) + return AnswerQueryResponse.to_json(response) + + +class _Handler(BaseHTTPRequestHandler): + calls: List[Dict[str, Any]] = [] + + def log_message(self, *args: Any) -> None: + pass + + def _send(self, status: int, body: str) -> None: + data = body.encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", "application/json; charset=UTF-8") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def do_POST(self) -> None: + body = json.loads(self.rfile.read(int(self.headers.get("Content-Length") or 0)) or b"{}") + type(self).calls.append({"path": self.path, "headers": dict(self.headers), "body": body}) + match = ROUTE.match(self.path.split("?", 1)[0]) + if not match: + self._send(404, json.dumps({"error": {"code": 404, "message": "no route", "status": "NOT_FOUND"}})) + return + query = body.get("query") + text = query.get("text", "") if isinstance(query, dict) else (query or "") + live = next((key for key in LIVE_ERRORS if text.startswith(key)), None) + if live is not None: + status, error_body = LIVE_ERRORS[live] + self._send(status, json.dumps(error_body)) + elif text.startswith("fail-denied"): + self._send( + 403, + json.dumps( + { + "error": { + "code": 403, + "message": "Permission denied for query '{0}' with key {1}".format( + text, self.headers.get("x-goog-api-key") + ), + "status": "PERMISSION_DENIED", + } + } + ), + ) + elif text.startswith("fail-odd-status"): + self._send( + 400, json.dumps({"error": {"code": 400, "message": "bad", "status": "free text " + text}}) + ) + elif match.group(2) == "answer": + self._send(200, _answer_body()) + else: + self._send(200, _search_body()) + + +@pytest.fixture() +def endpoint(): + _Handler.calls = [] + server = ThreadingHTTPServer(("127.0.0.1", 0), _Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield "http://127.0.0.1:{0}".format(server.server_address[1]) + finally: + server.shutdown() + server.server_close() + + +def _clients(endpoint: str, **options: Any): + from google.cloud.discoveryengine_v1 import ConversationalSearchServiceClient, SearchServiceClient + + client_options = dict({"api_endpoint": endpoint}, **options) + credentials = None if "api_key" in options else AnonymousCredentials() + return ( + SearchServiceClient(credentials=credentials, transport="rest", client_options=client_options), + ConversationalSearchServiceClient( + credentials=credentials, transport="rest", client_options=client_options + ), + ) + + +def test_rest_calls_get_the_same_spans_as_grpc(endpoint): + search, answers = _clients(endpoint) + with instrumented() as traced: + pager = search.search(request={"serving_config": SERVING_CONFIG, "query": QUERY}) + assert len(pager.results) == 3 + search.search_lite(request={"serving_config": SERVING_CONFIG, "query": QUERY}) + response = answers.answer_query( + request={"serving_config": SERVING_CONFIG, "query": {"text": QUERY}} + ) + assert response.answer.answer_text == ANSWER_TEXT + + assert [call["path"].split("?")[0].rsplit(":", 1)[1] for call in _Handler.calls] == [ + "search", + "searchLite", + "answer", + ] + search_span, lite_span, answer_span = traced.spans() + for span, name in ((search_span, "search"), (lite_span, "search_lite")): + assert span.name == "discoveryengine." + name + assert attrs(span) == { + "gen_ai.span.kind": "RETRIEVER", + "discoveryengine.serving_config": SERVING_CONFIG, + "discoveryengine.result_count": 3, + "gen_ai.retrieval.query": QUERY, + "input.value": QUERY, + } + assert span.status.status_code is StatusCode.OK + assert attrs(answer_span) == { + "gen_ai.span.kind": "RETRIEVER", + "discoveryengine.serving_config": SERVING_CONFIG, + "discoveryengine.result_count": ANSWER_REFERENCES, + "discoveryengine.answer.length": len(ANSWER_TEXT), + "discoveryengine.answer.state": "SUCCEEDED", + "discoveryengine.session": SESSION_NAME, + "gen_ai.retrieval.query": QUERY, + "input.value": QUERY, + } + wire = traced.wire() + for marker in CONTENT_MARKERS: + assert marker not in wire, marker + + +def test_a_rest_error_records_the_status_from_the_json_body(endpoint): + # google-api-core maps an HTTP 403 to Forbidden, which has no gRPC + # status; the server's "status" field is recorded instead. + search, _ = _clients(endpoint) + with instrumented() as traced: + with pytest.raises(core_exceptions.Forbidden) as raised: + search.search(request={"serving_config": SERVING_CONFIG, "query": "fail-denied " + QUERY}) + + assert raised.value.grpc_status_code is None + span = traced.one() + values = attrs(span) + assert values["discoveryengine.error.status"] == "PERMISSION_DENIED" + assert values["discoveryengine.error.code"] == 403 + assert span.status.status_code is StatusCode.ERROR + assert span.status.description.startswith("Forbidden: 403 POST ") + assert sorted(event(span, "exception")) == ["exception.message", "exception.type"] + # Default: the query is recorded, so the server's echo of it stays. + assert QUERY in span.status.description + + +def test_with_hide_inputs_the_query_is_removed_from_rest_error_text(endpoint, monkeypatch): + monkeypatch.setenv("FI_HIDE_INPUTS", "true") + search, _ = _clients(endpoint) + with instrumented() as traced: + with pytest.raises(core_exceptions.Forbidden): + search.search(request={"serving_config": SERVING_CONFIG, "query": "fail-denied " + QUERY}) + + span = traced.one() + assert attrs(span)["input.value"] == REDACTED_VALUE + assert "gen_ai.retrieval.query" not in attrs(span) + assert REDACTED_VALUE in span.status.description + assert QUERY not in traced.wire() + + +def test_a_rest_api_key_is_never_exported(endpoint): + # search_lite with client_options.api_key, the set-up Google documents + # for public website search; the key travels in the x-goog-api-key header. + search, _ = _clients(endpoint, api_key=API_KEY) + with instrumented() as traced: + search.search_lite(request={"serving_config": SERVING_CONFIG, "query": QUERY}) + with pytest.raises(core_exceptions.Forbidden) as raised: + search.search_lite(request={"serving_config": SERVING_CONFIG, "query": "fail-denied " + QUERY}) + + assert API_KEY in str(raised.value) + assert all(call["headers"].get("x-goog-api-key") == API_KEY for call in _Handler.calls) + assert all(API_KEY not in call["path"] for call in _Handler.calls) + assert API_KEY not in traced.wire() + assert "[redacted]" in traced.spans()[1].status.description + + +def test_only_a_canonical_rest_status_name_is_recorded(endpoint): + search, _ = _clients(endpoint) + with instrumented() as traced: + with pytest.raises(core_exceptions.BadRequest): + search.search(request={"serving_config": SERVING_CONFIG, "query": "fail-odd-status " + QUERY}) + + values = attrs(traced.one()) + assert "discoveryengine.error.status" not in values + assert values["discoveryengine.error.code"] == 400 + + +@pytest.mark.parametrize( + "key, operation, error_class, status, code", + [ + ("live-answer-standard-tier", "answer_query", core_exceptions.BadRequest, "FAILED_PRECONDITION", 400), + ("live-search-lite-not-a-website", "search_lite", core_exceptions.BadRequest, "FAILED_PRECONDITION", 400), + ("live-missing-engine", "search", core_exceptions.NotFound, "NOT_FOUND", 404), + ], +) +def test_live_rest_error_bodies_record_status_and_code(endpoint, key, operation, error_class, status, code): + # The three error bodies the live API returned on 2026-10-18. On REST a 400 + # is a plain BadRequest (no gRPC status), so FAILED_PRECONDITION must come + # from the body; a 404 is NotFound, whose class carries NOT_FOUND. + search, answers = _clients(endpoint) + query = key + " " + QUERY + with instrumented() as traced: + with pytest.raises(core_exceptions.GoogleAPICallError) as raised: + if operation == "answer_query": + answers.answer_query(request={"serving_config": SERVING_CONFIG, "query": {"text": query}}) + else: + getattr(search, operation)(request={"serving_config": SERVING_CONFIG, "query": query}) + + assert type(raised.value) is error_class + span = traced.one() + assert span.name == "discoveryengine." + operation + values = attrs(span) + assert values["discoveryengine.error.status"] == status + assert values["discoveryengine.error.code"] == code + assert values["gen_ai.retrieval.query"] == query + assert "discoveryengine.result_count" not in values + assert span.status.status_code is StatusCode.ERROR + assert span.status.description.startswith("{0}: {1} POST ".format(error_class.__name__, code)) + assert LIVE_ERRORS[key][1]["error"]["message"] in span.status.description + assert sorted(event(span, "exception")) == ["exception.message", "exception.type"] diff --git a/python/frameworks/discoveryengine/tests/test_discoveryengine_scope.py b/python/frameworks/discoveryengine/tests/test_discoveryengine_scope.py new file mode 100644 index 000000000..1046175eb --- /dev/null +++ b/python/frameworks/discoveryengine/tests/test_discoveryengine_scope.py @@ -0,0 +1,141 @@ +"""Scope: only v1 search, search_lite and answer_query (sync and async) are traced. + +AC-05: ``create_conversation`` and the other admin RPCs emit no span. +AC-09: instrumenting Discovery Engine does not import or instrument +``google.genai`` or ``traceai_google_genai``. +""" + +from __future__ import annotations + +import os +import subprocess +import sys +import textwrap + +import pytest +from google.api_core import exceptions as core_exceptions + +from _discoveryengine_support import ( + CONVERSATION_PARENT, + FakeDiscoveryEngine, + answer_client, + answer_request, + instrumented, + search_request, +) + +WRAPPED = { + ("discoveryengine_v1", "SearchServiceClient", "search"), + ("discoveryengine_v1", "SearchServiceClient", "search_lite"), + ("discoveryengine_v1", "SearchServiceAsyncClient", "search"), + ("discoveryengine_v1", "SearchServiceAsyncClient", "search_lite"), + ("discoveryengine_v1", "ConversationalSearchServiceClient", "answer_query"), + ("discoveryengine_v1", "ConversationalSearchServiceAsyncClient", "answer_query"), +} + + +@pytest.fixture() +def fake(): + with FakeDiscoveryEngine() as server: + yield server + + +def _client_methods(): + """Every function defined on every client class of every API version.""" + import importlib + + methods = {} + for version in ("discoveryengine_v1", "discoveryengine_v1alpha", "discoveryengine_v1beta"): + module = importlib.import_module("google.cloud." + version) + for name in module.__all__: + cls = getattr(module, name) + if not (isinstance(cls, type) and name.endswith("Client")): + continue + for attribute, value in vars(cls).items(): + if callable(value): + methods[(version, name, attribute)] = value + return methods + + +def test_only_the_six_v1_methods_are_wrapped(): + before = _client_methods() + with instrumented(): + during = _client_methods() + after = _client_methods() + + assert len(before) > 500 + changed = {key for key in before if during[key] is not before[key]} + assert changed == WRAPPED + assert all(after[key] is before[key] for key in before) + + +def test_the_unversioned_alias_is_v1beta_and_is_not_traced(fake): + # google.cloud.discoveryengine re-exports the v1beta clients at 0.20.5. + # The package targets v1 only (PRD section 3), so a client imported from + # the alias produces no span. The fake serves v1 only, so the call fails, + # and even that failure is not traced. + import grpc + from google.cloud import discoveryengine + from google.cloud.discoveryengine_v1beta.services.search_service.transports.grpc import ( + SearchServiceGrpcTransport, + ) + + assert discoveryengine.SearchServiceClient.__module__.startswith( + "google.cloud.discoveryengine_v1beta." + ) + client = discoveryengine.SearchServiceClient( + transport=SearchServiceGrpcTransport(channel=grpc.insecure_channel(fake.target)) + ) + with instrumented() as traced: + with pytest.raises(core_exceptions.MethodNotImplemented): + client.search(request=search_request()) + + assert traced.spans() == [] + + +def test_create_conversation_emits_no_span(fake): + with instrumented() as traced: + conversation = answer_client(fake).create_conversation( + parent=CONVERSATION_PARENT, conversation={} + ) + + assert conversation.name == CONVERSATION_PARENT + "/conversations/conversation-1" + assert fake.methods() == ["CreateConversation"] + assert traced.spans() == [] + + +def test_stream_answer_query_is_not_traced(fake): + # The architecture lists search, search_lite and answer_query only. + with instrumented() as traced: + responses = list(answer_client(fake).stream_answer_query(request=answer_request())) + + assert [response.answer.state.name for response in responses] == ["STREAMING", "SUCCEEDED"] + assert fake.methods() == ["StreamAnswerQuery"] + assert traced.spans() == [] + + +def test_instrumenting_does_not_import_or_instrument_google_genai(): + # AC-09, in a fresh interpreter so no other test's imports count. + code = textwrap.dedent( + """ + import sys + from opentelemetry.sdk.trace import TracerProvider + from traceai_discoveryengine import DiscoveryEngineInstrumentor + + instrumentor = DiscoveryEngineInstrumentor() + instrumentor.instrument(tracer_provider=TracerProvider()) + loaded = sorted( + name for name in sys.modules + if name == "google.genai" or name.startswith(("google.genai.", "traceai_google_genai")) + ) + instrumentor.uninstrument() + print("loaded=" + ",".join(loaded)) + """ + ) + env = dict(os.environ, PYTHONPATH=os.pathsep.join(sys.path)) + result = subprocess.run( + [sys.executable, "-c", code], env=env, capture_output=True, text=True, timeout=120 + ) + + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "loaded=" diff --git a/python/frameworks/discoveryengine/tests/test_discoveryengine_search.py b/python/frameworks/discoveryengine/tests/test_discoveryengine_search.py new file mode 100644 index 000000000..815213339 --- /dev/null +++ b/python/frameworks/discoveryengine/tests/test_discoveryengine_search.py @@ -0,0 +1,148 @@ +"""J1, J2, J4 and AC-03: SearchServiceClient.search and search_lite on the real gRPC transport.""" + +from __future__ import annotations + +import pytest +from google.api_core import exceptions as core_exceptions +from google.api_core import retry as retries +from google.cloud.discoveryengine_v1 import SearchRequest +from opentelemetry.trace import StatusCode + +from _discoveryengine_support import ( + CONTENT_MARKERS, + PAGES, + SEARCH_RESULTS, + SECOND_PAGE_RESULTS, + SERVING_CONFIG, + UNAVAILABLE_ONCE, + FakeDiscoveryEngine, + attrs, + instrumented, + search_client, + search_request, +) + +QUERY = "SEARCH-QUERY-MARKER" + + +@pytest.fixture() +def fake(): + with FakeDiscoveryEngine() as server: + yield server + + +@pytest.mark.parametrize("method", ["search", "search_lite"]) +def test_one_retriever_span_with_serving_config_and_result_count(fake, method): + with instrumented() as traced: + pager = getattr(search_client(fake), method)(request=search_request(QUERY)) + + assert len(pager.results) == SEARCH_RESULTS + span = traced.one() + assert span.name == "discoveryengine.{0}".format(method) + values = attrs(span) + assert values == { + "gen_ai.span.kind": "RETRIEVER", + "discoveryengine.serving_config": SERVING_CONFIG, + "discoveryengine.result_count": SEARCH_RESULTS, + # TraceConfig default: the query (the span's input) is recorded. + "gen_ai.retrieval.query": QUERY, + "input.value": QUERY, + } + assert span.status.status_code is StatusCode.OK + assert span.events == () + # Result content is never recorded, only the count. + wire = traced.wire() + for marker in CONTENT_MARKERS: + assert marker not in wire, marker + + +def test_search_lite_and_search_are_distinguished_by_span_name_only(fake): + with instrumented() as traced: + client = search_client(fake) + client.search(request=search_request()) + client.search_lite(request=search_request()) + + search, lite = traced.spans() + assert (search.name, lite.name) == ("discoveryengine.search", "discoveryengine.search_lite") + assert attrs(search) == attrs(lite) + assert fake.methods() == ["Search", "SearchLite"] + + +@pytest.mark.parametrize( + "call", + [ + pytest.param(lambda client, req: client.search(req), id="positional-dict"), + pytest.param(lambda client, req: client.search(request=SearchRequest(req)), id="keyword-message"), + pytest.param(lambda client, req: client.search(SearchRequest(req)), id="positional-message"), + pytest.param( + lambda client, req: type(client).search(client, request=req), id="called-through-the-class" + ), + ], +) +def test_the_request_is_read_however_it_is_passed(fake, call): + with instrumented() as traced: + call(search_client(fake), search_request("how is it passed")) + + values = attrs(traced.one()) + assert values["discoveryengine.serving_config"] == SERVING_CONFIG + assert values["input.value"] == "how is it passed" + + +def test_a_request_without_serving_config_or_query_records_neither(fake): + with instrumented() as traced: + search_client(fake).search() + + values = attrs(traced.one()) + assert values == {"gen_ai.span.kind": "RETRIEVER", "discoveryengine.result_count": SEARCH_RESULTS} + + +def test_iterating_a_paged_result_keeps_one_span_per_search_call(fake): + # J4: the pager fetches the next page with the same RPC, not through + # search(), so the second RPC is not a second span. The count is the + # first page's, which is all the call itself returned. + with instrumented() as traced: + pager = search_client(fake).search(request=search_request(PAGES)) + results = list(pager) + + assert len(results) == SEARCH_RESULTS + SECOND_PAGE_RESULTS + assert fake.methods() == ["Search", "Search"] + span = traced.one() + assert span.name == "discoveryengine.search" + assert attrs(span)["discoveryengine.result_count"] == SEARCH_RESULTS + + +def test_client_retries_stay_inside_one_span(fake): + retry = retries.Retry( + predicate=retries.if_exception_type(core_exceptions.ServiceUnavailable), + initial=0.01, + maximum=0.02, + timeout=10, + ) + with instrumented() as traced: + pager = search_client(fake).search(request=search_request(UNAVAILABLE_ONCE), retry=retry) + + assert len(pager.results) == SEARCH_RESULTS + assert fake.methods() == ["Search", "Search"] + span = traced.one() + assert span.status.status_code is StatusCode.OK + assert span.events == () + + +def test_the_serving_config_is_capped_at_1_kb(fake): + from traceai_discoveryengine import _wrappers + + long_config = SERVING_CONFIG + "/" + "x" * 2000 + with instrumented() as traced: + search_client(fake).search(request=search_request(serving_config=long_config)) + + assert fake.calls[0].request.serving_config == long_config + value = attrs(traced.one())["discoveryengine.serving_config"] + assert value == long_config[: _wrappers.MAX_VALUE_BYTES] + + +def test_no_model_token_or_cost_attributes(fake): + with instrumented() as traced: + search_client(fake).search(request=search_request()) + + for key in attrs(traced.one()): + assert not any(word in key for word in ("model", "token", "cost", "usage")), key diff --git a/python/frameworks/discoveryengine/tests/test_discoveryengine_stock_spans.py b/python/frameworks/discoveryengine/tests/test_discoveryengine_stock_spans.py new file mode 100644 index 000000000..79db868e5 --- /dev/null +++ b/python/frameworks/discoveryengine/tests/test_discoveryengine_stock_spans.py @@ -0,0 +1,164 @@ +"""AC-01 (D-8332): measure the stock spans before any wrapper exists. + +The architecture asks for this test first: construct ``SearchServiceClient`` +with a ``unittest.mock`` transport, call ``search`` and count the spans the +process emits, with ``InMemorySpanExporter`` on the global tracer provider. +A ``control`` span around the call proves the export path delivers, so a +zero count for the library is a measurement, not a broken exporter. + +The second test repeats the measurement with the real gRPC transports against +the loopback fake for every method the package may wrap (``search``, +``search_lite``, ``answer_query`` and the async twins). + +google-api-core 2.40 carries an experimental, opt-in tracing path +(``GOOGLE_SDK_EXPERIMENTAL_PYTHON_TRACING_ENABLED`` plus +``opentelemetry-instrumentation-grpc``) that only traces methods whose GAPIC +transport passes ``method_name`` to ``wrap_method``. google-cloud-discoveryengine +0.20.5 does not pass it, so the measurement is repeated with the variable set. + +Expected and measured result at google-cloud-discoveryengine 0.20.5: the +client emits no span, so it has no attribute keys to report. +""" + +from __future__ import annotations + +import asyncio +import importlib.util +from unittest import mock + +import grpc +import pytest + +from _discoveryengine_support import ( + SEARCH_RESULTS, + FakeDiscoveryEngine, + Traced, + answer_client, + answer_request, + async_answer_client, + async_search_client, + new_provider, + search_client, + search_request, +) + +EXPERIMENTAL_TRACING = "GOOGLE_SDK_EXPERIMENTAL_PYTHON_TRACING_ENABLED" + + +@pytest.fixture() +def global_provider(monkeypatch): + """An in-memory provider installed as the process-wide tracer provider. + + Any library span (Google's, gRPC's or ours) started through the global + API lands here. The OTel globals are restored after the test. + """ + from opentelemetry import trace + from opentelemetry.util._once import Once + + exporter, provider = new_provider() + monkeypatch.setattr(trace, "_TRACER_PROVIDER_SET_ONCE", Once()) + monkeypatch.setattr(trace, "_TRACER_PROVIDER", None) + trace.set_tracer_provider(provider) + assert trace.get_tracer_provider() is provider + yield Traced(exporter, provider) + + +@pytest.fixture(params=[None, "true"], ids=["default", "experimental-tracing-env"]) +def tracing_env(request, monkeypatch): + if request.param is None: + monkeypatch.delenv(EXPERIMENTAL_TRACING, raising=False) + else: + monkeypatch.setenv(EXPERIMENTAL_TRACING, request.param) + return request.param + + +def _assert_vendor_methods_are_not_wrapped(): + from google.cloud.discoveryengine_v1 import ( + ConversationalSearchServiceAsyncClient, + ConversationalSearchServiceClient, + SearchServiceAsyncClient, + SearchServiceClient, + ) + + for cls, name in ( + (SearchServiceClient, "search"), + (SearchServiceClient, "search_lite"), + (SearchServiceAsyncClient, "search"), + (SearchServiceAsyncClient, "search_lite"), + (ConversationalSearchServiceClient, "answer_query"), + (ConversationalSearchServiceAsyncClient, "answer_query"), + ): + method = vars(cls)[name] + # A plain function defined by the vendor module: nothing wrapped it. + assert type(method).__name__ == "function", (cls.__name__, name, type(method)) + assert method.__module__.startswith("google.cloud.discoveryengine_v1."), method.__module__ + + +def test_unwrapped_search_on_a_mock_transport_emits_no_span(global_provider, tracing_env): + from google.cloud.discoveryengine_v1 import SearchResponse, SearchServiceClient + from google.cloud.discoveryengine_v1.services.search_service.transports.grpc import ( + SearchServiceGrpcTransport, + ) + + _assert_vendor_methods_are_not_wrapped() + # The channel is never connected: the stub call below is a unittest.mock. + transport = SearchServiceGrpcTransport(channel=grpc.insecure_channel("127.0.0.1:9")) + client = SearchServiceClient(transport=transport) + tracer = global_provider.provider.get_tracer("control") + + with mock.patch.object(type(client.transport.search), "__call__") as call: + call.return_value = SearchResponse( + results=[SearchResponse.SearchResult(id="doc-{0}".format(i)) for i in range(SEARCH_RESULTS)], + total_size=SEARCH_RESULTS, + ) + with tracer.start_as_current_span("control"): + pager = client.search(request=search_request()) + + assert call.call_count == 1 + assert len(pager.results) == SEARCH_RESULTS + # Only the control span: the client and google-api-core emitted none. + assert global_provider.names() == ["control"] + + +def test_unwrapped_clients_on_the_real_grpc_transport_emit_no_span(global_provider, tracing_env): + _assert_vendor_methods_are_not_wrapped() + tracer = global_provider.provider.get_tracer("control") + + async def call_async(fake): + search = async_search_client(fake) + answer = async_answer_client(fake) + try: + pager = await search.search(request=search_request()) + await search.search_lite(request=search_request()) + await answer.answer_query(request=answer_request()) + return pager + finally: + await search.transport.close() + await answer.transport.close() + + with FakeDiscoveryEngine() as fake: + with tracer.start_as_current_span("control"): + pager = search_client(fake).search(request=search_request()) + search_client(fake).search_lite(request=search_request()) + answer_client(fake).answer_query(request=answer_request()) + async_pager = asyncio.run(call_async(fake)) + + assert fake.methods() == ["Search", "SearchLite", "AnswerQuery"] * 2 + assert len(pager.results) == SEARCH_RESULTS + assert len(async_pager.results) == SEARCH_RESULTS + assert global_provider.names() == ["control"] + + +def test_the_experimental_switch_state_is_part_of_the_measurement(tracing_env): + # google-api-core turns its experimental path on only when the variable + # is true and opentelemetry-instrumentation-grpc is importable. The + # default test environment does not install the latter; the README's + # second test command runs this file with it installed, where the switch + # is on and the two tests above still count zero. + observability = pytest.importorskip( + "google.api_core._observability", + reason="google-api-core without the experimental tracing path", + ) + installed = importlib.util.find_spec("opentelemetry.instrumentation.grpc") is not None + enabled = observability.is_otel_capabilities_enabled() + assert enabled is (installed and tracing_env == "true") diff --git a/python/frameworks/discoveryengine/tests/test_discoveryengine_uninstrument.py b/python/frameworks/discoveryengine/tests/test_discoveryengine_uninstrument.py new file mode 100644 index 000000000..73172bbda --- /dev/null +++ b/python/frameworks/discoveryengine/tests/test_discoveryengine_uninstrument.py @@ -0,0 +1,120 @@ +"""uninstrument() restores every wrapped method by identity and stops tracing.""" + +from __future__ import annotations + +import asyncio + +from google.cloud.discoveryengine_v1 import ( + ConversationalSearchServiceAsyncClient, + ConversationalSearchServiceClient, + SearchServiceAsyncClient, + SearchServiceClient, +) + +from _discoveryengine_support import ( + FakeDiscoveryEngine, + answer_client, + answer_request, + async_answer_client, + async_search_client, + instrumented, + search_client, + search_request, +) + +METHODS = ( + (SearchServiceClient, "search"), + (SearchServiceClient, "search_lite"), + (SearchServiceAsyncClient, "search"), + (SearchServiceAsyncClient, "search_lite"), + (ConversationalSearchServiceClient, "answer_query"), + (ConversationalSearchServiceAsyncClient, "answer_query"), +) + + +def _class_methods(): + return {(cls.__name__, name): vars(cls)[name] for cls, name in METHODS} + + +def _every_call(fake): + search_client(fake).search(request=search_request()) + search_client(fake).search_lite(request=search_request()) + answer_client(fake).answer_query(request=answer_request()) + + async def call(): + search = async_search_client(fake) + answer = async_answer_client(fake) + try: + await search.search(request=search_request()) + await search.search_lite(request=search_request()) + await answer.answer_query(request=answer_request()) + finally: + await search.transport.close() + await answer.transport.close() + + asyncio.run(call()) + + +def test_every_wrapped_method_is_defined_on_its_own_class(): + # The async clients define their own methods (they call the transport, + # not the sync client), so wrapping both classes cannot double-wrap one + # function. + methods = _class_methods() + assert len(methods) == 6 + assert len({id(method) for method in methods.values()}) == 6 + + +def test_instrument_wraps_and_uninstrument_restores_all_six_methods(): + originals = _class_methods() + + with instrumented(): + wrapped = _class_methods() + assert all(wrapped[key] is not originals[key] for key in originals) + assert all(wrapped[key].__wrapped__ is originals[key] for key in originals) + + restored = _class_methods() + assert all(restored[key] is originals[key] for key in originals), [ + key for key in originals if restored[key] is not originals[key] + ] + + +def test_no_spans_after_uninstrument(): + with FakeDiscoveryEngine() as fake: + with instrumented() as traced: + pass + _every_call(fake) + + assert len(fake.calls) == 6 + assert traced.spans() == [] + + +def test_instrument_uninstrument_cycles_do_not_stack_wrappers(): + originals = _class_methods() + with FakeDiscoveryEngine() as fake: + for _ in range(3): + with instrumented() as traced: + _every_call(fake) + assert len(traced.spans()) == 6 + assert _class_methods() == originals + + +def test_a_client_made_before_instrument_is_traced(): + # The methods are wrapped on the class, so construction order does not matter. + with FakeDiscoveryEngine() as fake: + client = search_client(fake) + with instrumented() as traced: + client.search(request=search_request()) + + assert traced.names() == ["discoveryengine.search"] + + +def test_a_method_bound_while_instrumented_stops_tracing_after_uninstrument(): + with FakeDiscoveryEngine() as fake: + client = search_client(fake) + with instrumented() as traced: + bound = client.search + bound(request=search_request()) + bound(request=search_request()) + + assert len(fake.calls) == 2 + assert traced.names() == ["discoveryengine.search"] diff --git a/python/frameworks/discoveryengine/traceai_discoveryengine/__init__.py b/python/frameworks/discoveryengine/traceai_discoveryengine/__init__.py new file mode 100644 index 000000000..f0d907f29 --- /dev/null +++ b/python/frameworks/discoveryengine/traceai_discoveryengine/__init__.py @@ -0,0 +1,136 @@ +"""OpenTelemetry instrumentation for Google Discovery Engine (google-cloud-discoveryengine). + +Wraps the v1 ``SearchServiceClient.search``, ``SearchServiceClient.search_lite``, +``ConversationalSearchServiceClient.answer_query`` and their async twins. +Conversation and session CRUD, ``stream_answer_query``, document ingestion +and the other admin RPCs are not traced. The unversioned +``google.cloud.discoveryengine`` alias re-exports the v1beta clients and is +not traced either. +""" + +import logging +from importlib import import_module +from typing import Any, Collection, Dict, Tuple + +from fi_instrumentation import FITracer, TraceConfig +from opentelemetry import trace as trace_api +from opentelemetry.instrumentation.instrumentor import BaseInstrumentor +from wrapt import wrap_function_wrapper + +from traceai_discoveryengine._wrappers import ( + ANSWER_QUERY, + SEARCH, + SEARCH_LITE, + AsyncOperationWrapper, + OperationWrapper, + Options, + _State, +) +from traceai_discoveryengine.package import _instruments +from traceai_discoveryengine.version import __version__ + +logger = logging.getLogger(__name__) +logger.addHandler(logging.NullHandler()) + +_SERVICES = "google.cloud.discoveryengine_v1.services." +# (module, class, is_async, operations) +_TARGETS = ( + (_SERVICES + "search_service.client", "SearchServiceClient", False, (SEARCH, SEARCH_LITE)), + ( + _SERVICES + "search_service.async_client", + "SearchServiceAsyncClient", + True, + (SEARCH, SEARCH_LITE), + ), + ( + _SERVICES + "conversational_search_service.client", + "ConversationalSearchServiceClient", + False, + (ANSWER_QUERY,), + ), + ( + _SERVICES + "conversational_search_service.async_client", + "ConversationalSearchServiceAsyncClient", + True, + (ANSWER_QUERY,), + ), +) + + +class DiscoveryEngineInstrumentor(BaseInstrumentor): # type: ignore[misc] + """Instrument Discovery Engine v1 search and answer client methods. + + ``instrument()`` accepts ``tracer_provider`` and ``config`` (a + ``fi_instrumentation.TraceConfig``, read from the ``FI_HIDE_*`` and + ``FI_PII_REDACTION`` environment variables when omitted). The query is + recorded by default, like the input of every traceAI instrumentor, and + ``hide_inputs`` / ``FI_HIDE_INPUTS`` hides it everywhere, including + server error text. Spans come from an ``FITracer``, so ``using_session`` / + ``using_user`` / ``using_metadata`` / ``using_tags`` / ``using_attributes`` + context attributes are stamped on them. + """ + + def instrumentation_dependencies(self) -> Collection[str]: + return _instruments + + def _instrument(self, **kwargs: Any) -> None: + config = kwargs.get("config") + if config is None: + config = TraceConfig() + elif not isinstance(config, TraceConfig): + raise TypeError( + "config must be a fi_instrumentation.TraceConfig, got {0}".format( + type(config).__name__ + ) + ) + if "capture_query" in kwargs: + # Removed before release: the query follows TraceConfig now. + logger.warning( + "capture_query is ignored: the query is recorded unless " + "TraceConfig(hide_inputs=True) or FI_HIDE_INPUTS=true is set" + ) + options = Options( + hide_inputs=bool(config.hide_inputs), + hide_outputs=bool(config.hide_outputs), + pii_redaction=bool(config.pii_redaction), + ) + tracer_provider = kwargs.get("tracer_provider") or trace_api.get_tracer_provider() + # FITracer applies the config's masking and PII redaction to every + # attribute and stamps the using_* context attributes. + tracer = FITracer( + trace_api.get_tracer(__name__, __version__, tracer_provider), config=config + ) + + self._originals: Dict[Tuple[type, str], Any] = {} + self._state = _State() + for module_name, class_name, is_async, operations in _TARGETS: + try: + client = getattr(import_module(module_name), class_name, None) + except ImportError: + client = None + for operation in operations: + if client is None or operation not in vars(client): + logger.warning( + "Discovery Engine instrumentation skipped %s.%s: not defined on the class", + class_name, + operation, + ) + continue + wrapper_type = AsyncOperationWrapper if is_async else OperationWrapper + wrapper = wrapper_type(tracer, operation, options, self._state) + self._originals[(client, operation)] = vars(client)[operation] + wrap_function_wrapper(client, operation, wrapper) + logger.debug("Discovery Engine instrumentation enabled") + + def _uninstrument(self, **kwargs: Any) -> None: + state = getattr(self, "_state", None) + if state is not None: + # Bound methods read while instrumented stop tracing. + state.enabled = False + for (client, operation), original in getattr(self, "_originals", {}).items(): + setattr(client, operation, original) + self._originals = {} + logger.debug("Discovery Engine instrumentation disabled") + + +__all__ = ["DiscoveryEngineInstrumentor", "__version__"] diff --git a/python/frameworks/discoveryengine/traceai_discoveryengine/_wrappers.py b/python/frameworks/discoveryengine/traceai_discoveryengine/_wrappers.py new file mode 100644 index 000000000..49cd25589 --- /dev/null +++ b/python/frameworks/discoveryengine/traceai_discoveryengine/_wrappers.py @@ -0,0 +1,764 @@ +"""wrapt wrappers for Discovery Engine search, search_lite and answer_query (sync and async).""" + +from __future__ import annotations + +import asyncio +import contextlib +import contextvars +import json +import logging +import re +from dataclasses import dataclass +from typing import Any, Callable, Dict, Iterator, List, Mapping, Optional, Sequence, Tuple + +from fi_instrumentation import REDACTED_VALUE +from fi_instrumentation.instrumentation.pii_redaction import redact_pii_in_string +from opentelemetry import context as context_api +from opentelemetry import trace as trace_api +from opentelemetry.trace import Span, Status, StatusCode, Tracer + +logger = logging.getLogger(__name__) +logger.addHandler(logging.NullHandler()) + +SPAN_KIND = "gen_ai.span.kind" +RETRIEVER = "RETRIEVER" +INPUT_VALUE = "input.value" +RETRIEVAL_QUERY = "gen_ai.retrieval.query" +SERVING_CONFIG = "discoveryengine.serving_config" +RESULT_COUNT = "discoveryengine.result_count" +SESSION = "discoveryengine.session" +ANSWER_LENGTH = "discoveryengine.answer.length" +ANSWER_STATE = "discoveryengine.answer.state" +ERROR_STATUS = "discoveryengine.error.status" +ERROR_CODE = "discoveryengine.error.code" +CANCELLED = "discoveryengine.cancelled" + +SEARCH = "search" +SEARCH_LITE = "search_lite" +ANSWER_QUERY = "answer_query" +REDACTED = "[redacted]" +# Recorded instead of server-written text when the credentials the client +# holds could not be read, so they cannot be removed from it. +UNREADABLE = "[not recorded: the client credentials could not be read]" +MAX_VALUE_BYTES = 1024 +_FAILED_STATE = "FAILED" + +# gRPC metadata keys that carry a credential, compared in lower case. +_AUTH_METADATA = frozenset( + {"authorization", "proxy-authorization", "x-goog-api-key", "x-goog-iam-authorization-token"} +) +# Credential attributes of google.auth credentials that may hold a secret. +_CREDENTIAL_FIELDS = ("token", "refresh_token", "client_secret") +# Google credential shapes, removed even when the package never held the +# value (a token minted by a refresh, a key quoted by the server). +_TOKEN_SHAPES = ( + re.compile(r"ya29\.[0-9A-Za-z_\-.~+/]+=*"), # OAuth 2.0 access token + re.compile(r"AIza[0-9A-Za-z_\-]{35}"), # API key + re.compile(r"1//[0-9A-Za-z_\-]{20,}"), # OAuth 2.0 refresh token + # JWT, such as a service account's self-signed token: three base64url + # parts, the first a JSON object ("eyJ" is base64 of '{"'). + re.compile(r"(? None: + self.enabled = True + + +def _cap(value: str, limit: int = MAX_VALUE_BYTES) -> str: + """Return the longest whole-character prefix of at most ``limit`` UTF-8 bytes.""" + # A cut inside a multi-byte character leaves an invalid tail; "ignore" + # drops it. "replace" keeps a lone surrogate from raising. + return value.encode("utf-8", "replace")[:limit].decode("utf-8", "ignore") + + +def _scrub(value: str, secrets: Sequence[str], server: bool = False) -> str: + """Replace every credential the client holds, then Google credential shapes. + + ``server`` also replaces the value after ``Bearer``; that pass runs only + on server-written text, where a quoted header is the likely source. + """ + for secret in secrets: + value = value.replace(secret, REDACTED) + for shape in _TOKEN_SHAPES: + value = shape.sub(REDACTED, value) + if server: + value = _BEARER.sub(lambda match: match.group(1) + REDACTED, value) + return value + + +# The query text to remove from server-written text: () when it is recorded, +# or None when it must be removed but could not be read (then no such text +# is kept). +Hidden = Optional[Tuple[str, ...]] + + +def _hide(value: str, hidden: Hidden) -> str: + """Replace every occurrence of a hidden input's copies with ``__REDACTED__``, in one pass.""" + if hidden is None: + return REDACTED_VALUE + if not hidden: + return value + pattern = "|".join(re.escape(text) for text in hidden) + return re.sub(pattern, lambda _: REDACTED_VALUE, value) + + +def _clean( + value: str, + secrets: Sequence[str], + limit: int = MAX_VALUE_BYTES, + pii: bool = False, + hidden: Hidden = (), + server: bool = False, +) -> str: + """Remove credentials, then the hidden query, then PII when enabled, then cap. + + Removing before the cap means a credential, the query or an email cut + by the cap cannot leave a prefix. ``FITracer`` applies its PII pass to + attributes again, after the cap; span events and the status are written + only from here. + """ + value = _hide(_scrub(value, secrets, server), hidden) + if pii: + value = redact_pii_in_string(value) + return _cap(value, limit) + + +def _field(message: Any, name: str) -> Any: + """A request or response field from a proto-plus message, raw protobuf or dict.""" + if message is None: + return None + if isinstance(message, Mapping): + return message.get(name) + return getattr(message, name, None) + + +def _has(message: Any, name: str) -> bool: + """Whether a message field is set (proto-plus returns an empty default otherwise).""" + if message is None: + return False + if isinstance(message, Mapping): + return message.get(name) is not None + try: + return name in message + except TypeError: + pass + has_field = getattr(type(message), "HasField", None) + if has_field is not None: + return bool(has_field(message, name)) + return getattr(message, name, None) is not None + + +def _count(value: Any) -> Optional[int]: + """Length of a repeated field; None (never 0) when the shape is unknown.""" + if value is None or isinstance(value, (str, bytes, bytearray, Mapping)): + return None + try: + return len(value) + except TypeError: + return None + + +def _replay(items: Sequence[Any], error: Exception) -> Iterator[Any]: + """The pairs read before ``error``, then ``error``: what the vendor would have read.""" + yield from items + raise error + + +def _read_metadata( + kwargs: Mapping[str, Any], +) -> Tuple[Mapping[str, Any], Optional[Tuple[Any, ...]]]: + """Read the call's ``metadata`` once; return the kwargs to call with and its pairs. + + ``metadata`` may be a one-shot iterable such as a generator, so it is + read here once and the vendor gets a tuple of the same pairs (it makes + a tuple of it too). If reading raises, the vendor gets the pairs read so + far and then the same error, as without the wrapper, and the pairs are + None: the credential lookup then fails closed. + """ + if kwargs.get("metadata") is None: + return kwargs, () + items: List[Any] = [] + try: + for item in kwargs["metadata"]: + items.append(item) + except Exception as error: + return dict(kwargs, metadata=_replay(items, error)), None + pairs = tuple(items) + return dict(kwargs, metadata=pairs), pairs + + +def _credential_values(instance: Any, metadata: Optional[Sequence[Any]]) -> List[str]: + """Every credential this call could send, longest first. + + The clients keep the google.auth credentials on the transport + (``client._transport._credentials``; the async clients wrap a sync client + in ``_client``) and ``client_options.api_key`` in ``_client_options``. + A caller can also pass auth headers per call through ``metadata``, the + pairs ``_read_metadata`` read (None if they could not be read). + Anything that cannot be read raises: the caller then records no free + text for the call. + """ + if metadata is None: + raise LookupError("the call's metadata could not be read") + values: List[str] = [] + + def add(value: Any) -> None: + if isinstance(value, str) and value and value not in values: + values.append(value) + + clients = [instance] + inner = getattr(instance, "_client", None) + if inner is not None: + clients.append(inner) + for client in clients: + credentials = getattr(getattr(client, "_transport", None), "_credentials", None) + for name in _CREDENTIAL_FIELDS: + add(getattr(credentials, name, None)) + add(_field(getattr(client, "_client_options", None), "api_key")) + + for item in metadata: + if not (isinstance(item, (tuple, list)) and len(item) == 2): + continue + key, value = item + if isinstance(key, str) and key.lower() in _AUTH_METADATA and isinstance(value, str): + add(value) + parts = value.split(None, 1) + if len(parts) == 2: + add(parts[1]) # the token after "Bearer" + return sorted(values, key=len, reverse=True) + + +def _request(args: Tuple[Any, ...], kwargs: Mapping[str, Any]) -> Any: + return args[0] if args else kwargs.get("request") + + +def _query_text(operation: str, request: Any) -> Optional[str]: + """``SearchRequest.query`` or ``AnswerQueryRequest.query.text``; None when empty.""" + query = _field(request, "query") + if operation == ANSWER_QUERY: + query = _field(query, "text") + return query if isinstance(query, str) and query else None + + +def _resource_name(value: Any, secrets: Sequence[str], pii: bool) -> Optional[str]: + if isinstance(value, str) and value and not value.endswith("/-"): + return _clean(value, secrets, pii=pii) + return None + + +def _request_attributes( + operation: str, request: Any, secrets: Optional[Sequence[str]], options: Options +) -> Dict[str, Any]: + pii = options.pii_redaction + attributes: Dict[str, Any] = {SPAN_KIND: RETRIEVER} + serving_config = _resource_name(_field(request, "serving_config"), secrets or (), pii) + if serving_config is not None: + attributes[SERVING_CONFIG] = serving_config + if operation == ANSWER_QUERY: + # "sessions/-" asks the server to start a session; the response names it. + session = _resource_name(_field(request, "session"), secrets or (), pii) + if session is not None: + attributes[SESSION] = session + + query = _query_text(operation, request) + if query and options.hide_inputs: + # The placeholder FITracer writes for a hidden input.value; the + # query text never reaches the span. + attributes[INPUT_VALUE] = REDACTED_VALUE + elif query and secrets is not None: + text = _clean(query, secrets, pii=pii) + attributes[RETRIEVAL_QUERY] = text + # The backend input panel reads input.value; keep both keys equal. + attributes[INPUT_VALUE] = text + return attributes + + +def _hides_query(options: Options) -> bool: + return options.hide_inputs + + +def _c_escaped(text: str) -> str: + """``text`` as protobuf text format writes a string field (non-ASCII kept). + + ``\\n``, ``\\r``, ``\\t``, quotes and backslash get their short escapes; + other ASCII control characters and DEL become 3-digit octal (``\\013``), + as upb's ``_upb_TextEncode_Escaped`` writes them. + """ + return "".join( + _C_ESCAPES.get(char) + or ("\\{0:03o}".format(ord(char)) if ord(char) < 0x20 or char == "\x7f" else char) + for char in text + ) + + +def _c_hex_escaped(text: str) -> str: + """``text`` as absl ``CHexEscape`` writes it: bytes outside printable ASCII as ``\\xHH``. + + Like absl, it also escapes a hex digit that follows a ``\\xHH``. + """ + pieces: List[str] = [] + after_hex = False + for byte in text.encode("utf-8", "replace"): + char = chr(byte) + if char in _C_ESCAPES: + pieces.append(_C_ESCAPES[char]) + after_hex = False + elif not 0x20 <= byte < 0x7F or (after_hex and char in _HEX_DIGITS): + pieces.append("\\x{0:02x}".format(byte)) + after_hex = True + else: + pieces.append(char) + after_hex = False + return "".join(pieces) + + +def _hidden_inputs( + operation: str, request: Any, secrets: Optional[Sequence[str]], options: Options +) -> Tuple[str, ...]: + """The query text to remove from server-written text, as it reads after scrubbing. + + Besides the verbatim text, the copies the client libraries escape: the + Python repr and JSON string forms, protobuf text format (a status + detail ``str(error)`` appends) and gRPC core's CHexEscape (the chained + gRPC error's ``debug_error_string`` in older releases). Longest first, + so a longer copy wins where two match at the same place. Empty when the + query is recorded (``hide_inputs`` off, the default) or empty. + """ + if not _hides_query(options): + return () + query = _query_text(operation, request) + if not query: + return () + copies = ( + query, + repr(query)[1:-1], + json.dumps(query)[1:-1], + _c_escaped(query), + _c_hex_escaped(query), + ) + texts = dict.fromkeys(_scrub(copy, secrets or (), server=True) for copy in copies) + return tuple(sorted((text for text in texts if text), key=len, reverse=True)) + + +def _response_attributes( + operation: str, result: Any, secrets: Sequence[str], options: Options +) -> Tuple[Dict[str, Any], Optional[str]]: + """Counts, the answer state and the session; never result or answer text. + + Returns the attributes and, for an answer whose state is FAILED, the + error status description. + """ + attributes: Dict[str, Any] = {} + if operation != ANSWER_QUERY: + # The pager reads through to the first SearchResponse. + results = _count(_field(result, "results")) + if results is not None: + attributes[RESULT_COUNT] = results + return attributes, None + + failure = None + if _has(result, "answer"): + answer = _field(result, "answer") + references = _count(_field(answer, "references")) + if references is not None: + attributes[RESULT_COUNT] = references + text = _field(answer, "answer_text") + if isinstance(text, str): + attributes[ANSWER_LENGTH] = len(text) + state = getattr(_field(answer, "state"), "name", None) + if isinstance(state, str): + attributes[ANSWER_STATE] = state + if state == _FAILED_STATE: + failure = "answer state FAILED" + if _has(result, "session"): + session = _resource_name( + _field(_field(result, "session"), "name"), secrets, options.pii_redaction + ) + if session is not None: + attributes[SESSION] = session + return attributes, failure + + +def _describe(error: BaseException) -> str: + try: + return str(error) + except Exception: + return "".format(type(error).__name__) + + +# The canonical google.rpc.Code names; a REST error's "status" is one of them. +_STATUS_NAMES = frozenset( + { + "OK", + "CANCELLED", + "UNKNOWN", + "INVALID_ARGUMENT", + "DEADLINE_EXCEEDED", + "NOT_FOUND", + "ALREADY_EXISTS", + "PERMISSION_DENIED", + "RESOURCE_EXHAUSTED", + "FAILED_PRECONDITION", + "ABORTED", + "OUT_OF_RANGE", + "UNIMPLEMENTED", + "INTERNAL", + "UNAVAILABLE", + "DATA_LOSS", + "UNAUTHENTICATED", + } +) + + +def _rest_status(error: Any) -> Optional[str]: + """The ``error.status`` of a REST error body, such as ``PERMISSION_DENIED``. + + On the REST transport ``google-api-core`` picks the exception class from + the HTTP code (403 is ``Forbidden``, which has no gRPC status), so the + status name comes from the JSON body the server sent. Only a canonical + status name is returned, never other body text. + """ + response = getattr(error, "response", None) + json_method = getattr(response, "json", None) + if not callable(json_method): + return None + try: + body = json_method() + except Exception: + return None + status = _field(_field(body, "error"), "status") if isinstance(body, Mapping) else None + return status if isinstance(status, str) and status in _STATUS_NAMES else None + + +def _error_attributes(error: BaseException) -> Dict[str, Any]: + """The gRPC status name and HTTP-style code of a ``google.api_core`` error. + + A ``RetryError`` has neither; the last attempt's error (``cause``) does. + A REST-transport error has no gRPC status; its JSON body's status is used. + """ + source: Any = error + if getattr(error, "grpc_status_code", None) is None and getattr(error, "cause", None) is not None: + source = error.cause # type: ignore[attr-defined] + attributes: Dict[str, Any] = {} + status = getattr(getattr(source, "grpc_status_code", None), "name", None) + if not isinstance(status, str): + status = _rest_status(source) + if isinstance(status, str): + attributes[ERROR_STATUS] = status + code = getattr(source, "code", None) + if isinstance(code, int) and not isinstance(code, bool): + attributes[ERROR_CODE] = int(code) + return attributes + + +def _exception_attributes( + error: BaseException, + secrets: Optional[Sequence[str]], + pii: bool = False, + hidden: Hidden = (), +) -> Dict[str, Any]: + """The OTel exception event: the type and the cleaned message, never a stack trace. + + Credentials are removed from the message, then the hidden query and PII + when enabled, then it is cut to 1 KB of UTF-8 (a server error can echo a + large request). ``exception.stacktrace`` is never written: traceback + frames carry local file paths, which include the user name. Without + readable credentials only the type is kept. + """ + error_type = type(error) + module = error_type.__module__ + qualified = ( + "{0}.{1}".format(module, error_type.__qualname__) + if module and module != "builtins" + else error_type.__qualname__ + ) + if secrets is None: + return {"exception.type": qualified, "exception.message": UNREADABLE} + return { + "exception.type": qualified, + "exception.message": _clean(_describe(error), secrets, pii=pii, hidden=hidden, server=True), + } + + +@contextlib.contextmanager +def _current(span: Span) -> Iterator[None]: + """Make ``span`` current for the vendor call so gRPC client spans nest under it. + + Also marks the call active, so a wrapped method called inside it opens + no second span. The span is ended by the wrapper, not on exit, and + errors are recorded once by ``_Call.error``. A failure to attach the + context leaves the call running without it rather than raising into the + caller. + """ + manager = None + token = None + try: + token = _ACTIVE.set(True) + manager = trace_api.use_span( + span, end_on_exit=False, record_exception=False, set_status_on_exception=False + ) + manager.__enter__() + except Exception: + logger.debug("Could not make the Discovery Engine span current", exc_info=True) + manager = None + try: + yield + finally: + if manager is not None: + try: + manager.__exit__(None, None, None) + except Exception: + logger.debug("Could not restore the trace context", exc_info=True) + if token is not None: + try: + _ACTIVE.reset(token) + except Exception: + logger.debug("Could not reset the active-call marker", exc_info=True) + + +class _Call: + """One traced Discovery Engine call. Every method is isolated and ends the span once.""" + + def __init__( + self, + span: Span, + operation: str, + secrets: Optional[List[str]], + options: Options, + hidden: Hidden = (), + instance: Any = None, + metadata: Optional[Tuple[Any, ...]] = (), + ) -> None: + self.span = span + self.operation = operation + self.secrets = secrets + self.options = options + self.hidden = hidden + self.instance = instance + self.metadata = metadata + + def ok(self, result: Any) -> None: + failure = None + try: + attributes, failure = _response_attributes( + self.operation, result, self.secrets or [], self.options + ) + for key, value in attributes.items(): + self.span.set_attribute(key, value) + except Exception: + logger.debug("Could not read the Discovery Engine response", exc_info=True) + try: + if failure is None: + self.span.set_status(Status(StatusCode.OK)) + else: + # Returned, not raised: no exception event. + self.span.set_status(Status(StatusCode.ERROR, failure)) + except Exception: + logger.debug("Could not set the span status", exc_info=True) + self._end() + + def _secrets_after_call(self) -> Optional[List[str]]: + """The credentials read before the call and any set during it, longest first. + + A token minted during the call (first use or a refresh) is set on + the credentials the client holds after they were first read, so + they are read again (attribute reads only; nothing is refreshed). + None, failing closed, if either read failed. + """ + if self.secrets is None: + return None + try: + current = _credential_values(self.instance, self.metadata) + except Exception: + logger.debug("Could not read the Discovery Engine credentials again", exc_info=True) + return None + return sorted(dict.fromkeys(self.secrets + current), key=len, reverse=True) + + def error(self, error: BaseException) -> None: + # No result count: nothing was returned, so the count is unknown. + secrets = self._secrets_after_call() + try: + if secrets is None: + message = UNREADABLE + else: + message = _clean( + _describe(error), + secrets, + pii=self.options.pii_redaction, + hidden=self.hidden, + server=True, + ) + self.span.set_status( + Status(StatusCode.ERROR, "{0}: {1}".format(type(error).__name__, message)) + ) + except Exception: + logger.debug("Could not set the error status", exc_info=True) + try: + for key, value in _error_attributes(error).items(): + self.span.set_attribute(key, value) + except Exception: + logger.debug("Could not read the error status code", exc_info=True) + try: + self.span.add_event( + "exception", + _exception_attributes(error, secrets, self.options.pii_redaction, self.hidden), + ) + except Exception: + logger.debug("Could not record the exception", exc_info=True) + self._end() + + def cancelled(self) -> None: + # Cancellation is not an exception: no event and no result count. + try: + self.span.set_attribute(CANCELLED, True) + self.span.set_status(Status(StatusCode.ERROR, "cancelled")) + except Exception: + logger.debug("Could not mark the span cancelled", exc_info=True) + self._end() + + def _end(self) -> None: + try: + self.span.end() + except Exception: + logger.debug("Could not end the Discovery Engine span", exc_info=True) + + +class _BaseWrapper: + def __init__( + self, + tracer: Tracer, + operation: str, + options: Options, + state: Optional[_State] = None, + ) -> None: + self._tracer = tracer + self._operation = operation + self._span_name = "discoveryengine.{0}".format(operation) + self._options = options + self._state = state or _State() + + def _start( + self, instance: Any, args: Tuple[Any, ...], kwargs: Mapping[str, Any] + ) -> Tuple[Optional[_Call], Mapping[str, Any]]: + """Start the span (None to run the call untraced); return it and the kwargs to call with. + + The kwargs differ from the caller's only in ``metadata``, read once + by ``_read_metadata``. + """ + if not self._state.enabled or _ACTIVE.get(): + return None, kwargs + if context_api.get_value(context_api._SUPPRESS_INSTRUMENTATION_KEY): + return None, kwargs + request = _request(args, kwargs) + kwargs, metadata = _read_metadata(kwargs) + secrets: Optional[List[str]] + try: + secrets = _credential_values(instance, metadata) + except Exception: + # Fail closed: without the credentials, no free text is recorded. + logger.debug("Could not read the Discovery Engine credentials", exc_info=True) + secrets = None + try: + attributes = _request_attributes(self._operation, request, secrets, self._options) + except Exception: # an attribute must never break the user's call + logger.debug("Could not read the Discovery Engine request", exc_info=True) + attributes = {SPAN_KIND: RETRIEVER} + hidden: Hidden = () + try: + hidden = _hidden_inputs(self._operation, request, secrets, self._options) + except Exception: + # The query must be hidden but is unreadable: keep no server text. + logger.debug("Could not read the Discovery Engine query to hide", exc_info=True) + hidden = None if _hides_query(self._options) else () + try: + span = self._tracer.start_span(self._span_name, attributes=attributes) + except Exception: + logger.debug("Could not start the Discovery Engine span", exc_info=True) + return None, kwargs + call = _Call(span, self._operation, secrets, self._options, hidden, instance, metadata) + return call, kwargs + + +class OperationWrapper(_BaseWrapper): + """Trace a synchronous ``search`` / ``search_lite`` / ``answer_query`` call.""" + + def __call__( + self, + wrapped: Callable[..., Any], + instance: Any, + args: Tuple[Any, ...], + kwargs: Mapping[str, Any], + ) -> Any: + call, kwargs = self._start(instance, args, kwargs) + if call is None: + return wrapped(*args, **kwargs) + try: + with _current(call.span): + result = wrapped(*args, **kwargs) + except BaseException as error: + call.error(error) + raise + call.ok(result) + return result + + +class AsyncOperationWrapper(_BaseWrapper): + """Trace an async ``search`` / ``search_lite`` / ``answer_query`` call.""" + + async def __call__( + self, + wrapped: Callable[..., Any], + instance: Any, + args: Tuple[Any, ...], + kwargs: Mapping[str, Any], + ) -> Any: + call, kwargs = self._start(instance, args, kwargs) + if call is None: + return await wrapped(*args, **kwargs) + try: + with _current(call.span): + result = await wrapped(*args, **kwargs) + except asyncio.CancelledError: + call.cancelled() + raise + except BaseException as error: + call.error(error) + raise + call.ok(result) + return result diff --git a/python/frameworks/discoveryengine/traceai_discoveryengine/package.py b/python/frameworks/discoveryengine/traceai_discoveryengine/package.py new file mode 100644 index 000000000..1c5dd5665 --- /dev/null +++ b/python/frameworks/discoveryengine/traceai_discoveryengine/package.py @@ -0,0 +1 @@ +_instruments = ("google-cloud-discoveryengine >= 0.20.5, < 1",) diff --git a/python/frameworks/discoveryengine/traceai_discoveryengine/py.typed b/python/frameworks/discoveryengine/traceai_discoveryengine/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/python/frameworks/discoveryengine/traceai_discoveryengine/version.py b/python/frameworks/discoveryengine/traceai_discoveryengine/version.py new file mode 100644 index 000000000..3dc1f76bc --- /dev/null +++ b/python/frameworks/discoveryengine/traceai_discoveryengine/version.py @@ -0,0 +1 @@ +__version__ = "0.1.0"