diff --git a/.github/workflows/openai-compatible-recipe-tests.yml b/.github/workflows/openai-compatible-recipe-tests.yml new file mode 100644 index 000000000..0e2587e85 --- /dev/null +++ b/.github/workflows/openai-compatible-recipe-tests.yml @@ -0,0 +1,106 @@ +name: OpenAI-compatible recipe tests + +on: + pull_request: + branches: [main, dev] + paths: + - 'python/examples/**' + - 'python/fi_instrumentation/**' + - 'python/frameworks/openai/**' + - 'python/tests/harness/**' + - '.github/workflows/openai-compatible-recipe-tests.yml' + push: + branches: [dev] + paths: + - 'python/examples/**' + - 'python/fi_instrumentation/**' + - 'python/frameworks/openai/**' + - 'python/tests/harness/**' + - '.github/workflows/openai-compatible-recipe-tests.yml' + workflow_dispatch: + +permissions: + contents: read + +jobs: + plan: + name: Recipe plan + runs-on: ubuntu-24.04 + outputs: + matrix: ${{ steps.plan.outputs.matrix }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + - id: plan + run: | + python3 - <<'PY' + import json, os + from pathlib import Path + matrix = [] + for recipe in ('databricks', 'lepton', 'orcarouter', 'anannas-ai', 'doubleword'): + root = Path('python/examples') / recipe + if not root.exists(): + continue # Recipes may land independently. + assert (root / 'src/app.py').is_file(), f'{recipe}: missing app' + assert (root / 'requirements.txt').is_file(), f'{recipe}: missing requirements' + assert list((root / 'tests').glob('test_*.py')), f'{recipe}: missing tests' + for sdk, mode in (('3.24.0', 'source'), ('1.69.0', 'source'), ('3.24.0', 'published')): + matrix.append({'recipe': recipe, 'sdk': sdk, 'mode': mode}) + assert matrix, 'No recipe tests found' + with open(os.environ['GITHUB_OUTPUT'], 'a') as out: + out.write('matrix=' + json.dumps({'include': matrix}) + '\n') + PY + recipe: + name: Recipe (${{ matrix.recipe }}, openai ${{ matrix.sdk }}, ${{ matrix.mode }}) + needs: plan + runs-on: ubuntu-24.04 + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} + env: + PYTHONDONTWRITEBYTECODE: '1' + FI_BASE_URL: http://127.0.0.1:9 + FI_GRPC_URL: http://127.0.0.1:9 + RECIPE: ${{ matrix.recipe }} + SDK_VERSION: ${{ matrix.sdk }} + PACKAGE_MODE: ${{ matrix.mode }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: '3.11.9' + - name: Install SDK and harness dependencies + run: | + python -m pip install "openai==$SDK_VERSION" httpx 'wrapt<2' \ + opentelemetry-api opentelemetry-sdk opentelemetry-instrumentation \ + opentelemetry-exporter-otlp-proto-http requests protobuf \ + opentelemetry-proto pydantic jsonschema pytest + if [ "$PACKAGE_MODE" = published ]; then + python -m pip install 'traceAI-openai==0.1.10' 'fi-instrumentation-otel==1.1.0' + fi + - name: Run isolated recipe contracts + run: | + if [ "$PACKAGE_MODE" = published ]; then + export PYTHONPATH="python/examples/$RECIPE/src:python/tests" + else + export PYTHONPATH="python/examples/$RECIPE/src:python:python/frameworks/openai:python/tests" + fi + python -m pytest "python/examples/$RECIPE/tests" -q \ + --noconftest -p no:cacheprovider -o addopts='' -rfEs + result: + name: OpenAI-compatible recipes + if: always() + needs: [plan, recipe] + runs-on: ubuntu-24.04 + steps: + - name: Require every planned recipe contract + env: + PLAN_RESULT: ${{ needs.plan.result }} + RECIPE_RESULT: ${{ needs.recipe.result }} + run: test "$PLAN_RESULT" = success && test "$RECIPE_RESULT" = success diff --git a/python/examples/databricks/README.md b/python/examples/databricks/README.md new file mode 100644 index 000000000..ec4b6e55d --- /dev/null +++ b/python/examples/databricks/README.md @@ -0,0 +1,181 @@ +# Databricks (OpenAI-compatible) with traceAI + +Trace Chat Completions through the official `openai` Python SDK at your Databricks workspace's documented OpenAI-compatible base URL. This recipe uses `traceai-openai`; it needs no Databricks tracing package. + +## Install + +From this directory, install the pinned dependencies: + +```bash +pip install -r requirements.txt +``` + +Or install the packages directly: + +```bash +pip install traceAI-openai fi-instrumentation-otel openai +``` + +The requirements pin `openai==3.24.0`, `traceAI-openai==0.1.10`, and `fi-instrumentation-otel==1.1.0`. The `traceai-openai` SDK floor is `openai==1.69.0`. + +## Configure + +| Environment variable | Required | Purpose | +|---|---|---| +| `FI_API_KEY` | Yes | Future AGI API key; tracer only. | +| `FI_SECRET_KEY` | Yes | Future AGI secret key; tracer only. | +| `DATABRICKS_TOKEN` | Yes | Databricks token from your secret store; OpenAI client only. Never commit it. | +| `DATABRICKS_MODEL` | Yes, unless using `--model` | Your model service name or serving endpoint name. | +| `DATABRICKS_BASE_URL` | Yes | Your workspace-specific SDK base URL. There is no public default. | +| `FI_BASE_URL` | No | Future AGI collector origin; uses the tracer's default when unset. | +| `FI_PROJECT_NAME` | No | Observe project name; defaults to `databricks-openai-recipe`; tracer only. | + +Choose one of these two different surfaces. Replace `` with the host from your workspace URL: + +| Surface | Documented base URL form | `model` example | +|---|---|---| +| Unity Gateway model services | `https:///ai-gateway/mlflow/v1` | `system.ai.claude-sonnet-4-5` | +| Model Serving serving endpoints | `https:///serving-endpoints` | `openai-chat-endpoint` | + +Use your own model service or endpoint name. Both names above are examples. `DATABRICKS_BASE_URL` is required and replaces the base URL for either surface. + +## Run + +Replace all key and workspace placeholders before running. This example chooses model services; use the second URL form and your endpoint name for Model Serving. + +```bash +export FI_API_KEY="placeholder-fi-api-key" +export FI_SECRET_KEY="placeholder-fi-secret-key" +export DATABRICKS_TOKEN="placeholder-databricks-key" +export DATABRICKS_BASE_URL="https:///ai-gateway/mlflow/v1" +export DATABRICKS_MODEL="system.ai.claude-sonnet-4-5" +python src/app.py +python src/app.py --stream +``` + +You can override the model with `--model` and the question with `--prompt`. Set `FI_PROJECT_NAME` to send the traces to another Observe project; unset or empty, they go to `databricks-openai-recipe`. Missing configuration or a refused URL exits with code 2 before tracing starts. The URL check returns allowed URLs unchanged. The OpenAI SDK then adds a trailing slash to its stored `base_url` and appends `/chat/completions` for this call. + +## Code + +Instrument before creating the client. Validate the workspace URL with the helper in [src/app.py](src/app.py). + +```python +import os +from openai import OpenAI +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType +from traceai_openai import OpenAIInstrumentor +from app import check_base_url +provider = register(project_name=os.environ.get("FI_PROJECT_NAME") or "databricks-openai-recipe", # as in src/app.py + project_type=ProjectType.OBSERVE, set_global_tracer_provider=False, verbose=False) +OpenAIInstrumentor().instrument(tracer_provider=provider) +client = OpenAI(api_key=os.environ["DATABRICKS_TOKEN"], + base_url=check_base_url(os.environ["DATABRICKS_BASE_URL"])) +answer = client.chat.completions.create(model=os.environ["DATABRICKS_MODEL"], + messages=[{"role": "user", "content": "What is a lakehouse?"}]) +print(answer.choices[0].message.content) +provider.force_flush() +``` + +## What you see in Future AGI + +A Chat Completions call produces one `ChatCompletion` span with `gen_ai.span.kind=LLM`. On a successful non-streamed response, `gen_ai.request.model` is the model id the provider returns in its response. The fixtures return the model service or endpoint name sent in the request. + +Up to `traceai-openai` 0.1.10, the provider field says `openai` (`gen_ai.provider.name`), the shared instrumentor's label for this recipe. Later versions (future-agi/traceAI#249) say `databricks` for workspace hosts (`*.cloud.databricks.com`, `*.azuredatabricks.net`, `*.gcp.databricks.com`); other base URLs keep `openai`. + +Token usage appears in `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens`, and `gen_ai.usage.total_tokens` when the response has usage. Missing usage is omitted, never filled with zero. + +Streaming accumulates the answer in `output.value` through the same instrumentor. A default stream without usage exports no usage attributes. With `stream_options={"include_usage": True}` and a final usage chunk, all three usage attributes are exported. Your endpoint must support that option to use it. + +`traceai-openai` 0.1.10 and earlier omit `gen_ai.request.model` on streamed and failed calls; later versions record the requested model there. The requested model remains in the `gen_ai.request.parameters` JSON unless `FI_HIDE_LLM_INVOCATION_PARAMETERS=true`. Errors have span status ERROR and an exception event. + +## Provider specifics + +### Model services + +For Unity Gateway model services, use `/ai-gateway/mlflow/v1` and send your model service name as `model`. The Databricks documentation's example.staging.cloud.databricks.com is a placeholder, not a real host. Replace it with your workspace host; the app refuses the sample host. + +### Serving endpoints + +For Model Serving serving endpoints, use `/serving-endpoints` and send your endpoint name as `model`. These two suffixes select different surfaces. Do not combine them. The token comes from `DATABRICKS_TOKEN`; the SDK sends `Authorization: Bearer `. Keep it in a secret store. Databricks recommends machine-to-machine OAuth for production workloads. + +`model` is the model service or endpoint name, not a frozen foundation-model id. A pricing row keyed by a foundation-model id will not match an endpoint name. An endpoint name needs a matching pricing row for cost lookup. + +The app refuses workspace roots, `/api/2.0/serving-endpoints`, and `/serving-endpoints//invocations`. The last form is a REST invocation URL, not an SDK base URL. It also refuses `/ai-gateway/gemini` and `/ai-gateway/anthropic`, which are gateway native APIs for other SDKs. Databricks workspace hosts must use `https://`; plain `http://` would send the token in cleartext before any redirect. A query string or fragment on a workspace URL is refused. Placeholder checks include percent-encoded forms. Host checks ignore case and trailing dots. Customer proxies and loopback URLs are allowed unchanged. + +### Embeddings, only when the endpoint task is embeddings + +Use `client.embeddings.create(model="system.ai.gte-large-en", input="A short example.")` only when your selected endpoint task is embeddings. Use your own embedding service name. A chat endpoint can reject this call; that is an endpoint task mismatch. + +The gateway fixture produces one `CreateEmbeddingResponse` span with `gen_ai.span.kind=EMBEDDING` and provider `openai` (`databricks` in later versions). Up to 0.1.10 it records the response model in `embedding.model_name`, not `gen_ai.request.model`; later versions record it in both and replace the vectors in `output.value` with a short JSON summary. It exports input and total token usage, with no output token usage. Input text is exported in `input.value` and `embedding.embeddings.0.embedding.text` by default. The recipe's CLI runs Chat Completions. + +### Inside the workspace + +This recipe constructs the stock `openai.OpenAI` client directly; `databricks-openai` is not a recipe dependency. + +## Privacy + +For Chat Completions, set `FI_HIDE_INPUTS=true` or `FI_HIDE_OUTPUTS=true` before tracing starts to keep prompt or response text out of the exported span. `TraceConfig(hide_inputs=True)` passed to the instrumentor is another way to hide chat inputs. The provider still receives the prompt. Future AGI masking does not change the provider's own logging or Databricks payload logs. + +The embedding fixture pins exported input text separately. These chat masking tests do not establish masking of embedding-specific text attributes. + +## Limits / not covered + +- SQL `ai_query` and AI Functions. +- The MLflow Deployments SDK and Databricks-internal MLflow trace forwarding. +- Agent / Genie code and other agent frameworks. +- The Gemini/Anthropic gateway native APIs. +- The `DatabricksOpenAI` helper. +- Direct REST invocation calls and workspace provisioning. + +The fixtures use these syntactic test hosts, not real ones, with `httpx.MockTransport`: + +```text +https://dbc-00000000-0000.cloud.databricks.com/ai-gateway/mlflow/v1 +https://dbc-00000000-0000.cloud.databricks.com/serving-endpoints +``` + +## Tests + +Run from the repository root. The suite checks both surfaces, credentials, spans, usage, streaming, errors, chat masking, embeddings, URL refusal, and CLI subprocesses. Vendor requests use fixtures or a server on `127.0.0.1`. Subprocess guards refuse other hosts before DNS and record refused attempts. A negative control proves the guard is active. + +Repository source: + +```bash +env -u PYTHONPATH PYTHONDONTWRITEBYTECODE=1 \ + PYTHONPATH="python/examples/databricks/src:python:python/frameworks/openai:python/tests" \ + uv run --no-project --python 3.11 \ + --with 'openai==3.24.0' --with httpx --with 'wrapt<2' \ + --with opentelemetry-api --with opentelemetry-sdk --with opentelemetry-instrumentation \ + --with opentelemetry-exporter-otlp-proto-http --with requests --with protobuf \ + --with opentelemetry-proto --with pydantic --with jsonschema --with pytest \ + pytest python/examples/databricks/tests -q -p no:cacheprovider --noconftest -o addopts= -rfEs +``` + +Published packages: + +```bash +env -u PYTHONPATH PYTHONDONTWRITEBYTECODE=1 \ + PYTHONPATH="python/examples/databricks/src:python/tests" \ + uv run --no-project --python 3.11 \ + --with 'openai==3.24.0' --with httpx --with 'wrapt<2' \ + --with 'traceAI-openai==0.1.10' --with 'fi-instrumentation-otel==1.1.0' \ + --with opentelemetry-api --with opentelemetry-sdk --with opentelemetry-instrumentation \ + --with opentelemetry-exporter-otlp-proto-http --with requests --with protobuf \ + --with opentelemetry-proto --with pydantic --with jsonschema --with pytest \ + pytest python/examples/databricks/tests -q -p no:cacheprovider --noconftest -o addopts= -rfEs +``` + +Dependency variants for local checks (for the floor row, change `openai==3.24.0` to `openai==1.69.0` in the repository-source command): + +| Python | `openai` | traceAI packages | Local check configuration | +|---|---|---|---| +| 3.10, 3.11, 3.12, 3.13 | 3.24.0 | `traceai-openai` and `fi_instrumentation` from this repository | Commands above | +| 3.11 | 1.69.0 (the `traceai-openai` floor) | from this repository | Commands above | +| 3.11 | 3.24.0 | published `traceAI-openai==0.1.10` and `fi-instrumentation-otel==1.1.0` from PyPI | Commands above | + +## Official API references + +- [Query model services](https://docs.databricks.com/aws/en/ai-gateway/query-model-services) +- [Migrate to Unity Gateway](https://docs.databricks.com/aws/en/ai-gateway/migrate-to-unity-gateway) +- [Foundation Model REST API](https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/api-reference) diff --git a/python/examples/databricks/requirements.txt b/python/examples/databricks/requirements.txt new file mode 100644 index 000000000..19274c298 --- /dev/null +++ b/python/examples/databricks/requirements.txt @@ -0,0 +1,4 @@ +# OpenAI SDK and published traceAI package pins. +openai==3.24.0 +traceAI-openai==0.1.10 +fi-instrumentation-otel==1.1.0 diff --git a/python/examples/databricks/src/app.py b/python/examples/databricks/src/app.py new file mode 100644 index 000000000..4f9aedc9b --- /dev/null +++ b/python/examples/databricks/src/app.py @@ -0,0 +1,124 @@ +"""Trace Databricks Chat Completions with the stock OpenAI client.""" + +import argparse +import os +import sys +from urllib.parse import unquote, urlsplit + +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType +from openai import OpenAI +from traceai_openai import OpenAIInstrumentor + +DEFAULT_BASE_URL = None +DOCUMENTED_BASE_URL_FORMS = ( + "https:///ai-gateway/mlflow/v1", + "https:///serving-endpoints", +) +API_KEY_ENV = "DATABRICKS_TOKEN" +BASE_URL_ENV = "DATABRICKS_BASE_URL" +MODEL_ENV = "DATABRICKS_MODEL" +_PATHS = ("/ai-gateway/mlflow/v1", "/serving-endpoints") +_HOST_SUFFIXES = (".cloud.databricks.com", ".azuredatabricks.net", ".gcp.databricks.com") + + +def check_base_url(url: str) -> str: + if not url: + raise ValueError(f"Set {BASE_URL_ENV} to your workspace's OpenAI SDK base URL.") + decoded = url + while (next_value := unquote(decoded)) != decoded: + decoded = next_value + suffixes = "use /ai-gateway/mlflow/v1 or /serving-endpoints" + if "<" in decoded or ">" in decoded: + reason = "Replace URL placeholders with your workspace host" + if "/invocations" in decoded: + reason = "REST invocation URLs are not SDK base URLs; replace URL placeholders" + raise ValueError(f"{reason}; {suffixes} in {BASE_URL_ENV}.") + try: + # Host and path checks use the URL as given; only the placeholder check decodes. + parts = urlsplit(url) + host = (parts.hostname or "").lower().rstrip(".") + except ValueError: + raise ValueError(f"Set {BASE_URL_ENV} to an absolute HTTP(S) URL.") from None + if parts.scheme not in ("http", "https") or not host: + raise ValueError(f"Set {BASE_URL_ENV} to an absolute HTTP(S) URL.") + if host == "example.staging.cloud.databricks.com": + raise ValueError( + f"The sample host is a placeholder; set {BASE_URL_ENV} to your workspace; {suffixes}." + ) + if host.endswith(_HOST_SUFFIXES) and parts.scheme != "https": + raise ValueError(f"Databricks workspace URLs must use https; {suffixes} in {BASE_URL_ENV}.") + if host.endswith(_HOST_SUFFIXES) and (parts.query or parts.fragment): + raise ValueError(f"Remove the query or fragment; {suffixes} in {BASE_URL_ENV}.") + if host.endswith(_HOST_SUFFIXES) and parts.path.rstrip("/") not in _PATHS: + reason = "Unsupported Databricks SDK base URL" + if parts.path.startswith("/serving-endpoints/") and parts.path.rstrip("/").endswith( + "/invocations" + ): + reason = "REST invocation URLs are not SDK base URLs" + elif parts.path.rstrip("/") in ("/ai-gateway/gemini", "/ai-gateway/anthropic"): + reason = "Gateway native APIs for other SDKs are not covered" + raise ValueError(f"{reason}; {suffixes} in {BASE_URL_ENV}.") + return url + + +def make_client( + base_url: str | None = None, api_key: str | None = None, http_client=None +) -> OpenAI: + url = check_base_url( + base_url if base_url is not None else os.environ.get(BASE_URL_ENV) or DEFAULT_BASE_URL + ) + key = os.environ[API_KEY_ENV] if api_key is None else api_key + return OpenAI(base_url=url, api_key=key, http_client=http_client) + + +def setup_tracing(project_name: str | None = None): + provider = register( + project_name=project_name or os.environ.get("FI_PROJECT_NAME") or "databricks-openai-recipe", + project_type=ProjectType.OBSERVE, + set_global_tracer_provider=False, + verbose=False, + ) + OpenAIInstrumentor().instrument(tracer_provider=provider) + return provider + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--stream", action="store_true") + parser.add_argument("--prompt", default="What is a lakehouse? Answer in one sentence.") + parser.add_argument("--model") + args = parser.parse_args(argv) + try: + url = check_base_url(os.environ.get(BASE_URL_ENV) or DEFAULT_BASE_URL) + model = args.model or os.environ.get(MODEL_ENV) + if not model: + raise ValueError(f"Set {MODEL_ENV} or pass --model.") + key = os.environ.get(API_KEY_ENV) + if not key: + raise ValueError(f"Set {API_KEY_ENV} to a Databricks token from your secret store.") + except ValueError as error: + print(str(error), file=sys.stderr) + return 2 + provider = setup_tracing() + try: + with make_client(base_url=url, api_key=key) as client: + result = client.chat.completions.create( + model=model, + messages=[{"role": "user", "content": args.prompt}], + stream=args.stream, + ) + if args.stream: + for chunk in result: + if chunk.choices: + print(chunk.choices[0].delta.content or "", end="", flush=True) + print() + else: + print(result.choices[0].message.content or "") + finally: + provider.force_flush() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/python/examples/databricks/tests/_fake_openai.py b/python/examples/databricks/tests/_fake_openai.py new file mode 100644 index 000000000..61642e6a6 --- /dev/null +++ b/python/examples/databricks/tests/_fake_openai.py @@ -0,0 +1,99 @@ +"""OpenAI-shaped fixtures and a loopback server; no upstream forwarding.""" + +import copy +import json +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +ANSWER = "A lakehouse combines data lake storage with warehouse features." +USAGE = {"prompt_tokens": 7, "completion_tokens": 11, "total_tokens": 18} +AUTH_ERROR = {"error_code": "UNAUTHENTICATED", "message": "Invalid access token."} + + +def chat_response(model, usage=True): + response = { + "id": "chatcmpl-placeholder", + "object": "chat.completion", + "created": 1, + "model": model, + "choices": [{"index": 0, "message": {"role": "assistant", "content": ANSWER}, "finish_reason": "stop"}], + } + if usage: + response["usage"] = dict(USAGE) + return response + + +def stream_response(model, include_usage=False): + chunks = [] + for index, text in enumerate(("A lakehouse combines ", "data lake storage with warehouse features.")): + chunks.append({ + "id": "chatcmpl-placeholder", "object": "chat.completion.chunk", "created": 1, + "model": model, + "choices": [{"index": 0, "delta": {"content": text}, "finish_reason": "stop" if index else None}], + }) + if include_usage: + chunks.append({ + "id": "chatcmpl-placeholder", "object": "chat.completion.chunk", "created": 1, + "model": model, "choices": [], "usage": dict(USAGE), + }) + return ("".join("data: " + json.dumps(chunk) + "\n\n" for chunk in chunks) + "data: [DONE]\n\n").encode() + + +def embedding_response(model): + return { + "object": "list", "model": model, + "data": [{"object": "embedding", "index": 0, "embedding": [0.125, 0.25, 0.5]}], + "usage": {"prompt_tokens": 7, "total_tokens": 7}, + } + + +class FakeOpenAI: + def __init__(self, error=False): + self._requests = [] + self._lock = threading.Lock() + owner = self + + class Handler(BaseHTTPRequestHandler): + def do_POST(self): + body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + with owner._lock: + owner._requests.append({ + "path": self.path, + "headers": {key.lower(): value for key, value in self.headers.items()}, + "body": body, + }) + if self.path != "/v1/chat/completions": + self.send_error(404) + return + if error: + status, content_type, payload = 401, "application/json", json.dumps(AUTH_ERROR).encode() + elif body.get("stream"): + status, content_type = 200, "text/event-stream" + payload = stream_response(body["model"], body.get("stream_options", {}).get("include_usage", False)) + else: + status, content_type, payload = 200, "application/json", json.dumps(chat_response(body["model"])).encode() + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def log_message(self, _format, *args): + pass + + self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.base_url = f"http://127.0.0.1:{self._server.server_port}/v1" + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + self._thread.start() + + def requests(self): + with self._lock: + return copy.deepcopy(self._requests) + + def __enter__(self): + return self + + def __exit__(self, *args): + self._server.shutdown() + self._server.server_close() + self._thread.join() diff --git a/python/examples/databricks/tests/loopback_guard/sitecustomize.py b/python/examples/databricks/tests/loopback_guard/sitecustomize.py new file mode 100644 index 000000000..9b9a00db2 --- /dev/null +++ b/python/examples/databricks/tests/loopback_guard/sitecustomize.py @@ -0,0 +1,42 @@ +"""Refuse non-loopback connections before DNS in recipe subprocesses.""" + +import os +import socket +from pathlib import Path + +_connect = socket.socket.connect +_connect_ex = socket.socket.connect_ex +_getaddrinfo = socket.getaddrinfo + + +def _check(host): + if isinstance(host, bytes): + host = host.decode("ascii") + if host not in ("127.0.0.1", "localhost"): + log = os.environ.get("LOOPBACK_GUARD_LOG") + if log: + with open(log, "a", encoding="utf-8") as file: + file.write(f"REFUSED {host}\n") + raise OSError("Loopback guard refused a non-loopback host") + + +def _guarded_connect(self, address): + _check(address[0] if isinstance(address, tuple) else address) + return _connect(self, address) + + +def _guarded_connect_ex(self, address): + _check(address[0] if isinstance(address, tuple) else address) + return _connect_ex(self, address) + + +def _guarded_getaddrinfo(host, *args, **kwargs): + _check(host) + return _getaddrinfo(host, *args, **kwargs) + + +socket.socket.connect = _guarded_connect +socket.socket.connect_ex = _guarded_connect_ex +socket.getaddrinfo = _guarded_getaddrinfo +if ready := os.environ.get("LOOPBACK_GUARD_READY"): + Path(ready).write_text("LOOPBACK_GUARD_READY\n", encoding="utf-8") diff --git a/python/examples/databricks/tests/test_databricks_recipe.py b/python/examples/databricks/tests/test_databricks_recipe.py new file mode 100644 index 000000000..08e584471 --- /dev/null +++ b/python/examples/databricks/tests/test_databricks_recipe.py @@ -0,0 +1,664 @@ +"""Databricks recipe contracts: HTTP fixtures and loopback only.""" + +import atexit +import json +import os +from pathlib import Path +import signal +import socket +import subprocess +import sys +import uuid + +import app +import fi_instrumentation +from harness import Receiver +import httpx +import openai +import pytest +import traceai_openai +from traceai_openai import OpenAIInstrumentor + +from _fake_openai import ( + ANSWER, AUTH_ERROR, FakeOpenAI, chat_response, embedding_response, stream_response, +) + +# traceai-openai 0.1.10 and earlier label every span "openai"; later versions +# (future-agi/traceAI#249) use the vendor's name for its documented API host. +# They also omit gen_ai.request.model on streamed and failed calls, where later +# versions record the requested model. Accept both so this recipe passes on either version. +PROVIDER_NAMES = ("openai", "databricks") + +RECIPE = Path(__file__).resolve().parents[1] +GUARD = RECIPE / "tests" / "loopback_guard" +# Syntactic test hosts, not real ones. MockTransport never resolves these hosts. +GATEWAY = "https://dbc-00000000-0000.cloud.databricks.com/ai-gateway/mlflow/v1" +SERVING = "https://dbc-00000000-0000.cloud.databricks.com/serving-endpoints" +GATEWAY_MODEL = "system.ai.claude-sonnet-4-5" +ENDPOINT_MODEL = "openai-chat-endpoint" +# Synthetic. Not a documented Databricks rewrite. No live call. +RETURNED_MODEL = "system.ai.claude-sonnet-4-5-2026-09-01" +VENDOR_KEY = "placeholder-databricks-key" +FI_KEY = "placeholder-fi-api-key" +FI_SECRET = "placeholder-fi-secret-key" +PROMPT = "unique-prompt-marker-databricks" +FORMS = ( + "https:///ai-gateway/mlflow/v1", + "https:///serving-endpoints", +) + + +def attributes(span): + return {item["key"]: next(iter(item["value"].values())) for item in span["attributes"]} + + +def only_span(receiver, provider=None, kind="LLM", name="ChatCompletion"): + if provider is not None: + assert provider.force_flush() + spans = receiver.spans() + assert len(spans) == 1 + span = spans[0] + assert span["name"] == name + assert attributes(span)["gen_ai.span.kind"] == kind + assert attributes(span)["gen_ai.provider.name"] in PROVIDER_NAMES + return span + + +def usage_attributes(span): + return {key: int(value) for key, value in attributes(span).items() if key.startswith("gen_ai.usage.")} + + +def assert_usage(span): + assert usage_attributes(span) == { + "gen_ai.usage.input_tokens": 7, + "gen_ai.usage.output_tokens": 11, + "gen_ai.usage.total_tokens": 18, + } + + +def assert_vendor_request(request, base, model): + assert str(request.url) == base.rstrip("/") + "/chat/completions" + assert request.headers["authorization"] == f"Bearer {VENDOR_KEY}" + assert "x-api-key" not in request.headers + assert "x-secret-key" not in request.headers + assert FI_KEY not in str(request.headers) + assert FI_SECRET not in str(request.headers) + body = json.loads(request.content) + assert body["model"] == model + assert body["messages"][0]["content"] == PROMPT + + +def assert_export(receiver, project): + exports = receiver.requests() + assert len(exports) == 1 + export = exports[0] + assert export["path"] == "/tracer/v1/traces" + # Positive controls: the Future AGI keys reach the collector, and the project exists. + assert export["headers"]["x-api-key"] == FI_KEY + assert export["headers"]["x-secret-key"] == FI_SECRET + assert export["resource_attributes"][0]["project_name"] == project + assert export["resource_attributes"][0]["project_type"] == "observe" + # The corresponding positive control is the vendor Authorization assertion. + assert VENDOR_KEY not in json.dumps(receiver.spans()) + assert VENDOR_KEY not in json.dumps(exports) + + +@pytest.fixture(autouse=True) +def isolated_environment(monkeypatch): + for name in tuple(os.environ): + if name.startswith(("FI_", "OTEL_", "OPENAI_")) or name.lower() in ( + "http_proxy", "https_proxy", "all_proxy", "no_proxy", + ): + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv("FI_API_KEY", FI_KEY) + monkeypatch.setenv("FI_SECRET_KEY", FI_SECRET) + monkeypatch.setenv("FI_BASE_URL", "http://127.0.0.1:9") + monkeypatch.setenv(app.API_KEY_ENV, VENDOR_KEY) + monkeypatch.setenv(app.BASE_URL_ENV, GATEWAY) + monkeypatch.setenv(app.MODEL_ENV, GATEWAY_MODEL) + + original_connect = socket.socket.connect + original_connect_ex = socket.socket.connect_ex + original_getaddrinfo = socket.getaddrinfo + + def check(host): + if host not in ("127.0.0.1", "localhost"): + raise OSError("Tests allow only loopback network connections") + + def connect(self, address): + check(address[0] if isinstance(address, tuple) else address) + return original_connect(self, address) + + def connect_ex(self, address): + check(address[0] if isinstance(address, tuple) else address) + return original_connect_ex(self, address) + + def getaddrinfo(host, *args, **kwargs): + check(host) + return original_getaddrinfo(host, *args, **kwargs) + + monkeypatch.setattr(socket.socket, "connect", connect) + monkeypatch.setattr(socket.socket, "connect_ex", connect_ex) + monkeypatch.setattr(socket, "getaddrinfo", getaddrinfo) + yield + instrumentor = OpenAIInstrumentor() + if instrumentor.is_instrumented_by_opentelemetry: + instrumentor.uninstrument() + + +@pytest.fixture +def tracing(monkeypatch): + providers = [] + handlers = {sig: signal.getsignal(sig) for sig in (signal.SIGINT, signal.SIGTERM)} + with Receiver() as receiver: + monkeypatch.setenv("FI_BASE_URL", receiver.origin) + + def start(): + project = "databricks-recipe-test-" + uuid.uuid4().hex + provider = app.setup_tracing(project) + providers.append(provider) + return receiver, provider, project + + yield start + if OpenAIInstrumentor().is_instrumented_by_opentelemetry: + OpenAIInstrumentor().uninstrument() + for provider in providers: + provider.shutdown() + atexit.unregister(provider.shutdown) + for sig, handler in handlers.items(): + signal.signal(sig, handler) + + +def complete(client, model=GATEWAY_MODEL, **kwargs): + return client.chat.completions.create( + model=model, messages=[{"role": "user", "content": PROMPT}], **kwargs, + ) + + +def test_documented_forms_and_required_base_url(monkeypatch): + assert app.DEFAULT_BASE_URL is None + assert app.DOCUMENTED_BASE_URL_FORMS == FORMS + readme = (RECIPE / "README.md").read_text() + for form in FORMS: + assert form in readme + with pytest.raises(ValueError, match="placeholder"): + app.check_base_url(form) + monkeypatch.delenv(app.BASE_URL_ENV) + with pytest.raises(ValueError, match=app.BASE_URL_ENV): + app.make_client() + + +@pytest.mark.parametrize("base,model", [(GATEWAY, GATEWAY_MODEL), (SERVING, ENDPOINT_MODEL)]) +def test_both_surfaces_chat_and_key_separation(base, model, monkeypatch, tracing): + monkeypatch.setenv(app.BASE_URL_ENV, base) + receiver, provider, project = tracing() + requests = [] + + def handler(request): + requests.append(request) + assert_vendor_request(request, base, model) + return httpx.Response(200, json=chat_response(model)) + + with app.make_client(http_client=httpx.Client(transport=httpx.MockTransport(handler))) as client: + # OpenAI adds a trailing slash; the recipe passes the original URL unchanged. + assert str(client.base_url) == base + "/" + assert complete(client, model).choices[0].message.content == ANSWER + assert len(requests) == 1 + span = only_span(receiver, provider) + assert attributes(span)["gen_ai.request.model"] == model + assert attributes(span)["input.value"] == PROMPT + assert attributes(span)["output.value"] == ANSWER + assert_usage(span) + assert_export(receiver, project) + + +def test_response_model_differs_from_request(monkeypatch, tracing): + """Span stores the response model, not the gateway request name.""" + assert RETURNED_MODEL not in (GATEWAY_MODEL, ENDPOINT_MODEL) + monkeypatch.setenv(app.BASE_URL_ENV, GATEWAY) + receiver, provider, project = tracing() + requests = [] + + def handler(request): + requests.append(request) + return httpx.Response(200, json=chat_response(RETURNED_MODEL)) + + with app.make_client(http_client=httpx.Client(transport=httpx.MockTransport(handler))) as client: + response = complete(client, GATEWAY_MODEL) + assert response.model == RETURNED_MODEL + assert json.loads(requests[0].content)["model"] == GATEWAY_MODEL + span = only_span(receiver, provider) + assert attributes(span)["gen_ai.request.model"] == RETURNED_MODEL + body = chat_response(RETURNED_MODEL) + assert body["object"] == "chat.completion" + assert body["model"] == RETURNED_MODEL + assert set(body["usage"]) == {"prompt_tokens", "completion_tokens", "total_tokens"} + assert_export(receiver, project) + + +@pytest.mark.parametrize("env_project, explicit, expected", [ + (None, None, "databricks-openai-recipe"), + ("", None, "databricks-openai-recipe"), + ("databricks-from-env", None, "databricks-from-env"), + ("databricks-from-env", "databricks-explicit", "databricks-explicit"), +], ids=["default", "empty", "env", "explicit"]) +def test_setup_tracing_project_name(env_project, explicit, expected, monkeypatch): + # FI_PROJECT_NAME overrides the recipe default; an explicit argument wins over both. + # Never fall through to register()'s generic DEFAULT_PROJECT_NAME. + if env_project is not None: + monkeypatch.setenv("FI_PROJECT_NAME", env_project) + handlers = {sig: signal.getsignal(sig) for sig in (signal.SIGINT, signal.SIGTERM)} + with Receiver() as receiver: + monkeypatch.setenv("FI_BASE_URL", receiver.origin) + provider = app.setup_tracing(explicit) + try: + transport = httpx.MockTransport(lambda request: httpx.Response(200, json=chat_response(GATEWAY_MODEL))) + with app.make_client(http_client=httpx.Client(transport=transport)) as client: + complete(client) + only_span(receiver, provider) + assert_export(receiver, expected) + finally: + OpenAIInstrumentor().uninstrument() + provider.shutdown() + atexit.unregister(provider.shutdown) + for sig, handler in handlers.items(): + signal.signal(sig, handler) + + +def test_make_client_explicit_overrides_and_missing_key(monkeypatch): + def handler(request): + raise AssertionError("Creating a client must not make a request") + + with app.make_client( + base_url=SERVING, api_key="placeholder-explicit-key", + http_client=httpx.Client(transport=httpx.MockTransport(handler)), + ) as client: + assert str(client.base_url) == SERVING + "/" + assert client.api_key == "placeholder-explicit-key" + monkeypatch.delenv(app.API_KEY_ENV) + with pytest.raises(KeyError, match=app.API_KEY_ENV): + app.make_client() + + +def test_usage_absent_is_omitted(tracing): + receiver, provider, project = tracing() + seen = [] + + def handler(request): + seen.append(request) + assert_vendor_request(request, GATEWAY, GATEWAY_MODEL) + return httpx.Response(200, json=chat_response(GATEWAY_MODEL, usage=False)) + + with app.make_client(http_client=httpx.Client(transport=httpx.MockTransport(handler))) as client: + assert complete(client).choices[0].message.content == ANSWER + span = only_span(receiver, provider) + assert len(seen) == 1 + assert attributes(span)["gen_ai.request.model"] == GATEWAY_MODEL + assert usage_attributes(span) == {} + assert_export(receiver, project) + # test_both_surfaces_chat_and_key_separation is the positive usage control. + + +@pytest.mark.parametrize("include_usage", [False, True], ids=["default", "final-usage-chunk"]) +def test_stream_accumulation_and_usage(include_usage, tracing): + receiver, provider, project = tracing() + seen = [] + + def handler(request): + seen.append(request) + assert_vendor_request(request, GATEWAY, GATEWAY_MODEL) + body = json.loads(request.content) + assert body["stream"] is True + if include_usage: + assert body["stream_options"] == {"include_usage": True} + else: + assert "stream_options" not in body + return httpx.Response(200, headers={"Content-Type": "text/event-stream"}, content=stream_response(GATEWAY_MODEL, include_usage)) + + with app.make_client(http_client=httpx.Client(transport=httpx.MockTransport(handler))) as client: + options = {"stream_options": {"include_usage": True}} if include_usage else {} + chunks = list(complete(client, stream=True, **options)) + assert len(seen) == 1 + assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks if chunk.choices) == ANSWER + span = only_span(receiver, provider) + attrs = attributes(span) + assert attrs["output.value"] == ANSWER + # 0.1.10 records the model only from non-streamed responses; later versions record the requested one. + assert attrs.get("gen_ai.request.model") in (None, GATEWAY_MODEL) + assert json.loads(attrs["gen_ai.request.parameters"])["model"] == GATEWAY_MODEL + if include_usage: + assert_usage(span) + else: + assert usage_attributes(span) == {} + assert_export(receiver, project) + + +def test_authentication_error_span(tracing): + receiver, provider, project = tracing() + seen = [] + + def handler(request): + seen.append(request) + assert_vendor_request(request, GATEWAY, GATEWAY_MODEL) + return httpx.Response(401, json=AUTH_ERROR) + + with app.make_client(http_client=httpx.Client(transport=httpx.MockTransport(handler))) as client: + with pytest.raises(openai.AuthenticationError) as error: + complete(client) + assert len(seen) == 1 + assert "Invalid access token." in str(error.value) + span = only_span(receiver, provider) + assert span["status"]["code"] == "STATUS_CODE_ERROR" + events = [event for event in span["events"] if event["name"] == "exception"] + assert events + assert "AuthenticationError" in json.dumps(events) + assert "Invalid access token." in json.dumps(span["status"]) + # 0.1.10 records the model only from non-streamed responses; later versions record the requested one. + assert attributes(span).get("gen_ai.request.model") in (None, GATEWAY_MODEL) + assert json.loads(attributes(span)["gen_ai.request.parameters"])["model"] == GATEWAY_MODEL + assert VENDOR_KEY not in str(error.value) + assert_export(receiver, project) + + +def test_hide_inputs_and_outputs_have_visible_controls(monkeypatch, tracing): + def handler(request): + # Masking the export does not change the prompt sent to the provider. + assert_vendor_request(request, GATEWAY, GATEWAY_MODEL) + return httpx.Response(200, json=chat_response(GATEWAY_MODEL)) + + for hidden in (False, True): + monkeypatch.setenv("FI_HIDE_INPUTS", str(hidden).lower()) + monkeypatch.setenv("FI_HIDE_OUTPUTS", str(hidden).lower()) + receiver, provider, project = tracing() + with app.make_client(http_client=httpx.Client(transport=httpx.MockTransport(handler))) as client: + assert complete(client).choices[0].message.content == ANSWER + span = only_span(receiver, provider) + exported = json.dumps(span) + assert (PROMPT in exported) is (not hidden) + assert (ANSWER in exported) is (not hidden) + assert_usage(span) + assert_export(receiver, project) + OpenAIInstrumentor().uninstrument() + receiver.clear() + + +def test_embeddings_gateway_span(tracing): + receiver, provider, project = tracing() + model = "system.ai.gte-large-en" + marker = "unique-embedding-input-databricks" + seen = [] + + def handler(request): + seen.append(request) + assert str(request.url) == GATEWAY + "/embeddings" + assert request.headers["authorization"] == f"Bearer {VENDOR_KEY}" + assert "x-api-key" not in request.headers + assert "x-secret-key" not in request.headers + assert json.loads(request.content)["input"] == marker + assert json.loads(request.content)["model"] == model + return httpx.Response(200, json=embedding_response(model)) + + with app.make_client(http_client=httpx.Client(transport=httpx.MockTransport(handler))) as client: + response = client.embeddings.create(model=model, input=marker) + assert response.data[0].embedding == [0.125, 0.25, 0.5] + assert len(seen) == 1 + span = only_span(receiver, provider, kind="EMBEDDING", name="CreateEmbeddingResponse") + attrs = attributes(span) + assert attrs["embedding.model_name"] == model + # 0.1.10 sets only embedding.model_name; later versions also make it the span model. + assert attrs.get("gen_ai.request.model") in (None, model) + assert json.loads(attrs["gen_ai.request.parameters"])["model"] == model + assert usage_attributes(span) == {"gen_ai.usage.input_tokens": 7, "gen_ai.usage.total_tokens": 7} + # 0.1.10 JSON-quotes a plain-string input; later versions record the text itself. + assert attrs["input.value"] in (json.dumps(marker), marker) + assert attrs["embedding.embeddings.0.embedding.text"] == marker + assert marker in json.dumps(span) + assert_export(receiver, project) + + +REFUSED = [ + ("https:///ai-gateway/mlflow/v1", "placeholder"), + ("https:///serving-endpoints", "placeholder"), + ("https://.cloud.databricks.com/serving-endpoints", "placeholder"), + ("https://%3Cworkspace-host%3E/serving-endpoints", "placeholder"), + ("https://%253Cworkspace-host%253E/serving-endpoints", "placeholder"), + (GATEWAY + "/%3Cname%3E", "placeholder"), + (GATEWAY + "/%253Cname%253E", "placeholder"), + ("https://example.staging.cloud.databricks.com/ai-gateway/mlflow/v1", "sample host"), + ("https://EXAMPLE.STAGING.CLOUD.DATABRICKS.COM./serving-endpoints", "sample host"), +] +for host in ( + "dbc-00000000-0000.cloud.databricks.com", + "DBC-00000000-0000.CLOUD.DATABRICKS.COM", + "dbc-00000000-0000.cloud.databricks.com.", + "adb-0000000000000000.0.azuredatabricks.net", + "workspace-00000000.gcp.databricks.com", +): + for path, reason in ( + ("", "Unsupported"), ("/", "Unsupported"), + ("/serving-endpoints//invocations", "REST invocation"), + ("/serving-endpoints/openai-chat-endpoint/invocations", "REST invocation"), + ("/ai-gateway/gemini", "native APIs"), + ("/ai-gateway/anthropic", "native APIs"), + ("/api/2.0/serving-endpoints", "Unsupported"), + ): + REFUSED.append((f"https://{host}{path}", reason)) +# Plain http to a Databricks workspace would send the token in cleartext before any redirect. +for host in ( + "dbc-00000000-0000.cloud.databricks.com", + "adb-0000000000000000.0.azuredatabricks.net", + "workspace-00000000.gcp.databricks.com", +): + for path in ("/ai-gateway/mlflow/v1", "/serving-endpoints"): + REFUSED.append((f"http://{host}{path}", "must use https")) +REFUSED.extend([ + (SERVING + "?x=1", "query or fragment"), + (GATEWAY + "#frag", "query or fragment"), + ("https://dbc-00000000-0000.cloud.databricks.com/serving-endpoints%3F/x/invocations", "Unsupported"), +]) + + +@pytest.mark.parametrize("url,reason", REFUSED) +def test_refused_urls_are_actionable_and_not_rewritten(url, reason): + with pytest.raises(ValueError, match=reason) as error: + app.check_base_url(url) + message = str(error.value) + assert "\n" not in message + assert app.BASE_URL_ENV in message + assert "/ai-gateway/mlflow/v1" in message + assert "/serving-endpoints" in message + # Allowed spellings below prove this is a selective check, rather than refusing all URLs. + + +@pytest.mark.parametrize("url", [ + GATEWAY, SERVING, GATEWAY + "/", SERVING + "/", + GATEWAY.replace("dbc-00000000-0000.cloud.databricks.com", "DBC-00000000-0000.CLOUD.DATABRICKS.COM."), + "https://adb-0000000000000000.0.azuredatabricks.net/serving-endpoints", + "https://workspace-00000000.gcp.databricks.com/ai-gateway/mlflow/v1", + "https://customer-proxy.example/custom/prefix/", + "https://cloud.databricks.com.customer-proxy.example/custom/prefix", + "http://127.0.0.1:12345/v1/", "http://localhost:12345/custom/", +]) +def test_allowed_urls_are_returned_exactly(url): + assert app.check_base_url(url) == url + + +@pytest.mark.parametrize("url,reason", REFUSED) +def test_main_refuses_before_tracing_or_network(url, reason, monkeypatch, capsys): + def unexpected_tracing(*args, **kwargs): + pytest.fail("setup_tracing ran before configuration validation") + + monkeypatch.setattr(app, "setup_tracing", unexpected_tracing) + with Receiver() as receiver, FakeOpenAI() as fake: + monkeypatch.setenv("FI_BASE_URL", receiver.origin) + monkeypatch.setenv(app.BASE_URL_ENV, url) + assert app.main([]) == 2 + output = capsys.readouterr() + assert reason in output.err + assert app.BASE_URL_ENV in output.err + assert output.out == "" + assert receiver.spans() == [] + assert receiver.requests() == [] + assert fake.requests() == [] + + +@pytest.mark.parametrize("missing,empty", [ + (app.BASE_URL_ENV, False), (app.MODEL_ENV, False), (app.API_KEY_ENV, False), + (app.BASE_URL_ENV, True), (app.MODEL_ENV, True), (app.API_KEY_ENV, True), +]) +def test_main_missing_configuration_exits_before_tracing(missing, empty, monkeypatch, capsys): + def unexpected_tracing(*args, **kwargs): + pytest.fail("setup_tracing ran with missing configuration") + + monkeypatch.setattr(app, "setup_tracing", unexpected_tracing) + if empty: + monkeypatch.setenv(missing, "") + else: + monkeypatch.delenv(missing) + assert app.main([]) == 2 + output = capsys.readouterr() + assert missing in output.err + assert output.err.startswith("Set ") + assert output.out == "" + assert VENDOR_KEY not in output.err + + +def child_environment(tmp_path, **overrides): + child = os.environ.copy() + # Use the packages actually imported, so the same tests work with published wheels. + package_parents = [ + Path(fi_instrumentation.__file__).resolve().parent.parent, + Path(traceai_openai.__file__).resolve().parent.parent, + ] + harness_parent = Path(sys.modules[Receiver.__module__].__file__).resolve().parent.parent + paths = [GUARD, RECIPE / "src", harness_parent] + paths.extend(package_parents) + child.update({ + "PYTHONPATH": os.pathsep.join(dict.fromkeys(str(path) for path in paths)), + "PYTHONDONTWRITEBYTECODE": "1", + "LOOPBACK_GUARD_LOG": str(tmp_path / "guard.log"), + "LOOPBACK_GUARD_READY": str(tmp_path / "guard.ready"), + }) + (tmp_path / "guard.log").write_text("") + child.update(overrides) + return child + + +@pytest.mark.parametrize("env_project", [None, "databricks-subprocess-env"], ids=["default-project", "env-project"]) +@pytest.mark.parametrize("stream", [False, True], ids=["chat", "stream"]) +def test_app_subprocess_loopback(stream, env_project, tmp_path): + with Receiver() as receiver, FakeOpenAI() as fake: + child = child_environment(tmp_path, **{ + app.BASE_URL_ENV: fake.base_url, "FI_BASE_URL": receiver.origin, + }) + if env_project: + child["FI_PROJECT_NAME"] = env_project + command = [sys.executable, str(RECIPE / "src" / "app.py"), "--prompt", PROMPT] + if stream: + command.append("--stream") + result = subprocess.run(command, env=child, capture_output=True, text=True, timeout=30) + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == ANSWER + assert (tmp_path / "guard.ready").read_text() == "LOOPBACK_GUARD_READY\n" + assert (tmp_path / "guard.log").read_text() == "" + seen = fake.requests() + assert len(seen) == 1 + request = seen[0] + assert request["path"] == "/v1/chat/completions" + assert request["headers"]["authorization"] == f"Bearer {VENDOR_KEY}" + assert "x-api-key" not in request["headers"] + assert "x-secret-key" not in request["headers"] + assert request["body"]["model"] == GATEWAY_MODEL + assert request["body"]["messages"][0]["content"] == PROMPT + assert request["body"]["stream"] is stream + span = only_span(receiver) + assert attributes(span)["output.value"] == ANSWER + if stream: + # 0.1.10 records the model only from non-streamed responses; later versions record the requested one. + assert attributes(span).get("gen_ai.request.model") in (None, GATEWAY_MODEL) + assert json.loads(attributes(span)["gen_ai.request.parameters"])["model"] == GATEWAY_MODEL + assert usage_attributes(span) == {} + else: + assert attributes(span)["gen_ai.request.model"] == GATEWAY_MODEL + assert_usage(span) + assert_export(receiver, env_project or "databricks-openai-recipe") + for key in (VENDOR_KEY, FI_KEY, FI_SECRET): + assert key not in result.stdout + result.stderr + + +def test_model_flag_overrides_environment(monkeypatch, tracing, capsys): + receiver, provider, project = tracing() + monkeypatch.setattr(app, "setup_tracing", lambda: provider) + with FakeOpenAI() as fake: + monkeypatch.setenv(app.BASE_URL_ENV, fake.base_url) + monkeypatch.delenv(app.MODEL_ENV) + assert app.main(["--model", ENDPOINT_MODEL, "--prompt", PROMPT]) == 0 + assert capsys.readouterr().out.strip() == ANSWER + assert fake.requests()[0]["body"]["model"] == ENDPOINT_MODEL + assert attributes(only_span(receiver, provider))["gen_ai.request.model"] == ENDPOINT_MODEL + assert_export(receiver, project) + + +@pytest.mark.parametrize("operation", ["getaddrinfo", "connect", "connect_ex"]) +def test_guard_refuses_before_dns_with_loopback_control(operation, tmp_path): + host = "dbc-00000000-0000.cloud.databricks.com" # Syntactic test host, not real. + child = child_environment(tmp_path) + with FakeOpenAI() as fake: + port = int(fake.base_url.split(":")[2].split("/")[0]) + code = f""" +import socket +with socket.create_connection(('127.0.0.1', {port}), timeout=2): + print('loopback-connected') +if {operation!r} == 'getaddrinfo': + socket.getaddrinfo({host!r}, 443) +else: + with socket.socket() as sock: + getattr(sock, {operation!r})(({host!r}, 443)) +""" + result = subprocess.run([sys.executable, "-c", code], env=child, capture_output=True, text=True, timeout=10) + assert result.returncode != 0 + assert "loopback-connected" in result.stdout + assert "Loopback guard refused" in result.stderr + assert (tmp_path / "guard.ready").read_text() == "LOOPBACK_GUARD_READY\n" + assert (tmp_path / "guard.log").read_text() == f"REFUSED {host}\n" + + +SOURCE_TEST_COMMAND = """env -u PYTHONPATH PYTHONDONTWRITEBYTECODE=1 \\ + PYTHONPATH="python/examples/databricks/src:python:python/frameworks/openai:python/tests" \\ + uv run --no-project --python 3.11 \\ + --with 'openai==3.24.0' --with httpx --with 'wrapt<2' \\ + --with opentelemetry-api --with opentelemetry-sdk --with opentelemetry-instrumentation \\ + --with opentelemetry-exporter-otlp-proto-http --with requests --with protobuf \\ + --with opentelemetry-proto --with pydantic --with jsonschema --with pytest \\ + pytest python/examples/databricks/tests -q -p no:cacheprovider --noconftest -o addopts= -rfEs""" + + +def test_readme_and_requirement_pins(): + readme = (RECIPE / "README.md").read_text() + for form in FORMS: + assert form in readme + for name in (app.API_KEY_ENV, app.BASE_URL_ENV, app.MODEL_ENV, "FI_API_KEY", "FI_SECRET_KEY", "FI_BASE_URL", "FI_PROJECT_NAME", "FI_HIDE_INPUTS", "FI_HIDE_OUTPUTS"): + assert name in readme + assert "| `FI_PROJECT_NAME` | No | Observe project name; defaults to `databricks-openai-recipe`; tracer only. |" in readme + assert 'project_name=os.environ.get("FI_PROJECT_NAME") or "databricks-openai-recipe"' in readme + assert "the provider field says `openai`" in readme + assert "Databricks workspace hosts must use `https://`" in readme + assert "example.staging.cloud.databricks.com is a placeholder, not a real host." in readme + assert "This recipe constructs the stock `openai.OpenAI` client directly" in readme + assert "A pricing row keyed by a foundation-model id will not match an endpoint name." in readme + assert "Embeddings, only when the endpoint task is embeddings" in readme + assert "gen_ai.request.model" in readme + assert "trailing slash" in readme + assert SOURCE_TEST_COMMAND in readme + assert 'PYTHONPATH="python/examples/databricks/src:python/tests"' in readme + assert "--with 'traceAI-openai==0.1.10' --with 'fi-instrumentation-otel==1.1.0'" in readme + assert "TBD" not in readme + assert "| 3.10, 3.11, 3.12, 3.13 | 3.24.0 |" in readme + assert "| 3.11 | 1.69.0 (the `traceai-openai` floor) |" in readme + assert "published `traceAI-openai==0.1.10` and `fi-instrumentation-otel==1.1.0`" in readme + for key_prefix in ("dapi-", "sk-"): + assert key_prefix not in readme + requirements = (RECIPE / "requirements.txt").read_text().splitlines() + assert requirements[0].startswith("#") + assert requirements[1:] == ["openai==3.24.0", "traceAI-openai==0.1.10", "fi-instrumentation-otel==1.1.0"] + assert "databricks-openai" not in "\n".join(requirements)