diff --git a/python/examples/langsmith/README.md b/python/examples/langsmith/README.md new file mode 100644 index 000000000..8206c42bb --- /dev/null +++ b/python/examples/langsmith/README.md @@ -0,0 +1,427 @@ +# LangSmith → Future AGI (live trace forwarding recipe) + +There is no traceAI package for LangSmith, and this example adds none. With +its OpenTelemetry mode on, the LangSmith SDK turns every run into an +OpenTelemetry span. This recipe sends those spans to Future AGI live, while +the app runs. + +It is live forwarding only. It does not import historical LangSmith runs or +projects, it has no LangSmith REST client, and it puts no interceptor, proxy, +DNS override or certificate in front of api.smith.langchain.com. + +Pinned: `langsmith[otel]==0.14.4` (`Requires-Python: >=3.10`). Its wheel +METADATA declares `License: MIT`; it has no Trove classifiers and the wheel +ships no license file. The `otel` extra installs the OpenTelemetry +SDK and OTLP/HTTP exporter; `requirements.txt` pins both to 1.45.0, the +versions the tests ran with. These are dependencies of this example only, not +of any traceAI package. + +## What was tested, and what was not + +- **The SDK half was tested in a deliberately narrow app scope.** It runs the + real langsmith 0.14.4 SDK in OTEL-only mode with this recipe's + `@traceable` chain/tool/LLM stand-in and no LangSmith key. The pinned + OTLP/HTTP export path reached a loopback receiver and matched the hand-built + fixture exactly. The test guard covers Python's `connect`, `connect_ex` and + `getaddrinfo` paths; it is not an OS-, native-, UDP-, or every-DNS-isolation + guarantee. Python 3.10.17, 3.11.12, 3.12.10 and 3.13.15. +- **The SDK coverage does not generalize beyond that app scope.** It does not + test runs produced by LangChain or LangGraph callbacks, streaming, error + runs, other run types beyond the source maps below, any LangSmith SDK + version other than 0.14.4, or a real Future AGI collector, storage, + authentication or rendering path. +- **The collector half was tested against a stand-in.** The fixture was + posted to the shared harness receiver, not to a real fi-collector. What + fi-collector derives from it (span type, model, provider, tokens) was read + from fi-collector's source by opt-in tests, not observed in a running + fi-collector. +- **The contract tests do not call LangSmith.** In fan-out (hybrid) mode the + SDK also sends every run to LangSmith's REST API. In the contract tests those + calls go to a loopback stub that answers 200 to everything. The real LangSmith + API was exercised separately on 2026-10-19, see the next section. + +The contract tests make no call to LangSmith, Future AGI or a model. + +## Live vendor verification (2026-10-19) + +Fan-out mode against the real LangSmith API (US, `api.smith.langchain.com`, +personal access token), spans to a local Future AGI stack, using +`e2e/real_langchain_run.py`. LangSmith project `th8103-e2e`, Future AGI project +`th8103-livefill-th8335`. + +- Versions: langsmith 0.14.11, langchain 1.4.6, langchain-openai 1.7.2, + opentelemetry-sdk and exporter 1.45.1, Python 3.12. The README setup ran unchanged. +- LangSmith accepted the runs: `GET /info` 200, `POST /runs/multipart` 202, and + `POST /api/v1/runs/query` returned the whole run tree with token totals. +- Success: LangSmith trace `01a15682-3f4a-77ae-8845-0786985be886` = Future AGI + trace `40aac0f3-d1cf-e174-f734-f7e572573276` (7 spans, `llm` with + `gpt-5.6-terra` and tokens, `tool lookup_policy`). `fi_verify` PASS. +- Error: LangSmith trace `01a1568c-9cc1-7217-8cfb-0078e9343cfb` (`error`) = + Future AGI trace `f8033fac-dd16-c3e0-6ada-af81b58d49af` (5 spans, + `broken_lookup` and `tools` in ERROR). LangSmith keeps the error text and + traceback; Future AGI stores neither. +- The OpenTelemetry trace id is not the LangSmith run id. Matching is by start + time (identical to the microsecond) or the `lc_run--` text inside + the exported content. +- `LANGSMITH_HIDE_INPUTS` / `LANGSMITH_HIDE_OUTPUTS` hide the LangSmith copy + too (read back `inputs`/`outputs` null). Marker in 5 of 7 stored spans by + default, 0 of 7 with the switches; no username, host name, `/Users/` path, + `@`, traceback or error text in any stored column. +- The default 10 s OTLP exporter timeout was hit on a saturated local stack + ("Failed to export spans batch due to timeout"), with the batch stored + anyway. Set `OTEL_EXPORTER_OTLP_TIMEOUT` (seconds) if your collector is slow. +- Not proven: EU/self-hosted LangSmith, org-scoped keys + (`LANGSMITH_WORKSPACE_ID`), streaming, retriever/parser runs, a Collector fan-out. + +## Live check with a real LangChain agent (opt-in) + +`e2e/real_langchain_run.py` runs a real LangChain agent (real model call, one +tool) and a second run whose tool raises, through the same `init_tracing()`. +Set the README environment, `OPENAI_API_KEY` / `OPENAI_BASE_URL` and +`pip install langchain langchain-openai`, then `python e2e/real_langchain_run.py`. +Re-verified 2026-10-16 against a local Future AGI stack with langsmith 0.14.11 +(latest) and 0.14.4: root `chain` > `model`/`tools` chains > `llm` (model and +tokens filled) and `tool` spans, the failing tool as `ERROR`. The contract +suite also passes on 0.14.11 with `LANGSMITH_TEST_LATEST=1`. Known gap: the +trace view's Input/Output panels stay empty because content arrives as the +single `gen_ai.prompt` / `gen_ai.completion` attributes. + +## Two modes + +| Goal | LangSmith variables | LangSmith key | What langsmith 0.14.4 does | +|---|---|---|---| +| Future AGI only | `LANGSMITH_TRACING=true`, `LANGSMITH_OTEL_ENABLED=true`, `LANGSMITH_OTEL_ONLY=true` | not needed | Exports OpenTelemetry spans only. Makes no call to LangSmith (tested). | +| Fan-out: LangSmith and Future AGI | `LANGSMITH_TRACING=true`, `LANGSMITH_OTEL_ENABLED=true` | `LANGSMITH_API_KEY`, for the LangSmith copy | Calls LangSmith's REST API (`GET /info` once, then `POST /runs/multipart` with the runs) and exports the same runs as OpenTelemetry spans to Future AGI (seen against loopback stubs in the contract tests, and against the real LangSmith API on 2026-10-19). | + +The fan-out row's two variables are the ones LangSmith's OpenTelemetry page +names. In 0.14.4 the SDK calls the three modes `langsmith`, `otel` and +`hybrid`, set by `LANGSMITH_TRACING_MODE`. When that variable is unset it +reads `LANGSMITH_OTEL_ONLY=true` as `otel` and `LANGSMITH_OTEL_ENABLED=true` +as `hybrid`; if both kinds are set, `LANGSMITH_TRACING_MODE` wins and the SDK +warns (`langsmith/client.py:240-306`). `LANGSMITH_TRACING_MODE=otel` behaves +like the first row (tested). `LANGSMITH_OTEL_ONLY` is not on LangSmith's page; +it exists in 0.14.4, and the tests show that it stops the REST calls. + +`LANGSMITH_TRACING=true` is the default environment on switch in both modes. +Without an override, anything other than exactly `true` traces and exports +nothing (tested); the OTEL variables only choose where traces go. +`LANGSMITH_TRACING_V2`, when present, takes precedence; otherwise +`LANGCHAIN_TRACING_V2`, when present, takes precedence. Each must be exactly +`true` and overrides `LANGSMITH_TRACING`. `langsmith.configure(enabled=False)` +and `tracing_context(enabled=False)` override the environment switches too. + +Without a key the SDK only warns, and not at all in `otel` mode +(`client.py:734-760`). In `otel` mode it also skips the `GET /info` call +(`client.py:1918-1921`) and hands each batch only to the OpenTelemetry +exporter (`_internal/_background_thread.py:640-661`). + +## Run + +```bash +cd python/examples/langsmith +pip install -r requirements.txt + +export LANGSMITH_TRACING=true +export LANGSMITH_OTEL_ENABLED=true +export LANGSMITH_OTEL_ONLY=true # Future AGI only; remove it to fan out (needs LANGSMITH_API_KEY) + +export FI_API_KEY="YOUR_API_KEY" +export FI_SECRET_KEY="YOUR_SECRET_KEY" +export FI_PROJECT_NAME="my-langsmith-app" # resource attribute project_name +# Clear inherited trace-specific overrides before setting the general FI values. +unset OTEL_EXPORTER_OTLP_TRACES_ENDPOINT +unset OTEL_EXPORTER_OTLP_TRACES_HEADERS +export OTEL_EXPORTER_OTLP_ENDPOINT="https://YOUR_FI_COLLECTOR_ORIGIN" # no path +export OTEL_EXPORTER_OTLP_HEADERS="$(python -c 'import os; from urllib.parse import quote; print("x-api-key={0},x-secret-key={1}".format(quote(os.environ["FI_API_KEY"], safe=""), quote(os.environ["FI_SECRET_KEY"], safe="")))')" + +python src/app.py "What is the refund window?" +``` + +## The recipe + +`init_tracing()` in `src/app.py` is the whole integration: + +```python +provider = TracerProvider( + resource=Resource.create( + { + "project_name": os.environ["FI_PROJECT_NAME"], + "project_type": "observe", + } + ) +) +provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter())) +trace.set_tracer_provider(provider) +``` + +It is standard OpenTelemetry: the exporter reads its endpoint and headers +from the `OTEL_EXPORTER_OTLP_*` variables above. The rest of `src/app.py` is +an ordinary `@traceable` app: a chain run that calls a tool run and an LLM +run. The LLM run is a stand-in that returns what a chat model run returns; +put your model call there. + +These things about it are load-bearing: + +- **Install the provider before the first traced call.** When the LangSmith + client starts in `otel` or `hybrid` mode it uses the global tracer provider + if one is set; otherwise it installs its own (`client.py:1390-1418`). + `src/app.py` calls `init_tracing()` before it creates the client. +- **LangSmith's own provider cannot feed fi-collector.** Its resource is only + `service.name` (`OTEL_SERVICE_NAME`, default `langsmith`) and + `langsmith.internal_provider=true`. It is built without reading + `OTEL_RESOURCE_ATTRIBUTES` (`_internal/otel/_otel_client.py:124-131`), + so `project_name` cannot be added by environment (tested). + fi-collector rejects a batch whose resource has no `project_name` with + HTTP 400 (`fi-collector/pkg/auth/stamp.go:31-46`, `pkg/server/server.go:461-465`). +- **LangSmith's own provider sends your LangSmith key.** Unless + `OTEL_EXPORTER_OTLP_HEADERS` is set, it sends `x-api-key: ` + and `Langsmith-Project: ` to whatever OTLP endpoint is configured + (`_otel_client.py:116-122`; tested with no key set, so the header was + empty). fi-collector reads `X-Api-Key` as the Future AGI key + (`pkg/auth/middleware.go:45-46`). The recipe's provider sends only the + headers you set. +- **Flush before a short script exits.** `main()` calls `client.flush()`, + which hands LangSmith's queued runs to the provider, then + `provider.shutdown()`, which exports them. + +Another fan-out shape is an OpenTelemetry Collector that receives the app's +OTLP once and forwards it to both backends; it can add `project_name` with a +resource processor. This recipe does not build or test one. + +### Endpoint and headers + +`OTEL_EXPORTER_OTLP_ENDPOINT` is the collector origin with no path; the +exporter appends `/v1/traces` (tested). The signal-specific +`OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` takes precedence and is a full URL used +as given: `/tracer/v1/traces` posts there (tested). fi-collector +serves both paths (`pkg/server/server.go:233-234`). + +`OTEL_EXPORTER_OTLP_TRACES_HEADERS` likewise takes precedence over the +general `OTEL_EXPORTER_OTLP_HEADERS`. The Run block clears both +signal-specific variables before setting the general Future AGI endpoint and +headers. Without those resets, a stale trace endpoint can send the new Future +AGI credentials to another vendor, while stale trace headers can send another +vendor's credential to Future AGI. If you intentionally use signal-specific +settings, set both the full Future AGI trace endpoint and Future AGI headers +together; they override the general values. + +The exporter percent-decodes each header value. The test's secret contains +`,` and `=`; percent-encoded as in the `export` line above, it arrives intact +as `x-secret-key` (tested). + +## What is traced + +`python src/app.py` exports three spans in one trace, each with OTel kind +INTERNAL, status OK and no events: + +| Span | Parent | `langsmith.span.kind` | `gen_ai.operation.name` | Future AGI type | +|---|---|---|---|---| +| `support_request` | none | `chain` | `chain` | chain | +| `lookup_policy` | `support_request` | `tool` | `execute_tool` | tool | +| `chat_model` | `support_request` | `llm` | `chat` | llm | + +Attributes, from the 0.14.4 exporter (`_internal/otel/_otel_exporter.py:548-627`, `:721-848`) +and checked against the real export: + +- Every span: `langsmith.span.kind` (the run type), `langsmith.trace.name`, + `langsmith.trace.session_name` (the LangSmith project, `default` unless + `LANGSMITH_PROJECT` is set), `gen_ai.operation.name`, `gen_ai.system`, + `langsmith.metadata.*` (see [Privacy](#privacy)), `gen_ai.prompt` (the run's + inputs as JSON) and `gen_ai.completion` (its outputs as JSON). +- Tool span: `gen_ai.tool.name`. +- LLM span: `gen_ai.request.model` (from the run's `ls_model_name` metadata), + `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens` and + `gen_ai.usage.total_tokens` (from `usage_metadata` in its outputs), + `gen_ai.response.finish_reasons`, `gen_ai.serialized.name`. + +What that means in Future AGI (read from fi-collector source at future-agi +`main` 4af5338, not run): + +- **Type.** LangSmith sets none of `fi.span.kind`, `gen_ai.span.kind`, + `llm.request.type` and `openinference.span.kind`, so fi-collector takes the + type from `gen_ai.operation.name` (`exporter/clickhouse25exporter/converter.go:79-131`). + It does not read `langsmith.span.kind`. +- **Model, provider, tokens.** `gen_ai.request.model`, `gen_ai.system` and + `gen_ai.usage.*` are on fi-collector's alias lists (`pkg/adapter/adapter.go:279-310`), + so the model, provider and token columns fill on the LLM span. LangSmith + writes `gen_ai.system=langchain` on runs with no model name, or one it does + not recognise (`_otel_exporter.py:629-675`), so chain and tool spans show + provider `langchain`. + +How each LangSmith run type is typed. The middle column is LangSmith's map +(`_otel_exporter.py:117-127`); the last is fi-collector's result: + +| LangSmith `run_type` | `gen_ai.operation.name` | Future AGI type | +|---|---|---| +| `tool` | `execute_tool` | tool | +| `chain` | `chain` | chain | +| `llm` | `chat` | llm | +| `retriever` | `embeddings` | embedding | +| `embedding` | `embeddings` | embedding | +| `prompt` | `chat` | llm | +| `parser` | `parser` | unknown | + +So a retriever run is shown as an embedding and a prompt run as an LLM call. +Only the chain, tool and LLM rows were exported by the tests; the other rows +come from the two maps. + +`gen_ai.prompt` and `gen_ai.completion` are single JSON strings, not the +indexed `gen_ai.prompt..*` keys that the backend's OTel GenAI adapter turns +into input and output messages (`futureagi/tracer/utils/adapters/otel_genai.py:56-57`, +`:135-136`). Whether Future AGI renders them as the span's input and output +was not tested. + +## Privacy + +**Content is exported by default.** Every span carries its run's inputs in +`gen_ai.prompt` and its outputs in `gen_ai.completion`. For the LLM span that +is the whole message list, system prompt included, and the model's answer +(tested). In fan-out mode LangSmith receives the same content. + +`LANGSMITH_HIDE_INPUTS=true` and `LANGSMITH_HIDE_OUTPUTS=true` control those +input and output copies: together, every tested span's `gen_ai.prompt` and +`gen_ai.completion` is `{}`. They do not scrub metadata. The switches also +apply to the LangSmith copy (source reading: `client.py:2468-2475` runs before +either leg). The token counts come from the outputs, so when outputs are +hidden the LLM span has no `gen_ai.usage.*` and no +`gen_ai.response.finish_reasons`, +and the token columns stay empty. The model name stays, and +`langsmith.metadata.usage_metadata` still carries the counts as a JSON string +that fi-collector does not promote (tested). The `hide_inputs`, +`hide_outputs` and `anonymizer` arguments of `langsmith.Client(...)` accept a +function instead (`client.py:2958-2978`; not tested). + +Inputs can include `extra_headers`, `query` and `body` data (for example, +`extra_query` and `extra_body`); those values can survive in the raw +`gen_ai.prompt` JSON. A dict-valued +`extra_headers` is not guaranteed to become a valid separate OpenTelemetry +attribute, so do not assume headers land in a distinct attribute. Treat the +raw input copy as sensitive unless you have applied the input-hiding control. + +**Metadata.** Every span also carries, as `langsmith.metadata.`: + +- the run's metadata (`ls_method`, and on the LLM span `ls_provider`, + `ls_model_name` and `usage_metadata`); +- every `LANGSMITH_*` and `LANGCHAIN_*` environment variable, except names + containing `key`, `secret`, `token`, `password`, `passwd`, `pwd`, + `credential` or `email` and a few fixed names + (`env/_runtime_env.py:172-213`; the tests see `LANGSMITH_TRACING`, + `LANGSMITH_OTEL_ENABLED` and `LANGSMITH_OTEL_ONLY`); +- `revision_id`: `LANGCHAIN_REVISION_ID` if set, otherwise + `git describe --tags --always --dirty` of the working directory + (`env/_runtime_env.py:207-222`). + +That filter checks names, not values. For example, `LANGSMITH_ENDPOINT`, +`LANGSMITH_PROJECT` and other allowed variables can be exported as metadata; +a credential-bearing endpoint URL can therefore be exported. Review the values of allowed +LangSmith/LangChain variables rather than treating the name filter as a +privacy boundary. + +`LANGSMITH_HIDE_METADATA=true` drops all of it, and with it `ls_model_name`, +so the model column empties too (source reading: `client.py:2480-2486`, +`:2513-2519`, `_otel_exporter.py:516-522`). + +The tests use placeholder Future AGI credentials and set no LangSmith key. + +## Not included + +- No historical import. There is no LangSmith REST client, no project + backfill and no command that takes a LangSmith project. Future AGI's + Langfuse import page is not a template for this one. +- No interceptor, proxy, DNS override or certificate in front of + api.smith.langchain.com. +- No traceAI package, no fi-collector change and no span-kind mapping: + fi-collector's fallback already types chain, tool and LLM runs. + +## Troubleshooting + +- **No traces and no error.** `LANGSMITH_TRACING` is not exactly `true`; + `True` and `1` do not count (`utils.py:121-142`). Check + `LANGSMITH_TRACING_V2` first, then `LANGCHAIN_TRACING_V2`: when either is + present it overrides `LANGSMITH_TRACING` and must be exactly `true`. Also + check for `langsmith.configure(enabled=False)` or + `tracing_context(enabled=False)`. The OTEL variables alone trace nothing. +- **fi-collector answers 400, "no project_name".** The spans went through + LangSmith's own provider: `init_tracing()` ran after the LangSmith client + was created, or not at all. +- **Spans go to LangSmith's OTLP endpoint.** LangSmith's own provider, with + no `OTEL_EXPORTER_OTLP_ENDPOINT`, exports to + `/otel/v1/traces` + (`_otel_client.py:95-104`; source reading). Same fix as above. +- **A LangSmith key is used although `LANGSMITH_API_KEY` is unset.** A + LangSmith CLI profile in `~/.langsmith/config.json` (or `LANGSMITH_CONFIG_FILE`) + can supply a key and an endpoint (`_internal/_profiles.py:71-129`, + `client.py:1286-1299`; source reading). +- **Empty token columns.** `LANGSMITH_HIDE_OUTPUTS=true` is set, or the LLM + run's outputs have no `usage_metadata`. +- **Retriever runs shown as embeddings.** LangSmith exports `retriever` runs + with `gen_ai.operation.name=embeddings`; see the run-type table. +- **`Failed to export spans batch due to timeout, max retries or shutdown`.** + The exporter's default timeout is 10 s; raise `OTEL_EXPORTER_OTLP_TIMEOUT`. +- **Spans doubled.** Another instrumentor on the same provider (for example a + traceAI LangChain instrumentor) records the same calls again. Use one or the + other (not tested). + +## Tests + +`tests/test_langsmith_recipe.py` tests the two halves separately. + +- **Collector half.** `tests/fixtures/langsmith-0.14.4-run-tree.otlp.json` is + an OTLP/JSON body written by hand from the 0.14.4 exporter source: a + `support_request` chain run with a `lookup_policy` tool child and a + `chat_model` LLM child, with `project_name` and `project_type=observe` on + the resource. The tests post it with the harness `post_otlp()`, check it + with `compare()`, and assert one trace, the tool and LLM children, and the + model, provider and token keys. These need no LangSmith install. +- **SDK half.** `src/app.py` runs as written, in a subprocess, with the real + langsmith 0.14.4. `tests/_guarded_run.py` intercepts Python + `socket.connect`, `socket.connect_ex` and `socket.getaddrinfo` paths; its + positive control (`tests/_guard_probe.py`) exercises those paths for IPv4, + IPv6 and DNS attempts. It is not a guarantee of OS-, native-, UDP-, or every + DNS-path isolation. The pinned OTLP/HTTP export path is actually exercised: + spans go to the shared harness `Receiver` (`python/tests/harness`), which + serves `/v1/traces` and `/tracer/v1/traces` on 127.0.0.1. The recipe run + leaves `LANGSMITH_ENDPOINT` unset, so a LangSmith call on the tested Python + paths would be refused; none occurred. The export must equal the fixture + (names, attributes and values, statuses). Other runs check the request path, + headers and resource, the three modes against a loopback LangSmith stub, + tracing off, the hide switches, and LangSmith's own provider + (`tests/sdk_own_provider.py`, a fixture, not part of the recipe). + +Every child environment is built from scratch with a throwaway `HOME`, so no +LangSmith key or profile is inherited, and the first test fails if +`LANGSMITH_API_KEY` or `LANGCHAIN_API_KEY` is set. The FI keys are +placeholders. The tests pin `LANGCHAIN_REVISION_ID` so the exported +`revision_id` does not depend on the checkout. + +From the repository root, Python 3.11: + +```bash +env -u PYTHONPATH PYTHONPATH="python/tests" \ + uv run --no-project --python 3.11 \ + --with 'pytest==9.1.1' --with 'langsmith[otel]==0.14.4' \ + --with 'opentelemetry-sdk==1.45.0' --with 'opentelemetry-exporter-otlp-proto-http==1.45.0' \ + pytest python/examples/langsmith/tests --confcutdir python/examples/langsmith \ + -q -p no:cacheprovider --noconftest -o addopts= -rfEs +``` + +`--confcutdir` keeps pytest from importing `python/__init__.py`, which needs +traceAI's own dependencies. For Python 3.10, 3.12 or 3.13, replace +`--python 3.11`. Without +`--with 'langsmith[otel]==0.14.4'` (and the two OpenTelemetry pins) the +SDK-half tests are skipped and the fixture tests still run. + +Two tests are opt-in. They read fi-collector's span-type rules and alias +lists from its source (`exporter/clickhouse25exporter/converter.go` and +`pkg/adapter/adapter.go`) and check the "Future AGI type" columns above, the +fixture's types, and its model, provider and token values against them. +Point `FI_COLLECTOR_SRC` at a fi-collector checkout (the directory holding +`exporter/` and `pkg/`), then run the command above: + +```bash +export FI_COLLECTOR_SRC=/fi-collector +``` + +Without it they are skipped. No CI job runs this example, so run them +whenever fi-collector's rules or this recipe change; otherwise the tables +above can drift unnoticed. diff --git a/python/examples/langsmith/e2e/real_langchain_run.py b/python/examples/langsmith/e2e/real_langchain_run.py new file mode 100644 index 000000000..265f82bf9 --- /dev/null +++ b/python/examples/langsmith/e2e/real_langchain_run.py @@ -0,0 +1,61 @@ +"""Opt-in live check: a real LangChain agent (real model call, one tool) plus a failing run. + +Run by hand against a Future AGI collector, with the same environment as the +README. Needs an OpenAI-compatible endpoint (OPENAI_API_KEY / OPENAI_BASE_URL) +and `pip install langchain langchain-openai`. Not run in CI. + +Usage: python e2e/real_langchain_run.py [ok|fail|both] +""" + +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "src")) + +import langsmith # noqa: E402 +from app import init_tracing # noqa: E402 +from langchain.agents import create_agent # noqa: E402 +from langchain_core.tools import tool # noqa: E402 +from langchain_openai import ChatOpenAI # noqa: E402 + +MODEL = os.environ.get("E2E_MODEL", "gpt-5.6-terra") + + +@tool +def lookup_policy(topic: str) -> str: + """Return the refund policy for a topic.""" + return "Refunds are accepted within 30 days of purchase." + + +@tool +def broken_lookup(topic: str) -> str: + """Look up a policy (always fails).""" + raise RuntimeError("policy backend unavailable") + + +def run(agent_tools, question: str) -> None: + agent = create_agent(ChatOpenAI(model=MODEL, temperature=0), tools=agent_tools) + try: + out = agent.invoke({"messages": [{"role": "user", "content": question}]}) + print(out["messages"][-1].content) + except Exception as exc: # the failing tool run is exported with ERROR status + print("run failed:", type(exc).__name__) + + +def main() -> None: + mode = sys.argv[1] if len(sys.argv) > 1 else "both" + provider = init_tracing() + client = langsmith.Client() + langsmith.configure(client=client) + if mode in ("ok", "both"): + run([lookup_policy], os.environ.get("E2E_QUESTION", "Use the lookup_policy tool for 'refunds' and tell me the refund window.")) + if mode in ("fail", "both"): + run([broken_lookup], "Use the broken_lookup tool for 'refunds'.") + client.flush() + provider.shutdown() + + +if __name__ == "__main__": + main() diff --git a/python/examples/langsmith/requirements.txt b/python/examples/langsmith/requirements.txt new file mode 100644 index 000000000..be8d6f9c2 --- /dev/null +++ b/python/examples/langsmith/requirements.txt @@ -0,0 +1,6 @@ +# Not a traceAI package: these are the example's own dependencies. +# langsmith's otel extra pulls in the OpenTelemetry SDK and OTLP/HTTP exporter. +langsmith[otel]==0.14.4 +# src/app.py imports these directly; pinned to the versions the contract test ran with. +opentelemetry-sdk==1.45.0 +opentelemetry-exporter-otlp-proto-http==1.45.0 diff --git a/python/examples/langsmith/src/app.py b/python/examples/langsmith/src/app.py new file mode 100644 index 000000000..c3a601117 --- /dev/null +++ b/python/examples/langsmith/src/app.py @@ -0,0 +1,104 @@ +"""A LangSmith-traced app whose runs also reach Future AGI as OpenTelemetry spans. + +With OTEL mode on, the LangSmith SDK turns every run into an OpenTelemetry +span. ``init_tracing()`` installs the tracer provider those spans go to: the +Future AGI project on the resource, and an OTLP/HTTP exporter that reads its +endpoint and headers from the standard ``OTEL_EXPORTER_OTLP_*`` variables. +That is the whole integration; the rest is an ordinary ``@traceable`` app. + +Usage: python app.py "What is the refund window?" +""" + +from __future__ import annotations + +import os +import sys + +import langsmith +from langsmith import traceable +from opentelemetry import trace +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor + +MODEL = os.environ.get("APP_MODEL", "gpt-4o-mini") +POLICY = "Refunds are accepted within 30 days of purchase." + + +def init_tracing() -> TracerProvider: + """Install the global provider that LangSmith's OTEL exporter writes into. + + Call it before the first traced call. LangSmith adopts a global provider + that already exists; otherwise it installs its own, whose resource has no + ``project_name``, and fi-collector rejects that. + """ + provider = TracerProvider( + resource=Resource.create( + { + "project_name": os.environ["FI_PROJECT_NAME"], + "project_type": "observe", + } + ) + ) + provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter())) + trace.set_tracer_provider(provider) + return provider + + +@traceable(run_type="tool", name="lookup_policy") +def lookup_policy(topic: str) -> str: + return POLICY + + +@traceable( + run_type="llm", + name="chat_model", + metadata={"ls_provider": "openai", "ls_model_name": MODEL}, +) +def chat_model(messages: list[dict]) -> dict: + """Stand-in for your model call, returning what a chat model run returns. + + LangSmith reads the model name from ``ls_model_name`` and the token + counts from ``usage_metadata`` in the outputs. + """ + answer = "Refunds are accepted for 30 days." + return { + "choices": [ + { + "message": {"role": "assistant", "content": answer}, + "finish_reason": "stop", + } + ], + "usage_metadata": {"input_tokens": 21, "output_tokens": 9, "total_tokens": 30}, + } + + +@traceable(run_type="chain", name="support_request") +def support_request(question: str) -> str: + policy = lookup_policy("refunds") + response = chat_model( + [ + {"role": "system", "content": "Answer from this policy: " + policy}, + {"role": "user", "content": question}, + ] + ) + return response["choices"][0]["message"]["content"] + + +def main() -> None: + provider = init_tracing() + client = langsmith.Client() # created after init_tracing(), so it adopts the provider + langsmith.configure(client=client) + + question = sys.argv[1] if len(sys.argv) > 1 else "What is the refund window?" + print(support_request(question)) + + # A short script exits before the background queues drain: hand LangSmith's + # queued runs to the provider, then export them. + client.flush() + provider.shutdown() + + +if __name__ == "__main__": + main() diff --git a/python/examples/langsmith/tests/_guard_probe.py b/python/examples/langsmith/tests/_guard_probe.py new file mode 100644 index 000000000..5ebeff59d --- /dev/null +++ b/python/examples/langsmith/tests/_guard_probe.py @@ -0,0 +1,26 @@ +"""A test-only probe: try three non-loopback connections under the loopback guard. + +The contract test runs this through ``_guarded_run.py`` to prove the guard +really refuses and logs a non-loopback attempt, so the ``guard_attempts == []`` +assertions elsewhere cannot pass because the guard silently stopped working. +The targets are reserved and never reachable: TEST-NET-1 (RFC 5737), the IPv6 +documentation prefix (RFC 3849) and the ``.invalid`` TLD (RFC 2606). +Not part of the recipe. +""" + +from __future__ import annotations + +import socket + +refused = [] +for label, attempt in ( + ("connect", lambda: socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect(("192.0.2.1", 80))), + ("connect_ex", lambda: socket.socket(socket.AF_INET6, socket.SOCK_STREAM).connect_ex(("2001:db8::1", 80, 0, 0))), + ("getaddrinfo", lambda: socket.getaddrinfo("example.invalid", 443)), +): + try: + attempt() + except OSError as error: + if "blocked by loopback guard" in str(error): + refused.append(label) +print("refused:", ",".join(refused)) diff --git a/python/examples/langsmith/tests/_guarded_run.py b/python/examples/langsmith/tests/_guarded_run.py new file mode 100644 index 000000000..ae024d75c --- /dev/null +++ b/python/examples/langsmith/tests/_guarded_run.py @@ -0,0 +1,81 @@ +"""Run a script with every non-loopback network connection blocked and logged. + +Usage: python _guarded_run.py [args...] + +Before the script starts, ``socket.socket.connect``/``connect_ex`` and +``socket.getaddrinfo`` refuse any host other than 127.0.0.1, ::1 or +localhost. Each refused attempt is appended as a JSON line to the file named +by ``LOOPBACK_GUARD_LOG``, so a test can assert that the LangSmith SDK and +the example tried to reach nothing else (no api.smith.langchain.com, no +LangSmith OTLP endpoint). Unix sockets are not affected. +""" + +from __future__ import annotations + +import json +import os +import runpy +import socket +import sys + +_ALLOWED_HOSTS = {"127.0.0.1", "::1", "localhost"} +_LOG_PATH = os.environ.get("LOOPBACK_GUARD_LOG", "") + + +def _host(value: object) -> str: + if isinstance(value, bytes): + return value.decode("ascii", "replace") + return str(value) + + +def _refuse(kind: str, target: object) -> None: + if _LOG_PATH: + with open(_LOG_PATH, "a", encoding="utf-8") as log: + log.write(json.dumps({"kind": kind, "target": repr(target)}) + "\n") + + +_original_connect = socket.socket.connect +_original_connect_ex = socket.socket.connect_ex +_original_getaddrinfo = socket.getaddrinfo + + +def _is_blocked(sock: socket.socket, address: object) -> bool: + if sock.family not in (socket.AF_INET, socket.AF_INET6): + return False + host = address[0] if isinstance(address, tuple) else address + return _host(host) not in _ALLOWED_HOSTS + + +def _connect(self: socket.socket, address: object) -> None: + if _is_blocked(self, address): + _refuse("connect", address) + raise OSError("blocked by loopback guard: {0!r}".format(address)) + return _original_connect(self, address) + + +def _connect_ex(self: socket.socket, address: object) -> int: + if _is_blocked(self, address): + _refuse("connect_ex", address) + raise OSError("blocked by loopback guard: {0!r}".format(address)) + return _original_connect_ex(self, address) + + +def _getaddrinfo(host: object, *args: object, **kwargs: object): # type: ignore[no-untyped-def] + if host is not None and _host(host) not in _ALLOWED_HOSTS: + _refuse("getaddrinfo", host) + raise socket.gaierror("blocked by loopback guard: {0!r}".format(host)) + return _original_getaddrinfo(host, *args, **kwargs) + + +def main() -> None: + socket.socket.connect = _connect # type: ignore[method-assign] + socket.socket.connect_ex = _connect_ex # type: ignore[method-assign] + socket.getaddrinfo = _getaddrinfo # type: ignore[assignment] + script = sys.argv[1] + sys.argv = sys.argv[1:] + sys.path.insert(0, os.path.dirname(os.path.abspath(script))) + runpy.run_path(script, run_name="__main__") + + +if __name__ == "__main__": + main() diff --git a/python/examples/langsmith/tests/fixtures/langsmith-0.14.4-run-tree.otlp.json b/python/examples/langsmith/tests/fixtures/langsmith-0.14.4-run-tree.otlp.json new file mode 100644 index 000000000..24c9871d0 --- /dev/null +++ b/python/examples/langsmith/tests/fixtures/langsmith-0.14.4-run-tree.otlp.json @@ -0,0 +1,100 @@ +{ + "resourceSpans": [ + { + "resource": { + "attributes": [ + {"key": "project_name", "value": {"stringValue": "langsmith-recipe-contract"}}, + {"key": "project_type", "value": {"stringValue": "observe"}} + ] + }, + "scopeSpans": [ + { + "scope": {"name": "langsmith"}, + "spans": [ + { + "traceId": "6f1c2a9e0b3d4c5e8a7b9c0d1e2f3a4b", + "spanId": "a1b2c3d4e5f60718", + "name": "support_request", + "kind": 1, + "startTimeUnixNano": "1791187200000000000", + "endTimeUnixNano": "1791187200250000000", + "attributes": [ + {"key": "langsmith.span.kind", "value": {"stringValue": "chain"}}, + {"key": "langsmith.trace.name", "value": {"stringValue": "support_request"}}, + {"key": "langsmith.trace.session_name", "value": {"stringValue": "default"}}, + {"key": "gen_ai.operation.name", "value": {"stringValue": "chain"}}, + {"key": "gen_ai.system", "value": {"stringValue": "langchain"}}, + {"key": "langsmith.metadata.ls_method", "value": {"stringValue": "traceable"}}, + {"key": "langsmith.metadata.LANGSMITH_TRACING", "value": {"stringValue": "true"}}, + {"key": "langsmith.metadata.LANGSMITH_OTEL_ENABLED", "value": {"stringValue": "true"}}, + {"key": "langsmith.metadata.LANGSMITH_OTEL_ONLY", "value": {"stringValue": "true"}}, + {"key": "langsmith.metadata.revision_id", "value": {"stringValue": "fixture-revision"}}, + {"key": "gen_ai.prompt", "value": {"stringValue": "{\"question\":\"QMARK7c1e what is the refund window?\"}"}}, + {"key": "gen_ai.completion", "value": {"stringValue": "{\"output\":\"Refunds are accepted for 30 days.\"}"}} + ], + "status": {"code": 1} + }, + { + "traceId": "6f1c2a9e0b3d4c5e8a7b9c0d1e2f3a4b", + "spanId": "b2c3d4e5f6071829", + "parentSpanId": "a1b2c3d4e5f60718", + "name": "lookup_policy", + "kind": 1, + "startTimeUnixNano": "1791187200010000000", + "endTimeUnixNano": "1791187200020000000", + "attributes": [ + {"key": "langsmith.span.kind", "value": {"stringValue": "tool"}}, + {"key": "langsmith.trace.name", "value": {"stringValue": "lookup_policy"}}, + {"key": "langsmith.trace.session_name", "value": {"stringValue": "default"}}, + {"key": "gen_ai.operation.name", "value": {"stringValue": "execute_tool"}}, + {"key": "gen_ai.system", "value": {"stringValue": "langchain"}}, + {"key": "gen_ai.tool.name", "value": {"stringValue": "lookup_policy"}}, + {"key": "langsmith.metadata.ls_method", "value": {"stringValue": "traceable"}}, + {"key": "langsmith.metadata.LANGSMITH_TRACING", "value": {"stringValue": "true"}}, + {"key": "langsmith.metadata.LANGSMITH_OTEL_ENABLED", "value": {"stringValue": "true"}}, + {"key": "langsmith.metadata.LANGSMITH_OTEL_ONLY", "value": {"stringValue": "true"}}, + {"key": "langsmith.metadata.revision_id", "value": {"stringValue": "fixture-revision"}}, + {"key": "gen_ai.prompt", "value": {"stringValue": "{\"topic\":\"refunds\"}"}}, + {"key": "gen_ai.completion", "value": {"stringValue": "{\"output\":\"Refunds are accepted within 30 days of purchase.\"}"}} + ], + "status": {"code": 1} + }, + { + "traceId": "6f1c2a9e0b3d4c5e8a7b9c0d1e2f3a4b", + "spanId": "c3d4e5f60718293a", + "parentSpanId": "a1b2c3d4e5f60718", + "name": "chat_model", + "kind": 1, + "startTimeUnixNano": "1791187200030000000", + "endTimeUnixNano": "1791187200240000000", + "attributes": [ + {"key": "langsmith.span.kind", "value": {"stringValue": "llm"}}, + {"key": "langsmith.trace.name", "value": {"stringValue": "chat_model"}}, + {"key": "langsmith.trace.session_name", "value": {"stringValue": "default"}}, + {"key": "gen_ai.operation.name", "value": {"stringValue": "chat"}}, + {"key": "gen_ai.system", "value": {"stringValue": "openai"}}, + {"key": "gen_ai.request.model", "value": {"stringValue": "gpt-4o-mini"}}, + {"key": "langsmith.metadata.ls_method", "value": {"stringValue": "traceable"}}, + {"key": "langsmith.metadata.ls_provider", "value": {"stringValue": "openai"}}, + {"key": "langsmith.metadata.ls_model_name", "value": {"stringValue": "gpt-4o-mini"}}, + {"key": "langsmith.metadata.usage_metadata", "value": {"stringValue": "{\"input_tokens\":21,\"output_tokens\":9,\"total_tokens\":30}"}}, + {"key": "langsmith.metadata.LANGSMITH_TRACING", "value": {"stringValue": "true"}}, + {"key": "langsmith.metadata.LANGSMITH_OTEL_ENABLED", "value": {"stringValue": "true"}}, + {"key": "langsmith.metadata.LANGSMITH_OTEL_ONLY", "value": {"stringValue": "true"}}, + {"key": "langsmith.metadata.revision_id", "value": {"stringValue": "fixture-revision"}}, + {"key": "gen_ai.serialized.name", "value": {"stringValue": "chat_model"}}, + {"key": "gen_ai.prompt", "value": {"stringValue": "{\"messages\":[{\"role\":\"system\",\"content\":\"Answer from this policy: Refunds are accepted within 30 days of purchase.\"},{\"role\":\"user\",\"content\":\"QMARK7c1e what is the refund window?\"}]}"}}, + {"key": "gen_ai.usage.input_tokens", "value": {"intValue": "21"}}, + {"key": "gen_ai.usage.output_tokens", "value": {"intValue": "9"}}, + {"key": "gen_ai.usage.total_tokens", "value": {"intValue": "30"}}, + {"key": "gen_ai.response.finish_reasons", "value": {"arrayValue": {"values": [{"stringValue": "stop"}]}}}, + {"key": "gen_ai.completion", "value": {"stringValue": "{\"choices\":[{\"message\":{\"role\":\"assistant\",\"content\":\"Refunds are accepted for 30 days.\"},\"finish_reason\":\"stop\"}],\"usage_metadata\":{\"input_tokens\":21,\"output_tokens\":9,\"total_tokens\":30}}"}} + ], + "status": {"code": 1} + } + ] + } + ] + } + ] +} diff --git a/python/examples/langsmith/tests/sdk_own_provider.py b/python/examples/langsmith/tests/sdk_own_provider.py new file mode 100644 index 000000000..4a997bfdb --- /dev/null +++ b/python/examples/langsmith/tests/sdk_own_provider.py @@ -0,0 +1,26 @@ +"""A test fixture, not part of the recipe: the recipe's traced functions +exported through LangSmith's own tracer provider instead of ``init_tracing()``. + +The contract test runs it to show what that provider puts on the wire: a +resource without ``project_name`` even when ``OTEL_RESOURCE_ATTRIBUTES`` sets +one, and, without ``OTEL_EXPORTER_OTLP_HEADERS``, LangSmith's own +``x-api-key`` and ``Langsmith-Project`` headers. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src")) + +import langsmith # noqa: E402 +from app import support_request # noqa: E402 +from opentelemetry import trace # noqa: E402 + +# No global provider exists yet, so the client installs LangSmith's own. +client = langsmith.Client() +langsmith.configure(client=client) +print(support_request(sys.argv[1])) +client.flush() +trace.get_tracer_provider().shutdown() diff --git a/python/examples/langsmith/tests/test_langsmith_recipe.py b/python/examples/langsmith/tests/test_langsmith_recipe.py new file mode 100644 index 000000000..f97f9ace1 --- /dev/null +++ b/python/examples/langsmith/tests/test_langsmith_recipe.py @@ -0,0 +1,744 @@ +"""Contract test for the LangSmith trace-forwarding recipe. + +The two halves are tested separately: + +* Collector half (no LangSmith install needed). ``fixtures/`` holds a + hand-built OTLP body of a LangSmith run tree (a chain run with a tool child + and an LLM child), written from the langsmith 0.14.4 OTEL exporter source. + It is posted with the harness ``post_otlp()`` and checked with ``compare()``. +* SDK half (needs langsmith 0.14.4). ``src/app.py`` runs as written, in a + subprocess whose network is limited to 127.0.0.1 (``_guarded_run.py``), in + LangSmith's OTEL-only mode, with no LangSmith key. Its export must match the + fixture. + +Spans go to the shared harness ``Receiver``, which serves ``/v1/traces`` and +``/tracer/v1/traces`` like fi-collector's HTTP mux but does not authenticate, +stamp projects or store anything. Where LangSmith's REST API is needed, a +loopback stub stands in for it. Nothing here contacts LangSmith or Future AGI, +and no LangSmith key is set anywhere: every child environment is built from +scratch with a throwaway ``HOME``. +""" + +from __future__ import annotations + +import copy +import importlib.util +import json +import os +import re +import shlex +import sys +import threading +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from importlib.metadata import version +from pathlib import Path +from typing import Any, Callable, Optional +from urllib.parse import quote, urlsplit + +import pytest + +from harness import Receiver, compare, post_otlp, run + +TESTS_DIR = Path(__file__).resolve().parent +RECIPE_DIR = TESTS_DIR.parent +APP = RECIPE_DIR / "src" / "app.py" +README = RECIPE_DIR / "README.md" +FIXTURE = TESTS_DIR / "fixtures" / "langsmith-0.14.4-run-tree.otlp.json" +SDK_OWN_PROVIDER = TESTS_DIR / "sdk_own_provider.py" +GUARD = TESTS_DIR / "_guarded_run.py" +GUARD_PROBE = TESTS_DIR / "_guard_probe.py" + +PINNED_LANGSMITH = "0.14.4" +HAS_LANGSMITH = importlib.util.find_spec("langsmith") is not None +needs_sdk = pytest.mark.skipif(not HAS_LANGSMITH, reason="langsmith is not installed") + +LANGSMITH_KEY_VARS = ("LANGSMITH_API_KEY", "LANGCHAIN_API_KEY") +PROJECT = "langsmith-recipe-contract" +FI_API_KEY = "fi-api-placeholder-0000" +# A comma and an equals sign: the recipe's percent-encoding must carry them. +FI_SECRET_KEY = "fi-secret-placeholder,part=0000" +# LangSmith exports git describe of the working directory as revision_id +# unless this is set; the tests pin it so the export is reproducible. +REVISION = "fixture-revision" + +QUESTION = "QMARK7c1e what is the refund window?" +ANSWER = "Refunds are accepted for 30 days." +POLICY = "Refunds are accepted within 30 days of purchase." +CONTENT_MARKERS = ("QMARK7c1e", ANSWER, POLICY) + +ROOT, TOOL, LLM = "support_request", "lookup_policy", "chat_model" +SDK_KINDS = {ROOT: "chain", TOOL: "tool", LLM: "llm"} +OPERATIONS = {ROOT: "chain", TOOL: "execute_tool", LLM: "chat"} +# The four keys fi-collector reads a span kind from before it falls back to +# gen_ai.operation.name (fi-collector/exporter/clickhouse25exporter/converter.go). +SPAN_KIND_KEYS = ("fi.span.kind", "gen_ai.span.kind", "llm.request.type", "openinference.span.kind") +# OTLP/JSON writes enums as integers; protobuf decoded by MessageToDict uses names. +STATUS_CODES = {"STATUS_CODE_UNSET": 0, "STATUS_CODE_OK": 1, "STATUS_CODE_ERROR": 2} +SPAN_KINDS = { + "SPAN_KIND_UNSPECIFIED": 0, + "SPAN_KIND_INTERNAL": 1, + "SPAN_KIND_SERVER": 2, + "SPAN_KIND_CLIENT": 3, + "SPAN_KIND_PRODUCER": 4, + "SPAN_KIND_CONSUMER": 5, +} +# A hang guard, not a speed check. +RUN_TIMEOUT_SECONDS = 120 + + +def otlp_headers(api_key: str, secret_key: str) -> str: + """Build OTEL_EXPORTER_OTLP_HEADERS exactly as README.md tells users to.""" + return "x-api-key={0},x-secret-key={1}".format( + quote(api_key, safe=""), quote(secret_key, safe="") + ) + + +# -------------------------------------------------------------------------- +# Span helpers +# -------------------------------------------------------------------------- + + +def _value(value: dict[str, Any]) -> Any: + for kind in ("stringValue", "boolValue", "doubleValue"): + if kind in value: + return value[kind] + if "intValue" in value: + return int(value["intValue"]) + if "arrayValue" in value: + return [_value(item) for item in value["arrayValue"].get("values", [])] + return value + + +def _attrs(span: dict[str, Any]) -> dict[str, Any]: + return {item["key"]: _value(item.get("value", {})) for item in span.get("attributes", [])} + + +def _by_name(spans: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: + named = {span["name"]: span for span in spans} + assert len(named) == len(spans), sorted(span["name"] for span in spans) + return named + + +def _as_otlp_json(spans: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Protobuf-decoded spans with enums as OTLP/JSON integers, sorted by name.""" + converted = copy.deepcopy(spans) + for span in converted: + status = span.get("status", {}) + if isinstance(status.get("code"), str): + status["code"] = STATUS_CODES[status["code"]] + if isinstance(span.get("kind"), str): + span["kind"] = SPAN_KINDS[span["kind"]] + return sorted(converted, key=lambda span: span["name"]) + + +def _write_golden(directory: Path, spans: list[dict[str, Any]]) -> Path: + golden = directory / "spans.golden.json" + golden.write_text(json.dumps(sorted(spans, key=lambda s: s["name"]), indent=1), encoding="utf-8") + return golden + + +def _assert_run_tree(spans: list[dict[str, Any]]) -> None: + """One trace: the chain run is the root, the tool and LLM runs its children.""" + named = _by_name(spans) + assert set(named) == {ROOT, TOOL, LLM} + assert len({span["traceId"] for span in spans}) == 1 + root = named[ROOT] + assert not root.get("parentSpanId") + for child in (TOOL, LLM): + assert named[child]["parentSpanId"] == root["spanId"], child + for name, span in named.items(): + attrs = _attrs(span) + assert attrs.get("langsmith.span.kind") == SDK_KINDS[name], name + assert attrs.get("gen_ai.operation.name") == OPERATIONS[name], name + assert not set(SPAN_KIND_KEYS) & set(attrs), name + assert _as_otlp_json([span])[0]["status"] == {"code": 1}, name + + +def _readme_span_rows() -> dict[str, tuple[str, str, str, str]]: + text = README.read_text(encoding="utf-8") + rows = re.findall( + r"^\| `(\w+)` \| (none|`\w+`) \| `(\w+)` \| `(\w+)` \| (\w+) \|$", text, flags=re.MULTILINE + ) + return {name: (parent.strip("`"), kind, op, fi_type) for name, parent, kind, op, fi_type in rows} + + +def _readme_run_type_rows() -> dict[str, tuple[str, str]]: + text = README.read_text(encoding="utf-8") + rows = re.findall(r"^\| `(\w+)` \| `(\w+)` \| (\w+) \|$", text, flags=re.MULTILINE) + return {run_type: (op, fi_type) for run_type, op, fi_type in rows} + + +# -------------------------------------------------------------------------- +# The rule: no LangSmith key anywhere. +# -------------------------------------------------------------------------- + + +def test_no_langsmith_key_in_the_environment(tmp_path: Path) -> None: + for name in LANGSMITH_KEY_VARS: + assert name not in os.environ, "unset {0}: a LangSmith key is a spend hard stop".format(name) + env = _child_env(tmp_path, tmp_path / "guard.jsonl", "http://127.0.0.1:9") + assert not [name for name in env if name in LANGSMITH_KEY_VARS] + # A LangSmith CLI profile can also supply a key; the child HOME has none. + assert not (Path(env["HOME"]) / ".langsmith").exists() + assert "LANGSMITH_CONFIG_FILE" not in env + + +# -------------------------------------------------------------------------- +# Collector half: the hand-built fixture, no LangSmith needed. +# -------------------------------------------------------------------------- + + +@pytest.fixture(scope="module") +def fixture_body() -> dict[str, Any]: + return json.loads(FIXTURE.read_text(encoding="utf-8")) + + +@pytest.fixture(scope="module") +def fixture_spans(fixture_body: dict[str, Any]) -> list[dict[str, Any]]: + [resource_spans] = fixture_body["resourceSpans"] + [scope_spans] = resource_spans["scopeSpans"] + assert scope_spans["scope"] == {"name": "langsmith"} + return scope_spans["spans"] + + +def test_fixture_round_trips_through_the_harness( + fixture_body: dict[str, Any], fixture_spans: list[dict[str, Any]], tmp_path: Path +) -> None: + golden = _write_golden(tmp_path, fixture_spans) + with Receiver() as receiver: + assert post_otlp(fixture_body, receiver.collector_endpoint) == 200 + received = receiver.spans() + requests = receiver.requests() + assert len(received) == 3 + compare(sorted(received, key=lambda s: s["name"]), golden) + [request] = requests + assert request["path"] == "/tracer/v1/traces" + assert request["resource_attributes"] == [{"project_name": PROJECT, "project_type": "observe"}] + _assert_run_tree(received) + + +def test_fixture_is_valid_otlp_json(fixture_spans: list[dict[str, Any]]) -> None: + """OTLP/JSON: hex trace and span ids, integer enums, int64 as strings.""" + for span in fixture_spans: + assert re.fullmatch(r"[0-9a-f]{32}", span["traceId"]), span["name"] + assert re.fullmatch(r"[0-9a-f]{16}", span["spanId"]), span["name"] + assert re.fullmatch(r"[0-9a-f]{16}", span.get("parentSpanId", "0" * 16)), span["name"] + assert span["kind"] == 1 and span["status"] == {"code": 1}, span["name"] + assert int(span["startTimeUnixNano"]) < int(span["endTimeUnixNano"]), span["name"] + for item in span["attributes"]: + if "intValue" in item["value"]: + assert isinstance(item["value"]["intValue"], str), item["key"] + + +def test_fixture_is_one_trace_with_a_tool_and_an_llm_child( + fixture_spans: list[dict[str, Any]], +) -> None: + _assert_run_tree(fixture_spans) + + +def test_fixture_carries_the_model_provider_and_token_keys( + fixture_spans: list[dict[str, Any]], +) -> None: + named = {name: _attrs(span) for name, span in _by_name(fixture_spans).items()} + llm = named[LLM] + assert llm["gen_ai.request.model"] == "gpt-4o-mini" + assert llm["gen_ai.system"] == "openai" + assert ( + llm["gen_ai.usage.input_tokens"], + llm["gen_ai.usage.output_tokens"], + llm["gen_ai.usage.total_tokens"], + ) == (21, 9, 30) + assert named[TOOL]["gen_ai.tool.name"] == TOOL + # LangSmith's default system for runs without a model name. + assert named[ROOT]["gen_ai.system"] == named[TOOL]["gen_ai.system"] == "langchain" + for name in (ROOT, TOOL): + assert not [key for key in named[name] if key.startswith("gen_ai.usage.")], name + assert "gen_ai.request.model" not in named[name], name + + +def test_fixture_carries_content_and_no_secret(fixture_body: dict[str, Any]) -> None: + dump = json.dumps(fixture_body) + for marker in CONTENT_MARKERS: + assert marker in dump, marker + for secret in (FI_API_KEY, FI_SECRET_KEY): + assert secret not in dump + + +def test_readme_span_table_matches_the_fixture(fixture_spans: list[dict[str, Any]]) -> None: + rows = _readme_span_rows() + named = _by_name(fixture_spans) + assert set(rows) == set(named) + by_id = {span["spanId"]: name for name, span in named.items()} + for name, (parent, kind, operation, _fi_type) in rows.items(): + attrs = _attrs(named[name]) + assert parent == by_id.get(named[name].get("parentSpanId", ""), "none"), name + assert (kind, operation) == (attrs["langsmith.span.kind"], attrs["gen_ai.operation.name"]), name + + +# -------------------------------------------------------------------------- +# SDK half: src/app.py with langsmith 0.14.4, loopback only, no key. +# -------------------------------------------------------------------------- + + +class _LangSmithStub: + """Loopback stand-in for LangSmith's REST API: answers 200 ``{}`` to anything + and records each request's method and path.""" + + def __init__(self) -> None: + self.calls: list[tuple[str, str]] = [] + lock = threading.Lock() + owner = self + + class Handler(BaseHTTPRequestHandler): + def _answer(self) -> None: + length = int(self.headers.get("Content-Length", "0") or 0) + if length: + self.rfile.read(length) + with lock: + owner.calls.append((self.command, urlsplit(self.path).path)) + body = b"{}" + self.send_response(HTTPStatus.OK) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + do_GET = do_POST = do_PATCH = _answer # noqa: N815 + + def log_message(self, format: str, *args: Any) -> None: # noqa: A002 + return + + self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.origin = "http://127.0.0.1:{0}".format(self._server.server_port) + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + self._thread.start() + + def __enter__(self) -> "_LangSmithStub": + return self + + def __exit__(self, *_exc: Any) -> None: + self._server.shutdown() + self._server.server_close() + self._thread.join() + + +def _child_env(home: Path, guard_log: Path, otlp_origin: str, **overrides: Optional[str]) -> dict[str, str]: + """The recipe's environment, built from scratch (nothing inherited but PATH).""" + env = { + "PATH": os.environ.get("PATH", ""), + "HOME": str(home), + "LOOPBACK_GUARD_LOG": str(guard_log), + "LANGSMITH_TRACING": "true", + "LANGSMITH_OTEL_ENABLED": "true", + "LANGSMITH_OTEL_ONLY": "true", + "LANGCHAIN_REVISION_ID": REVISION, + "OTEL_EXPORTER_OTLP_ENDPOINT": otlp_origin, + "OTEL_EXPORTER_OTLP_HEADERS": otlp_headers(FI_API_KEY, FI_SECRET_KEY), + "FI_PROJECT_NAME": PROJECT, + } + for key, value in overrides.items(): + if value is None: + env.pop(key, None) + else: + env[key] = value + return env + + +def _run_script( + tmp_path: Path, + script: Path, + langsmith_endpoint: bool = False, + receiver_env: Optional[Callable[[Receiver], dict[str, Optional[str]]]] = None, + **overrides: Optional[str], +) -> dict[str, Any]: + """Run one script under the loopback guard; return everything the tests read. + + With ``langsmith_endpoint`` the run's LANGSMITH_ENDPOINT is a loopback + stub, so a REST call to LangSmith is recorded instead of refused. + ``receiver_env`` adds overrides that need the receiver's address. + """ + tmp_path.mkdir(parents=True, exist_ok=True) + guard_log = tmp_path / "guard.jsonl" + with Receiver() as receiver, _LangSmithStub() as stub: + if langsmith_endpoint: + overrides["LANGSMITH_ENDPOINT"] = stub.origin + if receiver_env is not None: + overrides.update(receiver_env(receiver)) + env = _child_env(tmp_path, guard_log, receiver.origin, **overrides) + assert not [name for name in env if name in LANGSMITH_KEY_VARS] + result = run( + [sys.executable, str(GUARD), str(script), QUESTION], + env=env, + stdin=None, + timeout=RUN_TIMEOUT_SECONDS, + ) + record = { + "result": result, + "stdout": result.stdout.decode("utf-8", "replace"), + "stderr": result.stderr.decode("utf-8", "replace"), + "requests": receiver.requests(), + "spans": receiver.spans(), + "langsmith_calls": list(stub.calls), + } + record["guard_attempts"] = ( + [json.loads(line) for line in guard_log.read_text().splitlines()] + if guard_log.exists() + else [] + ) + return record + + +def _assert_ran(record: dict[str, Any], exported: bool = True) -> None: + result = record["result"] + assert not result.timed_out, record["stderr"] + assert result.returncode == 0, record["stderr"] + assert record["guard_attempts"] == [], record["guard_attempts"] + assert ANSWER in record["stdout"] + if exported: + assert record["requests"], "no export reached the receiver" + assert len(record["spans"]) == 3, [span["name"] for span in record["spans"]] + + +def _export_dump(record: dict[str, Any]) -> str: + """Everything the export carried except HTTP headers: spans and resources.""" + return json.dumps( + {"spans": record["spans"], "resources": [r["resource_attributes"] for r in record["requests"]]}, + sort_keys=True, + ) + + +def test_guard_refuses_and_logs_non_loopback_connections(tmp_path: Path) -> None: + """Positive control for every ``guard_attempts == []`` assertion in this file.""" + guard_log = tmp_path / "guard.jsonl" + result = run( + [sys.executable, str(GUARD), str(GUARD_PROBE)], + env={"PATH": os.environ.get("PATH", ""), "HOME": str(tmp_path), "LOOPBACK_GUARD_LOG": str(guard_log)}, + stdin=None, + timeout=60, + ) + assert not result.timed_out + assert result.returncode == 0, result.stderr.decode("utf-8", "replace") + assert "refused: connect,connect_ex,getaddrinfo" in result.stdout.decode("utf-8", "replace") + logged = [json.loads(line) for line in guard_log.read_text().splitlines()] + assert [entry["kind"] for entry in logged] == ["connect", "connect_ex", "getaddrinfo"] + assert "192.0.2.1" in logged[0]["target"] + assert "2001:db8::1" in logged[1]["target"] + assert "example.invalid" in logged[2]["target"] + + +@needs_sdk +def test_installed_langsmith_is_the_pinned_version() -> None: + # LANGSMITH_TEST_LATEST=1 runs the whole suite against a newer langsmith + # (the "latest stable" matrix leg) instead of the pinned 0.14.4. + if os.environ.get("LANGSMITH_TEST_LATEST") == "1": + pytest.skip("running against a newer langsmith on purpose") + assert version("langsmith") == PINNED_LANGSMITH + + +@pytest.fixture(scope="module") +def recipe_run(tmp_path_factory: pytest.TempPathFactory) -> dict[str, Any]: + if not HAS_LANGSMITH: + pytest.skip("langsmith is not installed") + record = _run_script(tmp_path_factory.mktemp("recipe"), APP) + _assert_ran(record) + return record + + +@needs_sdk +def test_recipe_runs_with_loopback_network_only(recipe_run: dict[str, Any]) -> None: + # LANGSMITH_ENDPOINT is unset, so any LangSmith call would have gone to + # api.smith.langchain.com and been refused and logged by the guard. + assert recipe_run["guard_attempts"] == [] + assert recipe_run["langsmith_calls"] == [] + + +@needs_sdk +def test_sdk_export_matches_the_fixture( + recipe_run: dict[str, Any], fixture_spans: list[dict[str, Any]], tmp_path: Path +) -> None: + golden = _write_golden(tmp_path, fixture_spans) + compare(_as_otlp_json(recipe_run["spans"]), golden) + _assert_run_tree(recipe_run["spans"]) + for span in _as_otlp_json(recipe_run["spans"]): + assert span["kind"] == 1, span["name"] + assert not span.get("events"), span["name"] + + +@needs_sdk +def test_export_path_headers_and_resource(recipe_run: dict[str, Any]) -> None: + assert recipe_run["requests"], "no export reached the receiver" + for request in recipe_run["requests"]: + assert request["path"] == "/v1/traces" + headers = request["headers"] + assert headers.get("x-api-key") == FI_API_KEY + assert headers.get("x-secret-key") == FI_SECRET_KEY + assert "authorization" not in headers + assert "langsmith-project" not in headers + assert request["resource_attributes"], "export without a resource" + for resource in request["resource_attributes"]: + assert resource.get("project_name") == PROJECT + assert resource.get("project_type") == "observe" + + +@needs_sdk +def test_content_is_exported_by_default(recipe_run: dict[str, Any]) -> None: + """LangSmith's default: inputs and outputs on every span. Control for the hide test.""" + named = {name: _attrs(span) for name, span in _by_name(recipe_run["spans"]).items()} + assert "QMARK7c1e" in named[ROOT]["gen_ai.prompt"] + assert ANSWER in named[ROOT]["gen_ai.completion"] + assert POLICY in named[TOOL]["gen_ai.completion"] + assert POLICY in named[LLM]["gen_ai.prompt"] # the system prompt + assert "QMARK7c1e" in named[LLM]["gen_ai.prompt"] + assert ANSWER in named[LLM]["gen_ai.completion"] + + +@needs_sdk +def test_no_secret_in_the_export_or_output(recipe_run: dict[str, Any]) -> None: + dump = _export_dump(recipe_run) + output = recipe_run["stdout"] + recipe_run["stderr"] + for secret in (FI_API_KEY, FI_SECRET_KEY, quote(FI_SECRET_KEY, safe="")): + assert secret not in dump + assert secret not in output + + +@needs_sdk +@pytest.mark.parametrize( + ("mode", "overrides", "expected_calls"), + [ + # The recipe: OTEL only. The LangSmith endpoint is reachable and unused. + ("otel_only", {}, set()), + # The same mode under its 0.14.4 name. + ( + "tracing_mode_otel", + {"LANGSMITH_TRACING_MODE": "otel", "LANGSMITH_OTEL_ENABLED": None, "LANGSMITH_OTEL_ONLY": None}, + set(), + ), + # The official page's two flags: hybrid, LangSmith's REST API as well. + ("hybrid", {"LANGSMITH_OTEL_ONLY": None}, {("GET", "/info"), ("POST", "/runs/multipart")}), + ], +) +def test_which_modes_call_langsmith( + tmp_path: Path, mode: str, overrides: dict[str, Optional[str]], expected_calls: set[tuple[str, str]] +) -> None: + record = _run_script(tmp_path, APP, langsmith_endpoint=True, **overrides) + _assert_ran(record) + assert set(record["langsmith_calls"]) == expected_calls, mode + _assert_run_tree(record["spans"]) + + +@needs_sdk +def test_tracing_unset_exports_nothing(tmp_path: Path) -> None: + """LANGSMITH_TRACING is the on switch; the OTEL variables alone trace nothing.""" + record = _run_script(tmp_path, APP, LANGSMITH_TRACING=None) + _assert_ran(record, exported=False) + assert record["requests"] == [] + assert record["spans"] == [] + + +@needs_sdk +def test_hide_inputs_and_outputs_drop_content_and_token_counts(tmp_path: Path) -> None: + record = _run_script(tmp_path, APP, LANGSMITH_HIDE_INPUTS="true", LANGSMITH_HIDE_OUTPUTS="true") + _assert_ran(record) + dump = _export_dump(record) + for marker in CONTENT_MARKERS: + assert marker not in dump, marker + named = {name: _attrs(span) for name, span in _by_name(record["spans"]).items()} + for name, attrs in named.items(): + assert attrs["gen_ai.prompt"] == "{}", name + assert attrs["gen_ai.completion"] == "{}", name + llm = named[LLM] + # The token counts come from the outputs, so they go with them... + assert not [key for key in llm if key.startswith("gen_ai.usage.")] + assert "gen_ai.response.finish_reasons" not in llm + # ...while the model name (metadata) and the metadata copy of the usage stay. + assert llm["gen_ai.request.model"] == "gpt-4o-mini" + assert json.loads(llm["langsmith.metadata.usage_metadata"])["total_tokens"] == 30 + + +@needs_sdk +def test_traces_endpoint_is_used_as_given(tmp_path: Path) -> None: + """OTEL_EXPORTER_OTLP_TRACES_ENDPOINT is a full URL: fi-collector's /tracer path works.""" + record = _run_script( + tmp_path, + APP, + receiver_env=lambda receiver: { + "OTEL_EXPORTER_OTLP_ENDPOINT": None, + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": receiver.collector_endpoint, + }, + ) + _assert_ran(record) + assert {request["path"] for request in record["requests"]} == {"/tracer/v1/traces"} + + +def _readme_otlp_setup(env: dict[str, str], origin: str) -> dict[str, Optional[str]]: + """Exercise the shipped Run block's OTLP setup with only dummy credentials. + + Do not replace the recipe with a test-only environment reset. Both stale + signal-specific values are present before these exact shell lines run. + No pip, app, real endpoint or user-shell startup file is executed here. + """ + block = README.read_text(encoding="utf-8").split("## Run\n", 1)[1].split("```bash\n", 1)[1].split("```", 1)[0] + lines = [ + line for line in block.splitlines() + if line.startswith("export OTEL_EXPORTER_OTLP_") + or line.startswith("unset OTEL_EXPORTER_OTLP_") + ] + setup = "\n".join(lines).replace("https://YOUR_FI_COLLECTOR_ORIGIN", origin) + setup = setup.replace("python -c", shlex.quote(sys.executable) + " -c") + names = ( + "OTEL_EXPORTER_OTLP_ENDPOINT", "OTEL_EXPORTER_OTLP_HEADERS", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "OTEL_EXPORTER_OTLP_TRACES_HEADERS", + ) + # Serialize ONLY the four dummy test variables, never the ambient environment. + emit = "import json,os;print(json.dumps({k:os.environ.get(k) for k in " + repr(names) + "}))" + result = run( + ["/bin/bash", "--noprofile", "--norc", "-c", "set -eu\n" + setup + "\n" + shlex.quote(sys.executable) + " -c " + shlex.quote(emit)], + env={**env, "FI_API_KEY": FI_API_KEY, "FI_SECRET_KEY": FI_SECRET_KEY}, + stdin=None, timeout=20, + ) + assert not result.timed_out + assert result.returncode == 0, result.stderr.decode("utf-8", "replace") + return json.loads(result.stdout) + + +@needs_sdk +@pytest.mark.parametrize("stale", ["endpoint", "headers", "both"]) +def test_readme_setup_prevents_inherited_destination_and_key_crossover(tmp_path: Path, stale: str) -> None: + """The real exporter must honor the copied Run setup, not stale vendor config.""" + guard_log = tmp_path / "guard.jsonl" + with Receiver() as intended, Receiver() as other_vendor: + env = _child_env(tmp_path, guard_log, intended.origin) + if stale in ("endpoint", "both"): + env["OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"] = other_vendor.collector_endpoint + if stale in ("headers", "both"): + env["OTEL_EXPORTER_OTLP_TRACES_HEADERS"] = "x-api-key=other-vendor-placeholder" + setup = _readme_otlp_setup(env, intended.origin) + for name, value in setup.items(): + if value is None: + env.pop(name, None) + else: + env[name] = value + result = run( + [sys.executable, str(GUARD), str(APP), QUESTION], + env=env, stdin=None, timeout=RUN_TIMEOUT_SECONDS, + ) + assert not result.timed_out + assert result.returncode == 0, result.stderr.decode("utf-8", "replace") + # The unsafe old README delivers actual spans/keys to this wrong receiver. + assert other_vendor.requests() == [], "stale trace endpoint received the FI export" + requests = intended.requests() + assert requests, "no export reached the intended FI destination" + assert len(intended.spans()) == 3 + for request in requests: + assert request["path"] == "/v1/traces" + assert request["headers"].get("x-api-key") == FI_API_KEY + assert request["headers"].get("x-secret-key") == FI_SECRET_KEY + assert "other-vendor-placeholder" not in json.dumps(request["headers"]) + assert not guard_log.exists(), "a non-loopback request was attempted" + + +@needs_sdk +def test_langsmith_own_provider_has_no_project_and_sends_its_own_headers(tmp_path: Path) -> None: + """Why init_tracing() exists: without it, fi-collector would reject the batch.""" + record = _run_script( + tmp_path, + SDK_OWN_PROVIDER, + OTEL_RESOURCE_ATTRIBUTES="project_name={0},project_type=observe".format(PROJECT), + OTEL_EXPORTER_OTLP_HEADERS=None, + ) + _assert_ran(record) + _assert_run_tree(record["spans"]) + for request in record["requests"]: + assert request["path"] == "/v1/traces" + # OTEL_RESOURCE_ATTRIBUTES is not read; there is no project_name. + assert request["resource_attributes"] == [ + {"service.name": "langsmith", "langsmith.internal_provider": True} + ] + # LangSmith's key header (empty: no key is set) and project header. + assert request["headers"].get("x-api-key") == "" + assert request["headers"].get("langsmith-project") == "default" + assert "x-secret-key" not in request["headers"] + + +@needs_sdk +def test_readme_run_type_table_matches_the_sdk() -> None: + from typing import get_args + + from langsmith._internal.otel._otel_exporter import _get_operation_name + from langsmith.client import RUN_TYPE_T + + rows = _readme_run_type_rows() + assert set(rows) == set(get_args(RUN_TYPE_T)) + for run_type, (operation, _fi_type) in rows.items(): + assert operation == _get_operation_name(run_type), run_type + + +# -------------------------------------------------------------------------- +# Opt-in: read fi-collector's span-kind and alias rules instead of copying them. +# -------------------------------------------------------------------------- + + +def _go_block(text: str, name: str) -> str: + start = text.index(name + " = ") + opening = text.index("{\n", start) + closing = re.compile(r"\n\t?\}").search(text, opening) + assert closing is not None, name + return text[opening + 2 : closing.start()] + + +def _collector_rules() -> dict[str, Any]: + root = os.environ.get("FI_COLLECTOR_SRC") + if not root: + pytest.skip("set FI_COLLECTOR_SRC to a fi-collector checkout") + converter = (Path(root) / "exporter" / "clickhouse25exporter" / "converter.go").read_text(encoding="utf-8") + adapter = (Path(root) / "pkg" / "adapter" / "adapter.go").read_text(encoding="utf-8") + quoted = re.compile(r'"([^"]+)"') + rules: dict[str, Any] = { + "known": set(re.findall(r'"([^"]+)":\s*\{\}', _go_block(converter, "knownObservationTypes"))), + "kind_keys": quoted.findall(_go_block(converter, "spanKindAttrKeys")), + "operation_keys": quoted.findall(_go_block(converter, "operationNameAttrKeys")), + "synonyms": dict(re.findall(r'"([^"]+)":\s*"([^"]+)"', _go_block(converter, "spanKindSynonyms"))), + } + for name in ("modelNameKeys", "providerKeys", "inputTokenKeys", "outputTokenKeys", "totalTokenKeys"): + rules[name] = quoted.findall(_go_block(adapter, name)) + assert tuple(rules["kind_keys"]) == SPAN_KIND_KEYS + return rules + + +def _collector_type(attrs: dict[str, Any], rules: dict[str, Any]) -> str: + """resolveObservationType (converter.go), applied to rules read from the source.""" + raw = next((str(attrs[k]) for k in rules["kind_keys"] if attrs.get(k)), "") + if not raw: + raw = next((str(attrs[k]) for k in rules["operation_keys"] if attrs.get(k)), "") + kind = raw.strip().lower() + kind = rules["synonyms"].get(kind, kind) + return kind if kind in rules["known"] else "unknown" + + +def _first(attrs: dict[str, Any], keys: list[str]) -> Any: + return next((attrs[k] for k in keys if attrs.get(k) not in (None, "")), None) + + +def test_collector_types_and_hot_keys_for_the_fixture(fixture_spans: list[dict[str, Any]]) -> None: + rules = _collector_rules() + rows = _readme_span_rows() + for name, span in _by_name(fixture_spans).items(): + attrs = _attrs(span) + assert _collector_type(attrs, rules) == rows[name][3] == SDK_KINDS[name], name + llm = _attrs(_by_name(fixture_spans)[LLM]) + assert _first(llm, rules["modelNameKeys"]) == "gpt-4o-mini" + assert _first(llm, rules["providerKeys"]) == "openai" + assert _first(llm, rules["inputTokenKeys"]) == 21 + assert _first(llm, rules["outputTokenKeys"]) == 9 + assert _first(llm, rules["totalTokenKeys"]) == 30 + for name in (ROOT, TOOL): + attrs = _attrs(_by_name(fixture_spans)[name]) + assert _first(attrs, rules["modelNameKeys"]) is None, name + assert _first(attrs, rules["providerKeys"]) == "langchain", name + + +def test_readme_run_type_table_matches_the_collector() -> None: + rules = _collector_rules() + for run_type, (operation, fi_type) in _readme_run_type_rows().items(): + assert _collector_type({"gen_ai.operation.name": operation}, rules) == fi_type, run_type diff --git a/python/tests/harness/__init__.py b/python/tests/harness/__init__.py new file mode 100644 index 000000000..cf79047f1 --- /dev/null +++ b/python/tests/harness/__init__.py @@ -0,0 +1,238 @@ +"""Small, loopback-only utilities for OTLP contract tests.""" + +from __future__ import annotations + +import copy +import difflib +import json +import subprocess +import threading +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any, Mapping, Optional, Sequence, Union +from urllib.parse import urlsplit +from urllib.request import Request, urlopen + + +class Receiver: + """An in-memory OTLP/HTTP receiver for tests. + + The receiver only listens on loopback and never forwards received spans. + """ + + def __init__(self) -> None: + self._spans: list[dict[str, Any]] = [] + self._requests: list[dict[str, Any]] = [] + self._lock = threading.Lock() + owner = self + + class Handler(BaseHTTPRequestHandler): + def _read_body(self) -> bytes: + # The Node OTLP exporter streams with Transfer-Encoding: chunked + # and sends no Content-Length, so both framings are accepted. + if "chunked" in self.headers.get("Transfer-Encoding", "").lower(): + chunks = [] + while True: + size_line = self.rfile.readline().split(b";", 1)[0].strip() + size = int(size_line, 16) + if size == 0: + while self.rfile.readline() not in (b"\r\n", b"\n", b""): + pass + return b"".join(chunks) + chunks.append(self.rfile.read(size)) + self.rfile.readline() + length = int(self.headers.get("Content-Length", "0")) + return self.rfile.read(length) + + def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler + path = urlsplit(self.path).path + if path not in ("/v1/traces", "/tracer/v1/traces"): + self.send_error(HTTPStatus.NOT_FOUND) + return + + try: + body = self._read_body() + content_type = self.headers.get("Content-Type", "").lower() + + if content_type.startswith("application/json"): + request = json.loads(body.decode("utf-8")) + elif content_type.startswith(("application/x-protobuf", "application/protobuf")): + from google.protobuf.json_format import MessageToDict + from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ( + ExportTraceServiceRequest, + ) + + protobuf_request = ExportTraceServiceRequest() + protobuf_request.ParseFromString(body) + request = MessageToDict(protobuf_request) + else: + self.send_error(HTTPStatus.UNSUPPORTED_MEDIA_TYPE) + return + + spans: list[dict[str, Any]] = [] + resource_attributes: list[dict[str, Any]] = [] + for resource_spans in request.get( + "resourceSpans", request.get("resource_spans", []) + ): + resource_attributes.append( + _flatten_attributes( + resource_spans.get("resource", {}).get("attributes", []) + ) + ) + for scope_spans in resource_spans.get( + "scopeSpans", resource_spans.get("scope_spans", []) + ): + spans.extend(scope_spans.get("spans", [])) + except (ImportError, TypeError, UnicodeDecodeError, ValueError) as error: + self.send_error(HTTPStatus.BAD_REQUEST, str(error)) + return + + with owner._lock: + owner._spans.extend(copy.deepcopy(spans)) + owner._requests.append( + { + "path": path, + "headers": { + key.lower(): value for key, value in self.headers.items() + }, + "resource_attributes": resource_attributes, + } + ) + self.send_response(HTTPStatus.OK) + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, _format: str, *args: Any) -> None: + return + + self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.origin = "http://127.0.0.1:{0}".format(self._server.server_port) + self.endpoint = "{0}/v1/traces".format(self.origin) + self.collector_endpoint = "{0}/tracer/v1/traces".format(self.origin) + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + self._thread.start() + + def __enter__(self) -> "Receiver": + return self + + def __exit__(self, _type: Any, _value: Any, _traceback: Any) -> None: + self._server.shutdown() + self._server.server_close() + self._thread.join() + + def spans(self) -> list[dict[str, Any]]: + """Return a copy of the decoded spans received so far.""" + with self._lock: + return copy.deepcopy(self._spans) + + def requests(self) -> list[dict[str, Any]]: + """Return one record per accepted export: path, lower-cased headers, resource attributes.""" + with self._lock: + return copy.deepcopy(self._requests) + + def clear(self) -> None: + """Remove every decoded span and request record received so far.""" + with self._lock: + self._spans.clear() + self._requests.clear() + + +def _flatten_attributes(attributes: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + """Turn OTLP key/value attributes into a plain dict of scalar values.""" + flat: dict[str, Any] = {} + for attribute in attributes: + value = attribute.get("value", {}) + for kind in ("stringValue", "boolValue", "intValue", "doubleValue"): + if kind in value: + flat[attribute.get("key", "")] = value[kind] + break + else: + flat[attribute.get("key", "")] = value + return flat + + +def run( + argv: Sequence[str], + env: Mapping[str, str], + stdin: Optional[bytes], + timeout: float, +) -> subprocess.CompletedProcess[bytes]: + """Run one process and return its captured output, including after a timeout.""" + process = subprocess.Popen( + argv, + env=dict(env), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + timed_out = False + try: + stdout, stderr = process.communicate(input=stdin, timeout=timeout) + except subprocess.TimeoutExpired: + timed_out = True + process.kill() + stdout, stderr = process.communicate() + + result = subprocess.CompletedProcess(argv, process.returncode, stdout, stderr) + result.timed_out = timed_out + return result + + +def post_otlp(spans: dict[str, Any], endpoint: str) -> int: + """Post one JSON-encoded OTLP request to a loopback receiver.""" + parsed = urlsplit(endpoint) + if parsed.scheme != "http" or parsed.hostname != "127.0.0.1": + raise ValueError("post_otlp only sends to a loopback HTTP endpoint") + + request = Request( + endpoint, + data=json.dumps(spans).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urlopen(request, timeout=5) as response: + return response.status + + +def compare(actual: Sequence[dict[str, Any]], golden_path: Union[str, Path]) -> None: + """Fail with a diff unless span names, attributes, and statuses match a golden.""" + with Path(golden_path).open(encoding="utf-8") as golden_file: + expected = json.load(golden_file) + + def canonical(spans: Sequence[dict[str, Any]]) -> list[dict[str, Any]]: + normalized = [] + for span in spans: + attributes = span.get("attributes", []) + if isinstance(attributes, dict): + attributes = [ + {"key": key, "value": value} for key, value in attributes.items() + ] + normalized.append( + { + "name": span.get("name", ""), + "attributes": sorted( + attributes, + key=lambda attribute: json.dumps( + attribute, sort_keys=True, separators=(",", ":") + ), + ), + "status": span.get("status", {}), + } + ) + return normalized + + actual_normalized = canonical(actual) + expected_normalized = canonical(expected) + if actual_normalized != expected_normalized: + diff = "\n".join( + difflib.unified_diff( + json.dumps(expected_normalized, indent=2, sort_keys=True).splitlines(), + json.dumps(actual_normalized, indent=2, sort_keys=True).splitlines(), + fromfile=str(golden_path), + tofile="actual", + lineterm="", + ) + ) + print(diff) + raise AssertionError("spans do not match golden") diff --git a/python/tests/harness/test_harness.py b/python/tests/harness/test_harness.py new file mode 100644 index 000000000..bd61d5adc --- /dev/null +++ b/python/tests/harness/test_harness.py @@ -0,0 +1,189 @@ +"""Contract tests for the shared loopback-only harness.""" + +import json +import sys + +import pytest + +from tests.harness import Receiver, compare, post_otlp, run + + +def test_receiver_stores_posted_span_and_can_clear() -> None: + payload = { + "resourceSpans": [ + { + "scopeSpans": [ + { + "spans": [ + { + "traceId": "trace-id", + "spanId": "span-id", + "name": "harness.receiver", + "attributes": [ + {"key": "component", "value": {"stringValue": "test"}} + ], + "status": {"code": 1}, + } + ] + } + ] + } + ] + } + + with Receiver() as receiver: + assert post_otlp(payload, receiver.endpoint) == 200 + assert receiver.spans() == payload["resourceSpans"][0]["scopeSpans"][0]["spans"] + receiver.clear() + assert receiver.spans() == [] + + +def test_receiver_accepts_the_collector_path() -> None: + payload = { + "resourceSpans": [ + {"scopeSpans": [{"spans": [{"name": "collector.path"}]}]} + ] + } + + with Receiver() as receiver: + assert post_otlp(payload, receiver.collector_endpoint) == 200 + assert receiver.spans() == [{"name": "collector.path"}] + + +def _post_chunked(url: str, body: bytes, headers: dict) -> int: + """Send a body with Transfer-Encoding: chunked, as the Node OTLP exporter does.""" + import http.client + from urllib.parse import urlsplit + + parts = urlsplit(url) + connection = http.client.HTTPConnection(parts.hostname, parts.port, timeout=5) + try: + chunks = [body[: len(body) // 2], body[len(body) // 2 :]] + connection.request( + "POST", + parts.path, + body=iter(chunks), + headers={**headers, "Transfer-Encoding": "chunked"}, + encode_chunked=True, + ) + return connection.getresponse().status + finally: + connection.close() + + +def test_receiver_decodes_a_chunked_body() -> None: + payload = { + "resourceSpans": [ + {"scopeSpans": [{"spans": [{"name": "chunked.body"}]}]} + ] + } + + with Receiver() as receiver: + status = _post_chunked( + receiver.collector_endpoint, + json.dumps(payload).encode("utf-8"), + {"Content-Type": "application/json"}, + ) + assert status == 200 + assert receiver.spans() == [{"name": "chunked.body"}] + + +def test_receiver_records_path_headers_and_resource_attributes() -> None: + payload = { + "resourceSpans": [ + { + "resource": { + "attributes": [ + {"key": "project_name", "value": {"stringValue": "harness"}}, + {"key": "project_type", "value": {"stringValue": "observe"}}, + ] + }, + "scopeSpans": [{"spans": [{"name": "with.resource"}]}], + } + ] + } + + with Receiver() as receiver: + status = _post_chunked( + receiver.collector_endpoint, + json.dumps(payload).encode("utf-8"), + { + "Content-Type": "application/json", + "X-Api-Key": "test-key", + "X-Secret-Key": "test-secret", + }, + ) + assert status == 200 + [request] = receiver.requests() + assert request["path"] == "/tracer/v1/traces" + assert request["headers"]["x-api-key"] == "test-key" + assert request["headers"]["x-secret-key"] == "test-secret" + assert "authorization" not in request["headers"] + assert request["resource_attributes"] == [ + {"project_name": "harness", "project_type": "observe"} + ] + receiver.clear() + assert receiver.requests() == [] + + +def test_run_captures_output_and_kills_timed_out_process() -> None: + completed = run( + [ + sys.executable, + "-c", + "import sys; print('stdout'); sys.stderr.write('stderr\\n')", + ], + env={}, + stdin=b"", + timeout=1, + ) + assert completed.returncode == 0 + assert completed.stdout == b"stdout\n" + assert completed.stderr == b"stderr\n" + assert completed.timed_out is False + + timed_out = run( + [sys.executable, "-c", "import time; time.sleep(30)"], + env={}, + stdin=b"", + timeout=0.1, + ) + assert timed_out.timed_out is True + assert timed_out.returncode != 0 + + +def test_compare_accepts_normalized_match_and_prints_mismatch_diff(tmp_path, capsys) -> None: + golden = [ + { + "traceId": "golden-trace", + "spanId": "golden-span", + "startTimeUnixNano": "1", + "endTimeUnixNano": "2", + "name": "harness.compare", + "attributes": [ + {"key": "alpha", "value": {"stringValue": "a"}}, + {"key": "beta", "value": {"intValue": "2"}}, + ], + "status": {"code": 1}, + } + ] + golden_path = tmp_path / "spans.golden.json" + golden_path.write_text(json.dumps(golden), encoding="utf-8") + + actual = [ + { + "traceId": "other-trace", + "spanId": "other-span", + "startTimeUnixNano": "3", + "endTimeUnixNano": "4", + "name": "harness.compare", + "attributes": list(reversed(golden[0]["attributes"])), + "status": {"code": 1}, + } + ] + compare(actual, golden_path) + + actual[0]["name"] = "harness.mismatch" + with pytest.raises(AssertionError, match="spans do not match golden"): + compare(actual, golden_path) + assert "---" in capsys.readouterr().out