diff --git a/.github/assets/generate-coverage.py b/.github/assets/generate-coverage.py index 49155247e..6a15c57e7 100644 --- a/.github/assets/generate-coverage.py +++ b/.github/assets/generate-coverage.py @@ -30,7 +30,7 @@ "azure-openai": "Azure OpenAI", "azure-search": "Azure AI Search", "bedrock": "Amazon Bedrock", "beeai": "BeeAI", "cerebras": "Cerebras", "chromadb": "Chroma", "claude-agent-sdk": "Claude Agent SDK", "codex-sdk": "Codex SDK", - "cohere": "Cohere", "crewai": "CrewAI", "cursor-sdk": "Cursor SDK", + "cognee": "Cognee", "cohere": "Cohere", "crewai": "CrewAI", "cursor-sdk": "Cursor SDK", "deepseek": "DeepSeek", "discoveryengine": "Google Discovery Engine", "dspy": "DSPy", "eino": "Eino", "elasticsearch": "Elasticsearch", "elevenlabs-agents": "ElevenLabs Agents", "exa": "Exa", "firecrawl": "Firecrawl", "fireworks": "Fireworks AI", "genkit": "Genkit", @@ -69,7 +69,7 @@ "smolagents strands vercel voltagent"), ("rag-vector-databases", "RAG & vector databases", "Retrieval pipelines, vector stores, and search backends.", - "azure-search chromadb discoveryengine elasticsearch haystack lancedb llamaindex " + "azure-search chromadb cognee discoveryengine elasticsearch haystack lancedb llamaindex " "milvus mongodb pgvector pinecone qdrant redis weaviate"), ("tools-search", "Tools & search", "Tool connectivity, web search, crawling, and extraction.", diff --git a/.github/assets/integration-coverage.json b/.github/assets/integration-coverage.json index 089352d55..11886fa9b 100644 --- a/.github/assets/integration-coverage.json +++ b/.github/assets/integration-coverage.json @@ -1,5 +1,5 @@ { - "integration_families": 77, + "integration_families": 78, "languages": [ "Python", "TypeScript", @@ -7,7 +7,7 @@ "Go", "C#" ], - "method": "Union of normalized framework families from 65 Python framework manifests, 45 TypeScript instrumentor manifests, 23 Java integration modules and 5 Go module manifests. Runtime variants share a family; shared/core SDKs, BOMs, aggregators and examples do not count. C# has a manual-span core SDK and no framework auto-instrumentors.", + "method": "Union of normalized framework families from 66 Python framework manifests, 45 TypeScript instrumentor manifests, 23 Java integration modules and 5 Go module manifests. Runtime variants share a family; shared/core SDKs, BOMs, aggregators and examples do not count. C# has a manual-span core SDK and no framework auto-instrumentors.", "normalization": { "ag2-classic": "ag2", "google-genai-live": "google-genai", @@ -24,7 +24,7 @@ "fi-instrumentation-vertexai": "vertexai" }, "package_counts_in_matrix": { - "Python": 65, + "Python": 66, "TypeScript": 45, "Java": 23, "Go": 5, @@ -39,6 +39,10 @@ "manifest": "typescript/packages/fi-semantic-conventions/package.json", "reason": "shared SDK / semantic conventions" }, + { + "manifest": "java/traceai-bom/pom.xml", + "reason": "dependency-management / aggregator POM; not an integration" + }, { "manifest": "java/traceai-java-core/pom.xml", "reason": "shared SDK / BOM / example; not an integration" @@ -112,6 +116,7 @@ "families": [ "azure-search", "chromadb", + "cognee", "elasticsearch", "discoveryengine", "haystack", @@ -1067,6 +1072,20 @@ ] } }, + { + "id": "cognee", + "name": "Cognee", + "category": "rag-vector-databases", + "languages": { + "Python": [ + { + "manifest": "python/frameworks/cognee/pyproject.toml", + "package": "traceai-cognee", + "source": "python/frameworks/cognee/" + } + ] + } + }, { "id": "elasticsearch", "name": "Elasticsearch", diff --git a/.github/assets/test-generate-coverage.py b/.github/assets/test-generate-coverage.py index f7b7c50aa..40075c11b 100644 --- a/.github/assets/test-generate-coverage.py +++ b/.github/assets/test-generate-coverage.py @@ -70,7 +70,10 @@ def test_full_catalog_visible_once_with_language_links(self): section = readme.split("## Integrations\n", 1)[1].split("\n## ", 1)[0] self.assertNotRegex(section, r"(?i)<\s*/?\s*(?:details|summary)\b") self.assertIn(f'**{data["integration_families"]} integration families · 5 languages', readme) - self.assertEqual(data["integration_families"], 77) + self.assertEqual(data["integration_families"], 78) + cognee = next(family for family in data["families"] if family["id"] == "cognee") + self.assertEqual(cognee["category"], "rag-vector-databases") + self.assertEqual(cognee["languages"]["Python"][0]["package"], "traceai-cognee") self.assertEqual(len(data["categories"]), 9) keys = [key for category in data["categories"] for key in category["families"]] self.assertEqual(len(keys), len(set(keys))) diff --git a/.github/release/config.json b/.github/release/config.json index a3204edea..566951b5a 100644 --- a/.github/release/config.json +++ b/.github/release/config.json @@ -7,6 +7,7 @@ "_pypi_bootstrap_comment": "PyPI allows only three pending trusted publishers per organization, so these first-time projects are created by their first upload with the PYPI_BOOTSTRAP_TOKEN secret instead. Only a package in this list that has never been published uses the token. After the first release: add each project to the futureagi organization, add its trusted publisher (release.yml, blank environment), delete the token on PyPI and the secret, and empty this list.", "pypi_bootstrap_token": [ "traceAI-ag2", + "traceai-cognee", "traceAI-discoveryengine", "traceAI-elevenlabs-agents", "traceAI-exa", diff --git a/.release-please-manifest.json b/.release-please-manifest.json index abd32c28c..dc45f421c 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -12,6 +12,7 @@ "python/frameworks/chromadb": "0.1.1", "python/frameworks/claude-agent-sdk": "0.1.1", "python/frameworks/cohere": "0.1.1", + "python/frameworks/cognee": "0.1.0", "python/frameworks/crewai": "0.1.9", "python/frameworks/deepseek": "0.1.1", "python/frameworks/discoveryengine": "0.1.0", diff --git a/README.md b/README.md index 6e60999a7..33dea4b8f 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Trace LLM calls, agents, tools, and retrieval. Keep your application code and your observability stack. -**77 integration families · 5 languages · OpenTelemetry-native · Apache 2.0** +**78 integration families · 5 languages · OpenTelemetry-native · Apache 2.0** [Get started](#quickstart) · [Integrations](#integrations) · [Documentation](https://docs.futureagi.com/docs/tracing) · [Contribute](CONTRIBUTING.md) @@ -299,11 +299,11 @@ After a call finishes, open the `traceai-quickstart` project in **Observe** in F ## Integrations -**77 integration families, with language coverage you can inspect.** Each SDK link opens its package source. A dash means no dedicated framework package in this repository—not a promise of unsupported runtime behavior. +**78 integration families, with language coverage you can inspect.** Each SDK link opens its package source. A dash means no dedicated framework package in this repository—not a promise of unsupported runtime behavior. Python and TypeScript provide framework instrumentors; Java provides client wrappers and framework modules; Go provides middleware and framework integrations. **C# provides [manual spans and activities](csharp/src/FIInstrumentation/), not automatic framework instrumentation**, so its framework cells are intentionally empty. -All 77 families are listed below, grouped by what they integrate. Runtime variants share a row: OpenAI includes Realtime; Google GenAI includes Live; AG2 includes its classic SDK. LangGraph ships in the LangChain integration. Shared SDKs are not counted as integrations. Package availability is not a claim of complete upstream feature compatibility; check each package's source and documentation for its scope. +All 78 families are listed below, grouped by what they integrate. Runtime variants share a row: OpenAI includes Realtime; Google GenAI includes Live; AG2 includes its classic SDK. LangGraph ships in the LangChain integration. Shared SDKs are not counted as integrations. Package availability is not a claim of complete upstream feature compatibility; check each package's source and documentation for its scope. [Model providers](#model-providers) · [Agent frameworks](#agent-frameworks) · [RAG & vector databases](#rag-vector-databases) · [Tools & search](#tools-search) · [Voice & realtime](#voice-realtime) · [Gateways & proxies](#gateways-proxies) · [Coding agents](#coding-agents) · [Java frameworks](#java-frameworks) · [Other](#other) @@ -377,6 +377,7 @@ Retrieval pipelines, vector stores, and search backends. | :--- | :---: | :---: | :---: | :---: | :---: | | Azure AI Search | — | — | [SDK](java/traceai-java-azure-search/ "traceai-java-azure-search") | — | — | | Chroma | [SDK](python/frameworks/chromadb/ "traceAI-chromadb") | [SDK](typescript/packages/traceai_chromadb/ "@traceai/chromadb") | [SDK](java/traceai-java-chromadb/ "traceai-java-chromadb") | — | — | +| Cognee | [SDK](python/frameworks/cognee/ "traceai-cognee") | — | — | — | — | | Elasticsearch | — | — | [SDK](java/traceai-java-elasticsearch/ "traceai-java-elasticsearch") | — | — | | Google Discovery Engine | [SDK](python/frameworks/discoveryengine/ "traceAI-discoveryengine") | — | — | — | — | | Haystack | [SDK](python/frameworks/haystack/ "traceAI-haystack") | — | — | — | — | diff --git a/python/README.md b/python/README.md index 9302b17dd..7f6ddc490 100644 --- a/python/README.md +++ b/python/README.md @@ -96,6 +96,7 @@ Each package below links to its own installation instructions, supported SDK ver | `traceAI-cerebras` | [cerebras](frameworks/cerebras/) | | `traceAI-chromadb` | [chromadb](frameworks/chromadb/) | | `traceai-claude-agent-sdk` | [claude-agent-sdk](frameworks/claude-agent-sdk/) | +| `traceai-cognee` | [cognee](frameworks/cognee/) | | `traceAI-cohere` | [cohere](frameworks/cohere/) | | `traceAI-crewai` | [crewai](frameworks/crewai/) | | `traceAI-deepseek` | [deepseek](frameworks/deepseek/) | diff --git a/python/examples/cognee/README.md b/python/examples/cognee/README.md index a64026d25..06d6e6ca7 100644 --- a/python/examples/cognee/README.md +++ b/python/examples/cognee/README.md @@ -1,450 +1,26 @@ -# Cognee → Future AGI (recipe) +# Cognee example -There is no `traceai-cognee` package. Cognee creates its own OpenTelemetry -spans when `COGNEE_TRACING_ENABLED=true`; this recipe sends those spans to -Future AGI. It adds no instrumentor (a second one would double every span) -and never calls Cognee's `setup_tracing` (Cognee calls it on its first traced -operation). Option A adds one small export filter, -[`src/cognee_filter.py`](src/cognee_filter.py), that keeps Cognee's content, -and error text that can quote it, out of the export and marks search and LLM -spans. +Send Cognee's native add, cognify and search spans to Future AGI with the maintained `traceai-cognee` adapter. Do not add a second Cognee or LiteLLM instrumentor. -Pinned: `cognee==1.6.5` (Python >=3.10,<3.15), the latest release on -2026-10-16; `cognee==1.6.2` was tested too and behaves the same. Both wheels' -METADATA declare `License-Expression: Apache-2.0` (license files `LICENSE` -and `NOTICE.md`); cognee is a dependency of this example only, not of any -traceAI package. Option A also needs `fi-instrumentation-otel>=1.1.0,<1.2`, -which declares Python <3.14. The tests (see [Tests](#tests)) run on Python -3.10, 3.11 and 3.13 against loopback fakes, with no LLM key and no Future AGI -project. The recipe was also checked live on 2026-10-16, on Python 3.12, -against a local Future AGI stack with real OpenAI models (`gpt-4o-mini`, -`text-embedding-3-small`): option A with content off and on and a failing -model call (cognee 1.6.5 and 1.6.2), and option B (cognee 1.6.5) over gRPC to -the local collector and over HTTP with the `:443` URL below to Future AGI -cloud. Traces only: metrics and logs are not ingested by this recipe. -Statements marked "source reading" come from the cognee 1.6.5 code and are -not exercised by the tests. +## Run -## Option A: `register()` (recommended) +Use Python 3.10–3.13. Install the dependencies, then configure Cognee's model and embedding providers using the [Cognee documentation](https://docs.cognee.ai/). ```bash pip install -r requirements.txt - -export FI_API_KEY=... # sent as the X-Api-Key header -export FI_SECRET_KEY=... # sent as the X-Secret-Key header -export FI_PROJECT_NAME=my-cognee-app export COGNEE_TRACING_ENABLED=true -# Cognee's own model settings, unchanged by this recipe: -export LLM_API_KEY=... # plus LLM_MODEL, EMBEDDING_* as usual - +export FI_API_KEY="YOUR_FI_API_KEY" +export FI_SECRET_KEY="YOUR_FI_SECRET_KEY" +export FI_PROJECT_NAME="my-cognee-app" python src/app.py "Who works on Lighthouse?" ``` -`register()` posts OTLP/HTTP protobuf to `https://api.futureagi.com/tracer/v1/traces` -(override the origin with `FI_BASE_URL`) with the `X-Api-Key` and -`X-Secret-Key` headers, and stamps every batch with the resource attributes -`project_name` and `project_type=observe`. - -The whole integration is [`src/app.py`](src/app.py) plus -[`src/cognee_filter.py`](src/cognee_filter.py), which you copy next to it: - -```python -from fi_instrumentation import register -from fi_instrumentation.fi_types import ProjectType -from fi_instrumentation.otel import BatchSpanProcessor, HTTPSpanExporter - -from cognee_filter import CogneeExportFilter - -provider = register( - project_name=os.environ["FI_PROJECT_NAME"], - project_type=ProjectType.OBSERVE, - set_global_tracer_provider=True, # Cognee reuses the global provider - verbose=False, -) -exporter = CogneeExportFilter(HTTPSpanExporter()) -provider.add_span_processor(BatchSpanProcessor(exporter)) # required, see below - -with provider.get_tracer(__name__).start_as_current_span("remember_and_ask"): - asyncio.run(remember_and_ask(question, document)) # cognee.add / cognify / search -``` - -These things about it are load-bearing: - -- **Register before the first Cognee call.** When Cognee sets up tracing it - checks for a global provider. If one exists it attaches to it; if not it - creates and installs its own, and OpenTelemetry then ignores a later - `register(set_global_tracer_provider=True)`. -- **Keep the `add_span_processor(BatchSpanProcessor(exporter))` line.** - `register()` marks its exporter as a default that the next - `add_span_processor()` call replaces (`fi_instrumentation/otel.py`, - `TracerProvider.add_span_processor`; tracked as TH-8394). Cognee calls - `add_span_processor()` on the global provider to attach its in-memory span - buffer, so without this line the Future AGI exporter is dropped and - **nothing is exported**: Cognee still buffers its spans, but no request - leaves the process (the `no_readd` test shows this). Adding the processor - yourself makes it a regular one that Cognee's call leaves alone, and it is - where the export filter goes. Once TH-8394 is fixed, the re-add line goes - away; until then `requirements.txt` pins `fi-instrumentation-otel>=1.1.0,<1.2` - (the published 1.1.0 behaves this way; the tests use this repository's - source). -- **The re-added exporter does not inherit `register()`'s options.** - `HTTPSpanExporter()` reads `FI_BASE_URL`, `FI_API_KEY` and `FI_SECRET_KEY` - just as `register()`'s default exporter does, and `BatchSpanProcessor` uses - the default batch settings. `register()`'s own processor is discarded, so if - you pass exporter options to `register()`, pass the same ones here: - `headers` and `timeout` to `HTTPSpanExporter(...)`; for `span_exporter` or - gRPC transport, wrap that exporter (or `GRPCSpanExporter(...)`) in - `CogneeExportFilter`; the batch tuning to `BatchSpanProcessor(...)`, or use - `SimpleSpanProcessor` for `batch=False`. -- **Wrap the calls in one span if you want one trace.** Each Cognee API call - starts its own root span, and `cognee.add` ends its `memory.store` span - before ingestion runs (`cognee/api/v1/add/add.py`), so without a parent span - one add/cognify/search sequence arrives as five traces. With the parent span - it is one trace. -- **Do not call `cognee.disable_tracing()` (or `shutdown_tracing()`).** With - this option they shut down the provider Cognee attached to, which is your - global Future AGI provider, so your app's own spans stop too (source - reading: `cognee/modules/observability/trace_context.py` and `tracing.py`). - To turn Cognee's spans off, unset `COGNEE_TRACING_ENABLED` instead. - -Do not also set `OTEL_EXPORTER_OTLP_ENDPOINT` with this option. Cognee ignores -it for traces when a provider is already registered, but its metrics and log -bridge read it and would send to it (source reading of -`cognee/modules/observability/metrics.py` and `logs.py`; not exercised by the -tests). - -### The export filter - -`CogneeExportFilter` wraps the exporter and changes only what that exporter -sends. It exports copies, so Cognee's own span buffer and any other processor -or exporter still see the original spans. - -- **Content is off by default.** For the Cognee 1.6.5 add/cognify/search path - covered by this recipe's recorded fixture and loopback tests, it removes the - five content-bearing attributes listed in [Content](#content): - `langfuse.observation.input`, `langfuse.observation.output`, - `memory.query.text`, `cognee.search.query` and `cognee.db.query`. -- **Error detail is off by default too, on every span.** Cognee's errors can - quote content: a content-policy rejection raises an error whose message is - the whole prompt (a document chunk, or the question with the retrieved - context), and that message is recorded on the LLM span and again on each - pipeline-task span it passes through. On every span passed to this filter in - content-off mode, it keeps each `exception` event's `exception.type` and - `exception.escaped`, replaces `exception.message` with - `__REDACTED__ (content capture off)`, and omits - `exception.stacktrace` altogether. An `ERROR` status description becomes - ` (detail removed: content capture off)`, or - `__REDACTED__ (content capture off)` if the span recorded no exception - type. In every event, numbers and booleans are kept and any other string - value is dropped (the tested Cognee 1.6.5 recipe adds no events besides - `exception`). -- **Stack traces are never exported, in either mode.** `exception.stacktrace` - is dropped with content capture on as well: a serialized traceback names - local files (home directory, user name, virtualenv layout), and it has no - trustworthy boundary between genuine frames and traceback-shaped - exception-message content. Use Cognee's own log for frames. -- **Content capture turns both back on.** - `CogneeExportFilter(exporter, capture_content=True)`, or - `COGNEE_FI_CAPTURE_CONTENT=true` in the environment, exports the content - attributes and the error messages and status descriptions as Cognee - recorded them; only `exception.stacktrace` stays out. There is no separate - switch for error detail. It also copies that content to `input.value` and - `output.value` (with `input.mime_type` and `output.mime_type`), the only - keys fi-collector fills a span's Input and Output from: the LLM call's - arguments (`langfuse.observation.input`), or on search spans the question - (`memory.query.text`, else `cognee.search.query`), as Input, and the - response (`langfuse.observation.output`) as Output. Without these copies, - Future AGI would show captured content only as raw attributes, with Input - and Output empty. A span that already has `input.value` or `output.value` - keeps it. -- **Search and LLM spans get a type.** Cognee sets no span-kind attribute, - so the filter sets `fi.span.kind` by span name, only where none of - `fi.span.kind`, `gen_ai.span.kind`, `llm.request.type` and - `openinference.span.kind` is set: `RETRIEVER` on `memory.retrieve` and - every span whose name starts with `cognee.search` (in 1.6.5 and 1.6.2, - `cognee.search.authorize` and `cognee.search.dataset`), `LLM` on - `cognee.observe.acreate_structured_output` (every LLM call in Cognee 1.6.5). Embedding spans are left as they are: - fi-collector already types them from `gen_ai.operation.name=embeddings`. -- **Missing-collection probes are not errors.** In every search Cognee - probes vector collections that do not exist yet, and each probe's - `cognee.db.vector.search` span ends in `ERROR` with - `CollectionNotFoundError` (60 of the 66 vector searches in the recorded - run). Without the filter every successful trace shows dozens of errors and - inflates error rates. By default the filter exports a - `cognee.db.vector.search` span whose only error is - `CollectionNotFoundError` as `OK`, without its `exception` event, and with - `cognee.vector_search.collection_missing=true`; `cognee.vector.collection` - still names the collection. Any other error, a second exception on the - same span, or the same error on another span stays `ERROR`. To keep the - probes as errors, pass `collection_missing_as_error=True` or set - `COGNEE_FI_COLLECTION_MISSING_AS_ERROR=true`. -- **It fails closed.** If the filter raises on a span, that span is exported - without the content attributes, events or status description; if even that - fails, the span is dropped. The filter logs a warning and never raises into - the export path. - -The trade-off: with the defaults, error detail in Future AGI is -less useful for debugging. You see the exception type, but not its message or -traceback frames; use the existing content-capture opt-in to export the -original error messages (frames stay in Cognee's own log). The filter does not touch Cognee's own logging (for example, -`run_tasks_base.py` logs a failed pipeline task with `logger.exception`; -source reading). - -What it keeps: span names and event names (in Cognee 1.6.5, fixed strings and -function names), the other span attributes (counts, lengths, names and ids; -source reading of the attributes Cognee sets on the tested path). The tests -assert that the planted document, question and answer do not reach the -loopback receiver, and that an 8000+-character content-policy prompt with a -forged chained-traceback frame reaches neither events nor status when the -recorded LLM and pipeline-parent spans pass through the default filter. - -## Option B: Cognee's own OTLP exporter (no traceAI) - -Cognee can export by itself when `OTEL_EXPORTER_OTLP_ENDPOINT` is set and no -provider was registered. It needs Cognee's tracing extra, which installs the -OpenTelemetry SDK and exporters: - -```bash -pip install "cognee[tracing]==1.6.5" - -export COGNEE_TRACING_ENABLED=true -export OTEL_EXPORTER_OTLP_ENDPOINT=https://api.futureagi.com:443/tracer/v1/traces -export OTEL_EXPORTER_OTLP_HEADERS="X-Api-Key=$FI_API_KEY,X-Secret-Key=$FI_SECRET_KEY" -export OTEL_RESOURCE_ATTRIBUTES="project_name=my-cognee-app,project_type=observe" -``` - -**Option B exports content and has no kind mapping.** The export filter -cannot be attached to Cognee's own exporter, so prompts, document text, -answers and search questions are exported (see [Content](#content)), and -search and LLM spans are stored as `unknown`. Option A is the privacy-safe -path. +Open **Observe → Tracing** in Future AGI and select `my-cognee-app`. The application wraps its operations in one `remember_and_ask` parent span and flushes after the operations end. -- **Full path.** Cognee passes the endpoint string to the OTLP exporter - unchanged; it does not append `/v1/traces`. The test configures - `/tracer/v1/traces` and the request arrives at exactly - `/tracer/v1/traces`. -- **Keep `:443` in the URL.** Cognee picks HTTP only for URLs containing - `:4318`, `:443/` or a few vendor paths; for any other URL it uses the gRPC - exporter whenever `opentelemetry-exporter-otlp-proto-grpc` is installed - (`cognee[tracing]` installs it). Cognee's own comment warns that a gRPC - exporter pointed at an HTTP-only endpoint fails silently. - `https://api.futureagi.com/tracer/v1/traces` would go to gRPC; - `https://api.futureagi.com:443/tracer/v1/traces` stays on HTTP (the test - checks Cognee's own classifier for both). -- **`project_name` is required.** Cognee builds its resource with - OpenTelemetry's `Resource.create`, which reads `OTEL_RESOURCE_ATTRIBUTES`; - the test shows `project_name` and `project_type=observe` arrive that way. - fi-collector rejects a batch in which any resource lacks `project_name`. -- **Headers are split naively** (source reading). Cognee parses - `OTEL_EXPORTER_OTLP_HEADERS` itself: split on `,`, then on the first `=`, - with no URL decoding. A key that contains a comma cannot be passed this way; - use option A. -- **Self-hosted URL without `:443` or `:4318`: Cognee picks gRPC.** With - `cognee[tracing]` installed, a URL such as `http://localhost:14338/tracer/v1/traces` - goes to the gRPC exporter and nothing arrives (checked live: no project was - created). Point `OTEL_EXPORTER_OTLP_ENDPOINT` at the collector's OTLP gRPC - port instead (`http://:`, no path) and write the header - names in lower case, `x-api-key=...,x-secret-key=...`: gRPC metadata keys - must be lower case, and with `X-Api-Key` every export fails with - `Invalid metadata`. Checked live against the local collector's gRPC - receiver: 5 traces, 119 spans. -- **Expect 404s for logs.** Cognee also sends its log records to the same URL - with `/v1/traces` replaced by `/v1/logs`, and (source reading) its metrics - to `/v1/metrics`. fi-collector serves traces only; the test run logs - `Failed to export logs batch code: 404`. Spans are unaffected. +## Content and lifecycle -## Spans Cognee emits +Content capture is off by default. Pass `capture_content=True` to `configure` only for approved application data. Recognized credential fields are redacted; free text is not scanned for secrets. Error descriptions and stack traces stay excluded in both modes. Span names are retained, so keep sensitive data out of them. Other processors on the provider receive native, unmodified spans and need their own privacy controls. -From the recipe run (one `add`, one `cognify`, one `GRAPH_COMPLETION` -search). Cognee 1.6.5 and 1.6.2 emit the same span names, kinds, attribute -keys and statuses; only the number of `cognee.db.graph.query` spans differs -(17 on 1.6.5, 14 on 1.6.2). - -"Future AGI type" is what fi-collector derives: it reads the span -kind from `fi.span.kind`, `gen_ai.span.kind`, `llm.request.type` or -`openinference.span.kind`, falls back to `gen_ai.operation.name`, and -otherwise stores `unknown`. Cognee sets none of the first four; with option A -the export filter sets `fi.span.kind` on the spans marked below. - -| Span | OTel kind | Future AGI type (option A / option B) | Notable attributes | -|---|---|---|---| -| `remember_and_ask` | INTERNAL | unknown | the app's own parent span (option A example) | -| `memory.store` | INTERNAL | unknown | `memory.operation=store`, `cognee.dataset.name` | -| `memory.process` | INTERNAL | unknown | `memory.operation=process`, `cognee.pipeline.name=cognify` | -| `memory.retrieve` | INTERNAL | retriever / unknown | `memory.query.text`, `cognee.search.query`, `cognee.search.type`, `memory.result.count` | -| `cognee.pipeline.task.resolve_data_directories`, `cognee.pipeline.task.ingest_data`, `cognee.pipeline.task.classify_documents`, `cognee.pipeline.task.extract_chunks_from_documents`, `cognee.pipeline.task.extract_graph_and_summarize`, `cognee.pipeline.task.add_data_points` | INTERNAL | unknown | `cognee.pipeline.task_name`, `cognee.result.count`, `cognee.result.summary` | -| `cognee.observe.acreate_structured_output` (every LLM call) | CLIENT | llm / unknown | `gen_ai.request.model`, `gen_ai.system=litellm-native`, `langfuse.observation.input`, `langfuse.observation.output`; `cognee.llm.model` on schema-bound calls only | -| `cognee.observe.embed_text` (every embedding call) | CLIENT | embedding | `gen_ai.operation.name=embeddings`, `gen_ai.request.model`, `gen_ai.provider.name`, `gen_ai.embeddings.dimension.count` | -| `cognee.llm.completion` | INTERNAL | unknown | prompt file name and lengths only | -| `cognee.db.graph.query`, `cognee.db.vector.search` | INTERNAL | unknown | `cognee.db.system`, `cognee.db.query`, `cognee.vector.collection`; with option A, `cognee.vector_search.collection_missing` on a probe of a missing collection | -| `cognee.search.authorize`, `cognee.search.dataset` | INTERNAL | retriever / unknown | `cognee.search.query`, `cognee.search.type`, dataset name and id | -| `cognee.retrieval.get_objects`, `cognee.retrieval.triplet_search`, `cognee.retrieval.vector_search`, `cognee.retrieval.embed_query`, `cognee.retrieval.get_context`, `cognee.retrieval.get_completion` | INTERNAL | unknown | retriever name, counts and lengths | -| `cognee.session.get_session`, `cognee.session.add_qa` | INTERNAL | unknown | `cognee.session.id` | - -With option A, the content attributes in this table are removed unless you -turn content capture on. - -Not in the table: `cognee.api.recall` (`api/v1/recall/recall.py:540` in 1.6.5) and -`cognee.agent_memory.retrieve` (`modules/agent_memory/runtime.py:405`) are -retrieval entry points that the recipe's add/cognify/search path does not -reach (source reading). The filter does not map them, so they stay `unknown` -in Future AGI. Their query text is still removed: `cognee.api.recall` puts it -in `cognee.search.query`, and `cognee.agent_memory.retrieve` records only the -query length, while the search it runs puts the query on `memory.retrieve` -and `cognee.search.*` spans. - -What that means in Future AGI: - -- **Model column: populated.** Every LLM span carries `gen_ai.request.model`, - which fi-collector promotes. The provider comes from `gen_ai.system` - (`litellm-native`, Cognee's adapter name, not the upstream provider). No - processor that copies `cognee.llm.model` is needed, so none is written. -- **Token and cost columns: empty.** Cognee 1.6.5 records no token usage on - any span. -- **Input and Output: empty by default.** Content capture is off, so no span - carries content. With content capture on (option A), LLM spans show the - call's arguments and response and search spans show the question; see - [The export filter](#the-export-filter). With option B they stay empty: the - content arrives only as raw `langfuse.*`, `memory.*` and `cognee.*` - attributes. -- **Search is shown as a retriever, LLM calls as LLM (option A).** The - export filter sets `fi.span.kind`. With option B they stay `unknown`: no - span-kind key arrives and only embedding spans carry - `gen_ai.operation.name`. -- **No errors in a successful run (option A).** Cognee's own spans put - most `cognee.db.vector.search` spans in `ERROR`, each with an `exception` - event: 60 of 66 in the recorded run (60 of its 117 spans), because Cognee - probes collections that do not exist yet. The export filter sends those as - `OK` with `cognee.vector_search.collection_missing=true` (see - [The export filter](#the-export-filter)), so they do not count in Future - AGI's error filters and error rates. With option B, or with - `COGNEE_FI_COLLECTION_MISSING_AS_ERROR=true`, they arrive as errors; with - option A's other defaults their status then reads - `cognee.infrastructure.databases.vector.exceptions.exceptions.CollectionNotFoundError - (detail removed: content capture off)`, without the collection name. - -## Content - -With tracing on, Cognee 1.6.5 writes content to span attributes by default, -and it has no setting that keeps tracing on but leaves content out. Option A's -export filter removes these attributes before export unless content capture -is on; option B exports them: - -| Attribute | On | Contains | -|---|---|---| -| `langfuse.observation.input` | every LLM span | the LLM call's string arguments as JSON: Cognee's system prompt plus the document text (extraction, summarization), or the user question and retrieved graph context (search). First 8000 characters, after Cognee's regex secret redaction. | -| `langfuse.observation.output` | every LLM span | the model's response (extracted graph JSON, summaries, the answer). Same cap and redaction. | -| `memory.query.text`, `cognee.search.query` | `memory.retrieve`, `cognee.search.authorize` | the search question (first 500 characters). | -| `cognee.db.query` | `cognee.db.graph.query` | the graph query text (first 500 characters, redacted). In the test run these are parameterised queries without document text. | -| `exception.message`, `exception.stacktrace`, the status description | `exception` events and `ERROR` status of the span that raised and of each span the error passes through | the error message can quote content: a content-policy rejection (`litellm_native/native_adapter.py:532-560` in 1.6.5) quotes the whole prompt, past the 8000-character cap. Stack traces can also contain that serialized message and host paths. With option A's defaults the message and description are replaced; `exception.stacktrace` is omitted in both modes (see [The export filter](#the-export-filter)). | - -The test plants markers in the document and the question and asserts that, -with option A's defaults, none of them arrives anywhere (attributes, status, -events). With content capture on, it asserts that among span attributes they -arrive under exactly the attribute keys above plus the filter's `input.value` -and `output.value` copies, and no others; error messages and status -descriptions are then exported as Cognee recorded them, so they also carry -content whenever an error quotes it. Stack traces are not exported in either -mode. -Embedding spans carry metadata only, not the embedded text. - -Not exported: the LLM and embedding API keys and the Future AGI keys (those -travel only as request headers). The test asserts both. - -With option B, the only way to keep content out of Future AGI is to leave -`COGNEE_TRACING_ENABLED` unset: the `tracing_off` test shows Cognee then emits -no spans (only the app's own parent span remains). If `LANGFUSE_PUBLIC_KEY` -and `LANGFUSE_SECRET_KEY` are set, Cognee turns tracing on by itself; set -`COGNEE_TRACING_ENABLED=false` to veto that (source reading). - -`TELEMETRY_DISABLED=1` is a different switch: it turns off Cognee's anonymous -product telemetry, not OpenTelemetry tracing. - -## Troubleshooting - -- **Dozens of `cognee.db.vector.search` errors in every trace.** The - exporter is not wrapped in `CogneeExportFilter` (or you use option B), or - `COGNEE_FI_COLLECTION_MISSING_AS_ERROR=true` is set. They are Cognee's - probes of collections that do not exist yet. -- **No traces at all with option A.** The `add_span_processor(BatchSpanProcessor(exporter))` - line is missing, or `register()` ran after Cognee's first traced call. -- **Option A traces stop part-way.** Something called - `cognee.disable_tracing()` or `shutdown_tracing()`, which shut down the - Future AGI provider. Unset `COGNEE_TRACING_ENABLED` instead. -- **Prompts or documents visible with option A.** `COGNEE_FI_CAPTURE_CONTENT=true` - is set, `capture_content=True` is passed, or the exporter is not wrapped in - `CogneeExportFilter`. -- **Error messages read `__REDACTED__ (content capture off)` and no traceback - appears.** That is option A's default, because serialized error detail can - quote content. Turn content capture on to export the original error - messages; tracebacks are never exported (they name local files), so check - Cognee's own log for frames. -- **Spans doubled.** Another instrumentor is wrapping Cognee or LiteLLM on top - of Cognee's own spans. Remove it. -- **Option B exports nothing, with no error.** Either the OpenTelemetry - packages are missing (plain `cognee==1.6.5` has none; with tracing on, - Cognee then silently uses no-op spans; install `cognee[tracing]==1.6.5`), - or the URL has no `:443`, so Cognee chose gRPC (for a self-hosted - collector, see option B above). -- **404 on `/tracer/v1/traces/v1/traces`.** Something appended the path to a - URL that already had it. Cognee does not append; give the full path once. -- **Five traces per request.** Wrap the Cognee calls in one parent span. -- **No token counts.** Cognee 1.6.5 does not record them. - -## Tests - -`tests/test_cognee_recipe.py` runs, side by side, six subprocesses with the -real `cognee==1.6.5` (the suite passes on `cognee==1.6.2` too). Each loads -`tests/loopback_guard/sitecustomize.py` -through `PYTHONPATH`, and so do the worker processes Cognee spawns for -LanceDB and Ladybug: in all of them, Python-level connections and DNS lookups -to any host other than 127.0.0.1 are refused and logged, and the test asserts -the log is empty and that the guard was installed in Cognee's workers. Native -code that opens sockets itself (for example a database extension download) is -not intercepted. Cognee's LLM and embedding endpoints are a loopback fake of -the OpenAI API (`tests/_fake_openai.py`), and spans go to the shared harness -`Receiver` (`python/tests/harness`), which serves `/v1/traces` and -`/tracer/v1/traces` like fi-collector but does not authenticate or store -anything: - -- `recipe`: `src/app.py` as written (option A, content off). -- `capture`: the same, with `COGNEE_FI_CAPTURE_CONTENT=true`. -- `probes_as_errors`: the same, with `COGNEE_FI_COLLECTION_MISSING_AS_ERROR=true`. -- `tracing_off`: the same, with `COGNEE_TRACING_ENABLED` unset. -- `no_readd`: `register()` without the re-add line, then one `cognee.add`. -- `otlp_env`: option B's environment variables, then one `cognee.add`. - -`tests/fixtures/cognee-1.6.2-recipe-spans.json` is Cognee's own spans from -the recipe run on cognee 1.6.2 as the Receiver decoded them, before the -export filter -(host paths replaced by ``, ``, `` and ``). The -fixture tests post it with the harness `post_otlp()`, check it with -`compare()`, run the same mapping and content assertions on it, and send it -through the export filter, with no Cognee installed. The live runs must match -the filtered fixture's shape (span names, kinds, attribute keys, statuses and -counts; the number of `cognee.db.graph.query` spans is compared only as -"at least one", because it changes between patch releases). The scenarios -point `TIKTOKEN_CACHE_DIR` at LiteLLM's bundled tokenizer files, because -cognee 1.6.5 loads tiktoken before LiteLLM sets that and would otherwise -download `cl100k_base`, which the loopback guard refuses; the recipe itself -needs no such setting. Re-record with `COGNEE_RECORD_FIXTURE=1` (it records the `capture` -run without the `fi.span.kind` the filter adds). - -From the repository root: - -```bash -env -u PYTHONPATH PYTHONPATH="python/examples/cognee:python:python/tests" \ - uv run --no-project --python 3.11 \ - --with pytest --with pytest-asyncio --with opentelemetry-api \ - --with opentelemetry-sdk --with opentelemetry-instrumentation \ - --with opentelemetry-exporter-otlp-proto-http --with wrapt --with requests \ - --with jsonschema --with protobuf --with opentelemetry-proto \ - --with 'cognee==1.6.5' \ - pytest python/examples/cognee/tests -q -p no:cacheprovider --noconftest -o addopts= -rfEs -``` +Configure before the first Cognee operation. End spans and call `handle.force_flush()` before `handle.shutdown()`; a span still open at shutdown is not exported. The handle owns only the adapter's exporter, not other processors on the provider. Cognee's `cognee.disable_tracing()` and `cognee.shutdown_tracing()` shut down the shared provider too, so do not use them while other application tracing is active. -Importing Cognee takes about a minute, so a full run takes a few minutes. -`register()` comes from this repository's `python/fi_instrumentation`, not a -published release. The environment above has no gRPC exporter installed; -option B's gRPC choice is covered by calling Cognee's URL classifier, not by a -gRPC export. +Search spans are mapped to `RETRIEVER` and structured-output model calls to `LLM`. Native error statuses, including missing-collection probes, are preserved. See the [adapter reference](../../frameworks/cognee/) for configuration options and lifecycle limits. diff --git a/python/examples/cognee/requirements.txt b/python/examples/cognee/requirements.txt index 32cbe0c32..935cf48ae 100644 --- a/python/examples/cognee/requirements.txt +++ b/python/examples/cognee/requirements.txt @@ -1,7 +1,2 @@ -# Not a traceAI package: these are the example's own dependencies. -cognee==1.6.5 -# Provides fi_instrumentation.register(); pulls in opentelemetry-sdk and the -# OTLP/HTTP exporter that Cognee's tracing needs. Pinned because app.py's -# re-add line relies on register() marking its exporter as a replaceable -# default (TH-8394); once that is fixed, the line goes away and the pin can move. -fi-instrumentation-otel>=1.1.0,<1.2 +cognee[tracing]==1.6.5 +traceai-cognee==0.1.0 diff --git a/python/examples/cognee/src/app.py b/python/examples/cognee/src/app.py index 23e2ebc9e..f495b45e9 100644 --- a/python/examples/cognee/src/app.py +++ b/python/examples/cognee/src/app.py @@ -1,83 +1,33 @@ -"""Send Cognee's own OpenTelemetry spans to Future AGI. - -Recipe example, not a package. Cognee 1.6.5 creates its spans itself when -``COGNEE_TRACING_ENABLED=true``. This file registers a Future AGI tracer -provider as the global OpenTelemetry provider before Cognee's first traced -call, so Cognee attaches to that provider instead of creating its own -(``setup_tracing`` in ``cognee/modules/observability/tracing.py``). It adds -no instrumentor and never calls ``setup_tracing``. The Future AGI exporter is -wrapped in ``CogneeExportFilter`` (``cognee_filter.py``, next to this file), -which keeps Cognee's prompts, documents, answers and queries, plus error text -and serialized exception stack traces that can quote them, out of the export -and marks search and LLM spans. - - COGNEE_TRACING_ENABLED=true python src/app.py "Who works on Lighthouse?" - -FI_API_KEY, FI_SECRET_KEY, FI_PROJECT_NAME and (optionally) FI_BASE_URL are -read from the environment; so are Cognee's own LLM_* and EMBEDDING_* -settings, and COGNEE_FI_CAPTURE_CONTENT=true to export content after all. -See README.md. -""" - -from __future__ import annotations - +"""Trace native Cognee operations using the maintained adapter.""" import asyncio import os import sys +from traceai_cognee import configure -from fi_instrumentation import register -from fi_instrumentation.fi_types import ProjectType -from fi_instrumentation.otel import BatchSpanProcessor, HTTPSpanExporter - -from cognee_filter import CogneeExportFilter - -DOCUMENT = "Ada works on the Lighthouse project." - +DOCUMENT = 'Ada works on the Lighthouse project.' def init_tracing(): - """Register the Future AGI provider. Call once, before any Cognee call.""" - provider = register( - project_name=os.environ["FI_PROJECT_NAME"], - project_type=ProjectType.OBSERVE, - # Cognee reuses the global provider when one is set. - set_global_tracer_provider=True, - verbose=False, - ) - # register() marks its exporter as a default that the next - # add_span_processor() call replaces (TH-8394). On its first traced call - # Cognee calls add_span_processor() on the global provider to attach its - # in-memory buffer, which would silently drop the Future AGI exporter. - # Adding the exporter again makes it a regular processor that Cognee's - # call leaves in place. HTTPSpanExporter() reads FI_BASE_URL, FI_API_KEY - # and FI_SECRET_KEY like register()'s own exporter; if you pass exporter - # or batch options to register(), pass the same ones here. - exporter = CogneeExportFilter(HTTPSpanExporter()) - provider.add_span_processor(BatchSpanProcessor(exporter)) - return provider - + return configure(project_name=os.environ['FI_PROJECT_NAME']) -async def remember_and_ask(question: str, document: str) -> list: +async def remember_and_ask(question, document): import cognee - await cognee.add(document) await cognee.cognify() return await cognee.search(question, query_type=cognee.SearchType.GRAPH_COMPLETION) - -def main(argv: list[str]) -> int: - provider = init_tracing() - question = argv[1] if len(argv) > 1 else "Who works on Lighthouse?" - document = argv[2] if len(argv) > 2 else DOCUMENT - # Cognee starts a new root span for each API call (and `add` ends its - # memory.store span before ingestion runs), so one parent span here is - # what puts add, cognify and search in one trace. - with provider.get_tracer(__name__).start_as_current_span("remember_and_ask"): - results = asyncio.run(remember_and_ask(question, document)) - for result in results: - print(result) - provider.force_flush() +def main(argv): + handle = init_tracing() + try: + question = argv[1] if len(argv) > 1 else 'Who works on Lighthouse?' + document = argv[2] if len(argv) > 2 else DOCUMENT + with handle.provider.get_tracer(__name__).start_as_current_span('remember_and_ask'): + results = asyncio.run(remember_and_ask(question, document)) + for result in results: + print(result) + handle.force_flush() + finally: + handle.shutdown() return 0 - -if __name__ == "__main__": +if __name__ == '__main__': sys.exit(main(sys.argv)) diff --git a/python/examples/cognee/src/cognee_filter.py b/python/examples/cognee/src/cognee_filter.py deleted file mode 100644 index 1ecfada1a..000000000 --- a/python/examples/cognee/src/cognee_filter.py +++ /dev/null @@ -1,275 +0,0 @@ -"""Export filter for Cognee's spans: content off by default, search and LLM kinds set. - -Recipe code, not a package: copy this file next to your app. It wraps the -exporter that sends spans to Future AGI (see app.py):: - - exporter = CogneeExportFilter(HTTPSpanExporter()) - provider.add_span_processor(BatchSpanProcessor(exporter)) - -and changes only what that exporter sends. It exports copies, so Cognee's -own span buffer and any other processor still see the original spans. - -Content. Cognee 1.6.5 writes prompts, document text, answers and search -questions to span attributes and has no setting to stop that, and its -errors can quote them (a content-policy rejection quotes the whole prompt). -By default the filter removes those attributes (``CONTENT_KEYS``) and, on -every span, all free-text error detail: exception messages and status -descriptions are replaced, exception stack traces are omitted, and other event -values are kept only if they are not text. Pass ``capture_content=True``, or set -``COGNEE_FI_CAPTURE_CONTENT=true``, to export content and error messages too. -With capture on, the filter also copies that content to ``input.value`` and -``output.value`` (``IO_SOURCES``), the keys Future AGI shows as a span's Input -and Output. - -Stack traces. ``exception.stacktrace`` is never exported, with or without -content capture: a serialized traceback carries local file paths (home -directory, user name, virtualenv layout) and can quote content. - -Span kinds. Cognee sets no span-kind attribute, so Future AGI would store -search and LLM spans as ``unknown``. The filter sets ``fi.span.kind`` by span -name (``span_kind``), only where none of ``SPAN_KIND_KEYS`` is set. - -Missing collections. In every search Cognee probes vector collections that -do not exist yet, and each probe's ``cognee.db.vector.search`` span ends in -ERROR with ``CollectionNotFoundError``, so every successful trace would show -dozens of errors. By default the filter exports a vector-search span whose -only error is that one as OK, without its exception event, and with -``cognee.vector_search.collection_missing=true``. Every other error stays -ERROR. Pass ``collection_missing_as_error=True``, or set -``COGNEE_FI_COLLECTION_MISSING_AS_ERROR=true``, to keep them as ERROR. - -If the filter fails on a span, it exports that span without content -attributes, events or status description; if even that fails, it drops the -span. It never raises into the export path. -""" - -from __future__ import annotations - -import json -import logging -import os -from typing import Any, Mapping, Optional, Sequence - -from opentelemetry.sdk.trace import Event, ReadableSpan -from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult -from opentelemetry.trace import Status, StatusCode - -logger = logging.getLogger(__name__) - -# Attributes in which Cognee 1.6.x exports content (paths in the cognee wheel). -CONTENT_KEYS = frozenset( - { - # LLM prompt arguments and response: modules/observability/get_observe.py - "langfuse.observation.input", - "langfuse.observation.output", - # The search question: api/v1/search/search.py, modules/search/methods/search.py - "memory.query.text", - "cognee.search.query", - # Graph query text: databases/graph/ladybug/adapter.py, neo4j_driver/adapter.py - "cognee.db.query", - } -) -# With content capture on, where a span's Input and Output come from: fi-collector -# fills them from input.value and output.value only. The first source key that -# is set wins; a span that already has the target key keeps it. -IO_SOURCES = ( - ("input", ("langfuse.observation.input", "memory.query.text", "cognee.search.query")), - ("output", ("langfuse.observation.output",)), -) -# fi-collector takes the span type from the first of these keys that is set. -SPAN_KIND_KEYS = ( - "fi.span.kind", - "gen_ai.span.kind", - "llm.request.type", - "openinference.span.kind", -) -# Cognee names a span after the decorated method; every -# @observe(as_type="generation") method in Cognee 1.6.5 is acreate_structured_output. -LLM_SPANS = frozenset({"cognee.observe.acreate_structured_output"}) -# What replaces error text by default. -DETAIL_REMOVED = "__REDACTED__ (content capture off)" -TYPE_ONLY = "{0} (detail removed: content capture off)" -# A probe of a vector collection that does not exist yet (see the module docstring). -VECTOR_SEARCH_SPAN = "cognee.db.vector.search" -COLLECTION_MISSING = ( - "cognee.infrastructure.databases.vector.exceptions.exceptions.CollectionNotFoundError" -) -COLLECTION_MISSING_KEY = "cognee.vector_search.collection_missing" -# The only event keys whose string values are kept: the exception's class name -# and "True"/"False". Cognee 1.6.5 adds no events other than OpenTelemetry's -# "exception" events; any other event gets the same allowlist. -SAFE_EVENT_KEYS = frozenset({"exception.type", "exception.escaped"}) - - -def span_kind(name: str) -> Optional[str]: - """The fi.span.kind for a Cognee span name, or None to leave the span as it is.""" - if name == "memory.retrieve" or name.startswith("cognee.search"): - return "RETRIEVER" - if name in LLM_SPANS: - return "LLM" - # Embedding spans already carry gen_ai.operation.name=embeddings. - return None - - -class CogneeExportFilter(SpanExporter): - """Wrap a span exporter; see the module docstring.""" - - def __init__( - self, - exporter: SpanExporter, - capture_content: Optional[bool] = None, - collection_missing_as_error: Optional[bool] = None, - ) -> None: - if capture_content is None: - capture_content = os.environ.get("COGNEE_FI_CAPTURE_CONTENT", "").lower() == "true" - if collection_missing_as_error is None: - collection_missing_as_error = ( - os.environ.get("COGNEE_FI_COLLECTION_MISSING_AS_ERROR", "").lower() == "true" - ) - self._exporter = exporter - self._capture_content = capture_content - self._collection_missing_as_error = collection_missing_as_error - - def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult: - out = [] - for span in spans: - try: - out.append(self._filtered(span)) - except Exception as error: - logger.warning( - "CogneeExportFilter: %s; exporting the span without content", - type(error).__name__, - ) - try: - out.append(self._stripped(span)) - except Exception as error: - logger.warning( - "CogneeExportFilter: %s; dropping the span", type(error).__name__ - ) - return self._exporter.export(out) - - def shutdown(self) -> None: - self._exporter.shutdown() - - def force_flush(self, timeout_millis: int = 30000) -> bool: - return self._exporter.force_flush(timeout_millis) - - def _filtered(self, span: ReadableSpan) -> ReadableSpan: - attributes = dict(span.attributes or {}) - kind = span_kind(span.name) - if kind and not any(key in attributes for key in SPAN_KIND_KEYS): - attributes["fi.span.kind"] = kind - if not self._collection_missing_as_error and _only_collection_missing(span): - attributes[COLLECTION_MISSING_KEY] = True - if self._capture_content: - _add_input_output(attributes) - else: - for key in CONTENT_KEYS: - attributes.pop(key, None) - return _copy(span, attributes, (), Status(StatusCode.OK)) - if self._capture_content: - _add_input_output(attributes) - events = [ - Event(e.name, _without_stacktrace(e.attributes), e.timestamp) for e in span.events - ] - return _copy(span, attributes, events, span.status) - for key in CONTENT_KEYS: - attributes.pop(key, None) - events = [Event(e.name, _without_detail(e.attributes), e.timestamp) for e in span.events] - status = span.status - if status.description: - # Only an ERROR status has a description; name the type of the - # exception that ended the span, if the span recorded one. - types = [e.attributes.get("exception.type") for e in events if e.name == "exception"] - if types and isinstance(types[-1], str): - status = Status(status.status_code, TYPE_ONLY.format(types[-1])) - else: - status = Status(status.status_code, DETAIL_REMOVED) - return _copy(span, attributes, events, status) - - def _stripped(self, span: ReadableSpan) -> ReadableSpan: - """The fail-closed copy: no content attributes, no events, no status description.""" - attributes = {k: v for k, v in (span.attributes or {}).items() if k not in CONTENT_KEYS} - return _copy(span, attributes, (), Status(span.status.status_code)) - - -def _add_input_output(attributes: dict) -> None: - """Copy captured content to the keys Future AGI shows as Input and Output.""" - for prefix, sources in IO_SOURCES: - if prefix + ".value" in attributes: - continue - value = next((attributes[k] for k in sources if isinstance(attributes.get(k), str)), None) - if value is None: - continue - attributes[prefix + ".value"] = value - attributes.setdefault( - prefix + ".mime_type", "application/json" if _is_json(value) else "text/plain" - ) - - -def _is_json(value: str) -> bool: - """True for a JSON object or array (Cognee's serialized LLM arguments).""" - try: - return isinstance(json.loads(value), (dict, list)) - except ValueError: - return False - - -def _only_collection_missing(span: ReadableSpan) -> bool: - """True for a vector-search span whose only error is a missing collection.""" - if span.name != VECTOR_SEARCH_SPAN or span.status.status_code != StatusCode.ERROR: - return False - return bool(span.events) and all( - event.name == "exception" - and (event.attributes or {}).get("exception.type") == COLLECTION_MISSING - for event in span.events - ) - - -def _without_stacktrace(attributes: Optional[Mapping[str, Any]]) -> dict: - """An event's attributes as recorded, minus the serialized stack trace (local paths).""" - return {k: v for k, v in (attributes or {}).items() if k != "exception.stacktrace"} - - -def _without_detail(attributes: Optional[Mapping[str, Any]]) -> dict: - """An event's attributes without free text: an allowlist of keys and value types.""" - attributes = attributes or {} - out = {} - for key, value in attributes.items(): - if key == "exception.message": - out[key] = DETAIL_REMOVED - elif key == "exception.stacktrace": - # A serialized traceback has no trustworthy boundary between its - # real frames and traceback-shaped exception-message content. - continue - elif key in SAFE_EVENT_KEYS or _not_text(value): - out[key] = value - return out - - -def _not_text(value: Any) -> bool: - """True for a number or boolean, or a sequence of them.""" - if isinstance(value, (bool, int, float)): - return True - return isinstance(value, (list, tuple)) and all( - isinstance(item, (bool, int, float)) for item in value - ) - - -def _copy( - span: ReadableSpan, attributes: Mapping[str, Any], events: Sequence[Event], status: Status -) -> ReadableSpan: - return ReadableSpan( - name=span.name, - context=span.context, - parent=span.parent, - resource=span.resource, - attributes=attributes, - events=events, - links=span.links, - kind=span.kind, - status=status, - start_time=span.start_time, - end_time=span.end_time, - instrumentation_scope=span.instrumentation_scope, - ) diff --git a/python/examples/cognee/tests/_run_app.py b/python/examples/cognee/tests/_run_app.py new file mode 100644 index 000000000..1456cd4fa --- /dev/null +++ b/python/examples/cognee/tests/_run_app.py @@ -0,0 +1,19 @@ +"""Run the maintained example with optional content capture in a fresh process.""" +import importlib.util +import os +from pathlib import Path +import sys + +import traceai_cognee + +app_path = Path(__file__).resolve().parents[1] / "src" / "app.py" +spec = importlib.util.spec_from_file_location("cognee_example", app_path) +app = importlib.util.module_from_spec(spec) +spec.loader.exec_module(app) + +# Keep the application path unchanged; opt in only for the capture regression. +if os.environ["TEST_CAPTURE_CONTENT"] == "true": + app.init_tracing = lambda: traceai_cognee.configure( + project_name=os.environ["FI_PROJECT_NAME"], capture_content=True + ) +sys.exit(app.main([str(app_path), *sys.argv[1:]])) diff --git a/python/examples/cognee/tests/cognee_add.py b/python/examples/cognee/tests/cognee_add.py deleted file mode 100644 index bd6120937..000000000 --- a/python/examples/cognee/tests/cognee_add.py +++ /dev/null @@ -1,59 +0,0 @@ -"""Scenario script: one ``cognee.add`` call, with tracing set up per argv[1]. - - python cognee_add.py otlp-env [URL ...] # no register(); Cognee's own OTLP exporter - python cognee_add.py no-readd # register() without the recipe's re-add line - -Prints one JSON line with the number of spans Cognee's in-memory buffer holds, -so the test can tell "Cognee made no spans" apart from "spans were made but -not exported". In otlp-env mode it also reports, for each URL argument, -whether Cognee 1.6.2 treats it as HTTP-only (otherwise Cognee picks the gRPC -exporter when that package is installed). -""" - -from __future__ import annotations - -import asyncio -import json -import os -import sys - -DOCUMENT = "Ada works on the Lighthouse project." - - -def main(argv: list[str]) -> int: - mode = argv[1] - provider = None - if mode == "no-readd": - from fi_instrumentation import register - from fi_instrumentation.fi_types import ProjectType - - provider = register( - project_name=os.environ["FI_PROJECT_NAME"], - project_type=ProjectType.OBSERVE, - set_global_tracer_provider=True, - verbose=False, - ) - elif mode != "otlp-env": - raise SystemExit("unknown mode: " + mode) - - import cognee - - asyncio.run(cognee.add(DOCUMENT)) - report = { - "mode": mode, - "buffered_spans": sum(len(trace.spans()) for trace in cognee.get_all_traces()), - } - if mode == "otlp-env": - from cognee.modules.observability.tracing import _requires_http_exporter - - report["http_only"] = {url: _requires_http_exporter(url) for url in argv[2:]} - print(json.dumps(report)) - if provider is not None: - provider.force_flush() - # In otlp-env mode Cognee's own provider flushes its BatchSpanProcessor - # at interpreter exit. - return 0 - - -if __name__ == "__main__": - sys.exit(main(sys.argv)) diff --git a/python/examples/cognee/tests/loopback_guard/sitecustomize.py b/python/examples/cognee/tests/loopback_guard/sitecustomize.py deleted file mode 100644 index 5f8abd1c2..000000000 --- a/python/examples/cognee/tests/loopback_guard/sitecustomize.py +++ /dev/null @@ -1,93 +0,0 @@ -"""Block and log every non-loopback network connection, in every test process. - -The test scenarios put this directory first on ``PYTHONPATH``, so Python's -``site`` imports this file at the start of each interpreter they start: -the scenario script, and the multiprocessing "spawn" workers in which Cognee -1.6.2 runs LanceDB and Ladybug (they inherit the environment). This file -then runs the interpreter's own ``sitecustomize``, if it has one (Homebrew's -Python does), so nothing else changes. - -When ``LOOPBACK_GUARD_LOG`` is set, ``socket.socket.connect``/``connect_ex`` -and ``socket.getaddrinfo`` refuse any host other than 127.0.0.1, ::1 or -localhost. Each refused attempt, and each process the guard is installed in, -is appended as a JSON line to that file, so a test can assert that Cognee, -LiteLLM and the example tried to reach nothing else (no LLM provider, no -Cognee telemetry, no Future AGI). Unix sockets are not affected, and neither -is native code that opens sockets itself. -""" - -from __future__ import annotations - -import importlib.machinery -import importlib.util -import json -import os -import socket -import sys - -_ALLOWED_HOSTS = {"127.0.0.1", "::1", "localhost"} -_LOG_PATH = os.environ.get("LOOPBACK_GUARD_LOG", "") - - -def _host(value: object) -> str: - if isinstance(value, bytes): - return value.decode("ascii", "replace") - return str(value) - - -def _log(record: dict) -> None: - with open(_LOG_PATH, "a", encoding="utf-8") as log: - log.write(json.dumps(record) + "\n") - - -def _refuse(kind: str, target: object) -> None: - _log({"kind": kind, "target": repr(target), "pid": os.getpid()}) - - -_original_connect = socket.socket.connect -_original_connect_ex = socket.socket.connect_ex -_original_getaddrinfo = socket.getaddrinfo - - -def _is_blocked(sock: socket.socket, address: object) -> bool: - if sock.family not in (socket.AF_INET, socket.AF_INET6): - return False - host = address[0] if isinstance(address, tuple) else address - return _host(host) not in _ALLOWED_HOSTS - - -def _connect(self: socket.socket, address: object) -> None: - if _is_blocked(self, address): - _refuse("connect", address) - raise OSError("blocked by loopback guard: {0!r}".format(address)) - return _original_connect(self, address) - - -def _connect_ex(self: socket.socket, address: object) -> int: - if _is_blocked(self, address): - _refuse("connect_ex", address) - raise OSError("blocked by loopback guard: {0!r}".format(address)) - return _original_connect_ex(self, address) - - -def _getaddrinfo(host: object, *args: object, **kwargs: object): # type: ignore[no-untyped-def] - if host is not None and _host(host) not in _ALLOWED_HOSTS: - _refuse("getaddrinfo", host) - raise socket.gaierror("blocked by loopback guard: {0!r}".format(host)) - return _original_getaddrinfo(host, *args, **kwargs) - - -def _run_shadowed_sitecustomize() -> None: - here = os.path.dirname(os.path.abspath(__file__)) - rest = [entry for entry in sys.path if os.path.abspath(entry or os.curdir) != here] - spec = importlib.machinery.PathFinder.find_spec("sitecustomize", rest) - if spec is not None and spec.loader is not None: - spec.loader.exec_module(importlib.util.module_from_spec(spec)) - - -if _LOG_PATH: - socket.socket.connect = _connect # type: ignore[method-assign] - socket.socket.connect_ex = _connect_ex # type: ignore[method-assign] - socket.getaddrinfo = _getaddrinfo # type: ignore[assignment] - _log({"kind": "installed", "pid": os.getpid(), "argv": list(sys.argv)}) -_run_shadowed_sitecustomize() diff --git a/python/examples/cognee/tests/test_cognee_adapter.py b/python/examples/cognee/tests/test_cognee_adapter.py new file mode 100644 index 000000000..61fdcc83e --- /dev/null +++ b/python/examples/cognee/tests/test_cognee_adapter.py @@ -0,0 +1,161 @@ +"""Maintained adapter contracts using real Cognee and synthetic localhost providers. + +Run with the adapter, Cognee's tracing extra and pytest installed, under an OS +sandbox denying outbound network except localhost. No real provider keys are used. +""" +import base64 +import importlib.util +import json +from pathlib import Path +import subprocess +import sys + +import pytest +from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import Event, ReadableSpan +from opentelemetry.trace import SpanContext, SpanKind, Status, StatusCode, TraceFlags +from traceai_cognee._exporter import project + +TESTS = Path(__file__).resolve().parent +EXAMPLE = TESTS.parent +spec = importlib.util.spec_from_file_location("fake_openai", TESTS / "_fake_openai.py") +fake_module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(fake_module) + + +def test_example_uses_maintained_adapter(): + requirements = (EXAMPLE / "requirements.txt").read_text().splitlines() + assert "cognee[tracing]==1.6.5" in requirements + assert "traceai-cognee==0.1.0" in requirements + source = (EXAMPLE / "src/app.py").read_text() + assert "from traceai_cognee import configure" in source + assert "cognee_filter" not in source + readme = (EXAMPLE / "README.md").read_text() + assert "capture_content=True" in readme + assert "COGNEE_TRACING_ENABLED=true" in readme + assert "cognee.disable_tracing()" in readme + + +@pytest.mark.parametrize("capture", [False, True]) +def test_recorded_native_spans_keep_identity_status_and_privacy(capture): + recorded = json.loads((TESTS / "fixtures/cognee-1.6.2-recipe-spans.json").read_text()) + kinds = {"SPAN_KIND_INTERNAL": SpanKind.INTERNAL, "SPAN_KIND_CLIENT": SpanKind.CLIENT} + statuses = {"STATUS_CODE_OK": StatusCode.OK, "STATUS_CODE_ERROR": StatusCode.ERROR} + + def context(trace_id, span_id): + return SpanContext(int.from_bytes(base64.b64decode(trace_id), "big"), + int.from_bytes(base64.b64decode(span_id), "big"), + False, TraceFlags(1)) + + def attrs(items): + return {item["key"]: next(iter(item["value"].values())) for item in items} + + output = [] + for item in recorded: + parent = context(item["traceId"], item["parentSpanId"]) if item.get("parentSpanId") else None + status = item.get("status", {}) + native = ReadableSpan( + name=item["name"], context=context(item["traceId"], item["spanId"]), parent=parent, + resource=Resource({"private": "unapproved-resource"}), + attributes=attrs(item.get("attributes", [])), + events=[Event(e["name"], attrs(e.get("attributes", [])), int(e["timeUnixNano"])) + for e in item.get("events", [])], + kind=kinds[item["kind"]], status=Status(statuses.get(status.get("code"), StatusCode.UNSET), status.get("message")), + start_time=int(item["startTimeUnixNano"]), end_time=int(item["endTimeUnixNano"]), + ) + before = dict(native.attributes) + mapped = project(native, "synthetic-project", capture) + assert mapped.context == native.context and mapped.parent == native.parent + assert mapped.start_time == native.start_time and mapped.end_time == native.end_time + assert mapped.status.status_code == native.status.status_code + assert mapped.status.description is None + assert dict(native.attributes) == before + assert dict(mapped.resource.attributes) == {"project_name": "synthetic-project", "project_type": "observe"} + assert all(not event.attributes for event in mapped.events) + if item["name"] == "memory.retrieve": + assert mapped.attributes["fi.span.kind"] == "RETRIEVER" + if item["name"] == "cognee.observe.acreate_structured_output": + assert mapped.attributes["fi.span.kind"] == "LLM" + if not capture: + assert not {"langfuse.observation.input", "langfuse.observation.output", "memory.query.text", + "cognee.search.query", "cognee.db.query", "input.value", "output.value"}.intersection(mapped.attributes) + output.append(mapped) + assert any(span.status.status_code == StatusCode.ERROR for span in output) + assert len(output) == len(recorded) + + +@pytest.mark.parametrize("capture", [False, True]) +def test_example_native_operations_export_once_with_content_policy(tmp_path, capture): + # The process runs real Cognee; only model responses and OTLP transport are local fakes. + fake = fake_module.FakeOpenAI() + bodies = [] + auth_headers = [] + + class CombinedHandler(fake._server.RequestHandlerClass): + def do_POST(self): + if self.path != "/tracer/v1/traces": + return super().do_POST() + length = int(self.headers.get("Content-Length", "0")) + if not 0 < length <= 4_000_000: + self.send_error(413) + return + bodies.append(self.rfile.read(length)) + auth_headers.append((self.headers.get("X-Api-Key"), self.headers.get("X-Secret-Key"))) + self.send_response(200) + self.send_header("Content-Length", "0") + self.end_headers() + + fake._server.RequestHandlerClass = CombinedHandler + for name in ("home", "tmp", "cache", "config", "data", "system", "logs"): + (tmp_path / name).mkdir() + env = { + "HOME": str(tmp_path / "home"), "TMPDIR": str(tmp_path / "tmp"), + "XDG_CACHE_HOME": str(tmp_path / "cache"), "XDG_CONFIG_HOME": str(tmp_path / "config"), + "PATH": "/usr/bin:/bin", "LANG": "C.UTF-8", "NO_COLOR": "1", + "TELEMETRY_DISABLED": "1", "LITELLM_LOCAL_MODEL_COST_MAP": "True", + "OTEL_TRACES_EXPORTER": "none", "OTEL_METRICS_EXPORTER": "none", "OTEL_LOGS_EXPORTER": "none", + "COGNEE_TRACING_ENABLED": "true", "TEST_CAPTURE_CONTENT": str(capture).lower(), + "LLM_PROVIDER": "openai", "LLM_MODEL": "openai/gpt-4o-mini", "LLM_ENDPOINT": fake.base_url, + "LLM_API_KEY": "synthetic-model-key", "OPENAI_API_KEY": "synthetic-model-key", + "EMBEDDING_PROVIDER": "openai", "EMBEDDING_MODEL": "openai/text-embedding-3-small", + "EMBEDDING_ENDPOINT": fake.base_url, "EMBEDDING_API_KEY": "synthetic-model-key", "EMBEDDING_DIMENSIONS": "16", + "DATA_ROOT_DIRECTORY": str(tmp_path / "data"), "SYSTEM_ROOT_DIRECTORY": str(tmp_path / "system"), + "CACHE_ROOT_DIRECTORY": str(tmp_path / "cache"), "COGNEE_LOGS_DIR": str(tmp_path / "logs"), + "FI_BASE_URL": fake.base_url.removesuffix("/v1"), "FI_PROJECT_NAME": "synthetic-project", + "FI_API_KEY": "synthetic-fi-api", "FI_SECRET_KEY": "synthetic-fi-secret", + } + import litellm + tokenizers = Path(litellm.__file__).parent / "litellm_core_utils/tokenizers" + env["TIKTOKEN_CACHE_DIR"] = str(tokenizers) + try: + result = subprocess.run( + [sys.executable, "-I", "-B", str(TESTS / "_run_app.py"), + "Who works on Lighthouse? QUERY-MARKER-4c9", "Ada works on Lighthouse. DOC-MARKER-3b7"], + cwd=tmp_path, env=env, capture_output=True, timeout=240, + ) + assert result.returncode == 0, result.stderr.decode(errors="replace")[-4000:] + assert fake_module.ANSWER_MARKER.encode() in result.stdout + finally: + fake.close() + assert bodies and all(headers == ("synthetic-fi-api", "synthetic-fi-secret") for headers in auth_headers) + spans = [] + for body in bodies: + request = ExportTraceServiceRequest.FromString(body) + for resource in request.resource_spans: + resource_attrs = {a.key: a.value.string_value for a in resource.resource.attributes} + assert resource_attrs == {"project_name": "synthetic-project", "project_type": "observe"} + spans.extend(span for scope in resource.scope_spans for span in scope.spans) + identities = {(s.trace_id, s.span_id) for s in spans} + assert len(identities) == len(spans) + assert len({s.trace_id for s in spans}) == 1 + for name in ("remember_and_ask", "memory.store", "memory.process", "memory.retrieve"): + assert sum(span.name == name for span in spans) == 1 + assert any(s.status.code == 2 for s in spans) # Do not rewrite native errors to OK. + assert all(not s.status.message and all(not e.attributes for e in s.events) for s in spans) + wire = b"".join(bodies) + for marker in (b"QUERY-MARKER-4c9", fake_module.ANSWER_MARKER.encode()): + assert (marker in wire) is capture + assert all(secret not in wire for secret in (b"synthetic-model-key", b"synthetic-fi-api", b"synthetic-fi-secret")) + for name, kind in (("memory.retrieve", "RETRIEVER"), ("cognee.observe.acreate_structured_output", "LLM")): + assert any(s.name == name and any(a.key == "fi.span.kind" and a.value.string_value == kind for a in s.attributes) for s in spans) diff --git a/python/examples/cognee/tests/test_cognee_recipe.py b/python/examples/cognee/tests/test_cognee_recipe.py deleted file mode 100644 index 3c06ce6c6..000000000 --- a/python/examples/cognee/tests/test_cognee_recipe.py +++ /dev/null @@ -1,1719 +0,0 @@ -"""Contract test for the Cognee tracing recipe. - -Runs ``src/app.py`` as written with the real ``cognee`` (pinned 1.6.5; 1.6.2 -also passes) and traceAI's ``register()``, plus two one-call scenario -scripts, each in a subprocess whose network, and that of the worker -processes Cognee spawns, is limited to -127.0.0.1 (``loopback_guard/sitecustomize.py``). Cognee's LLM and -embedding endpoints are a loopback fake of the OpenAI API -(``_fake_openai.py``). Spans go to the shared harness ``Receiver``, which -serves ``/v1/traces`` and ``/tracer/v1/traces`` like fi-collector's HTTP mux -but does not authenticate, stamp projects or store anything. - -The recorded fixture (``fixtures/cognee-1.6.2-recipe-spans.json``) is -Cognee's own spans from the recipe run, before the export filter -(``src/cognee_filter.py``) changes them. The fixture tests post it with -``post_otlp()`` and check it with ``compare()``, run the filter over it, and -compare the live runs with the filtered result, without Cognee installed. -Regenerate it with ``COGNEE_RECORD_FIXTURE=1``. - -All keys are placeholders. Nothing here contacts an LLM provider, Cognee's -telemetry host or Future AGI. -""" - -from __future__ import annotations - -import base64 -import concurrent.futures -import copy -import importlib.util -import json -import os -import re -import sys -import traceback -from pathlib import Path -from typing import Any - -import pytest -from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter -from opentelemetry.sdk.resources import Resource -from opentelemetry.sdk.trace import Event, ReadableSpan, TracerProvider -from opentelemetry.sdk.trace.export import SimpleSpanProcessor, SpanExportResult -from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter -from opentelemetry.trace import SpanContext, SpanKind, Status, StatusCode, TraceFlags - -from harness import Receiver, compare, post_otlp, run - -TESTS_DIR = Path(__file__).resolve().parent -RECIPE_DIR = TESTS_DIR.parent -SRC_DIR = RECIPE_DIR / "src" -PYTHON_DIR = RECIPE_DIR.parents[1] -REPO_ROOT = PYTHON_DIR.parent -APP = SRC_DIR / "app.py" -README = RECIPE_DIR / "README.md" -REQUIREMENTS = RECIPE_DIR / "requirements.txt" -ADD_SCRIPT = TESTS_DIR / "cognee_add.py" -# Put first on a scenario's PYTHONPATH, so every interpreter it starts loads the guard. -GUARD_DIR = TESTS_DIR / "loopback_guard" -FIXTURE = TESTS_DIR / "fixtures" / "cognee-1.6.2-recipe-spans.json" - -sys.path.insert(0, str(TESTS_DIR)) -sys.path.insert(0, str(SRC_DIR)) -from _fake_openai import ANSWER_MARKER, FakeOpenAI # noqa: E402 - -PROJECT = "cognee-recipe-contract" -FI_API_KEY = "fi-api-placeholder-0000" -FI_SECRET_KEY = "fi-secret-placeholder-0000" -LLM_API_KEY = "llm-placeholder-key-0000" -LLM_MODEL = "openai/gpt-4o-mini" -EMBEDDING_MODEL = "openai/text-embedding-3-small" - -# Content markers: the document text, the search question and the fake -# model's answer. -DOC_MARKER = "DMARK7f3a" -QUERY_MARKER = "QMARK91c2" -DOCUMENT = "Ada works on the Lighthouse project. " + DOC_MARKER -QUESTION = "Who works on Lighthouse? " + QUERY_MARKER -MARKERS = (DOC_MARKER, QUERY_MARKER, ANSWER_MARKER) - -# Which attribute keys carry each kind of content in Cognee's own spans -# (README.md, "Content"). With capture_content=True the filter keeps them. -CONTENT_KEYS = { - DOC_MARKER: {"langfuse.observation.input"}, - QUERY_MARKER: {"langfuse.observation.input", "cognee.search.query", "memory.query.text"}, - ANSWER_MARKER: {"langfuse.observation.output"}, -} -# What the filter removes by default: every key above, plus the graph query -# text (cognee.db.query, which held no marker in the recorded run). -FILTERED_KEYS = set().union(*CONTENT_KEYS.values()) | {"cognee.db.query"} -# With capture_content=True the filter also copies that content to the keys -# fi-collector shows as a span's Input and Output (README.md, "The export -# filter"): the prompt or the question to input.value, the response to -# output.value. -CAPTURE_IO_KEYS = { - DOC_MARKER: {"input.value"}, - QUERY_MARKER: {"input.value"}, - ANSWER_MARKER: {"output.value"}, -} -# Every key the filter adds in capture mode (none of them is Cognee's own). -IO_KEYS = ("input.value", "input.mime_type", "output.value", "output.mime_type") -# Cognee probes vector collections that do not exist yet; each probe span -# ends in ERROR with this exception in every successful run. By default the -# filter exports such a span as OK with MISSING_KEY (README.md, "The export -# filter"); COGNEE_FI_COLLECTION_MISSING_AS_ERROR=true keeps it ERROR. -VECTOR_SEARCH = "cognee.db.vector.search" -COLLECTION_MISSING = ( - "cognee.infrastructure.databases.vector.exceptions.exceptions.CollectionNotFoundError" -) -MISSING_KEY = "cognee.vector_search.collection_missing" -MISSING_SWITCH = "COGNEE_FI_COLLECTION_MISSING_AS_ERROR" -# What the filter puts in place of error text by default. -DETAIL_REMOVED = "__REDACTED__ (content capture off)" -TYPE_ONLY = " (detail removed: content capture off)" -TRACEBACK_HEADER = "Traceback (most recent call last):" - -APP_SPAN = "remember_and_ask" -INTERNAL = "SPAN_KIND_INTERNAL" -CLIENT = "SPAN_KIND_CLIENT" -GENERATION_SPAN = "cognee.observe.acreate_structured_output" -EMBEDDING_SPAN = "cognee.observe.embed_text" -# Every span name the recipe run produces, with its OTel span kind -# (README.md, "Spans Cognee 1.6.2 emits"). -SPAN_KINDS = { - APP_SPAN: INTERNAL, - "memory.store": INTERNAL, - "memory.process": INTERNAL, - "memory.retrieve": INTERNAL, - "cognee.pipeline.task.resolve_data_directories": INTERNAL, - "cognee.pipeline.task.ingest_data": INTERNAL, - "cognee.pipeline.task.classify_documents": INTERNAL, - "cognee.pipeline.task.extract_chunks_from_documents": INTERNAL, - "cognee.pipeline.task.extract_graph_and_summarize": INTERNAL, - "cognee.pipeline.task.add_data_points": INTERNAL, - GENERATION_SPAN: CLIENT, - EMBEDDING_SPAN: CLIENT, - "cognee.llm.completion": INTERNAL, - "cognee.db.graph.query": INTERNAL, - "cognee.db.vector.search": INTERNAL, - "cognee.search.authorize": INTERNAL, - "cognee.search.dataset": INTERNAL, - "cognee.retrieval.get_objects": INTERNAL, - "cognee.retrieval.triplet_search": INTERNAL, - "cognee.retrieval.vector_search": INTERNAL, - "cognee.retrieval.embed_query": INTERNAL, - "cognee.retrieval.get_context": INTERNAL, - "cognee.retrieval.get_completion": INTERNAL, - "cognee.session.get_session": INTERNAL, - "cognee.session.add_qa": INTERNAL, -} -COGNEE_ROOTS = ("memory.store", "memory.process", "memory.retrieve") -# fi-collector reads the span kind from these keys, then falls back to -# gen_ai.operation.name (exporter/clickhouse25exporter/converter.go). -SPAN_KIND_KEYS = ("fi.span.kind", "gen_ai.span.kind", "llm.request.type", "openinference.span.kind") -# The fi.span.kind the filter sets, by span name, on the recipe run's spans. -FILTER_KINDS = { - "memory.retrieve": "RETRIEVER", - "cognee.search.authorize": "RETRIEVER", - "cognee.search.dataset": "RETRIEVER", - GENERATION_SPAN: "LLM", -} -TOKEN_KEYS = ( - "llm.token_count.prompt", - "llm.token_count.completion", - "llm.token_count.total", - "gen_ai.usage.input_tokens", - "gen_ai.usage.output_tokens", - "gen_ai.usage.total_tokens", - "llm.usage.prompt_tokens", - "llm.usage.completion_tokens", - "llm.usage.total_tokens", -) - -# Production URL forms for the no-register() option, and whether Cognee -# 1.6.2 treats each as HTTP-only (tracing.py _requires_http_exporter). -OTLP_URL_FORMS = { - "https://api.futureagi.com/tracer/v1/traces": False, - "https://api.futureagi.com:443/tracer/v1/traces": True, -} -RUN_TIMEOUT_SECONDS = 900 - -COGNEE_INSTALLED = importlib.util.find_spec("cognee") is not None - - -def litellm_tokenizers_dir() -> str | None: - """LiteLLM's bundled tiktoken files, so Cognee never downloads them. - - Cognee 1.6.5 builds its embedding tokenizer with tiktoken before LiteLLM - has pointed TIKTOKEN_CACHE_DIR at the BPE files it ships, so tiktoken would - fetch cl100k_base from openaipublic.blob.core.windows.net, which the - loopback guard refuses. A test-environment setting only: the recipe itself - needs no such variable. - """ - spec = importlib.util.find_spec("litellm") - for location in (spec.submodule_search_locations or []) if spec else []: - tokenizers = Path(location) / "litellm_core_utils" / "tokenizers" - if tokenizers.is_dir(): - return str(tokenizers) - return None - - -TOKENIZERS_DIR = litellm_tokenizers_dir() - - -# -------------------------------------------------------------------------- -# Helpers shared by the live run and the recorded fixture -# -------------------------------------------------------------------------- - - -def attributes(span: dict[str, Any]) -> dict[str, Any]: - flat = {} - for attribute in span.get("attributes", []): - value = attribute.get("value", {}) - flat[attribute["key"]] = next(iter(value.values())) if value else None - return flat - - -def spans_named(spans: list[dict[str, Any]], name: str) -> list[dict[str, Any]]: - return [span for span in spans if span["name"] == name] - - -# Spans whose number depends on the Cognee patch release rather than on the -# recipe: one add/cognify/search run issues 14 graph queries on cognee 1.6.2 -# and 17 on 1.6.5. shape() compares their names, kinds, keys and statuses, not -# their count. -VERSION_DEPENDENT_COUNTS = frozenset({"cognee.db.graph.query"}) - - -def shape(spans: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: - """Per span name: kinds, attribute keys, status codes and count.""" - out: dict[str, dict[str, Any]] = {} - for span in spans: - entry = out.setdefault( - span["name"], {"kinds": set(), "keys": set(), "status": set(), "count": 0} - ) - entry["kinds"].add(span.get("kind")) - entry["keys"] |= set(attributes(span)) - entry["status"].add(span.get("status", {}).get("code", "STATUS_CODE_UNSET")) - entry["count"] += 1 - for name in VERSION_DEPENDENT_COUNTS & set(out): - out[name]["count"] = "at least 1" - return out - - -def host_paths() -> tuple[str, ...]: - """Machine-local path fragments that must not reach a recorded fixture.""" - return (str(Path.home()), "/Users/", "/home/", "/tmp/pytest-of-", "/var/folders", "/private/") - - -def scrub(spans: list[dict[str, Any]], tmp_base: Path) -> list[dict[str, Any]]: - """Replace machine-local paths (stack traces, data dirs) before recording.""" - text = json.dumps(spans) - for path, label in ( - (os.path.realpath(tmp_base), ""), - (str(tmp_base), ""), - (str(REPO_ROOT), ""), - ): - text = text.replace(path, label) - text = re.sub(r'File \\"[^"\\]*?/lib/python3\.\d+/', r'File \\"/', text) - text = text.replace(str(Path.home()), "") - return json.loads(text) - - -def without_filter_kinds(spans: list[dict[str, Any]]) -> list[dict[str, Any]]: - """Drop the keys the filter adds (fi.span.kind, capture-mode I/O), leaving Cognee's spans.""" - added = {"fi.span.kind", *IO_KEYS} - spans = copy.deepcopy(spans) - for span in spans: - span["attributes"] = [a for a in span.get("attributes", []) if a["key"] not in added] - return spans - - -def assert_span_names_and_kinds(spans: list[dict[str, Any]]) -> None: - assert {span["name"]: span["kind"] for span in spans} == SPAN_KINDS - for span in spans: - assert span["kind"] == SPAN_KINDS[span["name"]], span["name"] - - -def assert_collector_mapping_inputs(spans: list[dict[str, Any]], kinds: dict[str, str]) -> None: - """Assert the keys fi-collector maps from, as README.md's table states. - - ``kinds`` is the fi.span.kind expected per span name: empty for Cognee's - own spans, FILTER_KINDS after the recipe's export filter. - """ - for span in spans: - attrs = attributes(span) - expected_kind = {"fi.span.kind": kinds[span["name"]]} if span["name"] in kinds else {} - assert {key: attrs[key] for key in SPAN_KIND_KEYS if key in attrs} == expected_kind - assert not set(TOKEN_KEYS) & set(attrs), span["name"] - if span["name"] == EMBEDDING_SPAN: - assert attrs["gen_ai.operation.name"] == "embeddings" - assert attrs["gen_ai.request.model"] == EMBEDDING_MODEL - assert attrs["gen_ai.provider.name"] == "openai" - assert attrs["cognee.span.category"] == "embeddings" - else: - assert "gen_ai.operation.name" not in attrs, span["name"] - if span["name"] == GENERATION_SPAN: - # AC-04: the model key is present, so no copy processor is written. - assert attrs["gen_ai.request.model"] == LLM_MODEL - assert attrs["gen_ai.system"] == "litellm-native" - assert attrs["cognee.span.category"] == "generation" - assert attrs["langfuse.observation.type"] == "generation" - generation_spans = spans_named(spans, GENERATION_SPAN) - assert generation_spans - # cognee.llm.model is set only on schema-bound (structured) calls. - with_cognee_model = [s for s in generation_spans if "cognee.llm.model" in attributes(s)] - assert 0 < len(with_cognee_model) < len(generation_spans) - (search,) = spans_named(spans, "memory.retrieve") - search_attrs = attributes(search) - assert search_attrs["cognee.search.type"] == "GRAPH_COMPLETION" - assert search_attrs["memory.query.type"] == "GRAPH_COMPLETION" - assert search_attrs["memory.operation"] == "retrieve" - - -def assert_default_content(spans: list[dict[str, Any]], io_copies: bool = True) -> None: - """Among span attributes, captured content is under exactly the documented keys. - - Those are Cognee's own keys plus, after the filter in capture mode - (``io_copies``), its input.value/output.value copies. Error text - (exception events, status) is not checked here: with content capture on - it is exported as Cognee recorded it (README.md, "Content"). - """ - for marker, expected_keys in CONTENT_KEYS.items(): - found = { - key - for span in spans - for key, value in attributes(span).items() - if marker in str(value) - } - assert found == expected_keys | (CAPTURE_IO_KEYS[marker] if io_copies else set()), marker - for span in spans_named(spans, EMBEDDING_SPAN): - assert DOC_MARKER not in json.dumps(span) - - -def found_in(payload: Any, needles: Any) -> list[str]: - """The needles that occur in payload's JSON (a list keeps pytest's diff of it small).""" - text = json.dumps(payload) - return [needle for needle in needles if needle in text] - - -def assert_no_content(spans: list[dict[str, Any]]) -> None: - """No marker anywhere on the wire (attributes, status, events), no content key.""" - assert found_in(spans, MARKERS) == [] - for span in spans: - keys = set(attributes(span)) - for event in span.get("events", []): - keys |= set(attributes(event)) - assert not keys & FILTERED_KEYS, span["name"] - - -def assert_no_secrets(payload: Any) -> None: - assert found_in(payload, (FI_API_KEY, FI_SECRET_KEY, LLM_API_KEY)) == [] - - -def assert_one_trace_under_app_span(spans: list[dict[str, Any]]) -> None: - assert len({span["traceId"] for span in spans}) == 1 - roots = [span for span in spans if not span.get("parentSpanId")] - assert [root["name"] for root in roots] == [APP_SPAN] - root_id = roots[0]["spanId"] - for name in COGNEE_ROOTS: - (span,) = spans_named(spans, name) - assert span["parentSpanId"] == root_id, name - by_id = {span["spanId"]: span for span in spans} - assert len(by_id) == len(spans) - for span in spans: - if span.get("parentSpanId"): - assert span["parentSpanId"] in by_id, span["name"] - - -# -------------------------------------------------------------------------- -# Recorded spans as SDK spans, sent through the export filter -# -------------------------------------------------------------------------- - - -def otlp_value(value: dict[str, Any]) -> Any: - ((kind, raw),) = value.items() - if kind == "intValue": - return int(raw) - if kind == "arrayValue": - return [otlp_value(item) for item in raw.get("values", [])] - return raw - - -def sdk_span(span: dict[str, Any]) -> ReadableSpan: - """Rebuild an SDK span from its Receiver-decoded (OTLP JSON) form.""" - - def span_id(key: str) -> int: - return int.from_bytes(base64.b64decode(span[key]), "big") - - def flat(items: list[dict[str, Any]]) -> dict[str, Any]: - return {item["key"]: otlp_value(item["value"]) for item in items} - - trace_id = span_id("traceId") - parent = None - if span.get("parentSpanId"): - parent = SpanContext(trace_id, span_id("parentSpanId"), is_remote=False) - status = span.get("status", {}) - return ReadableSpan( - name=span["name"], - context=SpanContext( - trace_id, span_id("spanId"), is_remote=False, trace_flags=TraceFlags(TraceFlags.SAMPLED) - ), - parent=parent, - resource=Resource({"project_name": PROJECT, "project_type": "observe"}), - attributes=flat(span.get("attributes", [])), - events=[ - Event(event["name"], flat(event.get("attributes", [])), int(event["timeUnixNano"])) - for event in span.get("events", []) - ], - kind=SpanKind[span["kind"].replace("SPAN_KIND_", "")], - status=Status( - StatusCode[status.get("code", "STATUS_CODE_UNSET").replace("STATUS_CODE_", "")], - status.get("message"), - ), - start_time=int(span["startTimeUnixNano"]), - end_time=int(span["endTimeUnixNano"]), - ) - - -def export_spans(spans: list[dict[str, Any]], wrap: Any = None) -> list[dict[str, Any]]: - """Export recorded spans over OTLP/HTTP, optionally through ``wrap(exporter)``.""" - with Receiver() as receiver: - exporter = OTLPSpanExporter(endpoint=receiver.collector_endpoint) - if wrap is not None: - exporter = wrap(exporter) - assert exporter.export([sdk_span(span) for span in spans]) == SpanExportResult.SUCCESS - exporter.shutdown() - return receiver.spans() - - -def filtered( - cognee_filter: Any, spans: list[dict[str, Any]], **options: Any -) -> list[dict[str, Any]]: - """Export recorded spans through src/cognee_filter.py; return what arrives.""" - return export_spans( - spans, lambda exporter: cognee_filter.CogneeExportFilter(exporter, **options) - ) - - -def error_names(spans: Any) -> list[str]: - return [s["name"] for s in spans if s.get("status", {}).get("code") == "STATUS_CODE_ERROR"] - - -def is_missing_collection_probe(span: dict[str, Any]) -> bool: - """A recorded vector search whose only error is CollectionNotFoundError.""" - if span["name"] != VECTOR_SEARCH: - return False - if span.get("status", {}).get("code") != "STATUS_CODE_ERROR": - return False - events = span.get("events", []) - return bool(events) and all( - event["name"] == "exception" - and attributes(event).get("exception.type") == COLLECTION_MISSING - for event in events - ) - - -def without_error_detail(span: dict[str, Any]) -> tuple[list[dict[str, Any]], dict[str, Any]]: - """A recorded span's events and status as the filter should export them by default. - - A missing-collection probe is exported as OK, without its exception event. - - Default content-off mode keeps the fixed marker, exception type and - exception.escaped, but omits serialized exception stack traces entirely. - """ - if is_missing_collection_probe(span): - return [], {"code": "STATUS_CODE_OK"} - events = copy.deepcopy(span.get("events", [])) - for event in events: - event["attributes"] = [ - attribute - for attribute in event["attributes"] - if attribute["key"] != "exception.stacktrace" - ] - for attribute in event["attributes"]: - if attribute["key"] == "exception.message": - attribute["value"] = {"stringValue": DETAIL_REMOVED} - status = dict(span.get("status", {})) - if "message" in status: - status["message"] = attributes(events[-1])["exception.type"] + TYPE_ONLY - return events, status - - -def without_stacktraces(events: list[dict[str, Any]]) -> list[dict[str, Any]]: - """Recorded events as the filter exports them with capture on. - - Content capture keeps exception messages, but exception.stacktrace is - never exported: serialized tracebacks carry local paths (and can quote - content). - """ - events = copy.deepcopy(events) - for event in events: - event["attributes"] = [ - attribute - for attribute in event["attributes"] - if attribute["key"] != "exception.stacktrace" - ] - return events - - -def filtered_attributes(span: dict[str, Any], capture_content: bool) -> dict[str, Any]: - """A recorded span's attributes as the filter should export them.""" - expected = { - key: value - for key, value in attributes(span).items() - if capture_content or key not in FILTERED_KEYS - } - if span["name"] in FILTER_KINDS: - expected["fi.span.kind"] = FILTER_KINDS[span["name"]] - if capture_content: - expected.update(expected_io(attributes(span))) - if is_missing_collection_probe(span): - expected[MISSING_KEY] = True - return expected - - -def expected_io(attrs: dict[str, Any]) -> dict[str, Any]: - """The Input/Output keys capture mode adds for a span with Cognee's ``attrs``.""" - def mime(value: str) -> str: - # Cognee serializes an LLM call's arguments as a JSON object, but cuts - # it at 8000 characters, so a long prompt is no longer valid JSON. - try: - is_json = isinstance(json.loads(value), (dict, list)) - except ValueError: - is_json = False - return "application/json" if is_json else "text/plain" - - io: dict[str, Any] = {} - prompt = attrs.get("langfuse.observation.input") - question = attrs.get("memory.query.text", attrs.get("cognee.search.query")) - response = attrs.get("langfuse.observation.output") - if prompt is not None: - io.update({"input.value": prompt, "input.mime_type": mime(prompt)}) - elif question is not None: - io.update({"input.value": question, "input.mime_type": mime(question)}) - if response is not None: - io.update({"output.value": response, "output.mime_type": mime(response)}) - return io - - -@pytest.fixture(scope="module") -def cognee_filter() -> Any: - """src/cognee_filter.py. Imported here so a missing filter fails only its tests.""" - import cognee_filter as module - - return module - - -# -------------------------------------------------------------------------- -# Live runs -# -------------------------------------------------------------------------- - - -class Scenario: - """One guarded subprocess with its own Receiver, fake model and data dir.""" - - def __init__(self, name: str, script: Path, args: list[str], work: Path) -> None: - self.name = name - self.script = script - self.args = args - self.work = work - self.guard_log = work / "guard.jsonl" - self.result: Any = None - self.spans: list[dict[str, Any]] = [] - self.requests: list[dict[str, Any]] = [] - self.fake_requests: list[dict[str, Any]] = [] - - def env(self, receiver: Receiver, fake: FakeOpenAI) -> dict[str, str]: - env = { - "PATH": os.environ.get("PATH", ""), - "HOME": str(self.work), - "PYTHONPATH": os.pathsep.join((str(GUARD_DIR), str(PYTHON_DIR))), - "LOOPBACK_GUARD_LOG": str(self.guard_log), - # Cognee's product telemetry is separate from OTel tracing. - "TELEMETRY_DISABLED": "1", - # Keep LiteLLM from fetching its model cost map at import. - "LITELLM_LOCAL_MODEL_COST_MAP": "True", - # Offline tokenizer files (see litellm_tokenizers_dir). - **({"TIKTOKEN_CACHE_DIR": TOKENIZERS_DIR} if TOKENIZERS_DIR else {}), - "LLM_PROVIDER": "openai", - "LLM_MODEL": LLM_MODEL, - "LLM_ENDPOINT": fake.base_url, - "LLM_API_KEY": LLM_API_KEY, - "EMBEDDING_PROVIDER": "openai", - "EMBEDDING_MODEL": EMBEDDING_MODEL, - "EMBEDDING_ENDPOINT": fake.base_url, - "EMBEDDING_API_KEY": LLM_API_KEY, - "EMBEDDING_DIMENSIONS": str(fake.dimensions), - "DATA_ROOT_DIRECTORY": str(self.work / "data"), - "SYSTEM_ROOT_DIRECTORY": str(self.work / "system"), - "CACHE_ROOT_DIRECTORY": str(self.work / "cache"), - "COGNEE_LOGS_DIR": str(self.work / "logs"), - } - if self.name != "tracing_off": - env["COGNEE_TRACING_ENABLED"] = "true" - if self.name == "capture": - env["COGNEE_FI_CAPTURE_CONTENT"] = "true" - if self.name == "probes_as_errors": - env[MISSING_SWITCH] = "true" - if self.name == "otlp_env": - # The no-register() option: Cognee's own exporter, configured - # exactly as README.md says (with the Receiver's origin). - env["OTEL_EXPORTER_OTLP_ENDPOINT"] = receiver.collector_endpoint - env["OTEL_EXPORTER_OTLP_HEADERS"] = "X-Api-Key={0},X-Secret-Key={1}".format( - FI_API_KEY, FI_SECRET_KEY - ) - env["OTEL_RESOURCE_ATTRIBUTES"] = "project_name={0},project_type=observe".format( - PROJECT - ) - else: - env.update( - { - "FI_BASE_URL": receiver.origin, - "FI_API_KEY": FI_API_KEY, - "FI_SECRET_KEY": FI_SECRET_KEY, - "FI_PROJECT_NAME": PROJECT, - } - ) - return env - - def execute(self) -> "Scenario": - with Receiver() as receiver: - fake = FakeOpenAI(dimensions=16) - try: - self.result = run( - [sys.executable, str(self.script), *self.args], - env=self.env(receiver, fake), - stdin=None, - timeout=RUN_TIMEOUT_SECONDS, - ) - self.spans = receiver.spans() - self.requests = receiver.requests() - self.fake_requests = fake.requests() - finally: - fake.close() - return self - - @property - def stdout(self) -> str: - return self.result.stdout.decode("utf-8", "replace") - - @property - def stderr(self) -> str: - return self.result.stderr.decode("utf-8", "replace") - - def guard_records(self) -> list[dict[str, Any]]: - if not self.guard_log.exists(): - return [] - lines = self.guard_log.read_text(encoding="utf-8").splitlines() - return [json.loads(line) for line in lines] - - def blocked(self) -> list[dict[str, Any]]: - return [record for record in self.guard_records() if record["kind"] != "installed"] - - def guarded_workers(self) -> list[dict[str, Any]]: - """Processes spawned by multiprocessing that loaded the guard.""" - return [ - record - for record in self.guard_records() - if record["kind"] == "installed" and "--multiprocessing-fork" in record["argv"] - ] - - def assert_ran_offline(self) -> None: - assert not self.result.timed_out, self.stderr[-4000:] - assert self.result.returncode == 0, self.stderr[-4000:] - assert self.blocked() == [] - - def json_line(self) -> dict[str, Any]: - lines = [line for line in self.stdout.splitlines() if line.startswith("{")] - return json.loads(lines[-1]) - - -@pytest.fixture(scope="module") -def runs(tmp_path_factory: pytest.TempPathFactory) -> dict[str, Scenario]: - if not COGNEE_INSTALLED: - pytest.skip("cognee is not installed") - scenarios = [ - Scenario("recipe", APP, [QUESTION, DOCUMENT], tmp_path_factory.mktemp("recipe")), - Scenario("capture", APP, [QUESTION, DOCUMENT], tmp_path_factory.mktemp("capture")), - Scenario( - "probes_as_errors", APP, [QUESTION, DOCUMENT], tmp_path_factory.mktemp("probes") - ), - Scenario("tracing_off", APP, [QUESTION, DOCUMENT], tmp_path_factory.mktemp("off")), - Scenario("no_readd", ADD_SCRIPT, ["no-readd"], tmp_path_factory.mktemp("noreadd")), - Scenario( - "otlp_env", ADD_SCRIPT, ["otlp-env", *OTLP_URL_FORMS], tmp_path_factory.mktemp("env") - ), - ] - # Importing cognee alone takes about a minute, so the processes run side - # by side. - with concurrent.futures.ThreadPoolExecutor(max_workers=len(scenarios)) as pool: - done = list(pool.map(Scenario.execute, scenarios)) - by_name = {scenario.name: scenario for scenario in done} - capture = by_name["capture"] - if os.environ.get("COGNEE_RECORD_FIXTURE") == "1" and capture.result.returncode == 0: - # The fixture is Cognee's own output: the capture run (all content - # kept) without the fi.span.kind the filter adds. - spans = scrub(without_filter_kinds(capture.spans), tmp_path_factory.getbasetemp()) - FIXTURE.write_text(json.dumps(spans, indent=1) + "\n", encoding="utf-8") - return by_name - - -def test_recipe_runs_offline_against_the_fakes(runs: dict[str, Scenario]) -> None: - recipe = runs["recipe"] - recipe.assert_ran_offline() - assert ANSWER_MARKER in recipe.stdout - paths = {request["path"] for request in recipe.fake_requests} - assert paths == {"/v1/chat/completions", "/v1/embeddings"} - - -def test_loopback_guard_also_runs_in_cognee_worker_processes(runs: dict[str, Scenario]) -> None: - # Cognee 1.6.2 runs LanceDB and Ladybug in multiprocessing "spawn" - # workers; the guard must be installed there too, not only in the - # scenario's main process (each scenario's assert_ran_offline checks - # that nothing was refused). - for name, scenario in runs.items(): - assert scenario.guarded_workers(), name - - -def test_recipe_exports_to_collector_path_with_both_auth_headers( - runs: dict[str, Scenario], -) -> None: - requests = runs["recipe"].requests - assert requests - for request in requests: - assert request["path"] == "/tracer/v1/traces" - assert request["headers"]["x-api-key"] == FI_API_KEY - assert request["headers"]["x-secret-key"] == FI_SECRET_KEY - assert request["headers"]["content-type"] == "application/x-protobuf" - assert LLM_API_KEY not in json.dumps(request["headers"]) - - -def test_recipe_resource_is_an_observe_project(runs: dict[str, Scenario]) -> None: - resources = [ - resource for request in runs["recipe"].requests for resource in request["resource_attributes"] - ] - assert resources - for resource in resources: - assert resource["project_name"] == PROJECT - assert resource["project_type"] == "observe" - # One provider: every export carries the same register() resource. - assert len({json.dumps(resource, sort_keys=True) for resource in resources}) == 1 - - -def test_recipe_spans_match_documented_names_and_kinds(runs: dict[str, Scenario]) -> None: - assert_span_names_and_kinds(runs["recipe"].spans) - - -def test_recipe_puts_add_cognify_search_in_one_trace(runs: dict[str, Scenario]) -> None: - assert_one_trace_under_app_span(runs["recipe"].spans) - - -def test_recipe_does_not_duplicate_spans(runs: dict[str, Scenario]) -> None: - recipe = runs["recipe"] - chat_calls = [r for r in recipe.fake_requests if r["path"] == "/v1/chat/completions"] - embedding_calls = [r for r in recipe.fake_requests if r["path"] == "/v1/embeddings"] - assert len(spans_named(recipe.spans, GENERATION_SPAN)) == len(chat_calls) - assert len(spans_named(recipe.spans, EMBEDDING_SPAN)) == len(embedding_calls) - assert len({span["spanId"] for span in recipe.spans}) == len(recipe.spans) - - -def test_recipe_marks_search_and_llm_spans_for_the_collector(runs: dict[str, Scenario]) -> None: - # memory.retrieve and cognee.search.* arrive as RETRIEVER, LLM calls as - # LLM; embedding spans keep gen_ai.operation.name only. - assert_collector_mapping_inputs(runs["recipe"].spans, FILTER_KINDS) - - -def test_recipe_exports_no_content_by_default(runs: dict[str, Scenario]) -> None: - assert_no_content(runs["recipe"].spans) - - -def test_recipe_exports_missing_collection_probes_as_ok(runs: dict[str, Scenario]) -> None: - # Cognee's vector-search probes of collections that do not exist yet fail - # in every run. By default they arrive as OK, flagged, and nothing in a - # successful run is an error. - for name in ("recipe", "capture"): - spans = runs[name].spans - assert error_names(spans) == [], name - flagged = [s for s in spans if attributes(s).get(MISSING_KEY) is True] - assert flagged, name - for span in flagged: - assert span["name"] == VECTOR_SEARCH - assert span["status"] == {"code": "STATUS_CODE_OK"} - assert span.get("events", []) == [] - - -def test_switch_keeps_probes_as_errors_without_detail( - runs: dict[str, Scenario], recorded: list[dict[str, Any]], cognee_filter: Any -) -> None: - # With COGNEE_FI_COLLECTION_MISSING_AS_ERROR=true the probes stay ERROR; - # with capture off only fixed markers and the exception type arrive. - run = runs["probes_as_errors"] - run.assert_ran_offline() - errors = [s for s in run.spans if s.get("status", {}).get("code") == "STATUS_CODE_ERROR"] - assert errors - for span in errors: - assert span["name"] == VECTOR_SEARCH - assert MISSING_KEY not in attributes(span) - assert span["events"], span["name"] - for event in span["events"]: - event_attrs = attributes(event) - assert event_attrs["exception.type"] == COLLECTION_MISSING - assert event_attrs["exception.message"] == DETAIL_REMOVED - assert span["status"]["message"] == COLLECTION_MISSING + TYPE_ONLY - assert "exception.stacktrace" not in event_attrs - assert shape(run.spans) == shape( - filtered(cognee_filter, recorded, collection_missing_as_error=True) - ) - - -def test_recipe_exports_no_secrets(runs: dict[str, Scenario]) -> None: - recipe = runs["recipe"] - assert_no_secrets(recipe.spans) - assert_no_secrets([request["resource_attributes"] for request in recipe.requests]) - - -def test_recipe_run_matches_recorded_fixture( - runs: dict[str, Scenario], recorded: list[dict[str, Any]], cognee_filter: Any -) -> None: - assert shape(runs["recipe"].spans) == shape(filtered(cognee_filter, recorded)) - - -def test_capture_content_exports_content_under_documented_keys( - runs: dict[str, Scenario], -) -> None: - capture = runs["capture"] - capture.assert_ran_offline() - assert_default_content(capture.spans) - assert_no_secrets(capture.spans) - - -def test_capture_run_matches_recorded_fixture( - runs: dict[str, Scenario], recorded: list[dict[str, Any]], cognee_filter: Any -) -> None: - capture = runs["capture"] - assert_collector_mapping_inputs(capture.spans, FILTER_KINDS) - assert shape(capture.spans) == shape(filtered(cognee_filter, recorded, capture_content=True)) - - -def test_tracing_off_exports_no_cognee_spans(runs: dict[str, Scenario]) -> None: - off = runs["tracing_off"] - off.assert_ran_offline() - assert ANSWER_MARKER in off.stdout - # Only the app's own parent span is left. - assert [span["name"] for span in off.spans] == [APP_SPAN] - - -def test_register_without_the_readd_line_exports_nothing(runs: dict[str, Scenario]) -> None: - no_readd = runs["no_readd"] - no_readd.assert_ran_offline() - # Cognee made spans (its in-memory buffer holds them) ... - assert no_readd.json_line()["buffered_spans"] > 0 - # ... but register()'s exporter was replaced, so none left the process. - assert no_readd.requests == [] - assert no_readd.spans == [] - - -def test_otlp_env_option_posts_to_the_configured_path(runs: dict[str, Scenario]) -> None: - otlp = runs["otlp_env"] - otlp.assert_ran_offline() - assert otlp.requests - for request in otlp.requests: - # Cognee passes the endpoint through unchanged: no second /v1/traces. - assert request["path"] == "/tracer/v1/traces" - assert request["headers"]["x-api-key"] == FI_API_KEY - assert request["headers"]["x-secret-key"] == FI_SECRET_KEY - for resource in request["resource_attributes"]: - assert resource["project_name"] == PROJECT - assert resource["project_type"] == "observe" - assert resource["service.name"] == "cognee" - names = [span["name"] for span in otlp.spans] - assert "memory.store" in names - assert len(names) == otlp.json_line()["buffered_spans"] - assert_no_secrets(otlp.spans) - - -def test_otlp_env_option_also_sends_logs_the_collector_rejects( - runs: dict[str, Scenario], -) -> None: - # Cognee derives /../v1/logs from the traces URL; fi-collector - # (and the Receiver) answer 404 there. - assert "Failed to export logs batch code: 404" in runs["otlp_env"].stderr - - -def test_otlp_env_transport_choice_for_documented_urls(runs: dict[str, Scenario]) -> None: - assert runs["otlp_env"].json_line()["http_only"] == OTLP_URL_FORMS - - -# -------------------------------------------------------------------------- -# Recorded fixture (no Cognee needed) -# -------------------------------------------------------------------------- - - -@pytest.fixture(scope="module") -def recorded() -> list[dict[str, Any]]: - return json.loads(FIXTURE.read_text(encoding="utf-8")) - - -def otlp_body(spans: list[dict[str, Any]]) -> dict[str, Any]: - resource = {"project_name": PROJECT, "project_type": "observe"} - return { - "resourceSpans": [ - { - "resource": { - "attributes": [ - {"key": key, "value": {"stringValue": value}} - for key, value in resource.items() - ] - }, - "scopeSpans": [{"spans": spans}], - } - ] - } - - -def test_recorded_fixture_round_trips_through_the_harness(recorded: list[dict[str, Any]]) -> None: - with Receiver() as receiver: - assert post_otlp(otlp_body(recorded), receiver.collector_endpoint) == 200 - compare(receiver.spans(), FIXTURE) - (request,) = receiver.requests() - assert request["path"] == "/tracer/v1/traces" - assert request["resource_attributes"] == [{"project_name": PROJECT, "project_type": "observe"}] - - -def test_recorded_fixture_matches_documented_mapping(recorded: list[dict[str, Any]]) -> None: - # Cognee's own spans: no span-kind key, content under the documented keys. - assert_span_names_and_kinds(recorded) - assert_one_trace_under_app_span(recorded) - assert_collector_mapping_inputs(recorded, {}) - assert_default_content(recorded, io_copies=False) - assert_no_secrets(recorded) - assert found_in(recorded, host_paths()) == [] - - -def test_recorded_error_spans_are_vector_search_probes(recorded: list[dict[str, Any]]) -> None: - errors = [s for s in recorded if s.get("status", {}).get("code") == "STATUS_CODE_ERROR"] - searches = spans_named(recorded, VECTOR_SEARCH) - assert {span["name"] for span in errors} == {VECTOR_SEARCH} - assert all(is_missing_collection_probe(span) for span in errors) - for span in errors: - assert [event["name"] for event in span["events"]] == ["exception"] - assert "{0} of {1}".format(len(errors), len(searches)) in README.read_text(encoding="utf-8") - - -def test_scrub_removes_host_and_tmp_paths(tmp_path_factory: pytest.TempPathFactory) -> None: - base = tmp_path_factory.getbasetemp() - planted = [ - str(base / "recipe0" / "data" / "file.txt"), - os.path.realpath(base) + "/recipe0/system", - str(REPO_ROOT / "python" / "x.py"), - str(Path.home() / ".cache" / "uv" / "x.py"), - 'File "{0}/.venv/lib/python3.11/site-packages/x.py"'.format(Path.home()), - ] - scrubbed = scrub([{"name": "x", "attributes": [], "planted": planted}], base) - assert found_in(scrubbed, host_paths()) == [] - - -def test_fixture_spans_rebuild_as_sdk_spans(recorded: list[dict[str, Any]]) -> None: - # The helper the filter tests rely on: rebuilt spans export back to the - # same names, attributes, statuses and events. - wire = export_spans(recorded) - compare(wire, FIXTURE) - for before, after in zip(recorded, wire): - assert after.get("events", []) == before.get("events", []), before["name"] - - -# -------------------------------------------------------------------------- -# The export filter (src/cognee_filter.py), on the recorded spans -# -------------------------------------------------------------------------- - - -def test_filter_lists_the_tested_content_keys_and_span_names(cognee_filter: Any) -> None: - assert cognee_filter.CONTENT_KEYS == FILTERED_KEYS - assert cognee_filter.SPAN_KIND_KEYS == SPAN_KIND_KEYS - for name in SPAN_KINDS: - assert cognee_filter.span_kind(name) == FILTER_KINDS.get(name), name - - -def test_filter_keeps_content_off_the_wire_by_default( - cognee_filter: Any, recorded: list[dict[str, Any]] -) -> None: - wire = filtered(cognee_filter, recorded) - assert len(wire) == len(recorded) - assert_no_content(wire) - # Error text is replaced and serialized exception stack traces are omitted; - # everything else is unchanged. - for before, after in zip(recorded, wire): - assert attributes(after) == filtered_attributes(before, capture_content=False) - events, status = without_error_detail(before) - assert after.get("events", []) == events - assert after.get("status") == status - - -@pytest.mark.parametrize("capture_content", [False, True]) -def test_filter_exports_missing_collection_probes_as_ok( - cognee_filter: Any, recorded: list[dict[str, Any]], capture_content: bool -) -> None: - probes = [span for span in recorded if is_missing_collection_probe(span)] - assert len(probes) == 60 - wire = { - span["spanId"]: span - for span in filtered(cognee_filter, recorded, capture_content=capture_content) - } - assert error_names(wire.values()) == [] - for probe in probes: - after = wire[probe["spanId"]] - assert after["status"] == {"code": "STATUS_CODE_OK"} - assert after.get("events", []) == [] - assert attributes(after)[MISSING_KEY] is True - # Which collection was missing is still there. - collection = attributes(probe)["cognee.vector.collection"] - assert attributes(after)["cognee.vector.collection"] == collection - for span in recorded: - if not is_missing_collection_probe(span): - assert MISSING_KEY not in attributes(wire[span["spanId"]]), span["name"] - - -@pytest.mark.parametrize( - ("env_value", "argument", "kept_as_error"), - [ - (None, None, False), - ("true", None, True), - ("TRUE", None, True), - ("false", None, False), - ("true", False, False), - (None, True, True), - ], -) -def test_filter_collection_missing_switch( - cognee_filter: Any, - recorded: list[dict[str, Any]], - monkeypatch: pytest.MonkeyPatch, - env_value: Any, - argument: Any, - kept_as_error: bool, -) -> None: - if env_value is None: - monkeypatch.delenv(MISSING_SWITCH, raising=False) - else: - monkeypatch.setenv(MISSING_SWITCH, env_value) - options = {} if argument is None else {"collection_missing_as_error": argument} - probe = next(span for span in recorded if is_missing_collection_probe(span)) - (after,) = filtered(cognee_filter, [probe], **options) - if kept_as_error: - assert after["status"] == { - "code": "STATUS_CODE_ERROR", - "message": COLLECTION_MISSING + TYPE_ONLY, - } - assert MISSING_KEY not in attributes(after) - assert [event["name"] for event in after["events"]] == ["exception"] - else: - assert after["status"] == {"code": "STATUS_CODE_OK"} - assert attributes(after)[MISSING_KEY] is True - - -def test_filter_keeps_every_other_error(cognee_filter: Any, recorded: list[dict[str, Any]]) -> None: - probe = next(span for span in recorded if is_missing_collection_probe(span)) - when = probe["endTimeUnixNano"] - other_error = copy.deepcopy(probe) - other_error["status"] = {"code": "STATUS_CODE_ERROR", "message": "ValueError: bad vector"} - other_error["events"] = [exception_event("ValueError", "bad vector", "trace", when)] - mixed = copy.deepcopy(probe) - mixed["events"].append(exception_event("RuntimeError", "lost connection", "trace", when)) - elsewhere = copy.deepcopy(probe) - elsewhere["name"] = "cognee.retrieval.vector_search" - no_event = copy.deepcopy(probe) - no_event["events"] = [] - for capture_content in (False, True): - spans = [other_error, mixed, elsewhere, no_event] - wire = filtered(cognee_filter, spans, capture_content=capture_content) - assert len(wire) == 4 - for span in wire: - assert span["status"]["code"] == "STATUS_CODE_ERROR", span["name"] - assert MISSING_KEY not in attributes(span) - - -def test_filter_sets_span_kinds_only_where_none_is_set( - cognee_filter: Any, recorded: list[dict[str, Any]] -) -> None: - assert_collector_mapping_inputs(filtered(cognee_filter, recorded), FILTER_KINDS) - (search,) = copy.deepcopy(spans_named(recorded, "memory.retrieve")) - search["attributes"].append({"key": "gen_ai.span.kind", "value": {"stringValue": "CHAIN"}}) - (wire,) = filtered(cognee_filter, [search]) - assert attributes(wire)["gen_ai.span.kind"] == "CHAIN" - assert "fi.span.kind" not in attributes(wire) - - -def test_filter_removes_error_text_quoting_content( - cognee_filter: Any, recorded: list[dict[str, Any]] -) -> None: - # Cognee did not quote content in errors in the recorded run; plant it. - (search,) = copy.deepcopy(spans_named(recorded, "memory.retrieve")) - generation = copy.deepcopy( - next(s for s in spans_named(recorded, GENERATION_SPAN) if DOC_MARKER in json.dumps(s)) - ) - for span, quoted in ((search, QUESTION), (generation, DOCUMENT)): - span["status"] = {"code": "STATUS_CODE_ERROR", "message": "ValueError: bad " + quoted} - span["events"] = [ - exception_event( - "ValueError", - "bad " + quoted, - '{0}\n File "/app/x.py", line 3, in run\nValueError: bad {1}\n'.format( - TRACEBACK_HEADER, quoted - ), - span["endTimeUnixNano"], - ) - ] - wire = filtered(cognee_filter, [search, generation]) - assert_no_content(wire) - for span in wire: - assert span["status"]["message"] == "ValueError" + TYPE_ONLY - assert attributes(span["events"][0]) == { - "exception.type": "ValueError", - "exception.message": DETAIL_REMOVED, - "exception.escaped": "False", - } - - -# A content-policy rejection as Cognee 1.6.2 raises it (litellm_native/ -# native_adapter.py:512-518): the message quotes the whole prompt. The payload -# is longer than Cognee's 8000-character attribute cap (get_observe.py:12, :46) -# and has several lines; the marker is on every line, so also past the cap. -PAYLOAD_MARKER = "PMARK2c6d" -PAYLOAD = "\n".join([PAYLOAD_MARKER + " Ada works on the Lighthouse project."] * 220) -POLICY_ERROR = "cognee.infrastructure.llm.exceptions.ContentPolicyFilterError" -TASK_SPAN = "cognee.pipeline.task.extract_graph_and_summarize" -CHAIN_FRAME_MARKER = "PROMPT_SECRET_FIXTURE_ONLY" -# This is the exact separator, blank line, header and fake frame from -# tickets/TH-8328/fix-r3/red-probe.py. It is quoted by the exception message -# below, so it also occurs in the serialized exception.stacktrace. -FORGED_CHAIN_FRAME = "\n".join( - ( - "The above exception was the direct cause of the following exception:", - "", - TRACEBACK_HEADER, - ' File "/private/PROMPT_SECRET_FIXTURE_ONLY.py", line 1, in ' - "PROMPT_SECRET_FIXTURE_ONLY", - "end of quoted prompt", - ) -) -FORGED_CHAIN_PAYLOAD = ("ordinary oversized prompt text " * 320) + "\n" + FORGED_CHAIN_FRAME - - -class ContentPolicyViolationError(Exception): - """Stands in for litellm's exception of that name.""" - - -class ContentPolicyFilterError(Exception): - """Stands in for Cognee's ContentPolicyFilterError.""" - - -def raise_policy_rejection(payload: str = PAYLOAD) -> None: - try: - raise ContentPolicyViolationError("litellm.ContentPolicyViolationError: content_filter") - except ContentPolicyViolationError as error: - # str() of Cognee's error: CogneeApiError.__str__ (cognee/exceptions/exceptions.py:62-66). - raise ContentPolicyFilterError( - "CogneeValidationError: The provided input contains content that is not aligned " - "with our content policy: {0} (Status code: 422)".format(payload) - ) from error - - -def exception_event(type_: str, message: str, stacktrace: str, time: str) -> dict[str, Any]: - """The event OpenTelemetry's record_exception() adds, in OTLP JSON form.""" - return { - "timeUnixNano": time, - "name": "exception", - "attributes": [ - {"key": "exception.type", "value": {"stringValue": type_}}, - {"key": "exception.message", "value": {"stringValue": message}}, - {"key": "exception.stacktrace", "value": {"stringValue": stacktrace}}, - {"key": "exception.escaped", "value": {"stringValue": "False"}}, - ], - } - - -def policy_rejection_spans( - recorded: list[dict[str, Any]], payload: str = PAYLOAD -) -> list[dict[str, Any]]: - """A recorded LLM span and its pipeline task span, as a content-policy rejection ends them. - - The LLM span's input attribute is the prompt JSON cut at 8000 characters, - which does not parse. The task span has no content attribute and records - the error twice: run_tasks_base.py:243-245, then new_span's - start_as_current_span (observability/__init__.py:110-118). - """ - llm = copy.deepcopy( - next( - span - for span in spans_named(recorded, GENERATION_SPAN) - for parent in recorded - if parent["spanId"] == span["parentSpanId"] and parent["name"] == TASK_SPAN - ) - ) - (task,) = [copy.deepcopy(s) for s in recorded if s["spanId"] == llm["parentSpanId"]] - try: - raise_policy_rejection(payload) - except ContentPolicyFilterError as raised: - error = raised - stacktrace = "".join(traceback.format_exception(type(error), error, error.__traceback__)) - prompt = json.dumps({"text_input": payload, "system_prompt": "Extract a graph."})[:8000] - with pytest.raises(ValueError): - json.loads(prompt) - for attribute in llm["attributes"]: - if attribute["key"] == "langfuse.observation.input": - attribute["value"] = {"stringValue": prompt} - for span, recorded_times in ((llm, 1), (task, 2)): - span["status"] = { - "code": "STATUS_CODE_ERROR", - "message": "ContentPolicyFilterError: " + str(error), - } - span["events"] = [ - exception_event(POLICY_ERROR, str(error), stacktrace, span["endTimeUnixNano"]) - ] * recorded_times - return [llm, task] - - -def test_filter_removes_error_detail_on_every_span_by_default( - cognee_filter: Any, recorded: list[dict[str, Any]] -) -> None: - # N1: the rejection quotes the whole prompt in the exception message, the - # stack trace and the status, on the LLM span (whose cut-off input does not - # parse) and again on its pipeline task span (which has no content key). - spans = policy_rejection_spans(recorded) - assert found_in(spans, [PAYLOAD_MARKER]) == [PAYLOAD_MARKER] - wire = filtered(cognee_filter, spans) - assert [span["name"] for span in wire] == [GENERATION_SPAN, TASK_SPAN] - assert found_in(wire, [PAYLOAD_MARKER]) == [] - assert_no_content(wire) - for before, after in zip(spans, wire): - assert after["status"] == { - "code": "STATUS_CODE_ERROR", - "message": POLICY_ERROR + TYPE_ONLY, - } - assert len(after["events"]) == len(before["events"]) - for event_after in after["events"]: - event_attrs = attributes(event_after) - assert event_attrs["exception.type"] == POLICY_ERROR - assert event_attrs["exception.escaped"] == "False" - assert event_attrs["exception.message"] == DETAIL_REMOVED - assert "exception.stacktrace" not in event_attrs - - -def test_filter_capture_content_keeps_error_detail( - cognee_filter: Any, recorded: list[dict[str, Any]] -) -> None: - spans = policy_rejection_spans(recorded) - wire = filtered(cognee_filter, spans, capture_content=True) - for before, after in zip(spans, wire): - assert attributes(after) == filtered_attributes(before, capture_content=True) - assert after["events"] == without_stacktraces(before["events"]) - assert after["status"] == before["status"] - assert [attributes(event)["exception.message"] for event in after["events"]] == [ - attributes(event)["exception.message"] for event in before["events"] - ] - for event in after["events"]: - assert "exception.stacktrace" not in attributes(event) - - -def test_filter_omits_forged_chained_traceback_on_every_span_by_default( - cognee_filter: Any, recorded: list[dict[str, Any]] -) -> None: - """Serialized exception text has no trustworthy frame provenance. - - The content-policy exception quotes an oversized prompt on the recorded - LLM span and its pipeline parent. Its quoted text contains the exact - traceback-shaped sequence from the red probe, including the forged frame. - """ - assert len(FORGED_CHAIN_PAYLOAD) > 8000 - spans = policy_rejection_spans(recorded, FORGED_CHAIN_PAYLOAD) - assert found_in(spans, [CHAIN_FRAME_MARKER]) == [CHAIN_FRAME_MARKER] - - wire = filtered(cognee_filter, spans) - assert [span["name"] for span in wire] == [GENERATION_SPAN, TASK_SPAN] - assert found_in(wire, [CHAIN_FRAME_MARKER]) == [] - for span in wire: - assert span["status"] == { - "code": "STATUS_CODE_ERROR", - "message": POLICY_ERROR + TYPE_ONLY, - } - for event in span["events"]: - event_attrs = attributes(event) - assert event_attrs["exception.type"] == POLICY_ERROR - assert event_attrs["exception.escaped"] == "False" - assert event_attrs["exception.message"] == DETAIL_REMOVED - assert "exception.stacktrace" not in event_attrs - - captured = filtered(cognee_filter, spans, capture_content=True) - for before, after in zip(spans, captured): - assert attributes(after) == filtered_attributes(before, capture_content=True) - assert after["events"] == without_stacktraces(before["events"]) - assert after["status"] == before["status"] - assert found_in(after, [CHAIN_FRAME_MARKER]) == [CHAIN_FRAME_MARKER] - - -def test_filter_removes_detail_of_a_raised_chained_exception(cognee_filter: Any) -> None: - # The real record_exception() and use_span() output on this Python version, - # for a chained exception with a multi-line message. - filtered_out, captured_out, raw_out = ( - InMemorySpanExporter(), - InMemorySpanExporter(), - InMemorySpanExporter(), - ) - provider = TracerProvider() - provider.add_span_processor(SimpleSpanProcessor(cognee_filter.CogneeExportFilter(filtered_out))) - provider.add_span_processor( - SimpleSpanProcessor(cognee_filter.CogneeExportFilter(captured_out, capture_content=True)) - ) - provider.add_span_processor(SimpleSpanProcessor(raw_out)) - with pytest.raises(ContentPolicyFilterError): - with provider.get_tracer("test").start_as_current_span(GENERATION_SPAN): - raise_policy_rejection() - (raw,) = raw_out.get_finished_spans() - (clean,) = filtered_out.get_finished_spans() - (captured,) = captured_out.get_finished_spans() - assert PAYLOAD_MARKER in raw.status.description - exported = { - "attributes": dict(clean.attributes), - "status": clean.status.description, - "events": [[event.name, dict(event.attributes)] for event in clean.events], - } - assert found_in(exported, [PAYLOAD_MARKER]) == [] - (event,) = clean.events - exception_type = event.attributes["exception.type"] - assert exception_type.endswith(".ContentPolicyFilterError") - assert clean.status.description == exception_type + TYPE_ONLY - assert "exception.stacktrace" not in event.attributes - # Capture mode keeps all content and the error message and status; it - # adds only the span-kind mapping (applied in both modes) and drops only - # the serialized stack trace, which carries local paths. - assert cognee_filter.span_kind(GENERATION_SPAN) == "LLM" - assert dict(captured.attributes) == {**dict(raw.attributes), "fi.span.kind": "LLM"} - assert [event.name for event in captured.events] == [event.name for event in raw.events] - for captured_event, raw_event in zip(captured.events, raw.events): - assert "exception.stacktrace" in raw_event.attributes - assert dict(captured_event.attributes) == { - key: value - for key, value in raw_event.attributes.items() - if key != "exception.stacktrace" - } - assert PAYLOAD_MARKER in captured.events[0].attributes["exception.message"] - assert captured.status == raw.status - - -MARK = "TMARK5e0a" - - -@pytest.mark.parametrize( - ("trace", "exception_type"), - [ - pytest.param( - [ - TRACEBACK_HEADER, - ' File "/app/client.py", line 12, in post', - " raise ContentPolicyViolationError(body) # " + MARK, - " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^", - "litellm.ContentPolicyViolationError: rejected " + MARK, - "a second message line " + MARK, - ' File "/forged.py", line 1, in ' + MARK, - "", - "The above exception was the direct cause of the following exception:", - "", - TRACEBACK_HEADER, - ' File "/app/adapter.py", line 515, in acreate_structured_output', - " raise ContentPolicyFilterError(message) from error", - "cognee.ContentPolicyFilterError: policy " + MARK, - MARK + " on a second line", - ], - "cognee.ContentPolicyFilterError", - id="direct-cause-multi-line-messages", - ), - pytest.param( - [ - TRACEBACK_HEADER, - ' File "/app/a.py", line 3, in run', - " step(" + MARK + ")", - ' File "/app/a.py", line 7, in step', - " step(" + MARK + ")", - " [Previous line repeated 2 more times]", - "KeyError: '" + MARK + "'", - "", - "During handling of the above exception, another exception occurred:", - "", - TRACEBACK_HEADER, - ' File "/app/a.py", line 5, in run', - " raise ValueError(" + MARK + ")", - "ValueError: " + MARK, - "a note added with add_note() " + MARK, - ], - "ValueError", - id="during-handling-with-repeats-and-notes", - ), - pytest.param( - [ - "ValueError: " + MARK, - "", - "The above exception was the direct cause of the following exception:", - "", - TRACEBACK_HEADER, - ' File "/app/a.py", line 3, in run', - "RuntimeError: " + MARK, - ], - "RuntimeError", - id="cause-without-traceback", - ), - pytest.param( - [ - TRACEBACK_HEADER, - ' File "/app/a.py", line 3, in run', - "ValueError: " + MARK, - TRACEBACK_HEADER, - ' File "/app/' + MARK + '.py", line 1, in ' + MARK, - MARK, - ], - "ValueError", - id="message-that-looks-like-a-traceback", - ), - pytest.param( - [ - " + Exception Group Traceback (most recent call last):", - ' | File "/app/a.py", line 3, in run', - ' | raise ExceptionGroup("' + MARK + '", errors)', - " | ExceptionGroup: " + MARK + " (1 sub-exception)", - " +-+---------------- 1 ----------------", - " | ValueError: " + MARK, - " +------------------------------------", - ], - "ExceptionGroup", - id="unrecognised-layout", - ), - ], -) - - -def test_filter_omits_serialized_stacktraces_in_both_modes( - cognee_filter: Any, trace: list[str], exception_type: str -) -> None: - exporter, captured_exporter = InMemorySpanExporter(), InMemorySpanExporter() - provider = TracerProvider() - provider.add_span_processor(SimpleSpanProcessor(cognee_filter.CogneeExportFilter(exporter))) - provider.add_span_processor( - SimpleSpanProcessor(cognee_filter.CogneeExportFilter(captured_exporter, capture_content=True)) - ) - with provider.get_tracer("test").start_as_current_span(TASK_SPAN) as span: - span.add_event( - "exception", - { - "exception.type": exception_type, - "exception.message": MARK, - "exception.stacktrace": "\n".join(trace) + "\n", - "exception.escaped": "False", - }, - ) - (exported,) = exporter.get_finished_spans() - (captured,) = captured_exporter.get_finished_spans() - assert dict(exported.events[0].attributes) == { - "exception.type": exception_type, - "exception.message": DETAIL_REMOVED, - "exception.escaped": "False", - } - assert found_in(dict(exported.events[0].attributes), [MARK]) == [] - # Capture mode keeps the message (content the user opted into) but never - # the serialized stack trace. - assert dict(captured.events[0].attributes) == { - "exception.type": exception_type, - "exception.message": MARK, - "exception.escaped": "False", - } - - -@pytest.mark.parametrize("capture_content", [False, True], ids=["default", "capture"]) -def test_filter_never_exports_stack_traces_or_local_paths( - cognee_filter: Any, recorded: list[dict[str, Any]], capture_content: bool -) -> None: - # The recorded run's vector-search probes carry Cognee's real stack - # traces, whose frames name the local Python installation (scrubbed to - # in the fixture). Kept as errors here, so their events reach - # the exporter, they must arrive without the trace in both modes. - local = ["", TRACEBACK_HEADER] - assert found_in(recorded, local) == local - wire = filtered( - cognee_filter, - recorded, - capture_content=capture_content, - collection_missing_as_error=True, - ) - assert error_names(wire) == error_names(recorded) - assert found_in(wire, ["", TRACEBACK_HEADER, "site-packages"]) == [] - for span in wire: - for event in span.get("events", []): - assert "exception.stacktrace" not in attributes(event) - assert "exception.type" in attributes(event) - - -def test_filter_keeps_only_known_keys_and_non_text_values_in_events(cognee_filter: Any) -> None: - # Cognee 1.6.2 adds no events other than "exception"; anything else gets - # the same allowlist. - exporter = InMemorySpanExporter() - provider = TracerProvider() - provider.add_span_processor(SimpleSpanProcessor(cognee_filter.CogneeExportFilter(exporter))) - with provider.get_tracer("test").start_as_current_span(APP_SPAN) as span: - span.add_event( - "retry", - { - "attempt": 2, - "delay.seconds": 1.5, - "final": False, - "scores": [0.5, 1.0], - "reason": "rejected " + MARK, - "inputs": ["first " + MARK, "second"], - "exception.type": "ValueError", - }, - ) - span.set_status(Status(StatusCode.ERROR, "gave up on " + MARK)) - (exported,) = exporter.get_finished_spans() - assert dict(exported.events[0].attributes) == { - "attempt": 2, - "delay.seconds": 1.5, - "final": False, - "scores": (0.5, 1.0), - "exception.type": "ValueError", - } - # An event that is not named "exception" does not give the status a type. - assert exported.status.description == DETAIL_REMOVED - - -def test_filter_capture_content_keeps_cognees_content( - cognee_filter: Any, recorded: list[dict[str, Any]] -) -> None: - wire = filtered(cognee_filter, recorded, capture_content=True) - assert_default_content(wire) - assert_collector_mapping_inputs(wire, FILTER_KINDS) - for before, after in zip(recorded, wire): - assert attributes(after) == filtered_attributes(before, capture_content=True) - - -def test_filter_capture_content_shows_content_as_span_input_and_output( - cognee_filter: Any, recorded: list[dict[str, Any]] -) -> None: - # fi-collector fills a span's Input and Output from input.value and - # output.value only; without these copies, captured prompts, questions and - # answers show up in Future AGI as raw attributes, not as Input/Output. - captured = filtered(cognee_filter, recorded, capture_content=True) - wire = {span["spanId"]: attributes(span) for span in captured} - generations = [span for span in recorded if span["name"] == GENERATION_SPAN] - assert generations - for span in generations: - attrs = wire[span["spanId"]] - assert attrs["input.value"] == attributes(span)["langfuse.observation.input"] - assert attrs["input.mime_type"] == "application/json" - assert attrs["output.value"] == attributes(span)["langfuse.observation.output"] - assert attrs["output.mime_type"] in ("application/json", "text/plain") - (search,) = spans_named(recorded, "memory.retrieve") - assert wire[search["spanId"]]["input.value"] == QUESTION - assert wire[search["spanId"]]["input.mime_type"] == "text/plain" - # Spans that carry no content get no Input/Output. - for span in spans_named(recorded, "cognee.db.vector.search"): - assert not set(IO_KEYS) & set(wire[span["spanId"]]) - # Default mode adds none of them. - for span in filtered(cognee_filter, recorded): - assert not set(IO_KEYS) & set(attributes(span)), span["name"] - - -def test_filter_capture_content_keeps_an_existing_span_input_and_output( - cognee_filter: Any, -) -> None: - exporter = InMemorySpanExporter() - provider = TracerProvider() - provider.add_span_processor( - SimpleSpanProcessor(cognee_filter.CogneeExportFilter(exporter, capture_content=True)) - ) - with provider.get_tracer("test").start_as_current_span(GENERATION_SPAN) as span: - span.set_attribute("langfuse.observation.input", '{"text_input": "cognee prompt"}') - span.set_attribute("langfuse.observation.output", "cognee answer") - span.set_attribute("input.value", "set by another instrumentor") - span.set_attribute("input.mime_type", "text/plain") - (exported,) = exporter.get_finished_spans() - assert exported.attributes["input.value"] == "set by another instrumentor" - assert exported.attributes["input.mime_type"] == "text/plain" - assert exported.attributes["output.value"] == "cognee answer" - assert exported.attributes["output.mime_type"] == "text/plain" - - -@pytest.mark.parametrize( - ("env_value", "argument", "kept"), - [ - (None, None, False), - ("true", None, True), - ("TRUE", None, True), - ("false", None, False), - ("true", False, False), - (None, True, True), - ], -) -def test_filter_capture_switch( - cognee_filter: Any, monkeypatch: pytest.MonkeyPatch, env_value: Any, argument: Any, kept: bool -) -> None: - if env_value is None: - monkeypatch.delenv("COGNEE_FI_CAPTURE_CONTENT", raising=False) - else: - monkeypatch.setenv("COGNEE_FI_CAPTURE_CONTENT", env_value) - exporter = InMemorySpanExporter() - options = {} if argument is None else {"capture_content": argument} - provider = TracerProvider() - provider.add_span_processor( - SimpleSpanProcessor(cognee_filter.CogneeExportFilter(exporter, **options)) - ) - with provider.get_tracer("test").start_as_current_span("memory.retrieve") as span: - span.set_attribute("memory.query.text", QUESTION) - (exported,) = exporter.get_finished_spans() - assert ("memory.query.text" in exported.attributes) is kept - - -def test_filter_does_not_change_spans_other_processors_see(cognee_filter: Any) -> None: - filtered_out, raw_out = InMemorySpanExporter(), InMemorySpanExporter() - provider = TracerProvider() - # The filter runs first; the plain exporter after it must see the original. - provider.add_span_processor(SimpleSpanProcessor(cognee_filter.CogneeExportFilter(filtered_out))) - provider.add_span_processor(SimpleSpanProcessor(raw_out)) - with provider.get_tracer("test").start_as_current_span("memory.retrieve") as span: - span.set_attribute("memory.query.text", QUESTION) - span.add_event("exception", {"exception.message": "bad " + QUESTION}) - span.set_status(Status(StatusCode.ERROR, "bad " + QUESTION)) - (raw,) = raw_out.get_finished_spans() - (clean,) = filtered_out.get_finished_spans() - assert dict(raw.attributes) == {"memory.query.text": QUESTION} - assert dict(raw.events[0].attributes) == {"exception.message": "bad " + QUESTION} - assert raw.status.description == "bad " + QUESTION - assert dict(clean.attributes) == {"fi.span.kind": "RETRIEVER"} - assert dict(clean.events[0].attributes) == {"exception.message": DETAIL_REMOVED} - # The exception event names no type, so the status cannot either. - assert clean.status.description == DETAIL_REMOVED - assert clean.context == raw.context and clean.parent == raw.parent - - -def test_filter_that_raises_still_exports_spans_without_content( - cognee_filter: Any, recorded: list[dict[str, Any]], monkeypatch: pytest.MonkeyPatch -) -> None: - def broken(*args: Any, **kwargs: Any) -> Any: - raise RuntimeError("filter bug") - - monkeypatch.setattr(cognee_filter.CogneeExportFilter, "_filtered", broken) - wire = filtered(cognee_filter, recorded) - assert [span["name"] for span in wire] == [span["name"] for span in recorded] - assert_no_content(wire) - for span in wire: - assert "events" not in span and "message" not in span.get("status", {}) - - -def test_filter_drops_a_span_it_cannot_strip( - cognee_filter: Any, recorded: list[dict[str, Any]], monkeypatch: pytest.MonkeyPatch -) -> None: - def broken(*args: Any, **kwargs: Any) -> Any: - raise RuntimeError("filter bug") - - monkeypatch.setattr(cognee_filter.CogneeExportFilter, "_filtered", broken) - monkeypatch.setattr(cognee_filter.CogneeExportFilter, "_stripped", broken) - assert filtered(cognee_filter, recorded) == [] - - -# -------------------------------------------------------------------------- -# README and requirements -# -------------------------------------------------------------------------- - - -def test_readme_states_what_the_tests_check() -> None: - readme = README.read_text(encoding="utf-8") - app = APP.read_text(encoding="utf-8") - for line in ( - "exporter = CogneeExportFilter(HTTPSpanExporter())", - "provider.add_span_processor(BatchSpanProcessor(exporter))", - ): - assert line in app and line in readme, line - for fact in ( - "cognee==1.6.5", - "COGNEE_TRACING_ENABLED", - "/tracer/v1/traces", - "X-Api-Key", - "X-Secret-Key", - "project_type=observe", - "https://api.futureagi.com:443/tracer/v1/traces", - "COGNEE_FI_CAPTURE_CONTENT=true", - MISSING_SWITCH + "=true", - MISSING_KEY, - "capture_content=True", - "fi.span.kind", - DETAIL_REMOVED, - TYPE_ONLY.strip(" ()"), - *FILTERED_KEYS, - *SPAN_KINDS, - ): - assert fact in readme, fact - - -@pytest.mark.parametrize( - "fact", - [ - pytest.param("TH-8394", id="R2-register-follow-up"), - pytest.param("pass the same ones", id="R3-same-exporter-options"), - pytest.param('pip install "cognee[tracing]==1.6.5"', id="R5-option-b-install"), - pytest.param( - "Stack traces are never exported, in either mode", id="RV-stacktrace-both-modes" - ), - pytest.param("x-api-key=...,x-secret-key=...", id="RV-option-b-grpc-lowercase-headers"), - pytest.param("cognee.disable_tracing()", id="R7-disable-tracing"), - pytest.param("License-Expression: Apache-2.0", id="D9-license-metadata"), - pytest.param("less useful for debugging", id="N1-error-detail-trade-off"), - pytest.param("among span attributes", id="F1-capture-claim-scope"), - pytest.param("`cognee.api.recall`", id="F2-recall-unmapped"), - pytest.param("`cognee.agent_memory.retrieve`", id="F2-agent-memory-unmapped"), - ], -) -def test_readme_covers_review_items(fact: str) -> None: - assert fact in README.read_text(encoding="utf-8") - - -def test_requirements_pin_the_register_behaviour_the_recipe_relies_on() -> None: - lines = REQUIREMENTS.read_text(encoding="utf-8").splitlines() - assert "fi-instrumentation-otel>=1.1.0,<1.2" in lines - assert "cognee==1.6.5" in lines diff --git a/python/frameworks/anthropic/poetry.lock b/python/frameworks/anthropic/poetry.lock index 65ee3277e..1ab16753a 100644 --- a/python/frameworks/anthropic/poetry.lock +++ b/python/frameworks/anthropic/poetry.lock @@ -226,14 +226,14 @@ test = ["pytest (>=6)"] [[package]] name = "fi-instrumentation-otel" -version = "1.1.0" -description = "OpenTelemetry instrumentation" +version = "1.2.2" +description = "OpenTelemetry tracing for AI applications: tracer setup, span helpers, exporters, and semantic conventions for Future AGI." optional = false python-versions = "<3.14,>3.9" groups = ["main"] files = [ - {file = "fi_instrumentation_otel-1.1.0-py3-none-any.whl", hash = "sha256:4fa2bf663e0ce09447aca92ae798cd7b0dee35c5c92c2e033f1d89d158a9e62c"}, - {file = "fi_instrumentation_otel-1.1.0.tar.gz", hash = "sha256:7ec324c3c9242aa87a88cf23a21c11919b8736c6451c5fb50917f592a853b059"}, + {file = "fi_instrumentation_otel-1.2.2-py3-none-any.whl", hash = "sha256:25cf88f93e33aa928d4fae0f631afe8a32565639490862b715869e0633becbaf"}, + {file = "fi_instrumentation_otel-1.2.2.tar.gz", hash = "sha256:b675df47d7a03c506fbd2e4b0420fe4f74d12a6353a88b219bfd97797b276d4c"}, ] [package.dependencies] @@ -1135,4 +1135,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = ">3.9, <3.14" -content-hash = "239c305494e211c7f0af947af955f6163a90bad1795aba8ce185de3a5846d963" +content-hash = "212f97a32a59ae314872c8a3eab2de167f6372210b691eb867b40f7620b933f4" diff --git a/python/frameworks/anthropic/pyproject.toml b/python/frameworks/anthropic/pyproject.toml index 7f73c2c0b..e1930f4ca 100644 --- a/python/frameworks/anthropic/pyproject.toml +++ b/python/frameworks/anthropic/pyproject.toml @@ -30,7 +30,7 @@ classifiers = [ [tool.poetry.dependencies] python = ">3.9, <3.14" anthropic = ">=0.41.0" -fi-instrumentation-otel = ">=0.1.7" +fi-instrumentation-otel = ">=1.2.2" [tool.poetry.urls] diff --git a/python/frameworks/anthropic/tests/test_r2_native_regression.py b/python/frameworks/anthropic/tests/test_r2_native_regression.py new file mode 100644 index 000000000..d55670c93 --- /dev/null +++ b/python/frameworks/anthropic/tests/test_r2_native_regression.py @@ -0,0 +1,165 @@ +"""Anthropic SDK offline regressions; no vendor calls are made.""" +import asyncio +import json + +import anthropic +import pytest +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from traceai_anthropic import AnthropicInstrumentor + +try: + import httpx2 as httpx +except ImportError: + import httpx + +MESSAGE = {"id": "msg_local", "type": "message", "role": "assistant", "content": [{"type": "text", "text": "answer"}], "model": "local", "stop_reason": "end_turn", "stop_sequence": None, "usage": {"input_tokens": 4, "output_tokens": 2}} +START = {"id": "msg_local", "type": "message", "role": "assistant", "content": [], "model": "local", "stop_reason": None, "stop_sequence": None, "usage": {"input_tokens": 4, "output_tokens": 0}} +EVENTS = [{"type": "message_start", "message": START}, {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "hello"}}, {"type": "content_block_stop", "index": 0}, {"type": "message_delta", "delta": {"stop_reason": "end_turn", "stop_sequence": None}, "usage": {"output_tokens": 2}}, {"type": "message_stop"}] +BODY = "".join("event: " + event["type"] + "\ndata: " + json.dumps(event) + "\n\n" for event in EVENTS) +PARAMS = {"model": "local", "max_tokens": 16, "messages": [{"role": "user", "content": "hello"}]} +TOOLS = [{"role": "assistant", "content": [{"type": "tool_use", "id": "tool1", "name": "lookup", "input": {"query": "abc"}}]}, {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "tool1", "content": "ok"}]}] + + +def setup(): + p = TracerProvider() + x = InMemorySpanExporter() + p.add_span_processor(SimpleSpanProcessor(x)) + i = AnthropicInstrumentor() + i.instrument(tracer_provider=p) + return p, x, i + + +def handler(req): + if json.loads(req.content).get("stream"): + return httpx.Response(200, headers={"content-type": "text/event-stream"}, text=BODY) + return httpx.Response(200, json=MESSAGE) + + +def test_native_tool_mapping_sync(): + c = anthropic.Anthropic(api_key="local", http_client=httpx.Client(transport=httpx.MockTransport(handler))) + params = dict(PARAMS, messages=TOOLS) + expected = c.messages.create(**params).model_dump() + p, x, i = setup() + try: + assert c.messages.create(**params).model_dump() == expected + assert len(x.get_finished_spans()) == 1 + finally: + i.uninstrument(); c.close(); p.shutdown() + + +def test_native_tool_mapping_async(): + async def run(): + c = anthropic.AsyncAnthropic(api_key="local", http_client=httpx.AsyncClient(transport=httpx.MockTransport(handler))) + params = dict(PARAMS, messages=TOOLS) + expected = (await c.messages.create(**params)).model_dump() + p, x, i = setup() + try: + assert (await c.messages.create(**params)).model_dump() == expected + assert len(x.get_finished_spans()) == 1 + finally: + i.uninstrument(); await c.close(); p.shutdown() + asyncio.run(run()) + + +@pytest.mark.parametrize("mode", ["full", "close", "context"]) +def test_native_sync_stream_lifecycle(mode): + p, x, i = setup() + c = anthropic.Anthropic(api_key="local", http_client=httpx.Client(transport=httpx.MockTransport(handler))) + try: + s = c.messages.create(**PARAMS, stream=True) + if mode == "full": + assert len(list(s)) == len(EVENTS) + elif mode == "close": + assert s.close() is None + assert s.close() is None + else: + with s as entered: + assert entered is s + assert next(iter(entered)).type == "message_start" + assert len(x.get_finished_spans()) == 1 + finally: + i.uninstrument(); c.close(); p.shutdown() + + +@pytest.mark.parametrize("mode", ["full", "close", "context"]) +def test_native_async_stream_lifecycle(mode): + async def run(): + p, x, i = setup() + c = anthropic.AsyncAnthropic(api_key="local", http_client=httpx.AsyncClient(transport=httpx.MockTransport(handler))) + try: + s = await c.messages.create(**PARAMS, stream=True) + if mode == "full": + assert len([e async for e in s]) == len(EVENTS) + elif mode == "close": + assert await s.close() is None + assert await s.close() is None + else: + async with s as entered: + assert entered is s + assert (await entered.__aiter__().__anext__()).type == "message_start" + assert len(x.get_finished_spans()) == 1 + finally: + i.uninstrument(); await c.close(); p.shutdown() + asyncio.run(run()) + + +def test_native_typed_tool_use(): + block = anthropic.types.ToolUseBlock(id="tool1", name="lookup", input={"query": "abc"}, type="tool_use") + params = dict(PARAMS, messages=[{"role": "assistant", "content": [block]}, TOOLS[1]]) + c = anthropic.Anthropic(api_key="local", http_client=httpx.Client(transport=httpx.MockTransport(handler))) + expected = c.messages.create(**params).model_dump() + p, x, i = setup() + try: + assert c.messages.create(**params).model_dump() == expected + assert len(x.get_finished_spans()) == 1 + finally: + i.uninstrument(); c.close(); p.shutdown() + + +def test_native_stream_error_is_hidden(monkeypatch, caplog): + secret = "R2_ERROR_SECRET_831" + monkeypatch.setenv("FI_HIDE_ERROR_DETAILS", "true") + body = 'event: error\ndata: ' + json.dumps({"type": "error", "error": {"type": "api_error", "message": secret}}) + '\n\n' + c = anthropic.Anthropic(api_key="local", http_client=httpx.Client(transport=httpx.MockTransport(lambda req: httpx.Response(200, headers={"content-type": "text/event-stream"}, text=body)))) + p, x, i = setup() + try: + with pytest.raises(Exception, match=secret): + list(c.messages.create(**PARAMS, stream=True)) + spans = x.get_finished_spans() + assert len(spans) == 1 + serialized = str(spans[0].status) + str(spans[0].attributes) + str([(e.name, dict(e.attributes)) for e in spans[0].events]) + caplog.text + assert secret not in serialized + finally: + i.uninstrument(); c.close(); p.shutdown() + + +def test_native_async_stream_cancel(): + async def run(): + entered = asyncio.Event() + class WaitingStream(httpx.AsyncByteStream): + async def __aiter__(self): + entered.set() + await asyncio.Event().wait() + yield b"" + def slow(req): + return httpx.Response(200, headers={"content-type": "text/event-stream"}, stream=WaitingStream()) + c = anthropic.AsyncAnthropic(api_key="local", http_client=httpx.AsyncClient(transport=httpx.MockTransport(slow))) + p, x, i = setup() + try: + s = await c.messages.create(**PARAMS, stream=True) + async def consume(): + async for _ in s: + pass + task = asyncio.create_task(consume()) + await entered.wait() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert len(x.get_finished_spans()) == 1 + await s.close() + assert len(x.get_finished_spans()) == 1 + finally: + i.uninstrument(); await c.close(); p.shutdown() + asyncio.run(run()) diff --git a/python/frameworks/anthropic/tests/test_r3_stream_resume.py b/python/frameworks/anthropic/tests/test_r3_stream_resume.py new file mode 100644 index 000000000..a046f94d0 --- /dev/null +++ b/python/frameworks/anthropic/tests/test_r3_stream_resume.py @@ -0,0 +1,249 @@ +"""Native SDK stream ownership regressions, using local mock HTTP only.""" +import asyncio +import gc +import json +from importlib.metadata import version + +import anthropic +import pytest +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import StatusCode +from traceai_anthropic import AnthropicInstrumentor + +try: + import httpx2 as httpx +except ImportError: + import httpx + +START = {'id': 'msg_1', 'type': 'message', 'role': 'assistant', 'content': [], 'model': 'local', 'stop_reason': None, 'stop_sequence': None, 'usage': {'input_tokens': 4, 'output_tokens': 0}} +EVENTS = [{'type': 'message_start', 'message': START}, {'type': 'content_block_start', 'index': 0, 'content_block': {'type': 'text', 'text': '', 'citations': []}}, {'type': 'content_block_delta', 'index': 0, 'delta': {'type': 'text_delta', 'text': 'hello'}}, {'type': 'content_block_stop', 'index': 0}, {'type': 'message_delta', 'delta': {'stop_reason': 'end_turn', 'stop_sequence': None}, 'usage': {'output_tokens': 2}}, {'type': 'message_stop'}] +CITATION = {'type': 'content_block_delta', 'index': 0, 'delta': {'type': 'citations_delta', 'citation': {'type': 'char_location', 'cited_text': 'fact', 'document_index': 0, 'document_title': 'doc', 'start_char_index': 0, 'end_char_index': 4}}} +PARAMS = dict(model='local', max_tokens=16, messages=[{'role': 'user', 'content': 'hi'}], stream=True) +SECRET = 'r3_hidden_error_sentinel' + + +def body(events): + return ''.join('event: ' + e['type'] + '\ndata: ' + json.dumps(e) + '\n\n' for e in events) + + +def make_client(events, asynchronous=False): + handler = lambda req: httpx.Response(200, headers={'content-type': 'text/event-stream'}, text=body(events)) + if asynchronous: + return anthropic.AsyncAnthropic(api_key='local', http_client=httpx.AsyncClient(transport=httpx.MockTransport(handler))) + return anthropic.Anthropic(api_key='local', http_client=httpx.Client(transport=httpx.MockTransport(handler))) + + +def setup(): + p = TracerProvider(); exporter = InMemorySpanExporter(); p.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = AnthropicInstrumentor(); instrumentor.instrument(tracer_provider=p) + return p, exporter, instrumentor + + +def assert_span(exporter, error=False, full=True): + spans = exporter.get_finished_spans() + assert len(spans) == 1 + span = spans[0] + assert span.status.status_code == (StatusCode.ERROR if error else StatusCode.OK) + if full: + attrs = dict(span.attributes) + assert 'hello' in str(attrs) + assert attrs['gen_ai.usage.input_tokens'] == 4 + assert attrs['gen_ai.usage.output_tokens'] == 2 + assert SECRET not in str(span.status) + str(span.attributes) + str([(e.name, dict(e.attributes)) for e in span.events]) + + +@pytest.mark.parametrize('mode', ['temporary', 'break', 'iterator_close', 'stream_close']) +def test_sync_stream_lifetime(mode): + c = make_client(EVENTS); p, x, i = setup() + try: + stream = c.messages.create(**PARAMS) + if mode == 'temporary': + assert next(iter(stream)).type == 'message_start'; gc.collect() + elif mode == 'break': + for event in stream: + assert event.type == 'message_start'; break + gc.collect() + else: + it = iter(stream); assert next(it).type == 'message_start'; it.close() + assert len(x.get_finished_spans()) == 0 + if mode == 'stream_close': + stream.close(); stream.close(); assert_span(x, full=False) + else: + assert list(stream)[-1].type == 'message_stop'; assert_span(x) + stream.close(); assert_span(x) + finally: + i.uninstrument(); c.close(); p.shutdown() + + +@pytest.mark.parametrize('mode', ['break', 'iterator_close', 'stream_close']) +def test_async_stream_lifetime(mode): + async def run(): + c = make_client(EVENTS, True); p, x, i = setup() + try: + stream = await c.messages.create(**PARAMS) + if mode == 'break': + async for event in stream: + assert event.type == 'message_start'; break + await asyncio.sleep(0); gc.collect(); await asyncio.sleep(0) + else: + it = stream.__aiter__(); assert (await it.__anext__()).type == 'message_start'; await it.aclose() + assert len(x.get_finished_spans()) == 0 + if mode == 'stream_close': + await stream.close(); await stream.close(); assert_span(x, full=False) + else: + assert [e async for e in stream][-1].type == 'message_stop'; assert_span(x) + await stream.close(); assert_span(x) + finally: + i.uninstrument(); await c.close(); p.shutdown() + asyncio.run(run()) + + +@pytest.mark.parametrize('asynchronous', [False, True]) +def test_resumed_api_error_preserves_partial_output_and_privacy(asynchronous, monkeypatch, caplog): + monkeypatch.setenv('FI_HIDE_ERROR_DETAILS', 'true') + events = EVENTS[:-1] + [{'type': 'error', 'error': {'type': 'api_error', 'message': SECRET}}] + async def run_async(): + c = make_client(events, True); p, x, i = setup() + try: + stream = await c.messages.create(**PARAMS) + it = stream.__aiter__(); await it.__anext__(); await it.aclose() + assert not x.get_finished_spans() + with pytest.raises(anthropic.APIStatusError, match=SECRET): + async for _ in stream: pass + assert_span(x, error=True); assert SECRET not in caplog.text + await stream.close(); assert_span(x, error=True) + finally: + i.uninstrument(); await c.close(); p.shutdown() + if asynchronous: + asyncio.run(run_async()); return + c = make_client(events); p, x, i = setup() + try: + stream = c.messages.create(**PARAMS); assert next(iter(stream)).type == 'message_start' + assert not x.get_finished_spans() + with pytest.raises(anthropic.APIStatusError, match=SECRET): list(stream) + assert_span(x, error=True); assert SECRET not in caplog.text + stream.close(); assert_span(x, error=True) + finally: + i.uninstrument(); c.close(); p.shutdown() + + +@pytest.mark.parametrize('asynchronous', [False, True]) +def test_native_citation_parity(asynchronous): + if version('anthropic') == '0.41.0': + pytest.skip('NA: 0.41.0 has no citations delta in its SDK type contract') + events = EVENTS[:3] + [CITATION] + EVENTS[3:] + async def run_async(): + c = make_client(events, True) + expected = [e.model_dump() async for e in await c.messages.create(**PARAMS)] + p, x, i = setup() + try: + actual = [e.model_dump() async for e in await c.messages.create(**PARAMS)] + assert actual == expected; assert_span(x) + finally: + i.uninstrument(); await c.close(); p.shutdown() + if asynchronous: + asyncio.run(run_async()); return + c = make_client(events); expected = [e.model_dump() for e in c.messages.create(**PARAMS)] + p, x, i = setup() + try: + assert [e.model_dump() for e in c.messages.create(**PARAMS)] == expected; assert_span(x) + finally: + i.uninstrument(); c.close(); p.shutdown() + + +@pytest.mark.parametrize('asynchronous', [False, True]) +def test_native_thinking_tool_and_unknown_delta_parity(asynchronous): + if version('anthropic') == '0.41.0': + pytest.skip('NA: thinking/unknown delta fixture is for current SDK') + extra = [ + {'type': 'content_block_start', 'index': 1, 'content_block': {'type': 'thinking', 'thinking': '', 'signature': ''}}, + {'type': 'content_block_delta', 'index': 1, 'delta': {'type': 'thinking_delta', 'thinking': 'reason'}}, + {'type': 'content_block_delta', 'index': 1, 'delta': {'type': 'signature_delta', 'signature': 'signed'}}, + {'type': 'content_block_stop', 'index': 1}, + {'type': 'content_block_start', 'index': 2, 'content_block': {'type': 'tool_use', 'id': 'tool1', 'name': 'lookup', 'input': {}}}, + {'type': 'content_block_delta', 'index': 2, 'delta': {'type': 'input_json_delta', 'partial_json': '{"x":1}'}}, + {'type': 'content_block_delta', 'index': 2, 'delta': {'type': 'future_nontext_delta', 'value': 'future'}}, + {'type': 'content_block_stop', 'index': 2}, + ] + events = EVENTS[:4] + extra + EVENTS[4:] + # Compare native models, not model_dump(): SDK 0.57.1 accepts unknown + # deltas but its Pydantic serializer cannot dump their fallback model. + async def run_async(): + c = make_client(events, True) + expected = [e async for e in await c.messages.create(**PARAMS)] + p, x, i = setup() + try: + assert [e async for e in await c.messages.create(**PARAMS)] == expected + assert_span(x) + assert 'lookup' in str(x.get_finished_spans()[0].attributes) + finally: + i.uninstrument(); await c.close(); p.shutdown() + if asynchronous: + asyncio.run(run_async()); return + c = make_client(events) + expected = list(c.messages.create(**PARAMS)) + p, x, i = setup() + try: + assert list(c.messages.create(**PARAMS)) == expected + assert_span(x) + assert 'lookup' in str(x.get_finished_spans()[0].attributes) + finally: + i.uninstrument(); c.close(); p.shutdown() + + +@pytest.mark.parametrize('asynchronous', [False, True]) +def test_resumed_stream_content_privacy_and_usage(asynchronous, monkeypatch): + monkeypatch.setenv('FI_HIDE_INPUTS', 'true') + monkeypatch.setenv('FI_HIDE_OUTPUTS', 'true') + params = dict(PARAMS, messages=[{'role': 'user', 'content': 'r3_private_prompt_sentinel'}]) + def check(x): + spans = x.get_finished_spans(); assert len(spans) == 1 + span = spans[0]; attrs = dict(span.attributes) + assert span.status.status_code == StatusCode.OK + assert 'r3_private_prompt_sentinel' not in str(attrs) + assert 'hello' not in str(attrs) + assert attrs['gen_ai.usage.input_tokens'] == 4 + assert attrs['gen_ai.usage.output_tokens'] == 2 + async def run_async(): + c = make_client(EVENTS, True); p, x, i = setup() + try: + s = await c.messages.create(**params); it = s.__aiter__(); await it.__anext__(); await it.aclose() + assert not x.get_finished_spans() + assert [event async for event in s][-1].type == 'message_stop'; check(x) + finally: + i.uninstrument(); await c.close(); p.shutdown() + if asynchronous: + asyncio.run(run_async()); return + c = make_client(EVENTS); p, x, i = setup() + try: + s = c.messages.create(**params); next(iter(s)); assert not x.get_finished_spans() + assert list(s)[-1].type == 'message_stop'; check(x) + finally: + i.uninstrument(); c.close(); p.shutdown() + + +@pytest.mark.parametrize('asynchronous', [False, True]) +def test_consumer_injected_exception_does_not_finish_stream(asynchronous): + class ConsumerError(Exception): + pass + async def run_async(): + c = make_client(EVENTS, True); p, x, i = setup() + try: + s = await c.messages.create(**PARAMS); it = s.__aiter__(); await it.__anext__() + with pytest.raises(ConsumerError): await it.athrow(ConsumerError('consumer')) + assert not x.get_finished_spans() + assert [event async for event in s][-1].type == 'message_stop'; assert_span(x) + finally: + i.uninstrument(); await c.close(); p.shutdown() + if asynchronous: + asyncio.run(run_async()); return + c = make_client(EVENTS); p, x, i = setup() + try: + s = c.messages.create(**PARAMS); it = iter(s); next(it) + with pytest.raises(ConsumerError): it.throw(ConsumerError('consumer')) + assert not x.get_finished_spans() + assert list(s)[-1].type == 'message_stop'; assert_span(x) + finally: + i.uninstrument(); c.close(); p.shutdown() diff --git a/python/frameworks/anthropic/tests/test_r4_direct_lifecycle.py b/python/frameworks/anthropic/tests/test_r4_direct_lifecycle.py new file mode 100644 index 000000000..97ea105d8 --- /dev/null +++ b/python/frameworks/anthropic/tests/test_r4_direct_lifecycle.py @@ -0,0 +1,225 @@ +"""Native SDK direct/mixed stream advancement and in-flight close ownership.""" +import asyncio +import json + +import anthropic +import pytest +from opentelemetry.trace import StatusCode + +from test_r3_stream_resume import EVENTS, PARAMS, SECRET, assert_span, httpx, make_client, setup + + +def test_mixed_sync_direct_iter_resume(): + client = make_client(EVENTS) + provider, exporter, instrumentor = setup() + try: + stream = client.messages.create(**PARAMS) + assert next(stream).type == 'message_start' + iterator = iter(stream) + assert next(iterator).type == 'content_block_start' + assert next(stream).type == 'content_block_delta' + iterator.close() + assert not exporter.get_finished_spans() + assert [event.type for event in stream] == ['content_block_stop', 'message_delta', 'message_stop'] + assert_span(exporter) + with pytest.raises(StopIteration): + next(stream) + assert_span(exporter) + finally: + instrumentor.uninstrument(); client.close(); provider.shutdown() + + +def test_mixed_async_direct_iter_resume(): + async def run(): + client = make_client(EVENTS, True) + provider, exporter, instrumentor = setup() + try: + stream = await client.messages.create(**PARAMS) + assert (await stream.__anext__()).type == 'message_start' + iterator = stream.__aiter__() + assert (await iterator.__anext__()).type == 'content_block_start' + assert (await stream.__anext__()).type == 'content_block_delta' + await iterator.aclose() + assert not exporter.get_finished_spans() + assert [event.type async for event in stream] == ['content_block_stop', 'message_delta', 'message_stop'] + assert_span(exporter) + with pytest.raises(StopAsyncIteration): + await stream.__anext__() + assert_span(exporter) + finally: + instrumentor.uninstrument(); await client.close(); provider.shutdown() + asyncio.run(run()) + + +@pytest.mark.parametrize('asynchronous', [False, True]) +def test_direct_partial_close(asynchronous): + async def run(): + client = make_client(EVENTS, True) + provider, exporter, instrumentor = setup() + try: + stream = await client.messages.create(**PARAMS) + for _ in range(3): + await stream.__anext__() + assert not exporter.get_finished_spans() + assert await stream.close() is None + assert_span(exporter, full=False) + attrs = exporter.get_finished_spans()[0].attributes + assert 'hello' in str(attrs) + assert attrs['gen_ai.usage.input_tokens'] == 4 + finally: + instrumentor.uninstrument(); await client.close(); provider.shutdown() + if asynchronous: + asyncio.run(run()); return + client = make_client(EVENTS) + provider, exporter, instrumentor = setup() + try: + stream = client.messages.create(**PARAMS) + for _ in range(3): + next(stream) + assert not exporter.get_finished_spans() + assert stream.close() is None + assert_span(exporter, full=False) + attrs = exporter.get_finished_spans()[0].attributes + assert 'hello' in str(attrs) + assert attrs['gen_ai.usage.input_tokens'] == 4 + finally: + instrumentor.uninstrument(); client.close(); provider.shutdown() + + +@pytest.mark.parametrize('traced', [False, True]) +@pytest.mark.parametrize('outcome', ['error', 'cancel']) +def test_close_returns_before_pending_error_and_multiple_closers(traced, outcome, monkeypatch): + monkeypatch.setenv('FI_HIDE_ERROR_DETAILS', 'true') + class Wire(httpx.AsyncByteStream): + def __init__(self): + self.pending = asyncio.Event(); self.release = asyncio.Event() + async def __aiter__(self): + yield ('event: message_start\ndata: ' + json.dumps(EVENTS[0]) + '\n\n').encode() + self.pending.set() + await self.release.wait() + raise RuntimeError(SECRET) + async def aclose(self): + pass + async def run(): + wire = Wire() + client = anthropic.AsyncAnthropic(api_key='local', http_client=httpx.AsyncClient(transport=httpx.MockTransport(lambda req: httpx.Response(200, headers={'content-type': 'text/event-stream'}, stream=wire)))) + if traced: + provider, exporter, instrumentor = setup() + try: + stream = await client.messages.create(**PARAMS) + assert (await stream.__anext__()).type == 'message_start' + pending = asyncio.create_task(stream.__anext__()) + await asyncio.wait_for(wire.pending.wait(), 2) + assert await asyncio.wait_for(asyncio.gather(stream.close(), stream.close()), 2) == [None, None] + if traced: + assert not exporter.get_finished_spans() + if outcome == 'cancel': + pending.cancel() + with pytest.raises(asyncio.CancelledError): + await pending + else: + wire.release.set() + with pytest.raises(RuntimeError, match=SECRET): + await pending + if traced: + assert_span(exporter, error=True, full=False) + finally: + if traced: + instrumentor.uninstrument(); provider.shutdown() + await client.close() + asyncio.run(run()) + + +def test_close_exception_preserves_native_error_and_return(): + from traceai_anthropic._stream import _MessagesStream + from traceai_anthropic._with_span import _WithSpan + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import SimpleSpanProcessor + from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + class Native: + def close(self): + raise ValueError('native close failure') + provider = TracerProvider(); exporter = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + stream = _MessagesStream(Native(), _WithSpan(provider.get_tracer(__name__).start_span('close'))) + try: + with pytest.raises(ValueError, match='native close failure'): + stream.close() + assert len(exporter.get_finished_spans()) == 1 + assert exporter.get_finished_spans()[0].status.status_code == StatusCode.ERROR + finally: + provider.shutdown() + + +@pytest.mark.parametrize('traced', [False, True]) +def test_sync_close_does_not_finish_before_pending_error(traced, monkeypatch): + import threading + monkeypatch.setenv('FI_HIDE_ERROR_DETAILS', 'true') + class Wire(httpx.SyncByteStream): + def __init__(self): + self.pending = threading.Event(); self.release = threading.Event() + def __iter__(self): + yield ('event: message_start\ndata: ' + json.dumps(EVENTS[0]) + '\n\n').encode() + self.pending.set() + if not self.release.wait(3): + raise TimeoutError('test release timed out') + raise RuntimeError(SECRET) + def close(self): + pass + wire = Wire() + client = anthropic.Anthropic(api_key='local', http_client=httpx.Client(transport=httpx.MockTransport(lambda req: httpx.Response(200, headers={'content-type': 'text/event-stream'}, stream=wire)))) + if traced: + provider, exporter, instrumentor = setup() + try: + stream = client.messages.create(**PARAMS) + assert next(stream).type == 'message_start' + errors = [] + def advance(): + try: + next(stream) + except BaseException as exc: + errors.append(exc) + thread = threading.Thread(target=advance) + thread.start() + assert wire.pending.wait(2) + assert stream.close() is None + assert stream.close() is None + if traced: + assert not exporter.get_finished_spans() + wire.release.set(); thread.join(2) + assert not thread.is_alive() + assert len(errors) == 1 and isinstance(errors[0], RuntimeError) + assert str(errors[0]) == SECRET + if traced: + assert_span(exporter, error=True, full=False) + finally: + wire.release.set() + if traced: + instrumentor.uninstrument(); provider.shutdown() + client.close() + + +@pytest.mark.parametrize('asynchronous', [False, True]) +def test_direct_advance_context_exception(asynchronous, monkeypatch): + monkeypatch.setenv('FI_HIDE_ERROR_DETAILS', 'true') + async def run(): + client = make_client(EVENTS, True); provider, exporter, instrumentor = setup() + try: + with pytest.raises(ValueError, match=SECRET): + async with await client.messages.create(**PARAMS) as stream: + assert (await stream.__anext__()).type == 'message_start' + raise ValueError(SECRET) + assert_span(exporter, error=True, full=False) + finally: + instrumentor.uninstrument(); await client.close(); provider.shutdown() + if asynchronous: + asyncio.run(run()); return + client = make_client(EVENTS); provider, exporter, instrumentor = setup() + try: + with pytest.raises(ValueError, match=SECRET): + with client.messages.create(**PARAMS) as stream: + assert next(stream).type == 'message_start' + raise ValueError(SECRET) + assert_span(exporter, error=True, full=False) + finally: + instrumentor.uninstrument(); client.close(); provider.shutdown() diff --git a/python/frameworks/anthropic/tests/test_r5_native_lifecycle.py b/python/frameworks/anthropic/tests/test_r5_native_lifecycle.py new file mode 100644 index 000000000..0258133ad --- /dev/null +++ b/python/frameworks/anthropic/tests/test_r5_native_lifecycle.py @@ -0,0 +1,270 @@ +import asyncio, json +import anthropic, pytest +try: + import httpx2 as httpx +except ImportError: + import httpx +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from traceai_anthropic import AnthropicInstrumentor + +START={'type':'message_start','message':{'id':'msg_1','type':'message','role':'assistant','content':[],'model':'local','stop_reason':None,'stop_sequence':None,'usage':{'input_tokens':4,'output_tokens':0}}} +DELTA={'type':'message_delta','delta':{'stop_reason':'end_turn','stop_sequence':None},'usage':{'output_tokens':2}} +PARAMS=dict(model='local',max_tokens=16,messages=[{'role':'user','content':'hi'}],stream=True) +def event(data): return ('event: '+data['type']+'\ndata: '+json.dumps(data)+'\n\n').encode() +def setup(): + p=TracerProvider(); x=InMemorySpanExporter(); p.add_span_processor(SimpleSpanProcessor(x)); i=AnthropicInstrumentor(); i.instrument(tracer_provider=p); return p,x,i + +@pytest.mark.parametrize('traced',[False,True]) +def test_close_during_advance_preserves_late_result_and_usage(traced): + class Wire(httpx.AsyncByteStream): + def __init__(self): self.pending=asyncio.Event(); self.release=asyncio.Event() + async def __aiter__(self): + yield event(START) + self.pending.set(); await self.release.wait() + yield event(DELTA) + async def aclose(self): pass + async def run(): + wire=Wire(); c=anthropic.AsyncAnthropic(api_key='local',http_client=httpx.AsyncClient(transport=httpx.MockTransport(lambda req:httpx.Response(200,headers={'content-type':'text/event-stream'},stream=wire)))) + if traced: p,x,i=setup() + try: + s=await c.messages.create(**PARAMS); assert (await s.__anext__()).type=='message_start' + pending=asyncio.create_task(s.__anext__()); await asyncio.wait_for(wire.pending.wait(),2) + assert await asyncio.wait_for(s.close(),2) is None + if traced: assert len(x.get_finished_spans())==0 + wire.release.set(); result=await asyncio.wait_for(pending,2); assert result.type=='message_delta' and result.usage.output_tokens==2 + if traced: + spans=x.get_finished_spans(); assert len(spans)==1 + assert spans[0].attributes['gen_ai.usage.input_tokens']==4 + assert spans[0].attributes['gen_ai.usage.output_tokens']==2 + assert spans[0].status.status_code.name=='OK' + await s.close(); assert len(x.get_finished_spans())==1 + finally: + wire.release.set() + if traced: i.uninstrument(); p.shutdown() + await c.close() + asyncio.run(run()) + +class NonStringifiableError(Exception): + def __str__(self): raise RuntimeError('str failed') + +@pytest.mark.parametrize('traced',[False,True]) +def test_advance_preserves_exact_exception_even_if_stringification_fails(traced, monkeypatch): + monkeypatch.setenv('FI_HIDE_ERROR_DETAILS','true') + err=NonStringifiableError() + class Wire(httpx.SyncByteStream): + def __iter__(self): + yield event(START) + raise err + def close(self): pass + c=anthropic.Anthropic(api_key='local',http_client=httpx.Client(transport=httpx.MockTransport(lambda req:httpx.Response(200,headers={'content-type':'text/event-stream'},stream=Wire())))) + if traced: p,x,i=setup() + try: + s=c.messages.create(**PARAMS); assert next(s).type=='message_start' + try: next(s) + except BaseException as got: assert got is err, f'expected original exception identity, got {type(got).__name__}' + else: pytest.fail('expected transport error') + finally: + if traced: i.uninstrument(); p.shutdown() + c.close() + +@pytest.mark.parametrize('traced',[False,True]) +def test_close_preserves_exact_exception_even_if_stringification_fails(traced, monkeypatch): + monkeypatch.setenv('FI_HIDE_ERROR_DETAILS','true') + err=NonStringifiableError() + class Wire(httpx.SyncByteStream): + def __iter__(self): yield event(START) + def close(self): raise err + c=anthropic.Anthropic(api_key='local',http_client=httpx.Client(transport=httpx.MockTransport(lambda req:httpx.Response(200,headers={'content-type':'text/event-stream'},stream=Wire())))) + if traced: p,x,i=setup() + try: + s=c.messages.create(**PARAMS) + try: s.close() + except BaseException as got: assert got is err, f'expected original exception identity, got {type(got).__name__}' + else: pytest.fail('expected transport error') + finally: + if traced: i.uninstrument(); p.shutdown() + c.close() + +@pytest.mark.parametrize('traced',[False,True]) +def test_close_is_not_blocked_by_other_thread_export(traced): + import threading + from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult + exporter_entered=threading.Event(); release=threading.Event(); close_done=threading.Event(); reader_errors=[]; close_errors=[] + class BlockingExporter(SpanExporter): + def export(self, spans): + exporter_entered.set() + if not release.wait(5): raise TimeoutError('test exporter timeout') + return SpanExportResult.SUCCESS + def shutdown(self): pass + body=event(START)+event(DELTA) + c=anthropic.Anthropic(api_key='local',http_client=httpx.Client(transport=httpx.MockTransport(lambda req:httpx.Response(200,headers={'content-type':'text/event-stream'},content=body)))) + if traced: + p=TracerProvider(); p.add_span_processor(SimpleSpanProcessor(BlockingExporter())); i=AnthropicInstrumentor(); i.instrument(tracer_provider=p) + try: + s=c.messages.create(**PARAMS) + def read(): + try: list(s) + except BaseException as exc: reader_errors.append(exc) + finally: + if not traced: exporter_entered.set() + def close(): + try: s.close() + except BaseException as exc: close_errors.append(exc) + finally: close_done.set() + t=threading.Thread(target=read); t.start(); assert exporter_entered.wait(2) + closer=threading.Thread(target=close); closer.start() + returned=close_done.wait(0.5) + release.set(); t.join(2); closer.join(2) + assert not t.is_alive() and not closer.is_alive() + assert not reader_errors and not close_errors + assert returned, 'native close blocked behind tracing export' + finally: + release.set() + if traced: i.uninstrument(); p.shutdown() + c.close() + +@pytest.mark.parametrize('hidden', [False, True]) +@pytest.mark.parametrize('traced', [False, True]) +@pytest.mark.parametrize('asynchronous', [False, True]) +def test_context_body_exception_identity_and_status(hidden, traced, asynchronous, monkeypatch): + monkeypatch.setenv('FI_HIDE_ERROR_DETAILS', str(hidden).lower()) + err = NonStringifiableError() + if asynchronous: + async def run(): + c = anthropic.AsyncAnthropic(api_key='local', http_client=httpx.AsyncClient(transport=httpx.MockTransport(lambda req: httpx.Response(200, headers={'content-type':'text/event-stream'}, content=event(START))))) + if traced: p,x,i=setup() + try: + s = await c.messages.create(**PARAMS) + try: + async with s: + raise err + except BaseException as got: assert got is err + else: pytest.fail('expected body error') + if traced: + spans=x.get_finished_spans(); assert len(spans)==1 + assert spans[0].status.status_code.name == 'ERROR' + if hidden: assert 'NonStringifiableError' not in str(spans[0].status.description) + finally: + if traced: i.uninstrument(); p.shutdown() + await c.close() + asyncio.run(run()); return + c = anthropic.Anthropic(api_key='local', http_client=httpx.Client(transport=httpx.MockTransport(lambda req: httpx.Response(200, headers={'content-type':'text/event-stream'}, content=event(START))))) + if traced: p,x,i=setup() + try: + s=c.messages.create(**PARAMS) + try: + with s: raise err + except BaseException as got: assert got is err + else: pytest.fail('expected body error') + if traced: + spans=x.get_finished_spans(); assert len(spans)==1 + assert spans[0].status.status_code.name == 'ERROR' + if hidden: assert 'NonStringifiableError' not in str(spans[0].status.description) + finally: + if traced: i.uninstrument(); p.shutdown() + c.close() + +@pytest.mark.parametrize('hidden', [False, True]) +@pytest.mark.parametrize('traced', [False, True]) +def test_request_error_identity(hidden, traced, monkeypatch): + monkeypatch.setenv('FI_HIDE_ERROR_DETAILS', str(hidden).lower()) + err=NonStringifiableError() + def transport(req): raise err + c=anthropic.Anthropic(api_key='local', max_retries=0, http_client=httpx.Client(transport=httpx.MockTransport(transport))) + if traced: p,x,i=setup() + try: + try: c.messages.create(**PARAMS) + except BaseException as got: + # The SDK wraps transport exceptions; compare its cause identity. + assert got.__cause__ is err + else: pytest.fail('expected transport error') + if traced: + spans=x.get_finished_spans(); assert len(spans)==1 + assert spans[0].status.status_code.name == 'ERROR' + finally: + if traced: i.uninstrument(); p.shutdown() + c.close() + +@pytest.mark.parametrize('hidden', [False, True]) +@pytest.mark.parametrize('where', ['advance', 'close']) +@pytest.mark.parametrize('traced', [False, True]) +def test_async_transport_exception_identity(hidden, where, traced, monkeypatch): + monkeypatch.setenv('FI_HIDE_ERROR_DETAILS', str(hidden).lower()) + err = NonStringifiableError() + class Wire(httpx.AsyncByteStream): + async def __aiter__(self): + yield event(START) + if where == 'advance': raise err + async def aclose(self): + if where == 'close': raise err + async def run(): + c=anthropic.AsyncAnthropic(api_key='local', http_client=httpx.AsyncClient(transport=httpx.MockTransport(lambda req:httpx.Response(200,headers={'content-type':'text/event-stream'},stream=Wire())))) + if traced: p,x,i=setup() + try: + s=await c.messages.create(**PARAMS) + if where == 'advance': assert (await s.__anext__()).type == 'message_start' + try: + if where == 'advance': await s.__anext__() + else: await s.close() + except BaseException as got: assert got is err + else: pytest.fail('expected transport error') + if traced: + spans=x.get_finished_spans(); assert len(spans)==1 + assert spans[0].status.status_code.name=='ERROR' + if hidden: assert 'NonStringifiableError' not in str(spans[0].status.description) + finally: + if traced: i.uninstrument(); p.shutdown() + await c.close() + asyncio.run(run()) + +@pytest.mark.parametrize('hidden', [False, True]) +def test_ordinary_error_diagnostics_and_redaction(hidden, monkeypatch): + monkeypatch.setenv('FI_HIDE_ERROR_DETAILS', str(hidden).lower()) + secret='private-error-diagnostic-r5' + err=ValueError(secret) + class Wire(httpx.SyncByteStream): + def __iter__(self): + yield event(START) + raise err + def close(self): pass + c=anthropic.Anthropic(api_key='local',http_client=httpx.Client(transport=httpx.MockTransport(lambda req:httpx.Response(200,headers={'content-type':'text/event-stream'},stream=Wire())))) + p,x,i=setup() + try: + s=c.messages.create(**PARAMS); assert next(s).type=='message_start' + with pytest.raises(ValueError) as caught: next(s) + assert caught.value is err + spans=x.get_finished_spans(); assert len(spans)==1 + span=spans[0]; assert span.status.status_code.name=='ERROR' + data=str(span.status.description)+str(span.events)+str(span.attributes) + assert (secret not in data) if hidden else (secret in data) + finally: + i.uninstrument(); p.shutdown(); c.close() + +def test_failing_exporter_does_not_replace_native_error(): + from opentelemetry import context as context_api + from opentelemetry.sdk.trace.export import SpanExporter + err = NonStringifiableError() + class Wire(httpx.SyncByteStream): + def __iter__(self): + yield event(START) + raise err + def close(self): pass + class BrokenExporter(SpanExporter): + def export(self, spans): raise KeyboardInterrupt('export callback') + def shutdown(self): pass + p=TracerProvider(); p.add_span_processor(SimpleSpanProcessor(BrokenExporter())) + i=AnthropicInstrumentor(); i.instrument(tracer_provider=p) + c=anthropic.Anthropic(api_key='local',http_client=httpx.Client(transport=httpx.MockTransport(lambda req:httpx.Response(200,headers={'content-type':'text/event-stream'},stream=Wire())))) + # Older OTel processors leave instrumentation suppressed if an exporter + # raises BaseException. Isolate this deliberate failure from later tests. + token = context_api.attach(context_api.get_current()) + try: + s=c.messages.create(**PARAMS); assert next(s).type=='message_start' + try: next(s) + except BaseException as got: assert got is err + else: pytest.fail('expected native error') + finally: + context_api.detach(token) + i.uninstrument(); c.close(); p.shutdown() diff --git a/python/frameworks/anthropic/tests/test_r6_telemetry_isolation.py b/python/frameworks/anthropic/tests/test_r6_telemetry_isolation.py new file mode 100644 index 000000000..b6adf5cbd --- /dev/null +++ b/python/frameworks/anthropic/tests/test_r6_telemetry_isolation.py @@ -0,0 +1,90 @@ +import json, threading +import anthropic, pytest +try: + import httpx2 as httpx +except ImportError: + import httpx +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from traceai_anthropic import AnthropicInstrumentor + +START={'type':'message_start','message':{'id':'msg_1','type':'message','role':'assistant','content':[],'model':'local','stop_reason':None,'stop_sequence':None,'usage':{'input_tokens':4,'output_tokens':0}}} +BLOCK={'type':'content_block_start','index':0,'content_block':{'type':'text','text':''}} +TEXT={'type':'content_block_delta','index':0,'delta':{'type':'text_delta','text':'hello'}} +PARAMS=dict(model='local',max_tokens=16,messages=[{'role':'user','content':'hi'}],stream=True) +def event(data): return ('event: '+data['type']+'\ndata: '+json.dumps(data)+'\n\n').encode() +def client(body): return anthropic.Anthropic(api_key='local',http_client=httpx.Client(transport=httpx.MockTransport(lambda req:httpx.Response(200,headers={'content-type':'text/event-stream'},content=body)))) +def setup(): + p=TracerProvider(); x=InMemorySpanExporter(); p.add_span_processor(SimpleSpanProcessor(x)); i=AnthropicInstrumentor(); i.instrument(tracer_provider=p); return p,x,i + +@pytest.mark.parametrize('traced',[False,True]) +def test_event_with_null_usage_reaches_caller_and_rest_is_traced(traced): + # The SDK accepts a message_start whose usage is null; telemetry must not replace it. + start=json.loads(json.dumps(START)); start['message']['usage']=None + c=client(event(start)+event(BLOCK)+event(TEXT)) + if traced: p,x,i=setup() + try: + s=c.messages.create(**PARAMS) + first=next(s); assert first.type=='message_start' and first.message.usage is None + assert [e.type for e in s]==['content_block_start','content_block_delta'] + assert s.close() is None + if traced: + spans=x.get_finished_spans(); assert len(spans)==1 + assert spans[0].status.status_code.name!='ERROR' + assert spans[0].attributes['gen_ai.output.messages.0.message.role']=='assistant' + assert spans[0].attributes['gen_ai.output.messages.0.message.content']=='hello' + assert spans[0].attributes['gen_ai.usage.input_tokens']==0 + finally: + if traced: i.uninstrument(); p.shutdown() + c.close() + +def test_unreadable_event_keeps_native_item_and_later_telemetry(monkeypatch): + from anthropic.types.raw_message_start_event import RawMessageStartEvent + class Broken(Exception): pass + def role(self): raise Broken() + c=client(event(START)+event(BLOCK)+event(TEXT)) + p,x,i=setup() + try: + s=c.messages.create(**PARAMS) + monkeypatch.setattr(RawMessageStartEvent,'message',property(role),raising=False) + first=next(s) + monkeypatch.undo() + assert first.type=='message_start' + assert [e.type for e in s]==['content_block_start','content_block_delta'] + assert s.close() is None + spans=x.get_finished_spans(); assert len(spans)==1 + assert spans[0].status.status_code.name!='ERROR' + assert 'hello' in spans[0].attributes['output.value'] + finally: + i.uninstrument(); p.shutdown(); c.close() + +@pytest.mark.parametrize('traced',[False,True]) +def test_sdk_serializer_runs_outside_lifecycle_lock(monkeypatch,traced): + try: + from anthropic.types import Completion + except ImportError: + pytest.skip('native SDK removed completions') + c=client(b'event: completion\ndata: {"id":"c1","type":"completion","model":"local","completion":"hello","stop_reason":"stop_sequence"}\n\n') + if not hasattr(c,'completions'): + c.close(); pytest.skip('native SDK removed completions') + original=Completion.model_dump; acquired=[]; holder={} + def model_dump(self,*args,**kwargs): + def probe(): + got=holder['stream']._self_lock.acquire(timeout=0.5); acquired.append(got) + if got: holder['stream']._self_lock.release() + t=threading.Thread(target=probe); t.start(); t.join(2) + return original(self,*args,**kwargs) + monkeypatch.setattr(Completion,'model_dump',model_dump) + if traced: p,x,i=setup() + try: + s=c.completions.create(model='local',prompt='Hi',max_tokens_to_sample=8,stream=True); holder['stream']=s + assert next(s).completion=='hello' + assert acquired==([True] if traced else []) + assert s.close() is None + if traced: + spans=x.get_finished_spans(); assert len(spans)==1 + assert 'hello' in spans[0].attributes['output.value'] + finally: + if traced: i.uninstrument(); p.shutdown() + c.close() diff --git a/python/frameworks/anthropic/tests/test_r7_merge_order.py b/python/frameworks/anthropic/tests/test_r7_merge_order.py new file mode 100644 index 000000000..e7f34ba11 --- /dev/null +++ b/python/frameworks/anthropic/tests/test_r7_merge_order.py @@ -0,0 +1,57 @@ +import threading +from traceai_anthropic._stream import _Stream + + +class Native: + """A thread-safe native iterator, so two threads can advance one stream.""" + def __init__(self, items): self.items = iter(items); self.lock = threading.Lock() + def __next__(self): + with self.lock: return next(self.items) + def close(self): return None + + +def lock_free(stream, seen): + # Record whether another thread can take the lifecycle lock right now. + def probe(): + got = stream._self_lock.acquire(timeout=0.5); seen.append(got) + if got: stream._self_lock.release() + t = threading.Thread(target=probe); t.start(); t.join(2) + + +def test_concurrent_advances_merge_in_arrival_order(): + ready, release = threading.Event(), threading.Event() + class Slow: + def model_dump(self, **kwargs): + ready.set(); release.wait(3); return {'completion': 'A'} + class Fast: + def model_dump(self, **kwargs): return {'completion': 'B'} + s = _Stream(Native([Slow(), Fast()]), None) + first = threading.Thread(target=lambda: next(s)); first.start(); assert ready.wait(2) + second = threading.Thread(target=lambda: next(s)); second.start(); second.join(2) + assert not second.is_alive() # the later item finished extracting first + release.set(); first.join(3) + assert s._response_accumulator._result() == {'completion': 'AB'} + assert not s._self_pending and not s._self_extracting + + +def test_serializer_result_runs_no_foreign_code_under_lock(): + seen = []; holder = {} + class Foreign: + def __deepcopy__(self, memo): lock_free(holder['s'], seen); return 'copied' + def __str__(self): lock_free(holder['s'], seen); return 'foreign' + class Result(dict): + def get(self, *a): lock_free(holder['s'], seen); return super().get(*a) + def items(self): lock_free(holder['s'], seen); return super().items() + def keys(self): lock_free(holder['s'], seen); return super().keys() + def __iter__(self): lock_free(holder['s'], seen); return super().__iter__() + def __getitem__(self, k): lock_free(holder['s'], seen); return super().__getitem__(k) + class Text(str): + def __add__(self, other): lock_free(holder['s'], seen); return str(self) + other + class Item: + def model_dump(self, **kwargs): return Result(completion=Text('hello'), extra=Foreign()) + s = _Stream(Native([Item()]), None); holder['s'] = s + next(s) + assert seen and all(seen), seen + result = s._response_accumulator._result() + assert result['completion'] == 'hello' and type(result['completion']) is str + assert result['extra'] == 'foreign' diff --git a/python/frameworks/anthropic/tests/test_r8_ticket_recovery.py b/python/frameworks/anthropic/tests/test_r8_ticket_recovery.py new file mode 100644 index 000000000..fdf864240 --- /dev/null +++ b/python/frameworks/anthropic/tests/test_r8_ticket_recovery.py @@ -0,0 +1,54 @@ +import threading +import pytest +from traceai_anthropic import _stream +from traceai_anthropic._stream import _Stream + + +class Native: + """A thread-safe native iterator, so two threads can advance one stream.""" + def __init__(self, items): self.items = iter(items); self.lock = threading.Lock() + def __next__(self): + with self.lock: return next(self.items) + def close(self): return None + + +class Item: + def __init__(self, text='x', ready=None, release=None): self.text = text; self.ready = ready; self.release = release + def model_dump(self, **kwargs): + if self.ready: + self.ready.set(); self.release.wait(5) + return {'completion': self.text} + + +MAX_PENDING = getattr(_stream, '_MAX_PENDING', 1024) + + +def test_interrupt_during_merge_leaves_nothing_stranded(monkeypatch): + s = _Stream(Native([Item('a')] + [Item() for _ in range(1000)]), None) + merge = _stream._ResponseAccumulator.merge + calls = [] + def interrupted(self, values): + calls.append(values) + if len(calls) == 1: + raise KeyboardInterrupt('async interrupt while merging') + return merge(self, values) + monkeypatch.setattr(_stream._ResponseAccumulator, 'merge', interrupted) + with pytest.raises(KeyboardInterrupt): + next(s) + for _ in range(1000): + next(s) + assert not s._self_pending and not getattr(s, '_self_extracting', ()) and s._self_active == 0 + assert s._response_accumulator._result() == {'completion': 'x' * 1000} + + +def test_stalled_extraction_keeps_the_queue_bounded(): + ready, release = threading.Event(), threading.Event() + s = _Stream(Native([Item('slow', ready, release)] + [Item() for _ in range(2 * MAX_PENDING)]), None) + slow = threading.Thread(target=lambda: next(s)); slow.start(); assert ready.wait(2) + for _ in range(2 * MAX_PENDING): + next(s) + assert len(s._self_pending) <= MAX_PENDING + 1 + release.set(); slow.join(5); assert not slow.is_alive() + assert not s._self_pending and not getattr(s, '_self_extracting', ()) and not getattr(s, '_self_skipped', ()) + # The stalled item lost its place, so its telemetry is dropped; every later item is kept. + assert s._response_accumulator._result() == {'completion': 'x' * (2 * MAX_PENDING)} diff --git a/python/frameworks/anthropic/tests/test_sdk_native_compat.py b/python/frameworks/anthropic/tests/test_sdk_native_compat.py new file mode 100644 index 000000000..cca377b9e --- /dev/null +++ b/python/frameworks/anthropic/tests/test_sdk_native_compat.py @@ -0,0 +1,108 @@ +"""Offline tests exercise the native SDK over its real HTTP transport interface.""" +import asyncio +import json + +import anthropic +import pytest +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from traceai_anthropic import AnthropicInstrumentor + +try: + import httpx2 as httpx +except ImportError: + import httpx + + +MESSAGE = { + "id": "msg_local", "type": "message", "role": "assistant", + "content": [{"type": "text", "text": "offline result"}], + "model": "local-test", "stop_reason": "end_turn", "stop_sequence": None, + "usage": {"input_tokens": 4, "output_tokens": 2}, +} + + +def fixture_provider(): + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + return provider, exporter + + +def transport_handler(request): + assert json.loads(request.content)["model"] == "local-test" + return httpx.Response(200, json=MESSAGE) + + +def test_real_sync_sdk_matches_untraced_and_exports_one_span(): + provider, exporter = fixture_provider() + client = anthropic.Anthropic(api_key="local-test", http_client=httpx.Client(transport=httpx.MockTransport(transport_handler))) + params = dict(model="local-test", max_tokens=16, messages=[{"role": "user", "content": "hello"}]) + expected = client.messages.create(**params).model_dump() + instrumentor = AnthropicInstrumentor() + instrumentor.instrument(tracer_provider=provider) + try: + actual = client.messages.create(**params).model_dump() + assert actual == expected + spans = exporter.get_finished_spans() + assert len(spans) == 1 + finally: + instrumentor.uninstrument() + client.close() + provider.shutdown() + + +def test_real_async_sdk_matches_untraced_and_exports_one_span(): + async def run(): + provider, exporter = fixture_provider() + client = anthropic.AsyncAnthropic(api_key="local-test", http_client=httpx.AsyncClient(transport=httpx.MockTransport(transport_handler))) + params = dict(model="local-test", max_tokens=16, messages=[{"role": "user", "content": "hello"}]) + expected = (await client.messages.create(**params)).model_dump() + instrumentor = AnthropicInstrumentor() + instrumentor.instrument(tracer_provider=provider) + try: + actual = (await client.messages.create(**params)).model_dump() + assert actual == expected + assert len(exporter.get_finished_spans()) == 1 + finally: + instrumentor.uninstrument() + await client.close() + provider.shutdown() + asyncio.run(run()) + + +def test_real_sdk_error_is_not_swallowed(): + def fail(request): + return httpx.Response(400, json={"type": "error", "error": {"type": "invalid_request_error", "message": "offline error"}}) + provider, exporter = fixture_provider() + client = anthropic.Anthropic(api_key="local-test", max_retries=0, http_client=httpx.Client(transport=httpx.MockTransport(fail))) + instrumentor = AnthropicInstrumentor() + instrumentor.instrument(tracer_provider=provider) + try: + with pytest.raises(anthropic.BadRequestError, match="offline error"): + client.messages.create(model="local-test", max_tokens=16, messages=[{"role": "user", "content": "hello"}]) + assert len(exporter.get_finished_spans()) == 1 + finally: + instrumentor.uninstrument() + client.close() + provider.shutdown() + + +def test_real_async_sdk_error_is_not_swallowed(): + async def run(): + def fail(request): + return httpx.Response(400, json={"type": "error", "error": {"type": "invalid_request_error", "message": "offline error"}}) + provider, exporter = fixture_provider() + client = anthropic.AsyncAnthropic(api_key="local-test", max_retries=0, http_client=httpx.AsyncClient(transport=httpx.MockTransport(fail))) + instrumentor = AnthropicInstrumentor() + instrumentor.instrument(tracer_provider=provider) + try: + with pytest.raises(anthropic.BadRequestError, match="offline error"): + await client.messages.create(model="local-test", max_tokens=16, messages=[{"role": "user", "content": "hello"}]) + assert len(exporter.get_finished_spans()) == 1 + finally: + instrumentor.uninstrument() + await client.close() + provider.shutdown() + asyncio.run(run()) diff --git a/python/frameworks/anthropic/traceai_anthropic/__init__.py b/python/frameworks/anthropic/traceai_anthropic/__init__.py index 7e1c13cca..e60118782 100644 --- a/python/frameworks/anthropic/traceai_anthropic/__init__.py +++ b/python/frameworks/anthropic/traceai_anthropic/__init__.py @@ -30,6 +30,7 @@ class AnthropicInstrumentor(BaseInstrumentor): # type: ignore[misc] "_original_completions_create", "_original_async_completions_create", "_original_messages_create", + "_original_messages_count_tokens", "_original_async_messages_create", "_original_protect", "_instruments", @@ -40,8 +41,16 @@ def instrumentation_dependencies(self) -> Collection[str]: return _instruments def _instrument(self, **kwargs: Any) -> None: - from anthropic.resources.completions import AsyncCompletions, Completions from anthropic.resources.messages import AsyncMessages, Messages + + try: + # anthropic>=1.0 removed the legacy Completions API (`/v1/complete`) + # entirely - see anthropic-sdk-python's MIGRATION.md, "Removed: the + # legacy Text Completions API". Messages is unaffected, so degrade to + # instrumenting only Messages instead of failing to instrument anything. + from anthropic.resources.completions import AsyncCompletions, Completions + except ImportError: + AsyncCompletions = Completions = None try: from fi.evals import Protect except ImportError: @@ -60,19 +69,25 @@ def _instrument(self, **kwargs: Any) -> None: config=config, ) - self._original_completions_create = Completions.create - wrap_function_wrapper( - module="anthropic.resources.completions", - name="Completions.create", - wrapper=_CompletionsWrapper(tracer=self._tracer), - ) + if Completions is not None: + self._original_completions_create = Completions.create + wrap_function_wrapper( + module="anthropic.resources.completions", + name="Completions.create", + wrapper=_CompletionsWrapper(tracer=self._tracer), + ) + else: + self._original_completions_create = None - self._original_async_completions_create = AsyncCompletions.create - wrap_function_wrapper( - module="anthropic.resources.completions", - name="AsyncCompletions.create", - wrapper=_AsyncCompletionsWrapper(tracer=self._tracer), - ) + if AsyncCompletions is not None: + self._original_async_completions_create = AsyncCompletions.create + wrap_function_wrapper( + module="anthropic.resources.completions", + name="AsyncCompletions.create", + wrapper=_AsyncCompletionsWrapper(tracer=self._tracer), + ) + else: + self._original_async_completions_create = None self._original_messages_create = Messages.create wrap_function_wrapper( @@ -88,6 +103,7 @@ def _instrument(self, **kwargs: Any) -> None: wrapper=_AsyncMessagesWrapper(tracer=self._tracer), ) + self._original_messages_count_tokens = Messages.count_tokens wrap_function_wrapper( module="anthropic.resources.messages", name="Messages.count_tokens", @@ -105,21 +121,30 @@ def _instrument(self, **kwargs: Any) -> None: self._original_protect = None def _uninstrument(self, **kwargs: Any) -> None: - from anthropic.resources.completions import AsyncCompletions, Completions from anthropic.resources.messages import AsyncMessages, Messages + + try: + from anthropic.resources.completions import AsyncCompletions, Completions + except ImportError: + AsyncCompletions = Completions = None try: from fi.evals import Protect except ImportError: logger.warning("ai-evaluation is not installed, please install it to trace protect") Protect = None - if self._original_completions_create is not None: + if Completions is not None and self._original_completions_create is not None: Completions.create = self._original_completions_create # type: ignore[method-assign] - if self._original_async_completions_create is not None: + if ( + AsyncCompletions is not None + and self._original_async_completions_create is not None + ): AsyncCompletions.create = self._original_async_completions_create # type: ignore[method-assign] if self._original_messages_create is not None: Messages.create = self._original_messages_create # type: ignore[method-assign] + if self._original_messages_count_tokens is not None: + Messages.count_tokens = self._original_messages_count_tokens # type: ignore[method-assign] if self._original_async_messages_create is not None: AsyncMessages.create = self._original_async_messages_create # type: ignore[method-assign] diff --git a/python/frameworks/anthropic/traceai_anthropic/_stream.py b/python/frameworks/anthropic/traceai_anthropic/_stream.py index d73e58f88..98f3fc212 100644 --- a/python/frameworks/anthropic/traceai_anthropic/_stream.py +++ b/python/frameworks/anthropic/traceai_anthropic/_stream.py @@ -1,5 +1,7 @@ from collections import defaultdict from copy import deepcopy +import inspect +import threading from typing import ( TYPE_CHECKING, Any, @@ -13,7 +15,6 @@ Mapping, Optional, Tuple, - Union, ) from fi_instrumentation import safe_json_dumps @@ -27,6 +28,7 @@ from opentelemetry.util.types import AttributeValue from traceai_anthropic._utils import ( _as_output_attributes, + _error_status, _finish_tracing, _ValueAndType, ) @@ -36,12 +38,224 @@ if TYPE_CHECKING: from anthropic import Stream from anthropic.types import Completion, RawMessageStreamEvent - from anthropic.types.raw_content_block_delta_event import RawContentBlockDeltaEvent - from anthropic.types.text_block import TextBlock - from anthropic.types.tool_use_block import ToolUseBlock -class _Stream(ObjectProxy): # type: ignore +# Settled items held back by a stalled earlier extraction before ordering is +# given up for that extraction. +_MAX_PENDING = 1024 + + +class _StreamLifecycle(ObjectProxy): + """Keep the tracing lifetime coupled to the SDK stream's native lifetime.""" + + def __init__(self, stream): + super().__init__(stream) + self._self_lock = threading.RLock() + self._self_active = 0 + self._self_terminal = False + self._self_error = None + self._self_finalizing = False + # Items are extracted outside the lock, possibly concurrently; tickets + # taken on arrival keep their merges in the order the stream produced them. + # A settled item waits only for earlier tickets still being extracted. + self._self_next_ticket = 0 + self._self_extracting = set() + self._self_pending = {} + self._self_skipped = set() + + def _begin_operation(self): + with self._self_lock: + self._self_active += 1 + + def _end_operation(self, *, terminal=False, error=None, item=None, has_item=False): + ticket = values = None + try: + if has_item: + with self._self_lock: + ticket = self._self_next_ticket + self._self_next_ticket += 1 + self._self_extracting.add(ticket) + values = self._extract(item) + finally: + self._settle(terminal, error, ticket, values) + + def _extract(self, item): + # SDK model methods and the values they return are external code, so read + # and copy the item before taking the lifecycle lock. A telemetry failure + # must not replace the native item. + try: + return _plain(self._response_accumulator.extract(item)) + except Exception: + return None + + def _merge(self, values): + # Once finalization is claimed, a subsequent native operation + # must not mutate the accumulator being read by the exporter. + if values is None or self._self_finalizing: + return + try: + self._response_accumulator.merge(values) + except Exception: + # Losing one item's telemetry must not change the native operation. + pass + + def _drain(self, everything=False): + # Merge settled items in ticket order, up to the earliest ticket still + # being extracted. Recomputed from state on every call, so an interrupt + # part-way through leaves nothing stranded. + floor = None + if not everything and self._self_extracting: + floor = min(self._self_extracting) + for ticket in sorted(self._self_pending): + if floor is not None and ticket > floor: + break + self._merge(self._self_pending.pop(ticket)) + + def _settle(self, terminal, error, ticket, values): + finalize = False + final_error = None + try: + with self._self_lock: + try: + if ticket is not None: + self._self_extracting.discard(ticket) + if ticket in self._self_skipped: + # Later items were already merged without it. + self._self_skipped.discard(ticket) + else: + self._self_pending[ticket] = values + if len(self._self_pending) > _MAX_PENDING: + # An earlier extraction is stalled; stop holding later + # items for it so the queue stays bounded. + latest = max(self._self_pending) + stalled = {t for t in self._self_extracting if t < latest} + self._self_skipped |= stalled + self._self_extracting -= stalled + self._drain() + finally: + if terminal: + self._self_terminal = True + if error is not None and self._self_error is None: + self._self_error = error + self._self_active -= 1 + if (self._self_terminal and not self._self_active + and not self._self_finalizing): + # Every operation has settled; merge whatever is left. + self._drain(everything=True) + self._self_finalizing = True + finalize = True + final_error = self._self_error + finally: + if finalize: + # Span processors/exporters are arbitrary synchronous user code. + # Claim exactly once under the lock, then release it before calling + # them so another thread can always reach the native close. + try: + if final_error is not None: + self._finish_error(final_error) + else: + self._finish_tracing(trace_api.Status(trace_api.StatusCode.OK)) + except BaseException: + # Telemetry must not replace the native operation's outcome. + pass + + def _advance(self, iterator): + self._begin_operation() + try: + item = next(iterator) + except StopIteration: + self._end_operation(terminal=True) + raise + except BaseException as exc: + self._end_operation(terminal=True, error=exc) + raise + self._end_operation(item=item, has_item=True) + return item + + async def _advance_async(self, iterator): + self._begin_operation() + try: + item = await iterator.__anext__() + except StopAsyncIteration: + self._end_operation(terminal=True) + raise + except BaseException as exc: + self._end_operation(terminal=True, error=exc) + raise + self._end_operation(item=item, has_item=True) + return item + + def close(self): + # Calling an async native close merely creates a coroutine; ownership + # starts when it is awaited, just as the underlying close does. + if inspect.iscoroutinefunction(self.__wrapped__.close): + async def close_async(): + self._begin_operation() + try: + result = await self.__wrapped__.close() + except BaseException as exc: + self._end_operation(terminal=True, error=exc) + raise + self._end_operation(terminal=True) + return result + return close_async() + self._begin_operation() + try: + result = self.__wrapped__.close() + except BaseException as exc: + self._end_operation(terminal=True, error=exc) + raise + if inspect.isawaitable(result): + async def wait_for_close(): + try: + value = await result + except BaseException as exc: + self._end_operation(terminal=True, error=exc) + raise + self._end_operation(terminal=True) + return value + return wait_for_close() + self._end_operation(terminal=True) + return result + + def __enter__(self): + self.__wrapped__.__enter__() + return self + + def __exit__(self, exc_type, exc, tb): + self._begin_operation() + try: + result = self.__wrapped__.__exit__(exc_type, exc, tb) + except BaseException as error: + self._end_operation(terminal=True, error=error) + raise + self._end_operation(terminal=True, error=exc) + return result + + async def __aenter__(self): + await self.__wrapped__.__aenter__() + return self + + async def __aexit__(self, exc_type, exc, tb): + self._begin_operation() + try: + result = await self.__wrapped__.__aexit__(exc_type, exc, tb) + except BaseException as error: + self._end_operation(terminal=True, error=error) + raise + self._end_operation(terminal=True, error=exc) + return result + + def _finish_error(self, exception): + status = _error_status(exception) + try: + self._with_span.record_exception(exception) + except BaseException: + pass + self._finish_tracing(status) + + +class _Stream(_StreamLifecycle): # type: ignore __slots__ = ( "_response_accumulator", "_with_span", @@ -57,42 +271,30 @@ def __init__( self._response_accumulator = _ResponseAccumulator() self._with_span = with_span + def __next__(self): + return self._advance(self.__wrapped__) + + async def __anext__(self): + return await self._advance_async(self.__wrapped__) + def __iter__(self) -> Iterator["Completion"]: - try: - for item in self.__wrapped__: - self._response_accumulator.process_chunk(item) - yield item - except Exception as exception: - status = trace_api.Status( - status_code=trace_api.StatusCode.ERROR, - description=f"{type(exception).__name__}: {exception}", - ) - self._with_span.record_exception(exception) - self._finish_tracing(status=status) - raise - status = trace_api.Status( - status_code=trace_api.StatusCode.OK, - ) - self._finish_tracing(status=status) + iterator = iter(self.__wrapped__) + while True: + try: + item = self._advance(iterator) + except StopIteration: + return + # Closing this iterator does not close the native stream. + yield item async def __aiter__(self) -> AsyncIterator["Completion"]: - try: - async for item in self.__wrapped__: - self._response_accumulator.process_chunk(item) - yield item - except Exception as exception: - status = trace_api.Status( - status_code=trace_api.StatusCode.ERROR, - description=f"{type(exception).__name__}: {exception}", - ) - self._with_span.record_exception(exception) - self._finish_tracing(status=status) - raise - # completed without exception - status = trace_api.Status( - status_code=trace_api.StatusCode.OK, - ) - self._finish_tracing(status=status) + iterator = self.__wrapped__.__aiter__() + while True: + try: + item = await self._advance_async(iterator) + except StopAsyncIteration: + return + yield item def _finish_tracing( self, @@ -121,9 +323,11 @@ def __init__(self) -> None: stop_reason=_SimpleStringReplace(), ) - def process_chunk(self, chunk: "Completion") -> None: + def extract(self, chunk: "Completion") -> Dict[str, Any]: + return chunk.model_dump(exclude_unset=True, warnings=False) + + def merge(self, values: Mapping[str, Any]) -> None: self._is_null = False - values = chunk.model_dump(exclude_unset=True, warnings=False) self._values += values def _result(self) -> Optional[Dict[str, Any]]: @@ -159,13 +363,21 @@ def get_extra_attributes(self) -> Iterator[Tuple[str, AttributeValue]]: yield SpanAttributes.GEN_AI_OUTPUT_MESSAGES, completion -class _MessagesStream(ObjectProxy): # type: ignore +class _MessagesStream(_StreamLifecycle): # type: ignore __slots__ = ( "_response_accumulator", "_with_span", "_is_finished", ) + def __del__(self): + # Preserve the prior best-effort cleanup for abandoned message streams. + try: + self._begin_operation() + self._end_operation(terminal=True) + except Exception: + pass + def __init__( self, stream: "Stream[RawMessageStreamEvent]", @@ -175,55 +387,30 @@ def __init__( self._response_accumulator = _MessageResponseAccumulator() self._with_span = with_span - def __iter__(self) -> Iterator["RawMessageStreamEvent"]: - try: - for item in self.__wrapped__: - self._response_accumulator.process_chunk(item) - yield item - except Exception as exception: - status = trace_api.Status( - status_code=trace_api.StatusCode.ERROR, - description=f"{type(exception).__name__}: {exception}", - ) - self._with_span.record_exception(exception) - self._finish_tracing(status=status) - raise - status = trace_api.Status( - status_code=trace_api.StatusCode.OK, - ) - self._finish_tracing(status=status) + def __next__(self): + return self._advance(self.__wrapped__) - async def __aiter__(self) -> AsyncIterator["RawMessageStreamEvent"]: - try: - async for item in self.__wrapped__: - self._response_accumulator.process_chunk(item) - yield item - except Exception as exception: - status = trace_api.Status( - status_code=trace_api.StatusCode.ERROR, - description=f"{type(exception).__name__}: {exception}", - ) - self._with_span.record_exception(exception) - self._finish_tracing(status=status) - raise - # completed without exception - status = trace_api.Status( - status_code=trace_api.StatusCode.OK, - ) - self._finish_tracing(status=status) + async def __anext__(self): + return await self._advance_async(self.__wrapped__) - async def aclose(self): - await self.__wrapped__.aclose() - if not self._is_finished: - self._is_finished = True - status = trace_api.Status(status_code=trace_api.StatusCode.OK) - self._finish_tracing(status=status) + def __iter__(self) -> Iterator["RawMessageStreamEvent"]: + iterator = iter(self.__wrapped__) + while True: + try: + item = self._advance(iterator) + except StopIteration: + return + # Closing this iterator does not close the native stream. + yield item - def __del__(self): - if not getattr(self, "_is_finished", False): - self._is_finished = True - status = trace_api.Status(status_code=trace_api.StatusCode.OK) - self._finish_tracing(status=status) + async def __aiter__(self) -> AsyncIterator["RawMessageStreamEvent"]: + iterator = self.__wrapped__.__aiter__() + while True: + try: + item = await self._advance_async(iterator) + except StopAsyncIteration: + return + yield item def _finish_tracing( self, @@ -238,19 +425,44 @@ def _finish_tracing( ) +def _token_count(value: Any) -> Optional[str]: + return None if value is None else str(value) + + +def _plain(value: Any) -> Any: + """Copy extracted values into built-in types, so merging runs no foreign code.""" + if value is None or type(value) in (bool, int, float, str): + return value + if isinstance(value, str): + return str.__str__(value) + if isinstance(value, int): + return int.__int__(value) + if isinstance(value, float): + return float.__float__(value) + if isinstance(value, Mapping): + return {_plain(key): _plain(item) for key, item in value.items()} + if isinstance(value, tuple): + return tuple(_plain(item) for item in value) + if isinstance(value, list): + return [_plain(item) for item in value] + return str(value) + + class _MessageResponseAccumulator: __slots__ = ( "_is_null", "_values", "_current_message_idx", - "_current_content_block_type", + "_current_block", ) def __init__(self) -> None: self._is_null = True self._current_message_idx = -1 - self._current_content_block_type: Union["TextBlock", "ToolUseBlock", None] = ( - None + # (kind, type, tool name) of the open content block; kind is "text", + # "tool_use" or None for blocks whose deltas are not recorded. + self._current_block: Tuple[Optional[str], Optional[str], Optional[str]] = ( + None, None, None ) self._values = _ValuesAccumulator( messages=_IndexedAccumulator( @@ -271,7 +483,8 @@ def __init__(self) -> None: ), ) - def process_chunk(self, chunk: "RawContentBlockDeltaEvent") -> None: + def extract(self, chunk: "RawMessageStreamEvent") -> Tuple[Any, ...]: + """Read an SDK event into plain values; merge applies them in order.""" from anthropic.types.raw_content_block_delta_event import ( RawContentBlockDeltaEvent, ) @@ -283,54 +496,75 @@ def process_chunk(self, chunk: "RawContentBlockDeltaEvent") -> None: from anthropic.types.text_block import TextBlock from anthropic.types.tool_use_block import ToolUseBlock - self._is_null = False if isinstance(chunk, RawMessageStartEvent): + # The SDK accepts events whose usage is null; keep the rest of the event. + usage = getattr(chunk.message, "usage", None) + return ("message_start", chunk.message.role, + _token_count(getattr(usage, "input_tokens", None))) + if isinstance(chunk, RawContentBlockStartEvent): + block = chunk.content_block + if isinstance(block, TextBlock): + return ("block_start", "text", block.type, None) + if isinstance(block, ToolUseBlock): + return ("block_start", "tool_use", block.type, block.name) + return ("block_start", None, None, None) + if isinstance(chunk, RawContentBlockDeltaEvent): + delta_type = getattr(chunk.delta, "type", None) + if delta_type == "text_delta": + return ("block_delta", chunk.index, delta_type, chunk.delta.text) # type: ignore + if delta_type == "input_json_delta": + return ("block_delta", chunk.index, delta_type, chunk.delta.partial_json) # type: ignore + return ("other",) + if isinstance(chunk, RawMessageDeltaEvent): + usage = getattr(chunk, "usage", None) + return ("message_delta", chunk.delta.stop_reason, + _token_count(getattr(usage, "output_tokens", None))) + return ("other",) + + def merge(self, event: Tuple[Any, ...]) -> None: + self._is_null = False + kind = event[0] + if kind == "message_start": + _, role, input_tokens = event self._current_message_idx += 1 - value = { + self._values += { "messages": { "index": str(self._current_message_idx), - "role": chunk.message.role, - "input_tokens": str(chunk.message.usage.input_tokens), + "role": role, + "input_tokens": input_tokens, } } - self._values += value - elif isinstance(chunk, RawContentBlockStartEvent): - self._current_content_block_type = chunk.content_block - elif isinstance(chunk, RawContentBlockDeltaEvent): - if isinstance(self._current_content_block_type, TextBlock): - value = { - "messages": { - "index": str(self._current_message_idx), - "content": { - "index": chunk.index, - "type": self._current_content_block_type.type, - "text": chunk.delta.text, # type: ignore - }, - } + elif kind == "block_start": + self._current_block = event[1:] + elif kind == "block_delta": + _, index, delta_type, fragment = event + block_kind, block_type, tool_name = self._current_block + if block_kind == "text" and delta_type == "text_delta": + content = {"index": index, "type": block_type, "text": fragment} + elif block_kind == "tool_use" and delta_type == "input_json_delta": + content = { + "index": index, + "type": block_type, + "tool_name": tool_name, + "tool_input": fragment, } - self._values += value - elif isinstance(self._current_content_block_type, ToolUseBlock): - value = { - "messages": { - "index": str(self._current_message_idx), - "content": { - "index": chunk.index, - "type": self._current_content_block_type.type, - "tool_name": self._current_content_block_type.name, - "tool_input": chunk.delta.partial_json, # type: ignore - }, - } + else: + return + self._values += { + "messages": { + "index": str(self._current_message_idx), + "content": content, } - self._values += value - elif isinstance(chunk, RawMessageDeltaEvent): - value = { + } + elif kind == "message_delta": + _, stop_reason, output_tokens = event + self._values += { "messages": { "index": str(self._current_message_idx), - "stop_reason": chunk.delta.stop_reason, - "output_tokens": str(chunk.usage.output_tokens), + "stop_reason": stop_reason, + "output_tokens": output_tokens, } } - self._values += value def _result(self) -> Optional[Dict[str, Any]]: if self._is_null: diff --git a/python/frameworks/anthropic/traceai_anthropic/_utils.py b/python/frameworks/anthropic/traceai_anthropic/_utils.py index 3e44de6e0..d49eae795 100644 --- a/python/frameworks/anthropic/traceai_anthropic/_utils.py +++ b/python/frameworks/anthropic/traceai_anthropic/_utils.py @@ -11,6 +11,30 @@ logger.addHandler(logging.NullHandler()) +def _error_status(exception: BaseException) -> trace_api.Status: + """Format useful diagnostics without trusting an exception's __str__.""" + name = type(exception).__name__ + try: + detail = str(exception) + except BaseException: + detail = "" + return trace_api.Status(trace_api.StatusCode.ERROR, f"{name}: {detail}") + + +def _finish_request_error(span: _WithSpan, exception: BaseException) -> None: + # Each operation is isolated: even a broken processor must not obscure the + # original SDK error or prevent a best-effort attempt to end the span. + for action in ( + lambda: span.set_status(_error_status(exception)), + lambda: span.record_exception(exception), + span.finish_tracing, + ): + try: + action() + except BaseException: + pass + + class _ValueAndType(NamedTuple): value: str type: FiMimeTypeValues @@ -30,13 +54,13 @@ def _finish_tracing( try: attributes: Attributes = dict(has_attributes.get_attributes()) except Exception as e: - logger.exception("Failed to get attributes") + logger.warning("Failed to get attributes") attributes = None try: extra_attributes: Attributes = dict(has_attributes.get_extra_attributes()) except Exception as e: - logger.exception("Failed to get extra attributes") + logger.warning("Failed to get extra attributes") extra_attributes = None try: @@ -46,8 +70,7 @@ def _finish_tracing( extra_attributes=extra_attributes, ) except Exception as e: - print(f"Failed to finish tracing: {e}") - logger.exception("Failed to finish tracing") + logger.warning("Failed to finish tracing") raise @@ -56,8 +79,7 @@ def _io_value_and_type(obj: Any) -> _ValueAndType: json_value = safe_json_dumps(obj) return _ValueAndType(json_value, FiMimeTypeValues.JSON) except Exception as e: - print(f"Failed to serialize as JSON: {e}") - logger.exception("Failed to get input attributes from request parameters.") + logger.warning("Failed to get input attributes from request parameters.") return _ValueAndType(str(obj), FiMimeTypeValues.TEXT) diff --git a/python/frameworks/anthropic/traceai_anthropic/_with_span.py b/python/frameworks/anthropic/traceai_anthropic/_with_span.py index f81d4e7ad..4b354ff4d 100644 --- a/python/frameworks/anthropic/traceai_anthropic/_with_span.py +++ b/python/frameworks/anthropic/traceai_anthropic/_with_span.py @@ -22,7 +22,7 @@ def __init__( try: self._is_finished = not self._span.is_recording() except Exception: - logger.exception("Failed to check if span is recording") + logger.warning("Failed to check if span is recording") print("Error checking span recording status, defaulting to finished") self._is_finished = True @@ -42,7 +42,7 @@ def record_exception(self, exception: Exception) -> None: try: self._span.record_exception(exception) except Exception: - logger.exception("Failed to record exception on span") + logger.warning("Failed to record exception on span") print("Error recording exception on span") def set_status(self, status: Union[trace_api.Status, trace_api.StatusCode]) -> None: @@ -51,7 +51,7 @@ def set_status(self, status: Union[trace_api.Status, trace_api.StatusCode]) -> N try: self._span.set_status(status=status) except Exception: - logger.exception("Failed to set status on span") + logger.warning("Failed to set status on span") print("Error setting status on span") def add_event(self, name: str) -> None: @@ -60,7 +60,7 @@ def add_event(self, name: str) -> None: try: self._span.add_event(name) except Exception: - logger.exception("Failed to add event to span") + logger.warning("Failed to add event to span") print("Error adding event to span") def finish_tracing( @@ -81,20 +81,20 @@ def finish_tracing( try: self._span.set_attribute(key, value) except Exception: - logger.exception("Failed to set attribute on span") + logger.warning("Failed to set attribute on span") print(f"Error setting attribute: {key}") if status is not None: try: self._span.set_status(status=status) except Exception: - logger.exception("Failed to set status code on span") + logger.warning("Failed to set status code on span") print("Error setting final status") try: self._span.end() except Exception: - logger.exception("Failed to end span") + logger.warning("Failed to end span") print("Error ending span") self._is_finished = True diff --git a/python/frameworks/anthropic/traceai_anthropic/_wrappers.py b/python/frameworks/anthropic/traceai_anthropic/_wrappers.py index 15b6aa3a2..c21251a9f 100644 --- a/python/frameworks/anthropic/traceai_anthropic/_wrappers.py +++ b/python/frameworks/anthropic/traceai_anthropic/_wrappers.py @@ -33,6 +33,7 @@ from opentelemetry import trace as trace_api from opentelemetry.trace import INVALID_SPAN from traceai_anthropic._stream import _MessagesStream, _Stream +from traceai_anthropic._utils import _finish_request_error from traceai_anthropic._with_span import _WithSpan if TYPE_CHECKING: @@ -64,7 +65,7 @@ def _start_as_current_span( attributes=attributes, ) except Exception as e: - print(f"Error creating span: {e}") + logger.warning("Failed to create span") span = INVALID_SPAN with trace_api.use_span( span, @@ -151,12 +152,8 @@ def __call__( ), ) - except Exception as exception: - print(f"Error in completion: {exception}") - span.set_status( - trace_api.Status(trace_api.StatusCode.ERROR, str(exception)) - ) - span.record_exception(exception) + except BaseException as exception: + _finish_request_error(span, exception) raise streaming = kwargs.get("stream", False) @@ -219,11 +216,8 @@ async def __call__( try: response = await wrapped(*args, **kwargs) span.set_attributes(dict(_get_raw_output(response))) - except Exception as exception: - span.set_status( - trace_api.Status(trace_api.StatusCode.ERROR, str(exception)) - ) - span.record_exception(exception) + except BaseException as exception: + _finish_request_error(span, exception) raise span.set_status(trace_api.StatusCode.OK) streaming = kwargs.get("stream", False) @@ -305,11 +299,8 @@ def __call__( # Add I/O to span attributes instead of sending separately self._add_io_to_span_attributes(span=span, output_data=output_content) - except Exception as exception: - span.set_status( - trace_api.Status(trace_api.StatusCode.ERROR, str(exception)) - ) - span.record_exception(exception) + except BaseException as exception: + _finish_request_error(span, exception) raise streaming = kwargs.get("stream", False) @@ -382,11 +373,8 @@ async def __call__( try: response = await wrapped(*args, **kwargs) span.set_attributes(dict(_get_raw_output(response))) - except Exception as exception: - span.set_status( - trace_api.Status(trace_api.StatusCode.ERROR, str(exception)) - ) - span.record_exception(exception) + except BaseException as exception: + _finish_request_error(span, exception) raise streaming = kwargs.get("stream", False) if streaming: @@ -454,11 +442,8 @@ def __call__( try: response = wrapped(*args, **kwargs) span.set_attributes(dict(_get_raw_output(response))) - except Exception as exception: - span.set_status( - trace_api.Status(trace_api.StatusCode.ERROR, str(exception)) - ) - span.record_exception(exception) + except BaseException as exception: + _finish_request_error(span, exception) raise span.set_status(trace_api.StatusCode.OK) @@ -569,7 +554,7 @@ def process_message_content(content): "query": str(eval_input[0]) if eval_input else None, } except Exception as e: - logger.exception(f"Error in _extract_image_data: {e}") + logger.warning("Failed to extract image data") return { "input_images": None, "filtered_messages": messages, @@ -699,17 +684,17 @@ def _get_llm_input_messages(messages: List[Dict[str, str]]) -> Any: def _get_tool_use_block( block: ToolUseBlock, msg_prefix: str, tool_index: int ) -> Iterator[Tuple[str, Any]]: - if tool_call_id := block.id: + if tool_call_id := (block.get("id") if isinstance(block, Mapping) else block.id): yield ( f"{msg_prefix}.{MESSAGE_TOOL_CALLS}.{tool_index}.{TOOL_CALL_ID}", tool_call_id, ) - if block_name := block.name: + if block_name := (block.get("name") if isinstance(block, Mapping) else block.name): yield ( f"{msg_prefix}.{MESSAGE_TOOL_CALLS}.{tool_index}.{TOOL_CALL_FUNCTION_NAME}", block_name, ) - if block_input := block.input: + if block_input := (block.get("input") if isinstance(block, Mapping) else block.input): yield ( f"{msg_prefix}.{MESSAGE_TOOL_CALLS}.{tool_index}.{TOOL_CALL_FUNCTION_ARGUMENTS_JSON}", safe_json_dumps(block_input), diff --git a/python/frameworks/cognee/LICENSE b/python/frameworks/cognee/LICENSE new file mode 100644 index 000000000..72ffed2f0 --- /dev/null +++ b/python/frameworks/cognee/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for describing the origin of the Work and + reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may accept and charge a + fee for acceptance of support, warranty, indemnity, or other liability + obligations and/or rights consistent with this License. However, in + accepting such obligations, You may act only on Your own behalf and on + Your sole responsibility, not on behalf of any other Contributor, and + only if You agree to indemnify, defend, and hold each Contributor + harmless for any liability incurred by, or claims asserted against, + such Contributor by reason of your accepting any such warranty or + additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2024-2026 Future AGI, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/python/frameworks/cognee/README.md b/python/frameworks/cognee/README.md new file mode 100644 index 000000000..d19fa5402 --- /dev/null +++ b/python/frameworks/cognee/README.md @@ -0,0 +1,39 @@ +# traceai-cognee + +Send Cognee's native OpenTelemetry spans to Future AGI without additional instrumentation. Enable `COGNEE_TRACING_ENABLED=true` before your first Cognee operation and configure once before that operation. + +Requires Python 3.10–3.13 and `fi-instrumentation-otel>=1.2.2,<2`. + +```bash +pip install "cognee[tracing]==1.6.5" "traceai-cognee==0.1.0" +export COGNEE_TRACING_ENABLED=true +export FI_API_KEY="YOUR_FI_API_KEY" +export FI_SECRET_KEY="YOUR_FI_SECRET_KEY" +export FI_PROJECT_NAME="my-cognee-app" +``` + +Configure your model and embedding providers using the [Cognee documentation](https://docs.cognee.ai/), then run your operations with the adapter attached: + +```python +import os +from traceai_cognee import configure + +handle = configure(project_name=os.environ['FI_PROJECT_NAME']) +# Run cognee.add, cognee.cognify and cognee.search here. +handle.force_flush() +handle.shutdown() +``` + +Set `FI_API_KEY` and `FI_SECRET_KEY` for the Future AGI exporter. `FI_BASE_URL` selects a different Future AGI origin. Alternatively supply an OTLP HTTP `endpoint` and `headers`, or an injected OpenTelemetry exporter. A supplied exporter is owned by the handle. `provider` can be the global SDK provider Cognee will use. Existing processors are not shut down; a provider whose default processor would be replaced on add is rejected. + +A repeated `configure` with the same options returns the active handle; different options require shutting it down first. Shutdown detaches the handle immediately and closes the exporter asynchronously, after in-flight callbacks return; it does not wait for exporter closure. Flush before shutdown when the queue must be drained. Each span is exported by the configuration that was active when it started: end spans and flush before shutdown, because a span still open at shutdown is not exported, and a later configuration does not adopt it. The adapter tracks at most 65,536 open spans; beyond that the oldest open span is dropped when it ends. Spans that share a trace ID, span ID and start time with another tracked span are not exported; such reuse only happens with a faulty ID generator, and once the limit evicts the earlier span's entry the protection no longer applies to it. If a custom provider raises from `add_span_processor` after attaching the processor, a retry reuses it; a provider that hides its processor list can keep an inactive extra processor for each failed attempt. Closure uses a non-daemon thread, so a stuck exporter can delay interpreter exit. An old handle cannot flush or shut down a later configuration. Shutting down the provider closes the adapter too and prevents reuse of that provider. Overlapping `configure` calls fail fast with `RuntimeError`; retry after the first call finishes. A successful `force_flush` drains the processor queue, not an acknowledgement of remote delivery. + +By default only allowlisted operational attributes are sent. `capture_content=True` includes recognized Cognee input/output fields. Error details and stack traces remain excluded. Span names are retained; keep sensitive data out of them. Other processors on the provider receive unmodified native spans and need their own privacy controls. + +### Structured content redaction + +With content capture enabled, the adapter redacts recognized credential field names in nested objects, arrays, JSON-serialized objects, and flattened paths. These include API keys, authorization headers, access/refresh/auth/bearer/ID/session/security tokens, client secrets, secret/private keys, passwords, credentials, and AWS access-key IDs and secret access keys. Common snake-case, camel-case, and dotted/bracketed path forms are recognized. Fields such as `input_tokens`, `token_count`, and `request_id` remain available. + +This is a field-name policy, not a universal secret detector. Free text and unknown encodings are not scanned for credentials; enable content capture only for approved application data. The adapter does not mutate the original content or inspect opaque runtime objects. Numeric attributes outside the OTLP signed 64-bit integer range, or non-finite floating-point values, are omitted before encoding. + +For a complete add/cognify/search application, see the [Cognee example](../../examples/cognee/). Search spans are mapped to `RETRIEVER` and structured-output model calls to `LLM`. Native span IDs, parent relationships and status codes are preserved, including Cognee's missing-collection probe errors. diff --git a/python/frameworks/cognee/pyproject.toml b/python/frameworks/cognee/pyproject.toml new file mode 100644 index 000000000..94ceef1eb --- /dev/null +++ b/python/frameworks/cognee/pyproject.toml @@ -0,0 +1,26 @@ +[build-system] +requires = ["hatchling>=1.27,<2"] +build-backend = "hatchling.build" + +[project] +name = "traceai-cognee" +version = "0.1.0" +description = "Future AGI exporter for Cognee native OpenTelemetry spans" +readme = "README.md" +license = "Apache-2.0" +authors = [{name = "Future AGI", email = "no-reply@futureagi.com"}] +requires-python = ">=3.10,<3.14" +dependencies = ["fi-instrumentation-otel>=1.2.2,<2", "opentelemetry-sdk>=1.27,<2"] + +[project.optional-dependencies] +dev = ["pytest>=8,<9"] + +[project.urls] +Repository = "https://github.com/future-agi/traceAI" +Documentation = "https://docs.futureagi.com/docs/integrations/traceai/cognee" + +[tool.hatch.build.targets.wheel] +packages = ["traceai_cognee"] + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/python/frameworks/cognee/tests/test_adapter.py b/python/frameworks/cognee/tests/test_adapter.py new file mode 100644 index 000000000..349dcc5ad --- /dev/null +++ b/python/frameworks/cognee/tests/test_adapter.py @@ -0,0 +1,96 @@ +import json +import unittest +from opentelemetry import trace +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import Status, StatusCode +from traceai_cognee import configure, __version__ + +class TestAdapter(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.provider = TracerProvider() + trace.set_tracer_provider(cls.provider) + + def setUp(self): + self.other = InMemorySpanExporter() + self.provider.add_span_processor(SimpleSpanProcessor(self.other)) + self.sink = InMemorySpanExporter() + + def emit(self, name, attrs): + with self.provider.get_tracer('test').start_as_current_span(name) as span: + for key, value in attrs.items(): span.set_attribute(key, value) + span.record_exception(ValueError('fake-private-error')) + span.set_status(Status(StatusCode.ERROR, 'fake-private-error')) + + def test_projection_copy_and_status(self): + handle = configure(project_name='sample', exporter=self.sink) + self.assertIs(handle, configure(project_name='sample', exporter=self.sink)) + with self.assertRaises(ValueError): configure(project_name='different', exporter=self.sink) + self.emit('cognee.search.query', {'cognee.search.query':'fake-private-query', 'custom.secret':'fake-secret', 'gen_ai.usage.input_tokens':7}) + self.assertTrue(handle.force_flush()) + src = self.other.get_finished_spans()[-1]; dst = self.sink.get_finished_spans()[-1] + self.assertEqual(src.attributes['cognee.search.query'], 'fake-private-query') + self.assertNotIn('cognee.search.query', dst.attributes) + self.assertNotIn('custom.secret', dst.attributes) + self.assertEqual(dst.attributes['gen_ai.usage.input_tokens'], 7) + self.assertEqual(dst.attributes['fi.span.kind'], 'RETRIEVER') + self.assertEqual(dst.status.status_code, StatusCode.ERROR) + self.assertIsNone(dst.status.description) + self.assertEqual(dst.context, src.context); self.assertEqual(dst.parent, src.parent) + self.assertEqual(dst.start_time, src.start_time); self.assertEqual(dst.end_time, src.end_time) + self.assertEqual(dst.resource.attributes['project_name'], 'sample') + self.assertNotIn('fake-private-error', str(dst.events)) + handle.shutdown() + self.emit('after', {}) + self.assertEqual(self.other.get_finished_spans()[-1].name, 'after') + + def test_content_redaction_and_llm(self): + handle = configure(project_name='sample', exporter=self.sink, capture_content=True) + value = json.dumps({'api_key':'fake-secret', 'question':'hello'}) + self.emit('cognee.observe.acreate_structured_output', {'langfuse.observation.input':value, 'langfuse.observation.output':'answer'}) + handle.force_flush(); out = self.sink.get_finished_spans()[-1] + self.assertEqual(out.attributes['fi.span.kind'], 'LLM') + self.assertEqual(json.loads(out.attributes['input.value'])['api_key'], '[REDACTED]') + self.assertEqual(out.attributes['output.value'], 'answer') + self.assertNotIn('fake-secret', str(out.attributes)) + handle.shutdown() + + def test_additional_processor_and_lifecycle(self): + handle = configure(project_name='sample', exporter=self.sink) + native_buffer = InMemorySpanExporter() + self.provider.add_span_processor(SimpleSpanProcessor(native_buffer)) + self.emit('memory.retrieve', {'memory.query.text': 'question'}) + self.assertTrue(handle.force_flush()) + self.assertEqual(len(self.sink.get_finished_spans()), 1) + self.assertEqual(native_buffer.get_finished_spans()[-1].attributes['memory.query.text'], 'question') + handle.shutdown() + handle.shutdown() + self.emit('later', {}) + self.assertTrue(self.provider.force_flush()) + self.assertEqual(native_buffer.get_finished_spans()[-1].name, 'later') + second = InMemorySpanExporter() + next_handle = configure(project_name='sample', exporter=second) + self.emit('reconfigured', {}) + self.assertTrue(self.provider.force_flush()) + self.assertEqual([span.name for span in second.get_finished_spans()], ['reconfigured']) + next_handle.shutdown() + self.assertTrue(self.provider.force_flush()) + + def test_replaceable_default_refused_without_replacement(self): + self.provider._default_processor = True + try: + before = self.provider._active_span_processor._span_processors + with self.assertRaisesRegex(RuntimeError, 'replaceable default'): + configure(project_name='sample', exporter=self.sink) + self.assertEqual(before, self.provider._active_span_processor._span_processors) + finally: + del self.provider._default_processor + + def test_default_exporter_constructs_without_sending(self): + handle = configure(project_name='sample', endpoint='http://127.0.0.1:1/v1/traces', headers={'x-api-key': 'synthetic'}) + self.assertTrue(handle.force_flush()) + handle.shutdown() + + def test_version(self): self.assertEqual(__version__, '0.1.0') diff --git a/python/frameworks/cognee/tests/test_generation_isolation.py b/python/frameworks/cognee/tests/test_generation_isolation.py new file mode 100644 index 000000000..c40576476 --- /dev/null +++ b/python/frameworks/cognee/tests/test_generation_isolation.py @@ -0,0 +1,201 @@ +"""Spans stay with the configuration active when they started; all inputs are synthetic.""" +import subprocess +import sys +import unittest + +ISOLATION = ''' +import threading +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +import traceai_cognee +from traceai_cognee import configure +LLM = 'cognee.observe.acreate_structured_output' +INPUT = 'langfuse.observation.input' +a, b = InMemorySpanExporter(), InMemorySpanExporter() +ha = configure(project_name='project-A', exporter=a, capture_content=True) +tracer = ha.provider.get_tracer('test') + +# Ends while A is active: exported by A. +with tracer.start_as_current_span(LLM, attributes={INPUT: 'kept-by-A'}): pass +# Starts under A, ends after A shuts down and B is configured: exported by neither. +late = tracer.start_span(LLM, attributes={INPUT: 'data-owned-by-A'}) +assert ha.force_flush() +ha.shutdown() +# Starts while nothing is configured: a later configuration does not adopt it. +orphan = tracer.start_span(LLM, attributes={INPUT: 'data-owned-by-nobody'}) +hb = configure(project_name='project-B', exporter=b, capture_content=True) +late.end(); orphan.end() +with tracer.start_as_current_span(LLM, attributes={INPUT: 'owned-by-B'}): pass +assert hb.force_flush() +seen_a = [s.attributes.get('input.value') for s in a.get_finished_spans()] +seen_b = [(s.resource.attributes['project_name'], s.attributes.get('input.value')) for s in b.get_finished_spans()] +assert seen_a == ['kept-by-A'], seen_a +assert seen_b == [('project-B', 'owned-by-B')], seen_b + +# Interleaved spans across concurrent threads under B are all exported once, by B. +def work(i): + with tracer.start_as_current_span('cognee.search.query', attributes={'gen_ai.usage.input_tokens': i}): pass +threads = [threading.Thread(target=work, args=(i,)) for i in range(64)] +[t.start() for t in threads]; [t.join(10) for t in threads] +assert hb.force_flush() +tokens = sorted(s.attributes['gen_ai.usage.input_tokens'] for s in b.get_finished_spans()[1:]) +assert tokens == list(range(64)), tokens +slot = traceai_cognee._slots[hb.provider] +assert not slot._open, len(slot._open) +hb.shutdown() +''' + +CAP = ''' +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +import traceai_cognee +from traceai_cognee import configure +traceai_cognee._MAX_OPEN_SPANS = 3 +sink = InMemorySpanExporter() +h = configure(project_name='cap', exporter=sink) +tracer = h.provider.get_tracer('test') +spans = [tracer.start_span('open' + str(i)) for i in range(5)] +slot = traceai_cognee._slots[h.provider] +assert len(slot._open) == 3, len(slot._open) +spans[0].end() # evicted while open: dropped, never re-attributed +spans[4].end() # still tracked: exported +assert h.force_flush() +assert [s.name for s in sink.get_finished_spans()] == ['open4'], sink.get_finished_spans() +h.shutdown() +# Spans still open at shutdown stay marked, unexported, until they end. +assert all(isinstance(v, traceai_cognee._Ambiguous) for v in slot._open.values()), slot._open +spans[2].end(); spans[3].end() +assert not slot._open +''' + +DUPLICATE_IDS = ''' +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.id_generator import IdGenerator +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from traceai_cognee import configure +LLM = 'cognee.observe.acreate_structured_output' +INPUT = 'langfuse.observation.input' +class Fixed(IdGenerator): + def generate_span_id(self): return 2 + def generate_trace_id(self): return 1 +p = TracerProvider(id_generator=Fixed(), shutdown_on_exit=False) +a, b = InMemorySpanExporter(), InMemorySpanExporter() +ha = configure(project_name='A', provider=p, exporter=a, capture_content=True) +tracer = p.get_tracer('test') +old = tracer.start_span(LLM, attributes={INPUT: 'data-from-A'}) +ha.shutdown() +hb = configure(project_name='B', provider=p, exporter=b, capture_content=True) +new = tracer.start_span(LLM, attributes={INPUT: 'data-from-B'}) +# The reused IDs must not let A's span take B's place. +old.end(); new.end() +assert hb.force_flush() +seen = [s.attributes.get('input.value') for s in b.get_finished_spans()] +assert seen == ['data-from-B'], seen +# Two open spans with the same identity cannot be told apart: neither is exported. +twins = [tracer.start_span('twin', start_time=10**18) for _ in range(2)] +twins[0].end(); twins[1].end() +assert hb.force_flush() +assert [s.name for s in b.get_finished_spans()] == [LLM], b.get_finished_spans() +hb.shutdown() +''' + +IDENTICAL_IDENTITY = ''' +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.id_generator import IdGenerator +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +import traceai_cognee +from traceai_cognee import configure +INPUT = 'langfuse.observation.input' +class Fixed(IdGenerator): + def generate_span_id(self): return 2 + def generate_trace_id(self): return 1 +p = TracerProvider(id_generator=Fixed(), shutdown_on_exit=False) +a, b = InMemorySpanExporter(), InMemorySpanExporter() +ha = configure(project_name='A', provider=p, exporter=a, capture_content=True) +tracer = p.get_tracer('test') +# Same IDs and start time across a reconfiguration: B must not adopt A's span. +old = tracer.start_span('old', start_time=10**18, attributes={INPUT: 'data-from-A'}) +ha.shutdown() +hb = configure(project_name='B', provider=p, exporter=b, capture_content=True) +new = tracer.start_span('new', start_time=10**18) +old.end(); new.end() +assert hb.force_flush() +assert not b.get_finished_spans(), [(s.name, s.attributes.get('input.value')) for s in b.get_finished_spans()] +# Overlapping duplicates stay unattributable until every one has ended. +first = tracer.start_span('first', start_time=2 * 10**18) +second = tracer.start_span('second', start_time=2 * 10**18) +first.end() +third = tracer.start_span('third', start_time=2 * 10**18) +second.end(); third.end() +fresh = tracer.start_span('fresh', start_time=2 * 10**18); fresh.end() +assert hb.force_flush() +assert [s.name for s in b.get_finished_spans()] == ['fresh'], [s.name for s in b.get_finished_spans()] +assert not traceai_cognee._slots[p]._open +hb.shutdown() +''' + +PARTIAL_ATTACH = ''' +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from traceai_cognee import configure +class AttachThenRaise(TracerProvider): + def add_span_processor(self, processor): + super().add_span_processor(processor) + raise RuntimeError('provider failed after attaching') +class RaiseBeforeAttach(TracerProvider): + failures = 2 + def add_span_processor(self, processor): + if RaiseBeforeAttach.failures: + RaiseBeforeAttach.failures -= 1 + raise RuntimeError('provider failed before attaching') + return super().add_span_processor(processor) +# Each case runs in its own process: the first configure sets the global provider. +cls, retries = {'attach_then_raise': (AttachThenRaise, 1), 'raise_before_attach': (RaiseBeforeAttach, 2)}[CASE] +provider = cls(shutdown_on_exit=False) +# A handle owns and closes its exporter, so each configuration gets its own. +for _ in range(retries): + try: + configure(project_name='p', provider=provider, exporter=InMemorySpanExporter()) + raise AssertionError('provider error was swallowed') + except RuntimeError as error: + assert 'provider failed' in str(error), error +handle = configure(project_name='p', provider=provider, exporter=InMemorySpanExporter()) +for _ in range(3): + handle.shutdown() + sink = InMemorySpanExporter() + handle = configure(project_name='p', provider=provider, exporter=sink) +assert len(provider._active_span_processor._span_processors) == 1, cls +with provider.get_tracer('t').start_as_current_span('after-retry'): pass +assert handle.force_flush() +assert [s.name for s in sink.get_finished_spans()] == ['after-retry'], cls +handle.shutdown() +''' + + +class GenerationIsolationTests(unittest.TestCase): + def run_fresh(self, code, case=None): + if case is not None: + code = 'CASE = %r\n' % case + code + result = subprocess.run([sys.executable, '-c', code], cwd=sys.prefix, + capture_output=True, text=True, timeout=30) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_spans_never_cross_configurations(self): + self.run_fresh(ISOLATION) + + def test_open_span_tracking_is_bounded(self): + self.run_fresh(CAP) + + def test_duplicate_span_ids_never_cross_configurations(self): + self.run_fresh(DUPLICATE_IDS) + + def test_identical_identity_is_never_adopted_or_reattributed(self): + self.run_fresh(IDENTICAL_IDENTITY) + + def test_attach_then_raise_retry_reuses_one_slot(self): + self.run_fresh(PARTIAL_ATTACH, 'attach_then_raise') + + def test_raise_before_attach_retry_reuses_one_slot(self): + self.run_fresh(PARTIAL_ATTACH, 'raise_before_attach') + + +if __name__ == '__main__': + unittest.main() diff --git a/python/frameworks/cognee/tests/test_interpreter_exit.py b/python/frameworks/cognee/tests/test_interpreter_exit.py new file mode 100644 index 000000000..873c72aee --- /dev/null +++ b/python/frameworks/cognee/tests/test_interpreter_exit.py @@ -0,0 +1,100 @@ +"""Check delivery and exporter closure, not just a successful process exit.""" +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + + +PROGRAM = r''' +import atexit +import json +from pathlib import Path +import sys +import time +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor, SpanExporter, SpanExportResult + +mode, filename = sys.argv[1:] +output = Path(filename) + +class Sink(SpanExporter): + def __init__(self, label): + self.label = label + + def record(self, operation, names=None): + with output.open('a') as stream: + stream.write(json.dumps([self.label, operation, names]) + '\n') + + def export(self, spans): + # Simulate buffered I/O that cannot finish before atexit returns unless + # shutdown actually waits. No real service or credentials are needed. + time.sleep(0.2 if self.label != 'foreign' else 0) + self.record('export', [span.name for span in spans]) + return SpanExportResult.SUCCESS + + def shutdown(self): + self.record('shutdown') + +if mode == 'baseline': + provider = TracerProvider() + provider.add_span_processor(BatchSpanProcessor(Sink('baseline'))) +else: + from traceai_cognee import configure + options = {} + if mode == 'existing-provider': + options['provider'] = TracerProvider() + handle = configure(project_name='synthetic', exporter=Sink('adapter'), **options) + provider = handle.provider + if mode in ('late-shutdown', 'closed-generation'): + # Registered later, this callback closes the generation after Python's + # normal thread-join phase but before the generation's own exit hook. + atexit.register(handle.shutdown) + +provider.add_span_processor(BatchSpanProcessor(Sink('foreign'))) +with provider.get_tracer('exit-regression').start_as_current_span('finished-before-exit'): + pass +if mode == 'closed-generation': + handle.shutdown() +# No force_flush or explicit provider.shutdown: use normal interpreter teardown. +''' + + +class InterpreterExitTests(unittest.TestCase): + def check_delivery(self, mode): + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / 'delivery.jsonl' + result = subprocess.run( + [sys.executable, '-I', '-c', PROGRAM, mode, str(output)], + cwd=directory, capture_output=True, text=True, timeout=20, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stderr, '') + events = [json.loads(line) for line in output.read_text().splitlines()] if output.exists() else [] + label = 'baseline' if mode == 'baseline' else 'adapter' + for exporter in (label, 'foreign'): + self.assertEqual( + [event[1:] for event in events if event[0] == exporter], + [['export', ['finished-before-exit']], ['shutdown', None]], + (mode, events), + ) + + def test_sdk_baseline_delivers_and_closes(self): + self.check_delivery('baseline') + + def test_default_provider_delivers_and_closes(self): + self.check_delivery('default-provider') + + def test_existing_provider_delivers_and_closes(self): + self.check_delivery('existing-provider') + + def test_shutdown_from_later_exit_callback_delivers_and_closes(self): + self.check_delivery('late-shutdown') + + def test_closed_generation_is_not_closed_twice_at_exit(self): + self.check_delivery('closed-generation') + + +if __name__ == '__main__': + unittest.main() diff --git a/python/frameworks/cognee/tests/test_lifecycle_regressions.py b/python/frameworks/cognee/tests/test_lifecycle_regressions.py new file mode 100644 index 000000000..5bac08357 --- /dev/null +++ b/python/frameworks/cognee/tests/test_lifecycle_regressions.py @@ -0,0 +1,207 @@ +"""Bounded lifecycle regressions; all exporters and inputs are synthetic.""" +import gc +import subprocess +import sys +import threading +import unittest +import weakref + +from opentelemetry import trace +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult +from traceai_cognee import configure + + +class Sink(SpanExporter): + def __init__(self): + self.names = [] + self.closed = 0 + self.closed_event = threading.Event() + def export(self, spans): + self.names.extend(s.name for s in spans) + return SpanExportResult.SUCCESS + def shutdown(self): + self.closed += 1 + self.closed_event.set() + + +class LifecycleTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + existing = trace.get_tracer_provider() + if isinstance(existing, TracerProvider): + cls.provider = existing + else: + cls.provider = TracerProvider(shutdown_on_exit=False) + trace.set_tracer_provider(cls.provider) + + def test_reentry_and_close_during_flush(self): + started, release, worker_done = (threading.Event() for _ in range(3)) + class Reentrant(Sink): + def export(s, spans): + if not started.is_set(): + started.set() + def work(): + with tracer.start_as_current_span('worker'): + pass + worker_done.set() + t = threading.Thread(target=work, daemon=True) + t.start() + if not worker_done.wait(2): + raise AssertionError('traced exporter worker blocked') + if not release.wait(2): + raise AssertionError('test did not release exporter') + return super().export(spans) + sink = Reentrant() + handle = configure(project_name='reentry', exporter=sink) + tracer = self.provider.get_tracer('test') + with tracer.start_as_current_span('original'): pass + flush = threading.Thread(target=handle.force_flush, daemon=True) + flush.start() + try: + self.assertTrue(started.wait(3)) + self.assertTrue(worker_done.wait(3)) + close = threading.Thread(target=handle.shutdown, daemon=True) + close.start(); close.join(1) + self.assertFalse(close.is_alive(), 'shutdown waited for exporter') + finally: + release.set(); flush.join(5) + self.assertFalse(flush.is_alive()) + self.assertTrue(sink.closed_event.wait(5)) + self.assertEqual(sink.closed, 1) + handle.shutdown() + self.assertEqual(sink.closed, 1) + + def test_reuse_old_handles_collection_and_provider_shutdown(self): + baseline = len(self.provider._active_span_processor._span_processors) + refs = [] + old = None + for index in range(25): + sink = Sink(); refs.append(weakref.ref(sink)) + h = configure(project_name='p' + str(index), exporter=sink) + if old: + old.shutdown() + self.assertTrue(old.force_flush()) + with self.provider.get_tracer('test').start_as_current_span('span' + str(index)): pass + self.assertTrue(h.force_flush()) + self.assertEqual(sink.names, ['span' + str(index)]) + h.shutdown(); self.assertTrue(sink.closed_event.wait(5)); self.assertEqual(sink.closed, 1) + old = h + del sink, h + gc.collect() + self.assertTrue(all(ref() is None for ref in refs)) + self.assertLessEqual(len(self.provider._active_span_processor._span_processors), baseline + 1) + + def test_raw_refusal_and_validation_leave_global_usable(self): + code = ''' +from opentelemetry import trace +from opentelemetry.trace import ProxyTracerProvider +from opentelemetry.sdk.trace import TracerProvider as Ordinary +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from fi_instrumentation.otel import TracerProvider +from traceai_cognee import configure +before = trace.get_tracer_provider() +assert isinstance(before, ProxyTracerProvider) +p = TracerProvider(verbose=False) +old = p._active_span_processor._span_processors +sink = InMemorySpanExporter() +try: + configure(project_name='x', provider=p, exporter=sink) + raise AssertionError('raw provider accepted') +except RuntimeError: + pass +assert trace.get_tracer_provider() is before +assert p._active_span_processor._span_processors == old and p._default_processor +try: + configure(project_name='x', headers=object()) + raise AssertionError('invalid headers accepted') +except (TypeError, AttributeError): + pass +assert trace.get_tracer_provider() is before +try: + configure(project_name='x', exporter=object()) + raise AssertionError('invalid exporter accepted') +except TypeError: + pass +assert trace.get_tracer_provider() is before +h = configure(project_name='valid', provider=Ordinary(shutdown_on_exit=False), exporter=sink) +assert trace.get_tracer_provider() is h.provider +h.shutdown(); p.shutdown() +''' + result = subprocess.run([sys.executable, '-c', code], cwd=sys.prefix, capture_output=True, text=True, timeout=20) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_background_exporter_can_join_shutdown_worker(self): + code = ''' +import threading +from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult +from traceai_cognee import configure +ready=threading.Event(); entered=threading.Event(); finished=threading.Event(); blocked=[] +class Sink(SpanExporter): + def export(self, spans): + entered.set() + assert ready.wait(5) + t=threading.Thread(target=h.shutdown, daemon=True); t.start(); t.join(1) + blocked.append(t.is_alive()) + finished.set() + return SpanExportResult.SUCCESS + def shutdown(self): pass +h=configure(project_name='x', exporter=Sink()) +tracer=h.provider.get_tracer('test') +for i in range(600): + with tracer.start_as_current_span('span'): pass +assert entered.wait(5) +ready.set() +assert finished.wait(5) +assert not any(blocked), 'background exporter waited on shutdown worker, which waited on exporter' +''' + result = subprocess.run([sys.executable, '-c', code], cwd=sys.prefix, capture_output=True, text=True, timeout=20) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_overlapping_configure_fails_fast_and_first_succeeds(self): + code = ''' +import threading +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from traceai_cognee import configure +entered=threading.Event(); release=threading.Event(); result=[] +class Provider(TracerProvider): + def add_span_processor(self, processor): + entered.set() + assert release.wait(3) + return super().add_span_processor(processor) +p=Provider(shutdown_on_exit=False); sink=InMemorySpanExporter() +def first(): + result.append(configure(project_name='x', provider=p, exporter=sink)) +t=threading.Thread(target=first, daemon=True); t.start(); assert entered.wait(3) +try: + configure(project_name='x', provider=p, exporter=sink) + raise AssertionError('overlapping configure accepted') +except RuntimeError as e: + assert 'configure is in progress' in str(e) +finally: + release.set() +t.join(5); assert not t.is_alive() and len(result)==1 +assert configure(project_name='x', provider=p, exporter=sink) is result[0] +result[0].shutdown(); p.shutdown() +''' + result = subprocess.run([sys.executable, '-c', code], cwd=sys.prefix, capture_output=True, text=True, timeout=20) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_provider_shutdown_closes_once_and_rejects_reuse(self): + code = ''' +from traceai_cognee import configure +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +s=InMemorySpanExporter(); h=configure(project_name='x', exporter=s) +h.provider.shutdown(); h.shutdown() +try: + configure(project_name='again', exporter=InMemorySpanExporter()) + raise AssertionError('shutdown provider accepted') +except RuntimeError: + pass +''' + result = subprocess.run([sys.executable, '-c', code], cwd=sys.prefix, capture_output=True, text=True, timeout=20) + self.assertEqual(result.returncode, 0, result.stderr) + +if __name__ == '__main__': + unittest.main() diff --git a/python/frameworks/cognee/tests/test_security_regressions.py b/python/frameworks/cognee/tests/test_security_regressions.py new file mode 100644 index 000000000..cdaffc2c9 --- /dev/null +++ b/python/frameworks/cognee/tests/test_security_regressions.py @@ -0,0 +1,94 @@ +"""Synthetic regressions for the native export projection.""" +import json +import math +import unittest + +from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +from traceai_cognee._exporter import project +from traceai_cognee._structured import redact_structured + + +class TestSecurityRegressions(unittest.TestCase): + def spans(self, attributes): + provider = TracerProvider(shutdown_on_exit=False) + sink = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(sink)) + try: + for attrs in attributes: + with provider.get_tracer('synthetic').start_as_current_span( + 'cognee.observe.acreate_structured_output', attributes=attrs + ): + pass + return sink.get_finished_spans() + finally: + provider.shutdown() + + def test_aws_credential_aliases_and_legitimate_fields(self): + credential = 'synthetic-aws-credential-should-be-redacted' + source = { + 'aws_secret_access_key': credential, + 'aws_access_key_id': credential, + 'aws_session_token': credential, + 'awsSecretAccessKey': credential, + 'credentials': {'accessKeyId': credential, 'secretAccessKey': credential}, + 'client.aws_secret_access_key.value': credential, + 'client[awsSessionToken][value]': credential, + 'nested': json.dumps({'aws_access_key_id': credential, 'question': 'hello'}), + 'input_token': 7, + 'token_count': 11, + 'request_id': 'request-1', + } + before = json.dumps(source) + sanitized = redact_structured(source) + self.assertEqual(json.dumps(source), before) + self.assertNotIn(credential, json.dumps(sanitized)) + self.assertEqual(sanitized['input_token'], 7) + self.assertEqual(sanitized['token_count'], 11) + self.assertEqual(sanitized['request_id'], 'request-1') + self.assertEqual(json.loads(sanitized['nested'])['question'], 'hello') + (span,) = self.spans([{'langfuse.observation.input': before}]) + projected = project(span, 'synthetic', True) + self.assertNotIn(credential.encode(), encode_spans([projected]).SerializeToString()) + self.assertEqual(json.loads(projected.attributes['input.value'])['request_id'], 'request-1') + + def test_integer_boundaries_encode_without_raw_value_logs(self): + for value in (-(2**63), 2**63 - 1, -(2**63) - 1, 2**63, 10**100): + with self.subTest(value_size=value.bit_length()): + spans = self.spans([ + {'gen_ai.usage.input_tokens': value}, + {'gen_ai.usage.output_tokens': 9}, + ]) + projected = [project(span, 'synthetic', False) for span in spans] + with self.assertNoLogs('opentelemetry.exporter.otlp.proto.common', level='ERROR'): + encoded = encode_spans(projected) + encoded.SerializeToString() + self.assertEqual(sum(len(s.spans) for r in encoded.resource_spans for s in r.scope_spans), 2) + if -(2**63) <= value < 2**63: + self.assertEqual(projected[0].attributes['gen_ai.usage.input_tokens'], value) + else: + self.assertNotIn('gen_ai.usage.input_tokens', projected[0].attributes) + self.assertEqual(projected[1].attributes['gen_ai.usage.output_tokens'], 9) + + def test_opt_in_numeric_content_is_wire_safe(self): + for value in (2**63, -(2**63) - 1, math.inf, -math.inf, math.nan): + with self.subTest(type=type(value).__name__): + (span,) = self.spans([{'langfuse.observation.output': value}]) + projected = project(span, 'synthetic', True) + self.assertNotIn('langfuse.observation.output', projected.attributes) + self.assertNotIn('output.value', projected.attributes) + with self.assertNoLogs('opentelemetry.exporter.otlp.proto.common', level='ERROR'): + encode_spans([projected]).SerializeToString() + + def test_finite_metadata_remains_available(self): + (span,) = self.spans([{'gen_ai.usage.input_tokens': 13, 'gen_ai.usage.output_tokens': 2.0}]) + projected = project(span, 'synthetic', False) + self.assertEqual(projected.attributes['gen_ai.usage.input_tokens'], 13) + self.assertEqual(projected.attributes['gen_ai.usage.output_tokens'], 2.0) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/frameworks/cognee/tests/test_structured.py b/python/frameworks/cognee/tests/test_structured.py new file mode 100644 index 000000000..305faed27 --- /dev/null +++ b/python/frameworks/cognee/tests/test_structured.py @@ -0,0 +1,70 @@ +"""Synthetic unit tests for the structural-redaction helper.""" +import copy +import importlib.util +import json +from pathlib import Path +import unittest + +path = Path(__file__).resolve().parents[1] / "traceai_cognee" / "_structured.py" +spec = importlib.util.spec_from_file_location("cognee_structured", path) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) + + +class TestStructuralRedaction(unittest.TestCase): + def test_nested_fields_and_counts(self): + source = { + "client": {"config": {"apiKey": "fake-secret-123", "model": "demo"}}, + "headers": {"Authorization": "fake-secret-123", "X-Api-Key": "fake-secret-123"}, + "client.config.refreshToken": "fake-secret-123", + "input_tokens": 42, + "token_count": 10, + "nested": json.dumps({"access_token": "fake-secret-123", "value": "kept"}), + } + before = copy.deepcopy(source) + result = module.redact_structured(source) + self.assertEqual(source, before) + self.assertNotIn("fake-secret-123", json.dumps(result)) + self.assertEqual(result["input_tokens"], 42) + self.assertEqual(result["token_count"], 10) + self.assertEqual(result["client"]["config"]["model"], "demo") + self.assertEqual(json.loads(result["nested"])["value"], "kept") + + def test_credential_paths_and_usage_metrics(self): + source = { + "http.request.header.authorization.value": "fake-secret-123", + "client[config][apiKey][value]": "fake-secret-123", + "client_config_api_key": "fake-secret-123", + "input_token": 5, + "input_tokens": 10, + "token_count": 15, + } + result = module.redact_structured(source) + for key in ("http.request.header.authorization.value", "client[config][apiKey][value]", "client_config_api_key"): + self.assertEqual(result[key], module.REDACTED) + for key in ("input_token", "input_tokens", "token_count"): + self.assertEqual(result[key], source[key]) + + def test_malformed_and_large_payloads_fail_closed(self): + self.assertEqual(module.redact_structured('{"api_key":"fake-secret-123"'), module.REDACTED) + self.assertEqual(module.redact_structured("x" * (module._MAX_TEXT + 1)), module.REDACTED) + self.assertEqual(module.redact_structured({"x" * 513: "fake-secret-123"}), module.REDACTED) + self.assertEqual(module.redact_structured({str(i): i for i in range(11000)}), module.REDACTED) + + def test_cycles_and_opaque_values_do_not_inspect_objects(self): + class Dangerous: + def __str__(self): + raise AssertionError("must not call str") + def __repr__(self): + raise AssertionError("must not call repr") + obj = [] + obj.append(obj) + self.assertEqual(module.redact_structured(obj), module.REDACTED) + self.assertEqual(module.redact_structured({"obj": Dangerous()}), module.REDACTED) + + def test_free_text_is_not_a_secret_detector(self): + self.assertEqual(module.redact_structured("fake-secret-123"), "fake-secret-123") + + +if __name__ == "__main__": + unittest.main() diff --git a/python/frameworks/cognee/traceai_cognee/__init__.py b/python/frameworks/cognee/traceai_cognee/__init__.py new file mode 100644 index 000000000..cbe2a3f5d --- /dev/null +++ b/python/frameworks/cognee/traceai_cognee/__init__.py @@ -0,0 +1,304 @@ +"""Export Cognee's native OpenTelemetry spans to Future AGI.""" +import atexit +from collections import OrderedDict +from collections.abc import Mapping +from threading import Event, RLock, Thread +from weakref import WeakKeyDictionary +from opentelemetry import trace +from opentelemetry.sdk.trace import TracerProvider, SpanProcessor +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.trace import ProxyTracerProvider +from ._exporter import CogneeExporter + +__version__ = '0.1.0' # x-release-please-version +_lock = RLock() +_slots = WeakKeyDictionary() +_configuring = False +# Spans that start but never end must not grow memory without bound. An evicted +# entry cannot protect against faulty reuse of its IDs and identical start time. +_MAX_OPEN_SPANS = 65536 + + +class _Ambiguous: + """Open spans sharing one identity, none of which can be attributed safely. + + Until every such span ends or the cap evicts the marker, a later span + reusing this identity cannot be matched with an earlier one's end. + """ + __slots__ = ('open',) + + def __init__(self, open_spans): + self.open = open_spans + + +def _span_key(span): + context = getattr(span, 'context', None) + if context is None: + return None + # The start time keeps a span from an earlier configuration apart from a + # later one that reuses its IDs (for example a faulty ID generator). + return context.trace_id, context.span_id, getattr(span, 'start_time', None) + + +def _attached(provider, slot): + # Read-only check of the SDK's processor list, used only after a provider + # raised from add_span_processor, to tell whether it attached the slot anyway. + active = getattr(provider, '_active_span_processor', None) + return any(processor is slot for processor in getattr(active, '_span_processors', ())) + + +class _Generation: + """Lease callbacks without holding a lock across downstream code. + + Closing detaches immediately; the last outstanding callback closes the + delegate. In particular, close never waits for a callback that may itself + be waiting for a traced worker. + """ + def __init__(self, delegate): + self._lock = RLock() + self._delegate = delegate + self._inflight = 0 + self._closed = False + self._exiting = False + self._shutdown_complete = Event() + # Registered after the SDK provider's hook: drain this generation before + # provider teardown, when Python's normal thread-join phase is already over. + atexit.register(self._shutdown_at_exit) + + def call(self, method, *args): + with self._lock: + if self._closed: + return True if method == 'force_flush' else None + delegate = self._delegate + self._inflight += 1 + try: + return getattr(delegate, method)(*args) + finally: + with self._lock: + self._inflight -= 1 + closing = self._closed and self._inflight == 0 + if closing: + self._delegate = None + if closing: + self._shutdown_delegate(delegate) + + def _shutdown_delegate(self, delegate): + with self._lock: + if not self._exiting: + # BatchSpanProcessor can be exporting on its own worker even when + # no callback is leased. Normal shutdown must not join that worker: + # its exporter may be waiting for the caller of Handle.shutdown(). + Thread(target=self._finish_shutdown, args=(delegate,), + name='traceai-cognee-close', daemon=False).start() + return + self._finish_shutdown(delegate) + + def _finish_shutdown(self, delegate): + try: + delegate.shutdown() + finally: + atexit.unregister(self._shutdown_at_exit) + self._shutdown_complete.set() + + def _shutdown_at_exit(self): + with self._lock: + self._exiting = True + self.close() + # An earlier exit callback may already have initiated asynchronous close. + # Waiting here also covers an outstanding leased callback finishing it. + self._shutdown_complete.wait() + + def close(self): + with self._lock: + if self._closed: + return + self._closed = True + delegate = self._delegate if not self._inflight else None + if delegate is not None: + self._delegate = None + if delegate is not None: + self._shutdown_delegate(delegate) + + +class _OwnedProcessor(SpanProcessor): + """One reusable provider-owned slot, independent of handle generations.""" + def __init__(self): + self._lock = RLock() + self._handle = None + self._shutdown = False + # Each open span stays with the generation active when it started, so a + # span that ends after reconfiguration never reaches the new project. + self._open = OrderedDict() + + def current(self): + with self._lock: + return self._handle + + def install(self, handle): + with self._lock: + if self._shutdown: + raise RuntimeError('provider processor has shut down') + if self._handle is not None: + raise RuntimeError('provider already configured') + self._handle = handle + + def detach(self, handle): + generation = handle._generation + with self._lock: + if self._handle is handle: + self._handle = None + # The detached generation's open spans are never exported. Keep their + # identities marked until they end, so a later configuration cannot + # adopt one through a reused identity. + for key in [key for key, owner in self._open.items() if owner is generation]: + self._open[key] = _Ambiguous(1) + generation.close() + + def on_start(self, span, parent_context=None): + key = _span_key(span) + with self._lock: + handle = self._handle + if handle is None or key is None: + return + entry = self._open.get(key) + if isinstance(entry, _Ambiguous): + entry.open += 1 + return + if entry is not None: + self._open[key] = _Ambiguous(2) + return + generation = handle._generation + if len(self._open) >= _MAX_OPEN_SPANS: + self._open.popitem(last=False) + self._open[key] = generation + generation.call('on_start', span, parent_context) + + def on_end(self, span): + key = _span_key(span) + with self._lock: + entry = self._open.get(key) if key is not None else None + if isinstance(entry, _Ambiguous): + entry.open -= 1 + if entry.open <= 0: + del self._open[key] + return + if entry is not None: + del self._open[key] + if entry is not None: + entry.call('on_end', span) + + def force_flush(self, timeout_millis=30000): + handle = self.current() + return True if handle is None else handle.force_flush(timeout_millis) + + def shutdown(self): + with self._lock: + self._shutdown = True + handle, self._handle = self._handle, None + self._open = OrderedDict() + if handle is not None: + handle._generation.close() + + +class Handle: + def __init__(self, provider, processor, generation, signature): + self.provider, self._processor = provider, processor + self._generation, self._signature = generation, signature + + def force_flush(self, timeout_millis=30000): + return self._generation.call('force_flush', timeout_millis) + + def shutdown(self): + self._processor.detach(self) + + +def configure(*, project_name, capture_content=False, endpoint=None, headers=None, + exporter=None, provider=None): + """Attach one exporter; call before Cognee first creates native spans. + + An injected exporter is transferred to this handle. Shutdown affects only this + adapter's processor, not the provider or its other processors. Shutdown may + defer delegate closure until an already-running callback returns. + """ + if not isinstance(project_name, str) or not project_name: + raise ValueError('project_name must be a nonempty string') + if not isinstance(capture_content, bool): + raise TypeError('capture_content must be a bool') + if provider is not None and not isinstance(provider, TracerProvider): + raise TypeError('Cognee adapter requires an OpenTelemetry SDK TracerProvider') + if exporter is not None and (endpoint is not None or headers is not None): + raise ValueError('use exporter or endpoint/headers, not both') + if exporter is not None and not all(callable(getattr(exporter, name, None)) + for name in ('export', 'force_flush', 'shutdown')): + raise TypeError('exporter must implement export, force_flush and shutdown') + if endpoint is not None and (not isinstance(endpoint, str) or not endpoint): + raise ValueError('endpoint must be a nonempty string') + if headers is not None and (not isinstance(headers, Mapping) or + not all(isinstance(k, str) and isinstance(v, str) + for k, v in headers.items())): + raise TypeError('headers must map strings to strings') + signature = (project_name, capture_content, endpoint, + tuple(sorted((headers or {}).items())), id(exporter) if exporter is not None else None) + global _configuring + with _lock: + if _configuring: + raise RuntimeError('another Cognee configure is in progress') + _configuring = True + generation = None + try: + global_provider = trace.get_tracer_provider() + if provider is None: + provider = TracerProvider() if isinstance(global_provider, ProxyTracerProvider) else global_provider + elif not isinstance(global_provider, ProxyTracerProvider) and global_provider is not provider: + raise ValueError('provider must be the global provider used by Cognee') + if not isinstance(provider, TracerProvider): + raise TypeError('Cognee adapter requires an OpenTelemetry SDK TracerProvider') + if getattr(provider, '_default_processor', False): + raise RuntimeError('provider has a replaceable default processor; use an ordinary OTel provider') + with _lock: + slot = _slots.get(provider) + existing = slot.current() if slot else None + if existing is not None: + if existing._signature != signature: + raise ValueError('provider already configured with different options') + return existing + if exporter is None: + from fi_instrumentation.otel import HTTPSpanExporter + exporter = HTTPSpanExporter(endpoint=endpoint, headers=headers) + generation = _Generation(BatchSpanProcessor(CogneeExporter(exporter, project_name, capture_content))) + new_slot = slot is None + if new_slot: + slot = _OwnedProcessor() + handle = Handle(provider, slot, generation, signature) + slot.install(handle) + if new_slot: + with _lock: + _slots[provider] = slot + try: + provider.add_span_processor(slot) + except BaseException: + # A provider may attach the slot and then raise. Keep an attached + # slot registered so a retry reuses it instead of adding another. + if not _attached(provider, slot): + with _lock: + if _slots.get(provider) is slot: + del _slots[provider] + raise + if isinstance(global_provider, ProxyTracerProvider): + trace.set_tracer_provider(provider) + if trace.get_tracer_provider() is not provider: + raise RuntimeError('could not install global tracer provider') + generation = None + return handle + except BaseException: + if generation is not None: + if 'handle' in locals(): + slot.detach(handle) + else: + generation.close() + raise + finally: + with _lock: + _configuring = False + +__all__ = ['configure', 'Handle', '__version__'] diff --git a/python/frameworks/cognee/traceai_cognee/_exporter.py b/python/frameworks/cognee/traceai_cognee/_exporter.py new file mode 100644 index 000000000..042dabd88 --- /dev/null +++ b/python/frameworks/cognee/traceai_cognee/_exporter.py @@ -0,0 +1,98 @@ +"""Copy and project native spans at the exporter boundary.""" +import json +import math +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import ReadableSpan, Event +from opentelemetry.sdk.trace.export import SpanExporter +from opentelemetry.trace import Status, Link +from ._structured import redact_structured + +# Explicit keys: unknown native attributes may contain user content or credentials. +_SAFE = frozenset({'gen_ai.operation.name', 'gen_ai.system', 'gen_ai.request.model', + 'gen_ai.response.model', 'gen_ai.usage.input_tokens', + 'gen_ai.usage.output_tokens', 'gen_ai.usage.total_tokens', + 'llm.token_count.prompt', 'llm.token_count.completion', + 'llm.token_count.total', 'cognee.search.type'}) +_CONTENT = ('langfuse.observation.input', 'langfuse.observation.output', + 'memory.query.text', 'cognee.search.query', 'cognee.db.query') +_KIND_KEYS = ('fi.span.kind', 'gen_ai.span.kind', 'llm.request.type', 'openinference.span.kind') + +def _wire_number(value): + if type(value) is bool: + return True + if type(value) is int: + return -(2**63) <= value < 2**63 + return type(value) is float and math.isfinite(value) + + +def _safe_value(value): + return _wire_number(value) or (isinstance(value, str) and len(value) <= 256 and all(c.isalnum() or c in '._-/' for c in value)) + +def _kind(name): + if name == 'memory.retrieve' or name.startswith('cognee.search'): + return 'RETRIEVER' + if name == 'cognee.observe.acreate_structured_output': + return 'LLM' + return None + +def _content(value): + if not isinstance(value, str) and not _wire_number(value): + return None + return redact_structured(value) + +def project(span, project_name, capture_content): + attrs = dict(span.attributes or {}) + out = {k: v for k, v in attrs.items() if k in _SAFE and _safe_value(v)} + for key in _KIND_KEYS: + if key in attrs and _safe_value(attrs[key]): + out[key] = attrs[key] + if not any(k in out for k in _KIND_KEYS) and _kind(span.name): + out['fi.span.kind'] = _kind(span.name) + if capture_content: + for key in _CONTENT: + value = _content(attrs.get(key)) + if value is not None: + out[key] = value + for dest, sources in (('input', ('langfuse.observation.input', 'memory.query.text', 'cognee.search.query')), + ('output', ('langfuse.observation.output',))): + for key in sources: + if key in out: + value = out[key] + out[dest + '.value'] = value + try: + mime = 'application/json' if isinstance(json.loads(value), (dict, list)) else 'text/plain' + except (TypeError, ValueError): + mime = 'text/plain' + out[dest + '.mime_type'] = mime + break + # Even content opt-in never exports error text, stacktraces or arbitrary events. + events = [Event('exception', {}, e.timestamp) for e in span.events if e.name == 'exception'] + links = [Link(link.context, {}) for link in span.links] + resource = Resource({'project_name': project_name, 'project_type': 'observe'}) + return ReadableSpan(name=span.name, context=span.context, parent=span.parent, + resource=resource, attributes=out, events=events, links=links, + kind=span.kind, status=Status(span.status.status_code), + start_time=span.start_time, end_time=span.end_time, + instrumentation_scope=None) + +class CogneeExporter(SpanExporter): + def __init__(self, exporter, project_name, capture_content=False): + self.exporter = exporter + self.project_name = project_name + self.capture_content = capture_content + + def export(self, spans): + # Fail closed: a malformed span is dropped rather than exporting native data. + projected = [] + for span in spans: + try: + projected.append(project(span, self.project_name, self.capture_content)) + except Exception: + continue + return self.exporter.export(projected) + + def force_flush(self, timeout_millis=30000): + return self.exporter.force_flush(timeout_millis) + + def shutdown(self): + self.exporter.shutdown() diff --git a/python/frameworks/cognee/traceai_cognee/_structured.py b/python/frameworks/cognee/traceai_cognee/_structured.py new file mode 100644 index 000000000..da4e356a9 --- /dev/null +++ b/python/frameworks/cognee/traceai_cognee/_structured.py @@ -0,0 +1,103 @@ +"""Bounded structural redaction for native span attributes. + +This helper is not an exporter or a complete privacy boundary. It +recognises credential *field names* in structured data, not secrets in free +text or arbitrary encodings. Inputs are never mutated. +""" +from __future__ import annotations + +import json +import re +from typing import Any + +REDACTED = "[REDACTED]" +_MAX_DEPTH = 24 +_MAX_NODES = 10000 +_MAX_TEXT = 1_000_000 +_MAX_KEY = 512 +_CAMEL = re.compile(r"(?<=[a-z0-9])(?=[A-Z])") +_PARTS = re.compile(r"[^a-z0-9]+") +_CREDENTIAL_NAMES = frozenset({ + "apikey", "xapikey", "authorization", "proxyauthorization", + "accesstoken", "refreshtoken", "authtoken", "bearertoken", "idtoken", + "token", "clientsecret", "secretkey", "clientkey", "password", "passwd", + "credential", "credentials", "privatekey", + "accesskeyid", "secretaccesskey", "sessiontoken", "securitytoken", +}) + + +def _credential_field(key: str) -> bool: + # Treat dots and brackets as path boundaries, not as separators inside a + # single name. A credential-bearing path can continue after the credential + # segment (e.g. headers.authorization.value). Matching complete segments + # avoids hiding useful metrics such as input_tokens or token_count. + for segment in re.split(r"[.\[\]]+", key): + parts = [p for p in _PARTS.split(_CAMEL.sub(" ", segment).lower()) if p] + if "".join(parts) in _CREDENTIAL_NAMES: + return True + # A flattened underscore form such as client_config_api_key has no + # path delimiter. Do not match bare "token" as a suffix: e.g. + # input_token is a legitimate usage metric. + if any("".join(parts[i:]) in (_CREDENTIAL_NAMES - {"token"}) + for i in range(1, len(parts))): + return True + return False + + +class _Limit(Exception): + pass + + +def redact_structured(value: Any) -> Any: + """Return an independent sanitized JSON-like value or REDACTED on limits. + + JSON-serialized objects and arrays nested inside other JSON values are also + inspected. Ordinary strings remain unchanged; callers must separately apply + the relevant content and error-detail policy before exporting them. + """ + remaining = [_MAX_NODES] + active: set[int] = set() + + def visit(obj: Any, depth: int) -> Any: + remaining[0] -= 1 + if depth > _MAX_DEPTH or remaining[0] < 0: + raise _Limit + if isinstance(obj, str): + if len(obj) > _MAX_TEXT: + raise _Limit + stripped = obj.strip() + if stripped.startswith(("{", "[")): + try: + parsed = json.loads(obj) + except (ValueError, RecursionError): + # An invalid or truncated structured-looking value must + # not fall back to publishing its potentially secret text. + return REDACTED + return json.dumps(visit(parsed, depth + 1), ensure_ascii=False, separators=(",", ":")) + return obj + if obj is None or isinstance(obj, (bool, int, float)): + return obj + if isinstance(obj, (dict, list, tuple)): + ident = id(obj) + if ident in active: + raise _Limit + active.add(ident) + try: + if isinstance(obj, dict): + result = {} + for key, item in obj.items(): + remaining[0] -= 1 + if remaining[0] < 0 or not isinstance(key, str) or len(key) > _MAX_KEY: + raise _Limit + result[key] = REDACTED if _credential_field(key) else visit(item, depth + 1) + return result + return [visit(item, depth + 1) for item in obj] + finally: + active.remove(ident) + # Do not call repr/str on runtime objects, which may contain credentials. + raise _Limit + + try: + return visit(value, 0) + except (ValueError, TypeError, RecursionError, _Limit): + return REDACTED diff --git a/python/frameworks/crewai/README.md b/python/frameworks/crewai/README.md index 7667b43be..d67d8e8ed 100644 --- a/python/frameworks/crewai/README.md +++ b/python/frameworks/crewai/README.md @@ -26,8 +26,138 @@ import os os.environ["FI_API_KEY"] = FI_API_KEY os.environ["FI_SECRET_KEY"] = FI_SECRET_KEY + +# Set these before importing CrewAI to opt out of its separate native telemetry. +os.environ["CREWAI_DISABLE_TELEMETRY"] = "true" +os.environ["CREWAI_DISABLE_TRACKING"] = "true" ``` +### Privacy and credentials + +#### Structural guarantee: framework configuration is not content + +The instrumentor never exports an LLM, Agent, Crew, Task, Tool or client object +itself, and never calls its `repr`, `str`, `model_dump`, `dict`, `vars` or generic +JSON serializer to capture configuration. LLM API keys, base URLs (including +URLs with userinfo), configuration headers and credentials are not selected. +This does not depend on finding a secret's spelling or value. + +The complete wrapper inventory is: + +| Wrapper | Explicit input projection | Explicit result projection | +| --- | --- | --- | +| `Task._execute_core` | Task description, expected output and application context; no agent or tool argument object | `TaskOutput.json_dict` when it is a plain dictionary, otherwise `TaskOutput.raw`; plain application results | +| `Crew.kickoff` | Application `inputs`; named agent/task business metadata below | `CrewOutput.json_dict` when it is a plain dictionary, otherwise `CrewOutput.raw`; prompt/completion/total usage counts | +| `ToolUsage._use` | `ToolCalling.tool_name`, `ToolCalling.arguments`; tool name only from the runtime tool | Plain application result; no tool configuration | + +Crew metadata selects only crew/task keys and UUIDs; agent role, goal, backstory, +key, UUID, verbosity, iteration/rate limits, delegation flag and tool names; +and task description, expected output, UUID, execution/human-input flags, +assigned agent role/key, context task descriptions and tool names. Task description +and expected output are also emitted as formatted aliases when `share_crew=True`. +`Agent.execute_task` and LLM calls have no additional wrappers in this package: +they execute inside the task span and their arguments/results are not serialized +independently. Separately installed LLM instrumentors have their own policies. + +Opaque runtime objects nested in application data are replaced with +`[omitted runtime object]`, without inspecting their attributes. The guarantee is +about framework configuration objects, not credentials deliberately copied into +application prose or a plain dictionary. Instrumentation preserves application +arguments, results and raised exceptions. + +#### Application content: a field-name policy, not a universal secret detector + +With content capture on, task/tool inputs and outputs are application content. +The following policy applies to plain dictionaries, lists and tuples (copied +without mutation), and complete JSON object/array strings. It also applies to +context and span/event attributes, including structured `log.body` attributes. +It does not filter independent application logging pipelines or console output. + +Names are case-insensitive; an initial `X-`/`X_` prefix and punctuation/separators +are normalized away. Dot/bracket field paths are inspected component by +component, including nested dictionary paths. A field and all its descendants +are replaced by `__REDACTED__` if the normalized component contains any of these +markers: + +- `apikey`, `accesskey`, `privatekey`, `clientkey`, `signingkey`, + `subscriptionkey`, `applicationkey`, `masterkey`; +- `token`, `secret`, `password`, `passwd`, `passphrase`, `credential`, `auth`, + `cookie`, `session`, `signature`, `bearer`, `jwt`, `otp`. + +`pin` matches a whole normalized component only. Adjacent components are also +combined for the eight `*key` markers above, so `client.api.key` and +`client[private][key]` match their single-field equivalents. + +Header interpretation requires an explicit field whose normalized name ends in +`headers` (for example `headers`, `request_headers`, `rawHeaders`): + +- Dictionaries with credential-named header keys; +- Two-item name/value pairs and collections of pairs; +- Alternating name/value lists such as Node's `rawHeaders`; +- Records with a `name`, `key` or `header` selector and a `value` field; +- Flattened dot/bracket records with those selectors, or `0`/`1` selectors, + including descendants of list-valued `value` fields in one attribute batch. + +A credential-named header selector preserves its name but redacts the entire +paired value. Outside a named headers field, pairs and `{name, value}` records +are ordinary business data, never inferred to be headers. For example, +`topics=['authentication', 'billing']`, business tables and product records keep +their values. + +The exact normalized metadata exceptions below remain visible (children are +still inspected). These cover standard token counters, generation limits, +tokenizer metadata, author metadata and non-secret session/service metadata: + +`acceptedpredictiontokens`, `apisecretid`, `audiotokens`, `author`, `authoremail`; +`authorid`, `authority`, `authorname`, `authors`, `authorship`; +`authorurl`, `cachecreationinputtokens`, `cachedcontenttokencount`, `cachedinputtokens`, `cachedtokencount`; +`cachedtokens`, `cachereadinputtokens`, `cachereadtokens`, `cachewriteinputtokens`, `cachewritetokens`; +`candidatestokencount`, `completiontokencount`, `completiontokens`, `completiontokensdetails`, `consultokenttl`; +`inputtokencount`, `inputtokens`, `inputtokensdetails`, `maxcompletiontokens`, `maxinputtokens`; +`maxnewtokens`, `maxoutputtokens`, `maxtokens`, `numtokens`, `outputtokencount`; +`outputtokens`, `outputtokensdetails`, `promptcachehittokens`, `promptcachemisstokens`, `prompttokencount`; +`prompttokens`, `prompttokensdetails`, `reasoningtokencount`, `reasoningtokens`, `rejectedpredictiontokens`; +`securitytokenid`, `servicetokencount`, `sessioncount`, `sessionduration`, `sessiondurationms`; +`subjecttokentype`, `thoughtstokencount`, `tokencount`, `tokenid`, `tokenizer`; +`tokenizername`, `tokenizerversion`, `tokensused`, `tokenusage`, `tooluseprompttokencount`; +`totaltokencount`, `totaltokens`, `vaulttokencount`. + +These are field-name rules, not value inspection. Unknown credential names +(for example an application-specific `pwd`, `sig` or vendor key header not +matching the markers), novel shapes, arbitrary free text and encodings other +than complete JSON objects/arrays are outside this policy. Do not put credentials +in content fields or metadata. Dictionaries beyond depth 20 are omitted, as are +structured strings larger than 1,000,000 characters or unparseable structured +strings; opaque objects are not dumped as a fallback. + +To disable content capture, set `FI_HIDE_INPUTS=true` and `FI_HIDE_OUTPUTS=true` +before instrumentation, or pass `TraceConfig(hide_inputs=True, hide_outputs=True)`. +Directional text/message hiding also covers CrewAI's content aliases. Wrapper +failures record ERROR status without copying exception messages or stacktraces. +For manually recorded error events, use `FI_HIDE_ERROR_DETAILS=true`. + +### CrewAI's separate native telemetry + +CrewAI can export through its own provider to `telemetry.crewai.com`, independently +of traceAI. **This instrumentor does not change environment variables, patch +CrewAI's telemetry classes, disable its provider or restore/change your settings +on `uninstrument()`.** Choose the native telemetry policy explicitly before +importing CrewAI. + +In CrewAI 0.152.0 and 1.15.33, `CREWAI_DISABLE_TELEMETRY=true` or +`CREWAI_DISABLE_TRACKING=true` disables native telemetry while leaving traceAI +available. `OTEL_SDK_DISABLED=true` is also honored, but disables the OTel SDK +and traceAI spans too; do not use it when you want traceAI tracing. +`CREWAI_TELEMETRY_OPT_OUT` and `DO_NOT_TRACK` are not native disable flags. + +CrewAI 0.41.1 has no native telemetry opt-out flag: its constructor creates an +exporter for `https://telemetry.crewai.com:4319/v1/traces` and ignores the flags +above. If you need native telemetry disabled, upgrade to a version that honors +`CREWAI_DISABLE_TELEMETRY` before running sensitive workloads. For legacy +workloads, enforce a network egress policy; traceAI's projections cannot protect +an independent provider's exports. `share_crew=False` is not a native telemetry +disable switch. + ## Quickstart ### Register Tracer Provider @@ -57,7 +187,6 @@ Set up your Crewai client with built-in observability. ```python from crewai import LLM, Agent, Crew, Process, Task -from crewai_tools import SerperDevTool def story_example(): llm = LLM( diff --git a/python/frameworks/crewai/poetry.lock b/python/frameworks/crewai/poetry.lock index 4d1eaa450..079b84875 100644 --- a/python/frameworks/crewai/poetry.lock +++ b/python/frameworks/crewai/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.1.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.2.1 and should not be changed by hand. [[package]] name = "aiohappyeyeballs" @@ -866,16 +866,19 @@ tests = ["asttokens (>=2.1.0)", "coverage", "coverage-enable-subprocess", "ipyth [[package]] name = "fi-instrumentation-otel" -version = "1.1.0" +version = "1.2.2" description = "OpenTelemetry instrumentation" optional = false -python-versions = "<3.14,>3.9" +python-versions = ">3.9,<3.14" groups = ["main"] -files = [ - {file = "fi_instrumentation_otel-1.1.0-py3-none-any.whl", hash = "sha256:4fa2bf663e0ce09447aca92ae798cd7b0dee35c5c92c2e033f1d89d158a9e62c"}, - {file = "fi_instrumentation_otel-1.1.0.tar.gz", hash = "sha256:7ec324c3c9242aa87a88cf23a21c11919b8736c6451c5fb50917f592a853b059"}, -] +[[package.files]] +file = "fi_instrumentation_otel-1.2.2-py3-none-any.whl" +hash = "sha256:25cf88f93e33aa928d4fae0f631afe8a32565639490862b715869e0633becbaf" + +[[package.files]] +file = "fi_instrumentation_otel-1.2.2.tar.gz" +hash = "sha256:b675df47d7a03c506fbd2e4b0420fe4f74d12a6353a88b219bfd97797b276d4c" [package.dependencies] jsonschema = ">=4.21.1,<5.0.0" opentelemetry-api = ">=1.29.0,<2.0.0" @@ -3999,6 +4002,27 @@ files = [ [package.dependencies] pyasn1 = ">=0.1.3" +[[package]] +name = "setuptools" +version = "80.10.2" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "setuptools-80.10.2-py3-none-any.whl", hash = "sha256:95b30ddfb717250edb492926c92b5221f7ef3fbcc2b07579bcd4a27da21d0173"}, + {file = "setuptools-80.10.2.tar.gz", hash = "sha256:8b0e9d10c784bf7d262c4e5ec5d4ec94127ce206e8738f29a437945fbc219b70"}, +] + +[package.extras] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\"", "ruff (>=0.8.0) ; sys_platform != \"cygwin\""] +core = ["importlib_metadata (>=6) ; python_version < \"3.10\"", "jaraco.functools (>=4)", "jaraco.text (>=3.7)", "more_itertools", "more_itertools (>=8.8)", "packaging (>=24.2)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1) ; python_version < \"3.11\"", "wheel (>=0.43.0)"] +cover = ["pytest-cov"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] +enabler = ["pytest-enabler (>=2.2)"] +test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21) ; python_version >= \"3.9\" and sys_platform != \"cygwin\"", "jaraco.envs (>=2.2)", "jaraco.path (>=3.7.2)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf ; sys_platform != \"cygwin\"", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] +type = ["importlib_metadata (>=7.0.2) ; python_version < \"3.10\"", "jaraco.develop (>=7.21) ; sys_platform != \"cygwin\"", "mypy (==1.14.*)", "pytest-mypy"] + [[package]] name = "shellingham" version = "1.5.4" @@ -4892,4 +4916,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = ">=3.10, <3.13" -content-hash = "7edf47743cfaec37f169b1e54bf5e9749fa128c99f157a0b90a0ea6928cdf1b8" +content-hash = "69b9e209bfe1b93924a5eaddeb64a1e73a5ccc1473083e22d1bf042372ed2ce3" diff --git a/python/frameworks/crewai/pyproject.toml b/python/frameworks/crewai/pyproject.toml index b3deb730a..aa0d19fac 100644 --- a/python/frameworks/crewai/pyproject.toml +++ b/python/frameworks/crewai/pyproject.toml @@ -27,8 +27,10 @@ classifiers = [ [tool.poetry.dependencies] python = ">=3.10, <3.13" -crewai = ">0.19.0" -fi-instrumentation-otel = ">=0.1.11" +crewai = ">=0.41.1" +fi-instrumentation-otel = ">=1.2.2" +# CrewAI 0.41.1 imports pkg_resources, removed in setuptools 81. +setuptools = ">=78.1.1,<81" [tool.poetry.urls] diff --git a/python/frameworks/crewai/tests/conftest.py b/python/frameworks/crewai/tests/conftest.py new file mode 100644 index 000000000..07ce107a8 --- /dev/null +++ b/python/frameworks/crewai/tests/conftest.py @@ -0,0 +1,114 @@ +"""Local synthetic tests only; never disable the OTel SDK's positive controls.""" + +import os + +import pytest + +# Must precede CrewAI imports, which can initialize a telemetry singleton. +os.environ["CREWAI_DISABLE_TELEMETRY"] = "true" +os.environ["CREWAI_DISABLE_TRACKING"] = "true" +os.environ["OTEL_SDK_DISABLED"] = "false" + + +@pytest.fixture(autouse=True) +def native_telemetry_off(request, monkeypatch): + from importlib.metadata import version + from crewai.telemetry import Telemetry + + if request.node.name.startswith("test_native_telemetry_"): + yield + return + if version("crewai") == "0.41.1": + # Test-only isolation: legacy CrewAI does not honor native disable flags. + # Production instrumentation must never patch the user's telemetry. + def disabled(instance): + instance.ready = False + instance.trace_set = False + + monkeypatch.setattr(Telemetry, "__init__", disabled) + else: + assert Telemetry._is_telemetry_disabled() + yield + + +@pytest.fixture +def service_credentials(): + return { + name: f"FAKE_SERVICE_CREDENTIAL_{index}_7f9b2a" + for index, name in enumerate( + ( + "X-Amz-Security-Token", + "X-Vault-Token", + "X-Consul-Token", + "X-Subject-Token", + "X-Api-Secret", + "X-Service-Token", + "security_token", + "X-Token", + "PRIVATE-TOKEN", + "private_token", + "X-Gitlab-Token", + "Circle-Token", + "Ocp-Apim-Subscription-Key", + "subscription_key", + "DD-APPLICATION-KEY", + "application_key", + "X-Parse-Master-Key", + "master_key", + "app_secret", + "X-Auth", + "Authorization", + "X-API-Key", + "vendor-api.key", + "client[private][key]", + "sessionid", + "passphrase", + "signature", + "bearer", + "jwt", + "otp", + "otp_code", + "totp", + "pin", + ) + ) + } + + +@pytest.fixture +def native_tool_usage(): + """Construct the real version-specific ToolUsage around a local function.""" + from importlib.metadata import version + from types import SimpleNamespace + + from crewai.tools.tool_usage import ToolUsage + + def create(function): + if version("crewai") == "0.41.1": + from crewai.agents.tools_handler import ToolsHandler + from langchain_core.tools import StructuredTool + + tool = StructuredTool.from_function(function) + usage = ToolUsage( + tools_handler=ToolsHandler(), + tools=[tool], + original_tools=[], + tools_description=function.__name__, + tools_names=function.__name__, + task=SimpleNamespace(used_tools=0), + function_calling_llm=None, + agent=SimpleNamespace(tools_results=[]), + action=None, + ) + usage._printer.print = lambda **kwargs: None + return usage + from crewai.tools.structured_tool import CrewStructuredTool + + return ToolUsage( + tools_handler=None, + tools=[CrewStructuredTool.from_function(function)], + task=None, + function_calling_llm=None, + ) + + return create diff --git a/python/frameworks/crewai/tests/test_batch_projection.py b/python/frameworks/crewai/tests/test_batch_projection.py new file mode 100644 index 000000000..d5d6c86f3 --- /dev/null +++ b/python/frameworks/crewai/tests/test_batch_projection.py @@ -0,0 +1,99 @@ +"""CrewAI-local batch adapter contracts; synthetic credentials only.""" + +import copy +import json +from types import MappingProxyType + +import pytest +from fi_instrumentation import TraceConfig +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, +) +from opentelemetry.sdk.trace.sampling import ALWAYS_ON, Sampler +from opentelemetry.trace import Link, SpanContext + +from traceai_crewai._privacy import REDACTED, _CrewAITraceConfig +from traceai_crewai._tracer import _CrewAITracer + +SECRET = "FAKE_BATCH_CREDENTIAL_7589a2" + + +@pytest.mark.parametrize("hide_inputs", [False, True]) +def test_batch_adapter_retains_core_policy_lazy_values_and_lifecycle( + hide_inputs, +): + observations = [] + + class InspectSampler(Sampler): + def should_sample(self, *args, **kwargs): + attributes = kwargs.get("attributes", args[4] if len(args) > 4 else {}) + links = kwargs.get("links", args[5] if len(args) > 5 else ()) + observations.append( + { + "attributes": dict(attributes), + "links": [dict(link.attributes) for link in links], + } + ) + return ALWAYS_ON.should_sample(*args, **kwargs) + + def get_description(self): + return "synthetic-batch-policy" + + provider = TracerProvider(sampler=InspectSampler(), shutdown_on_exit=False) + exporter = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + tracer = _CrewAITracer( + provider.get_tracer("batch"), + _CrewAITraceConfig(TraceConfig(hide_inputs=hide_inputs)), + ) + attributes = { + # Value first: never depend on selector iteration order. + "headers[0]['value']": SECRET, + "headers[0]['name']": "Authorization", + "headers[1]['value']": "application/json", + "headers[1]['name']": "Accept", + "request_id": "business-id", + } + before = copy.deepcopy(attributes) + calls = [] + + def lazy_input(): + calls.append("input") + return "useful input" + + link_context = SpanContext(trace_id=1, span_id=2, is_remote=False) + try: + with tracer.start_span( + "batch", + attributes={**attributes, "input.value": lazy_input}, + links=[Link(link_context, MappingProxyType(attributes))], + ) as span: + span.set_attributes(MappingProxyType(attributes)) + span.set_attributes({"business.lazy": lambda: "useful lazy"}) + span.add_event("batch", MappingProxyType(attributes)) + span.add_link(link_context, MappingProxyType(attributes)) + span.record_exception(RuntimeError("synthetic failure"), attributes) + assert attributes == before + assert calls == ([] if hide_inputs else ["input"]) + assert SECRET not in json.dumps(observations) + assert observations[0]["attributes"]["headers[0]['value']"] == REDACTED + assert observations[0]["links"][0]["headers[0]['value']"] == REDACTED + spans = exporter.get_finished_spans() + assert len(spans) == 1 + result = spans[0] + assert result.attributes["business.lazy"] == "useful lazy" + assert result.attributes["input.value"] == ( + REDACTED if hide_inputs else "useful input" + ) + for batch in ( + result.attributes, + *(event.attributes for event in result.events), + *(link.attributes for link in result.links), + ): + assert batch["headers[0]['value']"] == REDACTED + assert batch["headers[1]['value']"] == "application/json" + assert batch["request_id"] == "business-id" + finally: + provider.shutdown() diff --git a/python/frameworks/crewai/tests/test_core_policy_delegate.py b/python/frameworks/crewai/tests/test_core_policy_delegate.py new file mode 100644 index 000000000..0c5a07d12 --- /dev/null +++ b/python/frameworks/crewai/tests/test_core_policy_delegate.py @@ -0,0 +1,107 @@ +"""The CrewAI projection preserves every explicit core hide setting.""" + +import json + +import pytest +from fi_instrumentation import TraceConfig +from opentelemetry import trace +from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans + +from test_privacy_policy import SECRET, tracer_for + + +@pytest.mark.parametrize("enabled", [False, True]) +@pytest.mark.parametrize( + "flag,key,value", + [ + ("hide_input_images", "gen_ai.input.images.0", "https://example.invalid/image"), + ( + "hide_llm_invocation_parameters", + "gen_ai.request.parameters", + json.dumps({"temperature": 0.5, "api_key": SECRET}), + ), + ("hide_embedding_vectors", "gen_ai.embeddings.vectors", [0.1, 0.2]), + ( + "hide_embedding_vectors", + "embedding.embeddings.0.embedding.vector", + [0.1, 0.2], + ), + ], +) +def test_non_content_hide_flags_are_delegated_without_mutating( + flag, key, value, enabled +): + config = TraceConfig(**{flag: enabled}) + tracer, exporter, provider = tracer_for(config) + with tracer.start_as_current_span( + "policy", attributes={key: value, "client.config.api_key.value": SECRET} + ) as span: + span.add_event("policy.event", {key: value, "headers.Authorization.0": SECRET}) + spans = exporter.get_finished_spans() + assert SECRET.encode() not in encode_spans(spans).SerializeToString() + for attributes in (spans[0].attributes, spans[0].events[0].attributes): + if enabled: + assert key not in attributes + else: + assert key in attributes + assert getattr(config, flag) is enabled + provider.shutdown() + + +@pytest.mark.parametrize("enabled", [False, True]) +def test_pii_setting_is_preserved_and_credentials_are_always_redacted(enabled): + tracer, exporter, provider = tracer_for(TraceConfig(pii_redaction=enabled)) + with tracer.start_as_current_span("policy") as span: + span.add_event( + "log.record", + {"log.body": json.dumps({"api_key": SECRET, "email": "test@example.com"})}, + ) + wire = encode_spans(exporter.get_finished_spans()).SerializeToString() + assert SECRET.encode() not in wire + assert (b"test@example.com" in wire) is not enabled + provider.shutdown() + + +@pytest.mark.parametrize("enabled", [False, True]) +def test_manual_error_event_attributes_stay_structurally_redacted(enabled, caplog): + tracer, exporter, provider = tracer_for(TraceConfig(hide_error_details=enabled)) + with tracer.start_as_current_span("manual-error") as span: + # Free-text error details are controlled separately; structured exception + # attributes must be safe even when error-detail hiding is explicitly off. + span.record_exception( + RuntimeError("ordinary failure"), + attributes={"client.config.api_key.value": SECRET, "safe": "useful error"}, + ) + span.add_event( + "exception", + { + "exception.type": "RuntimeError", + "exception.message": "ordinary failure", + "log.body": json.dumps({"authorization": SECRET, "safe": "useful log"}), + }, + ) + span.set_status(trace.StatusCode.ERROR, "ordinary failure") + spans = exporter.get_finished_spans() + wire = encode_spans(spans).SerializeToString() + assert SECRET.encode() not in wire and SECRET not in caplog.text + assert b"useful error" in wire and b"useful log" in wire + assert (b"ordinary failure" in wire) is not enabled + assert (spans[0].status.description is None) is enabled + provider.shutdown() + + +@pytest.mark.parametrize( + "flag", + [ + "hide_input_images", + "hide_embedding_vectors", + "hide_llm_invocation_parameters", + "pii_redaction", + ], +) +def test_non_content_environment_flags_and_explicit_false_precedence(flag, monkeypatch): + from traceai_crewai._privacy import _CrewAITraceConfig + + monkeypatch.setenv("FI_" + flag.upper(), "true") + assert getattr(_CrewAITraceConfig(TraceConfig()), flag) is True + assert getattr(_CrewAITraceConfig(TraceConfig(**{flag: False})), flag) is False diff --git a/python/frameworks/crewai/tests/test_credential_patterns.py b/python/frameworks/crewai/tests/test_credential_patterns.py new file mode 100644 index 000000000..d997dc84c --- /dev/null +++ b/python/frameworks/crewai/tests/test_credential_patterns.py @@ -0,0 +1,241 @@ +"""Generated structural credential regressions; synthetic values only.""" + +import copy +import itertools +import json + +import pytest +from fi_instrumentation import TraceConfig + +from traceai_crewai._privacy import ( + REDACTED, + _CrewAITraceConfig, + _credential_key, + _safe_data, +) + +SECRET = "FAKE_PATTERN_CREDENTIAL_251f03" +# Independent expectations, not generated from the production allowlist. +SAFE_NAMES = ( + "input_tokens", + "output_tokens", + "total_tokens", + "prompt_tokens", + "completion_tokens", + "cached_tokens", + "cached_input_tokens", + "cache_read_tokens", + "cache_write_tokens", + "input_tokens_details", + "output_tokens_details", + "prompt_tokens_details", + "completion_tokens_details", + "num_tokens", + "author_id", + "author_name", + "authorship", + "api_secret_id", + "reasoning_tokens", + "accepted_prediction_tokens", + "rejected_prediction_tokens", + "candidates_token_count", + "thoughts_token_count", + "cached_content_token_count", + "max_new_tokens", + "token_count", + "max_tokens", + "max_completion_tokens", + "tokens_used", + "token_usage", + "tokenizer", + "author", + "authors", + "authority", + "session_count", + "session_duration_ms", + "token_id", + "security_token_id", + "vault_token_count", + "consul_token_ttl", + "subject_token_type", + "service_token_count", + "cache_creation_input_tokens", + "cache_read_input_tokens", + "audio_tokens", + "tokenizer_name", + "tokenizer_version", + "id", + "project_id", + "shipping", + "opportunity", + "purchase_order", + "query", + "answer", + "business_data", +) + + +@pytest.mark.parametrize("prefix", ["", "X-", "x_", "vendor-", "X-Service-"]) +@pytest.mark.parametrize("case", [str.lower, str.upper, str.title]) +def test_generated_credential_matrix(prefix, case): + concepts = ( + ("token",), + ("security", "token"), + ("private", "token"), + ("gitlab", "token"), + ("circle", "token"), + ("api", "secret"), + ("app", "secret"), + ("password",), + ("passwd",), + ("passphrase",), + ("api", "key"), + ("access", "key"), + ("private", "key"), + ("subscription", "key"), + ("application", "key"), + ("master", "key"), + ("ocp", "apim", "subscription", "key"), + ("dd", "application", "key"), + ("parse", "master", "key"), + ("credential",), + ("auth",), + ("authorization",), + ("authentication",), + ("cookie",), + ("session", "id"), + ("session", "token"), + ("signature",), + ("bearer",), + ("jwt",), + ("otp", "code"), + ("totp",), + ) + for words, separator in itertools.product( + concepts, ("", "-", "_", " ", "/", ":", ".") + ): + name = case(prefix + separator.join(words)) + assert _credential_key(name), name + payload = { + "headers": {name: SECRET, "Accept": "application/json"}, + f"client.config.{name}[0].value": SECRET, + "raw": json.dumps({name: SECRET, "answer": "useful answer"}), + "total_tokens": 42, + "id": "business-id", + } + before = copy.deepcopy(payload) + projected = _safe_data(payload) + assert SECRET not in json.dumps(projected), name + assert projected["headers"]["Accept"] == "application/json" + assert projected["total_tokens"] == 42 and projected["id"] == "business-id" + assert json.loads(projected["raw"])["answer"] == "useful answer" + assert payload == before + assert ( + _CrewAITraceConfig(TraceConfig()).mask( + f"client.config[{name}].value", SECRET + ) + == REDACTED + ) + + +@pytest.mark.parametrize("name", ["pin", "PIN", "x-pin", "X_OTP", "otp"]) +def test_short_credentials_match_whole_normalized_name(name): + assert _safe_data({name: SECRET}) == {name: REDACTED} + + +@pytest.mark.parametrize("name", SAFE_NAMES) +def test_explicit_metadata_exceptions_preserve_values_and_not_children(name): + for spelling in (name, name.upper(), name.replace("_", "-"), "X-" + name): + assert not _credential_key(spelling), spelling + payload = {spelling: 42, f"usage.{spelling}": 42, "id": "safe-id"} + assert _safe_data(payload) == payload + # An allowlisted parent cannot bypass inspection of its children. + assert SECRET not in json.dumps(_safe_data({spelling: {"api_key": SECRET}})) + assert _credential_key(f"{spelling}.api_key") + + +def test_unknown_vendor_token_metadata_is_not_a_generic_bypass(): + for name in ( + "private_token_count", + "vendor_secret_id", + "vendor_auth_type", + ): + assert _safe_data({name: SECRET})[name] == REDACTED + + +@pytest.mark.parametrize("shape", ["pairs", "tuples", "name", "key", "header"]) +def test_header_lists_and_serialized_lists(shape, service_credentials): + fields = list(service_credentials.items()) + [("Accept", "application/json")] + if shape == "pairs": + headers = [list(item) for item in fields] + elif shape == "tuples": + headers = fields + else: + headers = [ + {shape: key, "value": value, "description": "business data"} + for key, value in fields + ] + original = copy.deepcopy(headers) + projected = _safe_data( + {"headers": headers, "serialized_headers": json.dumps(headers)} + ) + rendered = json.dumps(projected) + assert all(value not in rendered for value in service_credentials.values()) + assert "application/json" in rendered + assert "Accept" in rendered + assert headers == original + if shape not in ("pairs", "tuples"): + assert all( + item["description"] == "business data" for item in projected["headers"] + ) + for name in service_credentials: + assert name in rendered + + +@pytest.mark.parametrize("selector", ["name", "key", "header", "0"]) +@pytest.mark.parametrize("brackets", [False, True]) +def test_flattened_header_records(selector, brackets): + value_field = "1" if selector == "0" else "value" + + def key(index, field): + return ( + f"headers[{index}]['{field}']" if brackets else f"headers.{index}.{field}" + ) + + original = { + key(0, selector): "Authorization", + key(0, value_field): SECRET, + key(1, selector): "Accept", + key(1, value_field): "application/json", + "headers['X-Gitlab-Token'][0]": SECRET, + "usage.total_tokens": 42, + "request_id": "business-id", + } + before = copy.deepcopy(original) + safe = _safe_data(original) + assert SECRET not in json.dumps(safe) + assert safe[key(0, selector)] == "Authorization" + assert safe[key(1, value_field)] == "application/json" + assert safe["usage.total_tokens"] == 42 + assert original == before + + +@pytest.mark.parametrize( + "first", ["authentication", "tokenization", "session", "author"] +) +def test_two_item_business_lists_are_not_header_pairs(first): + payload = { + "topics": [first, "billing"], + "categories": (first, "shipping"), + "serialized": json.dumps([first, "business data"]), + "total_tokens": 42, + } + before = copy.deepcopy(payload) + expected = {**payload, "categories": [first, "shipping"]} + assert _safe_data(payload) == expected + assert payload == before + # Only explicit header fields interpret pairs; bare business tables survive. + for headers in ([["Authorization", SECRET]], [("X-API-Key", SECRET)]): + assert _safe_data(headers) == [list(item) for item in headers] + assert SECRET not in json.dumps(_safe_data({"headers": headers})) + assert SECRET not in json.dumps(_safe_data({"headers": json.dumps(headers)})) diff --git a/python/frameworks/crewai/tests/test_credential_projection.py b/python/frameworks/crewai/tests/test_credential_projection.py new file mode 100644 index 000000000..6189fffcc --- /dev/null +++ b/python/frameworks/crewai/tests/test_credential_projection.py @@ -0,0 +1,99 @@ +"""Security regressions use synthetic values only and never contact a provider.""" + +import json +from unittest.mock import MagicMock + +from traceai_crewai._wrappers import ( + _safe_data, + _task_input, + _get_input_value, + _ExecuteCoreWrapper, +) + +SENTINEL = "FAKE_CREDENTIAL_ONLY_6c3f" + + +class ExplosiveRuntime: + def __repr__(self): + raise AssertionError("runtime repr evaluated") + + def to_dict(self): + raise AssertionError("runtime to_dict evaluated") + + def dict(self): + raise AssertionError("runtime dict evaluated") + + def model_dump(self): + raise AssertionError("runtime model_dump evaluated") + + +def test_opaque_objects_are_never_evaluated_even_when_nested(): + assert _safe_data({"value": [ExplosiveRuntime(), "user text"]}) == { + "value": ["[omitted runtime object]", "user text"] + } + + +def test_task_projection_supports_positional_and_keywords_without_agent_dump(): + def execute(agent=None, context=None, tools=None): + pass + + task = MagicMock(description="task text", expected_output="expected text") + positional = json.loads( + _task_input( + execute, + task, + (ExplosiveRuntime(), "context text", [ExplosiveRuntime()]), + {}, + ) + ) + keywords = json.loads( + _task_input( + execute, + task, + (), + { + "agent": ExplosiveRuntime(), + "context": "context text", + "tools": [ExplosiveRuntime()], + }, + ) + ) + assert ( + positional + == keywords + == { + "description": "task text", + "expected_output": "expected text", + "context": "context text", + } + ) + + +def test_generic_input_never_serializes_runtime_objects(): + def execute(arg, **kwargs): + pass + + payload = _get_input_value(execute, ExplosiveRuntime(), extra=ExplosiveRuntime()) + assert SENTINEL not in payload + assert json.loads(payload) == {} + + +def test_exception_does_not_enter_span_and_same_error_propagates(): + tracer = MagicMock() + span = tracer.start_as_current_span.return_value.__enter__.return_value + task = MagicMock( + description="task", expected_output="result", key="task-key", id="task-id" + ) + error = RuntimeError(SENTINEL) + + def execute(agent=None, context=None, tools=None): + raise error + + try: + _ExecuteCoreWrapper(tracer)(execute, task, (), {"context": "context"}) + except RuntimeError as actual: + assert actual is error + else: + raise AssertionError("application error was swallowed") + assert not span.record_exception.called + assert SENTINEL not in str(span.mock_calls) diff --git a/python/frameworks/crewai/tests/test_native_parity.py b/python/frameworks/crewai/tests/test_native_parity.py new file mode 100644 index 000000000..a4fedb708 --- /dev/null +++ b/python/frameworks/crewai/tests/test_native_parity.py @@ -0,0 +1,167 @@ +"""Real CrewAI Agent/Task/Crew and ToolUsage, using a local fake LLM/tool.""" + +import json +from importlib.metadata import version +from types import SimpleNamespace +from unittest.mock import patch + +import pytest +from crewai import Agent, Crew, Task +from crewai.tools.tool_calling import ToolCalling +from crewai.tools.tool_usage import ToolUsage +from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +from fi_instrumentation import TraceConfig +from traceai_crewai import CrewAIInstrumentor + +SECRET = "FAKE_NATIVE_CREDENTIAL_a7130d" +FLOOR = version("crewai") == "0.41.1" + + +def run_crew(): + response = json.dumps({"api_key": SECRET, "answer": "useful answer"}) + if FLOOR: + from langchain_core.language_models.fake import FakeListLLM + + llm = FakeListLLM(responses=["Final Answer: " + response]) + agent = Agent( + role="Synthetic worker", + goal="Answer", + backstory="Local test", + llm=llm, + allow_delegation=False, + ) + task = Task( + description="Return useful answer", + expected_output="JSON answer", + agent=agent, + ) + crew = Crew(agents=[agent], tasks=[task], share_crew=True) + result = crew.kickoff( + inputs={"client": {"config": {"api_key": SECRET}}, "tag": "useful input"} + ) + else: + from crewai import LLM + + llm = LLM( + model="openai/synthetic", api_key=SECRET, base_url="http://127.0.0.1:1/v1" + ) + agent = Agent( + role="Synthetic worker", + goal="Answer", + backstory="Local test", + llm=llm, + allow_delegation=False, + ) + task = Task( + description="Return useful answer", + expected_output="JSON answer", + agent=agent, + ) + crew = Crew(agents=[agent], tasks=[task], share_crew=True) + with patch.object(type(llm), "call", return_value=response) as call: + result = crew.kickoff( + inputs={ + "client": {"config": {"api_key": SECRET}}, + "tag": "useful input", + } + ) + assert call.call_count > 0 + assert json.loads(result.raw) == json.loads(response) + return result.raw + + +def run_tool(): + seen = [] + + def lookup(query: str, client: dict) -> str: + """Return a local synthetic result.""" + seen.append((query, client)) + return json.dumps( + {"api_key": SECRET, "answer": "useful tool answer", "total_tokens": 7} + ) + + if FLOOR: + from crewai.agents.tools_handler import ToolsHandler + from langchain_core.tools import StructuredTool + + tool = StructuredTool.from_function(lookup) + usage = ToolUsage( + tools_handler=ToolsHandler(), + tools=[tool], + original_tools=[], + tools_description="lookup", + tools_names="lookup", + task=SimpleNamespace(used_tools=0), + function_calling_llm=None, + agent=SimpleNamespace(tools_results=[]), + action=None, + ) + usage._printer.print = lambda **kwargs: None + else: + from crewai.tools.structured_tool import CrewStructuredTool + + tool = CrewStructuredTool.from_function(lookup) + usage = ToolUsage( + tools_handler=None, tools=[tool], task=None, function_calling_llm=None + ) + arguments = {"query": "useful query", "client": {"config": {"api_key": SECRET}}} + calling = ToolCalling(tool_name="lookup", arguments=arguments) + result = usage.use(calling, "lookup") + assert seen == [("useful query", {"config": {"api_key": SECRET}})] + assert ( + calling.arguments == arguments + and calling.arguments["client"]["config"]["api_key"] == SECRET + ) + assert json.loads(result) == { + "api_key": SECRET, + "answer": "useful tool answer", + "total_tokens": 7, + } + return result + + +@pytest.mark.parametrize( + "settings", + [ + {}, + {"hide_inputs": True}, + {"hide_outputs": True}, + {"hide_input_text": True, "hide_output_messages": True}, + ], +) +def test_native_instrumented_and_uninstrumented_outcomes_match(settings, caplog): + baseline_crew, baseline_tool = run_crew(), run_tool() + exporter = InMemorySpanExporter() + provider = TracerProvider(shutdown_on_exit=False) + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = CrewAIInstrumentor() + instrumentor.instrument(tracer_provider=provider, config=TraceConfig(**settings)) + try: + assert run_crew() == baseline_crew + assert run_tool() == baseline_tool + finally: + instrumentor.uninstrument() + spans = exporter.get_finished_spans() + assert {span.name for span in spans} >= { + "Task._execute_core", + "Crew.kickoff", + "ToolUsage._use", + } + assert not any("telemetry" in span.name.lower() for span in spans) + wire = encode_spans(spans).SerializeToString() + assert SECRET.encode() not in wire and SECRET not in caplog.text + if not settings: + assert all( + text in wire + for text in ( + b"useful query", + b"useful tool answer", + b"useful input", + b"Return useful answer", + ) + ) + provider.shutdown() diff --git a/python/frameworks/crewai/tests/test_native_telemetry.py b/python/frameworks/crewai/tests/test_native_telemetry.py new file mode 100644 index 000000000..be806e5fe --- /dev/null +++ b/python/frameworks/crewai/tests/test_native_telemetry.py @@ -0,0 +1,99 @@ +"""Native telemetry belongs to the application, not the instrumentor.""" + +import inspect +import os +from importlib import import_module +from importlib.metadata import version +from unittest.mock import patch + +import pytest +from crewai.telemetry import Telemetry +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +from traceai_crewai import CrewAIInstrumentor + + +@pytest.mark.parametrize("disabled", ["true", "false"]) +def test_native_telemetry_instrumentation_does_not_change_user_settings( + monkeypatch, disabled +): + for key in ( + "CREWAI_DISABLE_TELEMETRY", + "CREWAI_DISABLE_TRACKING", + "OTEL_SDK_DISABLED", + ): + monkeypatch.setenv(key, disabled) + before_env = dict(os.environ) + before = dict(vars(Telemetry)) + provider = TracerProvider(shutdown_on_exit=False) + exporter = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = CrewAIInstrumentor() + # Instrumentation must not instantiate or patch the native singleton/provider. + with patch.object( + Telemetry, "__init__", side_effect=AssertionError("native construction") + ): + init = Telemetry.__init__ + instrumentor.instrument(tracer_provider=provider) + try: + assert Telemetry.__init__ is init + assert dict(os.environ) == before_env + for key, value in before.items(): + if key != "__init__": + assert vars(Telemetry)[key] is value + with instrumentor._tracer.start_as_current_span("positive-control"): + pass + assert len(exporter.get_finished_spans()) == ( + 0 if disabled == "true" else 1 + ) + finally: + instrumentor.uninstrument() + assert dict(vars(Telemetry)) == before + assert dict(os.environ) == before_env + provider.shutdown() + + +def test_native_telemetry_installed_source_and_opt_out_flags(monkeypatch): + if version("crewai") == "0.41.1": + # This version has no native opt-out check at all. Do not advertise a + # variable it ignores, or instantiate its exporter to demonstrate it. + source = inspect.getsource(Telemetry) + assert not hasattr(Telemetry, "_is_telemetry_disabled") + for flag in ( + "CREWAI_DISABLE_TELEMETRY", + "CREWAI_DISABLE_TRACKING", + "OTEL_SDK_DISABLED", + "CREWAI_TELEMETRY_OPT_OUT", + "DO_NOT_TRACK", + ): + assert flag not in source + assert "https://telemetry.crewai.com:4319" in source + return + # Maintained CrewAI delegates to crewai-core; the repository-locked 0.152 + # implements the same checks in CrewAI itself. + cls = ( + import_module("crewai_core.telemetry").Telemetry + if version("crewai") == "1.15.33" + else Telemetry + ) + honored = ( + "CREWAI_DISABLE_TELEMETRY", + "CREWAI_DISABLE_TRACKING", + "OTEL_SDK_DISABLED", + ) + ignored = ("CREWAI_TELEMETRY_OPT_OUT", "DO_NOT_TRACK") + source = inspect.getsource(cls._is_telemetry_disabled) + for flag in honored + ignored: + monkeypatch.setenv(flag, "false") + assert not Telemetry._is_telemetry_disabled() + for flag in honored: + assert flag in source + monkeypatch.setenv(flag, "true") + assert Telemetry._is_telemetry_disabled() + monkeypatch.setenv(flag, "false") + for flag in ignored: + assert flag not in source + monkeypatch.setenv(flag, "true") + assert not Telemetry._is_telemetry_disabled() diff --git a/python/frameworks/crewai/tests/test_otlp_wire.py b/python/frameworks/crewai/tests/test_otlp_wire.py new file mode 100644 index 000000000..68052eb4c --- /dev/null +++ b/python/frameworks/crewai/tests/test_otlp_wire.py @@ -0,0 +1,776 @@ +"""Actual owned-loopback HTTP captures, not a gzip round-trip of span.to_json().""" + +import base64 +import copy +import gzip +import json +import os +import threading +from http.server import BaseHTTPRequestHandler, HTTPServer +from urllib.request import ProxyHandler, Request, build_opener +from typing import Any + +import pytest +from crewai.tools.tool_calling import ToolCalling +from fi_instrumentation import FITracer, TraceConfig +from google.protobuf.json_format import MessageToDict +from opentelemetry import trace +from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans +from opentelemetry.exporter.otlp.proto.http import Compression +from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( + OTLPSpanExporter, +) +from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ( + ExportTraceServiceRequest, +) +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import ( + SimpleSpanProcessor, + SpanExporter, + SpanExportResult, +) +from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, +) + +from traceai_crewai._privacy import _CrewAITraceConfig +from traceai_crewai._wrappers import _ToolUseWrapper + +SECRET = "FAKE_WIRE_CREDENTIAL_98cab1" + + +def otlp_json(spans): + # OTLP/JSON follows proto JSON except trace/span IDs are hex, not base64. + body = MessageToDict(encode_spans(spans), use_integers_for_enums=True) + + def convert(value): + if isinstance(value, dict): + return { + key: ( + base64.b64decode(item).hex() + if key in ("traceId", "spanId", "parentSpanId") + else convert(item) + ) + for key, item in value.items() + } + if isinstance(value, list): + return [convert(item) for item in value] + return value + + return json.dumps(convert(body)).encode() + + +class JsonHTTPExporter(SpanExporter): + """Test-only OTLP/JSON exporter; production HTTP exporter is protobuf.""" + + def __init__(self, endpoint, compressed): + self.endpoint, self.compressed = endpoint, compressed + + def export(self, spans): + data = otlp_json(spans) + headers = {"Content-Type": "application/json"} + if self.compressed: + data = gzip.compress(data) + headers["Content-Encoding"] = "gzip" + with build_opener(ProxyHandler({})).open( + Request(self.endpoint, data, headers), timeout=5 + ) as response: + assert response.status == 200 + return SpanExportResult.SUCCESS + + def shutdown(self): + pass + + +@pytest.mark.parametrize("fails", [False, True]) +@pytest.mark.parametrize("encoding", ["protobuf", "json"]) +@pytest.mark.parametrize("compressed", [False, True]) +@pytest.mark.parametrize( + "settings", + [ + {}, + {"hide_inputs": True}, + {"hide_outputs": True}, + {"hide_input_text": True, "hide_output_messages": True}, + ], +) +def test_real_otlp_http_wire( + encoding, compressed, settings, fails, otlp_collector, service_credentials +): + endpoint, captured = otlp_collector + provider = TracerProvider(shutdown_on_exit=False) + exporter = ( + OTLPSpanExporter( + endpoint=endpoint, + timeout=5, + compression=(Compression.Gzip if compressed else Compression.NoCompression), + ) + if encoding == "protobuf" + else JsonHTTPExporter(endpoint, compressed) + ) + provider.add_span_processor(SimpleSpanProcessor(exporter)) + tracer = FITracer( + provider.get_tracer("synthetic-wire"), + _CrewAITraceConfig(TraceConfig(**settings)), + ) + arguments = { + "client": {"config": {"api_key": SECRET}}, + "headers": service_credentials, + "query": "useful query", + } + result = { + "Authorization": SECRET, + "headers": service_credentials, + "answer": "useful answer", + "total_tokens": 13, + } + error = RuntimeError(SECRET) + + def use(calling, tool_string): + trace.get_current_span().add_event( + "tool.log", + { + "log.body": json.dumps( + { + "secret": SECRET, + "headers": service_credentials, + "safe": "useful log", + } + ) + }, + ) + if fails: + raise error + return result + + def invoke(): + return _ToolUseWrapper(tracer)( + use, + None, + (ToolCalling(tool_name="lookup", arguments=arguments), "lookup"), + {}, + ) + + try: + if fails: + with pytest.raises(RuntimeError) as caught: + invoke() + assert caught.value is error + else: + assert invoke() is result + assert result["Authorization"] == SECRET + assert arguments["client"]["config"]["api_key"] == SECRET + assert provider.force_flush() + assert len(captured) == 1 + headers, raw = captured[0] + assert headers["content-type"] == ( + "application/json" if encoding == "json" else "application/x-protobuf" + ) + assert (headers.get("content-encoding") == "gzip") is compressed + body = gzip.decompress(raw) if compressed else raw + assert SECRET.encode() not in body and SECRET.encode() not in raw + assert all( + value.encode() not in body and value.encode() not in raw + for value in service_credentials.values() + ) + assert b"useful log" in body + if settings.get("hide_inputs") or settings.get("hide_input_text"): + assert b"useful query" not in body + if settings.get("hide_outputs") or settings.get("hide_output_messages"): + assert b"useful answer" not in body + if not settings: + assert b"useful query" in body + assert (b"useful answer" in body) is not fails + if encoding == "json": + decoded = json.loads(body) + span = decoded["resourceSpans"][0]["scopeSpans"][0]["spans"][0] + assert len(span["traceId"]) == 32 and len(span["spanId"]) == 16 + assert span["status"]["code"] == (2 if fails else 1) + else: + decoded = ExportTraceServiceRequest.FromString(body) + spans = decoded.resource_spans[0].scope_spans[0].spans + assert len(spans) == 1 + assert spans[0].status.code == (2 if fails else 1) + finally: + provider.shutdown() + + +@pytest.fixture +def otlp_collector(): + captured = [] + + class Collector(BaseHTTPRequestHandler): + def do_POST(self): + data = self.rfile.read(int(self.headers["Content-Length"])) + captured.append( + ( + {key.lower(): value for key, value in self.headers.items()}, + data, + ) + ) + self.send_response(200) + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, *args): + pass + + server = HTTPServer(("127.0.0.1", 0), Collector) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_port}/v1/traces", captured + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + +@pytest.mark.parametrize("encoding", ["protobuf", "json"]) +@pytest.mark.parametrize("compressed", [False, True]) +@pytest.mark.parametrize( + "settings", + [ + {}, + {"hide_inputs": True}, + {"hide_outputs": True}, + {"hide_input_text": True, "hide_output_messages": True}, + ], +) +@pytest.mark.parametrize( + "shape", + ["dict", "pairs", "name", "key", "header", "serialized", "flattened"], +) +def test_native_service_credentials_never_reach_otlp_http( + shape, + encoding, + compressed, + settings, + otlp_collector, + native_tool_usage, + service_credentials, +): + from traceai_crewai import CrewAIInstrumentor + + endpoint, captured = otlp_collector + seen = [] + fields = {**service_credentials, "Accept": "application/json"} + if shape == "pairs": + tool_headers = [list(item) for item in fields.items()] + elif shape in ("name", "key", "header"): + tool_headers = [{shape: key, "value": value} for key, value in fields.items()] + elif shape == "serialized": + tool_headers = json.dumps( + [{"name": key, "value": value} for key, value in fields.items()] + ) + elif shape == "flattened": + tool_headers = {f"headers[{key}].value": value for key, value in fields.items()} + else: + tool_headers = fields + before = copy.deepcopy(tool_headers) + + def lookup(query: str, headers: Any) -> str: + """Return synthetic service credentials without altering application data.""" + seen.append((query, copy.deepcopy(headers))) + trace.get_current_span().add_event( + "tool.log", + {"log.body": json.dumps({"headers": headers, "message": "useful log"})}, + ) + return json.dumps( + {"headers": headers, "answer": "useful answer", "total_tokens": 13} + ) + + usage = native_tool_usage(lookup) + arguments = {"query": "useful query", "headers": tool_headers} + calling = ToolCalling(tool_name="lookup", arguments=arguments) + baseline = usage.use(calling, "lookup") + provider = TracerProvider(shutdown_on_exit=False) + exporter = ( + OTLPSpanExporter( + endpoint=endpoint, + timeout=5, + compression=(Compression.Gzip if compressed else Compression.NoCompression), + ) + if encoding == "protobuf" + else JsonHTTPExporter(endpoint, compressed) + ) + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = CrewAIInstrumentor() + instrumentor.instrument(tracer_provider=provider, config=TraceConfig(**settings)) + try: + # Use a fresh ToolUsage to exclude native caching from the parity check. + actual = native_tool_usage(lookup).use(calling, "lookup") + assert actual == baseline + assert json.loads(actual)["headers"] == before + assert seen == [("useful query", before)] * 2 + assert calling.arguments == arguments + assert arguments["headers"] == before + assert provider.force_flush() and len(captured) == 1 + headers, raw = captured[0] + assert headers["content-type"] == ( + "application/json" if encoding == "json" else "application/x-protobuf" + ) + assert (headers.get("content-encoding") == "gzip") is compressed + body = gzip.decompress(raw) if compressed else raw + for credential in service_credentials.values(): + assert credential.encode() not in body and credential.encode() not in raw + assert b"ToolUsage._use" in body and b"useful log" in body + assert ( + b"total_tokens" in body + or settings.get("hide_outputs") + or settings.get("hide_output_messages") + ) + assert (b"useful query" not in body) is bool( + settings.get("hide_inputs") or settings.get("hide_input_text") + ) + assert (b"useful answer" not in body) is bool( + settings.get("hide_outputs") or settings.get("hide_output_messages") + ) + finally: + instrumentor.uninstrument() + provider.shutdown() + + +@pytest.mark.parametrize("encoding", ["protobuf", "json"]) +@pytest.mark.parametrize("compressed", [False, True]) +@pytest.mark.parametrize("scenario", ["flat-attributes", "business-data"]) +def test_native_batch_projection_and_business_parity( + encoding, compressed, scenario, otlp_collector, native_tool_usage +): + from traceai_crewai import CrewAIInstrumentor + + # Independent, realistic OpenAI prediction details / Gemini usage / HF limit. + payload = { + "usage": { + "completion_tokens_details": { + "accepted_prediction_tokens": 3, + "rejected_prediction_tokens": 5, + }, + "candidates_token_count": 7, + "thoughts_token_count": 11, + "cached_content_token_count": 13, + "total_tokens": 42, + }, + "max_new_tokens": 64, + "topics": ["authentication", "billing"], + "categories": ["tokenization", "shipping"], + "id": "business-id", + } + before = copy.deepcopy(payload) + flat = {"request.id": "useful batch id", "usage.total_tokens": 42} + for selector in ("name", "key", "header", "0"): + value_field = "1" if selector == "0" else "value" + for bracketed in (False, True): + for reverse in (False, True): + root = f"request.headers.{selector}.{int(bracketed)}.{int(reverse)}" + + def path(index, field): + return ( + f"{root}[{index}]['{field}']" + if bracketed + else f"{root}.{index}.{field}" + ) + + records = [ + (path(0, selector), "Authorization"), + (path(0, value_field), SECRET), + (path(1, selector), "Accept"), + (path(1, value_field), "application/json"), + ] + flat.update(reversed(records) if reverse else records) + flat_before = copy.deepcopy(flat) + + def lookup(data: Any) -> str: + """Exercise the actual native ToolUsage and its current span.""" + span = trace.get_current_span() + if scenario == "flat-attributes": + span.set_attributes(flat) + span.add_event("batch.log", flat) + span.add_event("structured.log", {"log.body": json.dumps(data)}) + return json.dumps({"answer": "useful answer", **data}) + + calling = ToolCalling(tool_name="lookup", arguments={"data": payload}) + baseline = native_tool_usage(lookup).use(calling, "lookup") + endpoint, captured = otlp_collector + provider = TracerProvider(shutdown_on_exit=False) + exporter = ( + OTLPSpanExporter( + endpoint=endpoint, + timeout=5, + compression=(Compression.Gzip if compressed else Compression.NoCompression), + ) + if encoding == "protobuf" + else JsonHTTPExporter(endpoint, compressed) + ) + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = CrewAIInstrumentor() + instrumentor.instrument(tracer_provider=provider, config=TraceConfig()) + try: + assert native_tool_usage(lookup).use(calling, "lookup") == baseline + assert payload == before and flat == flat_before + assert provider.force_flush() and len(captured) == 1 + headers, raw = captured[0] + assert (headers.get("content-encoding") == "gzip") is compressed + body = gzip.decompress(raw) if compressed else raw + assert SECRET.encode() not in body and b"useful answer" in body + decoded = ( + json.loads(body) + if encoding == "json" + else MessageToDict( + ExportTraceServiceRequest.FromString(body), + use_integers_for_enums=True, + ) + ) + spans = [ + span + for resource in decoded["resourceSpans"] + for scope in resource["scopeSpans"] + for span in scope["spans"] + ] + assert len(spans) == 1 + + def attributes(items): + return { + item["key"]: item["value"].get( + "stringValue", item["value"].get("intValue") + ) + for item in items + } + + attrs = attributes(spans[0]["attributes"]) + output = json.loads(attrs["output.value"]) + if isinstance(output, str): + output = json.loads(output) + assert output == {"answer": "useful answer", **before} + assert json.loads(attrs["input.value"])["tool_args"]["data"] == before + events = { + event["name"]: attributes(event["attributes"]) + for event in spans[0]["events"] + } + assert json.loads(events["structured.log"]["log.body"]) == before + if scenario == "flat-attributes": + expected = { + key: "__REDACTED__" if value == SECRET else value + for key, value in flat.items() + } + for batch in (attrs, events["batch.log"]): + for key, value in expected.items(): + assert batch[key] == (str(value) if type(value) is int else value) + finally: + instrumentor.uninstrument() + provider.shutdown() + + +def test_raw_negative_control_detects_unprojected_synthetic_credential(): + # Do not send this deliberately unsafe baseline to even the local collector. + exporter = InMemorySpanExporter() + provider = TracerProvider(shutdown_on_exit=False) + provider.add_span_processor(SimpleSpanProcessor(exporter)) + with provider.get_tracer("raw-baseline").start_as_current_span( + "raw", attributes={"input.value": json.dumps({"api_key": SECRET})} + ): + pass + spans = exporter.get_finished_spans() + for data in (encode_spans(spans).SerializeToString(), otlp_json(spans)): + assert SECRET.encode() in data + assert SECRET.encode() in gzip.decompress(gzip.compress(data)) + provider.shutdown() + + +@pytest.mark.parametrize("encoding", ["protobuf", "json"]) +@pytest.mark.parametrize("compressed", [False, True]) +@pytest.mark.parametrize("capture_content", [False, True]) +def test_native_kickoff_structural_configuration_never_reaches_wire( + encoding, compressed, capture_content, otlp_collector, monkeypatch, caplog +): + """Real native kickoff, real framework credential fields, local fake LLM. + + Run this on the floor and current CrewAI. No vendor call/credential guessing: + these canaries exist only in configuration, not application prose/content. + """ + from importlib.metadata import version + from unittest.mock import patch + from crewai import Agent, Crew, Task + from traceai_crewai import CrewAIInstrumentor + + canaries = tuple(f"SYNTHETIC_CONFIG_{letter}_9ca427" for letter in "ABCD") + api_key, url_password, tool_key, environment_key = canaries + monkeypatch.setenv("OPENAI_API_KEY", environment_key) + base_url = f"https://u:{url_password}@example.invalid" + answer = "useful structural answer" + floor = version("crewai") == "0.41.1" + if floor: + # CrewAI 0.41.1 predates crewai.LLM and takes a LangChain LLM. + from langchain_openai import ChatOpenAI + from langchain_core.tools import BaseTool + from langchain_core.messages import AIMessage, AIMessageChunk + from langchain_core.outputs import ( + ChatGeneration, + ChatGenerationChunk, + ChatResult, + ) + from types import SimpleNamespace + + # CrewAI's token callback otherwise downloads a tokenizer on first use. + monkeypatch.setattr( + "tiktoken.encoding_for_model", + lambda _: SimpleNamespace(encode=lambda text: list(text)), + ) + + class LocalLLM(ChatOpenAI): + def _stream(self, *args, **kwargs): + trace.get_current_span().add_event( + "llm.log", {"log.body": "useful local LLM log"} + ) + yield ChatGenerationChunk( + message=AIMessageChunk(content="Final Answer: " + answer) + ) + + def _generate(self, *args, **kwargs): + trace.get_current_span().add_event( + "llm.log", {"log.body": "useful local LLM log"} + ) + return ChatResult( + generations=[ + ChatGeneration( + message=AIMessage(content="Final Answer: " + answer) + ) + ] + ) + + llm = LocalLLM(model="gpt-4o-mini", api_key=api_key, base_url=base_url) + assert llm.openai_api_key.get_secret_value() == api_key + assert llm.openai_api_base == base_url + else: + from crewai import LLM + from crewai.tools import BaseTool + + llm = LLM(model="openai/synthetic", api_key=api_key, base_url=base_url) + assert llm.api_key == api_key and llm.base_url == base_url + assert os.environ["OPENAI_API_KEY"] == environment_key + + class LocalTool(BaseTool): + name: str = "local_lookup" + description: str = "Look up a useful business result locally" + api_key: str = tool_key + + def _run(self, query: str) -> str: + return "useful business tool answer" + + def run(): + tool = LocalTool() + agent = Agent( + role="Useful analyst", + goal="Answer a business question", + backstory="Uses public business information", + llm=llm, + tools=[tool], + allow_delegation=False, + ) + task = Task( + description="Return useful business answer", + expected_output="A concise answer", + agent=agent, + tools=[tool], + ) + crew = Crew(agents=[agent], tasks=[task], share_crew=True) + inputs = { + "query": "useful business query", + "topics": ["authentication", "billing"], + } + before = copy.deepcopy(inputs) + if floor: + result = crew.kickoff(inputs=inputs) + else: + + def respond(*args, **kwargs): + trace.get_current_span().add_event( + "llm.log", {"log.body": "useful local LLM log"} + ) + return answer + + with patch.object(type(llm), "call", side_effect=respond) as call: + result = crew.kickoff(inputs=inputs) + assert call.called + assert inputs == before and tool.api_key == tool_key + return result.raw + + baseline = run() + assert baseline == answer + endpoint, captured = otlp_collector + provider = TracerProvider(shutdown_on_exit=False) + exporter = ( + OTLPSpanExporter( + endpoint=endpoint, + timeout=5, + compression=(Compression.Gzip if compressed else Compression.NoCompression), + ) + if encoding == "protobuf" + else JsonHTTPExporter(endpoint, compressed) + ) + provider.add_span_processor(SimpleSpanProcessor(exporter)) + config = TraceConfig( + hide_inputs=not capture_content, hide_outputs=not capture_content + ) + instrumentor = CrewAIInstrumentor() + instrumentor.instrument(tracer_provider=provider, config=config) + try: + assert run() == baseline + assert provider.force_flush() and len(captured) >= 2 + decoded_spans = [] + for headers, raw in captured: + assert (headers.get("content-encoding") == "gzip") is compressed + assert headers["content-type"] == ( + "application/json" if encoding == "json" else "application/x-protobuf" + ) + body = gzip.decompress(raw) if compressed else raw + for canary in canaries: + assert canary.encode() not in body and canary.encode() not in raw + assert canary not in caplog.text + # Ensure we're checking meaningful parsed OTLP, not empty captures. + decoded = ( + json.loads(body) + if encoding == "json" + else MessageToDict( + ExportTraceServiceRequest.FromString(body), + use_integers_for_enums=True, + ) + ) + decoded_spans.extend( + span + for resource in decoded["resourceSpans"] + for scope in resource["scopeSpans"] + for span in scope["spans"] + ) + assert {span["name"] for span in decoded_spans} >= { + "Task._execute_core", + "Crew.kickoff", + } + wire = b"".join( + gzip.decompress(raw) if compressed else raw for _, raw in captured + ) + assert b"useful local LLM log" in wire + assert (b"useful business query" in wire) is capture_content + assert (answer.encode() in wire) is capture_content + finally: + instrumentor.uninstrument() + provider.shutdown() + + +@pytest.mark.parametrize("encoding", ["protobuf", "json"]) +@pytest.mark.parametrize("compressed", [False, True]) +def test_contextual_payload_policy_business_and_header_shapes_on_wire( + encoding, compressed, otlp_collector, native_tool_usage +): + from traceai_crewai import CrewAIInstrumentor + + payload = { + "topics": ["authentication", "billing"], + "rows": [ + ["authentication", "public-billing"], + ["tokenization", "public-shipping"], + ], + "products": [ + {"name": "Authentication handbook", "value": "public-description"}, + {"key": "tokenization", "value": "public-category"}, + ], + "usage": { + "accepted_prediction_tokens": 1, + "rejected_prediction_tokens": 2, + "candidates_token_count": 3, + "thoughts_token_count": 4, + "cached_content_token_count": 5, + "max_new_tokens": 6, + "input_tokens": 7, + "output_tokens": 8, + "total_tokens": 15, + "prompt_cache_hit_tokens": 9, + "prompt_cache_miss_tokens": 10, + "cacheWriteInputTokens": 11, + "toolUsePromptTokenCount": 12, + }, + "author_email": "writer@example.invalid", + "author_url": "https://example.invalid/writer", + "client": {"api": {"key": SECRET}, "private": {"key": SECRET}}, + "rawHeaders": [ + "Authorization", + SECRET, + "X-API-Key", + SECRET, + "Accept", + "application/json", + ], + "headers": [{"name": "Authorization", "value": [SECRET]}], + # Undocumented names are intentionally outside the application policy. + "custom_vendor_field": "documented application limitation", + } + flat = { + "headers.0.name": "Authorization", + "headers.0.value.0": SECRET, + "headers[1]['key']": "X-API-Key", + "headers[1]['value'][0]": SECRET, + } + before = copy.deepcopy(payload) + + def lookup(data: Any) -> str: + """Emit synthetic application data on the actual native tool span.""" + span = trace.get_current_span() + span.set_attributes(flat) + span.add_event("batch.log", flat) + span.add_event("tool.log", {"log.body": json.dumps(data)}) + return json.dumps(data) + + endpoint, captured = otlp_collector + provider = TracerProvider(shutdown_on_exit=False) + exporter = ( + OTLPSpanExporter( + endpoint=endpoint, + timeout=5, + compression=(Compression.Gzip if compressed else Compression.NoCompression), + ) + if encoding == "protobuf" + else JsonHTTPExporter(endpoint, compressed) + ) + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = CrewAIInstrumentor() + instrumentor.instrument(tracer_provider=provider) + try: + calling = ToolCalling(tool_name="lookup", arguments={"data": payload}) + assert json.loads(native_tool_usage(lookup).use(calling, "lookup")) == before + assert payload == before + assert provider.force_flush() and len(captured) == 1 + _, raw = captured[0] + body = gzip.decompress(raw) if compressed else raw + assert SECRET.encode() not in body + decoded = ( + json.loads(body) + if encoding == "json" + else MessageToDict( + ExportTraceServiceRequest.FromString(body), use_integers_for_enums=True + ) + ) + span = decoded["resourceSpans"][0]["scopeSpans"][0]["spans"][0] + attrs = { + attr["key"]: attr["value"]["stringValue"] for attr in span["attributes"] + } + output = json.loads(attrs["output.value"]) + if isinstance(output, str): + output = json.loads(output) + for field in ( + "topics", + "rows", + "products", + "usage", + "author_email", + "author_url", + "custom_vendor_field", + ): + assert output[field] == before[field] + for event in span["events"]: + if event["name"] == "tool.log": + log = json.loads(event["attributes"][0]["value"]["stringValue"]) + for field in ("topics", "rows", "products", "usage"): + assert log[field] == before[field] + finally: + instrumentor.uninstrument() + provider.shutdown() diff --git a/python/frameworks/crewai/tests/test_privacy_policy.py b/python/frameworks/crewai/tests/test_privacy_policy.py new file mode 100644 index 000000000..9b020ff1b --- /dev/null +++ b/python/frameworks/crewai/tests/test_privacy_policy.py @@ -0,0 +1,357 @@ +"""Synthetic credentials, projected aliases, events, logs and error paths.""" + +import copy +import json +from types import SimpleNamespace + +import pytest +from fi_instrumentation import FITracer, TraceConfig +from opentelemetry import trace +from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +from traceai_crewai._privacy import _CrewAITraceConfig, _credential_key, _safe_data +from traceai_crewai._wrappers import ( + _ExecuteCoreWrapper, + _KickoffWrapper, + _ToolUseWrapper, +) + +SECRET = "FAKE_PRIVACY_CREDENTIAL_ef718a" +INPUT = "private input prose" +OUTPUT = "private output prose" +INPUT_ALIASES = ( + "input.value", + "raw.input", + "crew_inputs", + "crew_agents", + "crew_tasks", + "formatted_description", + "formatted_expected_output", + "gen_ai.input.messages.0.message.content", + "gen_ai.prompts", + "gen_ai.prompt", + "gen_ai.request.parameters", + "tool.parameters", + "gen_ai.tool.call.arguments", + "tool_call.function.arguments", + "message.function_call_arguments_json", +) +OUTPUT_ALIASES = ( + "output.value", + "raw.output", + "gen_ai.output.messages.0.message.content", + "gen_ai.completion", + "gen_ai.tool.call.result", +) +CONTENT_FLAGS = ( + "hide_inputs", + "hide_input_text", + "hide_input_messages", + "hide_outputs", + "hide_output_text", + "hide_output_messages", +) + + +def tracer_for(config): + exporter = InMemorySpanExporter() + provider = TracerProvider(shutdown_on_exit=False) + provider.add_span_processor(SimpleSpanProcessor(exporter)) + return ( + FITracer(provider.get_tracer("synthetic-crewai"), _CrewAITraceConfig(config)), + exporter, + provider, + ) + + +@pytest.mark.parametrize( + "key", + [ + "api_key", + "apiKey", + "API-KEY", + "X-API-Key", + "Authorization", + "Proxy-Authorization", + "access_token", + "refreshToken", + "auth_token", + "authentication_token", + "client_secret", + "clientSecretKey", + "secret_key", + "password", + "passwd", + "token", + "session_token", + "bearer_token", + "private_key", + "client.config.api_key", + "client.config.client_key", + "client.config.token", + "client.config.secret", + "headers.Authorization", + "aws_secret_access_key", + "aws_access_key_id", + "credentials", + "Cookie", + "Set-Cookie", + "headers.Authorization.0", + "headers[X-API-Key]", + "client.config.api_key.value", + "client.config.token.value", + "client.config.secret.value", + "client.config.secrets.value", + "apiSecret", + "secret_token", + "X-Amz-Security-Token", + "X-Vault-Token", + "X-Consul-Token", + "X-Subject-Token", + "X-Api-Secret", + "X-Service-Token", + "security_token", + "aws_security_token", + "vault_token", + "consul_token", + "subject_token", + "service_token", + "headers.x-aMz-SeCuRiTy-ToKeN", + "headers[X-Vault-Token][0]", + "headers.X-Consul-Token.value", + "headers[X-Subject-Token]", + "headers.X-Api-Secret.0", + "client.config.security_token.value", + ], +) +def test_credential_spellings_and_business_fields(key): + payload = { + key: SECRET, + "model": "safe-model", + "provider": "safe-provider", + "total_tokens": 42, + "prompt_tokens": 11, + "completion_tokens": 31, + "token_count": 42, + "tokenizer": "safe", + "max_tokens": 90, + "token_id": "safe-id", + "security_token_id": "safe-security-id", + "vault_token_count": 3, + "consul_token_ttl": 3600, + "subject_token_type": "safe-type", + "service_token_count": 2, + "api_secret_id": "safe-secret-id", + "id": "business-id", + } + before = copy.deepcopy(payload) + projected = _safe_data(payload) + assert SECRET not in json.dumps(projected) + assert projected[key] == "__REDACTED__" + assert {k: v for k, v in projected.items() if k != key} == { + k: v for k, v in payload.items() if k != key + } + assert payload == before + + +@pytest.mark.parametrize("flag", CONTENT_FLAGS) +@pytest.mark.parametrize("enabled", [True, False]) +def test_every_directional_hide_flag_covers_aliases_and_events( + flag, enabled, caplog, service_credentials +): + tracer, exporter, provider = tracer_for(TraceConfig(**{flag: enabled})) + attrs = { + **{k: INPUT for k in INPUT_ALIASES}, + **{k: OUTPUT for k in OUTPUT_ALIASES}, + "headers.Authorization": SECRET, + "gen_ai.usage.total_tokens": 42, + "log.body": json.dumps( + {"api_key": SECRET, "headers": service_credentials, "message": "useful log"} + ), + **{f"headers.{key}": value for key, value in service_credentials.items()}, + } + before = copy.deepcopy(attrs) + with tracer.start_as_current_span("privacy", attributes=attrs) as span: + span.set_attributes(attrs) + trace.get_current_span().add_event("tool.result", attributes=attrs) + spans = exporter.get_finished_spans() + wire = encode_spans(spans).SerializeToString() + assert SECRET.encode() not in wire + assert b"useful log" in wire and b"tool.result" in wire + for credential in service_credentials.values(): + assert credential.encode() not in wire and credential not in caplog.text + assert SECRET not in caplog.text + if enabled: + hidden, visible = (INPUT, OUTPUT) if "input" in flag else (OUTPUT, INPUT) + assert hidden.encode() not in wire and visible.encode() in wire + else: + assert INPUT.encode() in wire and OUTPUT.encode() in wire + assert attrs == before + assert spans[0].attributes["gen_ai.usage.total_tokens"] == 42 + provider.shutdown() + + +@pytest.mark.parametrize( + "kwargs", + [ + {}, + {"hide_inputs": False, "hide_outputs": False, "hide_error_details": False}, + {"hide_inputs": True, "hide_outputs": True}, + {"hide_input_messages": True, "hide_output_text": True}, + { + "hide_inputs": True, + "hide_output_messages": True, + "hide_error_details": False, + }, + {"hide_inputs": False, "hide_outputs": True, "hide_error_details": True}, + {"hide_input_images": True}, + {"hide_embedding_vectors": True}, + {"hide_llm_invocation_parameters": True}, + {"pii_redaction": True}, + ], +) +def test_default_and_explicit_combinations_always_redact_structured_aliases( + kwargs, service_credentials +): + tracer, exporter, provider = tracer_for(TraceConfig(**kwargs)) + structured = json.dumps( + { + "client": {"config": {"api_key": SECRET}}, + "headers": service_credentials, + "safe": "business", + } + ) + attrs = {k: structured for k in INPUT_ALIASES + OUTPUT_ALIASES} + with tracer.start_as_current_span("privacy", attributes=attrs) as span: + span.add_event("structured", attrs) + wire = encode_spans(exporter.get_finished_spans()).SerializeToString() + assert SECRET.encode() not in wire + assert all(value.encode() not in wire for value in service_credentials.values()) + assert ( + b"business" in wire + or (kwargs.get("hide_inputs") and kwargs.get("hide_outputs")) + or (kwargs.get("hide_input_messages") and kwargs.get("hide_output_text")) + or (kwargs.get("hide_inputs") and kwargs.get("hide_output_messages")) + ) + provider.shutdown() + + +@pytest.mark.parametrize("flag", CONTENT_FLAGS + ("hide_error_details",)) +def test_environment_flags_and_explicit_false_precedence(flag, monkeypatch): + monkeypatch.setenv("FI_" + flag.upper(), "true") + config = _CrewAITraceConfig(TraceConfig()) + assert getattr(config, flag) is True + explicit = _CrewAITraceConfig(TraceConfig(**{flag: False})) + assert getattr(explicit, flag) is False + assert explicit.mask("headers.Authorization", SECRET) == "__REDACTED__" + + +@pytest.mark.parametrize( + "wrapper_type", [_ExecuteCoreWrapper, _KickoffWrapper, _ToolUseWrapper] +) +@pytest.mark.parametrize( + "settings", + [ + {}, + {"hide_error_details": False}, + {"hide_error_details": True}, + {"hide_inputs": True, "hide_error_details": False}, + ], +) +def test_wrapper_errors_propagate_without_messages_events_or_logs( + wrapper_type, settings, caplog +): + tracer, exporter, provider = tracer_for(TraceConfig(**settings)) + instance = SimpleNamespace( + key="safe-key", + id="safe-id", + description="task", + expected_output="answer", + agents=[], + tasks=[], + usage_metrics=None, + ) + error = RuntimeError(SECRET) + + def fail(**kwargs): + raise error + + with pytest.raises(RuntimeError) as caught: + wrapper_type(tracer)(fail, instance, (), {}) + assert caught.value is error + spans = exporter.get_finished_spans() + assert len(spans) == 1 and spans[0].status.status_code is trace.StatusCode.ERROR + assert not spans[0].events and spans[0].status.description is None + assert SECRET.encode() not in encode_spans(spans).SerializeToString() + assert SECRET not in caplog.text + provider.shutdown() + + +def test_depth_cycles_and_opaque_objects_are_not_serialized(): + class Opaque: + def __getattribute__(self, name): + raise AssertionError("opaque object inspected") + + def __repr__(self): + raise AssertionError("opaque object stringified") + + cycle = {"safe": "business", "api_key": SECRET, "runtime": Opaque()} + cycle["cycle"] = cycle + assert SECRET not in json.dumps(_safe_data(cycle)) + assert _safe_data("free text " + SECRET) == "free text " + SECRET + assert not _credential_key("gen_ai.usage.total_tokens") + + +def test_context_is_redacted_before_sampler_and_at_final_setters(): + from fi_instrumentation import using_attributes + from opentelemetry.sdk.trace.sampling import Sampler, ALWAYS_ON + + observed = [] + + class InspectSampler(Sampler): + def should_sample(self, *args, **kwargs): + observed.append(dict(kwargs.get("attributes") or args[4])) + return ALWAYS_ON.should_sample(*args, **kwargs) + + def get_description(self): + return "synthetic-inspection" + + exporter = InMemorySpanExporter() + provider = TracerProvider(sampler=InspectSampler(), shutdown_on_exit=False) + provider.add_span_processor(SimpleSpanProcessor(exporter)) + tracer = FITracer(provider.get_tracer("context"), _CrewAITraceConfig(TraceConfig())) + instance = SimpleNamespace( + description="task", expected_output="answer", key="safe-key", id="safe-id" + ) + + def execute(agent=None, context=None): + return {"api_key": SECRET, "answer": "business output"} + + with using_attributes( + metadata={ + "client": {"config": {"api_key": SECRET}}, + "safe": "business metadata", + } + ): + _ExecuteCoreWrapper(tracer)(execute, instance, (), {}) + assert len(observed) == 1 + assert SECRET not in json.dumps(observed) + assert "business metadata" in json.dumps(observed) + wire = encode_spans(exporter.get_finished_spans()).SerializeToString() + assert ( + SECRET.encode() not in wire + and b"business metadata" in wire + and b"business output" in wire + ) + provider.shutdown() + + +def test_unparseable_or_oversized_structured_data_fails_closed(): + deeply_nested = "[" * 1100 + json.dumps({"api_key": SECRET}) + "]" * 1100 + malformed = '{"api_key": "' + SECRET + oversized = json.dumps({"api_key": SECRET, "text": "x" * 1_000_001}) + for value in (deeply_nested, malformed, oversized): + assert SECRET not in _safe_data(value) diff --git a/python/frameworks/crewai/tests/test_structural_privacy_r3.py b/python/frameworks/crewai/tests/test_structural_privacy_r3.py new file mode 100644 index 000000000..31406362f --- /dev/null +++ b/python/frameworks/crewai/tests/test_structural_privacy_r3.py @@ -0,0 +1,77 @@ +"""Synthetic-only structural privacy tests; no real credentials or network.""" + +import copy +import json + +from crewai.tools.tool_calling import ToolCalling +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans +from traceai_crewai._wrappers import _safe_data, _ToolUseWrapper + +SECRET = "FAKE_STRUCTURAL_CREDENTIAL_6dd845" + + +class Opaque: + def __repr__(self): + raise AssertionError("repr called") + + def __getattribute__(self, name): + raise AssertionError("runtime object inspected") + + +def test_structural_aliases_and_unchanged_input(): + item = { + "client": {"config": {"api_key": SECRET}}, + "apiKey": SECRET, + "headers": {"X-API-Key": SECRET, "Authorization": SECRET}, + "raw": json.dumps({"access_token": SECRET, "model": "safe-model"}), + "events": [{"client_secret": SECRET, "log": {"password": SECRET}}], + "client.config.api_key": SECRET, + "total_tokens": 42, + "id": "safe-id", + } + before = copy.deepcopy(item) + projected = _safe_data(item) + assert SECRET not in json.dumps(projected) + assert projected["total_tokens"] == 42 and projected["id"] == "safe-id" + assert json.loads(projected["raw"])["model"] == "safe-model" + assert item == before + assert _safe_data({"object": Opaque()}) == {"object": "[omitted runtime object]"} + assert ( + _safe_data("arbitrary free text " + SECRET) == "arbitrary free text " + SECRET + ) + + +def test_native_toolcalling_protobuf_and_original_response(): + arguments = { + "query": "useful query", + "client": {"config": {"api_key": SECRET}}, + "arguments": json.dumps({"refresh_token": SECRET, "safe": "ok"}), + "prompt_tokens": 12, + "opaque": Opaque(), + } + calling = ToolCalling(tool_name="search", arguments=arguments) + output = {"raw": {"apiKey": SECRET, "value": "useful output"}, "count": 7} + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + + def wrapped(calling, tool_string): + return output + + result = _ToolUseWrapper(provider.get_tracer("synthetic"))( + wrapped, None, (calling, "search"), {} + ) + spans = exporter.get_finished_spans() + assert len(spans) == 1 and result is output + assert calling.arguments["client"]["config"]["api_key"] == SECRET + assert output["raw"]["apiKey"] == SECRET + wire = encode_spans(spans).SerializeToString() + assert SECRET.encode() not in wire + assert ( + b"useful query" in wire + and b"useful output" in wire + and b"prompt_tokens" in wire + ) diff --git a/python/frameworks/crewai/traceai_crewai/__init__.py b/python/frameworks/crewai/traceai_crewai/__init__.py index 6872bcdd0..612c188d0 100644 --- a/python/frameworks/crewai/traceai_crewai/__init__.py +++ b/python/frameworks/crewai/traceai_crewai/__init__.py @@ -2,7 +2,7 @@ from importlib import import_module from typing import Any, Collection -from fi_instrumentation import FITracer, TraceConfig +from fi_instrumentation import TraceConfig from opentelemetry import trace as trace_api from opentelemetry.instrumentation.instrumentor import BaseInstrumentor # type: ignore from traceai_crewai._wrappers import ( @@ -10,6 +10,8 @@ _KickoffWrapper, _ToolUseWrapper, ) +from traceai_crewai._privacy import _CrewAITraceConfig +from traceai_crewai._tracer import _CrewAITracer from traceai_crewai.version import __version__ from wrapt import wrap_function_wrapper @@ -36,9 +38,9 @@ def _instrument(self, **kwargs: Any) -> None: config = TraceConfig() else: assert isinstance(config, TraceConfig) - self._tracer = FITracer( + self._tracer = _CrewAITracer( trace_api.get_tracer(__name__, __version__, tracer_provider), - config=config, + config=_CrewAITraceConfig(config), ) execute_core_wrapper = _ExecuteCoreWrapper(tracer=self._tracer) diff --git a/python/frameworks/crewai/traceai_crewai/_privacy.py b/python/frameworks/crewai/traceai_crewai/_privacy.py new file mode 100644 index 000000000..f49879616 --- /dev/null +++ b/python/frameworks/crewai/traceai_crewai/_privacy.py @@ -0,0 +1,333 @@ +"""CrewAI-specific structural projection; deliberately not free-text secret detection.""" + +import json +import re +from dataclasses import fields +from typing import Any + +from fi_instrumentation import TraceConfig + +REDACTED = "__REDACTED__" +# Match credential concepts, not a finite list of vendor header spellings. +_KEY_CREDENTIAL_MARKERS = ( + "apikey", + "accesskey", + "privatekey", + "clientkey", + "signingkey", + "subscriptionkey", + "applicationkey", + "masterkey", +) +_CREDENTIAL_MARKERS = _KEY_CREDENTIAL_MARKERS + ( + "token", + "secret", + "password", + "passwd", + "passphrase", + "credential", + "auth", + "cookie", + "session", + "signature", + "bearer", + "jwt", + "otp", +) +# Exact normalized exceptions only: never exempt arbitrary '*_count'/'*_id' +# names (e.g. private_token_count could still carry a credential). This documents +# the supported non-secret usage counters, model limits, tokenizer metadata, +# authorship, and service-token metadata. Extend deliberately with regressions. +_NON_SECRET_NAMES = frozenset( + { + "inputtokens", + "outputtokens", + "totaltokens", + "prompttokens", + "completiontokens", + "cachedtokens", + "cachedinputtokens", + "cachereadtokens", + "cachewritetokens", + "inputtokensdetails", + "outputtokensdetails", + "prompttokensdetails", + "completiontokensdetails", + "numtokens", + "cachecreationinputtokens", + "cachereadinputtokens", + "reasoningtokens", + # OpenAI prediction details, Gemini usage, and generation limits. + "acceptedpredictiontokens", + "rejectedpredictiontokens", + "candidatestokencount", + "thoughtstokencount", + "cachedcontenttokencount", + "maxnewtokens", + "audiotokens", + "tokencount", + "inputtokencount", + "outputtokencount", + "totaltokencount", + "prompttokencount", + "completiontokencount", + "cachedtokencount", + "reasoningtokencount", + "maxtokens", + "maxinputtokens", + "maxoutputtokens", + "maxcompletiontokens", + "tokensused", + "tokenusage", + "tokenizer", + "tokenizername", + "tokenizerversion", + "tokenid", + "securitytokenid", + "apisecretid", + "vaulttokencount", + "consultokenttl", + "subjecttokentype", + "servicetokencount", + "author", + "authors", + "authorid", + "authorname", + "authoremail", + "authorurl", + "promptcachehittokens", + "promptcachemisstokens", + "cachewriteinputtokens", + "tooluseprompttokencount", + "authorship", + "authority", + "sessioncount", + "sessionduration", + "sessiondurationms", + } +) +_INPUT_ALIASES = ( + "input.value", + "raw.input", + "crew_inputs", + "crew_agents", + "crew_tasks", + "formatted_description", + "formatted_expected_output", + "gen_ai.input.messages", + "gen_ai.prompts", + "gen_ai.prompt", + "gen_ai.request.parameters", + "gen_ai.tool.call.arguments", + "tool.parameters", + "tool_call.function.arguments", + "message.function_call_arguments_json", +) +_OUTPUT_ALIASES = ( + "output.value", + "raw.output", + "gen_ai.output.messages", + "gen_ai.completion", + "gen_ai.tool.call.result", +) + + +def _normalized_name(name: str) -> str: + name = re.sub(r"^x[\W_]+", "", name.lower().strip(" '\"")) + return re.sub(r"[^a-z0-9]", "", name) + + +def _credential_key(key: str) -> bool: + # Inspect every path component, including credential containers followed by + # a value/index. Exceptions apply to one complete field, never its children. + if _normalized_name(key) in _NON_SECRET_NAMES: + return False + parts = [_normalized_name(part) for part in re.split(r"[.\[\]]+", key) if part] + for normalized in parts: + if normalized in _NON_SECRET_NAMES: + continue + # PIN is whole-name only, so e.g. 'shipping' is not a credential. + if normalized == "pin" or any( + marker in normalized for marker in _CREDENTIAL_MARKERS + ): + return True + # A dot/bracket can separate words of a credential name as well as paths, + # e.g. headers.api.key or client[private][key]. + return any( + marker in left + right + for left, right in zip(parts, parts[1:]) + for marker in _KEY_CREDENTIAL_MARKERS + ) + + +def _field_path(key: str) -> tuple: + return tuple(_normalized_name(part) for part in re.split(r"[.\[\]]+", key) if part) + + +def _is_headers_field(key: str) -> bool: + parts = _field_path(key) + return bool(parts and parts[-1].endswith("headers")) + + +def _redact_header_records( + value: dict, *, _header_context: bool = False, _path: tuple = () +) -> dict: + """Project selector/value siblings only in explicitly named header fields. + + Flattened dot/bracket paths are supported, including descendants of a + list-valued value field. No inference from ordinary product records/pairs. + Unrelated values (including core's lazy callbacks) remain unevaluated. + """ + credential_values = set() + for key, item in value.items(): + if type(key) is not str or type(item) is not str: + continue + parts = _path + _field_path(key) + in_headers = _header_context or any( + part.endswith("headers") for part in parts[:-1] + ) + if ( + in_headers + and parts + and parts[-1] in ("name", "key", "header", "0") + and _credential_key(item) + ): + paired_field = "1" if parts[-1] == "0" else "value" + credential_values.add(parts[:-1] + (paired_field,)) + return { + key: ( + REDACTED + if type(key) is str + and any( + (_path + _field_path(key))[: len(prefix)] == prefix + for prefix in credential_values + ) + else item + ) + for key, item in value.items() + } + + +def _safe_data( + value: Any, + depth: int = 0, + *, + _header_context: bool = False, + _path: tuple = (), +) -> Any: + """Copy plain application data under a documented field-name policy. + + Never inspect, stringify or dump an opaque framework/configuration object. + Header records/pairs/alternating lists require a named headers context. + """ + if depth > 20: + return "[omitted]" + if type(value) is str: + if value.lstrip().startswith(("{", "[")): + if len(value) > 1_000_000: + return "[omitted oversized structured data]" + try: + parsed = json.loads(value) + except (ValueError, RecursionError): + return "[omitted unparseable structured data]" + if type(parsed) in (dict, list): + return json.dumps( + _safe_data( + parsed, + depth + 1, + _header_context=_header_context, + _path=_path, + ) + ) + return value + if value is None or type(value) in (int, float, bool): + return value + if type(value) is dict: + return { + key: ( + REDACTED + if _credential_key(".".join(_path + _field_path(key))) + else _safe_data( + item, + depth + 1, + _header_context=_header_context or _is_headers_field(key), + _path=_path + _field_path(key), + ) + ) + for key, item in _redact_header_records( + value, _header_context=_header_context, _path=_path + ).items() + if type(key) is str + } + if type(value) in (list, tuple): + # Includes standalone pairs and Node-style rawHeaders, but only in an + # explicit header field. Business tables are never promoted to headers. + if ( + _header_context + and value + and len(value) % 2 == 0 + and all(type(value[i]) is str for i in range(0, len(value), 2)) + ): + return [ + REDACTED + if i % 2 and _credential_key(value[i - 1]) + else _safe_data( + item, + depth + 1, + _header_context=_header_context, + _path=_path + (str(i),), + ) + for i, item in enumerate(value) + ] + return [ + _safe_data( + item, + depth + 1, + _header_context=_header_context, + _path=_path + (str(i),), + ) + for i, item in enumerate(value) + ] + return "[omitted runtime object]" + + +def _has_prefix(key: str, prefixes: tuple) -> bool: + return any(key == prefix or key.startswith(prefix + ".") for prefix in prefixes) + + +class _CrewAITraceConfig(TraceConfig): + """Keep the caller's core policy and cover CrewAI aliases/events as well. + + Applying projection at the mask boundary also protects context attributes + before samplers/processors see them, not just the wrapper's final setters. + """ + + def __init__(self, config: TraceConfig) -> None: + for field in fields(TraceConfig): + object.__setattr__(self, field.name, getattr(config, field.name)) + object.__setattr__(self, "_source", config) + + def mask(self, key: str, value: Any) -> Any: + hide_input = ( + self.hide_inputs or self.hide_input_text or self.hide_input_messages + ) + hide_output = ( + self.hide_outputs or self.hide_output_text or self.hide_output_messages + ) + if (hide_input and _has_prefix(key, _INPUT_ALIASES)) or ( + hide_output and _has_prefix(key, _OUTPUT_ALIASES) + ): + value = REDACTED + masked = self._source.mask(key, value) + if masked is None: + return None + if _credential_key(key): + return REDACTED + projected = _safe_data(masked, _header_context=_is_headers_field(key)) + # OTel attributes cannot carry dictionaries (nor lists of dictionaries). + if type(projected) is dict or ( + type(projected) is list + and any(type(item) in (dict, list) for item in projected) + ): + return json.dumps(projected) + return projected diff --git a/python/frameworks/crewai/traceai_crewai/_tracer.py b/python/frameworks/crewai/traceai_crewai/_tracer.py new file mode 100644 index 000000000..4a0ac1e4b --- /dev/null +++ b/python/frameworks/crewai/traceai_crewai/_tracer.py @@ -0,0 +1,83 @@ +"""CrewAI-local batch projection before core's per-attribute privacy policy.""" + +import wrapt +from fi_instrumentation import FITracer +from opentelemetry.trace import Link, SpanKind + +from ._privacy import _redact_header_records + + +def _project_attributes(attributes): + # Keep selector/value siblings together before core's scalar masking; leave + # unrelated values and core's lazy evaluation semantics unchanged. + return None if attributes is None else _redact_header_records(dict(attributes)) + + +class _CrewAISpan(wrapt.ObjectProxy): + """Retain core span lifecycle/error policy; project only attribute batches. + + No global FiSpan/SDK patches, shared selector state, or application mutation. + The complete batch is projected before core masks each scalar, regardless + of selector/value ordering in the mapping. + """ + + def __enter__(self): + self.__wrapped__.__enter__() + return self + + def __exit__(self, *args): + return self.__wrapped__.__exit__(*args) + + def set_attributes(self, attributes): + self.__wrapped__.set_attributes(_project_attributes(attributes)) + + def add_event(self, name, attributes=None, timestamp=None): + self.__wrapped__.add_event(name, _project_attributes(attributes), timestamp) + + def record_exception( + self, exception, attributes=None, timestamp=None, escaped=False + ): + self.__wrapped__.record_exception( + exception, _project_attributes(attributes), timestamp, escaped + ) + + def add_link(self, context, attributes=None): + self.__wrapped__.add_link(context, _project_attributes(attributes)) + + +class _CrewAITracer(FITracer): + def start_span( + self, + name, + context=None, + kind=SpanKind.INTERNAL, + attributes=None, + links=(), + start_time=None, + record_exception=True, + set_status_on_exception=True, + *, + fi_span_kind=None, + ): + # Creation attributes/links are projected before samplers/processors can + # observe them, not repaired after the raw span has already started. + projected_links = ( + None + if links is None + else [ + Link(link.context, _project_attributes(link.attributes)) + for link in links + ] + ) + span = super().start_span( + name=name, + context=context, + kind=kind, + attributes=_project_attributes(attributes), + links=projected_links, + start_time=start_time, + record_exception=record_exception, + set_status_on_exception=set_status_on_exception, + fi_span_kind=fi_span_kind, + ) + return _CrewAISpan(span) diff --git a/python/frameworks/crewai/traceai_crewai/_wrappers.py b/python/frameworks/crewai/traceai_crewai/_wrappers.py index 2ab524a76..aed1d7e0b 100644 --- a/python/frameworks/crewai/traceai_crewai/_wrappers.py +++ b/python/frameworks/crewai/traceai_crewai/_wrappers.py @@ -3,33 +3,18 @@ from inspect import signature from typing import Any, Callable, Iterator, List, Mapping, Optional, Tuple -from crewai import Agent from crewai.tools.tool_calling import ToolCalling -from fi_instrumentation import get_attributes_from_context, safe_json_dumps +from fi_instrumentation import get_attributes_from_context from fi_instrumentation.fi_types import FiSpanKindValues, SpanAttributes from opentelemetry import context as context_api from opentelemetry import trace as trace_api from opentelemetry.util.types import AttributeValue - -class SafeJSONEncoder(json.JSONEncoder): - """ - Safely encodes non-JSON-serializable objects. - """ - - def default(self, o: Any) -> Any: - try: - return super().default(o) - except TypeError: - if hasattr(o, "dict") and callable( - o.dict - ): # pydantic v1 models, e.g., from Cohere - return o.dict() - return repr(o) +from ._privacy import _credential_key, _safe_data def _flatten( - mapping: Optional[Mapping[str, Any]] + mapping: Optional[Mapping[str, Any]], ) -> Iterator[Tuple[str, AttributeValue]]: if not mapping: return @@ -51,64 +36,81 @@ def _flatten( yield key, value -def _get_input_value(method: Callable[..., Any], *args: Any, **kwargs: Any) -> str: - """ - Parses a method call's inputs into a JSON string. Ensures a consistent - output regardless of whether the those inputs are passed as positional or - keyword arguments. - """ +def _hide_input_content(tracer: Any) -> bool: + config = getattr(tracer, "_self_config", None) + return any( + getattr(config, key, False) is True + for key in ("hide_inputs", "hide_input_text", "hide_input_messages") + ) - # For typical class methods, the corresponding instance of inspect.Signature - # does not include the self parameter. However, the inspect.Signature - # instance for __call__ does include the self parameter. - tool = kwargs.get("calling") - if tool and isinstance(tool, ToolCalling): - return safe_json_dumps( - { - "tool_name": tool.tool_name, - "tool_args": tool.arguments, - }, - cls=SafeJSONEncoder, - ) +def _safe_context_attributes() -> dict: + return { + key: ("__REDACTED__" if _credential_key(key) else _safe_data(value)) + for key, value in get_attributes_from_context() + } - if args and args[0] and isinstance(args[0], Agent): - return safe_json_dumps( - { - "role": args[0].role, - "goal": args[0].goal, - "backstory": args[0].backstory, - }, - cls=SafeJSONEncoder, - ) - method_signature = signature(method) - first_parameter_name = next(iter(method_signature.parameters), None) - signature_contains_self_parameter = first_parameter_name in ["self"] - bound_arguments = method_signature.bind( - *( - [ - None - ] # the value bound to the method's self argument is discarded below, so pass None - if signature_contains_self_parameter - else [] # no self parameter, so no need to pass a value - ), - *args, - **kwargs, - ) - return safe_json_dumps( +def _safe_output(value: Any) -> Any: + cls = type(value) + if cls.__module__.startswith("crewai.") and cls.__name__ in ( + "TaskOutput", + "CrewOutput", + ): + content = value.json_dict if type(value.json_dict) is dict else value.raw + return _safe_data(content) + return _safe_data(value) + + +def _bound_call( + method: Callable[..., Any], args: Tuple[Any, ...], kwargs: Mapping[str, Any] +) -> dict: + try: + sig = signature(method) + needs_self = next(iter(sig.parameters), None) == "self" + return dict( + sig.bind_partial( + *((None,) if needs_self else ()), *args, **kwargs + ).arguments + ) + except (TypeError, ValueError): + return {} + + +def _task_input( + method: Callable[..., Any], + task: Any, + args: Tuple[Any, ...], + kwargs: Mapping[str, Any], +) -> str: + bound = _bound_call(method, args, kwargs) + return json.dumps( { - **{ - argument_name: argument_value - for argument_name, argument_value in bound_arguments.arguments.items() - if argument_name not in ["self", "kwargs"] - }, - **bound_arguments.arguments.get("kwargs", {}), - }, - cls=SafeJSONEncoder, + "description": _safe_data(getattr(task, "description", None)), + "expected_output": _safe_data(getattr(task, "expected_output", None)), + "context": _safe_data(bound.get("context")), + } ) +def _get_input_value(method: Callable[..., Any], *args: Any, **kwargs: Any) -> str: + """Project only ToolCalling's documented application fields. + + ToolUsage also receives the runtime tool and parsing state: none of those + arguments may enter a generic bound-arguments/object serialization fallback. + """ + bound = _bound_call(method, args, kwargs) + calling = bound.get("calling", kwargs.get("calling")) + if isinstance(calling, ToolCalling): + return json.dumps( + { + "tool_name": _safe_data(calling.tool_name), + "tool_args": _safe_data(calling.arguments), + } + ) + return json.dumps({}) + + class _ExecuteCoreWrapper: def __init__(self, tracer: trace_api.Tracer) -> None: self._tracer = tracer @@ -132,10 +134,10 @@ def __call__( _flatten( { GEN_AI_SPAN_KIND: FiSpanKindValues.AGENT, - SpanAttributes.INPUT_VALUE: _get_input_value( - wrapped, - *args, - **kwargs, + SpanAttributes.INPUT_VALUE: ( + "__REDACTED__" + if _hide_input_content(self._tracer) + else _task_input(wrapped, instance, args, kwargs) ), } ) @@ -143,36 +145,34 @@ def __call__( record_exception=False, set_status_on_exception=False, ) as span: - agent = args[0] if args else None - crew = agent.crew if agent else None + bound = _bound_call(wrapped, args, kwargs) + agent = bound.get("agent", args[0] if args else None) + crew = getattr(agent, "crew", None) if agent is not None else None task = instance - span.set_attribute( - SpanAttributes.INPUT_VALUE, - safe_json_dumps(_prepare_args_kwargs(args=args, **kwargs)), - ) - if crew: span.set_attribute("crew_key", crew.key) span.set_attribute("crew_id", str(crew.id)) span.set_attribute("task_key", task.key) span.set_attribute("task_id", str(task.id)) - if crew and crew.share_crew: - span.set_attribute("formatted_description", task.description) - span.set_attribute("formatted_expected_output", task.expected_output) + if crew and crew.share_crew and not _hide_input_content(self._tracer): + span.set_attribute( + "formatted_description", _safe_data(task.description) + ) + span.set_attribute( + "formatted_expected_output", _safe_data(task.expected_output) + ) try: response = wrapped(*args, **kwargs) - except Exception as exception: - span.set_status( - trace_api.Status(trace_api.StatusCode.ERROR, str(exception)) - ) - span.record_exception(exception) + except Exception: + # Provider exceptions may include credentials or request configuration. + span.set_status(trace_api.StatusCode.ERROR) raise span.set_status(trace_api.StatusCode.OK) - span.set_attribute(OUTPUT_VALUE, safe_json_dumps(response)) - span.set_attributes(dict(get_attributes_from_context())) + span.set_attribute(OUTPUT_VALUE, json.dumps(_safe_output(response))) + span.set_attributes(_safe_context_attributes()) return response @@ -206,66 +206,80 @@ def __call__( inputs = kwargs.get("inputs", None) or (args[0] if args else None) span.set_attribute("crew_key", crew.key) span.set_attribute("crew_id", str(crew.id)) - span.set_attribute("crew_inputs", json.dumps(inputs) if inputs else "") + # Only explicit kickoff inputs are content; checkpoint/file objects are runtime state. span.set_attribute( SpanAttributes.INPUT_VALUE, - safe_json_dumps(_prepare_args_kwargs(args=args, **kwargs)), - ) - span.set_attribute( - "crew_agents", - json.dumps( - [ - { - "key": agent.key, - "id": str(agent.id), - "role": agent.role, - "goal": agent.goal, - "backstory": agent.backstory, - "verbose?": agent.verbose, - "max_iter": agent.max_iter, - "max_rpm": agent.max_rpm, - "i18n": agent.i18n.prompt_file, - "delegation_enabled": agent.allow_delegation, - "tools_names": [ - tool.name.casefold() for tool in agent.tools or [] - ], - } - for agent in crew.agents - ] - ), - ) - span.set_attribute( - "crew_tasks", - json.dumps( - [ - { - "id": str(task.id), - "description": task.description, - "expected_output": task.expected_output, - "async_execution?": task.async_execution, - "human_input?": task.human_input, - "agent_role": task.agent.role if task.agent else "None", - "agent_key": task.agent.key if task.agent else None, - # CrewAI defaults Task.context to a NOT_SPECIFIED - # sentinel that is truthy but not iterable, so this - # must guard on type before iterating. - "context": ( - [ - context_task.description - for context_task in task.context - ] - if isinstance(task.context, (list, tuple)) - and task.context - else None - ), - "tools_names": [ - tool.name.casefold() for tool in task.tools or [] - ], - } - for task in crew.tasks - ] + ( + "__REDACTED__" + if _hide_input_content(self._tracer) + else json.dumps(_safe_data(inputs)) ), ) + if not _hide_input_content(self._tracer): + span.set_attribute("crew_inputs", json.dumps(_safe_data(inputs))) + if not _hide_input_content(self._tracer): + span.set_attribute( + "crew_agents", + json.dumps( + _safe_data( + [ + { + "key": agent.key, + "id": str(agent.id), + "role": agent.role, + "goal": agent.goal, + "backstory": agent.backstory, + "verbose?": agent.verbose, + "max_iter": agent.max_iter, + "max_rpm": agent.max_rpm, + "delegation_enabled": agent.allow_delegation, + "tools_names": [ + tool.name.casefold() + for tool in agent.tools or [] + ], + } + for agent in crew.agents + ] + ) + ), + ) + span.set_attribute( + "crew_tasks", + json.dumps( + _safe_data( + [ + { + "id": str(task.id), + "description": task.description, + "expected_output": task.expected_output, + "async_execution?": task.async_execution, + "human_input?": task.human_input, + "agent_role": ( + task.agent.role if task.agent else "None" + ), + "agent_key": task.agent.key if task.agent else None, + # CrewAI defaults Task.context to a NOT_SPECIFIED + # sentinel that is truthy but not iterable, so this + # must guard on type before iterating. + "context": ( + [ + context_task.description + for context_task in task.context + ] + if isinstance(task.context, (list, tuple)) + and task.context + else None + ), + "tools_names": [ + tool.name.casefold() + for tool in task.tools or [] + ], + } + for task in crew.tasks + ] + ) + ), + ) try: crew_output = wrapped(*args, **kwargs) usage_metrics = crew.usage_metrics @@ -273,7 +287,9 @@ def __call__( if ( prompt_tokens := usage_metrics.get("prompt_tokens") ) is not None: - span.set_attribute(GEN_AI_USAGE_INPUT_TOKENS, int(prompt_tokens)) + span.set_attribute( + GEN_AI_USAGE_INPUT_TOKENS, int(prompt_tokens) + ) if ( completion_tokens := usage_metrics.get("completion_tokens") ) is not None: @@ -294,20 +310,13 @@ def __call__( GEN_AI_USAGE_TOTAL_TOKENS, usage_metrics.total_tokens ) - except Exception as exception: - span.set_status( - trace_api.Status(trace_api.StatusCode.ERROR, str(exception)) - ) - span.record_exception(exception) + except Exception: + # Provider exceptions may include credentials or request configuration. + span.set_status(trace_api.StatusCode.ERROR) raise span.set_status(trace_api.StatusCode.OK) - if crew_output_dict := crew_output.to_dict(): - - span.set_attribute(OUTPUT_VALUE, json.dumps(crew_output_dict)) - span.set_attribute(OUTPUT_MIME_TYPE, "application/json") - else: - span.set_attribute(OUTPUT_VALUE, str(crew_output)) - span.set_attributes(dict(get_attributes_from_context())) + span.set_attribute(OUTPUT_VALUE, json.dumps(_safe_output(crew_output))) + span.set_attributes(_safe_context_attributes()) return crew_output @@ -334,10 +343,10 @@ def __call__( _flatten( { GEN_AI_SPAN_KIND: FiSpanKindValues.TOOL, - SpanAttributes.INPUT_VALUE: _get_input_value( - wrapped, - *args, - **kwargs, + SpanAttributes.INPUT_VALUE: ( + "__REDACTED__" + if _hide_input_content(self._tracer) + else _get_input_value(wrapped, *args, **kwargs) ), } ) @@ -345,62 +354,24 @@ def __call__( record_exception=False, set_status_on_exception=False, ) as span: - tool = kwargs.get("tool") - tool_name = "" - if tool: - tool_name = tool.name - span.set_attribute("function_calling_llm", instance.function_calling_llm) - span.set_attribute(SpanAttributes.GEN_AI_TOOL_NAME, tool_name) - - span.set_attribute( - SpanAttributes.INPUT_VALUE, - safe_json_dumps(_prepare_args_kwargs(args=args, **kwargs)), - ) + bound = _bound_call(wrapped, args, kwargs) + tool = bound.get("tool", kwargs.get("tool")) + tool_name = getattr(tool, "name", "") if tool is not None else "" + if type(tool_name) is str: + span.set_attribute(SpanAttributes.GEN_AI_TOOL_NAME, tool_name) try: response = wrapped(*args, **kwargs) - except Exception as exception: - span.set_status( - trace_api.Status(trace_api.StatusCode.ERROR, str(exception)) - ) - span.record_exception(exception) + except Exception: + # Provider exceptions may include credentials or request configuration. + span.set_status(trace_api.StatusCode.ERROR) raise span.set_status(trace_api.StatusCode.OK) - span.set_attribute(OUTPUT_VALUE, safe_json_dumps(response)) - span.set_attribute(OUTPUT_VALUE, safe_json_dumps(response)) - span.set_attributes(dict(get_attributes_from_context())) + span.set_attribute(OUTPUT_VALUE, json.dumps(_safe_output(response))) + span.set_attributes(_safe_context_attributes()) return response -def _convert_to_dict(obj: Any) -> Any: - """ - Recursively converts objects to dicts if they have a to_dict method or __dict__ attribute. - Handles nested lists and dictionaries. - """ - if hasattr(obj, "to_dict") and callable(obj.to_dict): - return obj.to_dict() - elif hasattr(obj, "__dict__"): - return obj.__dict__ - elif isinstance(obj, dict): - return {key: _convert_to_dict(value) for key, value in obj.items()} - elif isinstance(obj, (list, tuple)): - return [_convert_to_dict(item) for item in obj] - return obj - - -def _prepare_args_kwargs(args: Tuple[Any, ...], **kwargs: Any) -> dict: - """ - Prepares args and kwargs for JSON serialization by converting objects with to_dict - method to dictionaries. Maintains the original argument structure. - """ - processed_args = _convert_to_dict(args) - processed_kwargs = _convert_to_dict(kwargs) - return { - "args": processed_args, - **processed_kwargs, # Unpacks the processed kwargs directly - } - - INPUT_VALUE = SpanAttributes.INPUT_VALUE GEN_AI_SPAN_KIND = SpanAttributes.GEN_AI_SPAN_KIND OUTPUT_VALUE = SpanAttributes.OUTPUT_VALUE diff --git a/python/tests/test_framework_anthropic.py b/python/tests/test_framework_anthropic.py index 23e09b62e..9403c09c6 100644 --- a/python/tests/test_framework_anthropic.py +++ b/python/tests/test_framework_anthropic.py @@ -18,7 +18,7 @@ from fi_instrumentation.otel import register from fi_instrumentation.fi_types import ( EvalName, - EvalSpanKind, + EvalSpanKind, EvalTag, EvalTagType, ProjectType, @@ -26,11 +26,19 @@ ) from fi_instrumentation.instrumentation.context_attributes import using_attributes +try: + import anthropic.resources.completions # noqa: F401 + + _HAS_LEGACY_COMPLETIONS = True +except ImportError: + # anthropic>=1.0 removed the legacy Completions API entirely. + _HAS_LEGACY_COMPLETIONS = False + class TestAnthropicFramework: """Test Anthropic framework instrumentation.""" - @pytest.fixture(autouse=True) + @pytest.fixture(autouse=True) def setup_trace_provider(self): """Set up trace provider for Anthropic testing.""" eval_tags = [ @@ -43,7 +51,7 @@ def setup_trace_provider(self): custom_eval_name="anthropic_test_eval" ) ] - + with patch('fi_instrumentation.otel.check_custom_eval_config_exists') as mock_check: mock_check.return_value = False self.trace_provider = register( @@ -54,12 +62,12 @@ def setup_trace_provider(self): verbose=False ) yield - + @pytest.fixture def mock_anthropic_requests(self): """Mock Anthropic HTTP requests to avoid real API calls.""" from anthropic.types import Message, Usage, TextBlock - + # Create a proper Message object to return mock_usage = Usage(input_tokens=10, output_tokens=12) mock_content = [TextBlock(type="text", text="This is a test response from Claude.")] @@ -73,32 +81,55 @@ def mock_anthropic_requests(self): stop_sequence=None, usage=mock_usage ) - + with patch('anthropic.resources.messages.Messages.create') as mock_create: mock_create.return_value = mock_message yield mock_create - + def test_anthropic_import(self): """Test that we can import the Anthropic instrumentor.""" assert AnthropicInstrumentor is not None - + # Test basic instantiation instrumentor = AnthropicInstrumentor() assert instrumentor is not None - + + def test_instrument_without_legacy_completions_api(self): + """anthropic>=1.0 removed anthropic.resources.completions entirely + (the legacy Text Completions API). instrument() must degrade to + wrapping only Messages instead of raising ImportError and + instrumenting nothing - regression test for that failure mode. + """ + instrumentor = AnthropicInstrumentor() + + with patch.dict(sys.modules, {"anthropic.resources.completions": None}): + instrumentor.instrument(tracer_provider=self.trace_provider) + + try: + from anthropic.resources.messages import Messages + + assert type(Messages.create).__name__ == 'BoundFunctionWrapper' + assert instrumentor._original_messages_create is not None + assert instrumentor._original_completions_create is None + assert instrumentor._original_async_completions_create is None + finally: + instrumentor.uninstrument() + + assert type(Messages.create).__name__ != 'BoundFunctionWrapper' + def test_anthropic_basic_instrumentation(self, mock_anthropic_requests): """Test basic Anthropic messages instrumentation.""" # Initialize instrumentor instrumentor = AnthropicInstrumentor() instrumentor.instrument(tracer_provider=self.trace_provider) - + try: # Create client and make request client = anthropic.Anthropic(api_key="test-key") - + with using_attributes( session_id="test-session-123", - user_id="test-user-456", + user_id="test-user-456", metadata={"test_type": "instrumentation", "framework": "anthropic"}, tags=["anthropic", "test", "claude-3-sonnet"] ): @@ -109,24 +140,24 @@ def test_anthropic_basic_instrumentation(self, mock_anthropic_requests): {"role": "user", "content": "Write a short poem about AI testing."} ] ) - + # Verify the create method was called assert mock_anthropic_requests.called - + # Verify response structure (directly from mock Message object) assert response.content[0].text == "This is a test response from Claude." assert response.model == "claude-3-7-sonnet-latest" assert response.usage.input_tokens == 10 assert response.usage.output_tokens == 12 - + finally: # Clean up instrumentor.uninstrument() - + def test_anthropic_tool_use(self, mock_anthropic_requests): """Test Anthropic instrumentation with tool use.""" from anthropic.types import Message, Usage, ToolUseBlock - + # Set up tool use response mock_usage = Usage(input_tokens=15, output_tokens=8) mock_content = [ToolUseBlock( @@ -146,13 +177,13 @@ def test_anthropic_tool_use(self, mock_anthropic_requests): usage=mock_usage ) mock_anthropic_requests.return_value = mock_tool_message - + instrumentor = AnthropicInstrumentor() instrumentor.instrument(tracer_provider=self.trace_provider) - + try: client = anthropic.Anthropic(api_key="test-key") - + tools = [{ "name": "get_weather", "description": "Get current weather for a location", @@ -164,7 +195,7 @@ def test_anthropic_tool_use(self, mock_anthropic_requests): "required": ["location"] } }] - + with using_attributes(session_id="tool-test-session"): response = client.messages.create( model="claude-3-7-sonnet-latest", @@ -172,116 +203,116 @@ def test_anthropic_tool_use(self, mock_anthropic_requests): tools=tools, messages=[{"role": "user", "content": "What's the weather in San Francisco?"}] ) - + # Verify tool use was captured (directly from mock Message object) assert response.content[0].name == "get_weather" assert response.content[0].input["location"] == "San Francisco" assert mock_anthropic_requests.called - + finally: instrumentor.uninstrument() - + @pytest.mark.asyncio async def test_anthropic_async_instrumentation(self, mock_anthropic_requests): """Test Anthropic async instrumentation setup.""" # For async, let's just test that the instrumentor can handle async calls instrumentor = AnthropicInstrumentor() instrumentor.instrument(tracer_provider=self.trace_provider) - + try: client = anthropic.AsyncAnthropic(api_key="test-key") - + # Test that the async client is properly instrumented assert hasattr(client.messages, 'create') - + # Test context attributes work with using_attributes(session_id="async-test-session"): # Just verify the context is set up correctly without making a real call pass - + finally: instrumentor.uninstrument() - + + @pytest.mark.skipif( + not _HAS_LEGACY_COMPLETIONS, + reason="anthropic>=1.0 removed the legacy Completions API entirely", + ) def test_anthropic_completions_legacy(self, mock_anthropic_requests): """Test Anthropic completions (legacy) instrumentation setup.""" # Don't need to modify mock for this test - just testing setup - + instrumentor = AnthropicInstrumentor() instrumentor.instrument(tracer_provider=self.trace_provider) - + try: client = anthropic.Anthropic(api_key="test-key") - + # Verify the client is properly instrumented for completions assert hasattr(client.completions, 'create') - + # Test context attributes work with completions with using_attributes(session_id="completions-test-session"): # Just verify completions parameter handling without making real calls pass - + finally: instrumentor.uninstrument() - + def test_anthropic_streaming_setup(self, mock_anthropic_requests): """Test Anthropic streaming instrumentation setup.""" instrumentor = AnthropicInstrumentor() instrumentor.instrument(tracer_provider=self.trace_provider) - + try: client = anthropic.Anthropic(api_key="test-key") - + # Verify the client is properly instrumented for streaming assert hasattr(client.messages, 'create') - + # Test context attributes work with streaming with using_attributes(session_id="streaming-test-session"): # Just verify streaming parameter handling without making real calls pass - + finally: instrumentor.uninstrument() - + def test_anthropic_error_handling(self, mock_anthropic_requests): """Test Anthropic instrumentation with error scenarios.""" instrumentor = AnthropicInstrumentor() instrumentor.instrument(tracer_provider=self.trace_provider) - + try: client = anthropic.Anthropic(api_key="test-key") - + # Verify the client is properly instrumented even for error cases assert hasattr(client.messages, 'create') - + # Test context attributes work in error scenarios with using_attributes(session_id="error-test-session"): # Just verify the instrumentation is set up correctly pass - + finally: instrumentor.uninstrument() - + def test_instrumentor_uninstrumentation(self): - """Test that uninstrumentation properly restores original behavior.""" + """Messages must be restored on every SDK, including without completions.""" from anthropic.resources.messages import Messages - from anthropic.resources.completions import Completions - + if _HAS_LEGACY_COMPLETIONS: + from anthropic.resources.completions import Completions + original_completions_create = Completions.create + instrumentor = AnthropicInstrumentor() - - # Store original methods original_messages_create = Messages.create - original_completions_create = Completions.create - - # Instrument + original_count_tokens = Messages.count_tokens instrumentor.instrument(tracer_provider=self.trace_provider) - - # Methods should be wrapped (different types) assert type(Messages.create).__name__ == 'BoundFunctionWrapper' - assert type(Completions.create).__name__ == 'BoundFunctionWrapper' - - # Uninstrument + assert type(Messages.count_tokens).__name__ == 'BoundFunctionWrapper' + if _HAS_LEGACY_COMPLETIONS: + assert type(Completions.create).__name__ == 'BoundFunctionWrapper' instrumentor.uninstrument() - - # Methods should be restored (back to functions) assert Messages.create == original_messages_create - assert Completions.create == original_completions_create \ No newline at end of file + assert Messages.count_tokens == original_count_tokens + if _HAS_LEGACY_COMPLETIONS: + assert Completions.create == original_completions_create diff --git a/python/tests/test_framework_crewai.py b/python/tests/test_framework_crewai.py index 529c89748..c21220b3b 100644 --- a/python/tests/test_framework_crewai.py +++ b/python/tests/test_framework_crewai.py @@ -3,7 +3,7 @@ Tests the instrumentation of CrewAI components: - Task._execute_core (AGENT spans) -- Crew.kickoff (CHAIN spans) +- Crew.kickoff (CHAIN spans) - ToolUsage._use (TOOL spans) """ @@ -20,14 +20,12 @@ from traceai_crewai import CrewAIInstrumentor from traceai_crewai._wrappers import ( - SafeJSONEncoder, _ExecuteCoreWrapper, _KickoffWrapper, _ToolUseWrapper, - _convert_to_dict, _flatten, _get_input_value, - _prepare_args_kwargs, + _safe_data, ) @@ -48,24 +46,32 @@ def test_instrument_with_defaults(self, mock_wrap, mock_import): mock_task_module = Mock() mock_crew_module = Mock() mock_tool_module = Mock() - + mock_import.side_effect = [ mock_task_module, mock_crew_module, mock_tool_module, ] - + instrumentor = CrewAIInstrumentor() instrumentor._instrument(tracer_provider=mock_tracer_provider) - + # Should wrap all three methods assert mock_wrap.call_count == 3 - + # Verify correct module and method names calls = mock_wrap.call_args_list - assert any("crewai" in str(call) and "Task._execute_core" in str(call) for call in calls) - assert any("crewai" in str(call) and "Crew.kickoff" in str(call) for call in calls) - assert any("crewai.tools.tool_usage" in str(call) and "ToolUsage._use" in str(call) for call in calls) + assert any( + "crewai" in str(call) and "Task._execute_core" in str(call) + for call in calls + ) + assert any( + "crewai" in str(call) and "Crew.kickoff" in str(call) for call in calls + ) + assert any( + "crewai.tools.tool_usage" in str(call) and "ToolUsage._use" in str(call) + for call in calls + ) @patch("traceai_crewai.import_module") def test_uninstrument(self, mock_import): @@ -73,26 +79,26 @@ def test_uninstrument(self, mock_import): mock_task_module = Mock() mock_crew_module = Mock() mock_tool_module = Mock() - + mock_import.side_effect = [ mock_task_module, mock_crew_module, mock_tool_module, ] - + instrumentor = CrewAIInstrumentor() - + # Set up original methods original_execute = Mock() original_kickoff = Mock() original_tool_use = Mock() - + instrumentor._original_execute_core = original_execute instrumentor._original_kickoff = original_kickoff instrumentor._original_tool_use = original_tool_use - + instrumentor._uninstrument() - + # Verify methods are restored assert mock_task_module.Task._execute_core == original_execute assert mock_crew_module.Crew.kickoff == original_kickoff @@ -114,11 +120,11 @@ def setup_method(self): def test_suppress_instrumentation(self): """Test that instrumentation is suppressed when flag is set.""" - with patch.object(context_api, 'get_value', return_value=True): + with patch.object(context_api, "get_value", return_value=True): wrapped_func = Mock(return_value="result") - + result = self.wrapper(wrapped_func, None, (), {}) - + assert result == "result" wrapped_func.assert_called_once_with() self.mock_tracer.start_as_current_span.assert_not_called() @@ -131,18 +137,18 @@ def test_task_execution_basic(self): mock_task = Mock() mock_task.key = "task_key_123" mock_task.id = "task_id_456" - + wrapped_func = Mock(__name__="_execute_core", return_value="task_result") - + result = self.wrapper(wrapped_func, mock_task, (mock_agent,), {}) - + assert result == "task_result" - + # Verify span creation self.mock_tracer.start_as_current_span.assert_called_once() call_args = self.mock_tracer.start_as_current_span.call_args assert "Mock._execute_core" in call_args[0][0] - + # Verify attributes set self.mock_span.set_attribute.assert_any_call("task_key", "task_key_123") self.mock_span.set_attribute.assert_any_call("task_id", "task_id_456") @@ -154,27 +160,31 @@ def test_task_execution_with_crew(self): mock_crew.key = "crew_key_789" mock_crew.id = "crew_id_101" mock_crew.share_crew = True - + mock_agent = Mock() mock_agent.crew = mock_crew - + mock_task = Mock() mock_task.key = "task_key_123" mock_task.id = "task_id_456" mock_task.description = "Test task description" mock_task.expected_output = "Expected output format" - + wrapped_func = Mock(__name__="_execute_core", return_value="task_result") - + result = self.wrapper(wrapped_func, mock_task, (mock_agent,), {}) - + assert result == "task_result" - + # Verify crew attributes set self.mock_span.set_attribute.assert_any_call("crew_key", "crew_key_789") self.mock_span.set_attribute.assert_any_call("crew_id", "crew_id_101") - self.mock_span.set_attribute.assert_any_call("formatted_description", "Test task description") - self.mock_span.set_attribute.assert_any_call("formatted_expected_output", "Expected output format") + self.mock_span.set_attribute.assert_any_call( + "formatted_description", "Test task description" + ) + self.mock_span.set_attribute.assert_any_call( + "formatted_expected_output", "Expected output format" + ) def test_task_execution_error_handling(self): """Test error handling in task execution.""" @@ -183,39 +193,44 @@ def test_task_execution_error_handling(self): mock_task = Mock() mock_task.key = "task_key" mock_task.id = "task_id" - + test_exception = Exception("Task execution failed") wrapped_func = Mock(__name__="_execute_core", side_effect=test_exception) - + with pytest.raises(Exception, match="Task execution failed"): self.wrapper(wrapped_func, mock_task, (mock_agent,), {}) - + # Verify error handling self.mock_span.set_status.assert_called_once() call_args = self.mock_span.set_status.call_args[0][0] - assert call_args.status_code == StatusCode.ERROR - assert call_args.description == "Task execution failed" - self.mock_span.record_exception.assert_called_once_with(test_exception) + assert call_args == StatusCode.ERROR + self.mock_span.record_exception.assert_not_called() + assert "Task execution failed" not in str(self.mock_span.mock_calls) @patch("traceai_crewai._wrappers.get_attributes_from_context") def test_context_attributes(self, mock_get_attrs): """Test that context attributes are added to spans.""" - mock_get_attrs.return_value = {"session_id": "test_session", "user_id": "test_user"} - + mock_get_attrs.return_value = [ + ("session_id", "test_session"), + ("user_id", "test_user"), + ] + mock_agent = Mock() mock_agent.crew = None mock_task = Mock() mock_task.key = "task_key" mock_task.id = "task_id" - + wrapped_func = Mock(__name__="_execute_core", return_value="result") - + self.wrapper(wrapped_func, mock_task, (mock_agent,), {}) - + # Verify context attributes are set self.mock_span.set_attributes.assert_called_once() call_args = self.mock_span.set_attributes.call_args[0][0] - assert call_args["session_id"] == "test_session" + # Session identifiers can be session credentials under the structural + # policy; unrelated user/business identifiers remain available. + assert call_args["session_id"] == "__REDACTED__" assert call_args["user_id"] == "test_user" @@ -241,23 +256,27 @@ def test_crew_kickoff_basic(self): mock_crew.id = "crew_id_456" mock_crew.agents = [] mock_crew.tasks = [] - mock_crew.usage_metrics = {"prompt_tokens": 100, "completion_tokens": 50, "total_tokens": 150} - + mock_crew.usage_metrics = { + "prompt_tokens": 100, + "completion_tokens": 50, + "total_tokens": 150, + } + # Mock crew output mock_output = Mock() mock_output.to_dict.return_value = {"result": "crew completed successfully"} - + wrapped_func = Mock(__name__="kickoff", return_value=mock_output) - + result = self.wrapper(wrapped_func, mock_crew, (), {"inputs": {"task": "test"}}) - + assert result == mock_output - + # Verify span creation self.mock_tracer.start_as_current_span.assert_called_once() call_args = self.mock_tracer.start_as_current_span.call_args assert "Crew.kickoff" in call_args[0][0] - + # Verify basic attributes self.mock_span.set_attribute.assert_any_call("crew_key", "crew_key_123") self.mock_span.set_attribute.assert_any_call("crew_id", "crew_id_456") @@ -281,7 +300,7 @@ def test_crew_kickoff_with_agents_and_tasks(self): mock_tool.name = Mock() mock_tool.name.casefold.return_value = "search_tool" mock_agent.tools = [mock_tool] - + # Mock task mock_task = Mock() mock_task.id = "task_id_1" @@ -292,7 +311,7 @@ def test_crew_kickoff_with_agents_and_tasks(self): mock_task.agent = mock_agent mock_task.context = None mock_task.tools = [] - + # Mock crew mock_crew = Mock() mock_crew.__class__.__name__ = "Crew" @@ -300,17 +319,21 @@ def test_crew_kickoff_with_agents_and_tasks(self): mock_crew.id = "crew_id" mock_crew.agents = [mock_agent] mock_crew.tasks = [mock_task] - mock_crew.usage_metrics = {"prompt_tokens": 200, "completion_tokens": 100, "total_tokens": 300} - + mock_crew.usage_metrics = { + "prompt_tokens": 200, + "completion_tokens": 100, + "total_tokens": 300, + } + mock_output = Mock() mock_output.to_dict.return_value = {"result": "success"} - + wrapped_func = Mock(__name__="kickoff", return_value=mock_output) - + result = self.wrapper(wrapped_func, mock_crew, (), {}) - + assert result == mock_output - + # Verify token metrics are set self.mock_span.set_attribute.assert_any_call( SpanAttributes.GEN_AI_USAGE_INPUT_TOKENS, 200 @@ -330,21 +353,21 @@ def test_crew_kickoff_with_new_usage_metrics(self): mock_crew.id = "crew_id" mock_crew.agents = [] mock_crew.tasks = [] - + # New format usage metrics (object with attributes) mock_usage = Mock() mock_usage.prompt_tokens = 150 mock_usage.completion_tokens = 75 mock_usage.total_tokens = 225 mock_crew.usage_metrics = mock_usage - + mock_output = Mock() mock_output.to_dict.return_value = {"result": "success"} - + wrapped_func = Mock(__name__="kickoff", return_value=mock_output) - + self.wrapper(wrapped_func, mock_crew, (), {}) - + # Verify new format token metrics self.mock_span.set_attribute.assert_any_call( SpanAttributes.GEN_AI_USAGE_INPUT_TOKENS, 150 @@ -363,20 +386,20 @@ def test_crew_kickoff_error_handling(self): mock_crew.key = "crew_key" mock_crew.id = "crew_id" mock_crew.agents = [] # Empty list to avoid iteration issues - mock_crew.tasks = [] # Empty list to avoid iteration issues - + mock_crew.tasks = [] # Empty list to avoid iteration issues + test_exception = Exception("Crew kickoff failed") wrapped_func = Mock(__name__="kickoff", side_effect=test_exception) - + with pytest.raises(Exception, match="Crew kickoff failed"): self.wrapper(wrapped_func, mock_crew, (), {}) - + # Verify error handling self.mock_span.set_status.assert_called_once() call_args = self.mock_span.set_status.call_args[0][0] - assert call_args.status_code == StatusCode.ERROR - assert call_args.description == "Crew kickoff failed" - self.mock_span.record_exception.assert_called_once_with(test_exception) + assert call_args == StatusCode.ERROR + self.mock_span.record_exception.assert_not_called() + assert "Crew kickoff failed" not in str(self.mock_span.mock_calls) def test_crew_output_without_to_dict(self): """Test crew output that doesn't have to_dict method.""" @@ -387,19 +410,22 @@ def test_crew_output_without_to_dict(self): mock_crew.agents = [] mock_crew.tasks = [] mock_crew.usage_metrics = {} - + # Mock output without to_dict method mock_output = Mock() mock_output.to_dict.return_value = None - + wrapped_func = Mock(__name__="kickoff", return_value=mock_output) - + result = self.wrapper(wrapped_func, mock_crew, (), {}) - + assert result == mock_output - - # Should set string representation - self.mock_span.set_attribute.assert_any_call("output.value", str(mock_output)) + + # Unknown runtime outputs must not call serializers or repr. + self.mock_span.set_attribute.assert_any_call( + "output.value", json.dumps("[omitted runtime object]") + ) + mock_output.to_dict.assert_not_called() class _NotSpecifiedStub: @@ -508,24 +534,29 @@ def test_tool_use_basic(self): mock_instance = Mock() mock_instance.__class__.__name__ = "ToolUsage" mock_instance.function_calling_llm = "gpt-4" - + # Mock tool mock_tool = Mock() mock_tool.name = "search_tool" - - wrapped_func = Mock(__name__="_use", return_value={"status": "success", "result": "found data"}) - + + wrapped_func = Mock( + __name__="_use", return_value={"status": "success", "result": "found data"} + ) + result = self.wrapper(wrapped_func, mock_instance, (), {"tool": mock_tool}) - + assert result == {"status": "success", "result": "found data"} - + # Verify span creation self.mock_tracer.start_as_current_span.assert_called_once() call_args = self.mock_tracer.start_as_current_span.call_args assert "ToolUsage._use" in call_args[0][0] - + # Verify tool attributes - self.mock_span.set_attribute.assert_any_call("function_calling_llm", "gpt-4") + assert not any( + call.args[0] == "function_calling_llm" + for call in self.mock_span.set_attribute.call_args_list + ) self.mock_span.set_attribute.assert_any_call( SpanAttributes.GEN_AI_TOOL_NAME, "search_tool" ) @@ -535,48 +566,50 @@ def test_tool_use_without_tool(self): mock_instance = Mock() mock_instance.__class__.__name__ = "ToolUsage" mock_instance.function_calling_llm = "gpt-3.5-turbo" - + wrapped_func = Mock(__name__="_use", return_value="no tool result") - + result = self.wrapper(wrapped_func, mock_instance, (), {}) - + assert result == "no tool result" - + # Should set empty tool name - self.mock_span.set_attribute.assert_any_call(SpanAttributes.GEN_AI_TOOL_NAME, "") + self.mock_span.set_attribute.assert_any_call( + SpanAttributes.GEN_AI_TOOL_NAME, "" + ) def test_tool_use_error_handling(self): """Test error handling in tool usage.""" mock_instance = Mock() mock_instance.__class__.__name__ = "ToolUsage" mock_instance.function_calling_llm = "gpt-4" - + test_exception = Exception("Tool execution failed") wrapped_func = Mock(__name__="_use", side_effect=test_exception) - + with pytest.raises(Exception, match="Tool execution failed"): self.wrapper(wrapped_func, mock_instance, (), {}) - + # Verify error handling self.mock_span.set_status.assert_called_once() call_args = self.mock_span.set_status.call_args[0][0] - assert call_args.status_code == StatusCode.ERROR - assert call_args.description == "Tool execution failed" - self.mock_span.record_exception.assert_called_once_with(test_exception) + assert call_args == StatusCode.ERROR + self.mock_span.record_exception.assert_not_called() + assert "Tool execution failed" not in str(self.mock_span.mock_calls) @patch("traceai_crewai._wrappers.get_attributes_from_context") def test_tool_context_attributes(self, mock_get_attrs): """Test that context attributes are added to tool spans.""" - mock_get_attrs.return_value = {"session_id": "tool_session"} - + mock_get_attrs.return_value = [("session_id", "tool_session")] + mock_instance = Mock() mock_instance.__class__.__name__ = "ToolUsage" mock_instance.function_calling_llm = "gpt-4" - + wrapped_func = Mock(__name__="_use", return_value="result") - + self.wrapper(wrapped_func, mock_instance, (), {}) - + # Verify context attributes are set self.mock_span.set_attributes.assert_called_once() @@ -584,85 +617,71 @@ def test_tool_context_attributes(self, mock_get_attrs): class TestUtilityFunctions: """Test utility functions used by the wrappers.""" - def test_safe_json_encoder_basic(self): - """Test SafeJSONEncoder with basic serializable objects.""" - encoder = SafeJSONEncoder() - - # Should handle normal JSON types - result = json.dumps({"key": "value", "number": 42}, cls=SafeJSONEncoder) - assert "key" in result - assert "value" in result - - def test_safe_json_encoder_with_pydantic(self): - """Test SafeJSONEncoder with pydantic-like objects.""" - # Mock object with dict method - mock_obj = Mock() - mock_obj.dict.return_value = {"field": "value"} - - encoder = SafeJSONEncoder() - result = encoder.default(mock_obj) - - assert result == {"field": "value"} - - def test_safe_json_encoder_fallback(self): - """Test SafeJSONEncoder fallback to repr.""" - class CustomClass: + def test_projection_plain_data_and_nested_credentials(self): + payload = { + "safe": "business", + "client": {"config": {"api_key": "FAKE_SHARED_KEY"}}, + } + result = _safe_data(payload) + assert result == { + "safe": "business", + "client": {"config": {"api_key": "__REDACTED__"}}, + } + assert payload["client"]["config"]["api_key"] == "FAKE_SHARED_KEY" + + def test_projection_does_not_call_runtime_serializers(self): + runtime = Mock() + assert _safe_data(runtime) == "[omitted runtime object]" + runtime.dict.assert_not_called() + runtime.to_dict.assert_not_called() + runtime.model_dump.assert_not_called() + + def test_projection_does_not_call_repr(self): + class Runtime: def __repr__(self): - return "CustomClass()" - - encoder = SafeJSONEncoder() - result = encoder.default(CustomClass()) - - assert result == "CustomClass()" + raise AssertionError("runtime repr evaluated") + + assert _safe_data(Runtime()) == "[omitted runtime object]" def test_flatten_simple_mapping(self): """Test _flatten with simple mapping.""" data = {"key1": "value1", "key2": 42} result = dict(_flatten(data)) - + assert result == {"key1": "value1", "key2": 42} def test_flatten_nested_mapping(self): """Test _flatten with nested mapping.""" - data = { - "top": { - "nested": "value", - "deep": {"deeper": "deep_value"} - } - } + data = {"top": {"nested": "value", "deep": {"deeper": "deep_value"}}} result = dict(_flatten(data)) - + assert result["top.nested"] == "value" assert result["top.deep.deeper"] == "deep_value" def test_flatten_with_lists(self): """Test _flatten with lists of mappings.""" - data = { - "items": [ - {"name": "item1", "value": 1}, - {"name": "item2", "value": 2} - ] - } + data = {"items": [{"name": "item1", "value": 1}, {"name": "item2", "value": 2}]} result = dict(_flatten(data)) - + assert result["items.0.name"] == "item1" assert result["items.1.value"] == 2 def test_flatten_with_enum(self): """Test _flatten with enum values.""" + class TestEnum(Enum): VALUE = "test_value" - + data = {"enum_field": TestEnum.VALUE} result = dict(_flatten(data)) - + assert result["enum_field"] == "test_value" def test_flatten_with_none_values(self): """Test _flatten skips None values.""" data = {"valid": "value", "null": None} result = dict(_flatten(data)) - + assert "valid" in result assert "null" not in result @@ -672,78 +691,43 @@ def test_get_input_value_with_tool_calling(self): mock_tool_calling = Mock() mock_tool_calling.tool_name = "search" mock_tool_calling.arguments = {"query": "test"} - + # Import the real class for isinstance check with patch("traceai_crewai._wrappers.ToolCalling", mock_tool_calling.__class__): mock_method = Mock() - + result = _get_input_value(mock_method, calling=mock_tool_calling) - + parsed_result = json.loads(result) assert parsed_result["tool_name"] == "search" assert parsed_result["tool_args"] == {"query": "test"} - def test_get_input_value_with_agent(self): - """Test _get_input_value with Agent object.""" - # Mock Agent + def test_get_input_value_does_not_serialize_agent_arguments(self): + """Only explicit ToolCalling content belongs in a ToolUsage input span.""" mock_agent = Mock() - mock_agent.role = "researcher" - mock_agent.goal = "research topics" - mock_agent.backstory = "expert in research" - - # Import the real class for isinstance check - with patch("traceai_crewai._wrappers.Agent", mock_agent.__class__): - mock_method = Mock() - - result = _get_input_value(mock_method, mock_agent) - - parsed_result = json.loads(result) - assert parsed_result["role"] == "researcher" - assert parsed_result["goal"] == "research topics" - assert parsed_result["backstory"] == "expert in research" - - def test_convert_to_dict_with_to_dict_method(self): - """Test _convert_to_dict with object having to_dict method.""" - mock_obj = Mock() - mock_obj.to_dict.return_value = {"converted": True} - - result = _convert_to_dict(mock_obj) - assert result == {"converted": True} - - def test_convert_to_dict_with_dict_attr(self): - """Test _convert_to_dict with object having __dict__ attribute.""" - class TestObj: - def __init__(self): - self.field = "value" - - obj = TestObj() - result = _convert_to_dict(obj) - assert result == {"field": "value"} - - def test_convert_to_dict_with_nested_structures(self): - """Test _convert_to_dict with nested lists and dicts.""" - mock_obj = Mock() - mock_obj.to_dict.return_value = {"nested": True} - - data = { - "list": [mock_obj, "string"], - "dict": {"nested": mock_obj} + mock_agent.api_key = "FAKE_SHARED_AGENT_KEY" + mock_method = Mock() + assert json.loads(_get_input_value(mock_method, mock_agent)) == {} + mock_agent.to_dict.assert_not_called() + mock_agent.model_dump.assert_not_called() + + def test_projection_nested_containers_does_not_introspect_runtime(self): + runtime = Mock() + data = {"list": [runtime, "string"], "dict": {"nested": runtime}} + assert _safe_data(data) == { + "list": ["[omitted runtime object]", "string"], + "dict": {"nested": "[omitted runtime object]"}, } - - result = _convert_to_dict(data) - assert result["list"][0] == {"nested": True} - assert result["dict"]["nested"] == {"nested": True} - - def test_prepare_args_kwargs(self): - """Test _prepare_args_kwargs function.""" - mock_obj = Mock() - mock_obj.to_dict.return_value = {"converted": True} - - args = (mock_obj, "string") - kwargs = {"param": mock_obj} - - result = _prepare_args_kwargs(args, **kwargs) - - assert "args" in result - assert result["args"][0] == {"converted": True} - assert result["param"] == {"converted": True} \ No newline at end of file + runtime.to_dict.assert_not_called() + + def test_generic_call_projection_redacts_without_mutating(self): + runtime = Mock() + + def method(arg, **kwargs): + pass + + projected = json.loads( + _get_input_value(method, runtime, api_key="FAKE_SHARED_KEY") + ) + assert projected == {} + runtime.to_dict.assert_not_called() diff --git a/release-please-config.json b/release-please-config.json index a4cc5240c..aceb71c1c 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -114,6 +114,14 @@ "traceai_cohere/version.py" ] }, + "python/frameworks/cognee": { + "release-type": "python", + "component": "traceai-cognee", + "package-name": "traceai-cognee", + "extra-files": [ + "traceai_cognee/__init__.py" + ] + }, "python/frameworks/crewai": { "release-type": "python", "component": "traceai-crewai", diff --git a/typescript/packages/traceai_cerebras/package.json b/typescript/packages/traceai_cerebras/package.json index 6c413023f..ad057cb45 100644 --- a/typescript/packages/traceai_cerebras/package.json +++ b/typescript/packages/traceai_cerebras/package.json @@ -18,7 +18,7 @@ "scripts": { "build": "npm run build:cjs && npm run build:esm", "build:cjs": "tsc -p tsconfig.json", - "build:esm": "tsc -p tsconfig.esm.json", + "build:esm": "tsc -p tsconfig.esm.json && tsc-alias -p tsconfig.esm.json && echo '{\"type\": \"module\"}' > ./dist/esm/package.json", "test": "jest --config jest.config.js", "test:e2e": "jest --config jest.config.js --testMatch '**/*.e2e.test.ts'", "clean": "rimraf dist" @@ -56,6 +56,7 @@ "jest": "^29.7.0", "rimraf": "^6.0.1", "ts-jest": "^29.2.5", + "tsc-alias": "^1.8.16", "typescript": "^5.7.2" }, "peerDependencies": { diff --git a/typescript/packages/traceai_cerebras/tsconfig.esm.json b/typescript/packages/traceai_cerebras/tsconfig.esm.json index e8b9f2fd9..766751fa7 100644 --- a/typescript/packages/traceai_cerebras/tsconfig.esm.json +++ b/typescript/packages/traceai_cerebras/tsconfig.esm.json @@ -7,5 +7,8 @@ "exclude": [ "src/**/*.test.ts", "src/**/__tests__/**" - ] + ], + "tsc-alias": { + "resolveFullPaths": true + } } diff --git a/typescript/packages/traceai_deepseek/package.json b/typescript/packages/traceai_deepseek/package.json index caabf13dc..45baf16fc 100644 --- a/typescript/packages/traceai_deepseek/package.json +++ b/typescript/packages/traceai_deepseek/package.json @@ -18,7 +18,7 @@ "scripts": { "build": "npm run build:cjs && npm run build:esm", "build:cjs": "tsc -p tsconfig.json", - "build:esm": "tsc -p tsconfig.esm.json", + "build:esm": "tsc -p tsconfig.esm.json && tsc-alias -p tsconfig.esm.json && echo '{\"type\": \"module\"}' > ./dist/esm/package.json", "test": "jest --config jest.config.js", "test:e2e": "jest --config jest.config.js --testMatch '**/*.e2e.test.ts'", "clean": "rimraf dist" @@ -56,6 +56,7 @@ "openai": "^4.77.0", "rimraf": "^6.0.1", "ts-jest": "^29.2.5", + "tsc-alias": "^1.8.16", "typescript": "^5.7.2" }, "peerDependencies": { diff --git a/typescript/packages/traceai_deepseek/tsconfig.esm.json b/typescript/packages/traceai_deepseek/tsconfig.esm.json index e8b9f2fd9..766751fa7 100644 --- a/typescript/packages/traceai_deepseek/tsconfig.esm.json +++ b/typescript/packages/traceai_deepseek/tsconfig.esm.json @@ -7,5 +7,8 @@ "exclude": [ "src/**/*.test.ts", "src/**/__tests__/**" - ] + ], + "tsc-alias": { + "resolveFullPaths": true + } } diff --git a/typescript/packages/traceai_fireworks/package.json b/typescript/packages/traceai_fireworks/package.json index 3745d0a8f..f1d9cc12d 100644 --- a/typescript/packages/traceai_fireworks/package.json +++ b/typescript/packages/traceai_fireworks/package.json @@ -18,7 +18,7 @@ "scripts": { "build": "npm run build:cjs && npm run build:esm", "build:cjs": "tsc -p tsconfig.json", - "build:esm": "tsc -p tsconfig.esm.json", + "build:esm": "tsc -p tsconfig.esm.json && tsc-alias -p tsconfig.esm.json && echo '{\"type\": \"module\"}' > ./dist/esm/package.json", "test": "jest --config jest.config.js", "test:e2e": "jest --config jest.config.js --testMatch '**/*.e2e.test.ts'", "clean": "rimraf dist" @@ -56,6 +56,7 @@ "openai": "^4.77.0", "rimraf": "^6.0.1", "ts-jest": "^29.2.5", + "tsc-alias": "^1.8.16", "typescript": "^5.7.2" }, "peerDependencies": { diff --git a/typescript/packages/traceai_fireworks/tsconfig.esm.json b/typescript/packages/traceai_fireworks/tsconfig.esm.json index e8b9f2fd9..766751fa7 100644 --- a/typescript/packages/traceai_fireworks/tsconfig.esm.json +++ b/typescript/packages/traceai_fireworks/tsconfig.esm.json @@ -7,5 +7,8 @@ "exclude": [ "src/**/*.test.ts", "src/**/__tests__/**" - ] + ], + "tsc-alias": { + "resolveFullPaths": true + } } diff --git a/typescript/packages/traceai_huggingface/package.json b/typescript/packages/traceai_huggingface/package.json index 355fff64c..4d4fef47d 100644 --- a/typescript/packages/traceai_huggingface/package.json +++ b/typescript/packages/traceai_huggingface/package.json @@ -18,7 +18,7 @@ "scripts": { "build": "npm run build:cjs && npm run build:esm", "build:cjs": "tsc -p tsconfig.json", - "build:esm": "tsc -p tsconfig.esm.json", + "build:esm": "tsc -p tsconfig.esm.json && tsc-alias -p tsconfig.esm.json && echo '{\"type\": \"module\"}' > ./dist/esm/package.json", "test": "jest --config jest.config.js", "lint": "eslint src --ext .ts", "clean": "rm -rf dist" @@ -54,6 +54,7 @@ "@types/node": "^18.0.0", "jest": "^29.5.0", "ts-jest": "^29.1.0", + "tsc-alias": "^1.8.16", "typescript": "^5.0.0" }, "peerDependencies": { diff --git a/typescript/packages/traceai_huggingface/tsconfig.esm.json b/typescript/packages/traceai_huggingface/tsconfig.esm.json index e8b9f2fd9..766751fa7 100644 --- a/typescript/packages/traceai_huggingface/tsconfig.esm.json +++ b/typescript/packages/traceai_huggingface/tsconfig.esm.json @@ -7,5 +7,8 @@ "exclude": [ "src/**/*.test.ts", "src/**/__tests__/**" - ] + ], + "tsc-alias": { + "resolveFullPaths": true + } } diff --git a/typescript/packages/traceai_ollama/package.json b/typescript/packages/traceai_ollama/package.json index 0a04da5dd..ac7eeb544 100644 --- a/typescript/packages/traceai_ollama/package.json +++ b/typescript/packages/traceai_ollama/package.json @@ -18,7 +18,7 @@ "scripts": { "build": "npm run build:cjs && npm run build:esm", "build:cjs": "tsc -p tsconfig.json", - "build:esm": "tsc -p tsconfig.esm.json", + "build:esm": "tsc -p tsconfig.esm.json && tsc-alias -p tsconfig.esm.json && echo '{\"type\": \"module\"}' > ./dist/esm/package.json", "test": "jest --config jest.config.js", "lint": "eslint src --ext .ts", "clean": "rm -rf dist" @@ -54,6 +54,7 @@ "jest": "^29.5.0", "ollama": "^0.5.0", "ts-jest": "^29.1.0", + "tsc-alias": "^1.8.16", "typescript": "^5.0.0" }, "peerDependencies": { diff --git a/typescript/packages/traceai_ollama/tsconfig.esm.json b/typescript/packages/traceai_ollama/tsconfig.esm.json index e8b9f2fd9..766751fa7 100644 --- a/typescript/packages/traceai_ollama/tsconfig.esm.json +++ b/typescript/packages/traceai_ollama/tsconfig.esm.json @@ -7,5 +7,8 @@ "exclude": [ "src/**/*.test.ts", "src/**/__tests__/**" - ] + ], + "tsc-alias": { + "resolveFullPaths": true + } } diff --git a/typescript/packages/traceai_together/package.json b/typescript/packages/traceai_together/package.json index 0ba29d9d4..f0fe4a02c 100644 --- a/typescript/packages/traceai_together/package.json +++ b/typescript/packages/traceai_together/package.json @@ -18,7 +18,7 @@ "scripts": { "build": "npm run build:cjs && npm run build:esm", "build:cjs": "tsc -p tsconfig.json", - "build:esm": "tsc -p tsconfig.esm.json", + "build:esm": "tsc -p tsconfig.esm.json && tsc-alias -p tsconfig.esm.json && echo '{\"type\": \"module\"}' > ./dist/esm/package.json", "test": "jest --config jest.config.js", "lint": "eslint src --ext .ts", "clean": "rm -rf dist" @@ -54,6 +54,7 @@ "jest": "^29.5.0", "together-ai": "^0.7.0", "ts-jest": "^29.1.0", + "tsc-alias": "^1.8.16", "typescript": "^5.0.0" }, "peerDependencies": { diff --git a/typescript/packages/traceai_together/tsconfig.esm.json b/typescript/packages/traceai_together/tsconfig.esm.json index e8b9f2fd9..766751fa7 100644 --- a/typescript/packages/traceai_together/tsconfig.esm.json +++ b/typescript/packages/traceai_together/tsconfig.esm.json @@ -7,5 +7,8 @@ "exclude": [ "src/**/*.test.ts", "src/**/__tests__/**" - ] + ], + "tsc-alias": { + "resolveFullPaths": true + } } diff --git a/typescript/packages/traceai_vllm/package.json b/typescript/packages/traceai_vllm/package.json index 0ac06e385..27ed8907f 100644 --- a/typescript/packages/traceai_vllm/package.json +++ b/typescript/packages/traceai_vllm/package.json @@ -18,7 +18,7 @@ "scripts": { "build": "npm run build:cjs && npm run build:esm", "build:cjs": "tsc -p tsconfig.json", - "build:esm": "tsc -p tsconfig.esm.json", + "build:esm": "tsc -p tsconfig.esm.json && tsc-alias -p tsconfig.esm.json && echo '{\"type\": \"module\"}' > ./dist/esm/package.json", "test": "jest --config jest.config.js", "test:e2e": "jest --config jest.config.js --testMatch '**/*.e2e.test.ts'", "clean": "rimraf dist" @@ -56,6 +56,7 @@ "openai": "^4.77.0", "rimraf": "^6.0.1", "ts-jest": "^29.2.5", + "tsc-alias": "^1.8.16", "typescript": "^5.7.2" }, "peerDependencies": { diff --git a/typescript/packages/traceai_vllm/tsconfig.esm.json b/typescript/packages/traceai_vllm/tsconfig.esm.json index e8b9f2fd9..766751fa7 100644 --- a/typescript/packages/traceai_vllm/tsconfig.esm.json +++ b/typescript/packages/traceai_vllm/tsconfig.esm.json @@ -7,5 +7,8 @@ "exclude": [ "src/**/*.test.ts", "src/**/__tests__/**" - ] + ], + "tsc-alias": { + "resolveFullPaths": true + } } diff --git a/typescript/packages/traceai_xai/package.json b/typescript/packages/traceai_xai/package.json index 158eb5890..0b9d6f154 100644 --- a/typescript/packages/traceai_xai/package.json +++ b/typescript/packages/traceai_xai/package.json @@ -18,7 +18,7 @@ "scripts": { "build": "npm run build:cjs && npm run build:esm", "build:cjs": "tsc -p tsconfig.json", - "build:esm": "tsc -p tsconfig.esm.json", + "build:esm": "tsc -p tsconfig.esm.json && tsc-alias -p tsconfig.esm.json && echo '{\"type\": \"module\"}' > ./dist/esm/package.json", "test": "jest --config jest.config.js", "test:e2e": "jest --config jest.config.js --testMatch '**/*.e2e.test.ts'", "clean": "rimraf dist" @@ -57,6 +57,7 @@ "openai": "^4.77.0", "rimraf": "^6.0.1", "ts-jest": "^29.2.5", + "tsc-alias": "^1.8.16", "typescript": "^5.7.2" }, "peerDependencies": { diff --git a/typescript/packages/traceai_xai/tsconfig.esm.json b/typescript/packages/traceai_xai/tsconfig.esm.json index e8b9f2fd9..766751fa7 100644 --- a/typescript/packages/traceai_xai/tsconfig.esm.json +++ b/typescript/packages/traceai_xai/tsconfig.esm.json @@ -7,5 +7,8 @@ "exclude": [ "src/**/*.test.ts", "src/**/__tests__/**" - ] + ], + "tsc-alias": { + "resolveFullPaths": true + } } diff --git a/typescript/pnpm-lock.yaml b/typescript/pnpm-lock.yaml index 495e604cb..c6feab057 100644 --- a/typescript/pnpm-lock.yaml +++ b/typescript/pnpm-lock.yaml @@ -442,6 +442,9 @@ importers: ts-jest: specifier: ^29.2.5 version: 29.4.6(@babel/core@7.27.1)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.27.1))(jest-util@29.7.0)(jest@29.7.0(@types/node@22.15.33)(ts-node@10.9.2(@types/node@22.15.33)(typescript@5.8.3)))(typescript@5.8.3) + tsc-alias: + specifier: ^1.8.16 + version: 1.8.16 typescript: specifier: ^5.7.2 version: 5.8.3 @@ -706,6 +709,9 @@ importers: ts-jest: specifier: ^29.2.5 version: 29.4.6(@babel/core@7.27.1)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.27.1))(jest-util@29.7.0)(jest@29.7.0(@types/node@22.15.33)(ts-node@10.9.2(@types/node@22.15.33)(typescript@5.8.3)))(typescript@5.8.3) + tsc-alias: + specifier: ^1.8.16 + version: 1.8.16 typescript: specifier: ^5.7.2 version: 5.8.3 @@ -752,6 +758,9 @@ importers: ts-jest: specifier: ^29.2.5 version: 29.4.6(@babel/core@7.27.1)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.27.1))(jest-util@29.7.0)(jest@29.7.0(@types/node@22.15.33)(ts-node@10.9.2(@types/node@22.15.33)(typescript@5.8.3)))(typescript@5.8.3) + tsc-alias: + specifier: ^1.8.16 + version: 1.8.16 typescript: specifier: ^5.7.2 version: 5.8.3 @@ -1025,6 +1034,9 @@ importers: ts-jest: specifier: ^29.1.0 version: 29.4.6(@babel/core@7.27.1)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.27.1))(jest-util@29.7.0)(jest@29.7.0(@types/node@18.19.100)(ts-node@10.9.2(@types/node@22.15.33)(typescript@5.8.3)))(typescript@5.8.3) + tsc-alias: + specifier: ^1.8.16 + version: 1.8.16 typescript: specifier: ^5.0.0 version: 5.8.3 @@ -1534,6 +1546,9 @@ importers: ts-jest: specifier: ^29.1.0 version: 29.4.6(@babel/core@7.27.1)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.27.1))(jest-util@29.7.0)(jest@29.7.0(@types/node@18.19.100)(ts-node@10.9.2(@types/node@22.15.33)(typescript@5.8.3)))(typescript@5.8.3) + tsc-alias: + specifier: ^1.8.16 + version: 1.8.16 typescript: specifier: ^5.0.0 version: 5.8.3 @@ -2001,6 +2016,9 @@ importers: ts-jest: specifier: ^29.1.0 version: 29.4.6(@babel/core@7.27.1)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.27.1))(jest-util@29.7.0)(jest@29.7.0(@types/node@18.19.100)(ts-node@10.9.2(@types/node@22.15.33)(typescript@5.8.3)))(typescript@5.8.3) + tsc-alias: + specifier: ^1.8.16 + version: 1.8.16 typescript: specifier: ^5.0.0 version: 5.8.3 @@ -2118,6 +2136,9 @@ importers: ts-jest: specifier: ^29.2.5 version: 29.4.6(@babel/core@7.27.1)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.27.1))(jest-util@29.7.0)(jest@29.7.0(@types/node@22.15.33)(ts-node@10.9.2(@types/node@22.15.33)(typescript@5.8.3)))(typescript@5.8.3) + tsc-alias: + specifier: ^1.8.16 + version: 1.8.16 typescript: specifier: ^5.7.2 version: 5.8.3 @@ -2262,6 +2283,9 @@ importers: ts-jest: specifier: ^29.2.5 version: 29.4.6(@babel/core@7.27.1)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.27.1))(jest-util@29.7.0)(jest@29.7.0(@types/node@22.15.33)(ts-node@10.9.2(@types/node@22.15.33)(typescript@5.8.3)))(typescript@5.8.3) + tsc-alias: + specifier: ^1.8.16 + version: 1.8.16 typescript: specifier: ^5.7.2 version: 5.8.3