From 94e9d3bd92b41e0d0225a47cdddcbf62410f908c Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Fri, 30 Oct 2026 20:54:24 -0700 Subject: [PATCH 1/3] feat(cognee): add maintained native tracing adapter Co-Authored-By: Claude Code --- README.md | 2 +- python/README.md | 2 +- python/examples/cognee/README.md | 448 +---- python/examples/cognee/requirements.txt | 9 +- python/examples/cognee/src/app.py | 86 +- python/examples/cognee/src/cognee_filter.py | 275 --- python/examples/cognee/tests/_run_app.py | 19 + python/examples/cognee/tests/cognee_add.py | 59 - .../tests/loopback_guard/sitecustomize.py | 93 - .../cognee/tests/test_cognee_adapter.py | 161 ++ .../cognee/tests/test_cognee_recipe.py | 1719 ----------------- python/frameworks/cognee/CHANGELOG.md | 8 + python/frameworks/cognee/LICENSE | 201 ++ python/frameworks/cognee/README.md | 39 + python/frameworks/cognee/pyproject.toml | 26 + .../frameworks/cognee/tests/test_adapter.py | 96 + .../cognee/tests/test_generation_isolation.py | 201 ++ .../tests/test_lifecycle_regressions.py | 207 ++ .../cognee/tests/test_security_regressions.py | 94 + .../cognee/tests/test_structured.py | 70 + .../cognee/traceai_cognee/__init__.py | 279 +++ .../cognee/traceai_cognee/_exporter.py | 98 + .../cognee/traceai_cognee/_structured.py | 103 + 23 files changed, 1636 insertions(+), 2659 deletions(-) delete mode 100644 python/examples/cognee/src/cognee_filter.py create mode 100644 python/examples/cognee/tests/_run_app.py delete mode 100644 python/examples/cognee/tests/cognee_add.py delete mode 100644 python/examples/cognee/tests/loopback_guard/sitecustomize.py create mode 100644 python/examples/cognee/tests/test_cognee_adapter.py delete mode 100644 python/examples/cognee/tests/test_cognee_recipe.py create mode 100644 python/frameworks/cognee/CHANGELOG.md create mode 100644 python/frameworks/cognee/LICENSE create mode 100644 python/frameworks/cognee/README.md create mode 100644 python/frameworks/cognee/pyproject.toml create mode 100644 python/frameworks/cognee/tests/test_adapter.py create mode 100644 python/frameworks/cognee/tests/test_generation_isolation.py create mode 100644 python/frameworks/cognee/tests/test_lifecycle_regressions.py create mode 100644 python/frameworks/cognee/tests/test_security_regressions.py create mode 100644 python/frameworks/cognee/tests/test_structured.py create mode 100644 python/frameworks/cognee/traceai_cognee/__init__.py create mode 100644 python/frameworks/cognee/traceai_cognee/_exporter.py create mode 100644 python/frameworks/cognee/traceai_cognee/_structured.py diff --git a/README.md b/README.md index bd0e8e525..7fb865468 100644 --- a/README.md +++ b/README.md @@ -267,6 +267,7 @@ Catalog links point to the integration source and setup guidance in this reposit | [`traceAI-autogen`](python/frameworks/autogen/) | AutoGen | | [`traceai-strands`](python/frameworks/strands/) | AWS Strands Agents | | [`traceai-claude-agent-sdk`](python/frameworks/claude-agent-sdk/) | Claude Agent SDK | +| [`traceai-cognee`](python/frameworks/cognee/) | Cognee native tracing | | [`traceAI-crewai`](python/frameworks/crewai/) | CrewAI | | [`traceAI-elevenlabs-agents`](python/frameworks/elevenlabs-agents/) | ElevenLabs Agents | | [`traceAI-google-adk`](python/frameworks/google-adk/) | Google ADK | @@ -469,7 +470,6 @@ These recipes reuse SDK instrumentation or configure a native tracing/export pat | Anannas AI | [Setup](python/examples/anannas-ai/) | | Azure AI Foundry | [Setup](python/examples/azure-ai-foundry/) | | Baseten Model APIs | [Setup](python/examples/baseten/) | -| Cognee | [Setup](python/examples/cognee/) | | Databricks | [Setup](python/examples/databricks/) | | Doubleword | [Setup](python/examples/doubleword/) | | EverOS | [Setup](python/examples/everos/) | diff --git a/python/README.md b/python/README.md index ca0c4ad7e..f38fd39fb 100644 --- a/python/README.md +++ b/python/README.md @@ -193,6 +193,7 @@ Package names and links refer to the source in this repository. | [`traceAI-autogen`](../python/frameworks/autogen/) | AutoGen | | [`traceai-strands`](../python/frameworks/strands/) | AWS Strands Agents | | [`traceai-claude-agent-sdk`](../python/frameworks/claude-agent-sdk/) | Claude Agent SDK | +| [`traceai-cognee`](../python/frameworks/cognee/) | Cognee native tracing | | [`traceAI-crewai`](../python/frameworks/crewai/) | CrewAI | | [`traceAI-elevenlabs-agents`](../python/frameworks/elevenlabs-agents/) | ElevenLabs Agents | | [`traceAI-google-adk`](../python/frameworks/google-adk/) | Google ADK | @@ -249,7 +250,6 @@ These examples reuse existing instrumentation or configure tracing adapters. | Anannas AI | [Setup](examples/anannas-ai/) | | Azure AI Foundry | [Setup](examples/azure-ai-foundry/) | | Baseten Model APIs | [Setup](examples/baseten/) | -| Cognee | [Setup](examples/cognee/) | | Databricks | [Setup](examples/databricks/) | | Doubleword | [Setup](examples/doubleword/) | | EverOS | [Setup](examples/everos/) | diff --git a/python/examples/cognee/README.md b/python/examples/cognee/README.md index a64026d25..06d6e6ca7 100644 --- a/python/examples/cognee/README.md +++ b/python/examples/cognee/README.md @@ -1,450 +1,26 @@ -# Cognee → Future AGI (recipe) +# Cognee example -There is no `traceai-cognee` package. Cognee creates its own OpenTelemetry -spans when `COGNEE_TRACING_ENABLED=true`; this recipe sends those spans to -Future AGI. It adds no instrumentor (a second one would double every span) -and never calls Cognee's `setup_tracing` (Cognee calls it on its first traced -operation). Option A adds one small export filter, -[`src/cognee_filter.py`](src/cognee_filter.py), that keeps Cognee's content, -and error text that can quote it, out of the export and marks search and LLM -spans. +Send Cognee's native add, cognify and search spans to Future AGI with the maintained `traceai-cognee` adapter. Do not add a second Cognee or LiteLLM instrumentor. -Pinned: `cognee==1.6.5` (Python >=3.10,<3.15), the latest release on -2026-10-16; `cognee==1.6.2` was tested too and behaves the same. Both wheels' -METADATA declare `License-Expression: Apache-2.0` (license files `LICENSE` -and `NOTICE.md`); cognee is a dependency of this example only, not of any -traceAI package. Option A also needs `fi-instrumentation-otel>=1.1.0,<1.2`, -which declares Python <3.14. The tests (see [Tests](#tests)) run on Python -3.10, 3.11 and 3.13 against loopback fakes, with no LLM key and no Future AGI -project. The recipe was also checked live on 2026-10-16, on Python 3.12, -against a local Future AGI stack with real OpenAI models (`gpt-4o-mini`, -`text-embedding-3-small`): option A with content off and on and a failing -model call (cognee 1.6.5 and 1.6.2), and option B (cognee 1.6.5) over gRPC to -the local collector and over HTTP with the `:443` URL below to Future AGI -cloud. Traces only: metrics and logs are not ingested by this recipe. -Statements marked "source reading" come from the cognee 1.6.5 code and are -not exercised by the tests. +## Run -## Option A: `register()` (recommended) +Use Python 3.10–3.13. Install the dependencies, then configure Cognee's model and embedding providers using the [Cognee documentation](https://docs.cognee.ai/). ```bash pip install -r requirements.txt - -export FI_API_KEY=... # sent as the X-Api-Key header -export FI_SECRET_KEY=... # sent as the X-Secret-Key header -export FI_PROJECT_NAME=my-cognee-app export COGNEE_TRACING_ENABLED=true -# Cognee's own model settings, unchanged by this recipe: -export LLM_API_KEY=... # plus LLM_MODEL, EMBEDDING_* as usual - +export FI_API_KEY="YOUR_FI_API_KEY" +export FI_SECRET_KEY="YOUR_FI_SECRET_KEY" +export FI_PROJECT_NAME="my-cognee-app" python src/app.py "Who works on Lighthouse?" ``` -`register()` posts OTLP/HTTP protobuf to `https://api.futureagi.com/tracer/v1/traces` -(override the origin with `FI_BASE_URL`) with the `X-Api-Key` and -`X-Secret-Key` headers, and stamps every batch with the resource attributes -`project_name` and `project_type=observe`. - -The whole integration is [`src/app.py`](src/app.py) plus -[`src/cognee_filter.py`](src/cognee_filter.py), which you copy next to it: - -```python -from fi_instrumentation import register -from fi_instrumentation.fi_types import ProjectType -from fi_instrumentation.otel import BatchSpanProcessor, HTTPSpanExporter - -from cognee_filter import CogneeExportFilter - -provider = register( - project_name=os.environ["FI_PROJECT_NAME"], - project_type=ProjectType.OBSERVE, - set_global_tracer_provider=True, # Cognee reuses the global provider - verbose=False, -) -exporter = CogneeExportFilter(HTTPSpanExporter()) -provider.add_span_processor(BatchSpanProcessor(exporter)) # required, see below - -with provider.get_tracer(__name__).start_as_current_span("remember_and_ask"): - asyncio.run(remember_and_ask(question, document)) # cognee.add / cognify / search -``` - -These things about it are load-bearing: - -- **Register before the first Cognee call.** When Cognee sets up tracing it - checks for a global provider. If one exists it attaches to it; if not it - creates and installs its own, and OpenTelemetry then ignores a later - `register(set_global_tracer_provider=True)`. -- **Keep the `add_span_processor(BatchSpanProcessor(exporter))` line.** - `register()` marks its exporter as a default that the next - `add_span_processor()` call replaces (`fi_instrumentation/otel.py`, - `TracerProvider.add_span_processor`; tracked as TH-8394). Cognee calls - `add_span_processor()` on the global provider to attach its in-memory span - buffer, so without this line the Future AGI exporter is dropped and - **nothing is exported**: Cognee still buffers its spans, but no request - leaves the process (the `no_readd` test shows this). Adding the processor - yourself makes it a regular one that Cognee's call leaves alone, and it is - where the export filter goes. Once TH-8394 is fixed, the re-add line goes - away; until then `requirements.txt` pins `fi-instrumentation-otel>=1.1.0,<1.2` - (the published 1.1.0 behaves this way; the tests use this repository's - source). -- **The re-added exporter does not inherit `register()`'s options.** - `HTTPSpanExporter()` reads `FI_BASE_URL`, `FI_API_KEY` and `FI_SECRET_KEY` - just as `register()`'s default exporter does, and `BatchSpanProcessor` uses - the default batch settings. `register()`'s own processor is discarded, so if - you pass exporter options to `register()`, pass the same ones here: - `headers` and `timeout` to `HTTPSpanExporter(...)`; for `span_exporter` or - gRPC transport, wrap that exporter (or `GRPCSpanExporter(...)`) in - `CogneeExportFilter`; the batch tuning to `BatchSpanProcessor(...)`, or use - `SimpleSpanProcessor` for `batch=False`. -- **Wrap the calls in one span if you want one trace.** Each Cognee API call - starts its own root span, and `cognee.add` ends its `memory.store` span - before ingestion runs (`cognee/api/v1/add/add.py`), so without a parent span - one add/cognify/search sequence arrives as five traces. With the parent span - it is one trace. -- **Do not call `cognee.disable_tracing()` (or `shutdown_tracing()`).** With - this option they shut down the provider Cognee attached to, which is your - global Future AGI provider, so your app's own spans stop too (source - reading: `cognee/modules/observability/trace_context.py` and `tracing.py`). - To turn Cognee's spans off, unset `COGNEE_TRACING_ENABLED` instead. - -Do not also set `OTEL_EXPORTER_OTLP_ENDPOINT` with this option. Cognee ignores -it for traces when a provider is already registered, but its metrics and log -bridge read it and would send to it (source reading of -`cognee/modules/observability/metrics.py` and `logs.py`; not exercised by the -tests). - -### The export filter - -`CogneeExportFilter` wraps the exporter and changes only what that exporter -sends. It exports copies, so Cognee's own span buffer and any other processor -or exporter still see the original spans. - -- **Content is off by default.** For the Cognee 1.6.5 add/cognify/search path - covered by this recipe's recorded fixture and loopback tests, it removes the - five content-bearing attributes listed in [Content](#content): - `langfuse.observation.input`, `langfuse.observation.output`, - `memory.query.text`, `cognee.search.query` and `cognee.db.query`. -- **Error detail is off by default too, on every span.** Cognee's errors can - quote content: a content-policy rejection raises an error whose message is - the whole prompt (a document chunk, or the question with the retrieved - context), and that message is recorded on the LLM span and again on each - pipeline-task span it passes through. On every span passed to this filter in - content-off mode, it keeps each `exception` event's `exception.type` and - `exception.escaped`, replaces `exception.message` with - `__REDACTED__ (content capture off)`, and omits - `exception.stacktrace` altogether. An `ERROR` status description becomes - ` (detail removed: content capture off)`, or - `__REDACTED__ (content capture off)` if the span recorded no exception - type. In every event, numbers and booleans are kept and any other string - value is dropped (the tested Cognee 1.6.5 recipe adds no events besides - `exception`). -- **Stack traces are never exported, in either mode.** `exception.stacktrace` - is dropped with content capture on as well: a serialized traceback names - local files (home directory, user name, virtualenv layout), and it has no - trustworthy boundary between genuine frames and traceback-shaped - exception-message content. Use Cognee's own log for frames. -- **Content capture turns both back on.** - `CogneeExportFilter(exporter, capture_content=True)`, or - `COGNEE_FI_CAPTURE_CONTENT=true` in the environment, exports the content - attributes and the error messages and status descriptions as Cognee - recorded them; only `exception.stacktrace` stays out. There is no separate - switch for error detail. It also copies that content to `input.value` and - `output.value` (with `input.mime_type` and `output.mime_type`), the only - keys fi-collector fills a span's Input and Output from: the LLM call's - arguments (`langfuse.observation.input`), or on search spans the question - (`memory.query.text`, else `cognee.search.query`), as Input, and the - response (`langfuse.observation.output`) as Output. Without these copies, - Future AGI would show captured content only as raw attributes, with Input - and Output empty. A span that already has `input.value` or `output.value` - keeps it. -- **Search and LLM spans get a type.** Cognee sets no span-kind attribute, - so the filter sets `fi.span.kind` by span name, only where none of - `fi.span.kind`, `gen_ai.span.kind`, `llm.request.type` and - `openinference.span.kind` is set: `RETRIEVER` on `memory.retrieve` and - every span whose name starts with `cognee.search` (in 1.6.5 and 1.6.2, - `cognee.search.authorize` and `cognee.search.dataset`), `LLM` on - `cognee.observe.acreate_structured_output` (every LLM call in Cognee 1.6.5). Embedding spans are left as they are: - fi-collector already types them from `gen_ai.operation.name=embeddings`. -- **Missing-collection probes are not errors.** In every search Cognee - probes vector collections that do not exist yet, and each probe's - `cognee.db.vector.search` span ends in `ERROR` with - `CollectionNotFoundError` (60 of the 66 vector searches in the recorded - run). Without the filter every successful trace shows dozens of errors and - inflates error rates. By default the filter exports a - `cognee.db.vector.search` span whose only error is - `CollectionNotFoundError` as `OK`, without its `exception` event, and with - `cognee.vector_search.collection_missing=true`; `cognee.vector.collection` - still names the collection. Any other error, a second exception on the - same span, or the same error on another span stays `ERROR`. To keep the - probes as errors, pass `collection_missing_as_error=True` or set - `COGNEE_FI_COLLECTION_MISSING_AS_ERROR=true`. -- **It fails closed.** If the filter raises on a span, that span is exported - without the content attributes, events or status description; if even that - fails, the span is dropped. The filter logs a warning and never raises into - the export path. - -The trade-off: with the defaults, error detail in Future AGI is -less useful for debugging. You see the exception type, but not its message or -traceback frames; use the existing content-capture opt-in to export the -original error messages (frames stay in Cognee's own log). The filter does not touch Cognee's own logging (for example, -`run_tasks_base.py` logs a failed pipeline task with `logger.exception`; -source reading). - -What it keeps: span names and event names (in Cognee 1.6.5, fixed strings and -function names), the other span attributes (counts, lengths, names and ids; -source reading of the attributes Cognee sets on the tested path). The tests -assert that the planted document, question and answer do not reach the -loopback receiver, and that an 8000+-character content-policy prompt with a -forged chained-traceback frame reaches neither events nor status when the -recorded LLM and pipeline-parent spans pass through the default filter. - -## Option B: Cognee's own OTLP exporter (no traceAI) - -Cognee can export by itself when `OTEL_EXPORTER_OTLP_ENDPOINT` is set and no -provider was registered. It needs Cognee's tracing extra, which installs the -OpenTelemetry SDK and exporters: - -```bash -pip install "cognee[tracing]==1.6.5" - -export COGNEE_TRACING_ENABLED=true -export OTEL_EXPORTER_OTLP_ENDPOINT=https://api.futureagi.com:443/tracer/v1/traces -export OTEL_EXPORTER_OTLP_HEADERS="X-Api-Key=$FI_API_KEY,X-Secret-Key=$FI_SECRET_KEY" -export OTEL_RESOURCE_ATTRIBUTES="project_name=my-cognee-app,project_type=observe" -``` - -**Option B exports content and has no kind mapping.** The export filter -cannot be attached to Cognee's own exporter, so prompts, document text, -answers and search questions are exported (see [Content](#content)), and -search and LLM spans are stored as `unknown`. Option A is the privacy-safe -path. +Open **Observe → Tracing** in Future AGI and select `my-cognee-app`. The application wraps its operations in one `remember_and_ask` parent span and flushes after the operations end. -- **Full path.** Cognee passes the endpoint string to the OTLP exporter - unchanged; it does not append `/v1/traces`. The test configures - `/tracer/v1/traces` and the request arrives at exactly - `/tracer/v1/traces`. -- **Keep `:443` in the URL.** Cognee picks HTTP only for URLs containing - `:4318`, `:443/` or a few vendor paths; for any other URL it uses the gRPC - exporter whenever `opentelemetry-exporter-otlp-proto-grpc` is installed - (`cognee[tracing]` installs it). Cognee's own comment warns that a gRPC - exporter pointed at an HTTP-only endpoint fails silently. - `https://api.futureagi.com/tracer/v1/traces` would go to gRPC; - `https://api.futureagi.com:443/tracer/v1/traces` stays on HTTP (the test - checks Cognee's own classifier for both). -- **`project_name` is required.** Cognee builds its resource with - OpenTelemetry's `Resource.create`, which reads `OTEL_RESOURCE_ATTRIBUTES`; - the test shows `project_name` and `project_type=observe` arrive that way. - fi-collector rejects a batch in which any resource lacks `project_name`. -- **Headers are split naively** (source reading). Cognee parses - `OTEL_EXPORTER_OTLP_HEADERS` itself: split on `,`, then on the first `=`, - with no URL decoding. A key that contains a comma cannot be passed this way; - use option A. -- **Self-hosted URL without `:443` or `:4318`: Cognee picks gRPC.** With - `cognee[tracing]` installed, a URL such as `http://localhost:14338/tracer/v1/traces` - goes to the gRPC exporter and nothing arrives (checked live: no project was - created). Point `OTEL_EXPORTER_OTLP_ENDPOINT` at the collector's OTLP gRPC - port instead (`http://:`, no path) and write the header - names in lower case, `x-api-key=...,x-secret-key=...`: gRPC metadata keys - must be lower case, and with `X-Api-Key` every export fails with - `Invalid metadata`. Checked live against the local collector's gRPC - receiver: 5 traces, 119 spans. -- **Expect 404s for logs.** Cognee also sends its log records to the same URL - with `/v1/traces` replaced by `/v1/logs`, and (source reading) its metrics - to `/v1/metrics`. fi-collector serves traces only; the test run logs - `Failed to export logs batch code: 404`. Spans are unaffected. +## Content and lifecycle -## Spans Cognee emits +Content capture is off by default. Pass `capture_content=True` to `configure` only for approved application data. Recognized credential fields are redacted; free text is not scanned for secrets. Error descriptions and stack traces stay excluded in both modes. Span names are retained, so keep sensitive data out of them. Other processors on the provider receive native, unmodified spans and need their own privacy controls. -From the recipe run (one `add`, one `cognify`, one `GRAPH_COMPLETION` -search). Cognee 1.6.5 and 1.6.2 emit the same span names, kinds, attribute -keys and statuses; only the number of `cognee.db.graph.query` spans differs -(17 on 1.6.5, 14 on 1.6.2). - -"Future AGI type" is what fi-collector derives: it reads the span -kind from `fi.span.kind`, `gen_ai.span.kind`, `llm.request.type` or -`openinference.span.kind`, falls back to `gen_ai.operation.name`, and -otherwise stores `unknown`. Cognee sets none of the first four; with option A -the export filter sets `fi.span.kind` on the spans marked below. - -| Span | OTel kind | Future AGI type (option A / option B) | Notable attributes | -|---|---|---|---| -| `remember_and_ask` | INTERNAL | unknown | the app's own parent span (option A example) | -| `memory.store` | INTERNAL | unknown | `memory.operation=store`, `cognee.dataset.name` | -| `memory.process` | INTERNAL | unknown | `memory.operation=process`, `cognee.pipeline.name=cognify` | -| `memory.retrieve` | INTERNAL | retriever / unknown | `memory.query.text`, `cognee.search.query`, `cognee.search.type`, `memory.result.count` | -| `cognee.pipeline.task.resolve_data_directories`, `cognee.pipeline.task.ingest_data`, `cognee.pipeline.task.classify_documents`, `cognee.pipeline.task.extract_chunks_from_documents`, `cognee.pipeline.task.extract_graph_and_summarize`, `cognee.pipeline.task.add_data_points` | INTERNAL | unknown | `cognee.pipeline.task_name`, `cognee.result.count`, `cognee.result.summary` | -| `cognee.observe.acreate_structured_output` (every LLM call) | CLIENT | llm / unknown | `gen_ai.request.model`, `gen_ai.system=litellm-native`, `langfuse.observation.input`, `langfuse.observation.output`; `cognee.llm.model` on schema-bound calls only | -| `cognee.observe.embed_text` (every embedding call) | CLIENT | embedding | `gen_ai.operation.name=embeddings`, `gen_ai.request.model`, `gen_ai.provider.name`, `gen_ai.embeddings.dimension.count` | -| `cognee.llm.completion` | INTERNAL | unknown | prompt file name and lengths only | -| `cognee.db.graph.query`, `cognee.db.vector.search` | INTERNAL | unknown | `cognee.db.system`, `cognee.db.query`, `cognee.vector.collection`; with option A, `cognee.vector_search.collection_missing` on a probe of a missing collection | -| `cognee.search.authorize`, `cognee.search.dataset` | INTERNAL | retriever / unknown | `cognee.search.query`, `cognee.search.type`, dataset name and id | -| `cognee.retrieval.get_objects`, `cognee.retrieval.triplet_search`, `cognee.retrieval.vector_search`, `cognee.retrieval.embed_query`, `cognee.retrieval.get_context`, `cognee.retrieval.get_completion` | INTERNAL | unknown | retriever name, counts and lengths | -| `cognee.session.get_session`, `cognee.session.add_qa` | INTERNAL | unknown | `cognee.session.id` | - -With option A, the content attributes in this table are removed unless you -turn content capture on. - -Not in the table: `cognee.api.recall` (`api/v1/recall/recall.py:540` in 1.6.5) and -`cognee.agent_memory.retrieve` (`modules/agent_memory/runtime.py:405`) are -retrieval entry points that the recipe's add/cognify/search path does not -reach (source reading). The filter does not map them, so they stay `unknown` -in Future AGI. Their query text is still removed: `cognee.api.recall` puts it -in `cognee.search.query`, and `cognee.agent_memory.retrieve` records only the -query length, while the search it runs puts the query on `memory.retrieve` -and `cognee.search.*` spans. - -What that means in Future AGI: - -- **Model column: populated.** Every LLM span carries `gen_ai.request.model`, - which fi-collector promotes. The provider comes from `gen_ai.system` - (`litellm-native`, Cognee's adapter name, not the upstream provider). No - processor that copies `cognee.llm.model` is needed, so none is written. -- **Token and cost columns: empty.** Cognee 1.6.5 records no token usage on - any span. -- **Input and Output: empty by default.** Content capture is off, so no span - carries content. With content capture on (option A), LLM spans show the - call's arguments and response and search spans show the question; see - [The export filter](#the-export-filter). With option B they stay empty: the - content arrives only as raw `langfuse.*`, `memory.*` and `cognee.*` - attributes. -- **Search is shown as a retriever, LLM calls as LLM (option A).** The - export filter sets `fi.span.kind`. With option B they stay `unknown`: no - span-kind key arrives and only embedding spans carry - `gen_ai.operation.name`. -- **No errors in a successful run (option A).** Cognee's own spans put - most `cognee.db.vector.search` spans in `ERROR`, each with an `exception` - event: 60 of 66 in the recorded run (60 of its 117 spans), because Cognee - probes collections that do not exist yet. The export filter sends those as - `OK` with `cognee.vector_search.collection_missing=true` (see - [The export filter](#the-export-filter)), so they do not count in Future - AGI's error filters and error rates. With option B, or with - `COGNEE_FI_COLLECTION_MISSING_AS_ERROR=true`, they arrive as errors; with - option A's other defaults their status then reads - `cognee.infrastructure.databases.vector.exceptions.exceptions.CollectionNotFoundError - (detail removed: content capture off)`, without the collection name. - -## Content - -With tracing on, Cognee 1.6.5 writes content to span attributes by default, -and it has no setting that keeps tracing on but leaves content out. Option A's -export filter removes these attributes before export unless content capture -is on; option B exports them: - -| Attribute | On | Contains | -|---|---|---| -| `langfuse.observation.input` | every LLM span | the LLM call's string arguments as JSON: Cognee's system prompt plus the document text (extraction, summarization), or the user question and retrieved graph context (search). First 8000 characters, after Cognee's regex secret redaction. | -| `langfuse.observation.output` | every LLM span | the model's response (extracted graph JSON, summaries, the answer). Same cap and redaction. | -| `memory.query.text`, `cognee.search.query` | `memory.retrieve`, `cognee.search.authorize` | the search question (first 500 characters). | -| `cognee.db.query` | `cognee.db.graph.query` | the graph query text (first 500 characters, redacted). In the test run these are parameterised queries without document text. | -| `exception.message`, `exception.stacktrace`, the status description | `exception` events and `ERROR` status of the span that raised and of each span the error passes through | the error message can quote content: a content-policy rejection (`litellm_native/native_adapter.py:532-560` in 1.6.5) quotes the whole prompt, past the 8000-character cap. Stack traces can also contain that serialized message and host paths. With option A's defaults the message and description are replaced; `exception.stacktrace` is omitted in both modes (see [The export filter](#the-export-filter)). | - -The test plants markers in the document and the question and asserts that, -with option A's defaults, none of them arrives anywhere (attributes, status, -events). With content capture on, it asserts that among span attributes they -arrive under exactly the attribute keys above plus the filter's `input.value` -and `output.value` copies, and no others; error messages and status -descriptions are then exported as Cognee recorded them, so they also carry -content whenever an error quotes it. Stack traces are not exported in either -mode. -Embedding spans carry metadata only, not the embedded text. - -Not exported: the LLM and embedding API keys and the Future AGI keys (those -travel only as request headers). The test asserts both. - -With option B, the only way to keep content out of Future AGI is to leave -`COGNEE_TRACING_ENABLED` unset: the `tracing_off` test shows Cognee then emits -no spans (only the app's own parent span remains). If `LANGFUSE_PUBLIC_KEY` -and `LANGFUSE_SECRET_KEY` are set, Cognee turns tracing on by itself; set -`COGNEE_TRACING_ENABLED=false` to veto that (source reading). - -`TELEMETRY_DISABLED=1` is a different switch: it turns off Cognee's anonymous -product telemetry, not OpenTelemetry tracing. - -## Troubleshooting - -- **Dozens of `cognee.db.vector.search` errors in every trace.** The - exporter is not wrapped in `CogneeExportFilter` (or you use option B), or - `COGNEE_FI_COLLECTION_MISSING_AS_ERROR=true` is set. They are Cognee's - probes of collections that do not exist yet. -- **No traces at all with option A.** The `add_span_processor(BatchSpanProcessor(exporter))` - line is missing, or `register()` ran after Cognee's first traced call. -- **Option A traces stop part-way.** Something called - `cognee.disable_tracing()` or `shutdown_tracing()`, which shut down the - Future AGI provider. Unset `COGNEE_TRACING_ENABLED` instead. -- **Prompts or documents visible with option A.** `COGNEE_FI_CAPTURE_CONTENT=true` - is set, `capture_content=True` is passed, or the exporter is not wrapped in - `CogneeExportFilter`. -- **Error messages read `__REDACTED__ (content capture off)` and no traceback - appears.** That is option A's default, because serialized error detail can - quote content. Turn content capture on to export the original error - messages; tracebacks are never exported (they name local files), so check - Cognee's own log for frames. -- **Spans doubled.** Another instrumentor is wrapping Cognee or LiteLLM on top - of Cognee's own spans. Remove it. -- **Option B exports nothing, with no error.** Either the OpenTelemetry - packages are missing (plain `cognee==1.6.5` has none; with tracing on, - Cognee then silently uses no-op spans; install `cognee[tracing]==1.6.5`), - or the URL has no `:443`, so Cognee chose gRPC (for a self-hosted - collector, see option B above). -- **404 on `/tracer/v1/traces/v1/traces`.** Something appended the path to a - URL that already had it. Cognee does not append; give the full path once. -- **Five traces per request.** Wrap the Cognee calls in one parent span. -- **No token counts.** Cognee 1.6.5 does not record them. - -## Tests - -`tests/test_cognee_recipe.py` runs, side by side, six subprocesses with the -real `cognee==1.6.5` (the suite passes on `cognee==1.6.2` too). Each loads -`tests/loopback_guard/sitecustomize.py` -through `PYTHONPATH`, and so do the worker processes Cognee spawns for -LanceDB and Ladybug: in all of them, Python-level connections and DNS lookups -to any host other than 127.0.0.1 are refused and logged, and the test asserts -the log is empty and that the guard was installed in Cognee's workers. Native -code that opens sockets itself (for example a database extension download) is -not intercepted. Cognee's LLM and embedding endpoints are a loopback fake of -the OpenAI API (`tests/_fake_openai.py`), and spans go to the shared harness -`Receiver` (`python/tests/harness`), which serves `/v1/traces` and -`/tracer/v1/traces` like fi-collector but does not authenticate or store -anything: - -- `recipe`: `src/app.py` as written (option A, content off). -- `capture`: the same, with `COGNEE_FI_CAPTURE_CONTENT=true`. -- `probes_as_errors`: the same, with `COGNEE_FI_COLLECTION_MISSING_AS_ERROR=true`. -- `tracing_off`: the same, with `COGNEE_TRACING_ENABLED` unset. -- `no_readd`: `register()` without the re-add line, then one `cognee.add`. -- `otlp_env`: option B's environment variables, then one `cognee.add`. - -`tests/fixtures/cognee-1.6.2-recipe-spans.json` is Cognee's own spans from -the recipe run on cognee 1.6.2 as the Receiver decoded them, before the -export filter -(host paths replaced by ``, ``, `` and ``). The -fixture tests post it with the harness `post_otlp()`, check it with -`compare()`, run the same mapping and content assertions on it, and send it -through the export filter, with no Cognee installed. The live runs must match -the filtered fixture's shape (span names, kinds, attribute keys, statuses and -counts; the number of `cognee.db.graph.query` spans is compared only as -"at least one", because it changes between patch releases). The scenarios -point `TIKTOKEN_CACHE_DIR` at LiteLLM's bundled tokenizer files, because -cognee 1.6.5 loads tiktoken before LiteLLM sets that and would otherwise -download `cl100k_base`, which the loopback guard refuses; the recipe itself -needs no such setting. Re-record with `COGNEE_RECORD_FIXTURE=1` (it records the `capture` -run without the `fi.span.kind` the filter adds). - -From the repository root: - -```bash -env -u PYTHONPATH PYTHONPATH="python/examples/cognee:python:python/tests" \ - uv run --no-project --python 3.11 \ - --with pytest --with pytest-asyncio --with opentelemetry-api \ - --with opentelemetry-sdk --with opentelemetry-instrumentation \ - --with opentelemetry-exporter-otlp-proto-http --with wrapt --with requests \ - --with jsonschema --with protobuf --with opentelemetry-proto \ - --with 'cognee==1.6.5' \ - pytest python/examples/cognee/tests -q -p no:cacheprovider --noconftest -o addopts= -rfEs -``` +Configure before the first Cognee operation. End spans and call `handle.force_flush()` before `handle.shutdown()`; a span still open at shutdown is not exported. The handle owns only the adapter's exporter, not other processors on the provider. Cognee's `cognee.disable_tracing()` and `cognee.shutdown_tracing()` shut down the shared provider too, so do not use them while other application tracing is active. -Importing Cognee takes about a minute, so a full run takes a few minutes. -`register()` comes from this repository's `python/fi_instrumentation`, not a -published release. The environment above has no gRPC exporter installed; -option B's gRPC choice is covered by calling Cognee's URL classifier, not by a -gRPC export. +Search spans are mapped to `RETRIEVER` and structured-output model calls to `LLM`. Native error statuses, including missing-collection probes, are preserved. See the [adapter reference](../../frameworks/cognee/) for configuration options and lifecycle limits. diff --git a/python/examples/cognee/requirements.txt b/python/examples/cognee/requirements.txt index 32cbe0c32..935cf48ae 100644 --- a/python/examples/cognee/requirements.txt +++ b/python/examples/cognee/requirements.txt @@ -1,7 +1,2 @@ -# Not a traceAI package: these are the example's own dependencies. -cognee==1.6.5 -# Provides fi_instrumentation.register(); pulls in opentelemetry-sdk and the -# OTLP/HTTP exporter that Cognee's tracing needs. Pinned because app.py's -# re-add line relies on register() marking its exporter as a replaceable -# default (TH-8394); once that is fixed, the line goes away and the pin can move. -fi-instrumentation-otel>=1.1.0,<1.2 +cognee[tracing]==1.6.5 +traceai-cognee==0.1.0 diff --git a/python/examples/cognee/src/app.py b/python/examples/cognee/src/app.py index 23e2ebc9e..f495b45e9 100644 --- a/python/examples/cognee/src/app.py +++ b/python/examples/cognee/src/app.py @@ -1,83 +1,33 @@ -"""Send Cognee's own OpenTelemetry spans to Future AGI. - -Recipe example, not a package. Cognee 1.6.5 creates its spans itself when -``COGNEE_TRACING_ENABLED=true``. This file registers a Future AGI tracer -provider as the global OpenTelemetry provider before Cognee's first traced -call, so Cognee attaches to that provider instead of creating its own -(``setup_tracing`` in ``cognee/modules/observability/tracing.py``). It adds -no instrumentor and never calls ``setup_tracing``. The Future AGI exporter is -wrapped in ``CogneeExportFilter`` (``cognee_filter.py``, next to this file), -which keeps Cognee's prompts, documents, answers and queries, plus error text -and serialized exception stack traces that can quote them, out of the export -and marks search and LLM spans. - - COGNEE_TRACING_ENABLED=true python src/app.py "Who works on Lighthouse?" - -FI_API_KEY, FI_SECRET_KEY, FI_PROJECT_NAME and (optionally) FI_BASE_URL are -read from the environment; so are Cognee's own LLM_* and EMBEDDING_* -settings, and COGNEE_FI_CAPTURE_CONTENT=true to export content after all. -See README.md. -""" - -from __future__ import annotations - +"""Trace native Cognee operations using the maintained adapter.""" import asyncio import os import sys +from traceai_cognee import configure -from fi_instrumentation import register -from fi_instrumentation.fi_types import ProjectType -from fi_instrumentation.otel import BatchSpanProcessor, HTTPSpanExporter - -from cognee_filter import CogneeExportFilter - -DOCUMENT = "Ada works on the Lighthouse project." - +DOCUMENT = 'Ada works on the Lighthouse project.' def init_tracing(): - """Register the Future AGI provider. Call once, before any Cognee call.""" - provider = register( - project_name=os.environ["FI_PROJECT_NAME"], - project_type=ProjectType.OBSERVE, - # Cognee reuses the global provider when one is set. - set_global_tracer_provider=True, - verbose=False, - ) - # register() marks its exporter as a default that the next - # add_span_processor() call replaces (TH-8394). On its first traced call - # Cognee calls add_span_processor() on the global provider to attach its - # in-memory buffer, which would silently drop the Future AGI exporter. - # Adding the exporter again makes it a regular processor that Cognee's - # call leaves in place. HTTPSpanExporter() reads FI_BASE_URL, FI_API_KEY - # and FI_SECRET_KEY like register()'s own exporter; if you pass exporter - # or batch options to register(), pass the same ones here. - exporter = CogneeExportFilter(HTTPSpanExporter()) - provider.add_span_processor(BatchSpanProcessor(exporter)) - return provider - + return configure(project_name=os.environ['FI_PROJECT_NAME']) -async def remember_and_ask(question: str, document: str) -> list: +async def remember_and_ask(question, document): import cognee - await cognee.add(document) await cognee.cognify() return await cognee.search(question, query_type=cognee.SearchType.GRAPH_COMPLETION) - -def main(argv: list[str]) -> int: - provider = init_tracing() - question = argv[1] if len(argv) > 1 else "Who works on Lighthouse?" - document = argv[2] if len(argv) > 2 else DOCUMENT - # Cognee starts a new root span for each API call (and `add` ends its - # memory.store span before ingestion runs), so one parent span here is - # what puts add, cognify and search in one trace. - with provider.get_tracer(__name__).start_as_current_span("remember_and_ask"): - results = asyncio.run(remember_and_ask(question, document)) - for result in results: - print(result) - provider.force_flush() +def main(argv): + handle = init_tracing() + try: + question = argv[1] if len(argv) > 1 else 'Who works on Lighthouse?' + document = argv[2] if len(argv) > 2 else DOCUMENT + with handle.provider.get_tracer(__name__).start_as_current_span('remember_and_ask'): + results = asyncio.run(remember_and_ask(question, document)) + for result in results: + print(result) + handle.force_flush() + finally: + handle.shutdown() return 0 - -if __name__ == "__main__": +if __name__ == '__main__': sys.exit(main(sys.argv)) diff --git a/python/examples/cognee/src/cognee_filter.py b/python/examples/cognee/src/cognee_filter.py deleted file mode 100644 index 1ecfada1a..000000000 --- a/python/examples/cognee/src/cognee_filter.py +++ /dev/null @@ -1,275 +0,0 @@ -"""Export filter for Cognee's spans: content off by default, search and LLM kinds set. - -Recipe code, not a package: copy this file next to your app. It wraps the -exporter that sends spans to Future AGI (see app.py):: - - exporter = CogneeExportFilter(HTTPSpanExporter()) - provider.add_span_processor(BatchSpanProcessor(exporter)) - -and changes only what that exporter sends. It exports copies, so Cognee's -own span buffer and any other processor still see the original spans. - -Content. Cognee 1.6.5 writes prompts, document text, answers and search -questions to span attributes and has no setting to stop that, and its -errors can quote them (a content-policy rejection quotes the whole prompt). -By default the filter removes those attributes (``CONTENT_KEYS``) and, on -every span, all free-text error detail: exception messages and status -descriptions are replaced, exception stack traces are omitted, and other event -values are kept only if they are not text. Pass ``capture_content=True``, or set -``COGNEE_FI_CAPTURE_CONTENT=true``, to export content and error messages too. -With capture on, the filter also copies that content to ``input.value`` and -``output.value`` (``IO_SOURCES``), the keys Future AGI shows as a span's Input -and Output. - -Stack traces. ``exception.stacktrace`` is never exported, with or without -content capture: a serialized traceback carries local file paths (home -directory, user name, virtualenv layout) and can quote content. - -Span kinds. Cognee sets no span-kind attribute, so Future AGI would store -search and LLM spans as ``unknown``. The filter sets ``fi.span.kind`` by span -name (``span_kind``), only where none of ``SPAN_KIND_KEYS`` is set. - -Missing collections. In every search Cognee probes vector collections that -do not exist yet, and each probe's ``cognee.db.vector.search`` span ends in -ERROR with ``CollectionNotFoundError``, so every successful trace would show -dozens of errors. By default the filter exports a vector-search span whose -only error is that one as OK, without its exception event, and with -``cognee.vector_search.collection_missing=true``. Every other error stays -ERROR. Pass ``collection_missing_as_error=True``, or set -``COGNEE_FI_COLLECTION_MISSING_AS_ERROR=true``, to keep them as ERROR. - -If the filter fails on a span, it exports that span without content -attributes, events or status description; if even that fails, it drops the -span. It never raises into the export path. -""" - -from __future__ import annotations - -import json -import logging -import os -from typing import Any, Mapping, Optional, Sequence - -from opentelemetry.sdk.trace import Event, ReadableSpan -from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult -from opentelemetry.trace import Status, StatusCode - -logger = logging.getLogger(__name__) - -# Attributes in which Cognee 1.6.x exports content (paths in the cognee wheel). -CONTENT_KEYS = frozenset( - { - # LLM prompt arguments and response: modules/observability/get_observe.py - "langfuse.observation.input", - "langfuse.observation.output", - # The search question: api/v1/search/search.py, modules/search/methods/search.py - "memory.query.text", - "cognee.search.query", - # Graph query text: databases/graph/ladybug/adapter.py, neo4j_driver/adapter.py - "cognee.db.query", - } -) -# With content capture on, where a span's Input and Output come from: fi-collector -# fills them from input.value and output.value only. The first source key that -# is set wins; a span that already has the target key keeps it. -IO_SOURCES = ( - ("input", ("langfuse.observation.input", "memory.query.text", "cognee.search.query")), - ("output", ("langfuse.observation.output",)), -) -# fi-collector takes the span type from the first of these keys that is set. -SPAN_KIND_KEYS = ( - "fi.span.kind", - "gen_ai.span.kind", - "llm.request.type", - "openinference.span.kind", -) -# Cognee names a span after the decorated method; every -# @observe(as_type="generation") method in Cognee 1.6.5 is acreate_structured_output. -LLM_SPANS = frozenset({"cognee.observe.acreate_structured_output"}) -# What replaces error text by default. -DETAIL_REMOVED = "__REDACTED__ (content capture off)" -TYPE_ONLY = "{0} (detail removed: content capture off)" -# A probe of a vector collection that does not exist yet (see the module docstring). -VECTOR_SEARCH_SPAN = "cognee.db.vector.search" -COLLECTION_MISSING = ( - "cognee.infrastructure.databases.vector.exceptions.exceptions.CollectionNotFoundError" -) -COLLECTION_MISSING_KEY = "cognee.vector_search.collection_missing" -# The only event keys whose string values are kept: the exception's class name -# and "True"/"False". Cognee 1.6.5 adds no events other than OpenTelemetry's -# "exception" events; any other event gets the same allowlist. -SAFE_EVENT_KEYS = frozenset({"exception.type", "exception.escaped"}) - - -def span_kind(name: str) -> Optional[str]: - """The fi.span.kind for a Cognee span name, or None to leave the span as it is.""" - if name == "memory.retrieve" or name.startswith("cognee.search"): - return "RETRIEVER" - if name in LLM_SPANS: - return "LLM" - # Embedding spans already carry gen_ai.operation.name=embeddings. - return None - - -class CogneeExportFilter(SpanExporter): - """Wrap a span exporter; see the module docstring.""" - - def __init__( - self, - exporter: SpanExporter, - capture_content: Optional[bool] = None, - collection_missing_as_error: Optional[bool] = None, - ) -> None: - if capture_content is None: - capture_content = os.environ.get("COGNEE_FI_CAPTURE_CONTENT", "").lower() == "true" - if collection_missing_as_error is None: - collection_missing_as_error = ( - os.environ.get("COGNEE_FI_COLLECTION_MISSING_AS_ERROR", "").lower() == "true" - ) - self._exporter = exporter - self._capture_content = capture_content - self._collection_missing_as_error = collection_missing_as_error - - def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult: - out = [] - for span in spans: - try: - out.append(self._filtered(span)) - except Exception as error: - logger.warning( - "CogneeExportFilter: %s; exporting the span without content", - type(error).__name__, - ) - try: - out.append(self._stripped(span)) - except Exception as error: - logger.warning( - "CogneeExportFilter: %s; dropping the span", type(error).__name__ - ) - return self._exporter.export(out) - - def shutdown(self) -> None: - self._exporter.shutdown() - - def force_flush(self, timeout_millis: int = 30000) -> bool: - return self._exporter.force_flush(timeout_millis) - - def _filtered(self, span: ReadableSpan) -> ReadableSpan: - attributes = dict(span.attributes or {}) - kind = span_kind(span.name) - if kind and not any(key in attributes for key in SPAN_KIND_KEYS): - attributes["fi.span.kind"] = kind - if not self._collection_missing_as_error and _only_collection_missing(span): - attributes[COLLECTION_MISSING_KEY] = True - if self._capture_content: - _add_input_output(attributes) - else: - for key in CONTENT_KEYS: - attributes.pop(key, None) - return _copy(span, attributes, (), Status(StatusCode.OK)) - if self._capture_content: - _add_input_output(attributes) - events = [ - Event(e.name, _without_stacktrace(e.attributes), e.timestamp) for e in span.events - ] - return _copy(span, attributes, events, span.status) - for key in CONTENT_KEYS: - attributes.pop(key, None) - events = [Event(e.name, _without_detail(e.attributes), e.timestamp) for e in span.events] - status = span.status - if status.description: - # Only an ERROR status has a description; name the type of the - # exception that ended the span, if the span recorded one. - types = [e.attributes.get("exception.type") for e in events if e.name == "exception"] - if types and isinstance(types[-1], str): - status = Status(status.status_code, TYPE_ONLY.format(types[-1])) - else: - status = Status(status.status_code, DETAIL_REMOVED) - return _copy(span, attributes, events, status) - - def _stripped(self, span: ReadableSpan) -> ReadableSpan: - """The fail-closed copy: no content attributes, no events, no status description.""" - attributes = {k: v for k, v in (span.attributes or {}).items() if k not in CONTENT_KEYS} - return _copy(span, attributes, (), Status(span.status.status_code)) - - -def _add_input_output(attributes: dict) -> None: - """Copy captured content to the keys Future AGI shows as Input and Output.""" - for prefix, sources in IO_SOURCES: - if prefix + ".value" in attributes: - continue - value = next((attributes[k] for k in sources if isinstance(attributes.get(k), str)), None) - if value is None: - continue - attributes[prefix + ".value"] = value - attributes.setdefault( - prefix + ".mime_type", "application/json" if _is_json(value) else "text/plain" - ) - - -def _is_json(value: str) -> bool: - """True for a JSON object or array (Cognee's serialized LLM arguments).""" - try: - return isinstance(json.loads(value), (dict, list)) - except ValueError: - return False - - -def _only_collection_missing(span: ReadableSpan) -> bool: - """True for a vector-search span whose only error is a missing collection.""" - if span.name != VECTOR_SEARCH_SPAN or span.status.status_code != StatusCode.ERROR: - return False - return bool(span.events) and all( - event.name == "exception" - and (event.attributes or {}).get("exception.type") == COLLECTION_MISSING - for event in span.events - ) - - -def _without_stacktrace(attributes: Optional[Mapping[str, Any]]) -> dict: - """An event's attributes as recorded, minus the serialized stack trace (local paths).""" - return {k: v for k, v in (attributes or {}).items() if k != "exception.stacktrace"} - - -def _without_detail(attributes: Optional[Mapping[str, Any]]) -> dict: - """An event's attributes without free text: an allowlist of keys and value types.""" - attributes = attributes or {} - out = {} - for key, value in attributes.items(): - if key == "exception.message": - out[key] = DETAIL_REMOVED - elif key == "exception.stacktrace": - # A serialized traceback has no trustworthy boundary between its - # real frames and traceback-shaped exception-message content. - continue - elif key in SAFE_EVENT_KEYS or _not_text(value): - out[key] = value - return out - - -def _not_text(value: Any) -> bool: - """True for a number or boolean, or a sequence of them.""" - if isinstance(value, (bool, int, float)): - return True - return isinstance(value, (list, tuple)) and all( - isinstance(item, (bool, int, float)) for item in value - ) - - -def _copy( - span: ReadableSpan, attributes: Mapping[str, Any], events: Sequence[Event], status: Status -) -> ReadableSpan: - return ReadableSpan( - name=span.name, - context=span.context, - parent=span.parent, - resource=span.resource, - attributes=attributes, - events=events, - links=span.links, - kind=span.kind, - status=status, - start_time=span.start_time, - end_time=span.end_time, - instrumentation_scope=span.instrumentation_scope, - ) diff --git a/python/examples/cognee/tests/_run_app.py b/python/examples/cognee/tests/_run_app.py new file mode 100644 index 000000000..1456cd4fa --- /dev/null +++ b/python/examples/cognee/tests/_run_app.py @@ -0,0 +1,19 @@ +"""Run the maintained example with optional content capture in a fresh process.""" +import importlib.util +import os +from pathlib import Path +import sys + +import traceai_cognee + +app_path = Path(__file__).resolve().parents[1] / "src" / "app.py" +spec = importlib.util.spec_from_file_location("cognee_example", app_path) +app = importlib.util.module_from_spec(spec) +spec.loader.exec_module(app) + +# Keep the application path unchanged; opt in only for the capture regression. +if os.environ["TEST_CAPTURE_CONTENT"] == "true": + app.init_tracing = lambda: traceai_cognee.configure( + project_name=os.environ["FI_PROJECT_NAME"], capture_content=True + ) +sys.exit(app.main([str(app_path), *sys.argv[1:]])) diff --git a/python/examples/cognee/tests/cognee_add.py b/python/examples/cognee/tests/cognee_add.py deleted file mode 100644 index bd6120937..000000000 --- a/python/examples/cognee/tests/cognee_add.py +++ /dev/null @@ -1,59 +0,0 @@ -"""Scenario script: one ``cognee.add`` call, with tracing set up per argv[1]. - - python cognee_add.py otlp-env [URL ...] # no register(); Cognee's own OTLP exporter - python cognee_add.py no-readd # register() without the recipe's re-add line - -Prints one JSON line with the number of spans Cognee's in-memory buffer holds, -so the test can tell "Cognee made no spans" apart from "spans were made but -not exported". In otlp-env mode it also reports, for each URL argument, -whether Cognee 1.6.2 treats it as HTTP-only (otherwise Cognee picks the gRPC -exporter when that package is installed). -""" - -from __future__ import annotations - -import asyncio -import json -import os -import sys - -DOCUMENT = "Ada works on the Lighthouse project." - - -def main(argv: list[str]) -> int: - mode = argv[1] - provider = None - if mode == "no-readd": - from fi_instrumentation import register - from fi_instrumentation.fi_types import ProjectType - - provider = register( - project_name=os.environ["FI_PROJECT_NAME"], - project_type=ProjectType.OBSERVE, - set_global_tracer_provider=True, - verbose=False, - ) - elif mode != "otlp-env": - raise SystemExit("unknown mode: " + mode) - - import cognee - - asyncio.run(cognee.add(DOCUMENT)) - report = { - "mode": mode, - "buffered_spans": sum(len(trace.spans()) for trace in cognee.get_all_traces()), - } - if mode == "otlp-env": - from cognee.modules.observability.tracing import _requires_http_exporter - - report["http_only"] = {url: _requires_http_exporter(url) for url in argv[2:]} - print(json.dumps(report)) - if provider is not None: - provider.force_flush() - # In otlp-env mode Cognee's own provider flushes its BatchSpanProcessor - # at interpreter exit. - return 0 - - -if __name__ == "__main__": - sys.exit(main(sys.argv)) diff --git a/python/examples/cognee/tests/loopback_guard/sitecustomize.py b/python/examples/cognee/tests/loopback_guard/sitecustomize.py deleted file mode 100644 index 5f8abd1c2..000000000 --- a/python/examples/cognee/tests/loopback_guard/sitecustomize.py +++ /dev/null @@ -1,93 +0,0 @@ -"""Block and log every non-loopback network connection, in every test process. - -The test scenarios put this directory first on ``PYTHONPATH``, so Python's -``site`` imports this file at the start of each interpreter they start: -the scenario script, and the multiprocessing "spawn" workers in which Cognee -1.6.2 runs LanceDB and Ladybug (they inherit the environment). This file -then runs the interpreter's own ``sitecustomize``, if it has one (Homebrew's -Python does), so nothing else changes. - -When ``LOOPBACK_GUARD_LOG`` is set, ``socket.socket.connect``/``connect_ex`` -and ``socket.getaddrinfo`` refuse any host other than 127.0.0.1, ::1 or -localhost. Each refused attempt, and each process the guard is installed in, -is appended as a JSON line to that file, so a test can assert that Cognee, -LiteLLM and the example tried to reach nothing else (no LLM provider, no -Cognee telemetry, no Future AGI). Unix sockets are not affected, and neither -is native code that opens sockets itself. -""" - -from __future__ import annotations - -import importlib.machinery -import importlib.util -import json -import os -import socket -import sys - -_ALLOWED_HOSTS = {"127.0.0.1", "::1", "localhost"} -_LOG_PATH = os.environ.get("LOOPBACK_GUARD_LOG", "") - - -def _host(value: object) -> str: - if isinstance(value, bytes): - return value.decode("ascii", "replace") - return str(value) - - -def _log(record: dict) -> None: - with open(_LOG_PATH, "a", encoding="utf-8") as log: - log.write(json.dumps(record) + "\n") - - -def _refuse(kind: str, target: object) -> None: - _log({"kind": kind, "target": repr(target), "pid": os.getpid()}) - - -_original_connect = socket.socket.connect -_original_connect_ex = socket.socket.connect_ex -_original_getaddrinfo = socket.getaddrinfo - - -def _is_blocked(sock: socket.socket, address: object) -> bool: - if sock.family not in (socket.AF_INET, socket.AF_INET6): - return False - host = address[0] if isinstance(address, tuple) else address - return _host(host) not in _ALLOWED_HOSTS - - -def _connect(self: socket.socket, address: object) -> None: - if _is_blocked(self, address): - _refuse("connect", address) - raise OSError("blocked by loopback guard: {0!r}".format(address)) - return _original_connect(self, address) - - -def _connect_ex(self: socket.socket, address: object) -> int: - if _is_blocked(self, address): - _refuse("connect_ex", address) - raise OSError("blocked by loopback guard: {0!r}".format(address)) - return _original_connect_ex(self, address) - - -def _getaddrinfo(host: object, *args: object, **kwargs: object): # type: ignore[no-untyped-def] - if host is not None and _host(host) not in _ALLOWED_HOSTS: - _refuse("getaddrinfo", host) - raise socket.gaierror("blocked by loopback guard: {0!r}".format(host)) - return _original_getaddrinfo(host, *args, **kwargs) - - -def _run_shadowed_sitecustomize() -> None: - here = os.path.dirname(os.path.abspath(__file__)) - rest = [entry for entry in sys.path if os.path.abspath(entry or os.curdir) != here] - spec = importlib.machinery.PathFinder.find_spec("sitecustomize", rest) - if spec is not None and spec.loader is not None: - spec.loader.exec_module(importlib.util.module_from_spec(spec)) - - -if _LOG_PATH: - socket.socket.connect = _connect # type: ignore[method-assign] - socket.socket.connect_ex = _connect_ex # type: ignore[method-assign] - socket.getaddrinfo = _getaddrinfo # type: ignore[assignment] - _log({"kind": "installed", "pid": os.getpid(), "argv": list(sys.argv)}) -_run_shadowed_sitecustomize() diff --git a/python/examples/cognee/tests/test_cognee_adapter.py b/python/examples/cognee/tests/test_cognee_adapter.py new file mode 100644 index 000000000..61fdcc83e --- /dev/null +++ b/python/examples/cognee/tests/test_cognee_adapter.py @@ -0,0 +1,161 @@ +"""Maintained adapter contracts using real Cognee and synthetic localhost providers. + +Run with the adapter, Cognee's tracing extra and pytest installed, under an OS +sandbox denying outbound network except localhost. No real provider keys are used. +""" +import base64 +import importlib.util +import json +from pathlib import Path +import subprocess +import sys + +import pytest +from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import Event, ReadableSpan +from opentelemetry.trace import SpanContext, SpanKind, Status, StatusCode, TraceFlags +from traceai_cognee._exporter import project + +TESTS = Path(__file__).resolve().parent +EXAMPLE = TESTS.parent +spec = importlib.util.spec_from_file_location("fake_openai", TESTS / "_fake_openai.py") +fake_module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(fake_module) + + +def test_example_uses_maintained_adapter(): + requirements = (EXAMPLE / "requirements.txt").read_text().splitlines() + assert "cognee[tracing]==1.6.5" in requirements + assert "traceai-cognee==0.1.0" in requirements + source = (EXAMPLE / "src/app.py").read_text() + assert "from traceai_cognee import configure" in source + assert "cognee_filter" not in source + readme = (EXAMPLE / "README.md").read_text() + assert "capture_content=True" in readme + assert "COGNEE_TRACING_ENABLED=true" in readme + assert "cognee.disable_tracing()" in readme + + +@pytest.mark.parametrize("capture", [False, True]) +def test_recorded_native_spans_keep_identity_status_and_privacy(capture): + recorded = json.loads((TESTS / "fixtures/cognee-1.6.2-recipe-spans.json").read_text()) + kinds = {"SPAN_KIND_INTERNAL": SpanKind.INTERNAL, "SPAN_KIND_CLIENT": SpanKind.CLIENT} + statuses = {"STATUS_CODE_OK": StatusCode.OK, "STATUS_CODE_ERROR": StatusCode.ERROR} + + def context(trace_id, span_id): + return SpanContext(int.from_bytes(base64.b64decode(trace_id), "big"), + int.from_bytes(base64.b64decode(span_id), "big"), + False, TraceFlags(1)) + + def attrs(items): + return {item["key"]: next(iter(item["value"].values())) for item in items} + + output = [] + for item in recorded: + parent = context(item["traceId"], item["parentSpanId"]) if item.get("parentSpanId") else None + status = item.get("status", {}) + native = ReadableSpan( + name=item["name"], context=context(item["traceId"], item["spanId"]), parent=parent, + resource=Resource({"private": "unapproved-resource"}), + attributes=attrs(item.get("attributes", [])), + events=[Event(e["name"], attrs(e.get("attributes", [])), int(e["timeUnixNano"])) + for e in item.get("events", [])], + kind=kinds[item["kind"]], status=Status(statuses.get(status.get("code"), StatusCode.UNSET), status.get("message")), + start_time=int(item["startTimeUnixNano"]), end_time=int(item["endTimeUnixNano"]), + ) + before = dict(native.attributes) + mapped = project(native, "synthetic-project", capture) + assert mapped.context == native.context and mapped.parent == native.parent + assert mapped.start_time == native.start_time and mapped.end_time == native.end_time + assert mapped.status.status_code == native.status.status_code + assert mapped.status.description is None + assert dict(native.attributes) == before + assert dict(mapped.resource.attributes) == {"project_name": "synthetic-project", "project_type": "observe"} + assert all(not event.attributes for event in mapped.events) + if item["name"] == "memory.retrieve": + assert mapped.attributes["fi.span.kind"] == "RETRIEVER" + if item["name"] == "cognee.observe.acreate_structured_output": + assert mapped.attributes["fi.span.kind"] == "LLM" + if not capture: + assert not {"langfuse.observation.input", "langfuse.observation.output", "memory.query.text", + "cognee.search.query", "cognee.db.query", "input.value", "output.value"}.intersection(mapped.attributes) + output.append(mapped) + assert any(span.status.status_code == StatusCode.ERROR for span in output) + assert len(output) == len(recorded) + + +@pytest.mark.parametrize("capture", [False, True]) +def test_example_native_operations_export_once_with_content_policy(tmp_path, capture): + # The process runs real Cognee; only model responses and OTLP transport are local fakes. + fake = fake_module.FakeOpenAI() + bodies = [] + auth_headers = [] + + class CombinedHandler(fake._server.RequestHandlerClass): + def do_POST(self): + if self.path != "/tracer/v1/traces": + return super().do_POST() + length = int(self.headers.get("Content-Length", "0")) + if not 0 < length <= 4_000_000: + self.send_error(413) + return + bodies.append(self.rfile.read(length)) + auth_headers.append((self.headers.get("X-Api-Key"), self.headers.get("X-Secret-Key"))) + self.send_response(200) + self.send_header("Content-Length", "0") + self.end_headers() + + fake._server.RequestHandlerClass = CombinedHandler + for name in ("home", "tmp", "cache", "config", "data", "system", "logs"): + (tmp_path / name).mkdir() + env = { + "HOME": str(tmp_path / "home"), "TMPDIR": str(tmp_path / "tmp"), + "XDG_CACHE_HOME": str(tmp_path / "cache"), "XDG_CONFIG_HOME": str(tmp_path / "config"), + "PATH": "/usr/bin:/bin", "LANG": "C.UTF-8", "NO_COLOR": "1", + "TELEMETRY_DISABLED": "1", "LITELLM_LOCAL_MODEL_COST_MAP": "True", + "OTEL_TRACES_EXPORTER": "none", "OTEL_METRICS_EXPORTER": "none", "OTEL_LOGS_EXPORTER": "none", + "COGNEE_TRACING_ENABLED": "true", "TEST_CAPTURE_CONTENT": str(capture).lower(), + "LLM_PROVIDER": "openai", "LLM_MODEL": "openai/gpt-4o-mini", "LLM_ENDPOINT": fake.base_url, + "LLM_API_KEY": "synthetic-model-key", "OPENAI_API_KEY": "synthetic-model-key", + "EMBEDDING_PROVIDER": "openai", "EMBEDDING_MODEL": "openai/text-embedding-3-small", + "EMBEDDING_ENDPOINT": fake.base_url, "EMBEDDING_API_KEY": "synthetic-model-key", "EMBEDDING_DIMENSIONS": "16", + "DATA_ROOT_DIRECTORY": str(tmp_path / "data"), "SYSTEM_ROOT_DIRECTORY": str(tmp_path / "system"), + "CACHE_ROOT_DIRECTORY": str(tmp_path / "cache"), "COGNEE_LOGS_DIR": str(tmp_path / "logs"), + "FI_BASE_URL": fake.base_url.removesuffix("/v1"), "FI_PROJECT_NAME": "synthetic-project", + "FI_API_KEY": "synthetic-fi-api", "FI_SECRET_KEY": "synthetic-fi-secret", + } + import litellm + tokenizers = Path(litellm.__file__).parent / "litellm_core_utils/tokenizers" + env["TIKTOKEN_CACHE_DIR"] = str(tokenizers) + try: + result = subprocess.run( + [sys.executable, "-I", "-B", str(TESTS / "_run_app.py"), + "Who works on Lighthouse? QUERY-MARKER-4c9", "Ada works on Lighthouse. DOC-MARKER-3b7"], + cwd=tmp_path, env=env, capture_output=True, timeout=240, + ) + assert result.returncode == 0, result.stderr.decode(errors="replace")[-4000:] + assert fake_module.ANSWER_MARKER.encode() in result.stdout + finally: + fake.close() + assert bodies and all(headers == ("synthetic-fi-api", "synthetic-fi-secret") for headers in auth_headers) + spans = [] + for body in bodies: + request = ExportTraceServiceRequest.FromString(body) + for resource in request.resource_spans: + resource_attrs = {a.key: a.value.string_value for a in resource.resource.attributes} + assert resource_attrs == {"project_name": "synthetic-project", "project_type": "observe"} + spans.extend(span for scope in resource.scope_spans for span in scope.spans) + identities = {(s.trace_id, s.span_id) for s in spans} + assert len(identities) == len(spans) + assert len({s.trace_id for s in spans}) == 1 + for name in ("remember_and_ask", "memory.store", "memory.process", "memory.retrieve"): + assert sum(span.name == name for span in spans) == 1 + assert any(s.status.code == 2 for s in spans) # Do not rewrite native errors to OK. + assert all(not s.status.message and all(not e.attributes for e in s.events) for s in spans) + wire = b"".join(bodies) + for marker in (b"QUERY-MARKER-4c9", fake_module.ANSWER_MARKER.encode()): + assert (marker in wire) is capture + assert all(secret not in wire for secret in (b"synthetic-model-key", b"synthetic-fi-api", b"synthetic-fi-secret")) + for name, kind in (("memory.retrieve", "RETRIEVER"), ("cognee.observe.acreate_structured_output", "LLM")): + assert any(s.name == name and any(a.key == "fi.span.kind" and a.value.string_value == kind for a in s.attributes) for s in spans) diff --git a/python/examples/cognee/tests/test_cognee_recipe.py b/python/examples/cognee/tests/test_cognee_recipe.py deleted file mode 100644 index 3c06ce6c6..000000000 --- a/python/examples/cognee/tests/test_cognee_recipe.py +++ /dev/null @@ -1,1719 +0,0 @@ -"""Contract test for the Cognee tracing recipe. - -Runs ``src/app.py`` as written with the real ``cognee`` (pinned 1.6.5; 1.6.2 -also passes) and traceAI's ``register()``, plus two one-call scenario -scripts, each in a subprocess whose network, and that of the worker -processes Cognee spawns, is limited to -127.0.0.1 (``loopback_guard/sitecustomize.py``). Cognee's LLM and -embedding endpoints are a loopback fake of the OpenAI API -(``_fake_openai.py``). Spans go to the shared harness ``Receiver``, which -serves ``/v1/traces`` and ``/tracer/v1/traces`` like fi-collector's HTTP mux -but does not authenticate, stamp projects or store anything. - -The recorded fixture (``fixtures/cognee-1.6.2-recipe-spans.json``) is -Cognee's own spans from the recipe run, before the export filter -(``src/cognee_filter.py``) changes them. The fixture tests post it with -``post_otlp()`` and check it with ``compare()``, run the filter over it, and -compare the live runs with the filtered result, without Cognee installed. -Regenerate it with ``COGNEE_RECORD_FIXTURE=1``. - -All keys are placeholders. Nothing here contacts an LLM provider, Cognee's -telemetry host or Future AGI. -""" - -from __future__ import annotations - -import base64 -import concurrent.futures -import copy -import importlib.util -import json -import os -import re -import sys -import traceback -from pathlib import Path -from typing import Any - -import pytest -from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter -from opentelemetry.sdk.resources import Resource -from opentelemetry.sdk.trace import Event, ReadableSpan, TracerProvider -from opentelemetry.sdk.trace.export import SimpleSpanProcessor, SpanExportResult -from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter -from opentelemetry.trace import SpanContext, SpanKind, Status, StatusCode, TraceFlags - -from harness import Receiver, compare, post_otlp, run - -TESTS_DIR = Path(__file__).resolve().parent -RECIPE_DIR = TESTS_DIR.parent -SRC_DIR = RECIPE_DIR / "src" -PYTHON_DIR = RECIPE_DIR.parents[1] -REPO_ROOT = PYTHON_DIR.parent -APP = SRC_DIR / "app.py" -README = RECIPE_DIR / "README.md" -REQUIREMENTS = RECIPE_DIR / "requirements.txt" -ADD_SCRIPT = TESTS_DIR / "cognee_add.py" -# Put first on a scenario's PYTHONPATH, so every interpreter it starts loads the guard. -GUARD_DIR = TESTS_DIR / "loopback_guard" -FIXTURE = TESTS_DIR / "fixtures" / "cognee-1.6.2-recipe-spans.json" - -sys.path.insert(0, str(TESTS_DIR)) -sys.path.insert(0, str(SRC_DIR)) -from _fake_openai import ANSWER_MARKER, FakeOpenAI # noqa: E402 - -PROJECT = "cognee-recipe-contract" -FI_API_KEY = "fi-api-placeholder-0000" -FI_SECRET_KEY = "fi-secret-placeholder-0000" -LLM_API_KEY = "llm-placeholder-key-0000" -LLM_MODEL = "openai/gpt-4o-mini" -EMBEDDING_MODEL = "openai/text-embedding-3-small" - -# Content markers: the document text, the search question and the fake -# model's answer. -DOC_MARKER = "DMARK7f3a" -QUERY_MARKER = "QMARK91c2" -DOCUMENT = "Ada works on the Lighthouse project. " + DOC_MARKER -QUESTION = "Who works on Lighthouse? " + QUERY_MARKER -MARKERS = (DOC_MARKER, QUERY_MARKER, ANSWER_MARKER) - -# Which attribute keys carry each kind of content in Cognee's own spans -# (README.md, "Content"). With capture_content=True the filter keeps them. -CONTENT_KEYS = { - DOC_MARKER: {"langfuse.observation.input"}, - QUERY_MARKER: {"langfuse.observation.input", "cognee.search.query", "memory.query.text"}, - ANSWER_MARKER: {"langfuse.observation.output"}, -} -# What the filter removes by default: every key above, plus the graph query -# text (cognee.db.query, which held no marker in the recorded run). -FILTERED_KEYS = set().union(*CONTENT_KEYS.values()) | {"cognee.db.query"} -# With capture_content=True the filter also copies that content to the keys -# fi-collector shows as a span's Input and Output (README.md, "The export -# filter"): the prompt or the question to input.value, the response to -# output.value. -CAPTURE_IO_KEYS = { - DOC_MARKER: {"input.value"}, - QUERY_MARKER: {"input.value"}, - ANSWER_MARKER: {"output.value"}, -} -# Every key the filter adds in capture mode (none of them is Cognee's own). -IO_KEYS = ("input.value", "input.mime_type", "output.value", "output.mime_type") -# Cognee probes vector collections that do not exist yet; each probe span -# ends in ERROR with this exception in every successful run. By default the -# filter exports such a span as OK with MISSING_KEY (README.md, "The export -# filter"); COGNEE_FI_COLLECTION_MISSING_AS_ERROR=true keeps it ERROR. -VECTOR_SEARCH = "cognee.db.vector.search" -COLLECTION_MISSING = ( - "cognee.infrastructure.databases.vector.exceptions.exceptions.CollectionNotFoundError" -) -MISSING_KEY = "cognee.vector_search.collection_missing" -MISSING_SWITCH = "COGNEE_FI_COLLECTION_MISSING_AS_ERROR" -# What the filter puts in place of error text by default. -DETAIL_REMOVED = "__REDACTED__ (content capture off)" -TYPE_ONLY = " (detail removed: content capture off)" -TRACEBACK_HEADER = "Traceback (most recent call last):" - -APP_SPAN = "remember_and_ask" -INTERNAL = "SPAN_KIND_INTERNAL" -CLIENT = "SPAN_KIND_CLIENT" -GENERATION_SPAN = "cognee.observe.acreate_structured_output" -EMBEDDING_SPAN = "cognee.observe.embed_text" -# Every span name the recipe run produces, with its OTel span kind -# (README.md, "Spans Cognee 1.6.2 emits"). -SPAN_KINDS = { - APP_SPAN: INTERNAL, - "memory.store": INTERNAL, - "memory.process": INTERNAL, - "memory.retrieve": INTERNAL, - "cognee.pipeline.task.resolve_data_directories": INTERNAL, - "cognee.pipeline.task.ingest_data": INTERNAL, - "cognee.pipeline.task.classify_documents": INTERNAL, - "cognee.pipeline.task.extract_chunks_from_documents": INTERNAL, - "cognee.pipeline.task.extract_graph_and_summarize": INTERNAL, - "cognee.pipeline.task.add_data_points": INTERNAL, - GENERATION_SPAN: CLIENT, - EMBEDDING_SPAN: CLIENT, - "cognee.llm.completion": INTERNAL, - "cognee.db.graph.query": INTERNAL, - "cognee.db.vector.search": INTERNAL, - "cognee.search.authorize": INTERNAL, - "cognee.search.dataset": INTERNAL, - "cognee.retrieval.get_objects": INTERNAL, - "cognee.retrieval.triplet_search": INTERNAL, - "cognee.retrieval.vector_search": INTERNAL, - "cognee.retrieval.embed_query": INTERNAL, - "cognee.retrieval.get_context": INTERNAL, - "cognee.retrieval.get_completion": INTERNAL, - "cognee.session.get_session": INTERNAL, - "cognee.session.add_qa": INTERNAL, -} -COGNEE_ROOTS = ("memory.store", "memory.process", "memory.retrieve") -# fi-collector reads the span kind from these keys, then falls back to -# gen_ai.operation.name (exporter/clickhouse25exporter/converter.go). -SPAN_KIND_KEYS = ("fi.span.kind", "gen_ai.span.kind", "llm.request.type", "openinference.span.kind") -# The fi.span.kind the filter sets, by span name, on the recipe run's spans. -FILTER_KINDS = { - "memory.retrieve": "RETRIEVER", - "cognee.search.authorize": "RETRIEVER", - "cognee.search.dataset": "RETRIEVER", - GENERATION_SPAN: "LLM", -} -TOKEN_KEYS = ( - "llm.token_count.prompt", - "llm.token_count.completion", - "llm.token_count.total", - "gen_ai.usage.input_tokens", - "gen_ai.usage.output_tokens", - "gen_ai.usage.total_tokens", - "llm.usage.prompt_tokens", - "llm.usage.completion_tokens", - "llm.usage.total_tokens", -) - -# Production URL forms for the no-register() option, and whether Cognee -# 1.6.2 treats each as HTTP-only (tracing.py _requires_http_exporter). -OTLP_URL_FORMS = { - "https://api.futureagi.com/tracer/v1/traces": False, - "https://api.futureagi.com:443/tracer/v1/traces": True, -} -RUN_TIMEOUT_SECONDS = 900 - -COGNEE_INSTALLED = importlib.util.find_spec("cognee") is not None - - -def litellm_tokenizers_dir() -> str | None: - """LiteLLM's bundled tiktoken files, so Cognee never downloads them. - - Cognee 1.6.5 builds its embedding tokenizer with tiktoken before LiteLLM - has pointed TIKTOKEN_CACHE_DIR at the BPE files it ships, so tiktoken would - fetch cl100k_base from openaipublic.blob.core.windows.net, which the - loopback guard refuses. A test-environment setting only: the recipe itself - needs no such variable. - """ - spec = importlib.util.find_spec("litellm") - for location in (spec.submodule_search_locations or []) if spec else []: - tokenizers = Path(location) / "litellm_core_utils" / "tokenizers" - if tokenizers.is_dir(): - return str(tokenizers) - return None - - -TOKENIZERS_DIR = litellm_tokenizers_dir() - - -# -------------------------------------------------------------------------- -# Helpers shared by the live run and the recorded fixture -# -------------------------------------------------------------------------- - - -def attributes(span: dict[str, Any]) -> dict[str, Any]: - flat = {} - for attribute in span.get("attributes", []): - value = attribute.get("value", {}) - flat[attribute["key"]] = next(iter(value.values())) if value else None - return flat - - -def spans_named(spans: list[dict[str, Any]], name: str) -> list[dict[str, Any]]: - return [span for span in spans if span["name"] == name] - - -# Spans whose number depends on the Cognee patch release rather than on the -# recipe: one add/cognify/search run issues 14 graph queries on cognee 1.6.2 -# and 17 on 1.6.5. shape() compares their names, kinds, keys and statuses, not -# their count. -VERSION_DEPENDENT_COUNTS = frozenset({"cognee.db.graph.query"}) - - -def shape(spans: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: - """Per span name: kinds, attribute keys, status codes and count.""" - out: dict[str, dict[str, Any]] = {} - for span in spans: - entry = out.setdefault( - span["name"], {"kinds": set(), "keys": set(), "status": set(), "count": 0} - ) - entry["kinds"].add(span.get("kind")) - entry["keys"] |= set(attributes(span)) - entry["status"].add(span.get("status", {}).get("code", "STATUS_CODE_UNSET")) - entry["count"] += 1 - for name in VERSION_DEPENDENT_COUNTS & set(out): - out[name]["count"] = "at least 1" - return out - - -def host_paths() -> tuple[str, ...]: - """Machine-local path fragments that must not reach a recorded fixture.""" - return (str(Path.home()), "/Users/", "/home/", "/tmp/pytest-of-", "/var/folders", "/private/") - - -def scrub(spans: list[dict[str, Any]], tmp_base: Path) -> list[dict[str, Any]]: - """Replace machine-local paths (stack traces, data dirs) before recording.""" - text = json.dumps(spans) - for path, label in ( - (os.path.realpath(tmp_base), ""), - (str(tmp_base), ""), - (str(REPO_ROOT), ""), - ): - text = text.replace(path, label) - text = re.sub(r'File \\"[^"\\]*?/lib/python3\.\d+/', r'File \\"/', text) - text = text.replace(str(Path.home()), "") - return json.loads(text) - - -def without_filter_kinds(spans: list[dict[str, Any]]) -> list[dict[str, Any]]: - """Drop the keys the filter adds (fi.span.kind, capture-mode I/O), leaving Cognee's spans.""" - added = {"fi.span.kind", *IO_KEYS} - spans = copy.deepcopy(spans) - for span in spans: - span["attributes"] = [a for a in span.get("attributes", []) if a["key"] not in added] - return spans - - -def assert_span_names_and_kinds(spans: list[dict[str, Any]]) -> None: - assert {span["name"]: span["kind"] for span in spans} == SPAN_KINDS - for span in spans: - assert span["kind"] == SPAN_KINDS[span["name"]], span["name"] - - -def assert_collector_mapping_inputs(spans: list[dict[str, Any]], kinds: dict[str, str]) -> None: - """Assert the keys fi-collector maps from, as README.md's table states. - - ``kinds`` is the fi.span.kind expected per span name: empty for Cognee's - own spans, FILTER_KINDS after the recipe's export filter. - """ - for span in spans: - attrs = attributes(span) - expected_kind = {"fi.span.kind": kinds[span["name"]]} if span["name"] in kinds else {} - assert {key: attrs[key] for key in SPAN_KIND_KEYS if key in attrs} == expected_kind - assert not set(TOKEN_KEYS) & set(attrs), span["name"] - if span["name"] == EMBEDDING_SPAN: - assert attrs["gen_ai.operation.name"] == "embeddings" - assert attrs["gen_ai.request.model"] == EMBEDDING_MODEL - assert attrs["gen_ai.provider.name"] == "openai" - assert attrs["cognee.span.category"] == "embeddings" - else: - assert "gen_ai.operation.name" not in attrs, span["name"] - if span["name"] == GENERATION_SPAN: - # AC-04: the model key is present, so no copy processor is written. - assert attrs["gen_ai.request.model"] == LLM_MODEL - assert attrs["gen_ai.system"] == "litellm-native" - assert attrs["cognee.span.category"] == "generation" - assert attrs["langfuse.observation.type"] == "generation" - generation_spans = spans_named(spans, GENERATION_SPAN) - assert generation_spans - # cognee.llm.model is set only on schema-bound (structured) calls. - with_cognee_model = [s for s in generation_spans if "cognee.llm.model" in attributes(s)] - assert 0 < len(with_cognee_model) < len(generation_spans) - (search,) = spans_named(spans, "memory.retrieve") - search_attrs = attributes(search) - assert search_attrs["cognee.search.type"] == "GRAPH_COMPLETION" - assert search_attrs["memory.query.type"] == "GRAPH_COMPLETION" - assert search_attrs["memory.operation"] == "retrieve" - - -def assert_default_content(spans: list[dict[str, Any]], io_copies: bool = True) -> None: - """Among span attributes, captured content is under exactly the documented keys. - - Those are Cognee's own keys plus, after the filter in capture mode - (``io_copies``), its input.value/output.value copies. Error text - (exception events, status) is not checked here: with content capture on - it is exported as Cognee recorded it (README.md, "Content"). - """ - for marker, expected_keys in CONTENT_KEYS.items(): - found = { - key - for span in spans - for key, value in attributes(span).items() - if marker in str(value) - } - assert found == expected_keys | (CAPTURE_IO_KEYS[marker] if io_copies else set()), marker - for span in spans_named(spans, EMBEDDING_SPAN): - assert DOC_MARKER not in json.dumps(span) - - -def found_in(payload: Any, needles: Any) -> list[str]: - """The needles that occur in payload's JSON (a list keeps pytest's diff of it small).""" - text = json.dumps(payload) - return [needle for needle in needles if needle in text] - - -def assert_no_content(spans: list[dict[str, Any]]) -> None: - """No marker anywhere on the wire (attributes, status, events), no content key.""" - assert found_in(spans, MARKERS) == [] - for span in spans: - keys = set(attributes(span)) - for event in span.get("events", []): - keys |= set(attributes(event)) - assert not keys & FILTERED_KEYS, span["name"] - - -def assert_no_secrets(payload: Any) -> None: - assert found_in(payload, (FI_API_KEY, FI_SECRET_KEY, LLM_API_KEY)) == [] - - -def assert_one_trace_under_app_span(spans: list[dict[str, Any]]) -> None: - assert len({span["traceId"] for span in spans}) == 1 - roots = [span for span in spans if not span.get("parentSpanId")] - assert [root["name"] for root in roots] == [APP_SPAN] - root_id = roots[0]["spanId"] - for name in COGNEE_ROOTS: - (span,) = spans_named(spans, name) - assert span["parentSpanId"] == root_id, name - by_id = {span["spanId"]: span for span in spans} - assert len(by_id) == len(spans) - for span in spans: - if span.get("parentSpanId"): - assert span["parentSpanId"] in by_id, span["name"] - - -# -------------------------------------------------------------------------- -# Recorded spans as SDK spans, sent through the export filter -# -------------------------------------------------------------------------- - - -def otlp_value(value: dict[str, Any]) -> Any: - ((kind, raw),) = value.items() - if kind == "intValue": - return int(raw) - if kind == "arrayValue": - return [otlp_value(item) for item in raw.get("values", [])] - return raw - - -def sdk_span(span: dict[str, Any]) -> ReadableSpan: - """Rebuild an SDK span from its Receiver-decoded (OTLP JSON) form.""" - - def span_id(key: str) -> int: - return int.from_bytes(base64.b64decode(span[key]), "big") - - def flat(items: list[dict[str, Any]]) -> dict[str, Any]: - return {item["key"]: otlp_value(item["value"]) for item in items} - - trace_id = span_id("traceId") - parent = None - if span.get("parentSpanId"): - parent = SpanContext(trace_id, span_id("parentSpanId"), is_remote=False) - status = span.get("status", {}) - return ReadableSpan( - name=span["name"], - context=SpanContext( - trace_id, span_id("spanId"), is_remote=False, trace_flags=TraceFlags(TraceFlags.SAMPLED) - ), - parent=parent, - resource=Resource({"project_name": PROJECT, "project_type": "observe"}), - attributes=flat(span.get("attributes", [])), - events=[ - Event(event["name"], flat(event.get("attributes", [])), int(event["timeUnixNano"])) - for event in span.get("events", []) - ], - kind=SpanKind[span["kind"].replace("SPAN_KIND_", "")], - status=Status( - StatusCode[status.get("code", "STATUS_CODE_UNSET").replace("STATUS_CODE_", "")], - status.get("message"), - ), - start_time=int(span["startTimeUnixNano"]), - end_time=int(span["endTimeUnixNano"]), - ) - - -def export_spans(spans: list[dict[str, Any]], wrap: Any = None) -> list[dict[str, Any]]: - """Export recorded spans over OTLP/HTTP, optionally through ``wrap(exporter)``.""" - with Receiver() as receiver: - exporter = OTLPSpanExporter(endpoint=receiver.collector_endpoint) - if wrap is not None: - exporter = wrap(exporter) - assert exporter.export([sdk_span(span) for span in spans]) == SpanExportResult.SUCCESS - exporter.shutdown() - return receiver.spans() - - -def filtered( - cognee_filter: Any, spans: list[dict[str, Any]], **options: Any -) -> list[dict[str, Any]]: - """Export recorded spans through src/cognee_filter.py; return what arrives.""" - return export_spans( - spans, lambda exporter: cognee_filter.CogneeExportFilter(exporter, **options) - ) - - -def error_names(spans: Any) -> list[str]: - return [s["name"] for s in spans if s.get("status", {}).get("code") == "STATUS_CODE_ERROR"] - - -def is_missing_collection_probe(span: dict[str, Any]) -> bool: - """A recorded vector search whose only error is CollectionNotFoundError.""" - if span["name"] != VECTOR_SEARCH: - return False - if span.get("status", {}).get("code") != "STATUS_CODE_ERROR": - return False - events = span.get("events", []) - return bool(events) and all( - event["name"] == "exception" - and attributes(event).get("exception.type") == COLLECTION_MISSING - for event in events - ) - - -def without_error_detail(span: dict[str, Any]) -> tuple[list[dict[str, Any]], dict[str, Any]]: - """A recorded span's events and status as the filter should export them by default. - - A missing-collection probe is exported as OK, without its exception event. - - Default content-off mode keeps the fixed marker, exception type and - exception.escaped, but omits serialized exception stack traces entirely. - """ - if is_missing_collection_probe(span): - return [], {"code": "STATUS_CODE_OK"} - events = copy.deepcopy(span.get("events", [])) - for event in events: - event["attributes"] = [ - attribute - for attribute in event["attributes"] - if attribute["key"] != "exception.stacktrace" - ] - for attribute in event["attributes"]: - if attribute["key"] == "exception.message": - attribute["value"] = {"stringValue": DETAIL_REMOVED} - status = dict(span.get("status", {})) - if "message" in status: - status["message"] = attributes(events[-1])["exception.type"] + TYPE_ONLY - return events, status - - -def without_stacktraces(events: list[dict[str, Any]]) -> list[dict[str, Any]]: - """Recorded events as the filter exports them with capture on. - - Content capture keeps exception messages, but exception.stacktrace is - never exported: serialized tracebacks carry local paths (and can quote - content). - """ - events = copy.deepcopy(events) - for event in events: - event["attributes"] = [ - attribute - for attribute in event["attributes"] - if attribute["key"] != "exception.stacktrace" - ] - return events - - -def filtered_attributes(span: dict[str, Any], capture_content: bool) -> dict[str, Any]: - """A recorded span's attributes as the filter should export them.""" - expected = { - key: value - for key, value in attributes(span).items() - if capture_content or key not in FILTERED_KEYS - } - if span["name"] in FILTER_KINDS: - expected["fi.span.kind"] = FILTER_KINDS[span["name"]] - if capture_content: - expected.update(expected_io(attributes(span))) - if is_missing_collection_probe(span): - expected[MISSING_KEY] = True - return expected - - -def expected_io(attrs: dict[str, Any]) -> dict[str, Any]: - """The Input/Output keys capture mode adds for a span with Cognee's ``attrs``.""" - def mime(value: str) -> str: - # Cognee serializes an LLM call's arguments as a JSON object, but cuts - # it at 8000 characters, so a long prompt is no longer valid JSON. - try: - is_json = isinstance(json.loads(value), (dict, list)) - except ValueError: - is_json = False - return "application/json" if is_json else "text/plain" - - io: dict[str, Any] = {} - prompt = attrs.get("langfuse.observation.input") - question = attrs.get("memory.query.text", attrs.get("cognee.search.query")) - response = attrs.get("langfuse.observation.output") - if prompt is not None: - io.update({"input.value": prompt, "input.mime_type": mime(prompt)}) - elif question is not None: - io.update({"input.value": question, "input.mime_type": mime(question)}) - if response is not None: - io.update({"output.value": response, "output.mime_type": mime(response)}) - return io - - -@pytest.fixture(scope="module") -def cognee_filter() -> Any: - """src/cognee_filter.py. Imported here so a missing filter fails only its tests.""" - import cognee_filter as module - - return module - - -# -------------------------------------------------------------------------- -# Live runs -# -------------------------------------------------------------------------- - - -class Scenario: - """One guarded subprocess with its own Receiver, fake model and data dir.""" - - def __init__(self, name: str, script: Path, args: list[str], work: Path) -> None: - self.name = name - self.script = script - self.args = args - self.work = work - self.guard_log = work / "guard.jsonl" - self.result: Any = None - self.spans: list[dict[str, Any]] = [] - self.requests: list[dict[str, Any]] = [] - self.fake_requests: list[dict[str, Any]] = [] - - def env(self, receiver: Receiver, fake: FakeOpenAI) -> dict[str, str]: - env = { - "PATH": os.environ.get("PATH", ""), - "HOME": str(self.work), - "PYTHONPATH": os.pathsep.join((str(GUARD_DIR), str(PYTHON_DIR))), - "LOOPBACK_GUARD_LOG": str(self.guard_log), - # Cognee's product telemetry is separate from OTel tracing. - "TELEMETRY_DISABLED": "1", - # Keep LiteLLM from fetching its model cost map at import. - "LITELLM_LOCAL_MODEL_COST_MAP": "True", - # Offline tokenizer files (see litellm_tokenizers_dir). - **({"TIKTOKEN_CACHE_DIR": TOKENIZERS_DIR} if TOKENIZERS_DIR else {}), - "LLM_PROVIDER": "openai", - "LLM_MODEL": LLM_MODEL, - "LLM_ENDPOINT": fake.base_url, - "LLM_API_KEY": LLM_API_KEY, - "EMBEDDING_PROVIDER": "openai", - "EMBEDDING_MODEL": EMBEDDING_MODEL, - "EMBEDDING_ENDPOINT": fake.base_url, - "EMBEDDING_API_KEY": LLM_API_KEY, - "EMBEDDING_DIMENSIONS": str(fake.dimensions), - "DATA_ROOT_DIRECTORY": str(self.work / "data"), - "SYSTEM_ROOT_DIRECTORY": str(self.work / "system"), - "CACHE_ROOT_DIRECTORY": str(self.work / "cache"), - "COGNEE_LOGS_DIR": str(self.work / "logs"), - } - if self.name != "tracing_off": - env["COGNEE_TRACING_ENABLED"] = "true" - if self.name == "capture": - env["COGNEE_FI_CAPTURE_CONTENT"] = "true" - if self.name == "probes_as_errors": - env[MISSING_SWITCH] = "true" - if self.name == "otlp_env": - # The no-register() option: Cognee's own exporter, configured - # exactly as README.md says (with the Receiver's origin). - env["OTEL_EXPORTER_OTLP_ENDPOINT"] = receiver.collector_endpoint - env["OTEL_EXPORTER_OTLP_HEADERS"] = "X-Api-Key={0},X-Secret-Key={1}".format( - FI_API_KEY, FI_SECRET_KEY - ) - env["OTEL_RESOURCE_ATTRIBUTES"] = "project_name={0},project_type=observe".format( - PROJECT - ) - else: - env.update( - { - "FI_BASE_URL": receiver.origin, - "FI_API_KEY": FI_API_KEY, - "FI_SECRET_KEY": FI_SECRET_KEY, - "FI_PROJECT_NAME": PROJECT, - } - ) - return env - - def execute(self) -> "Scenario": - with Receiver() as receiver: - fake = FakeOpenAI(dimensions=16) - try: - self.result = run( - [sys.executable, str(self.script), *self.args], - env=self.env(receiver, fake), - stdin=None, - timeout=RUN_TIMEOUT_SECONDS, - ) - self.spans = receiver.spans() - self.requests = receiver.requests() - self.fake_requests = fake.requests() - finally: - fake.close() - return self - - @property - def stdout(self) -> str: - return self.result.stdout.decode("utf-8", "replace") - - @property - def stderr(self) -> str: - return self.result.stderr.decode("utf-8", "replace") - - def guard_records(self) -> list[dict[str, Any]]: - if not self.guard_log.exists(): - return [] - lines = self.guard_log.read_text(encoding="utf-8").splitlines() - return [json.loads(line) for line in lines] - - def blocked(self) -> list[dict[str, Any]]: - return [record for record in self.guard_records() if record["kind"] != "installed"] - - def guarded_workers(self) -> list[dict[str, Any]]: - """Processes spawned by multiprocessing that loaded the guard.""" - return [ - record - for record in self.guard_records() - if record["kind"] == "installed" and "--multiprocessing-fork" in record["argv"] - ] - - def assert_ran_offline(self) -> None: - assert not self.result.timed_out, self.stderr[-4000:] - assert self.result.returncode == 0, self.stderr[-4000:] - assert self.blocked() == [] - - def json_line(self) -> dict[str, Any]: - lines = [line for line in self.stdout.splitlines() if line.startswith("{")] - return json.loads(lines[-1]) - - -@pytest.fixture(scope="module") -def runs(tmp_path_factory: pytest.TempPathFactory) -> dict[str, Scenario]: - if not COGNEE_INSTALLED: - pytest.skip("cognee is not installed") - scenarios = [ - Scenario("recipe", APP, [QUESTION, DOCUMENT], tmp_path_factory.mktemp("recipe")), - Scenario("capture", APP, [QUESTION, DOCUMENT], tmp_path_factory.mktemp("capture")), - Scenario( - "probes_as_errors", APP, [QUESTION, DOCUMENT], tmp_path_factory.mktemp("probes") - ), - Scenario("tracing_off", APP, [QUESTION, DOCUMENT], tmp_path_factory.mktemp("off")), - Scenario("no_readd", ADD_SCRIPT, ["no-readd"], tmp_path_factory.mktemp("noreadd")), - Scenario( - "otlp_env", ADD_SCRIPT, ["otlp-env", *OTLP_URL_FORMS], tmp_path_factory.mktemp("env") - ), - ] - # Importing cognee alone takes about a minute, so the processes run side - # by side. - with concurrent.futures.ThreadPoolExecutor(max_workers=len(scenarios)) as pool: - done = list(pool.map(Scenario.execute, scenarios)) - by_name = {scenario.name: scenario for scenario in done} - capture = by_name["capture"] - if os.environ.get("COGNEE_RECORD_FIXTURE") == "1" and capture.result.returncode == 0: - # The fixture is Cognee's own output: the capture run (all content - # kept) without the fi.span.kind the filter adds. - spans = scrub(without_filter_kinds(capture.spans), tmp_path_factory.getbasetemp()) - FIXTURE.write_text(json.dumps(spans, indent=1) + "\n", encoding="utf-8") - return by_name - - -def test_recipe_runs_offline_against_the_fakes(runs: dict[str, Scenario]) -> None: - recipe = runs["recipe"] - recipe.assert_ran_offline() - assert ANSWER_MARKER in recipe.stdout - paths = {request["path"] for request in recipe.fake_requests} - assert paths == {"/v1/chat/completions", "/v1/embeddings"} - - -def test_loopback_guard_also_runs_in_cognee_worker_processes(runs: dict[str, Scenario]) -> None: - # Cognee 1.6.2 runs LanceDB and Ladybug in multiprocessing "spawn" - # workers; the guard must be installed there too, not only in the - # scenario's main process (each scenario's assert_ran_offline checks - # that nothing was refused). - for name, scenario in runs.items(): - assert scenario.guarded_workers(), name - - -def test_recipe_exports_to_collector_path_with_both_auth_headers( - runs: dict[str, Scenario], -) -> None: - requests = runs["recipe"].requests - assert requests - for request in requests: - assert request["path"] == "/tracer/v1/traces" - assert request["headers"]["x-api-key"] == FI_API_KEY - assert request["headers"]["x-secret-key"] == FI_SECRET_KEY - assert request["headers"]["content-type"] == "application/x-protobuf" - assert LLM_API_KEY not in json.dumps(request["headers"]) - - -def test_recipe_resource_is_an_observe_project(runs: dict[str, Scenario]) -> None: - resources = [ - resource for request in runs["recipe"].requests for resource in request["resource_attributes"] - ] - assert resources - for resource in resources: - assert resource["project_name"] == PROJECT - assert resource["project_type"] == "observe" - # One provider: every export carries the same register() resource. - assert len({json.dumps(resource, sort_keys=True) for resource in resources}) == 1 - - -def test_recipe_spans_match_documented_names_and_kinds(runs: dict[str, Scenario]) -> None: - assert_span_names_and_kinds(runs["recipe"].spans) - - -def test_recipe_puts_add_cognify_search_in_one_trace(runs: dict[str, Scenario]) -> None: - assert_one_trace_under_app_span(runs["recipe"].spans) - - -def test_recipe_does_not_duplicate_spans(runs: dict[str, Scenario]) -> None: - recipe = runs["recipe"] - chat_calls = [r for r in recipe.fake_requests if r["path"] == "/v1/chat/completions"] - embedding_calls = [r for r in recipe.fake_requests if r["path"] == "/v1/embeddings"] - assert len(spans_named(recipe.spans, GENERATION_SPAN)) == len(chat_calls) - assert len(spans_named(recipe.spans, EMBEDDING_SPAN)) == len(embedding_calls) - assert len({span["spanId"] for span in recipe.spans}) == len(recipe.spans) - - -def test_recipe_marks_search_and_llm_spans_for_the_collector(runs: dict[str, Scenario]) -> None: - # memory.retrieve and cognee.search.* arrive as RETRIEVER, LLM calls as - # LLM; embedding spans keep gen_ai.operation.name only. - assert_collector_mapping_inputs(runs["recipe"].spans, FILTER_KINDS) - - -def test_recipe_exports_no_content_by_default(runs: dict[str, Scenario]) -> None: - assert_no_content(runs["recipe"].spans) - - -def test_recipe_exports_missing_collection_probes_as_ok(runs: dict[str, Scenario]) -> None: - # Cognee's vector-search probes of collections that do not exist yet fail - # in every run. By default they arrive as OK, flagged, and nothing in a - # successful run is an error. - for name in ("recipe", "capture"): - spans = runs[name].spans - assert error_names(spans) == [], name - flagged = [s for s in spans if attributes(s).get(MISSING_KEY) is True] - assert flagged, name - for span in flagged: - assert span["name"] == VECTOR_SEARCH - assert span["status"] == {"code": "STATUS_CODE_OK"} - assert span.get("events", []) == [] - - -def test_switch_keeps_probes_as_errors_without_detail( - runs: dict[str, Scenario], recorded: list[dict[str, Any]], cognee_filter: Any -) -> None: - # With COGNEE_FI_COLLECTION_MISSING_AS_ERROR=true the probes stay ERROR; - # with capture off only fixed markers and the exception type arrive. - run = runs["probes_as_errors"] - run.assert_ran_offline() - errors = [s for s in run.spans if s.get("status", {}).get("code") == "STATUS_CODE_ERROR"] - assert errors - for span in errors: - assert span["name"] == VECTOR_SEARCH - assert MISSING_KEY not in attributes(span) - assert span["events"], span["name"] - for event in span["events"]: - event_attrs = attributes(event) - assert event_attrs["exception.type"] == COLLECTION_MISSING - assert event_attrs["exception.message"] == DETAIL_REMOVED - assert span["status"]["message"] == COLLECTION_MISSING + TYPE_ONLY - assert "exception.stacktrace" not in event_attrs - assert shape(run.spans) == shape( - filtered(cognee_filter, recorded, collection_missing_as_error=True) - ) - - -def test_recipe_exports_no_secrets(runs: dict[str, Scenario]) -> None: - recipe = runs["recipe"] - assert_no_secrets(recipe.spans) - assert_no_secrets([request["resource_attributes"] for request in recipe.requests]) - - -def test_recipe_run_matches_recorded_fixture( - runs: dict[str, Scenario], recorded: list[dict[str, Any]], cognee_filter: Any -) -> None: - assert shape(runs["recipe"].spans) == shape(filtered(cognee_filter, recorded)) - - -def test_capture_content_exports_content_under_documented_keys( - runs: dict[str, Scenario], -) -> None: - capture = runs["capture"] - capture.assert_ran_offline() - assert_default_content(capture.spans) - assert_no_secrets(capture.spans) - - -def test_capture_run_matches_recorded_fixture( - runs: dict[str, Scenario], recorded: list[dict[str, Any]], cognee_filter: Any -) -> None: - capture = runs["capture"] - assert_collector_mapping_inputs(capture.spans, FILTER_KINDS) - assert shape(capture.spans) == shape(filtered(cognee_filter, recorded, capture_content=True)) - - -def test_tracing_off_exports_no_cognee_spans(runs: dict[str, Scenario]) -> None: - off = runs["tracing_off"] - off.assert_ran_offline() - assert ANSWER_MARKER in off.stdout - # Only the app's own parent span is left. - assert [span["name"] for span in off.spans] == [APP_SPAN] - - -def test_register_without_the_readd_line_exports_nothing(runs: dict[str, Scenario]) -> None: - no_readd = runs["no_readd"] - no_readd.assert_ran_offline() - # Cognee made spans (its in-memory buffer holds them) ... - assert no_readd.json_line()["buffered_spans"] > 0 - # ... but register()'s exporter was replaced, so none left the process. - assert no_readd.requests == [] - assert no_readd.spans == [] - - -def test_otlp_env_option_posts_to_the_configured_path(runs: dict[str, Scenario]) -> None: - otlp = runs["otlp_env"] - otlp.assert_ran_offline() - assert otlp.requests - for request in otlp.requests: - # Cognee passes the endpoint through unchanged: no second /v1/traces. - assert request["path"] == "/tracer/v1/traces" - assert request["headers"]["x-api-key"] == FI_API_KEY - assert request["headers"]["x-secret-key"] == FI_SECRET_KEY - for resource in request["resource_attributes"]: - assert resource["project_name"] == PROJECT - assert resource["project_type"] == "observe" - assert resource["service.name"] == "cognee" - names = [span["name"] for span in otlp.spans] - assert "memory.store" in names - assert len(names) == otlp.json_line()["buffered_spans"] - assert_no_secrets(otlp.spans) - - -def test_otlp_env_option_also_sends_logs_the_collector_rejects( - runs: dict[str, Scenario], -) -> None: - # Cognee derives /../v1/logs from the traces URL; fi-collector - # (and the Receiver) answer 404 there. - assert "Failed to export logs batch code: 404" in runs["otlp_env"].stderr - - -def test_otlp_env_transport_choice_for_documented_urls(runs: dict[str, Scenario]) -> None: - assert runs["otlp_env"].json_line()["http_only"] == OTLP_URL_FORMS - - -# -------------------------------------------------------------------------- -# Recorded fixture (no Cognee needed) -# -------------------------------------------------------------------------- - - -@pytest.fixture(scope="module") -def recorded() -> list[dict[str, Any]]: - return json.loads(FIXTURE.read_text(encoding="utf-8")) - - -def otlp_body(spans: list[dict[str, Any]]) -> dict[str, Any]: - resource = {"project_name": PROJECT, "project_type": "observe"} - return { - "resourceSpans": [ - { - "resource": { - "attributes": [ - {"key": key, "value": {"stringValue": value}} - for key, value in resource.items() - ] - }, - "scopeSpans": [{"spans": spans}], - } - ] - } - - -def test_recorded_fixture_round_trips_through_the_harness(recorded: list[dict[str, Any]]) -> None: - with Receiver() as receiver: - assert post_otlp(otlp_body(recorded), receiver.collector_endpoint) == 200 - compare(receiver.spans(), FIXTURE) - (request,) = receiver.requests() - assert request["path"] == "/tracer/v1/traces" - assert request["resource_attributes"] == [{"project_name": PROJECT, "project_type": "observe"}] - - -def test_recorded_fixture_matches_documented_mapping(recorded: list[dict[str, Any]]) -> None: - # Cognee's own spans: no span-kind key, content under the documented keys. - assert_span_names_and_kinds(recorded) - assert_one_trace_under_app_span(recorded) - assert_collector_mapping_inputs(recorded, {}) - assert_default_content(recorded, io_copies=False) - assert_no_secrets(recorded) - assert found_in(recorded, host_paths()) == [] - - -def test_recorded_error_spans_are_vector_search_probes(recorded: list[dict[str, Any]]) -> None: - errors = [s for s in recorded if s.get("status", {}).get("code") == "STATUS_CODE_ERROR"] - searches = spans_named(recorded, VECTOR_SEARCH) - assert {span["name"] for span in errors} == {VECTOR_SEARCH} - assert all(is_missing_collection_probe(span) for span in errors) - for span in errors: - assert [event["name"] for event in span["events"]] == ["exception"] - assert "{0} of {1}".format(len(errors), len(searches)) in README.read_text(encoding="utf-8") - - -def test_scrub_removes_host_and_tmp_paths(tmp_path_factory: pytest.TempPathFactory) -> None: - base = tmp_path_factory.getbasetemp() - planted = [ - str(base / "recipe0" / "data" / "file.txt"), - os.path.realpath(base) + "/recipe0/system", - str(REPO_ROOT / "python" / "x.py"), - str(Path.home() / ".cache" / "uv" / "x.py"), - 'File "{0}/.venv/lib/python3.11/site-packages/x.py"'.format(Path.home()), - ] - scrubbed = scrub([{"name": "x", "attributes": [], "planted": planted}], base) - assert found_in(scrubbed, host_paths()) == [] - - -def test_fixture_spans_rebuild_as_sdk_spans(recorded: list[dict[str, Any]]) -> None: - # The helper the filter tests rely on: rebuilt spans export back to the - # same names, attributes, statuses and events. - wire = export_spans(recorded) - compare(wire, FIXTURE) - for before, after in zip(recorded, wire): - assert after.get("events", []) == before.get("events", []), before["name"] - - -# -------------------------------------------------------------------------- -# The export filter (src/cognee_filter.py), on the recorded spans -# -------------------------------------------------------------------------- - - -def test_filter_lists_the_tested_content_keys_and_span_names(cognee_filter: Any) -> None: - assert cognee_filter.CONTENT_KEYS == FILTERED_KEYS - assert cognee_filter.SPAN_KIND_KEYS == SPAN_KIND_KEYS - for name in SPAN_KINDS: - assert cognee_filter.span_kind(name) == FILTER_KINDS.get(name), name - - -def test_filter_keeps_content_off_the_wire_by_default( - cognee_filter: Any, recorded: list[dict[str, Any]] -) -> None: - wire = filtered(cognee_filter, recorded) - assert len(wire) == len(recorded) - assert_no_content(wire) - # Error text is replaced and serialized exception stack traces are omitted; - # everything else is unchanged. - for before, after in zip(recorded, wire): - assert attributes(after) == filtered_attributes(before, capture_content=False) - events, status = without_error_detail(before) - assert after.get("events", []) == events - assert after.get("status") == status - - -@pytest.mark.parametrize("capture_content", [False, True]) -def test_filter_exports_missing_collection_probes_as_ok( - cognee_filter: Any, recorded: list[dict[str, Any]], capture_content: bool -) -> None: - probes = [span for span in recorded if is_missing_collection_probe(span)] - assert len(probes) == 60 - wire = { - span["spanId"]: span - for span in filtered(cognee_filter, recorded, capture_content=capture_content) - } - assert error_names(wire.values()) == [] - for probe in probes: - after = wire[probe["spanId"]] - assert after["status"] == {"code": "STATUS_CODE_OK"} - assert after.get("events", []) == [] - assert attributes(after)[MISSING_KEY] is True - # Which collection was missing is still there. - collection = attributes(probe)["cognee.vector.collection"] - assert attributes(after)["cognee.vector.collection"] == collection - for span in recorded: - if not is_missing_collection_probe(span): - assert MISSING_KEY not in attributes(wire[span["spanId"]]), span["name"] - - -@pytest.mark.parametrize( - ("env_value", "argument", "kept_as_error"), - [ - (None, None, False), - ("true", None, True), - ("TRUE", None, True), - ("false", None, False), - ("true", False, False), - (None, True, True), - ], -) -def test_filter_collection_missing_switch( - cognee_filter: Any, - recorded: list[dict[str, Any]], - monkeypatch: pytest.MonkeyPatch, - env_value: Any, - argument: Any, - kept_as_error: bool, -) -> None: - if env_value is None: - monkeypatch.delenv(MISSING_SWITCH, raising=False) - else: - monkeypatch.setenv(MISSING_SWITCH, env_value) - options = {} if argument is None else {"collection_missing_as_error": argument} - probe = next(span for span in recorded if is_missing_collection_probe(span)) - (after,) = filtered(cognee_filter, [probe], **options) - if kept_as_error: - assert after["status"] == { - "code": "STATUS_CODE_ERROR", - "message": COLLECTION_MISSING + TYPE_ONLY, - } - assert MISSING_KEY not in attributes(after) - assert [event["name"] for event in after["events"]] == ["exception"] - else: - assert after["status"] == {"code": "STATUS_CODE_OK"} - assert attributes(after)[MISSING_KEY] is True - - -def test_filter_keeps_every_other_error(cognee_filter: Any, recorded: list[dict[str, Any]]) -> None: - probe = next(span for span in recorded if is_missing_collection_probe(span)) - when = probe["endTimeUnixNano"] - other_error = copy.deepcopy(probe) - other_error["status"] = {"code": "STATUS_CODE_ERROR", "message": "ValueError: bad vector"} - other_error["events"] = [exception_event("ValueError", "bad vector", "trace", when)] - mixed = copy.deepcopy(probe) - mixed["events"].append(exception_event("RuntimeError", "lost connection", "trace", when)) - elsewhere = copy.deepcopy(probe) - elsewhere["name"] = "cognee.retrieval.vector_search" - no_event = copy.deepcopy(probe) - no_event["events"] = [] - for capture_content in (False, True): - spans = [other_error, mixed, elsewhere, no_event] - wire = filtered(cognee_filter, spans, capture_content=capture_content) - assert len(wire) == 4 - for span in wire: - assert span["status"]["code"] == "STATUS_CODE_ERROR", span["name"] - assert MISSING_KEY not in attributes(span) - - -def test_filter_sets_span_kinds_only_where_none_is_set( - cognee_filter: Any, recorded: list[dict[str, Any]] -) -> None: - assert_collector_mapping_inputs(filtered(cognee_filter, recorded), FILTER_KINDS) - (search,) = copy.deepcopy(spans_named(recorded, "memory.retrieve")) - search["attributes"].append({"key": "gen_ai.span.kind", "value": {"stringValue": "CHAIN"}}) - (wire,) = filtered(cognee_filter, [search]) - assert attributes(wire)["gen_ai.span.kind"] == "CHAIN" - assert "fi.span.kind" not in attributes(wire) - - -def test_filter_removes_error_text_quoting_content( - cognee_filter: Any, recorded: list[dict[str, Any]] -) -> None: - # Cognee did not quote content in errors in the recorded run; plant it. - (search,) = copy.deepcopy(spans_named(recorded, "memory.retrieve")) - generation = copy.deepcopy( - next(s for s in spans_named(recorded, GENERATION_SPAN) if DOC_MARKER in json.dumps(s)) - ) - for span, quoted in ((search, QUESTION), (generation, DOCUMENT)): - span["status"] = {"code": "STATUS_CODE_ERROR", "message": "ValueError: bad " + quoted} - span["events"] = [ - exception_event( - "ValueError", - "bad " + quoted, - '{0}\n File "/app/x.py", line 3, in run\nValueError: bad {1}\n'.format( - TRACEBACK_HEADER, quoted - ), - span["endTimeUnixNano"], - ) - ] - wire = filtered(cognee_filter, [search, generation]) - assert_no_content(wire) - for span in wire: - assert span["status"]["message"] == "ValueError" + TYPE_ONLY - assert attributes(span["events"][0]) == { - "exception.type": "ValueError", - "exception.message": DETAIL_REMOVED, - "exception.escaped": "False", - } - - -# A content-policy rejection as Cognee 1.6.2 raises it (litellm_native/ -# native_adapter.py:512-518): the message quotes the whole prompt. The payload -# is longer than Cognee's 8000-character attribute cap (get_observe.py:12, :46) -# and has several lines; the marker is on every line, so also past the cap. -PAYLOAD_MARKER = "PMARK2c6d" -PAYLOAD = "\n".join([PAYLOAD_MARKER + " Ada works on the Lighthouse project."] * 220) -POLICY_ERROR = "cognee.infrastructure.llm.exceptions.ContentPolicyFilterError" -TASK_SPAN = "cognee.pipeline.task.extract_graph_and_summarize" -CHAIN_FRAME_MARKER = "PROMPT_SECRET_FIXTURE_ONLY" -# This is the exact separator, blank line, header and fake frame from -# tickets/TH-8328/fix-r3/red-probe.py. It is quoted by the exception message -# below, so it also occurs in the serialized exception.stacktrace. -FORGED_CHAIN_FRAME = "\n".join( - ( - "The above exception was the direct cause of the following exception:", - "", - TRACEBACK_HEADER, - ' File "/private/PROMPT_SECRET_FIXTURE_ONLY.py", line 1, in ' - "PROMPT_SECRET_FIXTURE_ONLY", - "end of quoted prompt", - ) -) -FORGED_CHAIN_PAYLOAD = ("ordinary oversized prompt text " * 320) + "\n" + FORGED_CHAIN_FRAME - - -class ContentPolicyViolationError(Exception): - """Stands in for litellm's exception of that name.""" - - -class ContentPolicyFilterError(Exception): - """Stands in for Cognee's ContentPolicyFilterError.""" - - -def raise_policy_rejection(payload: str = PAYLOAD) -> None: - try: - raise ContentPolicyViolationError("litellm.ContentPolicyViolationError: content_filter") - except ContentPolicyViolationError as error: - # str() of Cognee's error: CogneeApiError.__str__ (cognee/exceptions/exceptions.py:62-66). - raise ContentPolicyFilterError( - "CogneeValidationError: The provided input contains content that is not aligned " - "with our content policy: {0} (Status code: 422)".format(payload) - ) from error - - -def exception_event(type_: str, message: str, stacktrace: str, time: str) -> dict[str, Any]: - """The event OpenTelemetry's record_exception() adds, in OTLP JSON form.""" - return { - "timeUnixNano": time, - "name": "exception", - "attributes": [ - {"key": "exception.type", "value": {"stringValue": type_}}, - {"key": "exception.message", "value": {"stringValue": message}}, - {"key": "exception.stacktrace", "value": {"stringValue": stacktrace}}, - {"key": "exception.escaped", "value": {"stringValue": "False"}}, - ], - } - - -def policy_rejection_spans( - recorded: list[dict[str, Any]], payload: str = PAYLOAD -) -> list[dict[str, Any]]: - """A recorded LLM span and its pipeline task span, as a content-policy rejection ends them. - - The LLM span's input attribute is the prompt JSON cut at 8000 characters, - which does not parse. The task span has no content attribute and records - the error twice: run_tasks_base.py:243-245, then new_span's - start_as_current_span (observability/__init__.py:110-118). - """ - llm = copy.deepcopy( - next( - span - for span in spans_named(recorded, GENERATION_SPAN) - for parent in recorded - if parent["spanId"] == span["parentSpanId"] and parent["name"] == TASK_SPAN - ) - ) - (task,) = [copy.deepcopy(s) for s in recorded if s["spanId"] == llm["parentSpanId"]] - try: - raise_policy_rejection(payload) - except ContentPolicyFilterError as raised: - error = raised - stacktrace = "".join(traceback.format_exception(type(error), error, error.__traceback__)) - prompt = json.dumps({"text_input": payload, "system_prompt": "Extract a graph."})[:8000] - with pytest.raises(ValueError): - json.loads(prompt) - for attribute in llm["attributes"]: - if attribute["key"] == "langfuse.observation.input": - attribute["value"] = {"stringValue": prompt} - for span, recorded_times in ((llm, 1), (task, 2)): - span["status"] = { - "code": "STATUS_CODE_ERROR", - "message": "ContentPolicyFilterError: " + str(error), - } - span["events"] = [ - exception_event(POLICY_ERROR, str(error), stacktrace, span["endTimeUnixNano"]) - ] * recorded_times - return [llm, task] - - -def test_filter_removes_error_detail_on_every_span_by_default( - cognee_filter: Any, recorded: list[dict[str, Any]] -) -> None: - # N1: the rejection quotes the whole prompt in the exception message, the - # stack trace and the status, on the LLM span (whose cut-off input does not - # parse) and again on its pipeline task span (which has no content key). - spans = policy_rejection_spans(recorded) - assert found_in(spans, [PAYLOAD_MARKER]) == [PAYLOAD_MARKER] - wire = filtered(cognee_filter, spans) - assert [span["name"] for span in wire] == [GENERATION_SPAN, TASK_SPAN] - assert found_in(wire, [PAYLOAD_MARKER]) == [] - assert_no_content(wire) - for before, after in zip(spans, wire): - assert after["status"] == { - "code": "STATUS_CODE_ERROR", - "message": POLICY_ERROR + TYPE_ONLY, - } - assert len(after["events"]) == len(before["events"]) - for event_after in after["events"]: - event_attrs = attributes(event_after) - assert event_attrs["exception.type"] == POLICY_ERROR - assert event_attrs["exception.escaped"] == "False" - assert event_attrs["exception.message"] == DETAIL_REMOVED - assert "exception.stacktrace" not in event_attrs - - -def test_filter_capture_content_keeps_error_detail( - cognee_filter: Any, recorded: list[dict[str, Any]] -) -> None: - spans = policy_rejection_spans(recorded) - wire = filtered(cognee_filter, spans, capture_content=True) - for before, after in zip(spans, wire): - assert attributes(after) == filtered_attributes(before, capture_content=True) - assert after["events"] == without_stacktraces(before["events"]) - assert after["status"] == before["status"] - assert [attributes(event)["exception.message"] for event in after["events"]] == [ - attributes(event)["exception.message"] for event in before["events"] - ] - for event in after["events"]: - assert "exception.stacktrace" not in attributes(event) - - -def test_filter_omits_forged_chained_traceback_on_every_span_by_default( - cognee_filter: Any, recorded: list[dict[str, Any]] -) -> None: - """Serialized exception text has no trustworthy frame provenance. - - The content-policy exception quotes an oversized prompt on the recorded - LLM span and its pipeline parent. Its quoted text contains the exact - traceback-shaped sequence from the red probe, including the forged frame. - """ - assert len(FORGED_CHAIN_PAYLOAD) > 8000 - spans = policy_rejection_spans(recorded, FORGED_CHAIN_PAYLOAD) - assert found_in(spans, [CHAIN_FRAME_MARKER]) == [CHAIN_FRAME_MARKER] - - wire = filtered(cognee_filter, spans) - assert [span["name"] for span in wire] == [GENERATION_SPAN, TASK_SPAN] - assert found_in(wire, [CHAIN_FRAME_MARKER]) == [] - for span in wire: - assert span["status"] == { - "code": "STATUS_CODE_ERROR", - "message": POLICY_ERROR + TYPE_ONLY, - } - for event in span["events"]: - event_attrs = attributes(event) - assert event_attrs["exception.type"] == POLICY_ERROR - assert event_attrs["exception.escaped"] == "False" - assert event_attrs["exception.message"] == DETAIL_REMOVED - assert "exception.stacktrace" not in event_attrs - - captured = filtered(cognee_filter, spans, capture_content=True) - for before, after in zip(spans, captured): - assert attributes(after) == filtered_attributes(before, capture_content=True) - assert after["events"] == without_stacktraces(before["events"]) - assert after["status"] == before["status"] - assert found_in(after, [CHAIN_FRAME_MARKER]) == [CHAIN_FRAME_MARKER] - - -def test_filter_removes_detail_of_a_raised_chained_exception(cognee_filter: Any) -> None: - # The real record_exception() and use_span() output on this Python version, - # for a chained exception with a multi-line message. - filtered_out, captured_out, raw_out = ( - InMemorySpanExporter(), - InMemorySpanExporter(), - InMemorySpanExporter(), - ) - provider = TracerProvider() - provider.add_span_processor(SimpleSpanProcessor(cognee_filter.CogneeExportFilter(filtered_out))) - provider.add_span_processor( - SimpleSpanProcessor(cognee_filter.CogneeExportFilter(captured_out, capture_content=True)) - ) - provider.add_span_processor(SimpleSpanProcessor(raw_out)) - with pytest.raises(ContentPolicyFilterError): - with provider.get_tracer("test").start_as_current_span(GENERATION_SPAN): - raise_policy_rejection() - (raw,) = raw_out.get_finished_spans() - (clean,) = filtered_out.get_finished_spans() - (captured,) = captured_out.get_finished_spans() - assert PAYLOAD_MARKER in raw.status.description - exported = { - "attributes": dict(clean.attributes), - "status": clean.status.description, - "events": [[event.name, dict(event.attributes)] for event in clean.events], - } - assert found_in(exported, [PAYLOAD_MARKER]) == [] - (event,) = clean.events - exception_type = event.attributes["exception.type"] - assert exception_type.endswith(".ContentPolicyFilterError") - assert clean.status.description == exception_type + TYPE_ONLY - assert "exception.stacktrace" not in event.attributes - # Capture mode keeps all content and the error message and status; it - # adds only the span-kind mapping (applied in both modes) and drops only - # the serialized stack trace, which carries local paths. - assert cognee_filter.span_kind(GENERATION_SPAN) == "LLM" - assert dict(captured.attributes) == {**dict(raw.attributes), "fi.span.kind": "LLM"} - assert [event.name for event in captured.events] == [event.name for event in raw.events] - for captured_event, raw_event in zip(captured.events, raw.events): - assert "exception.stacktrace" in raw_event.attributes - assert dict(captured_event.attributes) == { - key: value - for key, value in raw_event.attributes.items() - if key != "exception.stacktrace" - } - assert PAYLOAD_MARKER in captured.events[0].attributes["exception.message"] - assert captured.status == raw.status - - -MARK = "TMARK5e0a" - - -@pytest.mark.parametrize( - ("trace", "exception_type"), - [ - pytest.param( - [ - TRACEBACK_HEADER, - ' File "/app/client.py", line 12, in post', - " raise ContentPolicyViolationError(body) # " + MARK, - " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^", - "litellm.ContentPolicyViolationError: rejected " + MARK, - "a second message line " + MARK, - ' File "/forged.py", line 1, in ' + MARK, - "", - "The above exception was the direct cause of the following exception:", - "", - TRACEBACK_HEADER, - ' File "/app/adapter.py", line 515, in acreate_structured_output', - " raise ContentPolicyFilterError(message) from error", - "cognee.ContentPolicyFilterError: policy " + MARK, - MARK + " on a second line", - ], - "cognee.ContentPolicyFilterError", - id="direct-cause-multi-line-messages", - ), - pytest.param( - [ - TRACEBACK_HEADER, - ' File "/app/a.py", line 3, in run', - " step(" + MARK + ")", - ' File "/app/a.py", line 7, in step', - " step(" + MARK + ")", - " [Previous line repeated 2 more times]", - "KeyError: '" + MARK + "'", - "", - "During handling of the above exception, another exception occurred:", - "", - TRACEBACK_HEADER, - ' File "/app/a.py", line 5, in run', - " raise ValueError(" + MARK + ")", - "ValueError: " + MARK, - "a note added with add_note() " + MARK, - ], - "ValueError", - id="during-handling-with-repeats-and-notes", - ), - pytest.param( - [ - "ValueError: " + MARK, - "", - "The above exception was the direct cause of the following exception:", - "", - TRACEBACK_HEADER, - ' File "/app/a.py", line 3, in run', - "RuntimeError: " + MARK, - ], - "RuntimeError", - id="cause-without-traceback", - ), - pytest.param( - [ - TRACEBACK_HEADER, - ' File "/app/a.py", line 3, in run', - "ValueError: " + MARK, - TRACEBACK_HEADER, - ' File "/app/' + MARK + '.py", line 1, in ' + MARK, - MARK, - ], - "ValueError", - id="message-that-looks-like-a-traceback", - ), - pytest.param( - [ - " + Exception Group Traceback (most recent call last):", - ' | File "/app/a.py", line 3, in run', - ' | raise ExceptionGroup("' + MARK + '", errors)', - " | ExceptionGroup: " + MARK + " (1 sub-exception)", - " +-+---------------- 1 ----------------", - " | ValueError: " + MARK, - " +------------------------------------", - ], - "ExceptionGroup", - id="unrecognised-layout", - ), - ], -) - - -def test_filter_omits_serialized_stacktraces_in_both_modes( - cognee_filter: Any, trace: list[str], exception_type: str -) -> None: - exporter, captured_exporter = InMemorySpanExporter(), InMemorySpanExporter() - provider = TracerProvider() - provider.add_span_processor(SimpleSpanProcessor(cognee_filter.CogneeExportFilter(exporter))) - provider.add_span_processor( - SimpleSpanProcessor(cognee_filter.CogneeExportFilter(captured_exporter, capture_content=True)) - ) - with provider.get_tracer("test").start_as_current_span(TASK_SPAN) as span: - span.add_event( - "exception", - { - "exception.type": exception_type, - "exception.message": MARK, - "exception.stacktrace": "\n".join(trace) + "\n", - "exception.escaped": "False", - }, - ) - (exported,) = exporter.get_finished_spans() - (captured,) = captured_exporter.get_finished_spans() - assert dict(exported.events[0].attributes) == { - "exception.type": exception_type, - "exception.message": DETAIL_REMOVED, - "exception.escaped": "False", - } - assert found_in(dict(exported.events[0].attributes), [MARK]) == [] - # Capture mode keeps the message (content the user opted into) but never - # the serialized stack trace. - assert dict(captured.events[0].attributes) == { - "exception.type": exception_type, - "exception.message": MARK, - "exception.escaped": "False", - } - - -@pytest.mark.parametrize("capture_content", [False, True], ids=["default", "capture"]) -def test_filter_never_exports_stack_traces_or_local_paths( - cognee_filter: Any, recorded: list[dict[str, Any]], capture_content: bool -) -> None: - # The recorded run's vector-search probes carry Cognee's real stack - # traces, whose frames name the local Python installation (scrubbed to - # in the fixture). Kept as errors here, so their events reach - # the exporter, they must arrive without the trace in both modes. - local = ["", TRACEBACK_HEADER] - assert found_in(recorded, local) == local - wire = filtered( - cognee_filter, - recorded, - capture_content=capture_content, - collection_missing_as_error=True, - ) - assert error_names(wire) == error_names(recorded) - assert found_in(wire, ["", TRACEBACK_HEADER, "site-packages"]) == [] - for span in wire: - for event in span.get("events", []): - assert "exception.stacktrace" not in attributes(event) - assert "exception.type" in attributes(event) - - -def test_filter_keeps_only_known_keys_and_non_text_values_in_events(cognee_filter: Any) -> None: - # Cognee 1.6.2 adds no events other than "exception"; anything else gets - # the same allowlist. - exporter = InMemorySpanExporter() - provider = TracerProvider() - provider.add_span_processor(SimpleSpanProcessor(cognee_filter.CogneeExportFilter(exporter))) - with provider.get_tracer("test").start_as_current_span(APP_SPAN) as span: - span.add_event( - "retry", - { - "attempt": 2, - "delay.seconds": 1.5, - "final": False, - "scores": [0.5, 1.0], - "reason": "rejected " + MARK, - "inputs": ["first " + MARK, "second"], - "exception.type": "ValueError", - }, - ) - span.set_status(Status(StatusCode.ERROR, "gave up on " + MARK)) - (exported,) = exporter.get_finished_spans() - assert dict(exported.events[0].attributes) == { - "attempt": 2, - "delay.seconds": 1.5, - "final": False, - "scores": (0.5, 1.0), - "exception.type": "ValueError", - } - # An event that is not named "exception" does not give the status a type. - assert exported.status.description == DETAIL_REMOVED - - -def test_filter_capture_content_keeps_cognees_content( - cognee_filter: Any, recorded: list[dict[str, Any]] -) -> None: - wire = filtered(cognee_filter, recorded, capture_content=True) - assert_default_content(wire) - assert_collector_mapping_inputs(wire, FILTER_KINDS) - for before, after in zip(recorded, wire): - assert attributes(after) == filtered_attributes(before, capture_content=True) - - -def test_filter_capture_content_shows_content_as_span_input_and_output( - cognee_filter: Any, recorded: list[dict[str, Any]] -) -> None: - # fi-collector fills a span's Input and Output from input.value and - # output.value only; without these copies, captured prompts, questions and - # answers show up in Future AGI as raw attributes, not as Input/Output. - captured = filtered(cognee_filter, recorded, capture_content=True) - wire = {span["spanId"]: attributes(span) for span in captured} - generations = [span for span in recorded if span["name"] == GENERATION_SPAN] - assert generations - for span in generations: - attrs = wire[span["spanId"]] - assert attrs["input.value"] == attributes(span)["langfuse.observation.input"] - assert attrs["input.mime_type"] == "application/json" - assert attrs["output.value"] == attributes(span)["langfuse.observation.output"] - assert attrs["output.mime_type"] in ("application/json", "text/plain") - (search,) = spans_named(recorded, "memory.retrieve") - assert wire[search["spanId"]]["input.value"] == QUESTION - assert wire[search["spanId"]]["input.mime_type"] == "text/plain" - # Spans that carry no content get no Input/Output. - for span in spans_named(recorded, "cognee.db.vector.search"): - assert not set(IO_KEYS) & set(wire[span["spanId"]]) - # Default mode adds none of them. - for span in filtered(cognee_filter, recorded): - assert not set(IO_KEYS) & set(attributes(span)), span["name"] - - -def test_filter_capture_content_keeps_an_existing_span_input_and_output( - cognee_filter: Any, -) -> None: - exporter = InMemorySpanExporter() - provider = TracerProvider() - provider.add_span_processor( - SimpleSpanProcessor(cognee_filter.CogneeExportFilter(exporter, capture_content=True)) - ) - with provider.get_tracer("test").start_as_current_span(GENERATION_SPAN) as span: - span.set_attribute("langfuse.observation.input", '{"text_input": "cognee prompt"}') - span.set_attribute("langfuse.observation.output", "cognee answer") - span.set_attribute("input.value", "set by another instrumentor") - span.set_attribute("input.mime_type", "text/plain") - (exported,) = exporter.get_finished_spans() - assert exported.attributes["input.value"] == "set by another instrumentor" - assert exported.attributes["input.mime_type"] == "text/plain" - assert exported.attributes["output.value"] == "cognee answer" - assert exported.attributes["output.mime_type"] == "text/plain" - - -@pytest.mark.parametrize( - ("env_value", "argument", "kept"), - [ - (None, None, False), - ("true", None, True), - ("TRUE", None, True), - ("false", None, False), - ("true", False, False), - (None, True, True), - ], -) -def test_filter_capture_switch( - cognee_filter: Any, monkeypatch: pytest.MonkeyPatch, env_value: Any, argument: Any, kept: bool -) -> None: - if env_value is None: - monkeypatch.delenv("COGNEE_FI_CAPTURE_CONTENT", raising=False) - else: - monkeypatch.setenv("COGNEE_FI_CAPTURE_CONTENT", env_value) - exporter = InMemorySpanExporter() - options = {} if argument is None else {"capture_content": argument} - provider = TracerProvider() - provider.add_span_processor( - SimpleSpanProcessor(cognee_filter.CogneeExportFilter(exporter, **options)) - ) - with provider.get_tracer("test").start_as_current_span("memory.retrieve") as span: - span.set_attribute("memory.query.text", QUESTION) - (exported,) = exporter.get_finished_spans() - assert ("memory.query.text" in exported.attributes) is kept - - -def test_filter_does_not_change_spans_other_processors_see(cognee_filter: Any) -> None: - filtered_out, raw_out = InMemorySpanExporter(), InMemorySpanExporter() - provider = TracerProvider() - # The filter runs first; the plain exporter after it must see the original. - provider.add_span_processor(SimpleSpanProcessor(cognee_filter.CogneeExportFilter(filtered_out))) - provider.add_span_processor(SimpleSpanProcessor(raw_out)) - with provider.get_tracer("test").start_as_current_span("memory.retrieve") as span: - span.set_attribute("memory.query.text", QUESTION) - span.add_event("exception", {"exception.message": "bad " + QUESTION}) - span.set_status(Status(StatusCode.ERROR, "bad " + QUESTION)) - (raw,) = raw_out.get_finished_spans() - (clean,) = filtered_out.get_finished_spans() - assert dict(raw.attributes) == {"memory.query.text": QUESTION} - assert dict(raw.events[0].attributes) == {"exception.message": "bad " + QUESTION} - assert raw.status.description == "bad " + QUESTION - assert dict(clean.attributes) == {"fi.span.kind": "RETRIEVER"} - assert dict(clean.events[0].attributes) == {"exception.message": DETAIL_REMOVED} - # The exception event names no type, so the status cannot either. - assert clean.status.description == DETAIL_REMOVED - assert clean.context == raw.context and clean.parent == raw.parent - - -def test_filter_that_raises_still_exports_spans_without_content( - cognee_filter: Any, recorded: list[dict[str, Any]], monkeypatch: pytest.MonkeyPatch -) -> None: - def broken(*args: Any, **kwargs: Any) -> Any: - raise RuntimeError("filter bug") - - monkeypatch.setattr(cognee_filter.CogneeExportFilter, "_filtered", broken) - wire = filtered(cognee_filter, recorded) - assert [span["name"] for span in wire] == [span["name"] for span in recorded] - assert_no_content(wire) - for span in wire: - assert "events" not in span and "message" not in span.get("status", {}) - - -def test_filter_drops_a_span_it_cannot_strip( - cognee_filter: Any, recorded: list[dict[str, Any]], monkeypatch: pytest.MonkeyPatch -) -> None: - def broken(*args: Any, **kwargs: Any) -> Any: - raise RuntimeError("filter bug") - - monkeypatch.setattr(cognee_filter.CogneeExportFilter, "_filtered", broken) - monkeypatch.setattr(cognee_filter.CogneeExportFilter, "_stripped", broken) - assert filtered(cognee_filter, recorded) == [] - - -# -------------------------------------------------------------------------- -# README and requirements -# -------------------------------------------------------------------------- - - -def test_readme_states_what_the_tests_check() -> None: - readme = README.read_text(encoding="utf-8") - app = APP.read_text(encoding="utf-8") - for line in ( - "exporter = CogneeExportFilter(HTTPSpanExporter())", - "provider.add_span_processor(BatchSpanProcessor(exporter))", - ): - assert line in app and line in readme, line - for fact in ( - "cognee==1.6.5", - "COGNEE_TRACING_ENABLED", - "/tracer/v1/traces", - "X-Api-Key", - "X-Secret-Key", - "project_type=observe", - "https://api.futureagi.com:443/tracer/v1/traces", - "COGNEE_FI_CAPTURE_CONTENT=true", - MISSING_SWITCH + "=true", - MISSING_KEY, - "capture_content=True", - "fi.span.kind", - DETAIL_REMOVED, - TYPE_ONLY.strip(" ()"), - *FILTERED_KEYS, - *SPAN_KINDS, - ): - assert fact in readme, fact - - -@pytest.mark.parametrize( - "fact", - [ - pytest.param("TH-8394", id="R2-register-follow-up"), - pytest.param("pass the same ones", id="R3-same-exporter-options"), - pytest.param('pip install "cognee[tracing]==1.6.5"', id="R5-option-b-install"), - pytest.param( - "Stack traces are never exported, in either mode", id="RV-stacktrace-both-modes" - ), - pytest.param("x-api-key=...,x-secret-key=...", id="RV-option-b-grpc-lowercase-headers"), - pytest.param("cognee.disable_tracing()", id="R7-disable-tracing"), - pytest.param("License-Expression: Apache-2.0", id="D9-license-metadata"), - pytest.param("less useful for debugging", id="N1-error-detail-trade-off"), - pytest.param("among span attributes", id="F1-capture-claim-scope"), - pytest.param("`cognee.api.recall`", id="F2-recall-unmapped"), - pytest.param("`cognee.agent_memory.retrieve`", id="F2-agent-memory-unmapped"), - ], -) -def test_readme_covers_review_items(fact: str) -> None: - assert fact in README.read_text(encoding="utf-8") - - -def test_requirements_pin_the_register_behaviour_the_recipe_relies_on() -> None: - lines = REQUIREMENTS.read_text(encoding="utf-8").splitlines() - assert "fi-instrumentation-otel>=1.1.0,<1.2" in lines - assert "cognee==1.6.5" in lines diff --git a/python/frameworks/cognee/CHANGELOG.md b/python/frameworks/cognee/CHANGELOG.md new file mode 100644 index 000000000..c331b1094 --- /dev/null +++ b/python/frameworks/cognee/CHANGELOG.md @@ -0,0 +1,8 @@ +## [0.1.0] + +### Added +- Export Cognee's native OpenTelemetry spans to Future AGI without duplicate instrumentation. +- Map native search and structured-output model calls to Future AGI span types while preserving IDs, parents and error statuses. +- Exclude content, error descriptions and stack traces by default; provide content opt-in with bounded structural credential-field redaction. +- Reuse a provider-owned processor across configurations, keep other exporters unchanged, and isolate spans across handle shutdown and reconfiguration. +- Require `fi-instrumentation-otel>=1.2.2,<2` and Python 3.10–3.13. diff --git a/python/frameworks/cognee/LICENSE b/python/frameworks/cognee/LICENSE new file mode 100644 index 000000000..72ffed2f0 --- /dev/null +++ b/python/frameworks/cognee/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for describing the origin of the Work and + reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may accept and charge a + fee for acceptance of support, warranty, indemnity, or other liability + obligations and/or rights consistent with this License. However, in + accepting such obligations, You may act only on Your own behalf and on + Your sole responsibility, not on behalf of any other Contributor, and + only if You agree to indemnify, defend, and hold each Contributor + harmless for any liability incurred by, or claims asserted against, + such Contributor by reason of your accepting any such warranty or + additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2024-2026 Future AGI, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/python/frameworks/cognee/README.md b/python/frameworks/cognee/README.md new file mode 100644 index 000000000..d19fa5402 --- /dev/null +++ b/python/frameworks/cognee/README.md @@ -0,0 +1,39 @@ +# traceai-cognee + +Send Cognee's native OpenTelemetry spans to Future AGI without additional instrumentation. Enable `COGNEE_TRACING_ENABLED=true` before your first Cognee operation and configure once before that operation. + +Requires Python 3.10–3.13 and `fi-instrumentation-otel>=1.2.2,<2`. + +```bash +pip install "cognee[tracing]==1.6.5" "traceai-cognee==0.1.0" +export COGNEE_TRACING_ENABLED=true +export FI_API_KEY="YOUR_FI_API_KEY" +export FI_SECRET_KEY="YOUR_FI_SECRET_KEY" +export FI_PROJECT_NAME="my-cognee-app" +``` + +Configure your model and embedding providers using the [Cognee documentation](https://docs.cognee.ai/), then run your operations with the adapter attached: + +```python +import os +from traceai_cognee import configure + +handle = configure(project_name=os.environ['FI_PROJECT_NAME']) +# Run cognee.add, cognee.cognify and cognee.search here. +handle.force_flush() +handle.shutdown() +``` + +Set `FI_API_KEY` and `FI_SECRET_KEY` for the Future AGI exporter. `FI_BASE_URL` selects a different Future AGI origin. Alternatively supply an OTLP HTTP `endpoint` and `headers`, or an injected OpenTelemetry exporter. A supplied exporter is owned by the handle. `provider` can be the global SDK provider Cognee will use. Existing processors are not shut down; a provider whose default processor would be replaced on add is rejected. + +A repeated `configure` with the same options returns the active handle; different options require shutting it down first. Shutdown detaches the handle immediately and closes the exporter asynchronously, after in-flight callbacks return; it does not wait for exporter closure. Flush before shutdown when the queue must be drained. Each span is exported by the configuration that was active when it started: end spans and flush before shutdown, because a span still open at shutdown is not exported, and a later configuration does not adopt it. The adapter tracks at most 65,536 open spans; beyond that the oldest open span is dropped when it ends. Spans that share a trace ID, span ID and start time with another tracked span are not exported; such reuse only happens with a faulty ID generator, and once the limit evicts the earlier span's entry the protection no longer applies to it. If a custom provider raises from `add_span_processor` after attaching the processor, a retry reuses it; a provider that hides its processor list can keep an inactive extra processor for each failed attempt. Closure uses a non-daemon thread, so a stuck exporter can delay interpreter exit. An old handle cannot flush or shut down a later configuration. Shutting down the provider closes the adapter too and prevents reuse of that provider. Overlapping `configure` calls fail fast with `RuntimeError`; retry after the first call finishes. A successful `force_flush` drains the processor queue, not an acknowledgement of remote delivery. + +By default only allowlisted operational attributes are sent. `capture_content=True` includes recognized Cognee input/output fields. Error details and stack traces remain excluded. Span names are retained; keep sensitive data out of them. Other processors on the provider receive unmodified native spans and need their own privacy controls. + +### Structured content redaction + +With content capture enabled, the adapter redacts recognized credential field names in nested objects, arrays, JSON-serialized objects, and flattened paths. These include API keys, authorization headers, access/refresh/auth/bearer/ID/session/security tokens, client secrets, secret/private keys, passwords, credentials, and AWS access-key IDs and secret access keys. Common snake-case, camel-case, and dotted/bracketed path forms are recognized. Fields such as `input_tokens`, `token_count`, and `request_id` remain available. + +This is a field-name policy, not a universal secret detector. Free text and unknown encodings are not scanned for credentials; enable content capture only for approved application data. The adapter does not mutate the original content or inspect opaque runtime objects. Numeric attributes outside the OTLP signed 64-bit integer range, or non-finite floating-point values, are omitted before encoding. + +For a complete add/cognify/search application, see the [Cognee example](../../examples/cognee/). Search spans are mapped to `RETRIEVER` and structured-output model calls to `LLM`. Native span IDs, parent relationships and status codes are preserved, including Cognee's missing-collection probe errors. diff --git a/python/frameworks/cognee/pyproject.toml b/python/frameworks/cognee/pyproject.toml new file mode 100644 index 000000000..94ceef1eb --- /dev/null +++ b/python/frameworks/cognee/pyproject.toml @@ -0,0 +1,26 @@ +[build-system] +requires = ["hatchling>=1.27,<2"] +build-backend = "hatchling.build" + +[project] +name = "traceai-cognee" +version = "0.1.0" +description = "Future AGI exporter for Cognee native OpenTelemetry spans" +readme = "README.md" +license = "Apache-2.0" +authors = [{name = "Future AGI", email = "no-reply@futureagi.com"}] +requires-python = ">=3.10,<3.14" +dependencies = ["fi-instrumentation-otel>=1.2.2,<2", "opentelemetry-sdk>=1.27,<2"] + +[project.optional-dependencies] +dev = ["pytest>=8,<9"] + +[project.urls] +Repository = "https://github.com/future-agi/traceAI" +Documentation = "https://docs.futureagi.com/docs/integrations/traceai/cognee" + +[tool.hatch.build.targets.wheel] +packages = ["traceai_cognee"] + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/python/frameworks/cognee/tests/test_adapter.py b/python/frameworks/cognee/tests/test_adapter.py new file mode 100644 index 000000000..349dcc5ad --- /dev/null +++ b/python/frameworks/cognee/tests/test_adapter.py @@ -0,0 +1,96 @@ +import json +import unittest +from opentelemetry import trace +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import Status, StatusCode +from traceai_cognee import configure, __version__ + +class TestAdapter(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.provider = TracerProvider() + trace.set_tracer_provider(cls.provider) + + def setUp(self): + self.other = InMemorySpanExporter() + self.provider.add_span_processor(SimpleSpanProcessor(self.other)) + self.sink = InMemorySpanExporter() + + def emit(self, name, attrs): + with self.provider.get_tracer('test').start_as_current_span(name) as span: + for key, value in attrs.items(): span.set_attribute(key, value) + span.record_exception(ValueError('fake-private-error')) + span.set_status(Status(StatusCode.ERROR, 'fake-private-error')) + + def test_projection_copy_and_status(self): + handle = configure(project_name='sample', exporter=self.sink) + self.assertIs(handle, configure(project_name='sample', exporter=self.sink)) + with self.assertRaises(ValueError): configure(project_name='different', exporter=self.sink) + self.emit('cognee.search.query', {'cognee.search.query':'fake-private-query', 'custom.secret':'fake-secret', 'gen_ai.usage.input_tokens':7}) + self.assertTrue(handle.force_flush()) + src = self.other.get_finished_spans()[-1]; dst = self.sink.get_finished_spans()[-1] + self.assertEqual(src.attributes['cognee.search.query'], 'fake-private-query') + self.assertNotIn('cognee.search.query', dst.attributes) + self.assertNotIn('custom.secret', dst.attributes) + self.assertEqual(dst.attributes['gen_ai.usage.input_tokens'], 7) + self.assertEqual(dst.attributes['fi.span.kind'], 'RETRIEVER') + self.assertEqual(dst.status.status_code, StatusCode.ERROR) + self.assertIsNone(dst.status.description) + self.assertEqual(dst.context, src.context); self.assertEqual(dst.parent, src.parent) + self.assertEqual(dst.start_time, src.start_time); self.assertEqual(dst.end_time, src.end_time) + self.assertEqual(dst.resource.attributes['project_name'], 'sample') + self.assertNotIn('fake-private-error', str(dst.events)) + handle.shutdown() + self.emit('after', {}) + self.assertEqual(self.other.get_finished_spans()[-1].name, 'after') + + def test_content_redaction_and_llm(self): + handle = configure(project_name='sample', exporter=self.sink, capture_content=True) + value = json.dumps({'api_key':'fake-secret', 'question':'hello'}) + self.emit('cognee.observe.acreate_structured_output', {'langfuse.observation.input':value, 'langfuse.observation.output':'answer'}) + handle.force_flush(); out = self.sink.get_finished_spans()[-1] + self.assertEqual(out.attributes['fi.span.kind'], 'LLM') + self.assertEqual(json.loads(out.attributes['input.value'])['api_key'], '[REDACTED]') + self.assertEqual(out.attributes['output.value'], 'answer') + self.assertNotIn('fake-secret', str(out.attributes)) + handle.shutdown() + + def test_additional_processor_and_lifecycle(self): + handle = configure(project_name='sample', exporter=self.sink) + native_buffer = InMemorySpanExporter() + self.provider.add_span_processor(SimpleSpanProcessor(native_buffer)) + self.emit('memory.retrieve', {'memory.query.text': 'question'}) + self.assertTrue(handle.force_flush()) + self.assertEqual(len(self.sink.get_finished_spans()), 1) + self.assertEqual(native_buffer.get_finished_spans()[-1].attributes['memory.query.text'], 'question') + handle.shutdown() + handle.shutdown() + self.emit('later', {}) + self.assertTrue(self.provider.force_flush()) + self.assertEqual(native_buffer.get_finished_spans()[-1].name, 'later') + second = InMemorySpanExporter() + next_handle = configure(project_name='sample', exporter=second) + self.emit('reconfigured', {}) + self.assertTrue(self.provider.force_flush()) + self.assertEqual([span.name for span in second.get_finished_spans()], ['reconfigured']) + next_handle.shutdown() + self.assertTrue(self.provider.force_flush()) + + def test_replaceable_default_refused_without_replacement(self): + self.provider._default_processor = True + try: + before = self.provider._active_span_processor._span_processors + with self.assertRaisesRegex(RuntimeError, 'replaceable default'): + configure(project_name='sample', exporter=self.sink) + self.assertEqual(before, self.provider._active_span_processor._span_processors) + finally: + del self.provider._default_processor + + def test_default_exporter_constructs_without_sending(self): + handle = configure(project_name='sample', endpoint='http://127.0.0.1:1/v1/traces', headers={'x-api-key': 'synthetic'}) + self.assertTrue(handle.force_flush()) + handle.shutdown() + + def test_version(self): self.assertEqual(__version__, '0.1.0') diff --git a/python/frameworks/cognee/tests/test_generation_isolation.py b/python/frameworks/cognee/tests/test_generation_isolation.py new file mode 100644 index 000000000..c40576476 --- /dev/null +++ b/python/frameworks/cognee/tests/test_generation_isolation.py @@ -0,0 +1,201 @@ +"""Spans stay with the configuration active when they started; all inputs are synthetic.""" +import subprocess +import sys +import unittest + +ISOLATION = ''' +import threading +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +import traceai_cognee +from traceai_cognee import configure +LLM = 'cognee.observe.acreate_structured_output' +INPUT = 'langfuse.observation.input' +a, b = InMemorySpanExporter(), InMemorySpanExporter() +ha = configure(project_name='project-A', exporter=a, capture_content=True) +tracer = ha.provider.get_tracer('test') + +# Ends while A is active: exported by A. +with tracer.start_as_current_span(LLM, attributes={INPUT: 'kept-by-A'}): pass +# Starts under A, ends after A shuts down and B is configured: exported by neither. +late = tracer.start_span(LLM, attributes={INPUT: 'data-owned-by-A'}) +assert ha.force_flush() +ha.shutdown() +# Starts while nothing is configured: a later configuration does not adopt it. +orphan = tracer.start_span(LLM, attributes={INPUT: 'data-owned-by-nobody'}) +hb = configure(project_name='project-B', exporter=b, capture_content=True) +late.end(); orphan.end() +with tracer.start_as_current_span(LLM, attributes={INPUT: 'owned-by-B'}): pass +assert hb.force_flush() +seen_a = [s.attributes.get('input.value') for s in a.get_finished_spans()] +seen_b = [(s.resource.attributes['project_name'], s.attributes.get('input.value')) for s in b.get_finished_spans()] +assert seen_a == ['kept-by-A'], seen_a +assert seen_b == [('project-B', 'owned-by-B')], seen_b + +# Interleaved spans across concurrent threads under B are all exported once, by B. +def work(i): + with tracer.start_as_current_span('cognee.search.query', attributes={'gen_ai.usage.input_tokens': i}): pass +threads = [threading.Thread(target=work, args=(i,)) for i in range(64)] +[t.start() for t in threads]; [t.join(10) for t in threads] +assert hb.force_flush() +tokens = sorted(s.attributes['gen_ai.usage.input_tokens'] for s in b.get_finished_spans()[1:]) +assert tokens == list(range(64)), tokens +slot = traceai_cognee._slots[hb.provider] +assert not slot._open, len(slot._open) +hb.shutdown() +''' + +CAP = ''' +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +import traceai_cognee +from traceai_cognee import configure +traceai_cognee._MAX_OPEN_SPANS = 3 +sink = InMemorySpanExporter() +h = configure(project_name='cap', exporter=sink) +tracer = h.provider.get_tracer('test') +spans = [tracer.start_span('open' + str(i)) for i in range(5)] +slot = traceai_cognee._slots[h.provider] +assert len(slot._open) == 3, len(slot._open) +spans[0].end() # evicted while open: dropped, never re-attributed +spans[4].end() # still tracked: exported +assert h.force_flush() +assert [s.name for s in sink.get_finished_spans()] == ['open4'], sink.get_finished_spans() +h.shutdown() +# Spans still open at shutdown stay marked, unexported, until they end. +assert all(isinstance(v, traceai_cognee._Ambiguous) for v in slot._open.values()), slot._open +spans[2].end(); spans[3].end() +assert not slot._open +''' + +DUPLICATE_IDS = ''' +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.id_generator import IdGenerator +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from traceai_cognee import configure +LLM = 'cognee.observe.acreate_structured_output' +INPUT = 'langfuse.observation.input' +class Fixed(IdGenerator): + def generate_span_id(self): return 2 + def generate_trace_id(self): return 1 +p = TracerProvider(id_generator=Fixed(), shutdown_on_exit=False) +a, b = InMemorySpanExporter(), InMemorySpanExporter() +ha = configure(project_name='A', provider=p, exporter=a, capture_content=True) +tracer = p.get_tracer('test') +old = tracer.start_span(LLM, attributes={INPUT: 'data-from-A'}) +ha.shutdown() +hb = configure(project_name='B', provider=p, exporter=b, capture_content=True) +new = tracer.start_span(LLM, attributes={INPUT: 'data-from-B'}) +# The reused IDs must not let A's span take B's place. +old.end(); new.end() +assert hb.force_flush() +seen = [s.attributes.get('input.value') for s in b.get_finished_spans()] +assert seen == ['data-from-B'], seen +# Two open spans with the same identity cannot be told apart: neither is exported. +twins = [tracer.start_span('twin', start_time=10**18) for _ in range(2)] +twins[0].end(); twins[1].end() +assert hb.force_flush() +assert [s.name for s in b.get_finished_spans()] == [LLM], b.get_finished_spans() +hb.shutdown() +''' + +IDENTICAL_IDENTITY = ''' +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.id_generator import IdGenerator +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +import traceai_cognee +from traceai_cognee import configure +INPUT = 'langfuse.observation.input' +class Fixed(IdGenerator): + def generate_span_id(self): return 2 + def generate_trace_id(self): return 1 +p = TracerProvider(id_generator=Fixed(), shutdown_on_exit=False) +a, b = InMemorySpanExporter(), InMemorySpanExporter() +ha = configure(project_name='A', provider=p, exporter=a, capture_content=True) +tracer = p.get_tracer('test') +# Same IDs and start time across a reconfiguration: B must not adopt A's span. +old = tracer.start_span('old', start_time=10**18, attributes={INPUT: 'data-from-A'}) +ha.shutdown() +hb = configure(project_name='B', provider=p, exporter=b, capture_content=True) +new = tracer.start_span('new', start_time=10**18) +old.end(); new.end() +assert hb.force_flush() +assert not b.get_finished_spans(), [(s.name, s.attributes.get('input.value')) for s in b.get_finished_spans()] +# Overlapping duplicates stay unattributable until every one has ended. +first = tracer.start_span('first', start_time=2 * 10**18) +second = tracer.start_span('second', start_time=2 * 10**18) +first.end() +third = tracer.start_span('third', start_time=2 * 10**18) +second.end(); third.end() +fresh = tracer.start_span('fresh', start_time=2 * 10**18); fresh.end() +assert hb.force_flush() +assert [s.name for s in b.get_finished_spans()] == ['fresh'], [s.name for s in b.get_finished_spans()] +assert not traceai_cognee._slots[p]._open +hb.shutdown() +''' + +PARTIAL_ATTACH = ''' +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from traceai_cognee import configure +class AttachThenRaise(TracerProvider): + def add_span_processor(self, processor): + super().add_span_processor(processor) + raise RuntimeError('provider failed after attaching') +class RaiseBeforeAttach(TracerProvider): + failures = 2 + def add_span_processor(self, processor): + if RaiseBeforeAttach.failures: + RaiseBeforeAttach.failures -= 1 + raise RuntimeError('provider failed before attaching') + return super().add_span_processor(processor) +# Each case runs in its own process: the first configure sets the global provider. +cls, retries = {'attach_then_raise': (AttachThenRaise, 1), 'raise_before_attach': (RaiseBeforeAttach, 2)}[CASE] +provider = cls(shutdown_on_exit=False) +# A handle owns and closes its exporter, so each configuration gets its own. +for _ in range(retries): + try: + configure(project_name='p', provider=provider, exporter=InMemorySpanExporter()) + raise AssertionError('provider error was swallowed') + except RuntimeError as error: + assert 'provider failed' in str(error), error +handle = configure(project_name='p', provider=provider, exporter=InMemorySpanExporter()) +for _ in range(3): + handle.shutdown() + sink = InMemorySpanExporter() + handle = configure(project_name='p', provider=provider, exporter=sink) +assert len(provider._active_span_processor._span_processors) == 1, cls +with provider.get_tracer('t').start_as_current_span('after-retry'): pass +assert handle.force_flush() +assert [s.name for s in sink.get_finished_spans()] == ['after-retry'], cls +handle.shutdown() +''' + + +class GenerationIsolationTests(unittest.TestCase): + def run_fresh(self, code, case=None): + if case is not None: + code = 'CASE = %r\n' % case + code + result = subprocess.run([sys.executable, '-c', code], cwd=sys.prefix, + capture_output=True, text=True, timeout=30) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_spans_never_cross_configurations(self): + self.run_fresh(ISOLATION) + + def test_open_span_tracking_is_bounded(self): + self.run_fresh(CAP) + + def test_duplicate_span_ids_never_cross_configurations(self): + self.run_fresh(DUPLICATE_IDS) + + def test_identical_identity_is_never_adopted_or_reattributed(self): + self.run_fresh(IDENTICAL_IDENTITY) + + def test_attach_then_raise_retry_reuses_one_slot(self): + self.run_fresh(PARTIAL_ATTACH, 'attach_then_raise') + + def test_raise_before_attach_retry_reuses_one_slot(self): + self.run_fresh(PARTIAL_ATTACH, 'raise_before_attach') + + +if __name__ == '__main__': + unittest.main() diff --git a/python/frameworks/cognee/tests/test_lifecycle_regressions.py b/python/frameworks/cognee/tests/test_lifecycle_regressions.py new file mode 100644 index 000000000..5bac08357 --- /dev/null +++ b/python/frameworks/cognee/tests/test_lifecycle_regressions.py @@ -0,0 +1,207 @@ +"""Bounded lifecycle regressions; all exporters and inputs are synthetic.""" +import gc +import subprocess +import sys +import threading +import unittest +import weakref + +from opentelemetry import trace +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult +from traceai_cognee import configure + + +class Sink(SpanExporter): + def __init__(self): + self.names = [] + self.closed = 0 + self.closed_event = threading.Event() + def export(self, spans): + self.names.extend(s.name for s in spans) + return SpanExportResult.SUCCESS + def shutdown(self): + self.closed += 1 + self.closed_event.set() + + +class LifecycleTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + existing = trace.get_tracer_provider() + if isinstance(existing, TracerProvider): + cls.provider = existing + else: + cls.provider = TracerProvider(shutdown_on_exit=False) + trace.set_tracer_provider(cls.provider) + + def test_reentry_and_close_during_flush(self): + started, release, worker_done = (threading.Event() for _ in range(3)) + class Reentrant(Sink): + def export(s, spans): + if not started.is_set(): + started.set() + def work(): + with tracer.start_as_current_span('worker'): + pass + worker_done.set() + t = threading.Thread(target=work, daemon=True) + t.start() + if not worker_done.wait(2): + raise AssertionError('traced exporter worker blocked') + if not release.wait(2): + raise AssertionError('test did not release exporter') + return super().export(spans) + sink = Reentrant() + handle = configure(project_name='reentry', exporter=sink) + tracer = self.provider.get_tracer('test') + with tracer.start_as_current_span('original'): pass + flush = threading.Thread(target=handle.force_flush, daemon=True) + flush.start() + try: + self.assertTrue(started.wait(3)) + self.assertTrue(worker_done.wait(3)) + close = threading.Thread(target=handle.shutdown, daemon=True) + close.start(); close.join(1) + self.assertFalse(close.is_alive(), 'shutdown waited for exporter') + finally: + release.set(); flush.join(5) + self.assertFalse(flush.is_alive()) + self.assertTrue(sink.closed_event.wait(5)) + self.assertEqual(sink.closed, 1) + handle.shutdown() + self.assertEqual(sink.closed, 1) + + def test_reuse_old_handles_collection_and_provider_shutdown(self): + baseline = len(self.provider._active_span_processor._span_processors) + refs = [] + old = None + for index in range(25): + sink = Sink(); refs.append(weakref.ref(sink)) + h = configure(project_name='p' + str(index), exporter=sink) + if old: + old.shutdown() + self.assertTrue(old.force_flush()) + with self.provider.get_tracer('test').start_as_current_span('span' + str(index)): pass + self.assertTrue(h.force_flush()) + self.assertEqual(sink.names, ['span' + str(index)]) + h.shutdown(); self.assertTrue(sink.closed_event.wait(5)); self.assertEqual(sink.closed, 1) + old = h + del sink, h + gc.collect() + self.assertTrue(all(ref() is None for ref in refs)) + self.assertLessEqual(len(self.provider._active_span_processor._span_processors), baseline + 1) + + def test_raw_refusal_and_validation_leave_global_usable(self): + code = ''' +from opentelemetry import trace +from opentelemetry.trace import ProxyTracerProvider +from opentelemetry.sdk.trace import TracerProvider as Ordinary +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from fi_instrumentation.otel import TracerProvider +from traceai_cognee import configure +before = trace.get_tracer_provider() +assert isinstance(before, ProxyTracerProvider) +p = TracerProvider(verbose=False) +old = p._active_span_processor._span_processors +sink = InMemorySpanExporter() +try: + configure(project_name='x', provider=p, exporter=sink) + raise AssertionError('raw provider accepted') +except RuntimeError: + pass +assert trace.get_tracer_provider() is before +assert p._active_span_processor._span_processors == old and p._default_processor +try: + configure(project_name='x', headers=object()) + raise AssertionError('invalid headers accepted') +except (TypeError, AttributeError): + pass +assert trace.get_tracer_provider() is before +try: + configure(project_name='x', exporter=object()) + raise AssertionError('invalid exporter accepted') +except TypeError: + pass +assert trace.get_tracer_provider() is before +h = configure(project_name='valid', provider=Ordinary(shutdown_on_exit=False), exporter=sink) +assert trace.get_tracer_provider() is h.provider +h.shutdown(); p.shutdown() +''' + result = subprocess.run([sys.executable, '-c', code], cwd=sys.prefix, capture_output=True, text=True, timeout=20) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_background_exporter_can_join_shutdown_worker(self): + code = ''' +import threading +from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult +from traceai_cognee import configure +ready=threading.Event(); entered=threading.Event(); finished=threading.Event(); blocked=[] +class Sink(SpanExporter): + def export(self, spans): + entered.set() + assert ready.wait(5) + t=threading.Thread(target=h.shutdown, daemon=True); t.start(); t.join(1) + blocked.append(t.is_alive()) + finished.set() + return SpanExportResult.SUCCESS + def shutdown(self): pass +h=configure(project_name='x', exporter=Sink()) +tracer=h.provider.get_tracer('test') +for i in range(600): + with tracer.start_as_current_span('span'): pass +assert entered.wait(5) +ready.set() +assert finished.wait(5) +assert not any(blocked), 'background exporter waited on shutdown worker, which waited on exporter' +''' + result = subprocess.run([sys.executable, '-c', code], cwd=sys.prefix, capture_output=True, text=True, timeout=20) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_overlapping_configure_fails_fast_and_first_succeeds(self): + code = ''' +import threading +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from traceai_cognee import configure +entered=threading.Event(); release=threading.Event(); result=[] +class Provider(TracerProvider): + def add_span_processor(self, processor): + entered.set() + assert release.wait(3) + return super().add_span_processor(processor) +p=Provider(shutdown_on_exit=False); sink=InMemorySpanExporter() +def first(): + result.append(configure(project_name='x', provider=p, exporter=sink)) +t=threading.Thread(target=first, daemon=True); t.start(); assert entered.wait(3) +try: + configure(project_name='x', provider=p, exporter=sink) + raise AssertionError('overlapping configure accepted') +except RuntimeError as e: + assert 'configure is in progress' in str(e) +finally: + release.set() +t.join(5); assert not t.is_alive() and len(result)==1 +assert configure(project_name='x', provider=p, exporter=sink) is result[0] +result[0].shutdown(); p.shutdown() +''' + result = subprocess.run([sys.executable, '-c', code], cwd=sys.prefix, capture_output=True, text=True, timeout=20) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_provider_shutdown_closes_once_and_rejects_reuse(self): + code = ''' +from traceai_cognee import configure +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +s=InMemorySpanExporter(); h=configure(project_name='x', exporter=s) +h.provider.shutdown(); h.shutdown() +try: + configure(project_name='again', exporter=InMemorySpanExporter()) + raise AssertionError('shutdown provider accepted') +except RuntimeError: + pass +''' + result = subprocess.run([sys.executable, '-c', code], cwd=sys.prefix, capture_output=True, text=True, timeout=20) + self.assertEqual(result.returncode, 0, result.stderr) + +if __name__ == '__main__': + unittest.main() diff --git a/python/frameworks/cognee/tests/test_security_regressions.py b/python/frameworks/cognee/tests/test_security_regressions.py new file mode 100644 index 000000000..cdaffc2c9 --- /dev/null +++ b/python/frameworks/cognee/tests/test_security_regressions.py @@ -0,0 +1,94 @@ +"""Synthetic regressions for the native export projection.""" +import json +import math +import unittest + +from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +from traceai_cognee._exporter import project +from traceai_cognee._structured import redact_structured + + +class TestSecurityRegressions(unittest.TestCase): + def spans(self, attributes): + provider = TracerProvider(shutdown_on_exit=False) + sink = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(sink)) + try: + for attrs in attributes: + with provider.get_tracer('synthetic').start_as_current_span( + 'cognee.observe.acreate_structured_output', attributes=attrs + ): + pass + return sink.get_finished_spans() + finally: + provider.shutdown() + + def test_aws_credential_aliases_and_legitimate_fields(self): + credential = 'synthetic-aws-credential-should-be-redacted' + source = { + 'aws_secret_access_key': credential, + 'aws_access_key_id': credential, + 'aws_session_token': credential, + 'awsSecretAccessKey': credential, + 'credentials': {'accessKeyId': credential, 'secretAccessKey': credential}, + 'client.aws_secret_access_key.value': credential, + 'client[awsSessionToken][value]': credential, + 'nested': json.dumps({'aws_access_key_id': credential, 'question': 'hello'}), + 'input_token': 7, + 'token_count': 11, + 'request_id': 'request-1', + } + before = json.dumps(source) + sanitized = redact_structured(source) + self.assertEqual(json.dumps(source), before) + self.assertNotIn(credential, json.dumps(sanitized)) + self.assertEqual(sanitized['input_token'], 7) + self.assertEqual(sanitized['token_count'], 11) + self.assertEqual(sanitized['request_id'], 'request-1') + self.assertEqual(json.loads(sanitized['nested'])['question'], 'hello') + (span,) = self.spans([{'langfuse.observation.input': before}]) + projected = project(span, 'synthetic', True) + self.assertNotIn(credential.encode(), encode_spans([projected]).SerializeToString()) + self.assertEqual(json.loads(projected.attributes['input.value'])['request_id'], 'request-1') + + def test_integer_boundaries_encode_without_raw_value_logs(self): + for value in (-(2**63), 2**63 - 1, -(2**63) - 1, 2**63, 10**100): + with self.subTest(value_size=value.bit_length()): + spans = self.spans([ + {'gen_ai.usage.input_tokens': value}, + {'gen_ai.usage.output_tokens': 9}, + ]) + projected = [project(span, 'synthetic', False) for span in spans] + with self.assertNoLogs('opentelemetry.exporter.otlp.proto.common', level='ERROR'): + encoded = encode_spans(projected) + encoded.SerializeToString() + self.assertEqual(sum(len(s.spans) for r in encoded.resource_spans for s in r.scope_spans), 2) + if -(2**63) <= value < 2**63: + self.assertEqual(projected[0].attributes['gen_ai.usage.input_tokens'], value) + else: + self.assertNotIn('gen_ai.usage.input_tokens', projected[0].attributes) + self.assertEqual(projected[1].attributes['gen_ai.usage.output_tokens'], 9) + + def test_opt_in_numeric_content_is_wire_safe(self): + for value in (2**63, -(2**63) - 1, math.inf, -math.inf, math.nan): + with self.subTest(type=type(value).__name__): + (span,) = self.spans([{'langfuse.observation.output': value}]) + projected = project(span, 'synthetic', True) + self.assertNotIn('langfuse.observation.output', projected.attributes) + self.assertNotIn('output.value', projected.attributes) + with self.assertNoLogs('opentelemetry.exporter.otlp.proto.common', level='ERROR'): + encode_spans([projected]).SerializeToString() + + def test_finite_metadata_remains_available(self): + (span,) = self.spans([{'gen_ai.usage.input_tokens': 13, 'gen_ai.usage.output_tokens': 2.0}]) + projected = project(span, 'synthetic', False) + self.assertEqual(projected.attributes['gen_ai.usage.input_tokens'], 13) + self.assertEqual(projected.attributes['gen_ai.usage.output_tokens'], 2.0) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/frameworks/cognee/tests/test_structured.py b/python/frameworks/cognee/tests/test_structured.py new file mode 100644 index 000000000..305faed27 --- /dev/null +++ b/python/frameworks/cognee/tests/test_structured.py @@ -0,0 +1,70 @@ +"""Synthetic unit tests for the structural-redaction helper.""" +import copy +import importlib.util +import json +from pathlib import Path +import unittest + +path = Path(__file__).resolve().parents[1] / "traceai_cognee" / "_structured.py" +spec = importlib.util.spec_from_file_location("cognee_structured", path) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) + + +class TestStructuralRedaction(unittest.TestCase): + def test_nested_fields_and_counts(self): + source = { + "client": {"config": {"apiKey": "fake-secret-123", "model": "demo"}}, + "headers": {"Authorization": "fake-secret-123", "X-Api-Key": "fake-secret-123"}, + "client.config.refreshToken": "fake-secret-123", + "input_tokens": 42, + "token_count": 10, + "nested": json.dumps({"access_token": "fake-secret-123", "value": "kept"}), + } + before = copy.deepcopy(source) + result = module.redact_structured(source) + self.assertEqual(source, before) + self.assertNotIn("fake-secret-123", json.dumps(result)) + self.assertEqual(result["input_tokens"], 42) + self.assertEqual(result["token_count"], 10) + self.assertEqual(result["client"]["config"]["model"], "demo") + self.assertEqual(json.loads(result["nested"])["value"], "kept") + + def test_credential_paths_and_usage_metrics(self): + source = { + "http.request.header.authorization.value": "fake-secret-123", + "client[config][apiKey][value]": "fake-secret-123", + "client_config_api_key": "fake-secret-123", + "input_token": 5, + "input_tokens": 10, + "token_count": 15, + } + result = module.redact_structured(source) + for key in ("http.request.header.authorization.value", "client[config][apiKey][value]", "client_config_api_key"): + self.assertEqual(result[key], module.REDACTED) + for key in ("input_token", "input_tokens", "token_count"): + self.assertEqual(result[key], source[key]) + + def test_malformed_and_large_payloads_fail_closed(self): + self.assertEqual(module.redact_structured('{"api_key":"fake-secret-123"'), module.REDACTED) + self.assertEqual(module.redact_structured("x" * (module._MAX_TEXT + 1)), module.REDACTED) + self.assertEqual(module.redact_structured({"x" * 513: "fake-secret-123"}), module.REDACTED) + self.assertEqual(module.redact_structured({str(i): i for i in range(11000)}), module.REDACTED) + + def test_cycles_and_opaque_values_do_not_inspect_objects(self): + class Dangerous: + def __str__(self): + raise AssertionError("must not call str") + def __repr__(self): + raise AssertionError("must not call repr") + obj = [] + obj.append(obj) + self.assertEqual(module.redact_structured(obj), module.REDACTED) + self.assertEqual(module.redact_structured({"obj": Dangerous()}), module.REDACTED) + + def test_free_text_is_not_a_secret_detector(self): + self.assertEqual(module.redact_structured("fake-secret-123"), "fake-secret-123") + + +if __name__ == "__main__": + unittest.main() diff --git a/python/frameworks/cognee/traceai_cognee/__init__.py b/python/frameworks/cognee/traceai_cognee/__init__.py new file mode 100644 index 000000000..4cd68278f --- /dev/null +++ b/python/frameworks/cognee/traceai_cognee/__init__.py @@ -0,0 +1,279 @@ +"""Export Cognee's native OpenTelemetry spans to Future AGI.""" +from collections import OrderedDict +from collections.abc import Mapping +from threading import RLock, Thread +from weakref import WeakKeyDictionary +from opentelemetry import trace +from opentelemetry.sdk.trace import TracerProvider, SpanProcessor +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.trace import ProxyTracerProvider +from ._exporter import CogneeExporter + +__version__ = '0.1.0' +_lock = RLock() +_slots = WeakKeyDictionary() +_configuring = False +# Spans that start but never end must not grow memory without bound. An evicted +# entry cannot protect against faulty reuse of its IDs and identical start time. +_MAX_OPEN_SPANS = 65536 + + +class _Ambiguous: + """Open spans sharing one identity, none of which can be attributed safely. + + Until every such span ends or the cap evicts the marker, a later span + reusing this identity cannot be matched with an earlier one's end. + """ + __slots__ = ('open',) + + def __init__(self, open_spans): + self.open = open_spans + + +def _span_key(span): + context = getattr(span, 'context', None) + if context is None: + return None + # The start time keeps a span from an earlier configuration apart from a + # later one that reuses its IDs (for example a faulty ID generator). + return context.trace_id, context.span_id, getattr(span, 'start_time', None) + + +def _attached(provider, slot): + # Read-only check of the SDK's processor list, used only after a provider + # raised from add_span_processor, to tell whether it attached the slot anyway. + active = getattr(provider, '_active_span_processor', None) + return any(processor is slot for processor in getattr(active, '_span_processors', ())) + + +class _Generation: + """Lease callbacks without holding a lock across downstream code. + + Closing detaches immediately; the last outstanding callback closes the + delegate. In particular, close never waits for a callback that may itself + be waiting for a traced worker. + """ + def __init__(self, delegate): + self._lock = RLock() + self._delegate = delegate + self._inflight = 0 + self._closed = False + + def call(self, method, *args): + with self._lock: + if self._closed: + return True if method == 'force_flush' else None + delegate = self._delegate + self._inflight += 1 + try: + return getattr(delegate, method)(*args) + finally: + with self._lock: + self._inflight -= 1 + closing = self._closed and self._inflight == 0 + if closing: + self._delegate = None + if closing: + self._shutdown_delegate(delegate) + + @staticmethod + def _shutdown_delegate(delegate): + # BatchSpanProcessor can be exporting on its own worker even when no + # processor callback is leased. Joining that worker here can deadlock + # if the exporter waits for the thread calling Handle.shutdown(). + Thread(target=delegate.shutdown, name='traceai-cognee-close', daemon=False).start() + + def close(self): + with self._lock: + if self._closed: + return + self._closed = True + delegate = self._delegate if not self._inflight else None + if delegate is not None: + self._delegate = None + if delegate is not None: + self._shutdown_delegate(delegate) + + +class _OwnedProcessor(SpanProcessor): + """One reusable provider-owned slot, independent of handle generations.""" + def __init__(self): + self._lock = RLock() + self._handle = None + self._shutdown = False + # Each open span stays with the generation active when it started, so a + # span that ends after reconfiguration never reaches the new project. + self._open = OrderedDict() + + def current(self): + with self._lock: + return self._handle + + def install(self, handle): + with self._lock: + if self._shutdown: + raise RuntimeError('provider processor has shut down') + if self._handle is not None: + raise RuntimeError('provider already configured') + self._handle = handle + + def detach(self, handle): + generation = handle._generation + with self._lock: + if self._handle is handle: + self._handle = None + # The detached generation's open spans are never exported. Keep their + # identities marked until they end, so a later configuration cannot + # adopt one through a reused identity. + for key in [key for key, owner in self._open.items() if owner is generation]: + self._open[key] = _Ambiguous(1) + generation.close() + + def on_start(self, span, parent_context=None): + key = _span_key(span) + with self._lock: + handle = self._handle + if handle is None or key is None: + return + entry = self._open.get(key) + if isinstance(entry, _Ambiguous): + entry.open += 1 + return + if entry is not None: + self._open[key] = _Ambiguous(2) + return + generation = handle._generation + if len(self._open) >= _MAX_OPEN_SPANS: + self._open.popitem(last=False) + self._open[key] = generation + generation.call('on_start', span, parent_context) + + def on_end(self, span): + key = _span_key(span) + with self._lock: + entry = self._open.get(key) if key is not None else None + if isinstance(entry, _Ambiguous): + entry.open -= 1 + if entry.open <= 0: + del self._open[key] + return + if entry is not None: + del self._open[key] + if entry is not None: + entry.call('on_end', span) + + def force_flush(self, timeout_millis=30000): + handle = self.current() + return True if handle is None else handle.force_flush(timeout_millis) + + def shutdown(self): + with self._lock: + self._shutdown = True + handle, self._handle = self._handle, None + self._open = OrderedDict() + if handle is not None: + handle._generation.close() + + +class Handle: + def __init__(self, provider, processor, generation, signature): + self.provider, self._processor = provider, processor + self._generation, self._signature = generation, signature + + def force_flush(self, timeout_millis=30000): + return self._generation.call('force_flush', timeout_millis) + + def shutdown(self): + self._processor.detach(self) + + +def configure(*, project_name, capture_content=False, endpoint=None, headers=None, + exporter=None, provider=None): + """Attach one exporter; call before Cognee first creates native spans. + + An injected exporter is transferred to this handle. Shutdown affects only this + adapter's processor, not the provider or its other processors. Shutdown may + defer delegate closure until an already-running callback returns. + """ + if not isinstance(project_name, str) or not project_name: + raise ValueError('project_name must be a nonempty string') + if not isinstance(capture_content, bool): + raise TypeError('capture_content must be a bool') + if provider is not None and not isinstance(provider, TracerProvider): + raise TypeError('Cognee adapter requires an OpenTelemetry SDK TracerProvider') + if exporter is not None and (endpoint is not None or headers is not None): + raise ValueError('use exporter or endpoint/headers, not both') + if exporter is not None and not all(callable(getattr(exporter, name, None)) + for name in ('export', 'force_flush', 'shutdown')): + raise TypeError('exporter must implement export, force_flush and shutdown') + if endpoint is not None and (not isinstance(endpoint, str) or not endpoint): + raise ValueError('endpoint must be a nonempty string') + if headers is not None and (not isinstance(headers, Mapping) or + not all(isinstance(k, str) and isinstance(v, str) + for k, v in headers.items())): + raise TypeError('headers must map strings to strings') + signature = (project_name, capture_content, endpoint, + tuple(sorted((headers or {}).items())), id(exporter) if exporter is not None else None) + global _configuring + with _lock: + if _configuring: + raise RuntimeError('another Cognee configure is in progress') + _configuring = True + generation = None + try: + global_provider = trace.get_tracer_provider() + if provider is None: + provider = TracerProvider() if isinstance(global_provider, ProxyTracerProvider) else global_provider + elif not isinstance(global_provider, ProxyTracerProvider) and global_provider is not provider: + raise ValueError('provider must be the global provider used by Cognee') + if not isinstance(provider, TracerProvider): + raise TypeError('Cognee adapter requires an OpenTelemetry SDK TracerProvider') + if getattr(provider, '_default_processor', False): + raise RuntimeError('provider has a replaceable default processor; use an ordinary OTel provider') + with _lock: + slot = _slots.get(provider) + existing = slot.current() if slot else None + if existing is not None: + if existing._signature != signature: + raise ValueError('provider already configured with different options') + return existing + if exporter is None: + from fi_instrumentation.otel import HTTPSpanExporter + exporter = HTTPSpanExporter(endpoint=endpoint, headers=headers) + generation = _Generation(BatchSpanProcessor(CogneeExporter(exporter, project_name, capture_content))) + new_slot = slot is None + if new_slot: + slot = _OwnedProcessor() + handle = Handle(provider, slot, generation, signature) + slot.install(handle) + if new_slot: + with _lock: + _slots[provider] = slot + try: + provider.add_span_processor(slot) + except BaseException: + # A provider may attach the slot and then raise. Keep an attached + # slot registered so a retry reuses it instead of adding another. + if not _attached(provider, slot): + with _lock: + if _slots.get(provider) is slot: + del _slots[provider] + raise + if isinstance(global_provider, ProxyTracerProvider): + trace.set_tracer_provider(provider) + if trace.get_tracer_provider() is not provider: + raise RuntimeError('could not install global tracer provider') + generation = None + return handle + except BaseException: + if generation is not None: + if 'handle' in locals(): + slot.detach(handle) + else: + generation.close() + raise + finally: + with _lock: + _configuring = False + +__all__ = ['configure', 'Handle', '__version__'] diff --git a/python/frameworks/cognee/traceai_cognee/_exporter.py b/python/frameworks/cognee/traceai_cognee/_exporter.py new file mode 100644 index 000000000..042dabd88 --- /dev/null +++ b/python/frameworks/cognee/traceai_cognee/_exporter.py @@ -0,0 +1,98 @@ +"""Copy and project native spans at the exporter boundary.""" +import json +import math +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import ReadableSpan, Event +from opentelemetry.sdk.trace.export import SpanExporter +from opentelemetry.trace import Status, Link +from ._structured import redact_structured + +# Explicit keys: unknown native attributes may contain user content or credentials. +_SAFE = frozenset({'gen_ai.operation.name', 'gen_ai.system', 'gen_ai.request.model', + 'gen_ai.response.model', 'gen_ai.usage.input_tokens', + 'gen_ai.usage.output_tokens', 'gen_ai.usage.total_tokens', + 'llm.token_count.prompt', 'llm.token_count.completion', + 'llm.token_count.total', 'cognee.search.type'}) +_CONTENT = ('langfuse.observation.input', 'langfuse.observation.output', + 'memory.query.text', 'cognee.search.query', 'cognee.db.query') +_KIND_KEYS = ('fi.span.kind', 'gen_ai.span.kind', 'llm.request.type', 'openinference.span.kind') + +def _wire_number(value): + if type(value) is bool: + return True + if type(value) is int: + return -(2**63) <= value < 2**63 + return type(value) is float and math.isfinite(value) + + +def _safe_value(value): + return _wire_number(value) or (isinstance(value, str) and len(value) <= 256 and all(c.isalnum() or c in '._-/' for c in value)) + +def _kind(name): + if name == 'memory.retrieve' or name.startswith('cognee.search'): + return 'RETRIEVER' + if name == 'cognee.observe.acreate_structured_output': + return 'LLM' + return None + +def _content(value): + if not isinstance(value, str) and not _wire_number(value): + return None + return redact_structured(value) + +def project(span, project_name, capture_content): + attrs = dict(span.attributes or {}) + out = {k: v for k, v in attrs.items() if k in _SAFE and _safe_value(v)} + for key in _KIND_KEYS: + if key in attrs and _safe_value(attrs[key]): + out[key] = attrs[key] + if not any(k in out for k in _KIND_KEYS) and _kind(span.name): + out['fi.span.kind'] = _kind(span.name) + if capture_content: + for key in _CONTENT: + value = _content(attrs.get(key)) + if value is not None: + out[key] = value + for dest, sources in (('input', ('langfuse.observation.input', 'memory.query.text', 'cognee.search.query')), + ('output', ('langfuse.observation.output',))): + for key in sources: + if key in out: + value = out[key] + out[dest + '.value'] = value + try: + mime = 'application/json' if isinstance(json.loads(value), (dict, list)) else 'text/plain' + except (TypeError, ValueError): + mime = 'text/plain' + out[dest + '.mime_type'] = mime + break + # Even content opt-in never exports error text, stacktraces or arbitrary events. + events = [Event('exception', {}, e.timestamp) for e in span.events if e.name == 'exception'] + links = [Link(link.context, {}) for link in span.links] + resource = Resource({'project_name': project_name, 'project_type': 'observe'}) + return ReadableSpan(name=span.name, context=span.context, parent=span.parent, + resource=resource, attributes=out, events=events, links=links, + kind=span.kind, status=Status(span.status.status_code), + start_time=span.start_time, end_time=span.end_time, + instrumentation_scope=None) + +class CogneeExporter(SpanExporter): + def __init__(self, exporter, project_name, capture_content=False): + self.exporter = exporter + self.project_name = project_name + self.capture_content = capture_content + + def export(self, spans): + # Fail closed: a malformed span is dropped rather than exporting native data. + projected = [] + for span in spans: + try: + projected.append(project(span, self.project_name, self.capture_content)) + except Exception: + continue + return self.exporter.export(projected) + + def force_flush(self, timeout_millis=30000): + return self.exporter.force_flush(timeout_millis) + + def shutdown(self): + self.exporter.shutdown() diff --git a/python/frameworks/cognee/traceai_cognee/_structured.py b/python/frameworks/cognee/traceai_cognee/_structured.py new file mode 100644 index 000000000..da4e356a9 --- /dev/null +++ b/python/frameworks/cognee/traceai_cognee/_structured.py @@ -0,0 +1,103 @@ +"""Bounded structural redaction for native span attributes. + +This helper is not an exporter or a complete privacy boundary. It +recognises credential *field names* in structured data, not secrets in free +text or arbitrary encodings. Inputs are never mutated. +""" +from __future__ import annotations + +import json +import re +from typing import Any + +REDACTED = "[REDACTED]" +_MAX_DEPTH = 24 +_MAX_NODES = 10000 +_MAX_TEXT = 1_000_000 +_MAX_KEY = 512 +_CAMEL = re.compile(r"(?<=[a-z0-9])(?=[A-Z])") +_PARTS = re.compile(r"[^a-z0-9]+") +_CREDENTIAL_NAMES = frozenset({ + "apikey", "xapikey", "authorization", "proxyauthorization", + "accesstoken", "refreshtoken", "authtoken", "bearertoken", "idtoken", + "token", "clientsecret", "secretkey", "clientkey", "password", "passwd", + "credential", "credentials", "privatekey", + "accesskeyid", "secretaccesskey", "sessiontoken", "securitytoken", +}) + + +def _credential_field(key: str) -> bool: + # Treat dots and brackets as path boundaries, not as separators inside a + # single name. A credential-bearing path can continue after the credential + # segment (e.g. headers.authorization.value). Matching complete segments + # avoids hiding useful metrics such as input_tokens or token_count. + for segment in re.split(r"[.\[\]]+", key): + parts = [p for p in _PARTS.split(_CAMEL.sub(" ", segment).lower()) if p] + if "".join(parts) in _CREDENTIAL_NAMES: + return True + # A flattened underscore form such as client_config_api_key has no + # path delimiter. Do not match bare "token" as a suffix: e.g. + # input_token is a legitimate usage metric. + if any("".join(parts[i:]) in (_CREDENTIAL_NAMES - {"token"}) + for i in range(1, len(parts))): + return True + return False + + +class _Limit(Exception): + pass + + +def redact_structured(value: Any) -> Any: + """Return an independent sanitized JSON-like value or REDACTED on limits. + + JSON-serialized objects and arrays nested inside other JSON values are also + inspected. Ordinary strings remain unchanged; callers must separately apply + the relevant content and error-detail policy before exporting them. + """ + remaining = [_MAX_NODES] + active: set[int] = set() + + def visit(obj: Any, depth: int) -> Any: + remaining[0] -= 1 + if depth > _MAX_DEPTH or remaining[0] < 0: + raise _Limit + if isinstance(obj, str): + if len(obj) > _MAX_TEXT: + raise _Limit + stripped = obj.strip() + if stripped.startswith(("{", "[")): + try: + parsed = json.loads(obj) + except (ValueError, RecursionError): + # An invalid or truncated structured-looking value must + # not fall back to publishing its potentially secret text. + return REDACTED + return json.dumps(visit(parsed, depth + 1), ensure_ascii=False, separators=(",", ":")) + return obj + if obj is None or isinstance(obj, (bool, int, float)): + return obj + if isinstance(obj, (dict, list, tuple)): + ident = id(obj) + if ident in active: + raise _Limit + active.add(ident) + try: + if isinstance(obj, dict): + result = {} + for key, item in obj.items(): + remaining[0] -= 1 + if remaining[0] < 0 or not isinstance(key, str) or len(key) > _MAX_KEY: + raise _Limit + result[key] = REDACTED if _credential_field(key) else visit(item, depth + 1) + return result + return [visit(item, depth + 1) for item in obj] + finally: + active.remove(ident) + # Do not call repr/str on runtime objects, which may contain credentials. + raise _Limit + + try: + return visit(value, 0) + except (ValueError, TypeError, RecursionError, _Limit): + return REDACTED From 78a5f42fd554d0fe727aeac3ea8194d8e56d3d83 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Fri, 30 Oct 2026 22:19:14 -0700 Subject: [PATCH 2/3] fix(cognee): drain native spans safely at interpreter exit Keep ordinary shutdown nonblocking while synchronously draining the owned processor at exit. Check actual span delivery and exporter closure in subprocess regressions, including late atexit shutdown and shared providers. Co-Authored-By: Claude Code --- python/frameworks/cognee/CHANGELOG.md | 3 + .../cognee/tests/test_interpreter_exit.py | 100 ++++++++++++++++++ .../cognee/traceai_cognee/__init__.py | 39 +++++-- 3 files changed, 135 insertions(+), 7 deletions(-) create mode 100644 python/frameworks/cognee/tests/test_interpreter_exit.py diff --git a/python/frameworks/cognee/CHANGELOG.md b/python/frameworks/cognee/CHANGELOG.md index c331b1094..5ec6c10b4 100644 --- a/python/frameworks/cognee/CHANGELOG.md +++ b/python/frameworks/cognee/CHANGELOG.md @@ -6,3 +6,6 @@ - Exclude content, error descriptions and stack traces by default; provide content opt-in with bounded structural credential-field redaction. - Reuse a provider-owned processor across configurations, keep other exporters unchanged, and isolate spans across handle shutdown and reconfiguration. - Require `fi-instrumentation-otel>=1.2.2,<2` and Python 3.10–3.13. + +### Fixed +- Drain ended spans and close the owned exporter during interpreter exit, while keeping normal handle shutdown nonblocking and other exporters unchanged. diff --git a/python/frameworks/cognee/tests/test_interpreter_exit.py b/python/frameworks/cognee/tests/test_interpreter_exit.py new file mode 100644 index 000000000..873c72aee --- /dev/null +++ b/python/frameworks/cognee/tests/test_interpreter_exit.py @@ -0,0 +1,100 @@ +"""Check delivery and exporter closure, not just a successful process exit.""" +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + + +PROGRAM = r''' +import atexit +import json +from pathlib import Path +import sys +import time +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor, SpanExporter, SpanExportResult + +mode, filename = sys.argv[1:] +output = Path(filename) + +class Sink(SpanExporter): + def __init__(self, label): + self.label = label + + def record(self, operation, names=None): + with output.open('a') as stream: + stream.write(json.dumps([self.label, operation, names]) + '\n') + + def export(self, spans): + # Simulate buffered I/O that cannot finish before atexit returns unless + # shutdown actually waits. No real service or credentials are needed. + time.sleep(0.2 if self.label != 'foreign' else 0) + self.record('export', [span.name for span in spans]) + return SpanExportResult.SUCCESS + + def shutdown(self): + self.record('shutdown') + +if mode == 'baseline': + provider = TracerProvider() + provider.add_span_processor(BatchSpanProcessor(Sink('baseline'))) +else: + from traceai_cognee import configure + options = {} + if mode == 'existing-provider': + options['provider'] = TracerProvider() + handle = configure(project_name='synthetic', exporter=Sink('adapter'), **options) + provider = handle.provider + if mode in ('late-shutdown', 'closed-generation'): + # Registered later, this callback closes the generation after Python's + # normal thread-join phase but before the generation's own exit hook. + atexit.register(handle.shutdown) + +provider.add_span_processor(BatchSpanProcessor(Sink('foreign'))) +with provider.get_tracer('exit-regression').start_as_current_span('finished-before-exit'): + pass +if mode == 'closed-generation': + handle.shutdown() +# No force_flush or explicit provider.shutdown: use normal interpreter teardown. +''' + + +class InterpreterExitTests(unittest.TestCase): + def check_delivery(self, mode): + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / 'delivery.jsonl' + result = subprocess.run( + [sys.executable, '-I', '-c', PROGRAM, mode, str(output)], + cwd=directory, capture_output=True, text=True, timeout=20, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stderr, '') + events = [json.loads(line) for line in output.read_text().splitlines()] if output.exists() else [] + label = 'baseline' if mode == 'baseline' else 'adapter' + for exporter in (label, 'foreign'): + self.assertEqual( + [event[1:] for event in events if event[0] == exporter], + [['export', ['finished-before-exit']], ['shutdown', None]], + (mode, events), + ) + + def test_sdk_baseline_delivers_and_closes(self): + self.check_delivery('baseline') + + def test_default_provider_delivers_and_closes(self): + self.check_delivery('default-provider') + + def test_existing_provider_delivers_and_closes(self): + self.check_delivery('existing-provider') + + def test_shutdown_from_later_exit_callback_delivers_and_closes(self): + self.check_delivery('late-shutdown') + + def test_closed_generation_is_not_closed_twice_at_exit(self): + self.check_delivery('closed-generation') + + +if __name__ == '__main__': + unittest.main() diff --git a/python/frameworks/cognee/traceai_cognee/__init__.py b/python/frameworks/cognee/traceai_cognee/__init__.py index 4cd68278f..7b91957fd 100644 --- a/python/frameworks/cognee/traceai_cognee/__init__.py +++ b/python/frameworks/cognee/traceai_cognee/__init__.py @@ -1,7 +1,8 @@ """Export Cognee's native OpenTelemetry spans to Future AGI.""" +import atexit from collections import OrderedDict from collections.abc import Mapping -from threading import RLock, Thread +from threading import Event, RLock, Thread from weakref import WeakKeyDictionary from opentelemetry import trace from opentelemetry.sdk.trace import TracerProvider, SpanProcessor @@ -58,6 +59,11 @@ def __init__(self, delegate): self._delegate = delegate self._inflight = 0 self._closed = False + self._exiting = False + self._shutdown_complete = Event() + # Registered after the SDK provider's hook: drain this generation before + # provider teardown, when Python's normal thread-join phase is already over. + atexit.register(self._shutdown_at_exit) def call(self, method, *args): with self._lock: @@ -76,12 +82,31 @@ def call(self, method, *args): if closing: self._shutdown_delegate(delegate) - @staticmethod - def _shutdown_delegate(delegate): - # BatchSpanProcessor can be exporting on its own worker even when no - # processor callback is leased. Joining that worker here can deadlock - # if the exporter waits for the thread calling Handle.shutdown(). - Thread(target=delegate.shutdown, name='traceai-cognee-close', daemon=False).start() + def _shutdown_delegate(self, delegate): + with self._lock: + if not self._exiting: + # BatchSpanProcessor can be exporting on its own worker even when + # no callback is leased. Normal shutdown must not join that worker: + # its exporter may be waiting for the caller of Handle.shutdown(). + Thread(target=self._finish_shutdown, args=(delegate,), + name='traceai-cognee-close', daemon=False).start() + return + self._finish_shutdown(delegate) + + def _finish_shutdown(self, delegate): + try: + delegate.shutdown() + finally: + atexit.unregister(self._shutdown_at_exit) + self._shutdown_complete.set() + + def _shutdown_at_exit(self): + with self._lock: + self._exiting = True + self.close() + # An earlier exit callback may already have initiated asynchronous close. + # Waiting here also covers an outstanding leased callback finishing it. + self._shutdown_complete.wait() def close(self): with self._lock: From daf8f2fd02847b03ae610b4760b42eb76a4fddd2 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Sat, 31 Oct 2026 01:14:28 -0700 Subject: [PATCH 3/3] fix(cognee): prepare first PyPI upload with published core Register the initial 0.1.0 package and runtime version with release-please, route its first PyPI upload through the bootstrap token, and remove the hand-written changelog so automated release PRs own future entries. Validate fi-instrumentation-otel 1.2.2 from PyPI on Cognee 1.6.5 and 1.6.6, plus clean wheel and sdist installs on Python 3.10 and 3.13. Co-Authored-By: Claude Opus 5.5 --- .github/release/config.json | 1 + .release-please-manifest.json | 1 + python/frameworks/cognee/CHANGELOG.md | 11 ----------- python/frameworks/cognee/traceai_cognee/__init__.py | 2 +- release-please-config.json | 8 ++++++++ 5 files changed, 11 insertions(+), 12 deletions(-) delete mode 100644 python/frameworks/cognee/CHANGELOG.md diff --git a/.github/release/config.json b/.github/release/config.json index a3204edea..566951b5a 100644 --- a/.github/release/config.json +++ b/.github/release/config.json @@ -7,6 +7,7 @@ "_pypi_bootstrap_comment": "PyPI allows only three pending trusted publishers per organization, so these first-time projects are created by their first upload with the PYPI_BOOTSTRAP_TOKEN secret instead. Only a package in this list that has never been published uses the token. After the first release: add each project to the futureagi organization, add its trusted publisher (release.yml, blank environment), delete the token on PyPI and the secret, and empty this list.", "pypi_bootstrap_token": [ "traceAI-ag2", + "traceai-cognee", "traceAI-discoveryengine", "traceAI-elevenlabs-agents", "traceAI-exa", diff --git a/.release-please-manifest.json b/.release-please-manifest.json index abd32c28c..dc45f421c 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -12,6 +12,7 @@ "python/frameworks/chromadb": "0.1.1", "python/frameworks/claude-agent-sdk": "0.1.1", "python/frameworks/cohere": "0.1.1", + "python/frameworks/cognee": "0.1.0", "python/frameworks/crewai": "0.1.9", "python/frameworks/deepseek": "0.1.1", "python/frameworks/discoveryengine": "0.1.0", diff --git a/python/frameworks/cognee/CHANGELOG.md b/python/frameworks/cognee/CHANGELOG.md deleted file mode 100644 index 5ec6c10b4..000000000 --- a/python/frameworks/cognee/CHANGELOG.md +++ /dev/null @@ -1,11 +0,0 @@ -## [0.1.0] - -### Added -- Export Cognee's native OpenTelemetry spans to Future AGI without duplicate instrumentation. -- Map native search and structured-output model calls to Future AGI span types while preserving IDs, parents and error statuses. -- Exclude content, error descriptions and stack traces by default; provide content opt-in with bounded structural credential-field redaction. -- Reuse a provider-owned processor across configurations, keep other exporters unchanged, and isolate spans across handle shutdown and reconfiguration. -- Require `fi-instrumentation-otel>=1.2.2,<2` and Python 3.10–3.13. - -### Fixed -- Drain ended spans and close the owned exporter during interpreter exit, while keeping normal handle shutdown nonblocking and other exporters unchanged. diff --git a/python/frameworks/cognee/traceai_cognee/__init__.py b/python/frameworks/cognee/traceai_cognee/__init__.py index 7b91957fd..cbe2a3f5d 100644 --- a/python/frameworks/cognee/traceai_cognee/__init__.py +++ b/python/frameworks/cognee/traceai_cognee/__init__.py @@ -10,7 +10,7 @@ from opentelemetry.trace import ProxyTracerProvider from ._exporter import CogneeExporter -__version__ = '0.1.0' +__version__ = '0.1.0' # x-release-please-version _lock = RLock() _slots = WeakKeyDictionary() _configuring = False diff --git a/release-please-config.json b/release-please-config.json index a4cc5240c..aceb71c1c 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -114,6 +114,14 @@ "traceai_cohere/version.py" ] }, + "python/frameworks/cognee": { + "release-type": "python", + "component": "traceai-cognee", + "package-name": "traceai-cognee", + "extra-files": [ + "traceai_cognee/__init__.py" + ] + }, "python/frameworks/crewai": { "release-type": "python", "component": "traceai-crewai",