From d6d64e41217fd33717c23bb7487d3e5c45cd8168 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Fri, 30 Oct 2026 22:12:31 -0700 Subject: [PATCH 1/5] fix(crewai): redact structured credentials and contain native telemetry Project business data without serializing runtime configuration; enforce structured credential redaction across CrewAI attributes, context and events. Honor directional privacy flags and preserve application outcomes while omitting automatic exception details. Disable native CrewAI telemetry across legacy and maintained versions without disabling traceAI spans. Declare setuptools<81 for clean floor installs, require core 1.2.2, and add installed-artifact privacy, native parity and actual OTLP JSON/protobuf gzip wire tests. Co-Authored-By: Claude Code --- python/frameworks/crewai/CHANGELOG.md | 11 + python/frameworks/crewai/README.md | 29 ++ python/frameworks/crewai/poetry.lock | 40 +- python/frameworks/crewai/pyproject.toml | 6 +- python/frameworks/crewai/tests/conftest.py | 31 ++ .../crewai/tests/test_core_policy_delegate.py | 107 +++++ .../tests/test_credential_projection.py | 99 +++++ .../crewai/tests/test_native_parity.py | 167 ++++++++ .../crewai/tests/test_native_telemetry.py | 93 ++++ .../frameworks/crewai/tests/test_otlp_wire.py | 202 +++++++++ .../crewai/tests/test_privacy_policy.py | 319 ++++++++++++++ .../tests/test_structural_privacy_r3.py | 77 ++++ .../crewai/traceai_crewai/__init__.py | 9 +- .../crewai/traceai_crewai/_privacy.py | 178 ++++++++ .../crewai/traceai_crewai/_telemetry.py | 66 +++ .../crewai/traceai_crewai/_wrappers.py | 386 ++++++++--------- python/tests/test_framework_crewai.py | 397 +++++++++--------- 17 files changed, 1806 insertions(+), 411 deletions(-) create mode 100644 python/frameworks/crewai/tests/conftest.py create mode 100644 python/frameworks/crewai/tests/test_core_policy_delegate.py create mode 100644 python/frameworks/crewai/tests/test_credential_projection.py create mode 100644 python/frameworks/crewai/tests/test_native_parity.py create mode 100644 python/frameworks/crewai/tests/test_native_telemetry.py create mode 100644 python/frameworks/crewai/tests/test_otlp_wire.py create mode 100644 python/frameworks/crewai/tests/test_privacy_policy.py create mode 100644 python/frameworks/crewai/tests/test_structural_privacy_r3.py create mode 100644 python/frameworks/crewai/traceai_crewai/_privacy.py create mode 100644 python/frameworks/crewai/traceai_crewai/_telemetry.py diff --git a/python/frameworks/crewai/CHANGELOG.md b/python/frameworks/crewai/CHANGELOG.md index 503789fd4..f6777a9b3 100644 --- a/python/frameworks/crewai/CHANGELOG.md +++ b/python/frameworks/crewai/CHANGELOG.md @@ -1,4 +1,15 @@ ## [0.1.9] - 2026-07-13 +### Security +- Redact credential-named fields in structured task, crew and tool trace data, + including JSON-serialized arguments, aliases, context and event attributes. + Project business content without serializing opaque runtime objects. +- Honor directional content hiding on CrewAI aliases and omit automatic + exception details while preserving application inputs, results and exceptions. +- Disable CrewAI's separate native telemetry while instrumented without disabling + the OTel SDK; support legacy versions that ignore native opt-out flags. +- Require fi-instrumentation-otel >=1.2.2 and declare the setuptools dependency + needed to import CrewAI 0.41.1 on a clean installation. + ### Fixed - Fix `crew.kickoff()` crashing with `TypeError: '_NotSpecified' object is not iterable` when a `Task` is created without an explicit `context`. CrewAI diff --git a/python/frameworks/crewai/README.md b/python/frameworks/crewai/README.md index 7667b43be..02501d349 100644 --- a/python/frameworks/crewai/README.md +++ b/python/frameworks/crewai/README.md @@ -26,8 +26,37 @@ import os os.environ["FI_API_KEY"] = FI_API_KEY os.environ["FI_SECRET_KEY"] = FI_SECRET_KEY + +# Set these before importing CrewAI to opt out of its separate native telemetry. +os.environ["CREWAI_DISABLE_TELEMETRY"] = "true" +os.environ["CREWAI_DISABLE_TRACKING"] = "true" ``` +### Privacy and credentials + +Task, crew and tool spans project business content rather than serializing agent, +LLM, client or tool configuration objects. Credential-named fields in structured +inputs and outputs, including complete JSON object/array strings, are redacted +by default. This includes nested `client.config.api_key`, API-key headers, +authorization, access/refresh tokens, secrets and passwords. Model/provider +identifiers, usage counts and unrelated business values remain visible. Tracing +does not change your inputs, tool results or raised exceptions. + +This is not arbitrary free-text or unknown-encoding secret detection. If prose +may contain secrets, set `FI_HIDE_INPUTS=true` and/or `FI_HIDE_OUTPUTS=true` +before instrumentation, or pass the corresponding `TraceConfig` settings. +Directional text/message hiding also covers CrewAI's content aliases. Automatic +wrapper failures record an ERROR status without copying exception messages or +stacktraces. For manually recorded error events, use `FI_HIDE_ERROR_DETAILS=true`. +This integration does not filter your application's own console output or logs. + +Call `instrument()` before constructing agents, tools or crews. While +instrumented, traceAI disables CrewAI's separate native telemetry, including the +legacy exporter in CrewAI 0.41.1, which ignores the environment flags above. +`uninstrument()` restores CrewAI's native behavior. This does not retract any +telemetry already queued before instrumentation. Do not set +`OTEL_SDK_DISABLED=true`: that also disables the traceAI spans you want to collect. + ## Quickstart ### Register Tracer Provider diff --git a/python/frameworks/crewai/poetry.lock b/python/frameworks/crewai/poetry.lock index 4d1eaa450..f4e648363 100644 --- a/python/frameworks/crewai/poetry.lock +++ b/python/frameworks/crewai/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.1.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.2.1 and should not be changed by hand. [[package]] name = "aiohappyeyeballs" @@ -866,16 +866,19 @@ tests = ["asttokens (>=2.1.0)", "coverage", "coverage-enable-subprocess", "ipyth [[package]] name = "fi-instrumentation-otel" -version = "1.1.0" +version = "1.2.2" description = "OpenTelemetry instrumentation" optional = false -python-versions = "<3.14,>3.9" +python-versions = ">3.9,<3.14" groups = ["main"] -files = [ - {file = "fi_instrumentation_otel-1.1.0-py3-none-any.whl", hash = "sha256:4fa2bf663e0ce09447aca92ae798cd7b0dee35c5c92c2e033f1d89d158a9e62c"}, - {file = "fi_instrumentation_otel-1.1.0.tar.gz", hash = "sha256:7ec324c3c9242aa87a88cf23a21c11919b8736c6451c5fb50917f592a853b059"}, -] +[[package.files]] +file = "fi_instrumentation_otel-1.2.2-py3-none-any.whl" +hash = "sha256:ba1003f5962303a0fb9f1587153d0c5bda3856018da00b36629aef96296604fc" + +[[package.files]] +file = "fi_instrumentation_otel-1.2.2.tar.gz" +hash = "sha256:13d16695511f91c6f16cfa52e4b3236193a734df85ce52b40d37ad52831962fd" [package.dependencies] jsonschema = ">=4.21.1,<5.0.0" opentelemetry-api = ">=1.29.0,<2.0.0" @@ -3999,6 +4002,27 @@ files = [ [package.dependencies] pyasn1 = ">=0.1.3" +[[package]] +name = "setuptools" +version = "80.10.2" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "setuptools-80.10.2-py3-none-any.whl", hash = "sha256:95b30ddfb717250edb492926c92b5221f7ef3fbcc2b07579bcd4a27da21d0173"}, + {file = "setuptools-80.10.2.tar.gz", hash = "sha256:8b0e9d10c784bf7d262c4e5ec5d4ec94127ce206e8738f29a437945fbc219b70"}, +] + +[package.extras] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\"", "ruff (>=0.8.0) ; sys_platform != \"cygwin\""] +core = ["importlib_metadata (>=6) ; python_version < \"3.10\"", "jaraco.functools (>=4)", "jaraco.text (>=3.7)", "more_itertools", "more_itertools (>=8.8)", "packaging (>=24.2)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1) ; python_version < \"3.11\"", "wheel (>=0.43.0)"] +cover = ["pytest-cov"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] +enabler = ["pytest-enabler (>=2.2)"] +test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21) ; python_version >= \"3.9\" and sys_platform != \"cygwin\"", "jaraco.envs (>=2.2)", "jaraco.path (>=3.7.2)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf ; sys_platform != \"cygwin\"", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] +type = ["importlib_metadata (>=7.0.2) ; python_version < \"3.10\"", "jaraco.develop (>=7.21) ; sys_platform != \"cygwin\"", "mypy (==1.14.*)", "pytest-mypy"] + [[package]] name = "shellingham" version = "1.5.4" @@ -4892,4 +4916,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = ">=3.10, <3.13" -content-hash = "7edf47743cfaec37f169b1e54bf5e9749fa128c99f157a0b90a0ea6928cdf1b8" +content-hash = "69b9e209bfe1b93924a5eaddeb64a1e73a5ccc1473083e22d1bf042372ed2ce3" diff --git a/python/frameworks/crewai/pyproject.toml b/python/frameworks/crewai/pyproject.toml index 7640ff726..4a4d4404f 100644 --- a/python/frameworks/crewai/pyproject.toml +++ b/python/frameworks/crewai/pyproject.toml @@ -11,8 +11,10 @@ packages = [ [tool.poetry.dependencies] python = ">=3.10, <3.13" -crewai = ">0.19.0" -fi-instrumentation-otel = ">=0.1.11" +crewai = ">=0.41.1" +fi-instrumentation-otel = ">=1.2.2" +# CrewAI 0.41.1 imports pkg_resources, removed in setuptools 81. +setuptools = ">=78.1.1,<81" [build-system] diff --git a/python/frameworks/crewai/tests/conftest.py b/python/frameworks/crewai/tests/conftest.py new file mode 100644 index 000000000..53272b99e --- /dev/null +++ b/python/frameworks/crewai/tests/conftest.py @@ -0,0 +1,31 @@ +"""Local synthetic tests only; never disable the OTel SDK's positive controls.""" + +import os + +import pytest + +# Must precede CrewAI imports, which can initialize a telemetry singleton. +os.environ["CREWAI_DISABLE_TELEMETRY"] = "true" +os.environ["CREWAI_DISABLE_TRACKING"] = "true" +os.environ["OTEL_SDK_DISABLED"] = "false" + + +@pytest.fixture(autouse=True) +def native_telemetry_off(request): + from crewai.telemetry import Telemetry + from traceai_crewai._telemetry import _NativeTelemetryGuard + + if request.node.name.startswith("test_guard_"): + yield + return + # Use the packaged adapter for baseline native calls too. Floor CrewAI + # ignores the environment flags; maintained versions use their own checks. + guard = _NativeTelemetryGuard() + guard.enable() + try: + telemetry = Telemetry() + assert not telemetry.ready + assert not getattr(telemetry, "provider", None) + yield + finally: + guard.disable() diff --git a/python/frameworks/crewai/tests/test_core_policy_delegate.py b/python/frameworks/crewai/tests/test_core_policy_delegate.py new file mode 100644 index 000000000..0c5a07d12 --- /dev/null +++ b/python/frameworks/crewai/tests/test_core_policy_delegate.py @@ -0,0 +1,107 @@ +"""The CrewAI projection preserves every explicit core hide setting.""" + +import json + +import pytest +from fi_instrumentation import TraceConfig +from opentelemetry import trace +from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans + +from test_privacy_policy import SECRET, tracer_for + + +@pytest.mark.parametrize("enabled", [False, True]) +@pytest.mark.parametrize( + "flag,key,value", + [ + ("hide_input_images", "gen_ai.input.images.0", "https://example.invalid/image"), + ( + "hide_llm_invocation_parameters", + "gen_ai.request.parameters", + json.dumps({"temperature": 0.5, "api_key": SECRET}), + ), + ("hide_embedding_vectors", "gen_ai.embeddings.vectors", [0.1, 0.2]), + ( + "hide_embedding_vectors", + "embedding.embeddings.0.embedding.vector", + [0.1, 0.2], + ), + ], +) +def test_non_content_hide_flags_are_delegated_without_mutating( + flag, key, value, enabled +): + config = TraceConfig(**{flag: enabled}) + tracer, exporter, provider = tracer_for(config) + with tracer.start_as_current_span( + "policy", attributes={key: value, "client.config.api_key.value": SECRET} + ) as span: + span.add_event("policy.event", {key: value, "headers.Authorization.0": SECRET}) + spans = exporter.get_finished_spans() + assert SECRET.encode() not in encode_spans(spans).SerializeToString() + for attributes in (spans[0].attributes, spans[0].events[0].attributes): + if enabled: + assert key not in attributes + else: + assert key in attributes + assert getattr(config, flag) is enabled + provider.shutdown() + + +@pytest.mark.parametrize("enabled", [False, True]) +def test_pii_setting_is_preserved_and_credentials_are_always_redacted(enabled): + tracer, exporter, provider = tracer_for(TraceConfig(pii_redaction=enabled)) + with tracer.start_as_current_span("policy") as span: + span.add_event( + "log.record", + {"log.body": json.dumps({"api_key": SECRET, "email": "test@example.com"})}, + ) + wire = encode_spans(exporter.get_finished_spans()).SerializeToString() + assert SECRET.encode() not in wire + assert (b"test@example.com" in wire) is not enabled + provider.shutdown() + + +@pytest.mark.parametrize("enabled", [False, True]) +def test_manual_error_event_attributes_stay_structurally_redacted(enabled, caplog): + tracer, exporter, provider = tracer_for(TraceConfig(hide_error_details=enabled)) + with tracer.start_as_current_span("manual-error") as span: + # Free-text error details are controlled separately; structured exception + # attributes must be safe even when error-detail hiding is explicitly off. + span.record_exception( + RuntimeError("ordinary failure"), + attributes={"client.config.api_key.value": SECRET, "safe": "useful error"}, + ) + span.add_event( + "exception", + { + "exception.type": "RuntimeError", + "exception.message": "ordinary failure", + "log.body": json.dumps({"authorization": SECRET, "safe": "useful log"}), + }, + ) + span.set_status(trace.StatusCode.ERROR, "ordinary failure") + spans = exporter.get_finished_spans() + wire = encode_spans(spans).SerializeToString() + assert SECRET.encode() not in wire and SECRET not in caplog.text + assert b"useful error" in wire and b"useful log" in wire + assert (b"ordinary failure" in wire) is not enabled + assert (spans[0].status.description is None) is enabled + provider.shutdown() + + +@pytest.mark.parametrize( + "flag", + [ + "hide_input_images", + "hide_embedding_vectors", + "hide_llm_invocation_parameters", + "pii_redaction", + ], +) +def test_non_content_environment_flags_and_explicit_false_precedence(flag, monkeypatch): + from traceai_crewai._privacy import _CrewAITraceConfig + + monkeypatch.setenv("FI_" + flag.upper(), "true") + assert getattr(_CrewAITraceConfig(TraceConfig()), flag) is True + assert getattr(_CrewAITraceConfig(TraceConfig(**{flag: False})), flag) is False diff --git a/python/frameworks/crewai/tests/test_credential_projection.py b/python/frameworks/crewai/tests/test_credential_projection.py new file mode 100644 index 000000000..28bea0e36 --- /dev/null +++ b/python/frameworks/crewai/tests/test_credential_projection.py @@ -0,0 +1,99 @@ +"""Security regressions use synthetic values only and never contact a provider.""" + +import json +from unittest.mock import MagicMock + +from traceai_crewai._wrappers import ( + _safe_data, + _task_input, + _get_input_value, + _ExecuteCoreWrapper, +) + +SENTINEL = "FAKE_CREDENTIAL_ONLY_6c3f" + + +class ExplosiveRuntime: + def __repr__(self): + raise AssertionError("runtime repr evaluated") + + def to_dict(self): + raise AssertionError("runtime to_dict evaluated") + + def dict(self): + raise AssertionError("runtime dict evaluated") + + def model_dump(self): + raise AssertionError("runtime model_dump evaluated") + + +def test_opaque_objects_are_never_evaluated_even_when_nested(): + assert _safe_data({"value": [ExplosiveRuntime(), "user text"]}) == { + "value": ["[omitted runtime object]", "user text"] + } + + +def test_task_projection_supports_positional_and_keywords_without_agent_dump(): + def execute(agent=None, context=None, tools=None): + pass + + task = MagicMock(description="task text", expected_output="expected text") + positional = json.loads( + _task_input( + execute, + task, + (ExplosiveRuntime(), "context text", [ExplosiveRuntime()]), + {}, + ) + ) + keywords = json.loads( + _task_input( + execute, + task, + (), + { + "agent": ExplosiveRuntime(), + "context": "context text", + "tools": [ExplosiveRuntime()], + }, + ) + ) + assert ( + positional + == keywords + == { + "description": "task text", + "expected_output": "expected text", + "context": "context text", + } + ) + + +def test_generic_input_never_serializes_runtime_objects(): + def execute(arg, **kwargs): + pass + + payload = _get_input_value(execute, ExplosiveRuntime(), extra=ExplosiveRuntime()) + assert SENTINEL not in payload + assert "omitted runtime object" in payload + + +def test_exception_does_not_enter_span_and_same_error_propagates(): + tracer = MagicMock() + span = tracer.start_as_current_span.return_value.__enter__.return_value + task = MagicMock( + description="task", expected_output="result", key="task-key", id="task-id" + ) + error = RuntimeError(SENTINEL) + + def execute(agent=None, context=None, tools=None): + raise error + + try: + _ExecuteCoreWrapper(tracer)(execute, task, (), {"context": "context"}) + except RuntimeError as actual: + assert actual is error + else: + raise AssertionError("application error was swallowed") + assert not span.record_exception.called + assert SENTINEL not in str(span.mock_calls) diff --git a/python/frameworks/crewai/tests/test_native_parity.py b/python/frameworks/crewai/tests/test_native_parity.py new file mode 100644 index 000000000..a4fedb708 --- /dev/null +++ b/python/frameworks/crewai/tests/test_native_parity.py @@ -0,0 +1,167 @@ +"""Real CrewAI Agent/Task/Crew and ToolUsage, using a local fake LLM/tool.""" + +import json +from importlib.metadata import version +from types import SimpleNamespace +from unittest.mock import patch + +import pytest +from crewai import Agent, Crew, Task +from crewai.tools.tool_calling import ToolCalling +from crewai.tools.tool_usage import ToolUsage +from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +from fi_instrumentation import TraceConfig +from traceai_crewai import CrewAIInstrumentor + +SECRET = "FAKE_NATIVE_CREDENTIAL_a7130d" +FLOOR = version("crewai") == "0.41.1" + + +def run_crew(): + response = json.dumps({"api_key": SECRET, "answer": "useful answer"}) + if FLOOR: + from langchain_core.language_models.fake import FakeListLLM + + llm = FakeListLLM(responses=["Final Answer: " + response]) + agent = Agent( + role="Synthetic worker", + goal="Answer", + backstory="Local test", + llm=llm, + allow_delegation=False, + ) + task = Task( + description="Return useful answer", + expected_output="JSON answer", + agent=agent, + ) + crew = Crew(agents=[agent], tasks=[task], share_crew=True) + result = crew.kickoff( + inputs={"client": {"config": {"api_key": SECRET}}, "tag": "useful input"} + ) + else: + from crewai import LLM + + llm = LLM( + model="openai/synthetic", api_key=SECRET, base_url="http://127.0.0.1:1/v1" + ) + agent = Agent( + role="Synthetic worker", + goal="Answer", + backstory="Local test", + llm=llm, + allow_delegation=False, + ) + task = Task( + description="Return useful answer", + expected_output="JSON answer", + agent=agent, + ) + crew = Crew(agents=[agent], tasks=[task], share_crew=True) + with patch.object(type(llm), "call", return_value=response) as call: + result = crew.kickoff( + inputs={ + "client": {"config": {"api_key": SECRET}}, + "tag": "useful input", + } + ) + assert call.call_count > 0 + assert json.loads(result.raw) == json.loads(response) + return result.raw + + +def run_tool(): + seen = [] + + def lookup(query: str, client: dict) -> str: + """Return a local synthetic result.""" + seen.append((query, client)) + return json.dumps( + {"api_key": SECRET, "answer": "useful tool answer", "total_tokens": 7} + ) + + if FLOOR: + from crewai.agents.tools_handler import ToolsHandler + from langchain_core.tools import StructuredTool + + tool = StructuredTool.from_function(lookup) + usage = ToolUsage( + tools_handler=ToolsHandler(), + tools=[tool], + original_tools=[], + tools_description="lookup", + tools_names="lookup", + task=SimpleNamespace(used_tools=0), + function_calling_llm=None, + agent=SimpleNamespace(tools_results=[]), + action=None, + ) + usage._printer.print = lambda **kwargs: None + else: + from crewai.tools.structured_tool import CrewStructuredTool + + tool = CrewStructuredTool.from_function(lookup) + usage = ToolUsage( + tools_handler=None, tools=[tool], task=None, function_calling_llm=None + ) + arguments = {"query": "useful query", "client": {"config": {"api_key": SECRET}}} + calling = ToolCalling(tool_name="lookup", arguments=arguments) + result = usage.use(calling, "lookup") + assert seen == [("useful query", {"config": {"api_key": SECRET}})] + assert ( + calling.arguments == arguments + and calling.arguments["client"]["config"]["api_key"] == SECRET + ) + assert json.loads(result) == { + "api_key": SECRET, + "answer": "useful tool answer", + "total_tokens": 7, + } + return result + + +@pytest.mark.parametrize( + "settings", + [ + {}, + {"hide_inputs": True}, + {"hide_outputs": True}, + {"hide_input_text": True, "hide_output_messages": True}, + ], +) +def test_native_instrumented_and_uninstrumented_outcomes_match(settings, caplog): + baseline_crew, baseline_tool = run_crew(), run_tool() + exporter = InMemorySpanExporter() + provider = TracerProvider(shutdown_on_exit=False) + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = CrewAIInstrumentor() + instrumentor.instrument(tracer_provider=provider, config=TraceConfig(**settings)) + try: + assert run_crew() == baseline_crew + assert run_tool() == baseline_tool + finally: + instrumentor.uninstrument() + spans = exporter.get_finished_spans() + assert {span.name for span in spans} >= { + "Task._execute_core", + "Crew.kickoff", + "ToolUsage._use", + } + assert not any("telemetry" in span.name.lower() for span in spans) + wire = encode_spans(spans).SerializeToString() + assert SECRET.encode() not in wire and SECRET not in caplog.text + if not settings: + assert all( + text in wire + for text in ( + b"useful query", + b"useful tool answer", + b"useful input", + b"Return useful answer", + ) + ) + provider.shutdown() diff --git a/python/frameworks/crewai/tests/test_native_telemetry.py b/python/frameworks/crewai/tests/test_native_telemetry.py new file mode 100644 index 000000000..5740cac19 --- /dev/null +++ b/python/frameworks/crewai/tests/test_native_telemetry.py @@ -0,0 +1,93 @@ +"""Verify the package's opt-out, including floor versions without disable flags.""" + +from importlib import import_module +from inspect import getattr_static +from unittest.mock import patch + +from crewai.telemetry import Telemetry +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +from traceai_crewai import CrewAIInstrumentor +from traceai_crewai._telemetry import _NativeTelemetryGuard + + +def test_guard_native_exporter_never_constructed_and_traceai_still_records(monkeypatch): + # The import was safe under pre-import flags; now prove instrument() itself + # disables native telemetry even when those flags are explicitly false. + for key in ( + "CREWAI_DISABLE_TELEMETRY", + "CREWAI_DISABLE_TRACKING", + "OTEL_SDK_DISABLED", + ): + monkeypatch.setenv(key, "false") + if hasattr(Telemetry, "_instance"): + monkeypatch.setattr(Telemetry, "_instance", None) + original = getattr_static(Telemetry, "__init__") + check = getattr_static(Telemetry, "_is_telemetry_disabled", None) + exporter = InMemorySpanExporter() + provider = TracerProvider(shutdown_on_exit=False) + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = CrewAIInstrumentor() + # A constructor spy is stronger than an unreachable network endpoint: a + # native exporter must not even be instantiated in the opted-out branch. + module = "crewai.telemetry.telemetry" + target = module + ( + ".SafeOTLPSpanExporter" + if hasattr(import_module(module), "SafeOTLPSpanExporter") + else ".OTLPSpanExporter" + ) + with patch( + target, side_effect=AssertionError("native exporter constructed") + ) as native_exporter: + instrumentor.instrument(tracer_provider=provider) + try: + telemetry = Telemetry() + assert not telemetry.ready + assert not getattr(telemetry, "provider", None) + if check is not None: + assert Telemetry._is_telemetry_disabled() + with instrumentor._tracer.start_as_current_span("positive-control"): + pass + assert len(exporter.get_finished_spans()) == 1 + assert not native_exporter.called + finally: + instrumentor.uninstrument() + assert getattr_static(Telemetry, "__init__") is original + assert getattr_static(Telemetry, "_is_telemetry_disabled", None) is check + provider.shutdown() + + +def test_guard_floor_blocks_existing_instance_and_restores(monkeypatch): + calls = [] + + class LegacyTelemetry: + def __init__(self): + self.ready, self.trace_set = True, True + calls.append("constructor") + + def crew_creation(self, *args): + calls.append("export") + + legacy = LegacyTelemetry() + from types import SimpleNamespace + import traceai_crewai._telemetry as module + + def fake_import(name): + if name == "crewai.telemetry": + return SimpleNamespace(Telemetry=LegacyTelemetry) + raise ModuleNotFoundError(name="crewai_core") + + monkeypatch.setattr(module, "import_module", fake_import) + guard = _NativeTelemetryGuard() + guard.enable() + try: + assert not LegacyTelemetry().ready + assert legacy.crew_creation(None, None) is None + assert calls == ["constructor"] + finally: + guard.disable() + assert LegacyTelemetry().ready + legacy.crew_creation(None, None) + assert calls == ["constructor", "constructor", "export"] diff --git a/python/frameworks/crewai/tests/test_otlp_wire.py b/python/frameworks/crewai/tests/test_otlp_wire.py new file mode 100644 index 000000000..3fa5d0a64 --- /dev/null +++ b/python/frameworks/crewai/tests/test_otlp_wire.py @@ -0,0 +1,202 @@ +"""Actual owned-loopback HTTP captures, not a gzip round-trip of span.to_json().""" + +import base64 +import gzip +import json +import threading +from http.server import BaseHTTPRequestHandler, HTTPServer +from urllib.request import ProxyHandler, Request, build_opener + +import pytest +from crewai.tools.tool_calling import ToolCalling +from fi_instrumentation import FITracer, TraceConfig +from google.protobuf.json_format import MessageToDict +from opentelemetry import trace +from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans +from opentelemetry.exporter.otlp.proto.http import Compression +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ( + ExportTraceServiceRequest, +) +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import ( + SimpleSpanProcessor, + SpanExporter, + SpanExportResult, +) +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +from traceai_crewai._privacy import _CrewAITraceConfig +from traceai_crewai._wrappers import _ToolUseWrapper + +SECRET = "FAKE_WIRE_CREDENTIAL_98cab1" + + +def otlp_json(spans): + # OTLP/JSON follows proto JSON except trace/span IDs are hex, not base64. + body = MessageToDict(encode_spans(spans), use_integers_for_enums=True) + + def convert(value): + if isinstance(value, dict): + return { + key: ( + base64.b64decode(item).hex() + if key in ("traceId", "spanId", "parentSpanId") + else convert(item) + ) + for key, item in value.items() + } + if isinstance(value, list): + return [convert(item) for item in value] + return value + + return json.dumps(convert(body)).encode() + + +class JsonHTTPExporter(SpanExporter): + """Test-only OTLP/JSON exporter; production HTTP exporter is protobuf.""" + + def __init__(self, endpoint, compressed): + self.endpoint, self.compressed = endpoint, compressed + + def export(self, spans): + data = otlp_json(spans) + headers = {"Content-Type": "application/json"} + if self.compressed: + data = gzip.compress(data) + headers["Content-Encoding"] = "gzip" + with build_opener(ProxyHandler({})).open( + Request(self.endpoint, data, headers), timeout=5 + ) as response: + assert response.status == 200 + return SpanExportResult.SUCCESS + + def shutdown(self): + pass + + +@pytest.mark.parametrize("fails", [False, True]) +@pytest.mark.parametrize("encoding", ["protobuf", "json"]) +@pytest.mark.parametrize("compressed", [False, True]) +@pytest.mark.parametrize( + "settings", + [ + {}, + {"hide_inputs": True}, + {"hide_outputs": True}, + {"hide_input_text": True, "hide_output_messages": True}, + ], +) +def test_real_otlp_http_wire(encoding, compressed, settings, fails): + captured = [] + + class Collector(BaseHTTPRequestHandler): + def do_POST(self): + data = self.rfile.read(int(self.headers["Content-Length"])) + captured.append( + ({key.lower(): value for key, value in self.headers.items()}, data) + ) + self.send_response(200) + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, *args): + pass + + server = HTTPServer(("127.0.0.1", 0), Collector) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + provider = TracerProvider(shutdown_on_exit=False) + endpoint = f"http://127.0.0.1:{server.server_port}/v1/traces" + exporter = ( + OTLPSpanExporter( + endpoint=endpoint, + timeout=5, + compression=Compression.Gzip if compressed else Compression.NoCompression, + ) + if encoding == "protobuf" + else JsonHTTPExporter(endpoint, compressed) + ) + provider.add_span_processor(SimpleSpanProcessor(exporter)) + tracer = FITracer( + provider.get_tracer("synthetic-wire"), + _CrewAITraceConfig(TraceConfig(**settings)), + ) + arguments = {"client": {"config": {"api_key": SECRET}}, "query": "useful query"} + result = {"Authorization": SECRET, "answer": "useful answer", "total_tokens": 13} + error = RuntimeError(SECRET) + + def use(calling, tool_string): + trace.get_current_span().add_event( + "tool.log", + {"log.body": json.dumps({"secret": SECRET, "safe": "useful log"})}, + ) + if fails: + raise error + return result + + def invoke(): + return _ToolUseWrapper(tracer)( + use, + None, + (ToolCalling(tool_name="lookup", arguments=arguments), "lookup"), + {}, + ) + + try: + if fails: + with pytest.raises(RuntimeError) as caught: + invoke() + assert caught.value is error + else: + assert invoke() is result + assert result["Authorization"] == SECRET + assert arguments["client"]["config"]["api_key"] == SECRET + assert provider.force_flush() + assert len(captured) == 1 + headers, raw = captured[0] + assert headers["content-type"] == ( + "application/json" if encoding == "json" else "application/x-protobuf" + ) + assert (headers.get("content-encoding") == "gzip") is compressed + body = gzip.decompress(raw) if compressed else raw + assert SECRET.encode() not in body and SECRET.encode() not in raw + assert b"useful log" in body + if settings.get("hide_inputs") or settings.get("hide_input_text"): + assert b"useful query" not in body + if settings.get("hide_outputs") or settings.get("hide_output_messages"): + assert b"useful answer" not in body + if not settings: + assert b"useful query" in body + assert (b"useful answer" in body) is not fails + if encoding == "json": + decoded = json.loads(body) + span = decoded["resourceSpans"][0]["scopeSpans"][0]["spans"][0] + assert len(span["traceId"]) == 32 and len(span["spanId"]) == 16 + assert span["status"]["code"] == (2 if fails else 1) + else: + decoded = ExportTraceServiceRequest.FromString(body) + spans = decoded.resource_spans[0].scope_spans[0].spans + assert len(spans) == 1 + assert spans[0].status.code == (2 if fails else 1) + finally: + provider.shutdown() + server.shutdown() + server.server_close() + thread.join(timeout=5) + + +def test_raw_negative_control_detects_unprojected_synthetic_credential(): + # Do not send this deliberately unsafe baseline to even the local collector. + exporter = InMemorySpanExporter() + provider = TracerProvider(shutdown_on_exit=False) + provider.add_span_processor(SimpleSpanProcessor(exporter)) + with provider.get_tracer("raw-baseline").start_as_current_span( + "raw", attributes={"input.value": json.dumps({"api_key": SECRET})} + ): + pass + spans = exporter.get_finished_spans() + for data in (encode_spans(spans).SerializeToString(), otlp_json(spans)): + assert SECRET.encode() in data + assert SECRET.encode() in gzip.decompress(gzip.compress(data)) + provider.shutdown() diff --git a/python/frameworks/crewai/tests/test_privacy_policy.py b/python/frameworks/crewai/tests/test_privacy_policy.py new file mode 100644 index 000000000..a1fc95324 --- /dev/null +++ b/python/frameworks/crewai/tests/test_privacy_policy.py @@ -0,0 +1,319 @@ +"""Synthetic credentials, projected aliases, events, logs and error paths.""" + +import copy +import json +from types import SimpleNamespace + +import pytest +from fi_instrumentation import FITracer, TraceConfig +from opentelemetry import trace +from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +from traceai_crewai._privacy import _CrewAITraceConfig, _credential_key, _safe_data +from traceai_crewai._wrappers import ( + _ExecuteCoreWrapper, + _KickoffWrapper, + _ToolUseWrapper, +) + +SECRET = "FAKE_PRIVACY_CREDENTIAL_ef718a" +INPUT = "private input prose" +OUTPUT = "private output prose" +INPUT_ALIASES = ( + "input.value", + "raw.input", + "crew_inputs", + "crew_agents", + "crew_tasks", + "formatted_description", + "formatted_expected_output", + "gen_ai.input.messages.0.message.content", + "gen_ai.prompts", + "gen_ai.prompt", + "gen_ai.request.parameters", + "tool.parameters", + "gen_ai.tool.call.arguments", + "tool_call.function.arguments", + "message.function_call_arguments_json", +) +OUTPUT_ALIASES = ( + "output.value", + "raw.output", + "gen_ai.output.messages.0.message.content", + "gen_ai.completion", + "gen_ai.tool.call.result", +) +CONTENT_FLAGS = ( + "hide_inputs", + "hide_input_text", + "hide_input_messages", + "hide_outputs", + "hide_output_text", + "hide_output_messages", +) + + +def tracer_for(config): + exporter = InMemorySpanExporter() + provider = TracerProvider(shutdown_on_exit=False) + provider.add_span_processor(SimpleSpanProcessor(exporter)) + return ( + FITracer(provider.get_tracer("synthetic-crewai"), _CrewAITraceConfig(config)), + exporter, + provider, + ) + + +@pytest.mark.parametrize( + "key", + [ + "api_key", + "apiKey", + "API-KEY", + "X-API-Key", + "Authorization", + "Proxy-Authorization", + "access_token", + "refreshToken", + "auth_token", + "authentication_token", + "client_secret", + "clientSecretKey", + "secret_key", + "password", + "passwd", + "token", + "session_token", + "bearer_token", + "private_key", + "client.config.api_key", + "client.config.client_key", + "client.config.token", + "client.config.secret", + "headers.Authorization", + "aws_secret_access_key", + "aws_access_key_id", + "credentials", + "Cookie", + "Set-Cookie", + "headers.Authorization.0", + "headers[X-API-Key]", + "client.config.api_key.value", + "client.config.token.value", + "client.config.secret.value", + "client.config.secrets.value", + "apiSecret", + "secret_token", + ], +) +def test_credential_spellings_and_business_fields(key): + payload = { + key: SECRET, + "model": "safe-model", + "provider": "safe-provider", + "total_tokens": 42, + "prompt_tokens": 11, + "completion_tokens": 31, + "token_count": 42, + "tokenizer": "safe", + "max_tokens": 90, + "token_id": "safe-id", + "id": "business-id", + } + before = copy.deepcopy(payload) + projected = _safe_data(payload) + assert SECRET not in json.dumps(projected) + assert projected[key] == "__REDACTED__" + assert {k: v for k, v in projected.items() if k != key} == { + k: v for k, v in payload.items() if k != key + } + assert payload == before + + +@pytest.mark.parametrize("flag", CONTENT_FLAGS) +@pytest.mark.parametrize("enabled", [True, False]) +def test_every_directional_hide_flag_covers_aliases_and_events(flag, enabled, caplog): + tracer, exporter, provider = tracer_for(TraceConfig(**{flag: enabled})) + attrs = { + **{k: INPUT for k in INPUT_ALIASES}, + **{k: OUTPUT for k in OUTPUT_ALIASES}, + "headers.Authorization": SECRET, + "gen_ai.usage.total_tokens": 42, + "log.body": json.dumps({"api_key": SECRET, "message": "useful log"}), + } + before = copy.deepcopy(attrs) + with tracer.start_as_current_span("privacy", attributes=attrs) as span: + span.set_attributes(attrs) + trace.get_current_span().add_event("tool.result", attributes=attrs) + spans = exporter.get_finished_spans() + wire = encode_spans(spans).SerializeToString() + assert SECRET.encode() not in wire + assert b"useful log" in wire and b"tool.result" in wire + assert SECRET not in caplog.text + if enabled: + hidden, visible = (INPUT, OUTPUT) if "input" in flag else (OUTPUT, INPUT) + assert hidden.encode() not in wire and visible.encode() in wire + else: + assert INPUT.encode() in wire and OUTPUT.encode() in wire + assert attrs == before + assert spans[0].attributes["gen_ai.usage.total_tokens"] == 42 + provider.shutdown() + + +@pytest.mark.parametrize( + "kwargs", + [ + {}, + {"hide_inputs": False, "hide_outputs": False, "hide_error_details": False}, + {"hide_inputs": True, "hide_outputs": True}, + {"hide_input_messages": True, "hide_output_text": True}, + { + "hide_inputs": True, + "hide_output_messages": True, + "hide_error_details": False, + }, + {"hide_inputs": False, "hide_outputs": True, "hide_error_details": True}, + {"hide_input_images": True}, + {"hide_embedding_vectors": True}, + {"hide_llm_invocation_parameters": True}, + {"pii_redaction": True}, + ], +) +def test_default_and_explicit_combinations_always_redact_structured_aliases(kwargs): + tracer, exporter, provider = tracer_for(TraceConfig(**kwargs)) + structured = json.dumps( + {"client": {"config": {"api_key": SECRET}}, "safe": "business"} + ) + attrs = {k: structured for k in INPUT_ALIASES + OUTPUT_ALIASES} + with tracer.start_as_current_span("privacy", attributes=attrs) as span: + span.add_event("structured", attrs) + wire = encode_spans(exporter.get_finished_spans()).SerializeToString() + assert SECRET.encode() not in wire + assert ( + b"business" in wire + or (kwargs.get("hide_inputs") and kwargs.get("hide_outputs")) + or (kwargs.get("hide_input_messages") and kwargs.get("hide_output_text")) + or (kwargs.get("hide_inputs") and kwargs.get("hide_output_messages")) + ) + provider.shutdown() + + +@pytest.mark.parametrize("flag", CONTENT_FLAGS + ("hide_error_details",)) +def test_environment_flags_and_explicit_false_precedence(flag, monkeypatch): + monkeypatch.setenv("FI_" + flag.upper(), "true") + config = _CrewAITraceConfig(TraceConfig()) + assert getattr(config, flag) is True + explicit = _CrewAITraceConfig(TraceConfig(**{flag: False})) + assert getattr(explicit, flag) is False + assert explicit.mask("headers.Authorization", SECRET) == "__REDACTED__" + + +@pytest.mark.parametrize( + "wrapper_type", [_ExecuteCoreWrapper, _KickoffWrapper, _ToolUseWrapper] +) +@pytest.mark.parametrize( + "settings", + [ + {}, + {"hide_error_details": False}, + {"hide_error_details": True}, + {"hide_inputs": True, "hide_error_details": False}, + ], +) +def test_wrapper_errors_propagate_without_messages_events_or_logs( + wrapper_type, settings, caplog +): + tracer, exporter, provider = tracer_for(TraceConfig(**settings)) + instance = SimpleNamespace( + key="safe-key", + id="safe-id", + description="task", + expected_output="answer", + agents=[], + tasks=[], + usage_metrics=None, + ) + error = RuntimeError(SECRET) + + def fail(**kwargs): + raise error + + with pytest.raises(RuntimeError) as caught: + wrapper_type(tracer)(fail, instance, (), {}) + assert caught.value is error + spans = exporter.get_finished_spans() + assert len(spans) == 1 and spans[0].status.status_code is trace.StatusCode.ERROR + assert not spans[0].events and spans[0].status.description is None + assert SECRET.encode() not in encode_spans(spans).SerializeToString() + assert SECRET not in caplog.text + provider.shutdown() + + +def test_depth_cycles_and_opaque_objects_are_not_serialized(): + class Opaque: + def __getattribute__(self, name): + raise AssertionError("opaque object inspected") + + def __repr__(self): + raise AssertionError("opaque object stringified") + + cycle = {"safe": "business", "api_key": SECRET, "runtime": Opaque()} + cycle["cycle"] = cycle + assert SECRET not in json.dumps(_safe_data(cycle)) + assert _safe_data("free text " + SECRET) == "free text " + SECRET + assert not _credential_key("gen_ai.usage.total_tokens") + + +def test_context_is_redacted_before_sampler_and_at_final_setters(): + from fi_instrumentation import using_attributes + from opentelemetry.sdk.trace.sampling import Sampler, ALWAYS_ON + + observed = [] + + class InspectSampler(Sampler): + def should_sample(self, *args, **kwargs): + observed.append(dict(kwargs.get("attributes") or args[4])) + return ALWAYS_ON.should_sample(*args, **kwargs) + + def get_description(self): + return "synthetic-inspection" + + exporter = InMemorySpanExporter() + provider = TracerProvider(sampler=InspectSampler(), shutdown_on_exit=False) + provider.add_span_processor(SimpleSpanProcessor(exporter)) + tracer = FITracer(provider.get_tracer("context"), _CrewAITraceConfig(TraceConfig())) + instance = SimpleNamespace( + description="task", expected_output="answer", key="safe-key", id="safe-id" + ) + + def execute(agent=None, context=None): + return {"api_key": SECRET, "answer": "business output"} + + with using_attributes( + metadata={ + "client": {"config": {"api_key": SECRET}}, + "safe": "business metadata", + } + ): + _ExecuteCoreWrapper(tracer)(execute, instance, (), {}) + assert len(observed) == 1 + assert SECRET not in json.dumps(observed) + assert "business metadata" in json.dumps(observed) + wire = encode_spans(exporter.get_finished_spans()).SerializeToString() + assert ( + SECRET.encode() not in wire + and b"business metadata" in wire + and b"business output" in wire + ) + provider.shutdown() + + +def test_unparseable_or_oversized_structured_data_fails_closed(): + deeply_nested = "[" * 1100 + json.dumps({"api_key": SECRET}) + "]" * 1100 + malformed = '{"api_key": "' + SECRET + oversized = json.dumps({"api_key": SECRET, "text": "x" * 1_000_001}) + for value in (deeply_nested, malformed, oversized): + assert SECRET not in _safe_data(value) diff --git a/python/frameworks/crewai/tests/test_structural_privacy_r3.py b/python/frameworks/crewai/tests/test_structural_privacy_r3.py new file mode 100644 index 000000000..31406362f --- /dev/null +++ b/python/frameworks/crewai/tests/test_structural_privacy_r3.py @@ -0,0 +1,77 @@ +"""Synthetic-only structural privacy tests; no real credentials or network.""" + +import copy +import json + +from crewai.tools.tool_calling import ToolCalling +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans +from traceai_crewai._wrappers import _safe_data, _ToolUseWrapper + +SECRET = "FAKE_STRUCTURAL_CREDENTIAL_6dd845" + + +class Opaque: + def __repr__(self): + raise AssertionError("repr called") + + def __getattribute__(self, name): + raise AssertionError("runtime object inspected") + + +def test_structural_aliases_and_unchanged_input(): + item = { + "client": {"config": {"api_key": SECRET}}, + "apiKey": SECRET, + "headers": {"X-API-Key": SECRET, "Authorization": SECRET}, + "raw": json.dumps({"access_token": SECRET, "model": "safe-model"}), + "events": [{"client_secret": SECRET, "log": {"password": SECRET}}], + "client.config.api_key": SECRET, + "total_tokens": 42, + "id": "safe-id", + } + before = copy.deepcopy(item) + projected = _safe_data(item) + assert SECRET not in json.dumps(projected) + assert projected["total_tokens"] == 42 and projected["id"] == "safe-id" + assert json.loads(projected["raw"])["model"] == "safe-model" + assert item == before + assert _safe_data({"object": Opaque()}) == {"object": "[omitted runtime object]"} + assert ( + _safe_data("arbitrary free text " + SECRET) == "arbitrary free text " + SECRET + ) + + +def test_native_toolcalling_protobuf_and_original_response(): + arguments = { + "query": "useful query", + "client": {"config": {"api_key": SECRET}}, + "arguments": json.dumps({"refresh_token": SECRET, "safe": "ok"}), + "prompt_tokens": 12, + "opaque": Opaque(), + } + calling = ToolCalling(tool_name="search", arguments=arguments) + output = {"raw": {"apiKey": SECRET, "value": "useful output"}, "count": 7} + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + + def wrapped(calling, tool_string): + return output + + result = _ToolUseWrapper(provider.get_tracer("synthetic"))( + wrapped, None, (calling, "search"), {} + ) + spans = exporter.get_finished_spans() + assert len(spans) == 1 and result is output + assert calling.arguments["client"]["config"]["api_key"] == SECRET + assert output["raw"]["apiKey"] == SECRET + wire = encode_spans(spans).SerializeToString() + assert SECRET.encode() not in wire + assert ( + b"useful query" in wire + and b"useful output" in wire + and b"prompt_tokens" in wire + ) diff --git a/python/frameworks/crewai/traceai_crewai/__init__.py b/python/frameworks/crewai/traceai_crewai/__init__.py index 6872bcdd0..541b448c3 100644 --- a/python/frameworks/crewai/traceai_crewai/__init__.py +++ b/python/frameworks/crewai/traceai_crewai/__init__.py @@ -10,6 +10,8 @@ _KickoffWrapper, _ToolUseWrapper, ) +from traceai_crewai._privacy import _CrewAITraceConfig +from traceai_crewai._telemetry import _NativeTelemetryGuard from traceai_crewai.version import __version__ from wrapt import wrap_function_wrapper @@ -24,6 +26,7 @@ class CrewAIInstrumentor(BaseInstrumentor): # type: ignore "_original_kickoff", "_original_tool_use", "_tracer", + "_native_telemetry_guard", ) def instrumentation_dependencies(self) -> Collection[str]: @@ -38,9 +41,12 @@ def _instrument(self, **kwargs: Any) -> None: assert isinstance(config, TraceConfig) self._tracer = FITracer( trace_api.get_tracer(__name__, __version__, tracer_provider), - config=config, + config=_CrewAITraceConfig(config), ) + self._native_telemetry_guard = _NativeTelemetryGuard() + self._native_telemetry_guard.enable() + execute_core_wrapper = _ExecuteCoreWrapper(tracer=self._tracer) self._original_execute_core = getattr( import_module("crewai").Task, "_execute_core", None @@ -70,6 +76,7 @@ def _instrument(self, **kwargs: Any) -> None: ) def _uninstrument(self, **kwargs: Any) -> None: + self._native_telemetry_guard.disable() if self._original_execute_core is not None: task_module = import_module("crewai") task_module.Task._execute_core = self._original_execute_core diff --git a/python/frameworks/crewai/traceai_crewai/_privacy.py b/python/frameworks/crewai/traceai_crewai/_privacy.py new file mode 100644 index 000000000..0b5f5d077 --- /dev/null +++ b/python/frameworks/crewai/traceai_crewai/_privacy.py @@ -0,0 +1,178 @@ +"""CrewAI-specific structural projection; deliberately not free-text secret detection.""" + +import json +import re +from dataclasses import fields +from typing import Any + +from fi_instrumentation import TraceConfig + +REDACTED = "__REDACTED__" +_CREDENTIAL_KEYS = frozenset( + { + "apikey", + "xapikey", + "authorization", + "proxyauthorization", + "accesstoken", + "refreshtoken", + "authtoken", + "authenticationtoken", + "bearertoken", + "sessiontoken", + "idtoken", + "apitoken", + "csrftoken", + "clientsecret", + "clientsecretkey", + "clientkey", + "secretkey", + "secretaccesskey", + "accesskey", + "accesskeyid", + "secret", + "secrets", + "apisecret", + "secrettoken", + "privatekey", + "signingkey", + "password", + "passwd", + "token", + "credential", + "credentials", + "cookie", + "setcookie", + } +) +_CREDENTIAL_SUFFIXES = ( + "apikey", + "authorization", + "accesstoken", + "refreshtoken", + "authtoken", + "authenticationtoken", + "bearertoken", + "sessiontoken", + "idtoken", + "apitoken", + "csrftoken", + "clientsecret", + "clientsecretkey", + "clientkey", + "secretkey", + "secretaccesskey", + "accesskey", + "accesskeyid", + "privatekey", + "signingkey", + "password", + "passwd", +) +_INPUT_ALIASES = ( + "input.value", + "raw.input", + "crew_inputs", + "crew_agents", + "crew_tasks", + "formatted_description", + "formatted_expected_output", + "gen_ai.input.messages", + "gen_ai.prompts", + "gen_ai.prompt", + "gen_ai.request.parameters", + "gen_ai.tool.call.arguments", + "tool.parameters", + "tool_call.function.arguments", + "message.function_call_arguments_json", +) +_OUTPUT_ALIASES = ( + "output.value", + "raw.output", + "gen_ai.output.messages", + "gen_ai.completion", + "gen_ai.tool.call.result", +) + + +def _credential_key(key: str) -> bool: + # Do not classify every field containing 'token': usage and IDs are useful. + # Flattened paths may put a value/index below a credential container, e.g. + # headers.Authorization.0 or client.config.api_key.value. + for part in re.split(r"[.\[\]]+", key.lower()): + normalized = re.sub(r"[^a-z0-9]", "", part) + if normalized in _CREDENTIAL_KEYS or normalized.endswith(_CREDENTIAL_SUFFIXES): + return True + return False + + +def _safe_data(value: Any, depth: int = 0) -> Any: + """Copy only plain business data, never calling runtime serializers or repr.""" + if depth > 20: + return "[omitted]" + if type(value) is str: + if value.lstrip().startswith(("{", "[")): + if len(value) > 1_000_000: + return "[omitted oversized structured data]" + try: + parsed = json.loads(value) + except (ValueError, RecursionError): + # Do not export a partially structured payload we cannot safely + # inspect (including valid JSON beyond parser resource limits). + return "[omitted unparseable structured data]" + if type(parsed) in (dict, list): + return json.dumps(_safe_data(parsed, depth + 1)) + return value + if value is None or type(value) in (int, float, bool): + return value + if type(value) is dict: + return { + key: REDACTED if _credential_key(key) else _safe_data(item, depth + 1) + for key, item in value.items() + if type(key) is str + } + if type(value) in (list, tuple): + return [_safe_data(item, depth + 1) for item in value] + return "[omitted runtime object]" + + +def _has_prefix(key: str, prefixes: tuple) -> bool: + return any(key == prefix or key.startswith(prefix + ".") for prefix in prefixes) + + +class _CrewAITraceConfig(TraceConfig): + """Keep the caller's core policy and cover CrewAI aliases/events as well. + + Applying projection at the mask boundary also protects context attributes + before samplers/processors see them, not just the wrapper's final setters. + """ + + def __init__(self, config: TraceConfig) -> None: + for field in fields(TraceConfig): + object.__setattr__(self, field.name, getattr(config, field.name)) + object.__setattr__(self, "_source", config) + + def mask(self, key: str, value: Any) -> Any: + hide_input = ( + self.hide_inputs or self.hide_input_text or self.hide_input_messages + ) + hide_output = ( + self.hide_outputs or self.hide_output_text or self.hide_output_messages + ) + if (hide_input and _has_prefix(key, _INPUT_ALIASES)) or ( + hide_output and _has_prefix(key, _OUTPUT_ALIASES) + ): + value = REDACTED + masked = self._source.mask(key, value) + if masked is None: + return None + if _credential_key(key): + return REDACTED + projected = _safe_data(masked) + # OTel attributes cannot carry dictionaries (nor lists of dictionaries). + if type(projected) is dict or ( + type(projected) is list + and any(type(item) in (dict, list) for item in projected) + ): + return json.dumps(projected) + return projected diff --git a/python/frameworks/crewai/traceai_crewai/_telemetry.py b/python/frameworks/crewai/traceai_crewai/_telemetry.py new file mode 100644 index 000000000..18603488c --- /dev/null +++ b/python/frameworks/crewai/traceai_crewai/_telemetry.py @@ -0,0 +1,66 @@ +"""Disable CrewAI's separate native exporter without disabling the OTel SDK.""" + +from importlib import import_module +from inspect import getattr_static +from typing import Any, List, Tuple + + +class _NativeTelemetryGuard: + """Reversible, version-aware opt-out while traceAI owns CrewAI tracing. + + Instrument before constructing agents/crews. This cannot retract telemetry + queued before instrumentation or stop unrelated application exporters. + """ + + def __init__(self) -> None: + self._originals: List[Tuple[Any, str, Any, bool]] = [] + + def _replace(self, owner: Any, name: str, replacement: Any) -> None: + owned = name in vars(owner) + self._originals.append((owner, name, getattr_static(owner, name), owned)) + setattr(owner, name, replacement) + + def enable(self) -> None: + telemetry = import_module("crewai.telemetry").Telemetry + classes = [telemetry] + # Maintained CrewAI factors the exporter/disable checks into crewai-core. + try: + classes.append(import_module("crewai_core.telemetry").Telemetry) + except ModuleNotFoundError as exc: + if exc.name not in ("crewai_core", "crewai_core.telemetry"): + raise + for cls in classes: + if hasattr(cls, "_is_telemetry_disabled"): + self._replace( + cls, "_is_telemetry_disabled", classmethod(lambda cls: True) + ) + else: + # CrewAI 0.41.1 ignores disable flags and creates an HTTP exporter + # in __init__. Do not construct that exporter in the first place. + def disabled(instance: Any) -> None: + instance.ready = False + instance.trace_set = False + + self._replace(cls, "__init__", disabled) + # Also stop future calls on instances constructed before instrument(). + for name in ( + "set_tracer", + "crew_creation", + "task_started", + "task_ended", + "tool_repeated_usage", + "tool_usage", + "tool_usage_error", + "crew_execution_span", + "end_crew", + ): + if hasattr(cls, name): + self._replace(cls, name, lambda *args, **kwargs: None) + + def disable(self) -> None: + for owner, name, original, owned in reversed(self._originals): + if owned: + setattr(owner, name, original) + else: + delattr(owner, name) + self._originals.clear() diff --git a/python/frameworks/crewai/traceai_crewai/_wrappers.py b/python/frameworks/crewai/traceai_crewai/_wrappers.py index 2ab524a76..a968d7764 100644 --- a/python/frameworks/crewai/traceai_crewai/_wrappers.py +++ b/python/frameworks/crewai/traceai_crewai/_wrappers.py @@ -5,31 +5,17 @@ from crewai import Agent from crewai.tools.tool_calling import ToolCalling -from fi_instrumentation import get_attributes_from_context, safe_json_dumps +from fi_instrumentation import get_attributes_from_context from fi_instrumentation.fi_types import FiSpanKindValues, SpanAttributes from opentelemetry import context as context_api from opentelemetry import trace as trace_api from opentelemetry.util.types import AttributeValue - -class SafeJSONEncoder(json.JSONEncoder): - """ - Safely encodes non-JSON-serializable objects. - """ - - def default(self, o: Any) -> Any: - try: - return super().default(o) - except TypeError: - if hasattr(o, "dict") and callable( - o.dict - ): # pydantic v1 models, e.g., from Cohere - return o.dict() - return repr(o) +from ._privacy import _credential_key, _safe_data def _flatten( - mapping: Optional[Mapping[str, Any]] + mapping: Optional[Mapping[str, Any]], ) -> Iterator[Tuple[str, AttributeValue]]: if not mapping: return @@ -51,6 +37,63 @@ def _flatten( yield key, value +def _hide_input_content(tracer: Any) -> bool: + config = getattr(tracer, "_self_config", None) + return any( + getattr(config, key, False) is True + for key in ("hide_inputs", "hide_input_text", "hide_input_messages") + ) + + +def _safe_context_attributes() -> dict: + return { + key: ("__REDACTED__" if _credential_key(key) else _safe_data(value)) + for key, value in get_attributes_from_context() + } + + +def _safe_output(value: Any) -> Any: + cls = type(value) + if cls.__module__.startswith("crewai.") and cls.__name__ in ( + "TaskOutput", + "CrewOutput", + ): + content = value.json_dict if type(value.json_dict) is dict else value.raw + return _safe_data(content) + return _safe_data(value) + + +def _bound_call( + method: Callable[..., Any], args: Tuple[Any, ...], kwargs: Mapping[str, Any] +) -> dict: + try: + sig = signature(method) + needs_self = next(iter(sig.parameters), None) == "self" + return dict( + sig.bind_partial( + *((None,) if needs_self else ()), *args, **kwargs + ).arguments + ) + except (TypeError, ValueError): + return {} + + +def _task_input( + method: Callable[..., Any], + task: Any, + args: Tuple[Any, ...], + kwargs: Mapping[str, Any], +) -> str: + bound = _bound_call(method, args, kwargs) + return json.dumps( + { + "description": _safe_data(getattr(task, "description", None)), + "expected_output": _safe_data(getattr(task, "expected_output", None)), + "context": _safe_data(bound.get("context")), + } + ) + + def _get_input_value(method: Callable[..., Any], *args: Any, **kwargs: Any) -> str: """ Parses a method call's inputs into a JSON string. Ensures a consistent @@ -62,51 +105,24 @@ def _get_input_value(method: Callable[..., Any], *args: Any, **kwargs: Any) -> s # does not include the self parameter. However, the inspect.Signature # instance for __call__ does include the self parameter. - tool = kwargs.get("calling") - if tool and isinstance(tool, ToolCalling): - return safe_json_dumps( + bound = _bound_call(method, args, kwargs) + calling = bound.get("calling", kwargs.get("calling")) + if isinstance(calling, ToolCalling): + return json.dumps( { - "tool_name": tool.tool_name, - "tool_args": tool.arguments, - }, - cls=SafeJSONEncoder, + "tool_name": _safe_data(calling.tool_name), + "tool_args": _safe_data(calling.arguments), + } ) - - if args and args[0] and isinstance(args[0], Agent): - return safe_json_dumps( + if args and isinstance(args[0], Agent): + return json.dumps( { - "role": args[0].role, - "goal": args[0].goal, - "backstory": args[0].backstory, - }, - cls=SafeJSONEncoder, + "role": _safe_data(args[0].role), + "goal": _safe_data(args[0].goal), + "backstory": _safe_data(args[0].backstory), + } ) - - method_signature = signature(method) - first_parameter_name = next(iter(method_signature.parameters), None) - signature_contains_self_parameter = first_parameter_name in ["self"] - bound_arguments = method_signature.bind( - *( - [ - None - ] # the value bound to the method's self argument is discarded below, so pass None - if signature_contains_self_parameter - else [] # no self parameter, so no need to pass a value - ), - *args, - **kwargs, - ) - return safe_json_dumps( - { - **{ - argument_name: argument_value - for argument_name, argument_value in bound_arguments.arguments.items() - if argument_name not in ["self", "kwargs"] - }, - **bound_arguments.arguments.get("kwargs", {}), - }, - cls=SafeJSONEncoder, - ) + return json.dumps(_safe_data({k: v for k, v in bound.items() if k != "self"})) class _ExecuteCoreWrapper: @@ -132,10 +148,10 @@ def __call__( _flatten( { GEN_AI_SPAN_KIND: FiSpanKindValues.AGENT, - SpanAttributes.INPUT_VALUE: _get_input_value( - wrapped, - *args, - **kwargs, + SpanAttributes.INPUT_VALUE: ( + "__REDACTED__" + if _hide_input_content(self._tracer) + else _task_input(wrapped, instance, args, kwargs) ), } ) @@ -143,36 +159,34 @@ def __call__( record_exception=False, set_status_on_exception=False, ) as span: - agent = args[0] if args else None - crew = agent.crew if agent else None + bound = _bound_call(wrapped, args, kwargs) + agent = bound.get("agent", args[0] if args else None) + crew = getattr(agent, "crew", None) if agent is not None else None task = instance - span.set_attribute( - SpanAttributes.INPUT_VALUE, - safe_json_dumps(_prepare_args_kwargs(args=args, **kwargs)), - ) - if crew: span.set_attribute("crew_key", crew.key) span.set_attribute("crew_id", str(crew.id)) span.set_attribute("task_key", task.key) span.set_attribute("task_id", str(task.id)) - if crew and crew.share_crew: - span.set_attribute("formatted_description", task.description) - span.set_attribute("formatted_expected_output", task.expected_output) + if crew and crew.share_crew and not _hide_input_content(self._tracer): + span.set_attribute( + "formatted_description", _safe_data(task.description) + ) + span.set_attribute( + "formatted_expected_output", _safe_data(task.expected_output) + ) try: response = wrapped(*args, **kwargs) - except Exception as exception: - span.set_status( - trace_api.Status(trace_api.StatusCode.ERROR, str(exception)) - ) - span.record_exception(exception) + except Exception: + # Provider exceptions may include credentials or request configuration. + span.set_status(trace_api.StatusCode.ERROR) raise span.set_status(trace_api.StatusCode.OK) - span.set_attribute(OUTPUT_VALUE, safe_json_dumps(response)) - span.set_attributes(dict(get_attributes_from_context())) + span.set_attribute(OUTPUT_VALUE, json.dumps(_safe_output(response))) + span.set_attributes(_safe_context_attributes()) return response @@ -206,66 +220,81 @@ def __call__( inputs = kwargs.get("inputs", None) or (args[0] if args else None) span.set_attribute("crew_key", crew.key) span.set_attribute("crew_id", str(crew.id)) - span.set_attribute("crew_inputs", json.dumps(inputs) if inputs else "") + # Only explicit kickoff inputs are content; checkpoint/file objects are runtime state. span.set_attribute( SpanAttributes.INPUT_VALUE, - safe_json_dumps(_prepare_args_kwargs(args=args, **kwargs)), - ) - span.set_attribute( - "crew_agents", - json.dumps( - [ - { - "key": agent.key, - "id": str(agent.id), - "role": agent.role, - "goal": agent.goal, - "backstory": agent.backstory, - "verbose?": agent.verbose, - "max_iter": agent.max_iter, - "max_rpm": agent.max_rpm, - "i18n": agent.i18n.prompt_file, - "delegation_enabled": agent.allow_delegation, - "tools_names": [ - tool.name.casefold() for tool in agent.tools or [] - ], - } - for agent in crew.agents - ] - ), - ) - span.set_attribute( - "crew_tasks", - json.dumps( - [ - { - "id": str(task.id), - "description": task.description, - "expected_output": task.expected_output, - "async_execution?": task.async_execution, - "human_input?": task.human_input, - "agent_role": task.agent.role if task.agent else "None", - "agent_key": task.agent.key if task.agent else None, - # CrewAI defaults Task.context to a NOT_SPECIFIED - # sentinel that is truthy but not iterable, so this - # must guard on type before iterating. - "context": ( - [ - context_task.description - for context_task in task.context - ] - if isinstance(task.context, (list, tuple)) - and task.context - else None - ), - "tools_names": [ - tool.name.casefold() for tool in task.tools or [] - ], - } - for task in crew.tasks - ] + ( + "__REDACTED__" + if _hide_input_content(self._tracer) + else json.dumps(_safe_data(inputs)) ), ) + if not _hide_input_content(self._tracer): + span.set_attribute("crew_inputs", json.dumps(_safe_data(inputs))) + if not _hide_input_content(self._tracer): + span.set_attribute( + "crew_agents", + json.dumps( + _safe_data( + [ + { + "key": agent.key, + "id": str(agent.id), + "role": agent.role, + "goal": agent.goal, + "backstory": agent.backstory, + "verbose?": agent.verbose, + "max_iter": agent.max_iter, + "max_rpm": agent.max_rpm, + "i18n": agent.i18n.prompt_file, + "delegation_enabled": agent.allow_delegation, + "tools_names": [ + tool.name.casefold() + for tool in agent.tools or [] + ], + } + for agent in crew.agents + ] + ) + ), + ) + span.set_attribute( + "crew_tasks", + json.dumps( + _safe_data( + [ + { + "id": str(task.id), + "description": task.description, + "expected_output": task.expected_output, + "async_execution?": task.async_execution, + "human_input?": task.human_input, + "agent_role": ( + task.agent.role if task.agent else "None" + ), + "agent_key": task.agent.key if task.agent else None, + # CrewAI defaults Task.context to a NOT_SPECIFIED + # sentinel that is truthy but not iterable, so this + # must guard on type before iterating. + "context": ( + [ + context_task.description + for context_task in task.context + ] + if isinstance(task.context, (list, tuple)) + and task.context + else None + ), + "tools_names": [ + tool.name.casefold() + for tool in task.tools or [] + ], + } + for task in crew.tasks + ] + ) + ), + ) try: crew_output = wrapped(*args, **kwargs) usage_metrics = crew.usage_metrics @@ -273,7 +302,9 @@ def __call__( if ( prompt_tokens := usage_metrics.get("prompt_tokens") ) is not None: - span.set_attribute(GEN_AI_USAGE_INPUT_TOKENS, int(prompt_tokens)) + span.set_attribute( + GEN_AI_USAGE_INPUT_TOKENS, int(prompt_tokens) + ) if ( completion_tokens := usage_metrics.get("completion_tokens") ) is not None: @@ -294,20 +325,13 @@ def __call__( GEN_AI_USAGE_TOTAL_TOKENS, usage_metrics.total_tokens ) - except Exception as exception: - span.set_status( - trace_api.Status(trace_api.StatusCode.ERROR, str(exception)) - ) - span.record_exception(exception) + except Exception: + # Provider exceptions may include credentials or request configuration. + span.set_status(trace_api.StatusCode.ERROR) raise span.set_status(trace_api.StatusCode.OK) - if crew_output_dict := crew_output.to_dict(): - - span.set_attribute(OUTPUT_VALUE, json.dumps(crew_output_dict)) - span.set_attribute(OUTPUT_MIME_TYPE, "application/json") - else: - span.set_attribute(OUTPUT_VALUE, str(crew_output)) - span.set_attributes(dict(get_attributes_from_context())) + span.set_attribute(OUTPUT_VALUE, json.dumps(_safe_output(crew_output))) + span.set_attributes(_safe_context_attributes()) return crew_output @@ -334,10 +358,10 @@ def __call__( _flatten( { GEN_AI_SPAN_KIND: FiSpanKindValues.TOOL, - SpanAttributes.INPUT_VALUE: _get_input_value( - wrapped, - *args, - **kwargs, + SpanAttributes.INPUT_VALUE: ( + "__REDACTED__" + if _hide_input_content(self._tracer) + else _get_input_value(wrapped, *args, **kwargs) ), } ) @@ -345,62 +369,24 @@ def __call__( record_exception=False, set_status_on_exception=False, ) as span: - tool = kwargs.get("tool") - tool_name = "" - if tool: - tool_name = tool.name - span.set_attribute("function_calling_llm", instance.function_calling_llm) - span.set_attribute(SpanAttributes.GEN_AI_TOOL_NAME, tool_name) - - span.set_attribute( - SpanAttributes.INPUT_VALUE, - safe_json_dumps(_prepare_args_kwargs(args=args, **kwargs)), - ) + bound = _bound_call(wrapped, args, kwargs) + tool = bound.get("tool", kwargs.get("tool")) + tool_name = getattr(tool, "name", "") if tool is not None else "" + if type(tool_name) is str: + span.set_attribute(SpanAttributes.GEN_AI_TOOL_NAME, tool_name) try: response = wrapped(*args, **kwargs) - except Exception as exception: - span.set_status( - trace_api.Status(trace_api.StatusCode.ERROR, str(exception)) - ) - span.record_exception(exception) + except Exception: + # Provider exceptions may include credentials or request configuration. + span.set_status(trace_api.StatusCode.ERROR) raise span.set_status(trace_api.StatusCode.OK) - span.set_attribute(OUTPUT_VALUE, safe_json_dumps(response)) - span.set_attribute(OUTPUT_VALUE, safe_json_dumps(response)) - span.set_attributes(dict(get_attributes_from_context())) + span.set_attribute(OUTPUT_VALUE, json.dumps(_safe_output(response))) + span.set_attributes(_safe_context_attributes()) return response -def _convert_to_dict(obj: Any) -> Any: - """ - Recursively converts objects to dicts if they have a to_dict method or __dict__ attribute. - Handles nested lists and dictionaries. - """ - if hasattr(obj, "to_dict") and callable(obj.to_dict): - return obj.to_dict() - elif hasattr(obj, "__dict__"): - return obj.__dict__ - elif isinstance(obj, dict): - return {key: _convert_to_dict(value) for key, value in obj.items()} - elif isinstance(obj, (list, tuple)): - return [_convert_to_dict(item) for item in obj] - return obj - - -def _prepare_args_kwargs(args: Tuple[Any, ...], **kwargs: Any) -> dict: - """ - Prepares args and kwargs for JSON serialization by converting objects with to_dict - method to dictionaries. Maintains the original argument structure. - """ - processed_args = _convert_to_dict(args) - processed_kwargs = _convert_to_dict(kwargs) - return { - "args": processed_args, - **processed_kwargs, # Unpacks the processed kwargs directly - } - - INPUT_VALUE = SpanAttributes.INPUT_VALUE GEN_AI_SPAN_KIND = SpanAttributes.GEN_AI_SPAN_KIND OUTPUT_VALUE = SpanAttributes.OUTPUT_VALUE diff --git a/python/tests/test_framework_crewai.py b/python/tests/test_framework_crewai.py index 529c89748..ea6d2cf7b 100644 --- a/python/tests/test_framework_crewai.py +++ b/python/tests/test_framework_crewai.py @@ -3,7 +3,7 @@ Tests the instrumentation of CrewAI components: - Task._execute_core (AGENT spans) -- Crew.kickoff (CHAIN spans) +- Crew.kickoff (CHAIN spans) - ToolUsage._use (TOOL spans) """ @@ -20,14 +20,12 @@ from traceai_crewai import CrewAIInstrumentor from traceai_crewai._wrappers import ( - SafeJSONEncoder, _ExecuteCoreWrapper, _KickoffWrapper, _ToolUseWrapper, - _convert_to_dict, _flatten, _get_input_value, - _prepare_args_kwargs, + _safe_data, ) @@ -40,32 +38,41 @@ def test_instrumentation_dependencies(self): deps = instrumentor.instrumentation_dependencies() assert "crewai >= 0.41.1" in deps + @patch("traceai_crewai._NativeTelemetryGuard") @patch("traceai_crewai.import_module") @patch("traceai_crewai.wrap_function_wrapper") - def test_instrument_with_defaults(self, mock_wrap, mock_import): + def test_instrument_with_defaults(self, mock_wrap, mock_import, mock_guard): """Test instrumentation with default parameters.""" mock_tracer_provider = Mock() mock_task_module = Mock() mock_crew_module = Mock() mock_tool_module = Mock() - + mock_import.side_effect = [ mock_task_module, mock_crew_module, mock_tool_module, ] - + instrumentor = CrewAIInstrumentor() instrumentor._instrument(tracer_provider=mock_tracer_provider) - + # Should wrap all three methods assert mock_wrap.call_count == 3 - + # Verify correct module and method names calls = mock_wrap.call_args_list - assert any("crewai" in str(call) and "Task._execute_core" in str(call) for call in calls) - assert any("crewai" in str(call) and "Crew.kickoff" in str(call) for call in calls) - assert any("crewai.tools.tool_usage" in str(call) and "ToolUsage._use" in str(call) for call in calls) + assert any( + "crewai" in str(call) and "Task._execute_core" in str(call) + for call in calls + ) + assert any( + "crewai" in str(call) and "Crew.kickoff" in str(call) for call in calls + ) + assert any( + "crewai.tools.tool_usage" in str(call) and "ToolUsage._use" in str(call) + for call in calls + ) @patch("traceai_crewai.import_module") def test_uninstrument(self, mock_import): @@ -73,26 +80,27 @@ def test_uninstrument(self, mock_import): mock_task_module = Mock() mock_crew_module = Mock() mock_tool_module = Mock() - + mock_import.side_effect = [ mock_task_module, mock_crew_module, mock_tool_module, ] - + instrumentor = CrewAIInstrumentor() - + # Set up original methods original_execute = Mock() original_kickoff = Mock() original_tool_use = Mock() - + instrumentor._original_execute_core = original_execute instrumentor._original_kickoff = original_kickoff instrumentor._original_tool_use = original_tool_use - + instrumentor._native_telemetry_guard = Mock() + instrumentor._uninstrument() - + # Verify methods are restored assert mock_task_module.Task._execute_core == original_execute assert mock_crew_module.Crew.kickoff == original_kickoff @@ -114,11 +122,11 @@ def setup_method(self): def test_suppress_instrumentation(self): """Test that instrumentation is suppressed when flag is set.""" - with patch.object(context_api, 'get_value', return_value=True): + with patch.object(context_api, "get_value", return_value=True): wrapped_func = Mock(return_value="result") - + result = self.wrapper(wrapped_func, None, (), {}) - + assert result == "result" wrapped_func.assert_called_once_with() self.mock_tracer.start_as_current_span.assert_not_called() @@ -131,18 +139,18 @@ def test_task_execution_basic(self): mock_task = Mock() mock_task.key = "task_key_123" mock_task.id = "task_id_456" - + wrapped_func = Mock(__name__="_execute_core", return_value="task_result") - + result = self.wrapper(wrapped_func, mock_task, (mock_agent,), {}) - + assert result == "task_result" - + # Verify span creation self.mock_tracer.start_as_current_span.assert_called_once() call_args = self.mock_tracer.start_as_current_span.call_args assert "Mock._execute_core" in call_args[0][0] - + # Verify attributes set self.mock_span.set_attribute.assert_any_call("task_key", "task_key_123") self.mock_span.set_attribute.assert_any_call("task_id", "task_id_456") @@ -154,27 +162,31 @@ def test_task_execution_with_crew(self): mock_crew.key = "crew_key_789" mock_crew.id = "crew_id_101" mock_crew.share_crew = True - + mock_agent = Mock() mock_agent.crew = mock_crew - + mock_task = Mock() mock_task.key = "task_key_123" mock_task.id = "task_id_456" mock_task.description = "Test task description" mock_task.expected_output = "Expected output format" - + wrapped_func = Mock(__name__="_execute_core", return_value="task_result") - + result = self.wrapper(wrapped_func, mock_task, (mock_agent,), {}) - + assert result == "task_result" - + # Verify crew attributes set self.mock_span.set_attribute.assert_any_call("crew_key", "crew_key_789") self.mock_span.set_attribute.assert_any_call("crew_id", "crew_id_101") - self.mock_span.set_attribute.assert_any_call("formatted_description", "Test task description") - self.mock_span.set_attribute.assert_any_call("formatted_expected_output", "Expected output format") + self.mock_span.set_attribute.assert_any_call( + "formatted_description", "Test task description" + ) + self.mock_span.set_attribute.assert_any_call( + "formatted_expected_output", "Expected output format" + ) def test_task_execution_error_handling(self): """Test error handling in task execution.""" @@ -183,35 +195,38 @@ def test_task_execution_error_handling(self): mock_task = Mock() mock_task.key = "task_key" mock_task.id = "task_id" - + test_exception = Exception("Task execution failed") wrapped_func = Mock(__name__="_execute_core", side_effect=test_exception) - + with pytest.raises(Exception, match="Task execution failed"): self.wrapper(wrapped_func, mock_task, (mock_agent,), {}) - + # Verify error handling self.mock_span.set_status.assert_called_once() call_args = self.mock_span.set_status.call_args[0][0] - assert call_args.status_code == StatusCode.ERROR - assert call_args.description == "Task execution failed" - self.mock_span.record_exception.assert_called_once_with(test_exception) + assert call_args == StatusCode.ERROR + self.mock_span.record_exception.assert_not_called() + assert "Task execution failed" not in str(self.mock_span.mock_calls) @patch("traceai_crewai._wrappers.get_attributes_from_context") def test_context_attributes(self, mock_get_attrs): """Test that context attributes are added to spans.""" - mock_get_attrs.return_value = {"session_id": "test_session", "user_id": "test_user"} - + mock_get_attrs.return_value = [ + ("session_id", "test_session"), + ("user_id", "test_user"), + ] + mock_agent = Mock() mock_agent.crew = None mock_task = Mock() mock_task.key = "task_key" mock_task.id = "task_id" - + wrapped_func = Mock(__name__="_execute_core", return_value="result") - + self.wrapper(wrapped_func, mock_task, (mock_agent,), {}) - + # Verify context attributes are set self.mock_span.set_attributes.assert_called_once() call_args = self.mock_span.set_attributes.call_args[0][0] @@ -241,23 +256,27 @@ def test_crew_kickoff_basic(self): mock_crew.id = "crew_id_456" mock_crew.agents = [] mock_crew.tasks = [] - mock_crew.usage_metrics = {"prompt_tokens": 100, "completion_tokens": 50, "total_tokens": 150} - + mock_crew.usage_metrics = { + "prompt_tokens": 100, + "completion_tokens": 50, + "total_tokens": 150, + } + # Mock crew output mock_output = Mock() mock_output.to_dict.return_value = {"result": "crew completed successfully"} - + wrapped_func = Mock(__name__="kickoff", return_value=mock_output) - + result = self.wrapper(wrapped_func, mock_crew, (), {"inputs": {"task": "test"}}) - + assert result == mock_output - + # Verify span creation self.mock_tracer.start_as_current_span.assert_called_once() call_args = self.mock_tracer.start_as_current_span.call_args assert "Crew.kickoff" in call_args[0][0] - + # Verify basic attributes self.mock_span.set_attribute.assert_any_call("crew_key", "crew_key_123") self.mock_span.set_attribute.assert_any_call("crew_id", "crew_id_456") @@ -281,7 +300,7 @@ def test_crew_kickoff_with_agents_and_tasks(self): mock_tool.name = Mock() mock_tool.name.casefold.return_value = "search_tool" mock_agent.tools = [mock_tool] - + # Mock task mock_task = Mock() mock_task.id = "task_id_1" @@ -292,7 +311,7 @@ def test_crew_kickoff_with_agents_and_tasks(self): mock_task.agent = mock_agent mock_task.context = None mock_task.tools = [] - + # Mock crew mock_crew = Mock() mock_crew.__class__.__name__ = "Crew" @@ -300,17 +319,21 @@ def test_crew_kickoff_with_agents_and_tasks(self): mock_crew.id = "crew_id" mock_crew.agents = [mock_agent] mock_crew.tasks = [mock_task] - mock_crew.usage_metrics = {"prompt_tokens": 200, "completion_tokens": 100, "total_tokens": 300} - + mock_crew.usage_metrics = { + "prompt_tokens": 200, + "completion_tokens": 100, + "total_tokens": 300, + } + mock_output = Mock() mock_output.to_dict.return_value = {"result": "success"} - + wrapped_func = Mock(__name__="kickoff", return_value=mock_output) - + result = self.wrapper(wrapped_func, mock_crew, (), {}) - + assert result == mock_output - + # Verify token metrics are set self.mock_span.set_attribute.assert_any_call( SpanAttributes.GEN_AI_USAGE_INPUT_TOKENS, 200 @@ -330,21 +353,21 @@ def test_crew_kickoff_with_new_usage_metrics(self): mock_crew.id = "crew_id" mock_crew.agents = [] mock_crew.tasks = [] - + # New format usage metrics (object with attributes) mock_usage = Mock() mock_usage.prompt_tokens = 150 mock_usage.completion_tokens = 75 mock_usage.total_tokens = 225 mock_crew.usage_metrics = mock_usage - + mock_output = Mock() mock_output.to_dict.return_value = {"result": "success"} - + wrapped_func = Mock(__name__="kickoff", return_value=mock_output) - + self.wrapper(wrapped_func, mock_crew, (), {}) - + # Verify new format token metrics self.mock_span.set_attribute.assert_any_call( SpanAttributes.GEN_AI_USAGE_INPUT_TOKENS, 150 @@ -363,20 +386,20 @@ def test_crew_kickoff_error_handling(self): mock_crew.key = "crew_key" mock_crew.id = "crew_id" mock_crew.agents = [] # Empty list to avoid iteration issues - mock_crew.tasks = [] # Empty list to avoid iteration issues - + mock_crew.tasks = [] # Empty list to avoid iteration issues + test_exception = Exception("Crew kickoff failed") wrapped_func = Mock(__name__="kickoff", side_effect=test_exception) - + with pytest.raises(Exception, match="Crew kickoff failed"): self.wrapper(wrapped_func, mock_crew, (), {}) - + # Verify error handling self.mock_span.set_status.assert_called_once() call_args = self.mock_span.set_status.call_args[0][0] - assert call_args.status_code == StatusCode.ERROR - assert call_args.description == "Crew kickoff failed" - self.mock_span.record_exception.assert_called_once_with(test_exception) + assert call_args == StatusCode.ERROR + self.mock_span.record_exception.assert_not_called() + assert "Crew kickoff failed" not in str(self.mock_span.mock_calls) def test_crew_output_without_to_dict(self): """Test crew output that doesn't have to_dict method.""" @@ -387,19 +410,22 @@ def test_crew_output_without_to_dict(self): mock_crew.agents = [] mock_crew.tasks = [] mock_crew.usage_metrics = {} - + # Mock output without to_dict method mock_output = Mock() mock_output.to_dict.return_value = None - + wrapped_func = Mock(__name__="kickoff", return_value=mock_output) - + result = self.wrapper(wrapped_func, mock_crew, (), {}) - + assert result == mock_output - - # Should set string representation - self.mock_span.set_attribute.assert_any_call("output.value", str(mock_output)) + + # Unknown runtime outputs must not call serializers or repr. + self.mock_span.set_attribute.assert_any_call( + "output.value", json.dumps("[omitted runtime object]") + ) + mock_output.to_dict.assert_not_called() class _NotSpecifiedStub: @@ -508,24 +534,29 @@ def test_tool_use_basic(self): mock_instance = Mock() mock_instance.__class__.__name__ = "ToolUsage" mock_instance.function_calling_llm = "gpt-4" - + # Mock tool mock_tool = Mock() mock_tool.name = "search_tool" - - wrapped_func = Mock(__name__="_use", return_value={"status": "success", "result": "found data"}) - + + wrapped_func = Mock( + __name__="_use", return_value={"status": "success", "result": "found data"} + ) + result = self.wrapper(wrapped_func, mock_instance, (), {"tool": mock_tool}) - + assert result == {"status": "success", "result": "found data"} - + # Verify span creation self.mock_tracer.start_as_current_span.assert_called_once() call_args = self.mock_tracer.start_as_current_span.call_args assert "ToolUsage._use" in call_args[0][0] - + # Verify tool attributes - self.mock_span.set_attribute.assert_any_call("function_calling_llm", "gpt-4") + assert not any( + call.args[0] == "function_calling_llm" + for call in self.mock_span.set_attribute.call_args_list + ) self.mock_span.set_attribute.assert_any_call( SpanAttributes.GEN_AI_TOOL_NAME, "search_tool" ) @@ -535,48 +566,50 @@ def test_tool_use_without_tool(self): mock_instance = Mock() mock_instance.__class__.__name__ = "ToolUsage" mock_instance.function_calling_llm = "gpt-3.5-turbo" - + wrapped_func = Mock(__name__="_use", return_value="no tool result") - + result = self.wrapper(wrapped_func, mock_instance, (), {}) - + assert result == "no tool result" - + # Should set empty tool name - self.mock_span.set_attribute.assert_any_call(SpanAttributes.GEN_AI_TOOL_NAME, "") + self.mock_span.set_attribute.assert_any_call( + SpanAttributes.GEN_AI_TOOL_NAME, "" + ) def test_tool_use_error_handling(self): """Test error handling in tool usage.""" mock_instance = Mock() mock_instance.__class__.__name__ = "ToolUsage" mock_instance.function_calling_llm = "gpt-4" - + test_exception = Exception("Tool execution failed") wrapped_func = Mock(__name__="_use", side_effect=test_exception) - + with pytest.raises(Exception, match="Tool execution failed"): self.wrapper(wrapped_func, mock_instance, (), {}) - + # Verify error handling self.mock_span.set_status.assert_called_once() call_args = self.mock_span.set_status.call_args[0][0] - assert call_args.status_code == StatusCode.ERROR - assert call_args.description == "Tool execution failed" - self.mock_span.record_exception.assert_called_once_with(test_exception) + assert call_args == StatusCode.ERROR + self.mock_span.record_exception.assert_not_called() + assert "Tool execution failed" not in str(self.mock_span.mock_calls) @patch("traceai_crewai._wrappers.get_attributes_from_context") def test_tool_context_attributes(self, mock_get_attrs): """Test that context attributes are added to tool spans.""" - mock_get_attrs.return_value = {"session_id": "tool_session"} - + mock_get_attrs.return_value = [("session_id", "tool_session")] + mock_instance = Mock() mock_instance.__class__.__name__ = "ToolUsage" mock_instance.function_calling_llm = "gpt-4" - + wrapped_func = Mock(__name__="_use", return_value="result") - + self.wrapper(wrapped_func, mock_instance, (), {}) - + # Verify context attributes are set self.mock_span.set_attributes.assert_called_once() @@ -584,85 +617,71 @@ def test_tool_context_attributes(self, mock_get_attrs): class TestUtilityFunctions: """Test utility functions used by the wrappers.""" - def test_safe_json_encoder_basic(self): - """Test SafeJSONEncoder with basic serializable objects.""" - encoder = SafeJSONEncoder() - - # Should handle normal JSON types - result = json.dumps({"key": "value", "number": 42}, cls=SafeJSONEncoder) - assert "key" in result - assert "value" in result - - def test_safe_json_encoder_with_pydantic(self): - """Test SafeJSONEncoder with pydantic-like objects.""" - # Mock object with dict method - mock_obj = Mock() - mock_obj.dict.return_value = {"field": "value"} - - encoder = SafeJSONEncoder() - result = encoder.default(mock_obj) - - assert result == {"field": "value"} - - def test_safe_json_encoder_fallback(self): - """Test SafeJSONEncoder fallback to repr.""" - class CustomClass: + def test_projection_plain_data_and_nested_credentials(self): + payload = { + "safe": "business", + "client": {"config": {"api_key": "FAKE_SHARED_KEY"}}, + } + result = _safe_data(payload) + assert result == { + "safe": "business", + "client": {"config": {"api_key": "__REDACTED__"}}, + } + assert payload["client"]["config"]["api_key"] == "FAKE_SHARED_KEY" + + def test_projection_does_not_call_runtime_serializers(self): + runtime = Mock() + assert _safe_data(runtime) == "[omitted runtime object]" + runtime.dict.assert_not_called() + runtime.to_dict.assert_not_called() + runtime.model_dump.assert_not_called() + + def test_projection_does_not_call_repr(self): + class Runtime: def __repr__(self): - return "CustomClass()" - - encoder = SafeJSONEncoder() - result = encoder.default(CustomClass()) - - assert result == "CustomClass()" + raise AssertionError("runtime repr evaluated") + + assert _safe_data(Runtime()) == "[omitted runtime object]" def test_flatten_simple_mapping(self): """Test _flatten with simple mapping.""" data = {"key1": "value1", "key2": 42} result = dict(_flatten(data)) - + assert result == {"key1": "value1", "key2": 42} def test_flatten_nested_mapping(self): """Test _flatten with nested mapping.""" - data = { - "top": { - "nested": "value", - "deep": {"deeper": "deep_value"} - } - } + data = {"top": {"nested": "value", "deep": {"deeper": "deep_value"}}} result = dict(_flatten(data)) - + assert result["top.nested"] == "value" assert result["top.deep.deeper"] == "deep_value" def test_flatten_with_lists(self): """Test _flatten with lists of mappings.""" - data = { - "items": [ - {"name": "item1", "value": 1}, - {"name": "item2", "value": 2} - ] - } + data = {"items": [{"name": "item1", "value": 1}, {"name": "item2", "value": 2}]} result = dict(_flatten(data)) - + assert result["items.0.name"] == "item1" assert result["items.1.value"] == 2 def test_flatten_with_enum(self): """Test _flatten with enum values.""" + class TestEnum(Enum): VALUE = "test_value" - + data = {"enum_field": TestEnum.VALUE} result = dict(_flatten(data)) - + assert result["enum_field"] == "test_value" def test_flatten_with_none_values(self): """Test _flatten skips None values.""" data = {"valid": "value", "null": None} result = dict(_flatten(data)) - + assert "valid" in result assert "null" not in result @@ -672,13 +691,13 @@ def test_get_input_value_with_tool_calling(self): mock_tool_calling = Mock() mock_tool_calling.tool_name = "search" mock_tool_calling.arguments = {"query": "test"} - + # Import the real class for isinstance check with patch("traceai_crewai._wrappers.ToolCalling", mock_tool_calling.__class__): mock_method = Mock() - + result = _get_input_value(mock_method, calling=mock_tool_calling) - + parsed_result = json.loads(result) assert parsed_result["tool_name"] == "search" assert parsed_result["tool_args"] == {"query": "test"} @@ -690,60 +709,38 @@ def test_get_input_value_with_agent(self): mock_agent.role = "researcher" mock_agent.goal = "research topics" mock_agent.backstory = "expert in research" - + # Import the real class for isinstance check with patch("traceai_crewai._wrappers.Agent", mock_agent.__class__): mock_method = Mock() - + result = _get_input_value(mock_method, mock_agent) - + parsed_result = json.loads(result) assert parsed_result["role"] == "researcher" assert parsed_result["goal"] == "research topics" assert parsed_result["backstory"] == "expert in research" - def test_convert_to_dict_with_to_dict_method(self): - """Test _convert_to_dict with object having to_dict method.""" - mock_obj = Mock() - mock_obj.to_dict.return_value = {"converted": True} - - result = _convert_to_dict(mock_obj) - assert result == {"converted": True} - - def test_convert_to_dict_with_dict_attr(self): - """Test _convert_to_dict with object having __dict__ attribute.""" - class TestObj: - def __init__(self): - self.field = "value" - - obj = TestObj() - result = _convert_to_dict(obj) - assert result == {"field": "value"} - - def test_convert_to_dict_with_nested_structures(self): - """Test _convert_to_dict with nested lists and dicts.""" - mock_obj = Mock() - mock_obj.to_dict.return_value = {"nested": True} - - data = { - "list": [mock_obj, "string"], - "dict": {"nested": mock_obj} + def test_projection_nested_containers_does_not_introspect_runtime(self): + runtime = Mock() + data = {"list": [runtime, "string"], "dict": {"nested": runtime}} + assert _safe_data(data) == { + "list": ["[omitted runtime object]", "string"], + "dict": {"nested": "[omitted runtime object]"}, + } + runtime.to_dict.assert_not_called() + + def test_generic_call_projection_redacts_without_mutating(self): + runtime = Mock() + + def method(arg, **kwargs): + pass + + projected = json.loads( + _get_input_value(method, runtime, api_key="FAKE_SHARED_KEY") + ) + assert projected == { + "arg": "[omitted runtime object]", + "kwargs": {"api_key": "__REDACTED__"}, } - - result = _convert_to_dict(data) - assert result["list"][0] == {"nested": True} - assert result["dict"]["nested"] == {"nested": True} - - def test_prepare_args_kwargs(self): - """Test _prepare_args_kwargs function.""" - mock_obj = Mock() - mock_obj.to_dict.return_value = {"converted": True} - - args = (mock_obj, "string") - kwargs = {"param": mock_obj} - - result = _prepare_args_kwargs(args, **kwargs) - - assert "args" in result - assert result["args"][0] == {"converted": True} - assert result["param"] == {"converted": True} \ No newline at end of file + runtime.to_dict.assert_not_called() From c84c1a07b1121ce55070dddf8da169cc1ee01f60 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Fri, 30 Oct 2026 22:52:01 -0700 Subject: [PATCH 2/5] fix(crewai): redact service credential headers Recognize security, Vault, Consul, subject and service tokens plus API-secret headers without hiding token counters or IDs. Cover actual native ToolUsage input/output and log events over OTLP JSON/protobuf, plain/gzip, alongside hide-setting and structured-alias regressions. Co-Authored-By: Claude Code --- python/frameworks/crewai/tests/conftest.py | 57 ++++++ .../frameworks/crewai/tests/test_otlp_wire.py | 162 +++++++++++++++--- .../crewai/tests/test_privacy_policy.py | 46 ++++- .../crewai/traceai_crewai/_privacy.py | 8 + 4 files changed, 246 insertions(+), 27 deletions(-) diff --git a/python/frameworks/crewai/tests/conftest.py b/python/frameworks/crewai/tests/conftest.py index 53272b99e..f842cbba1 100644 --- a/python/frameworks/crewai/tests/conftest.py +++ b/python/frameworks/crewai/tests/conftest.py @@ -29,3 +29,60 @@ def native_telemetry_off(request): yield finally: guard.disable() + + +@pytest.fixture +def service_credentials(): + return { + name: f"FAKE_SERVICE_CREDENTIAL_{index}_7f9b2a" + for index, name in enumerate( + ( + "X-Amz-Security-Token", + "X-Vault-Token", + "X-Consul-Token", + "X-Subject-Token", + "X-Api-Secret", + "X-Service-Token", + "security_token", + ) + ) + } + + +@pytest.fixture +def native_tool_usage(): + """Construct the real version-specific ToolUsage around a local function.""" + from importlib.metadata import version + from types import SimpleNamespace + + from crewai.tools.tool_usage import ToolUsage + + def create(function): + if version("crewai") == "0.41.1": + from crewai.agents.tools_handler import ToolsHandler + from langchain_core.tools import StructuredTool + + tool = StructuredTool.from_function(function) + usage = ToolUsage( + tools_handler=ToolsHandler(), + tools=[tool], + original_tools=[], + tools_description=function.__name__, + tools_names=function.__name__, + task=SimpleNamespace(used_tools=0), + function_calling_llm=None, + agent=SimpleNamespace(tools_results=[]), + action=None, + ) + usage._printer.print = lambda **kwargs: None + return usage + from crewai.tools.structured_tool import CrewStructuredTool + + return ToolUsage( + tools_handler=None, + tools=[CrewStructuredTool.from_function(function)], + task=None, + function_calling_llm=None, + ) + + return create diff --git a/python/frameworks/crewai/tests/test_otlp_wire.py b/python/frameworks/crewai/tests/test_otlp_wire.py index 3fa5d0a64..7f5216381 100644 --- a/python/frameworks/crewai/tests/test_otlp_wire.py +++ b/python/frameworks/crewai/tests/test_otlp_wire.py @@ -87,27 +87,11 @@ def shutdown(self): {"hide_input_text": True, "hide_output_messages": True}, ], ) -def test_real_otlp_http_wire(encoding, compressed, settings, fails): - captured = [] - - class Collector(BaseHTTPRequestHandler): - def do_POST(self): - data = self.rfile.read(int(self.headers["Content-Length"])) - captured.append( - ({key.lower(): value for key, value in self.headers.items()}, data) - ) - self.send_response(200) - self.send_header("Content-Length", "0") - self.end_headers() - - def log_message(self, *args): - pass - - server = HTTPServer(("127.0.0.1", 0), Collector) - thread = threading.Thread(target=server.serve_forever, daemon=True) - thread.start() +def test_real_otlp_http_wire( + encoding, compressed, settings, fails, otlp_collector, service_credentials +): + endpoint, captured = otlp_collector provider = TracerProvider(shutdown_on_exit=False) - endpoint = f"http://127.0.0.1:{server.server_port}/v1/traces" exporter = ( OTLPSpanExporter( endpoint=endpoint, @@ -122,14 +106,31 @@ def log_message(self, *args): provider.get_tracer("synthetic-wire"), _CrewAITraceConfig(TraceConfig(**settings)), ) - arguments = {"client": {"config": {"api_key": SECRET}}, "query": "useful query"} - result = {"Authorization": SECRET, "answer": "useful answer", "total_tokens": 13} + arguments = { + "client": {"config": {"api_key": SECRET}}, + "headers": service_credentials, + "query": "useful query", + } + result = { + "Authorization": SECRET, + "headers": service_credentials, + "answer": "useful answer", + "total_tokens": 13, + } error = RuntimeError(SECRET) def use(calling, tool_string): trace.get_current_span().add_event( "tool.log", - {"log.body": json.dumps({"secret": SECRET, "safe": "useful log"})}, + { + "log.body": json.dumps( + { + "secret": SECRET, + "headers": service_credentials, + "safe": "useful log", + } + ) + }, ) if fails: raise error @@ -161,6 +162,10 @@ def invoke(): assert (headers.get("content-encoding") == "gzip") is compressed body = gzip.decompress(raw) if compressed else raw assert SECRET.encode() not in body and SECRET.encode() not in raw + assert all( + value.encode() not in body and value.encode() not in raw + for value in service_credentials.values() + ) assert b"useful log" in body if settings.get("hide_inputs") or settings.get("hide_input_text"): assert b"useful query" not in body @@ -181,11 +186,122 @@ def invoke(): assert spans[0].status.code == (2 if fails else 1) finally: provider.shutdown() + + +@pytest.fixture +def otlp_collector(): + captured = [] + + class Collector(BaseHTTPRequestHandler): + def do_POST(self): + data = self.rfile.read(int(self.headers["Content-Length"])) + captured.append( + ({key.lower(): value for key, value in self.headers.items()}, data) + ) + self.send_response(200) + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, *args): + pass + + server = HTTPServer(("127.0.0.1", 0), Collector) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_port}/v1/traces", captured + finally: server.shutdown() server.server_close() thread.join(timeout=5) +@pytest.mark.parametrize("encoding", ["protobuf", "json"]) +@pytest.mark.parametrize("compressed", [False, True]) +@pytest.mark.parametrize( + "settings", + [ + {}, + {"hide_inputs": True}, + {"hide_outputs": True}, + {"hide_input_text": True, "hide_output_messages": True}, + ], +) +def test_native_service_credentials_never_reach_otlp_http( + encoding, + compressed, + settings, + otlp_collector, + native_tool_usage, + service_credentials, +): + from traceai_crewai import CrewAIInstrumentor + + endpoint, captured = otlp_collector + seen = [] + + def lookup(query: str, headers: dict) -> str: + """Return synthetic service credentials without altering application data.""" + seen.append((query, dict(headers))) + trace.get_current_span().add_event( + "tool.log", + {"log.body": json.dumps({"headers": headers, "message": "useful log"})}, + ) + return json.dumps( + {"headers": headers, "answer": "useful answer", "total_tokens": 13} + ) + + usage = native_tool_usage(lookup) + arguments = {"query": "useful query", "headers": dict(service_credentials)} + calling = ToolCalling(tool_name="lookup", arguments=arguments) + baseline = usage.use(calling, "lookup") + provider = TracerProvider(shutdown_on_exit=False) + exporter = ( + OTLPSpanExporter( + endpoint=endpoint, + timeout=5, + compression=Compression.Gzip if compressed else Compression.NoCompression, + ) + if encoding == "protobuf" + else JsonHTTPExporter(endpoint, compressed) + ) + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = CrewAIInstrumentor() + instrumentor.instrument(tracer_provider=provider, config=TraceConfig(**settings)) + try: + # Use a fresh ToolUsage to exclude native caching from the parity check. + actual = native_tool_usage(lookup).use(calling, "lookup") + assert actual == baseline + assert json.loads(actual)["headers"] == service_credentials + assert seen == [("useful query", service_credentials)] * 2 + assert calling.arguments == arguments + assert arguments["headers"] == service_credentials + assert provider.force_flush() and len(captured) == 1 + headers, raw = captured[0] + assert headers["content-type"] == ( + "application/json" if encoding == "json" else "application/x-protobuf" + ) + assert (headers.get("content-encoding") == "gzip") is compressed + body = gzip.decompress(raw) if compressed else raw + for credential in service_credentials.values(): + assert credential.encode() not in body and credential.encode() not in raw + assert b"ToolUsage._use" in body and b"useful log" in body + assert ( + b"total_tokens" in body + or settings.get("hide_outputs") + or settings.get("hide_output_messages") + ) + assert (b"useful query" not in body) is bool( + settings.get("hide_inputs") or settings.get("hide_input_text") + ) + assert (b"useful answer" not in body) is bool( + settings.get("hide_outputs") or settings.get("hide_output_messages") + ) + finally: + instrumentor.uninstrument() + provider.shutdown() + + def test_raw_negative_control_detects_unprojected_synthetic_credential(): # Do not send this deliberately unsafe baseline to even the local collector. exporter = InMemorySpanExporter() diff --git a/python/frameworks/crewai/tests/test_privacy_policy.py b/python/frameworks/crewai/tests/test_privacy_policy.py index a1fc95324..9b020ff1b 100644 --- a/python/frameworks/crewai/tests/test_privacy_policy.py +++ b/python/frameworks/crewai/tests/test_privacy_policy.py @@ -107,6 +107,24 @@ def tracer_for(config): "client.config.secrets.value", "apiSecret", "secret_token", + "X-Amz-Security-Token", + "X-Vault-Token", + "X-Consul-Token", + "X-Subject-Token", + "X-Api-Secret", + "X-Service-Token", + "security_token", + "aws_security_token", + "vault_token", + "consul_token", + "subject_token", + "service_token", + "headers.x-aMz-SeCuRiTy-ToKeN", + "headers[X-Vault-Token][0]", + "headers.X-Consul-Token.value", + "headers[X-Subject-Token]", + "headers.X-Api-Secret.0", + "client.config.security_token.value", ], ) def test_credential_spellings_and_business_fields(key): @@ -121,6 +139,12 @@ def test_credential_spellings_and_business_fields(key): "tokenizer": "safe", "max_tokens": 90, "token_id": "safe-id", + "security_token_id": "safe-security-id", + "vault_token_count": 3, + "consul_token_ttl": 3600, + "subject_token_type": "safe-type", + "service_token_count": 2, + "api_secret_id": "safe-secret-id", "id": "business-id", } before = copy.deepcopy(payload) @@ -135,14 +159,19 @@ def test_credential_spellings_and_business_fields(key): @pytest.mark.parametrize("flag", CONTENT_FLAGS) @pytest.mark.parametrize("enabled", [True, False]) -def test_every_directional_hide_flag_covers_aliases_and_events(flag, enabled, caplog): +def test_every_directional_hide_flag_covers_aliases_and_events( + flag, enabled, caplog, service_credentials +): tracer, exporter, provider = tracer_for(TraceConfig(**{flag: enabled})) attrs = { **{k: INPUT for k in INPUT_ALIASES}, **{k: OUTPUT for k in OUTPUT_ALIASES}, "headers.Authorization": SECRET, "gen_ai.usage.total_tokens": 42, - "log.body": json.dumps({"api_key": SECRET, "message": "useful log"}), + "log.body": json.dumps( + {"api_key": SECRET, "headers": service_credentials, "message": "useful log"} + ), + **{f"headers.{key}": value for key, value in service_credentials.items()}, } before = copy.deepcopy(attrs) with tracer.start_as_current_span("privacy", attributes=attrs) as span: @@ -152,6 +181,8 @@ def test_every_directional_hide_flag_covers_aliases_and_events(flag, enabled, ca wire = encode_spans(spans).SerializeToString() assert SECRET.encode() not in wire assert b"useful log" in wire and b"tool.result" in wire + for credential in service_credentials.values(): + assert credential.encode() not in wire and credential not in caplog.text assert SECRET not in caplog.text if enabled: hidden, visible = (INPUT, OUTPUT) if "input" in flag else (OUTPUT, INPUT) @@ -182,16 +213,23 @@ def test_every_directional_hide_flag_covers_aliases_and_events(flag, enabled, ca {"pii_redaction": True}, ], ) -def test_default_and_explicit_combinations_always_redact_structured_aliases(kwargs): +def test_default_and_explicit_combinations_always_redact_structured_aliases( + kwargs, service_credentials +): tracer, exporter, provider = tracer_for(TraceConfig(**kwargs)) structured = json.dumps( - {"client": {"config": {"api_key": SECRET}}, "safe": "business"} + { + "client": {"config": {"api_key": SECRET}}, + "headers": service_credentials, + "safe": "business", + } ) attrs = {k: structured for k in INPUT_ALIASES + OUTPUT_ALIASES} with tracer.start_as_current_span("privacy", attributes=attrs) as span: span.add_event("structured", attrs) wire = encode_spans(exporter.get_finished_spans()).SerializeToString() assert SECRET.encode() not in wire + assert all(value.encode() not in wire for value in service_credentials.values()) assert ( b"business" in wire or (kwargs.get("hide_inputs") and kwargs.get("hide_outputs")) diff --git a/python/frameworks/crewai/traceai_crewai/_privacy.py b/python/frameworks/crewai/traceai_crewai/_privacy.py index 0b5f5d077..5ca0f993b 100644 --- a/python/frameworks/crewai/traceai_crewai/_privacy.py +++ b/python/frameworks/crewai/traceai_crewai/_privacy.py @@ -57,6 +57,14 @@ "idtoken", "apitoken", "csrftoken", + # Specific service credential forms, not arbitrary '*token' fields: token + # counters, IDs, and tokenizer metadata must remain useful business data. + "securitytoken", + "vaulttoken", + "consultoken", + "subjecttoken", + "servicetoken", + "apisecret", "clientsecret", "clientsecretkey", "clientkey", From d280d4cae96dde075433756260e2825d0f340198 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Fri, 30 Oct 2026 23:42:53 -0700 Subject: [PATCH 3/5] fix(crewai): normalize credentials and redact structured header lists Replace vendor-specific credential names with normalized credential markers and explicit safe metadata exceptions. Project pair and name/value headers, serialized JSON, and flattened paths without mutating application data. Exercise generated spellings and real ToolUsage exports in JSON/protobuf plain/gzip at the supported floor and current CrewAI. Co-Authored-By: Claude Code --- python/frameworks/crewai/tests/conftest.py | 20 ++ .../crewai/tests/test_credential_patterns.py | 202 ++++++++++++++++++ .../frameworks/crewai/tests/test_otlp_wire.py | 32 ++- .../crewai/traceai_crewai/_privacy.py | 197 ++++++++++------- python/tests/test_framework_crewai.py | 4 +- 5 files changed, 376 insertions(+), 79 deletions(-) create mode 100644 python/frameworks/crewai/tests/test_credential_patterns.py diff --git a/python/frameworks/crewai/tests/conftest.py b/python/frameworks/crewai/tests/conftest.py index f842cbba1..3f8ef2292 100644 --- a/python/frameworks/crewai/tests/conftest.py +++ b/python/frameworks/crewai/tests/conftest.py @@ -44,6 +44,26 @@ def service_credentials(): "X-Api-Secret", "X-Service-Token", "security_token", + "X-Token", + "PRIVATE-TOKEN", + "private_token", + "X-Gitlab-Token", + "Circle-Token", + "app_secret", + "X-Auth", + "Authorization", + "X-API-Key", + "vendor-api.key", + "client[private][key]", + "sessionid", + "passphrase", + "signature", + "bearer", + "jwt", + "otp", + "otp_code", + "totp", + "pin", ) ) } diff --git a/python/frameworks/crewai/tests/test_credential_patterns.py b/python/frameworks/crewai/tests/test_credential_patterns.py new file mode 100644 index 000000000..64c99b48d --- /dev/null +++ b/python/frameworks/crewai/tests/test_credential_patterns.py @@ -0,0 +1,202 @@ +"""Generated structural credential regressions; synthetic values only.""" + +import copy +import itertools +import json + +import pytest +from fi_instrumentation import TraceConfig + +from traceai_crewai._privacy import ( + REDACTED, + _CrewAITraceConfig, + _credential_key, + _safe_data, +) + +SECRET = "FAKE_PATTERN_CREDENTIAL_251f03" +# Independent expectations, not generated from the production allowlist. +SAFE_NAMES = ( + "input_tokens", + "output_tokens", + "total_tokens", + "prompt_tokens", + "completion_tokens", + "cached_tokens", + "cached_input_tokens", + "cache_read_tokens", + "cache_write_tokens", + "input_tokens_details", + "output_tokens_details", + "prompt_tokens_details", + "completion_tokens_details", + "num_tokens", + "author_id", + "author_name", + "authorship", + "api_secret_id", + "reasoning_tokens", + "token_count", + "max_tokens", + "max_completion_tokens", + "tokens_used", + "token_usage", + "tokenizer", + "author", + "authors", + "authority", + "session_count", + "session_duration_ms", + "token_id", + "security_token_id", + "vault_token_count", + "consul_token_ttl", + "subject_token_type", + "service_token_count", + "cache_creation_input_tokens", + "cache_read_input_tokens", + "audio_tokens", + "tokenizer_name", + "tokenizer_version", + "id", + "project_id", + "shipping", + "opportunity", + "purchase_order", + "query", + "answer", + "business_data", +) + + +@pytest.mark.parametrize("prefix", ["", "X-", "x_", "vendor-", "X-Service-"]) +@pytest.mark.parametrize("case", [str.lower, str.upper, str.title]) +def test_generated_credential_matrix(prefix, case): + concepts = ( + ("token",), + ("security", "token"), + ("private", "token"), + ("gitlab", "token"), + ("circle", "token"), + ("api", "secret"), + ("app", "secret"), + ("password",), + ("passwd",), + ("passphrase",), + ("api", "key"), + ("access", "key"), + ("private", "key"), + ("credential",), + ("auth",), + ("authorization",), + ("authentication",), + ("cookie",), + ("session", "id"), + ("session", "token"), + ("signature",), + ("bearer",), + ("jwt",), + ("otp", "code"), + ("totp",), + ) + for words, separator in itertools.product( + concepts, ("", "-", "_", " ", "/", ":", ".") + ): + name = case(prefix + separator.join(words)) + assert _credential_key(name), name + payload = { + "headers": {name: SECRET, "Accept": "application/json"}, + f"client.config.{name}[0].value": SECRET, + "raw": json.dumps({name: SECRET, "answer": "useful answer"}), + "total_tokens": 42, + "id": "business-id", + } + before = copy.deepcopy(payload) + projected = _safe_data(payload) + assert SECRET not in json.dumps(projected), name + assert projected["headers"]["Accept"] == "application/json" + assert projected["total_tokens"] == 42 and projected["id"] == "business-id" + assert json.loads(projected["raw"])["answer"] == "useful answer" + assert payload == before + assert ( + _CrewAITraceConfig(TraceConfig()).mask( + f"client.config[{name}].value", SECRET + ) + == REDACTED + ) + + +@pytest.mark.parametrize("name", ["pin", "PIN", "x-pin", "X_OTP", "otp"]) +def test_short_credentials_match_whole_normalized_name(name): + assert _safe_data({name: SECRET}) == {name: REDACTED} + + +@pytest.mark.parametrize("name", SAFE_NAMES) +def test_explicit_metadata_exceptions_preserve_values_and_not_children(name): + for spelling in (name, name.upper(), name.replace("_", "-"), "X-" + name): + assert not _credential_key(spelling), spelling + payload = {spelling: 42, f"usage.{spelling}": 42, "id": "safe-id"} + assert _safe_data(payload) == payload + # An allowlisted parent cannot bypass inspection of its children. + assert SECRET not in json.dumps(_safe_data({spelling: {"api_key": SECRET}})) + assert _credential_key(f"{spelling}.api_key") + + +def test_unknown_vendor_token_metadata_is_not_a_generic_bypass(): + for name in ("private_token_count", "vendor_secret_id", "vendor_auth_type"): + assert _safe_data({name: SECRET})[name] == REDACTED + + +@pytest.mark.parametrize("shape", ["pairs", "tuples", "name", "key", "header"]) +def test_header_lists_and_serialized_lists(shape, service_credentials): + fields = list(service_credentials.items()) + [("Accept", "application/json")] + if shape == "pairs": + headers = [list(item) for item in fields] + elif shape == "tuples": + headers = fields + else: + headers = [ + {shape: key, "value": value, "description": "business data"} + for key, value in fields + ] + original = copy.deepcopy(headers) + projected = _safe_data({"headers": headers, "json": json.dumps(headers)}) + rendered = json.dumps(projected) + assert all(value not in rendered for value in service_credentials.values()) + assert "application/json" in rendered + assert "Accept" in rendered + assert headers == original + if shape not in ("pairs", "tuples"): + assert all( + item["description"] == "business data" for item in projected["headers"] + ) + for name in service_credentials: + assert name in rendered + + +@pytest.mark.parametrize("selector", ["name", "key", "header", "0"]) +@pytest.mark.parametrize("brackets", [False, True]) +def test_flattened_header_records(selector, brackets): + value_field = "1" if selector == "0" else "value" + + def key(index, field): + return ( + f"headers[{index}]['{field}']" if brackets else f"headers.{index}.{field}" + ) + + original = { + key(0, selector): "Authorization", + key(0, value_field): SECRET, + key(1, selector): "Accept", + key(1, value_field): "application/json", + "headers['X-Gitlab-Token'][0]": SECRET, + "usage.total_tokens": 42, + "request_id": "business-id", + } + before = copy.deepcopy(original) + safe = _safe_data(original) + assert SECRET not in json.dumps(safe) + assert safe[key(0, selector)] == "Authorization" + assert safe[key(1, value_field)] == "application/json" + assert safe["usage.total_tokens"] == 42 + assert original == before diff --git a/python/frameworks/crewai/tests/test_otlp_wire.py b/python/frameworks/crewai/tests/test_otlp_wire.py index 7f5216381..d824e95e0 100644 --- a/python/frameworks/crewai/tests/test_otlp_wire.py +++ b/python/frameworks/crewai/tests/test_otlp_wire.py @@ -1,11 +1,13 @@ """Actual owned-loopback HTTP captures, not a gzip round-trip of span.to_json().""" import base64 +import copy import gzip import json import threading from http.server import BaseHTTPRequestHandler, HTTPServer from urllib.request import ProxyHandler, Request, build_opener +from typing import Any import pytest from crewai.tools.tool_calling import ToolCalling @@ -227,7 +229,11 @@ def log_message(self, *args): {"hide_input_text": True, "hide_output_messages": True}, ], ) +@pytest.mark.parametrize( + "shape", ["dict", "pairs", "name", "key", "header", "serialized", "flattened"] +) def test_native_service_credentials_never_reach_otlp_http( + shape, encoding, compressed, settings, @@ -239,10 +245,24 @@ def test_native_service_credentials_never_reach_otlp_http( endpoint, captured = otlp_collector seen = [] + fields = {**service_credentials, "Accept": "application/json"} + if shape == "pairs": + tool_headers = [list(item) for item in fields.items()] + elif shape in ("name", "key", "header"): + tool_headers = [{shape: key, "value": value} for key, value in fields.items()] + elif shape == "serialized": + tool_headers = json.dumps( + [{"name": key, "value": value} for key, value in fields.items()] + ) + elif shape == "flattened": + tool_headers = {f"headers[{key}].value": value for key, value in fields.items()} + else: + tool_headers = fields + before = copy.deepcopy(tool_headers) - def lookup(query: str, headers: dict) -> str: + def lookup(query: str, headers: Any) -> str: """Return synthetic service credentials without altering application data.""" - seen.append((query, dict(headers))) + seen.append((query, copy.deepcopy(headers))) trace.get_current_span().add_event( "tool.log", {"log.body": json.dumps({"headers": headers, "message": "useful log"})}, @@ -252,7 +272,7 @@ def lookup(query: str, headers: dict) -> str: ) usage = native_tool_usage(lookup) - arguments = {"query": "useful query", "headers": dict(service_credentials)} + arguments = {"query": "useful query", "headers": tool_headers} calling = ToolCalling(tool_name="lookup", arguments=arguments) baseline = usage.use(calling, "lookup") provider = TracerProvider(shutdown_on_exit=False) @@ -272,10 +292,10 @@ def lookup(query: str, headers: dict) -> str: # Use a fresh ToolUsage to exclude native caching from the parity check. actual = native_tool_usage(lookup).use(calling, "lookup") assert actual == baseline - assert json.loads(actual)["headers"] == service_credentials - assert seen == [("useful query", service_credentials)] * 2 + assert json.loads(actual)["headers"] == before + assert seen == [("useful query", before)] * 2 assert calling.arguments == arguments - assert arguments["headers"] == service_credentials + assert arguments["headers"] == before assert provider.force_flush() and len(captured) == 1 headers, raw = captured[0] assert headers["content-type"] == ( diff --git a/python/frameworks/crewai/traceai_crewai/_privacy.py b/python/frameworks/crewai/traceai_crewai/_privacy.py index 5ca0f993b..818971e22 100644 --- a/python/frameworks/crewai/traceai_crewai/_privacy.py +++ b/python/frameworks/crewai/traceai_crewai/_privacy.py @@ -8,74 +8,85 @@ from fi_instrumentation import TraceConfig REDACTED = "__REDACTED__" -_CREDENTIAL_KEYS = frozenset( - { - "apikey", - "xapikey", - "authorization", - "proxyauthorization", - "accesstoken", - "refreshtoken", - "authtoken", - "authenticationtoken", - "bearertoken", - "sessiontoken", - "idtoken", - "apitoken", - "csrftoken", - "clientsecret", - "clientsecretkey", - "clientkey", - "secretkey", - "secretaccesskey", - "accesskey", - "accesskeyid", - "secret", - "secrets", - "apisecret", - "secrettoken", - "privatekey", - "signingkey", - "password", - "passwd", - "token", - "credential", - "credentials", - "cookie", - "setcookie", - } -) -_CREDENTIAL_SUFFIXES = ( +# Match credential concepts, not a finite list of vendor header spellings. +_CREDENTIAL_MARKERS = ( + "token", + "secret", + "password", + "passwd", + "passphrase", "apikey", - "authorization", - "accesstoken", - "refreshtoken", - "authtoken", - "authenticationtoken", - "bearertoken", - "sessiontoken", - "idtoken", - "apitoken", - "csrftoken", - # Specific service credential forms, not arbitrary '*token' fields: token - # counters, IDs, and tokenizer metadata must remain useful business data. - "securitytoken", - "vaulttoken", - "consultoken", - "subjecttoken", - "servicetoken", - "apisecret", - "clientsecret", - "clientsecretkey", - "clientkey", - "secretkey", - "secretaccesskey", "accesskey", - "accesskeyid", "privatekey", + "credential", + "auth", + "cookie", + "session", + "signature", + "bearer", + "jwt", + "otp", + "clientkey", "signingkey", - "password", - "passwd", +) +# Exact normalized exceptions only: never exempt arbitrary '*_count'/'*_id' +# names (e.g. private_token_count could still carry a credential). This documents +# the supported non-secret usage counters, model limits, tokenizer metadata, +# authorship, and service-token metadata. Extend deliberately with regressions. +_NON_SECRET_NAMES = frozenset( + { + "inputtokens", + "outputtokens", + "totaltokens", + "prompttokens", + "completiontokens", + "cachedtokens", + "cachedinputtokens", + "cachereadtokens", + "cachewritetokens", + "inputtokensdetails", + "outputtokensdetails", + "prompttokensdetails", + "completiontokensdetails", + "numtokens", + "cachecreationinputtokens", + "cachereadinputtokens", + "reasoningtokens", + "audiotokens", + "tokencount", + "inputtokencount", + "outputtokencount", + "totaltokencount", + "prompttokencount", + "completiontokencount", + "cachedtokencount", + "reasoningtokencount", + "maxtokens", + "maxinputtokens", + "maxoutputtokens", + "maxcompletiontokens", + "tokensused", + "tokenusage", + "tokenizer", + "tokenizername", + "tokenizerversion", + "tokenid", + "securitytokenid", + "apisecretid", + "vaulttokencount", + "consultokenttl", + "subjecttokentype", + "servicetokencount", + "author", + "authors", + "authorid", + "authorname", + "authorship", + "authority", + "sessioncount", + "sessionduration", + "sessiondurationms", + } ) _INPUT_ALIASES = ( "input.value", @@ -103,15 +114,32 @@ ) +def _normalized_name(name: str) -> str: + name = re.sub(r"^x[\W_]+", "", name.lower().strip(" '\"")) + return re.sub(r"[^a-z0-9]", "", name) + + def _credential_key(key: str) -> bool: - # Do not classify every field containing 'token': usage and IDs are useful. - # Flattened paths may put a value/index below a credential container, e.g. - # headers.Authorization.0 or client.config.api_key.value. - for part in re.split(r"[.\[\]]+", key.lower()): - normalized = re.sub(r"[^a-z0-9]", "", part) - if normalized in _CREDENTIAL_KEYS or normalized.endswith(_CREDENTIAL_SUFFIXES): + # Inspect every path component, including credential containers followed by + # a value/index. Exceptions apply to one complete field, never its children. + if _normalized_name(key) in _NON_SECRET_NAMES: + return False + parts = [_normalized_name(part) for part in re.split(r"[.\[\]]+", key) if part] + for normalized in parts: + if normalized in _NON_SECRET_NAMES: + continue + # PIN is whole-name only, so e.g. 'shipping' is not a credential. + if normalized == "pin" or any( + marker in normalized for marker in _CREDENTIAL_MARKERS + ): return True - return False + # A dot/bracket can separate words of a credential name as well as paths, + # e.g. headers.api.key or client[private][key]. + return any( + marker in left + right + for left, right in zip(parts, parts[1:]) + for marker in ("apikey", "accesskey", "privatekey", "clientkey", "signingkey") + ) def _safe_data(value: Any, depth: int = 0) -> Any: @@ -134,12 +162,37 @@ def _safe_data(value: Any, depth: int = 0) -> Any: if value is None or type(value) in (int, float, bool): return value if type(value) is dict: + # Name/value header records, including their flattened representations. + # Keep the selector and unrelated fields; redact only the paired value. + def path(key: str) -> tuple: + return tuple( + _normalized_name(part) for part in re.split(r"[.\[\]]+", key) if part + ) + + credential_values = set() + for key, item in value.items(): + if type(key) is not str or type(item) is not str: + continue + parts = path(key) + if ( + parts + and parts[-1] in ("name", "key", "header", "0") + and _credential_key(item) + ): + paired_field = "1" if parts[-1] == "0" else "value" + credential_values.add(parts[:-1] + (paired_field,)) return { - key: REDACTED if _credential_key(key) else _safe_data(item, depth + 1) + key: ( + REDACTED + if _credential_key(key) or path(key) in credential_values + else _safe_data(item, depth + 1) + ) for key, item in value.items() if type(key) is str } if type(value) in (list, tuple): + if len(value) == 2 and type(value[0]) is str and _credential_key(value[0]): + return [value[0], REDACTED] return [_safe_data(item, depth + 1) for item in value] return "[omitted runtime object]" diff --git a/python/tests/test_framework_crewai.py b/python/tests/test_framework_crewai.py index ea6d2cf7b..5947b35ad 100644 --- a/python/tests/test_framework_crewai.py +++ b/python/tests/test_framework_crewai.py @@ -230,7 +230,9 @@ def test_context_attributes(self, mock_get_attrs): # Verify context attributes are set self.mock_span.set_attributes.assert_called_once() call_args = self.mock_span.set_attributes.call_args[0][0] - assert call_args["session_id"] == "test_session" + # Session identifiers can be session credentials under the structural + # policy; unrelated user/business identifiers remain available. + assert call_args["session_id"] == "__REDACTED__" assert call_args["user_id"] == "test_user" From e664853559e8f32a3e98664b3951241a0e59501f Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sat, 31 Oct 2026 00:31:10 -0700 Subject: [PATCH 4/5] fix(crewai): project flattened header batches and preserve business metadata Cover subscription, application and master key credentials; retain provider token counters and ordinary business lists. Apply complete header record projection in the CrewAI-local tracer before scalar core policy without changing global spans or lazy value semantics. Co-Authored-By: Claude Code --- python/frameworks/crewai/tests/conftest.py | 6 + .../crewai/tests/test_batch_projection.py | 101 +++++++++ .../crewai/tests/test_credential_patterns.py | 58 ++++- .../frameworks/crewai/tests/test_otlp_wire.py | 199 ++++++++++++++++-- .../crewai/traceai_crewai/__init__.py | 5 +- .../crewai/traceai_crewai/_privacy.py | 151 +++++++++---- .../crewai/traceai_crewai/_tracer.py | 89 ++++++++ 7 files changed, 547 insertions(+), 62 deletions(-) create mode 100644 python/frameworks/crewai/tests/test_batch_projection.py create mode 100644 python/frameworks/crewai/traceai_crewai/_tracer.py diff --git a/python/frameworks/crewai/tests/conftest.py b/python/frameworks/crewai/tests/conftest.py index 3f8ef2292..6e3c4e581 100644 --- a/python/frameworks/crewai/tests/conftest.py +++ b/python/frameworks/crewai/tests/conftest.py @@ -49,6 +49,12 @@ def service_credentials(): "private_token", "X-Gitlab-Token", "Circle-Token", + "Ocp-Apim-Subscription-Key", + "subscription_key", + "DD-APPLICATION-KEY", + "application_key", + "X-Parse-Master-Key", + "master_key", "app_secret", "X-Auth", "Authorization", diff --git a/python/frameworks/crewai/tests/test_batch_projection.py b/python/frameworks/crewai/tests/test_batch_projection.py new file mode 100644 index 000000000..60cb8393a --- /dev/null +++ b/python/frameworks/crewai/tests/test_batch_projection.py @@ -0,0 +1,101 @@ +"""CrewAI-local batch adapter contracts; synthetic credentials only.""" + +import copy +import json +from types import MappingProxyType + +import pytest +from fi_instrumentation import TraceConfig +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, +) +from opentelemetry.sdk.trace.sampling import ALWAYS_ON, Sampler +from opentelemetry.trace import Link, SpanContext + +from traceai_crewai._privacy import REDACTED, _CrewAITraceConfig +from traceai_crewai._tracer import _CrewAITracer + +SECRET = "FAKE_BATCH_CREDENTIAL_7589a2" + + +@pytest.mark.parametrize("hide_inputs", [False, True]) +def test_batch_adapter_retains_core_policy_lazy_values_and_lifecycle( + hide_inputs, +): + observations = [] + + class InspectSampler(Sampler): + def should_sample(self, *args, **kwargs): + attributes = kwargs.get("attributes", args[4] if len(args) > 4 else {}) + links = kwargs.get("links", args[5] if len(args) > 5 else ()) + observations.append( + { + "attributes": dict(attributes), + "links": [dict(link.attributes) for link in links], + } + ) + return ALWAYS_ON.should_sample(*args, **kwargs) + + def get_description(self): + return "synthetic-batch-policy" + + provider = TracerProvider(sampler=InspectSampler(), shutdown_on_exit=False) + exporter = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + tracer = _CrewAITracer( + provider.get_tracer("batch"), + _CrewAITraceConfig(TraceConfig(hide_inputs=hide_inputs)), + ) + attributes = { + # Value first: never depend on selector iteration order. + "headers[0]['value']": SECRET, + "headers[0]['name']": "Authorization", + "headers[1]['value']": "application/json", + "headers[1]['name']": "Accept", + "request_id": "business-id", + } + before = copy.deepcopy(attributes) + calls = [] + + def lazy_input(): + calls.append("input") + return "useful input" + + link_context = SpanContext(trace_id=1, span_id=2, is_remote=False) + try: + with tracer.start_span( + "batch", + attributes={**attributes, "input.value": lazy_input}, + links=[Link(link_context, MappingProxyType(attributes))], + ) as span: + span.set_attributes(MappingProxyType(attributes)) + span.set_attributes({"business.lazy": lambda: "useful lazy"}) + span.add_event("batch", MappingProxyType(attributes)) + span.add_link(link_context, MappingProxyType(attributes)) + span.record_exception( + RuntimeError("synthetic failure"), attributes + ) + assert attributes == before + assert calls == ([] if hide_inputs else ["input"]) + assert SECRET not in json.dumps(observations) + assert observations[0]["attributes"]["headers[0]['value']"] == REDACTED + assert observations[0]["links"][0]["headers[0]['value']"] == REDACTED + spans = exporter.get_finished_spans() + assert len(spans) == 1 + result = spans[0] + assert result.attributes["business.lazy"] == "useful lazy" + assert result.attributes["input.value"] == ( + REDACTED if hide_inputs else "useful input" + ) + for batch in ( + result.attributes, + *(event.attributes for event in result.events), + *(link.attributes for link in result.links), + ): + assert batch["headers[0]['value']"] == REDACTED + assert batch["headers[1]['value']"] == "application/json" + assert batch["request_id"] == "business-id" + finally: + provider.shutdown() diff --git a/python/frameworks/crewai/tests/test_credential_patterns.py b/python/frameworks/crewai/tests/test_credential_patterns.py index 64c99b48d..9dd49325e 100644 --- a/python/frameworks/crewai/tests/test_credential_patterns.py +++ b/python/frameworks/crewai/tests/test_credential_patterns.py @@ -36,6 +36,12 @@ "authorship", "api_secret_id", "reasoning_tokens", + "accepted_prediction_tokens", + "rejected_prediction_tokens", + "candidates_token_count", + "thoughts_token_count", + "cached_content_token_count", + "max_new_tokens", "token_count", "max_tokens", "max_completion_tokens", @@ -86,6 +92,12 @@ def test_generated_credential_matrix(prefix, case): ("api", "key"), ("access", "key"), ("private", "key"), + ("subscription", "key"), + ("application", "key"), + ("master", "key"), + ("ocp", "apim", "subscription", "key"), + ("dd", "application", "key"), + ("parse", "master", "key"), ("credential",), ("auth",), ("authorization",), @@ -115,7 +127,10 @@ def test_generated_credential_matrix(prefix, case): projected = _safe_data(payload) assert SECRET not in json.dumps(projected), name assert projected["headers"]["Accept"] == "application/json" - assert projected["total_tokens"] == 42 and projected["id"] == "business-id" + assert ( + projected["total_tokens"] == 42 + and projected["id"] == "business-id" + ) assert json.loads(projected["raw"])["answer"] == "useful answer" assert payload == before assert ( @@ -138,18 +153,26 @@ def test_explicit_metadata_exceptions_preserve_values_and_not_children(name): payload = {spelling: 42, f"usage.{spelling}": 42, "id": "safe-id"} assert _safe_data(payload) == payload # An allowlisted parent cannot bypass inspection of its children. - assert SECRET not in json.dumps(_safe_data({spelling: {"api_key": SECRET}})) + assert SECRET not in json.dumps( + _safe_data({spelling: {"api_key": SECRET}}) + ) assert _credential_key(f"{spelling}.api_key") def test_unknown_vendor_token_metadata_is_not_a_generic_bypass(): - for name in ("private_token_count", "vendor_secret_id", "vendor_auth_type"): + for name in ( + "private_token_count", + "vendor_secret_id", + "vendor_auth_type", + ): assert _safe_data({name: SECRET})[name] == REDACTED @pytest.mark.parametrize("shape", ["pairs", "tuples", "name", "key", "header"]) def test_header_lists_and_serialized_lists(shape, service_credentials): - fields = list(service_credentials.items()) + [("Accept", "application/json")] + fields = list(service_credentials.items()) + [ + ("Accept", "application/json") + ] if shape == "pairs": headers = [list(item) for item in fields] elif shape == "tuples": @@ -168,7 +191,8 @@ def test_header_lists_and_serialized_lists(shape, service_credentials): assert headers == original if shape not in ("pairs", "tuples"): assert all( - item["description"] == "business data" for item in projected["headers"] + item["description"] == "business data" + for item in projected["headers"] ) for name in service_credentials: assert name in rendered @@ -181,7 +205,9 @@ def test_flattened_header_records(selector, brackets): def key(index, field): return ( - f"headers[{index}]['{field}']" if brackets else f"headers.{index}.{field}" + f"headers[{index}]['{field}']" + if brackets + else f"headers.{index}.{field}" ) original = { @@ -200,3 +226,23 @@ def key(index, field): assert safe[key(1, value_field)] == "application/json" assert safe["usage.total_tokens"] == 42 assert original == before + + +@pytest.mark.parametrize( + "first", ["authentication", "tokenization", "session", "author"] +) +def test_two_item_business_lists_are_not_header_pairs(first): + payload = { + "topics": [first, "billing"], + "categories": (first, "shipping"), + "serialized": json.dumps([first, "business data"]), + "total_tokens": 42, + } + before = copy.deepcopy(payload) + expected = {**payload, "categories": [first, "shipping"]} + assert _safe_data(payload) == expected + assert payload == before + # Recognizable pair collections still work, including bare serialized JSON. + for headers in ([["Authorization", SECRET]], [("X-API-Key", SECRET)]): + assert SECRET not in json.dumps(_safe_data(headers)) + assert SECRET not in _safe_data(json.dumps(headers)) diff --git a/python/frameworks/crewai/tests/test_otlp_wire.py b/python/frameworks/crewai/tests/test_otlp_wire.py index d824e95e0..d7d3a00ec 100644 --- a/python/frameworks/crewai/tests/test_otlp_wire.py +++ b/python/frameworks/crewai/tests/test_otlp_wire.py @@ -16,7 +16,9 @@ from opentelemetry import trace from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans from opentelemetry.exporter.otlp.proto.http import Compression -from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( + OTLPSpanExporter, +) from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ( ExportTraceServiceRequest, ) @@ -26,7 +28,9 @@ SpanExporter, SpanExportResult, ) -from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, +) from traceai_crewai._privacy import _CrewAITraceConfig from traceai_crewai._wrappers import _ToolUseWrapper @@ -98,7 +102,9 @@ def test_real_otlp_http_wire( OTLPSpanExporter( endpoint=endpoint, timeout=5, - compression=Compression.Gzip if compressed else Compression.NoCompression, + compression=( + Compression.Gzip if compressed else Compression.NoCompression + ), ) if encoding == "protobuf" else JsonHTTPExporter(endpoint, compressed) @@ -159,7 +165,9 @@ def invoke(): assert len(captured) == 1 headers, raw = captured[0] assert headers["content-type"] == ( - "application/json" if encoding == "json" else "application/x-protobuf" + "application/json" + if encoding == "json" + else "application/x-protobuf" ) assert (headers.get("content-encoding") == "gzip") is compressed body = gzip.decompress(raw) if compressed else raw @@ -171,7 +179,9 @@ def invoke(): assert b"useful log" in body if settings.get("hide_inputs") or settings.get("hide_input_text"): assert b"useful query" not in body - if settings.get("hide_outputs") or settings.get("hide_output_messages"): + if settings.get("hide_outputs") or settings.get( + "hide_output_messages" + ): assert b"useful answer" not in body if not settings: assert b"useful query" in body @@ -198,7 +208,13 @@ class Collector(BaseHTTPRequestHandler): def do_POST(self): data = self.rfile.read(int(self.headers["Content-Length"])) captured.append( - ({key.lower(): value for key, value in self.headers.items()}, data) + ( + { + key.lower(): value + for key, value in self.headers.items() + }, + data, + ) ) self.send_response(200) self.send_header("Content-Length", "0") @@ -230,7 +246,8 @@ def log_message(self, *args): ], ) @pytest.mark.parametrize( - "shape", ["dict", "pairs", "name", "key", "header", "serialized", "flattened"] + "shape", + ["dict", "pairs", "name", "key", "header", "serialized", "flattened"], ) def test_native_service_credentials_never_reach_otlp_http( shape, @@ -249,13 +266,17 @@ def test_native_service_credentials_never_reach_otlp_http( if shape == "pairs": tool_headers = [list(item) for item in fields.items()] elif shape in ("name", "key", "header"): - tool_headers = [{shape: key, "value": value} for key, value in fields.items()] + tool_headers = [ + {shape: key, "value": value} for key, value in fields.items() + ] elif shape == "serialized": tool_headers = json.dumps( [{"name": key, "value": value} for key, value in fields.items()] ) elif shape == "flattened": - tool_headers = {f"headers[{key}].value": value for key, value in fields.items()} + tool_headers = { + f"headers[{key}].value": value for key, value in fields.items() + } else: tool_headers = fields before = copy.deepcopy(tool_headers) @@ -265,7 +286,11 @@ def lookup(query: str, headers: Any) -> str: seen.append((query, copy.deepcopy(headers))) trace.get_current_span().add_event( "tool.log", - {"log.body": json.dumps({"headers": headers, "message": "useful log"})}, + { + "log.body": json.dumps( + {"headers": headers, "message": "useful log"} + ) + }, ) return json.dumps( {"headers": headers, "answer": "useful answer", "total_tokens": 13} @@ -280,14 +305,18 @@ def lookup(query: str, headers: Any) -> str: OTLPSpanExporter( endpoint=endpoint, timeout=5, - compression=Compression.Gzip if compressed else Compression.NoCompression, + compression=( + Compression.Gzip if compressed else Compression.NoCompression + ), ) if encoding == "protobuf" else JsonHTTPExporter(endpoint, compressed) ) provider.add_span_processor(SimpleSpanProcessor(exporter)) instrumentor = CrewAIInstrumentor() - instrumentor.instrument(tracer_provider=provider, config=TraceConfig(**settings)) + instrumentor.instrument( + tracer_provider=provider, config=TraceConfig(**settings) + ) try: # Use a fresh ToolUsage to exclude native caching from the parity check. actual = native_tool_usage(lookup).use(calling, "lookup") @@ -299,12 +328,17 @@ def lookup(query: str, headers: Any) -> str: assert provider.force_flush() and len(captured) == 1 headers, raw = captured[0] assert headers["content-type"] == ( - "application/json" if encoding == "json" else "application/x-protobuf" + "application/json" + if encoding == "json" + else "application/x-protobuf" ) assert (headers.get("content-encoding") == "gzip") is compressed body = gzip.decompress(raw) if compressed else raw for credential in service_credentials.values(): - assert credential.encode() not in body and credential.encode() not in raw + assert ( + credential.encode() not in body + and credential.encode() not in raw + ) assert b"ToolUsage._use" in body and b"useful log" in body assert ( b"total_tokens" in body @@ -315,8 +349,143 @@ def lookup(query: str, headers: Any) -> str: settings.get("hide_inputs") or settings.get("hide_input_text") ) assert (b"useful answer" not in body) is bool( - settings.get("hide_outputs") or settings.get("hide_output_messages") + settings.get("hide_outputs") + or settings.get("hide_output_messages") + ) + finally: + instrumentor.uninstrument() + provider.shutdown() + + +@pytest.mark.parametrize("encoding", ["protobuf", "json"]) +@pytest.mark.parametrize("compressed", [False, True]) +@pytest.mark.parametrize("scenario", ["flat-attributes", "business-data"]) +def test_native_batch_projection_and_business_parity( + encoding, compressed, scenario, otlp_collector, native_tool_usage +): + from traceai_crewai import CrewAIInstrumentor + + # Independent, realistic OpenAI prediction details / Gemini usage / HF limit. + payload = { + "usage": { + "completion_tokens_details": { + "accepted_prediction_tokens": 3, + "rejected_prediction_tokens": 5, + }, + "candidates_token_count": 7, + "thoughts_token_count": 11, + "cached_content_token_count": 13, + "total_tokens": 42, + }, + "max_new_tokens": 64, + "topics": ["authentication", "billing"], + "categories": ["tokenization", "shipping"], + "id": "business-id", + } + before = copy.deepcopy(payload) + flat = {"request.id": "useful batch id", "usage.total_tokens": 42} + for selector in ("name", "key", "header", "0"): + value_field = "1" if selector == "0" else "value" + for bracketed in (False, True): + for reverse in (False, True): + root = f"request.headers.{selector}.{int(bracketed)}.{int(reverse)}" + + def path(index, field): + return ( + f"{root}[{index}]['{field}']" + if bracketed + else f"{root}.{index}.{field}" + ) + + records = [ + (path(0, selector), "Authorization"), + (path(0, value_field), SECRET), + (path(1, selector), "Accept"), + (path(1, value_field), "application/json"), + ] + flat.update(reversed(records) if reverse else records) + flat_before = copy.deepcopy(flat) + + def lookup(data: Any) -> str: + """Exercise the actual native ToolUsage and its current span.""" + span = trace.get_current_span() + if scenario == "flat-attributes": + span.set_attributes(flat) + span.add_event("batch.log", flat) + span.add_event("structured.log", {"log.body": json.dumps(data)}) + return json.dumps({"answer": "useful answer", **data}) + + calling = ToolCalling(tool_name="lookup", arguments={"data": payload}) + baseline = native_tool_usage(lookup).use(calling, "lookup") + endpoint, captured = otlp_collector + provider = TracerProvider(shutdown_on_exit=False) + exporter = ( + OTLPSpanExporter( + endpoint=endpoint, + timeout=5, + compression=( + Compression.Gzip if compressed else Compression.NoCompression + ), + ) + if encoding == "protobuf" + else JsonHTTPExporter(endpoint, compressed) + ) + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = CrewAIInstrumentor() + instrumentor.instrument(tracer_provider=provider, config=TraceConfig()) + try: + assert native_tool_usage(lookup).use(calling, "lookup") == baseline + assert payload == before and flat == flat_before + assert provider.force_flush() and len(captured) == 1 + headers, raw = captured[0] + assert (headers.get("content-encoding") == "gzip") is compressed + body = gzip.decompress(raw) if compressed else raw + assert SECRET.encode() not in body and b"useful answer" in body + decoded = ( + json.loads(body) + if encoding == "json" + else MessageToDict( + ExportTraceServiceRequest.FromString(body), + use_integers_for_enums=True, + ) ) + spans = [ + span + for resource in decoded["resourceSpans"] + for scope in resource["scopeSpans"] + for span in scope["spans"] + ] + assert len(spans) == 1 + + def attributes(items): + return { + item["key"]: item["value"].get( + "stringValue", item["value"].get("intValue") + ) + for item in items + } + + attrs = attributes(spans[0]["attributes"]) + output = json.loads(attrs["output.value"]) + if isinstance(output, str): + output = json.loads(output) + assert output == {"answer": "useful answer", **before} + assert json.loads(attrs["input.value"])["tool_args"]["data"] == before + events = { + event["name"]: attributes(event["attributes"]) + for event in spans[0]["events"] + } + assert json.loads(events["structured.log"]["log.body"]) == before + if scenario == "flat-attributes": + expected = { + key: "__REDACTED__" if value == SECRET else value + for key, value in flat.items() + } + for batch in (attrs, events["batch.log"]): + for key, value in expected.items(): + assert batch[key] == ( + str(value) if type(value) is int else value + ) finally: instrumentor.uninstrument() provider.shutdown() diff --git a/python/frameworks/crewai/traceai_crewai/__init__.py b/python/frameworks/crewai/traceai_crewai/__init__.py index 541b448c3..08b1dfc4a 100644 --- a/python/frameworks/crewai/traceai_crewai/__init__.py +++ b/python/frameworks/crewai/traceai_crewai/__init__.py @@ -2,7 +2,7 @@ from importlib import import_module from typing import Any, Collection -from fi_instrumentation import FITracer, TraceConfig +from fi_instrumentation import TraceConfig from opentelemetry import trace as trace_api from opentelemetry.instrumentation.instrumentor import BaseInstrumentor # type: ignore from traceai_crewai._wrappers import ( @@ -11,6 +11,7 @@ _ToolUseWrapper, ) from traceai_crewai._privacy import _CrewAITraceConfig +from traceai_crewai._tracer import _CrewAITracer from traceai_crewai._telemetry import _NativeTelemetryGuard from traceai_crewai.version import __version__ from wrapt import wrap_function_wrapper @@ -39,7 +40,7 @@ def _instrument(self, **kwargs: Any) -> None: config = TraceConfig() else: assert isinstance(config, TraceConfig) - self._tracer = FITracer( + self._tracer = _CrewAITracer( trace_api.get_tracer(__name__, __version__, tracer_provider), config=_CrewAITraceConfig(config), ) diff --git a/python/frameworks/crewai/traceai_crewai/_privacy.py b/python/frameworks/crewai/traceai_crewai/_privacy.py index 818971e22..3983a908b 100644 --- a/python/frameworks/crewai/traceai_crewai/_privacy.py +++ b/python/frameworks/crewai/traceai_crewai/_privacy.py @@ -9,15 +9,22 @@ REDACTED = "__REDACTED__" # Match credential concepts, not a finite list of vendor header spellings. -_CREDENTIAL_MARKERS = ( +_KEY_CREDENTIAL_MARKERS = ( + "apikey", + "accesskey", + "privatekey", + "clientkey", + "signingkey", + "subscriptionkey", + "applicationkey", + "masterkey", +) +_CREDENTIAL_MARKERS = _KEY_CREDENTIAL_MARKERS + ( "token", "secret", "password", "passwd", "passphrase", - "apikey", - "accesskey", - "privatekey", "credential", "auth", "cookie", @@ -26,8 +33,6 @@ "bearer", "jwt", "otp", - "clientkey", - "signingkey", ) # Exact normalized exceptions only: never exempt arbitrary '*_count'/'*_id' # names (e.g. private_token_count could still carry a credential). This documents @@ -52,6 +57,13 @@ "cachecreationinputtokens", "cachereadinputtokens", "reasoningtokens", + # OpenAI prediction details, Gemini usage, and generation limits. + "acceptedpredictiontokens", + "rejectedpredictiontokens", + "candidatestokencount", + "thoughtstokencount", + "cachedcontenttokencount", + "maxnewtokens", "audiotokens", "tokencount", "inputtokencount", @@ -124,7 +136,9 @@ def _credential_key(key: str) -> bool: # a value/index. Exceptions apply to one complete field, never its children. if _normalized_name(key) in _NON_SECRET_NAMES: return False - parts = [_normalized_name(part) for part in re.split(r"[.\[\]]+", key) if part] + parts = [ + _normalized_name(part) for part in re.split(r"[.\[\]]+", key) if part + ] for normalized in parts: if normalized in _NON_SECRET_NAMES: continue @@ -138,12 +152,58 @@ def _credential_key(key: str) -> bool: return any( marker in left + right for left, right in zip(parts, parts[1:]) - for marker in ("apikey", "accesskey", "privatekey", "clientkey", "signingkey") + for marker in _KEY_CREDENTIAL_MARKERS + ) + + +def _field_path(key: str) -> tuple: + return tuple( + _normalized_name(part) for part in re.split(r"[.\[\]]+", key) if part ) -def _safe_data(value: Any, depth: int = 0) -> Any: - """Copy only plain business data, never calling runtime serializers or repr.""" +def _is_headers_field(key: str) -> bool: + parts = _field_path(key) + return bool(parts and parts[-1].endswith("headers")) + + +def _redact_header_records(value: dict) -> dict: + """Shallow projection of complete selector/value records, in any order. + + Leave unrelated values (including core's lazy callbacks) unevaluated. The + caller's per-attribute mask remains responsible for individual values. + """ + credential_values = set() + for key, item in value.items(): + if type(key) is not str or type(item) is not str: + continue + parts = _field_path(key) + if ( + parts + and parts[-1] in ("name", "key", "header", "0") + and _credential_key(item) + ): + paired_field = "1" if parts[-1] == "0" else "value" + credential_values.add(parts[:-1] + (paired_field,)) + return { + key: ( + REDACTED + if type(key) is str and _field_path(key) in credential_values + else item + ) + for key, item in value.items() + } + + +def _safe_data( + value: Any, depth: int = 0, *, _header_context: bool = False +) -> Any: + """Copy plain data; interpret pairs only inside a header collection. + + Header context comes from a headers field or a collection of two-item pairs, + including serialized collections. A standalone two-string business list is + not a header just because its first element mentions authentication/tokens. + """ if depth > 20: return "[omitted]" if type(value) is str: @@ -157,48 +217,57 @@ def _safe_data(value: Any, depth: int = 0) -> Any: # inspect (including valid JSON beyond parser resource limits). return "[omitted unparseable structured data]" if type(parsed) in (dict, list): - return json.dumps(_safe_data(parsed, depth + 1)) + return json.dumps( + _safe_data( + parsed, depth + 1, _header_context=_header_context + ) + ) return value if value is None or type(value) in (int, float, bool): return value if type(value) is dict: - # Name/value header records, including their flattened representations. - # Keep the selector and unrelated fields; redact only the paired value. - def path(key: str) -> tuple: - return tuple( - _normalized_name(part) for part in re.split(r"[.\[\]]+", key) if part - ) - - credential_values = set() - for key, item in value.items(): - if type(key) is not str or type(item) is not str: - continue - parts = path(key) - if ( - parts - and parts[-1] in ("name", "key", "header", "0") - and _credential_key(item) - ): - paired_field = "1" if parts[-1] == "0" else "value" - credential_values.add(parts[:-1] + (paired_field,)) return { key: ( REDACTED - if _credential_key(key) or path(key) in credential_values - else _safe_data(item, depth + 1) + if _credential_key(key) + else _safe_data( + item, + depth + 1, + _header_context=_is_headers_field(key), + ) ) - for key, item in value.items() + for key, item in _redact_header_records(value).items() if type(key) is str } if type(value) in (list, tuple): - if len(value) == 2 and type(value[0]) is str and _credential_key(value[0]): + if ( + _header_context + and len(value) == 2 + and type(value[0]) is str + and _credential_key(value[0]) + ): return [value[0], REDACTED] - return [_safe_data(item, depth + 1) for item in value] + pair_collection = bool(value) and all( + type(item) in (list, tuple) + and len(item) == 2 + and type(item[0]) is str + for item in value + ) + return [ + _safe_data( + item, + depth + 1, + _header_context=pair_collection or _header_context, + ) + for item in value + ] return "[omitted runtime object]" def _has_prefix(key: str, prefixes: tuple) -> bool: - return any(key == prefix or key.startswith(prefix + ".") for prefix in prefixes) + return any( + key == prefix or key.startswith(prefix + ".") for prefix in prefixes + ) class _CrewAITraceConfig(TraceConfig): @@ -215,10 +284,14 @@ def __init__(self, config: TraceConfig) -> None: def mask(self, key: str, value: Any) -> Any: hide_input = ( - self.hide_inputs or self.hide_input_text or self.hide_input_messages + self.hide_inputs + or self.hide_input_text + or self.hide_input_messages ) hide_output = ( - self.hide_outputs or self.hide_output_text or self.hide_output_messages + self.hide_outputs + or self.hide_output_text + or self.hide_output_messages ) if (hide_input and _has_prefix(key, _INPUT_ALIASES)) or ( hide_output and _has_prefix(key, _OUTPUT_ALIASES) @@ -229,7 +302,7 @@ def mask(self, key: str, value: Any) -> Any: return None if _credential_key(key): return REDACTED - projected = _safe_data(masked) + projected = _safe_data(masked, _header_context=_is_headers_field(key)) # OTel attributes cannot carry dictionaries (nor lists of dictionaries). if type(projected) is dict or ( type(projected) is list diff --git a/python/frameworks/crewai/traceai_crewai/_tracer.py b/python/frameworks/crewai/traceai_crewai/_tracer.py new file mode 100644 index 000000000..ee60cebb5 --- /dev/null +++ b/python/frameworks/crewai/traceai_crewai/_tracer.py @@ -0,0 +1,89 @@ +"""CrewAI-local batch projection before core's per-attribute privacy policy.""" + +import wrapt +from fi_instrumentation import FITracer +from opentelemetry.trace import Link, SpanKind + +from ._privacy import _redact_header_records + + +def _project_attributes(attributes): + # Keep selector/value siblings together before core's scalar masking; leave + # unrelated values and core's lazy evaluation semantics unchanged. + return ( + None + if attributes is None + else _redact_header_records(dict(attributes)) + ) + + +class _CrewAISpan(wrapt.ObjectProxy): + """Retain core span lifecycle/error policy; project only attribute batches. + + No global FiSpan/SDK patches, shared selector state, or application mutation. + The complete batch is projected before core masks each scalar, regardless + of selector/value ordering in the mapping. + """ + + def __enter__(self): + self.__wrapped__.__enter__() + return self + + def __exit__(self, *args): + return self.__wrapped__.__exit__(*args) + + def set_attributes(self, attributes): + self.__wrapped__.set_attributes(_project_attributes(attributes)) + + def add_event(self, name, attributes=None, timestamp=None): + self.__wrapped__.add_event( + name, _project_attributes(attributes), timestamp + ) + + def record_exception( + self, exception, attributes=None, timestamp=None, escaped=False + ): + self.__wrapped__.record_exception( + exception, _project_attributes(attributes), timestamp, escaped + ) + + def add_link(self, context, attributes=None): + self.__wrapped__.add_link(context, _project_attributes(attributes)) + + +class _CrewAITracer(FITracer): + def start_span( + self, + name, + context=None, + kind=SpanKind.INTERNAL, + attributes=None, + links=(), + start_time=None, + record_exception=True, + set_status_on_exception=True, + *, + fi_span_kind=None, + ): + # Creation attributes/links are projected before samplers/processors can + # observe them, not repaired after the raw span has already started. + projected_links = ( + None + if links is None + else [ + Link(link.context, _project_attributes(link.attributes)) + for link in links + ] + ) + span = super().start_span( + name=name, + context=context, + kind=kind, + attributes=_project_attributes(attributes), + links=projected_links, + start_time=start_time, + record_exception=record_exception, + set_status_on_exception=set_status_on_exception, + fi_span_kind=fi_span_kind, + ) + return _CrewAISpan(span) From 44ceaf3821bfdacd5c95c62db9bab3a292f3c880 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Sat, 31 Oct 2026 01:25:12 -0700 Subject: [PATCH 5/5] fix(crewai): project content without changing native telemetry Separate framework configuration exclusion from the documented application field-name policy. Scope header interpretation to explicit header contexts, preserve business records and usage metadata, and validate native kickoff canaries on real OTLP wire. Remove manual changelog edits and refresh released core hashes. Co-Authored-By: Claude Code --- python/frameworks/crewai/CHANGELOG.md | 11 - python/frameworks/crewai/README.md | 146 +++++-- python/frameworks/crewai/poetry.lock | 4 +- python/frameworks/crewai/tests/conftest.py | 28 +- .../crewai/tests/test_batch_projection.py | 4 +- .../crewai/tests/test_credential_patterns.py | 31 +- .../tests/test_credential_projection.py | 2 +- .../crewai/tests/test_native_telemetry.py | 134 +++---- .../frameworks/crewai/tests/test_otlp_wire.py | 359 +++++++++++++++--- .../crewai/traceai_crewai/__init__.py | 6 - .../crewai/traceai_crewai/_privacy.py | 111 +++--- .../crewai/traceai_crewai/_telemetry.py | 66 ---- .../crewai/traceai_crewai/_tracer.py | 10 +- .../crewai/traceai_crewai/_wrappers.py | 25 +- python/tests/test_framework_crewai.py | 33 +- 15 files changed, 619 insertions(+), 351 deletions(-) delete mode 100644 python/frameworks/crewai/traceai_crewai/_telemetry.py diff --git a/python/frameworks/crewai/CHANGELOG.md b/python/frameworks/crewai/CHANGELOG.md index f6777a9b3..503789fd4 100644 --- a/python/frameworks/crewai/CHANGELOG.md +++ b/python/frameworks/crewai/CHANGELOG.md @@ -1,15 +1,4 @@ ## [0.1.9] - 2026-07-13 -### Security -- Redact credential-named fields in structured task, crew and tool trace data, - including JSON-serialized arguments, aliases, context and event attributes. - Project business content without serializing opaque runtime objects. -- Honor directional content hiding on CrewAI aliases and omit automatic - exception details while preserving application inputs, results and exceptions. -- Disable CrewAI's separate native telemetry while instrumented without disabling - the OTel SDK; support legacy versions that ignore native opt-out flags. -- Require fi-instrumentation-otel >=1.2.2 and declare the setuptools dependency - needed to import CrewAI 0.41.1 on a clean installation. - ### Fixed - Fix `crew.kickoff()` crashing with `TypeError: '_NotSpecified' object is not iterable` when a `Task` is created without an explicit `context`. CrewAI diff --git a/python/frameworks/crewai/README.md b/python/frameworks/crewai/README.md index 02501d349..d67d8e8ed 100644 --- a/python/frameworks/crewai/README.md +++ b/python/frameworks/crewai/README.md @@ -34,28 +34,129 @@ os.environ["CREWAI_DISABLE_TRACKING"] = "true" ### Privacy and credentials -Task, crew and tool spans project business content rather than serializing agent, -LLM, client or tool configuration objects. Credential-named fields in structured -inputs and outputs, including complete JSON object/array strings, are redacted -by default. This includes nested `client.config.api_key`, API-key headers, -authorization, access/refresh tokens, secrets and passwords. Model/provider -identifiers, usage counts and unrelated business values remain visible. Tracing -does not change your inputs, tool results or raised exceptions. - -This is not arbitrary free-text or unknown-encoding secret detection. If prose -may contain secrets, set `FI_HIDE_INPUTS=true` and/or `FI_HIDE_OUTPUTS=true` -before instrumentation, or pass the corresponding `TraceConfig` settings. -Directional text/message hiding also covers CrewAI's content aliases. Automatic -wrapper failures record an ERROR status without copying exception messages or -stacktraces. For manually recorded error events, use `FI_HIDE_ERROR_DETAILS=true`. -This integration does not filter your application's own console output or logs. - -Call `instrument()` before constructing agents, tools or crews. While -instrumented, traceAI disables CrewAI's separate native telemetry, including the -legacy exporter in CrewAI 0.41.1, which ignores the environment flags above. -`uninstrument()` restores CrewAI's native behavior. This does not retract any -telemetry already queued before instrumentation. Do not set -`OTEL_SDK_DISABLED=true`: that also disables the traceAI spans you want to collect. +#### Structural guarantee: framework configuration is not content + +The instrumentor never exports an LLM, Agent, Crew, Task, Tool or client object +itself, and never calls its `repr`, `str`, `model_dump`, `dict`, `vars` or generic +JSON serializer to capture configuration. LLM API keys, base URLs (including +URLs with userinfo), configuration headers and credentials are not selected. +This does not depend on finding a secret's spelling or value. + +The complete wrapper inventory is: + +| Wrapper | Explicit input projection | Explicit result projection | +| --- | --- | --- | +| `Task._execute_core` | Task description, expected output and application context; no agent or tool argument object | `TaskOutput.json_dict` when it is a plain dictionary, otherwise `TaskOutput.raw`; plain application results | +| `Crew.kickoff` | Application `inputs`; named agent/task business metadata below | `CrewOutput.json_dict` when it is a plain dictionary, otherwise `CrewOutput.raw`; prompt/completion/total usage counts | +| `ToolUsage._use` | `ToolCalling.tool_name`, `ToolCalling.arguments`; tool name only from the runtime tool | Plain application result; no tool configuration | + +Crew metadata selects only crew/task keys and UUIDs; agent role, goal, backstory, +key, UUID, verbosity, iteration/rate limits, delegation flag and tool names; +and task description, expected output, UUID, execution/human-input flags, +assigned agent role/key, context task descriptions and tool names. Task description +and expected output are also emitted as formatted aliases when `share_crew=True`. +`Agent.execute_task` and LLM calls have no additional wrappers in this package: +they execute inside the task span and their arguments/results are not serialized +independently. Separately installed LLM instrumentors have their own policies. + +Opaque runtime objects nested in application data are replaced with +`[omitted runtime object]`, without inspecting their attributes. The guarantee is +about framework configuration objects, not credentials deliberately copied into +application prose or a plain dictionary. Instrumentation preserves application +arguments, results and raised exceptions. + +#### Application content: a field-name policy, not a universal secret detector + +With content capture on, task/tool inputs and outputs are application content. +The following policy applies to plain dictionaries, lists and tuples (copied +without mutation), and complete JSON object/array strings. It also applies to +context and span/event attributes, including structured `log.body` attributes. +It does not filter independent application logging pipelines or console output. + +Names are case-insensitive; an initial `X-`/`X_` prefix and punctuation/separators +are normalized away. Dot/bracket field paths are inspected component by +component, including nested dictionary paths. A field and all its descendants +are replaced by `__REDACTED__` if the normalized component contains any of these +markers: + +- `apikey`, `accesskey`, `privatekey`, `clientkey`, `signingkey`, + `subscriptionkey`, `applicationkey`, `masterkey`; +- `token`, `secret`, `password`, `passwd`, `passphrase`, `credential`, `auth`, + `cookie`, `session`, `signature`, `bearer`, `jwt`, `otp`. + +`pin` matches a whole normalized component only. Adjacent components are also +combined for the eight `*key` markers above, so `client.api.key` and +`client[private][key]` match their single-field equivalents. + +Header interpretation requires an explicit field whose normalized name ends in +`headers` (for example `headers`, `request_headers`, `rawHeaders`): + +- Dictionaries with credential-named header keys; +- Two-item name/value pairs and collections of pairs; +- Alternating name/value lists such as Node's `rawHeaders`; +- Records with a `name`, `key` or `header` selector and a `value` field; +- Flattened dot/bracket records with those selectors, or `0`/`1` selectors, + including descendants of list-valued `value` fields in one attribute batch. + +A credential-named header selector preserves its name but redacts the entire +paired value. Outside a named headers field, pairs and `{name, value}` records +are ordinary business data, never inferred to be headers. For example, +`topics=['authentication', 'billing']`, business tables and product records keep +their values. + +The exact normalized metadata exceptions below remain visible (children are +still inspected). These cover standard token counters, generation limits, +tokenizer metadata, author metadata and non-secret session/service metadata: + +`acceptedpredictiontokens`, `apisecretid`, `audiotokens`, `author`, `authoremail`; +`authorid`, `authority`, `authorname`, `authors`, `authorship`; +`authorurl`, `cachecreationinputtokens`, `cachedcontenttokencount`, `cachedinputtokens`, `cachedtokencount`; +`cachedtokens`, `cachereadinputtokens`, `cachereadtokens`, `cachewriteinputtokens`, `cachewritetokens`; +`candidatestokencount`, `completiontokencount`, `completiontokens`, `completiontokensdetails`, `consultokenttl`; +`inputtokencount`, `inputtokens`, `inputtokensdetails`, `maxcompletiontokens`, `maxinputtokens`; +`maxnewtokens`, `maxoutputtokens`, `maxtokens`, `numtokens`, `outputtokencount`; +`outputtokens`, `outputtokensdetails`, `promptcachehittokens`, `promptcachemisstokens`, `prompttokencount`; +`prompttokens`, `prompttokensdetails`, `reasoningtokencount`, `reasoningtokens`, `rejectedpredictiontokens`; +`securitytokenid`, `servicetokencount`, `sessioncount`, `sessionduration`, `sessiondurationms`; +`subjecttokentype`, `thoughtstokencount`, `tokencount`, `tokenid`, `tokenizer`; +`tokenizername`, `tokenizerversion`, `tokensused`, `tokenusage`, `tooluseprompttokencount`; +`totaltokencount`, `totaltokens`, `vaulttokencount`. + +These are field-name rules, not value inspection. Unknown credential names +(for example an application-specific `pwd`, `sig` or vendor key header not +matching the markers), novel shapes, arbitrary free text and encodings other +than complete JSON objects/arrays are outside this policy. Do not put credentials +in content fields or metadata. Dictionaries beyond depth 20 are omitted, as are +structured strings larger than 1,000,000 characters or unparseable structured +strings; opaque objects are not dumped as a fallback. + +To disable content capture, set `FI_HIDE_INPUTS=true` and `FI_HIDE_OUTPUTS=true` +before instrumentation, or pass `TraceConfig(hide_inputs=True, hide_outputs=True)`. +Directional text/message hiding also covers CrewAI's content aliases. Wrapper +failures record ERROR status without copying exception messages or stacktraces. +For manually recorded error events, use `FI_HIDE_ERROR_DETAILS=true`. + +### CrewAI's separate native telemetry + +CrewAI can export through its own provider to `telemetry.crewai.com`, independently +of traceAI. **This instrumentor does not change environment variables, patch +CrewAI's telemetry classes, disable its provider or restore/change your settings +on `uninstrument()`.** Choose the native telemetry policy explicitly before +importing CrewAI. + +In CrewAI 0.152.0 and 1.15.33, `CREWAI_DISABLE_TELEMETRY=true` or +`CREWAI_DISABLE_TRACKING=true` disables native telemetry while leaving traceAI +available. `OTEL_SDK_DISABLED=true` is also honored, but disables the OTel SDK +and traceAI spans too; do not use it when you want traceAI tracing. +`CREWAI_TELEMETRY_OPT_OUT` and `DO_NOT_TRACK` are not native disable flags. + +CrewAI 0.41.1 has no native telemetry opt-out flag: its constructor creates an +exporter for `https://telemetry.crewai.com:4319/v1/traces` and ignores the flags +above. If you need native telemetry disabled, upgrade to a version that honors +`CREWAI_DISABLE_TELEMETRY` before running sensitive workloads. For legacy +workloads, enforce a network egress policy; traceAI's projections cannot protect +an independent provider's exports. `share_crew=False` is not a native telemetry +disable switch. ## Quickstart @@ -86,7 +187,6 @@ Set up your Crewai client with built-in observability. ```python from crewai import LLM, Agent, Crew, Process, Task -from crewai_tools import SerperDevTool def story_example(): llm = LLM( diff --git a/python/frameworks/crewai/poetry.lock b/python/frameworks/crewai/poetry.lock index f4e648363..079b84875 100644 --- a/python/frameworks/crewai/poetry.lock +++ b/python/frameworks/crewai/poetry.lock @@ -874,11 +874,11 @@ groups = ["main"] [[package.files]] file = "fi_instrumentation_otel-1.2.2-py3-none-any.whl" -hash = "sha256:ba1003f5962303a0fb9f1587153d0c5bda3856018da00b36629aef96296604fc" +hash = "sha256:25cf88f93e33aa928d4fae0f631afe8a32565639490862b715869e0633becbaf" [[package.files]] file = "fi_instrumentation_otel-1.2.2.tar.gz" -hash = "sha256:13d16695511f91c6f16cfa52e4b3236193a734df85ce52b40d37ad52831962fd" +hash = "sha256:b675df47d7a03c506fbd2e4b0420fe4f74d12a6353a88b219bfd97797b276d4c" [package.dependencies] jsonschema = ">=4.21.1,<5.0.0" opentelemetry-api = ">=1.29.0,<2.0.0" diff --git a/python/frameworks/crewai/tests/conftest.py b/python/frameworks/crewai/tests/conftest.py index 6e3c4e581..07ce107a8 100644 --- a/python/frameworks/crewai/tests/conftest.py +++ b/python/frameworks/crewai/tests/conftest.py @@ -11,24 +11,24 @@ @pytest.fixture(autouse=True) -def native_telemetry_off(request): +def native_telemetry_off(request, monkeypatch): + from importlib.metadata import version from crewai.telemetry import Telemetry - from traceai_crewai._telemetry import _NativeTelemetryGuard - if request.node.name.startswith("test_guard_"): + if request.node.name.startswith("test_native_telemetry_"): yield return - # Use the packaged adapter for baseline native calls too. Floor CrewAI - # ignores the environment flags; maintained versions use their own checks. - guard = _NativeTelemetryGuard() - guard.enable() - try: - telemetry = Telemetry() - assert not telemetry.ready - assert not getattr(telemetry, "provider", None) - yield - finally: - guard.disable() + if version("crewai") == "0.41.1": + # Test-only isolation: legacy CrewAI does not honor native disable flags. + # Production instrumentation must never patch the user's telemetry. + def disabled(instance): + instance.ready = False + instance.trace_set = False + + monkeypatch.setattr(Telemetry, "__init__", disabled) + else: + assert Telemetry._is_telemetry_disabled() + yield @pytest.fixture diff --git a/python/frameworks/crewai/tests/test_batch_projection.py b/python/frameworks/crewai/tests/test_batch_projection.py index 60cb8393a..d5d6c86f3 100644 --- a/python/frameworks/crewai/tests/test_batch_projection.py +++ b/python/frameworks/crewai/tests/test_batch_projection.py @@ -74,9 +74,7 @@ def lazy_input(): span.set_attributes({"business.lazy": lambda: "useful lazy"}) span.add_event("batch", MappingProxyType(attributes)) span.add_link(link_context, MappingProxyType(attributes)) - span.record_exception( - RuntimeError("synthetic failure"), attributes - ) + span.record_exception(RuntimeError("synthetic failure"), attributes) assert attributes == before assert calls == ([] if hide_inputs else ["input"]) assert SECRET not in json.dumps(observations) diff --git a/python/frameworks/crewai/tests/test_credential_patterns.py b/python/frameworks/crewai/tests/test_credential_patterns.py index 9dd49325e..d997dc84c 100644 --- a/python/frameworks/crewai/tests/test_credential_patterns.py +++ b/python/frameworks/crewai/tests/test_credential_patterns.py @@ -127,10 +127,7 @@ def test_generated_credential_matrix(prefix, case): projected = _safe_data(payload) assert SECRET not in json.dumps(projected), name assert projected["headers"]["Accept"] == "application/json" - assert ( - projected["total_tokens"] == 42 - and projected["id"] == "business-id" - ) + assert projected["total_tokens"] == 42 and projected["id"] == "business-id" assert json.loads(projected["raw"])["answer"] == "useful answer" assert payload == before assert ( @@ -153,9 +150,7 @@ def test_explicit_metadata_exceptions_preserve_values_and_not_children(name): payload = {spelling: 42, f"usage.{spelling}": 42, "id": "safe-id"} assert _safe_data(payload) == payload # An allowlisted parent cannot bypass inspection of its children. - assert SECRET not in json.dumps( - _safe_data({spelling: {"api_key": SECRET}}) - ) + assert SECRET not in json.dumps(_safe_data({spelling: {"api_key": SECRET}})) assert _credential_key(f"{spelling}.api_key") @@ -170,9 +165,7 @@ def test_unknown_vendor_token_metadata_is_not_a_generic_bypass(): @pytest.mark.parametrize("shape", ["pairs", "tuples", "name", "key", "header"]) def test_header_lists_and_serialized_lists(shape, service_credentials): - fields = list(service_credentials.items()) + [ - ("Accept", "application/json") - ] + fields = list(service_credentials.items()) + [("Accept", "application/json")] if shape == "pairs": headers = [list(item) for item in fields] elif shape == "tuples": @@ -183,7 +176,9 @@ def test_header_lists_and_serialized_lists(shape, service_credentials): for key, value in fields ] original = copy.deepcopy(headers) - projected = _safe_data({"headers": headers, "json": json.dumps(headers)}) + projected = _safe_data( + {"headers": headers, "serialized_headers": json.dumps(headers)} + ) rendered = json.dumps(projected) assert all(value not in rendered for value in service_credentials.values()) assert "application/json" in rendered @@ -191,8 +186,7 @@ def test_header_lists_and_serialized_lists(shape, service_credentials): assert headers == original if shape not in ("pairs", "tuples"): assert all( - item["description"] == "business data" - for item in projected["headers"] + item["description"] == "business data" for item in projected["headers"] ) for name in service_credentials: assert name in rendered @@ -205,9 +199,7 @@ def test_flattened_header_records(selector, brackets): def key(index, field): return ( - f"headers[{index}]['{field}']" - if brackets - else f"headers.{index}.{field}" + f"headers[{index}]['{field}']" if brackets else f"headers.{index}.{field}" ) original = { @@ -242,7 +234,8 @@ def test_two_item_business_lists_are_not_header_pairs(first): expected = {**payload, "categories": [first, "shipping"]} assert _safe_data(payload) == expected assert payload == before - # Recognizable pair collections still work, including bare serialized JSON. + # Only explicit header fields interpret pairs; bare business tables survive. for headers in ([["Authorization", SECRET]], [("X-API-Key", SECRET)]): - assert SECRET not in json.dumps(_safe_data(headers)) - assert SECRET not in _safe_data(json.dumps(headers)) + assert _safe_data(headers) == [list(item) for item in headers] + assert SECRET not in json.dumps(_safe_data({"headers": headers})) + assert SECRET not in json.dumps(_safe_data({"headers": json.dumps(headers)})) diff --git a/python/frameworks/crewai/tests/test_credential_projection.py b/python/frameworks/crewai/tests/test_credential_projection.py index 28bea0e36..6189fffcc 100644 --- a/python/frameworks/crewai/tests/test_credential_projection.py +++ b/python/frameworks/crewai/tests/test_credential_projection.py @@ -75,7 +75,7 @@ def execute(arg, **kwargs): payload = _get_input_value(execute, ExplosiveRuntime(), extra=ExplosiveRuntime()) assert SENTINEL not in payload - assert "omitted runtime object" in payload + assert json.loads(payload) == {} def test_exception_does_not_enter_span_and_same_error_propagates(): diff --git a/python/frameworks/crewai/tests/test_native_telemetry.py b/python/frameworks/crewai/tests/test_native_telemetry.py index 5740cac19..be806e5fe 100644 --- a/python/frameworks/crewai/tests/test_native_telemetry.py +++ b/python/frameworks/crewai/tests/test_native_telemetry.py @@ -1,93 +1,99 @@ -"""Verify the package's opt-out, including floor versions without disable flags.""" +"""Native telemetry belongs to the application, not the instrumentor.""" +import inspect +import os from importlib import import_module -from inspect import getattr_static +from importlib.metadata import version from unittest.mock import patch +import pytest from crewai.telemetry import Telemetry from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import SimpleSpanProcessor from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter from traceai_crewai import CrewAIInstrumentor -from traceai_crewai._telemetry import _NativeTelemetryGuard -def test_guard_native_exporter_never_constructed_and_traceai_still_records(monkeypatch): - # The import was safe under pre-import flags; now prove instrument() itself - # disables native telemetry even when those flags are explicitly false. +@pytest.mark.parametrize("disabled", ["true", "false"]) +def test_native_telemetry_instrumentation_does_not_change_user_settings( + monkeypatch, disabled +): for key in ( "CREWAI_DISABLE_TELEMETRY", "CREWAI_DISABLE_TRACKING", "OTEL_SDK_DISABLED", ): - monkeypatch.setenv(key, "false") - if hasattr(Telemetry, "_instance"): - monkeypatch.setattr(Telemetry, "_instance", None) - original = getattr_static(Telemetry, "__init__") - check = getattr_static(Telemetry, "_is_telemetry_disabled", None) - exporter = InMemorySpanExporter() + monkeypatch.setenv(key, disabled) + before_env = dict(os.environ) + before = dict(vars(Telemetry)) provider = TracerProvider(shutdown_on_exit=False) + exporter = InMemorySpanExporter() provider.add_span_processor(SimpleSpanProcessor(exporter)) instrumentor = CrewAIInstrumentor() - # A constructor spy is stronger than an unreachable network endpoint: a - # native exporter must not even be instantiated in the opted-out branch. - module = "crewai.telemetry.telemetry" - target = module + ( - ".SafeOTLPSpanExporter" - if hasattr(import_module(module), "SafeOTLPSpanExporter") - else ".OTLPSpanExporter" - ) - with patch( - target, side_effect=AssertionError("native exporter constructed") - ) as native_exporter: + # Instrumentation must not instantiate or patch the native singleton/provider. + with patch.object( + Telemetry, "__init__", side_effect=AssertionError("native construction") + ): + init = Telemetry.__init__ instrumentor.instrument(tracer_provider=provider) try: - telemetry = Telemetry() - assert not telemetry.ready - assert not getattr(telemetry, "provider", None) - if check is not None: - assert Telemetry._is_telemetry_disabled() + assert Telemetry.__init__ is init + assert dict(os.environ) == before_env + for key, value in before.items(): + if key != "__init__": + assert vars(Telemetry)[key] is value with instrumentor._tracer.start_as_current_span("positive-control"): pass - assert len(exporter.get_finished_spans()) == 1 - assert not native_exporter.called + assert len(exporter.get_finished_spans()) == ( + 0 if disabled == "true" else 1 + ) finally: instrumentor.uninstrument() - assert getattr_static(Telemetry, "__init__") is original - assert getattr_static(Telemetry, "_is_telemetry_disabled", None) is check + assert dict(vars(Telemetry)) == before + assert dict(os.environ) == before_env provider.shutdown() -def test_guard_floor_blocks_existing_instance_and_restores(monkeypatch): - calls = [] - - class LegacyTelemetry: - def __init__(self): - self.ready, self.trace_set = True, True - calls.append("constructor") - - def crew_creation(self, *args): - calls.append("export") - - legacy = LegacyTelemetry() - from types import SimpleNamespace - import traceai_crewai._telemetry as module - - def fake_import(name): - if name == "crewai.telemetry": - return SimpleNamespace(Telemetry=LegacyTelemetry) - raise ModuleNotFoundError(name="crewai_core") - - monkeypatch.setattr(module, "import_module", fake_import) - guard = _NativeTelemetryGuard() - guard.enable() - try: - assert not LegacyTelemetry().ready - assert legacy.crew_creation(None, None) is None - assert calls == ["constructor"] - finally: - guard.disable() - assert LegacyTelemetry().ready - legacy.crew_creation(None, None) - assert calls == ["constructor", "constructor", "export"] +def test_native_telemetry_installed_source_and_opt_out_flags(monkeypatch): + if version("crewai") == "0.41.1": + # This version has no native opt-out check at all. Do not advertise a + # variable it ignores, or instantiate its exporter to demonstrate it. + source = inspect.getsource(Telemetry) + assert not hasattr(Telemetry, "_is_telemetry_disabled") + for flag in ( + "CREWAI_DISABLE_TELEMETRY", + "CREWAI_DISABLE_TRACKING", + "OTEL_SDK_DISABLED", + "CREWAI_TELEMETRY_OPT_OUT", + "DO_NOT_TRACK", + ): + assert flag not in source + assert "https://telemetry.crewai.com:4319" in source + return + # Maintained CrewAI delegates to crewai-core; the repository-locked 0.152 + # implements the same checks in CrewAI itself. + cls = ( + import_module("crewai_core.telemetry").Telemetry + if version("crewai") == "1.15.33" + else Telemetry + ) + honored = ( + "CREWAI_DISABLE_TELEMETRY", + "CREWAI_DISABLE_TRACKING", + "OTEL_SDK_DISABLED", + ) + ignored = ("CREWAI_TELEMETRY_OPT_OUT", "DO_NOT_TRACK") + source = inspect.getsource(cls._is_telemetry_disabled) + for flag in honored + ignored: + monkeypatch.setenv(flag, "false") + assert not Telemetry._is_telemetry_disabled() + for flag in honored: + assert flag in source + monkeypatch.setenv(flag, "true") + assert Telemetry._is_telemetry_disabled() + monkeypatch.setenv(flag, "false") + for flag in ignored: + assert flag not in source + monkeypatch.setenv(flag, "true") + assert not Telemetry._is_telemetry_disabled() diff --git a/python/frameworks/crewai/tests/test_otlp_wire.py b/python/frameworks/crewai/tests/test_otlp_wire.py index d7d3a00ec..68052eb4c 100644 --- a/python/frameworks/crewai/tests/test_otlp_wire.py +++ b/python/frameworks/crewai/tests/test_otlp_wire.py @@ -4,6 +4,7 @@ import copy import gzip import json +import os import threading from http.server import BaseHTTPRequestHandler, HTTPServer from urllib.request import ProxyHandler, Request, build_opener @@ -102,9 +103,7 @@ def test_real_otlp_http_wire( OTLPSpanExporter( endpoint=endpoint, timeout=5, - compression=( - Compression.Gzip if compressed else Compression.NoCompression - ), + compression=(Compression.Gzip if compressed else Compression.NoCompression), ) if encoding == "protobuf" else JsonHTTPExporter(endpoint, compressed) @@ -165,9 +164,7 @@ def invoke(): assert len(captured) == 1 headers, raw = captured[0] assert headers["content-type"] == ( - "application/json" - if encoding == "json" - else "application/x-protobuf" + "application/json" if encoding == "json" else "application/x-protobuf" ) assert (headers.get("content-encoding") == "gzip") is compressed body = gzip.decompress(raw) if compressed else raw @@ -179,9 +176,7 @@ def invoke(): assert b"useful log" in body if settings.get("hide_inputs") or settings.get("hide_input_text"): assert b"useful query" not in body - if settings.get("hide_outputs") or settings.get( - "hide_output_messages" - ): + if settings.get("hide_outputs") or settings.get("hide_output_messages"): assert b"useful answer" not in body if not settings: assert b"useful query" in body @@ -209,10 +204,7 @@ def do_POST(self): data = self.rfile.read(int(self.headers["Content-Length"])) captured.append( ( - { - key.lower(): value - for key, value in self.headers.items() - }, + {key.lower(): value for key, value in self.headers.items()}, data, ) ) @@ -266,17 +258,13 @@ def test_native_service_credentials_never_reach_otlp_http( if shape == "pairs": tool_headers = [list(item) for item in fields.items()] elif shape in ("name", "key", "header"): - tool_headers = [ - {shape: key, "value": value} for key, value in fields.items() - ] + tool_headers = [{shape: key, "value": value} for key, value in fields.items()] elif shape == "serialized": tool_headers = json.dumps( [{"name": key, "value": value} for key, value in fields.items()] ) elif shape == "flattened": - tool_headers = { - f"headers[{key}].value": value for key, value in fields.items() - } + tool_headers = {f"headers[{key}].value": value for key, value in fields.items()} else: tool_headers = fields before = copy.deepcopy(tool_headers) @@ -286,11 +274,7 @@ def lookup(query: str, headers: Any) -> str: seen.append((query, copy.deepcopy(headers))) trace.get_current_span().add_event( "tool.log", - { - "log.body": json.dumps( - {"headers": headers, "message": "useful log"} - ) - }, + {"log.body": json.dumps({"headers": headers, "message": "useful log"})}, ) return json.dumps( {"headers": headers, "answer": "useful answer", "total_tokens": 13} @@ -305,18 +289,14 @@ def lookup(query: str, headers: Any) -> str: OTLPSpanExporter( endpoint=endpoint, timeout=5, - compression=( - Compression.Gzip if compressed else Compression.NoCompression - ), + compression=(Compression.Gzip if compressed else Compression.NoCompression), ) if encoding == "protobuf" else JsonHTTPExporter(endpoint, compressed) ) provider.add_span_processor(SimpleSpanProcessor(exporter)) instrumentor = CrewAIInstrumentor() - instrumentor.instrument( - tracer_provider=provider, config=TraceConfig(**settings) - ) + instrumentor.instrument(tracer_provider=provider, config=TraceConfig(**settings)) try: # Use a fresh ToolUsage to exclude native caching from the parity check. actual = native_tool_usage(lookup).use(calling, "lookup") @@ -328,17 +308,12 @@ def lookup(query: str, headers: Any) -> str: assert provider.force_flush() and len(captured) == 1 headers, raw = captured[0] assert headers["content-type"] == ( - "application/json" - if encoding == "json" - else "application/x-protobuf" + "application/json" if encoding == "json" else "application/x-protobuf" ) assert (headers.get("content-encoding") == "gzip") is compressed body = gzip.decompress(raw) if compressed else raw for credential in service_credentials.values(): - assert ( - credential.encode() not in body - and credential.encode() not in raw - ) + assert credential.encode() not in body and credential.encode() not in raw assert b"ToolUsage._use" in body and b"useful log" in body assert ( b"total_tokens" in body @@ -349,8 +324,7 @@ def lookup(query: str, headers: Any) -> str: settings.get("hide_inputs") or settings.get("hide_input_text") ) assert (b"useful answer" not in body) is bool( - settings.get("hide_outputs") - or settings.get("hide_output_messages") + settings.get("hide_outputs") or settings.get("hide_output_messages") ) finally: instrumentor.uninstrument() @@ -423,9 +397,7 @@ def lookup(data: Any) -> str: OTLPSpanExporter( endpoint=endpoint, timeout=5, - compression=( - Compression.Gzip if compressed else Compression.NoCompression - ), + compression=(Compression.Gzip if compressed else Compression.NoCompression), ) if encoding == "protobuf" else JsonHTTPExporter(endpoint, compressed) @@ -483,9 +455,7 @@ def attributes(items): } for batch in (attrs, events["batch.log"]): for key, value in expected.items(): - assert batch[key] == ( - str(value) if type(value) is int else value - ) + assert batch[key] == (str(value) if type(value) is int else value) finally: instrumentor.uninstrument() provider.shutdown() @@ -505,3 +475,302 @@ def test_raw_negative_control_detects_unprojected_synthetic_credential(): assert SECRET.encode() in data assert SECRET.encode() in gzip.decompress(gzip.compress(data)) provider.shutdown() + + +@pytest.mark.parametrize("encoding", ["protobuf", "json"]) +@pytest.mark.parametrize("compressed", [False, True]) +@pytest.mark.parametrize("capture_content", [False, True]) +def test_native_kickoff_structural_configuration_never_reaches_wire( + encoding, compressed, capture_content, otlp_collector, monkeypatch, caplog +): + """Real native kickoff, real framework credential fields, local fake LLM. + + Run this on the floor and current CrewAI. No vendor call/credential guessing: + these canaries exist only in configuration, not application prose/content. + """ + from importlib.metadata import version + from unittest.mock import patch + from crewai import Agent, Crew, Task + from traceai_crewai import CrewAIInstrumentor + + canaries = tuple(f"SYNTHETIC_CONFIG_{letter}_9ca427" for letter in "ABCD") + api_key, url_password, tool_key, environment_key = canaries + monkeypatch.setenv("OPENAI_API_KEY", environment_key) + base_url = f"https://u:{url_password}@example.invalid" + answer = "useful structural answer" + floor = version("crewai") == "0.41.1" + if floor: + # CrewAI 0.41.1 predates crewai.LLM and takes a LangChain LLM. + from langchain_openai import ChatOpenAI + from langchain_core.tools import BaseTool + from langchain_core.messages import AIMessage, AIMessageChunk + from langchain_core.outputs import ( + ChatGeneration, + ChatGenerationChunk, + ChatResult, + ) + from types import SimpleNamespace + + # CrewAI's token callback otherwise downloads a tokenizer on first use. + monkeypatch.setattr( + "tiktoken.encoding_for_model", + lambda _: SimpleNamespace(encode=lambda text: list(text)), + ) + + class LocalLLM(ChatOpenAI): + def _stream(self, *args, **kwargs): + trace.get_current_span().add_event( + "llm.log", {"log.body": "useful local LLM log"} + ) + yield ChatGenerationChunk( + message=AIMessageChunk(content="Final Answer: " + answer) + ) + + def _generate(self, *args, **kwargs): + trace.get_current_span().add_event( + "llm.log", {"log.body": "useful local LLM log"} + ) + return ChatResult( + generations=[ + ChatGeneration( + message=AIMessage(content="Final Answer: " + answer) + ) + ] + ) + + llm = LocalLLM(model="gpt-4o-mini", api_key=api_key, base_url=base_url) + assert llm.openai_api_key.get_secret_value() == api_key + assert llm.openai_api_base == base_url + else: + from crewai import LLM + from crewai.tools import BaseTool + + llm = LLM(model="openai/synthetic", api_key=api_key, base_url=base_url) + assert llm.api_key == api_key and llm.base_url == base_url + assert os.environ["OPENAI_API_KEY"] == environment_key + + class LocalTool(BaseTool): + name: str = "local_lookup" + description: str = "Look up a useful business result locally" + api_key: str = tool_key + + def _run(self, query: str) -> str: + return "useful business tool answer" + + def run(): + tool = LocalTool() + agent = Agent( + role="Useful analyst", + goal="Answer a business question", + backstory="Uses public business information", + llm=llm, + tools=[tool], + allow_delegation=False, + ) + task = Task( + description="Return useful business answer", + expected_output="A concise answer", + agent=agent, + tools=[tool], + ) + crew = Crew(agents=[agent], tasks=[task], share_crew=True) + inputs = { + "query": "useful business query", + "topics": ["authentication", "billing"], + } + before = copy.deepcopy(inputs) + if floor: + result = crew.kickoff(inputs=inputs) + else: + + def respond(*args, **kwargs): + trace.get_current_span().add_event( + "llm.log", {"log.body": "useful local LLM log"} + ) + return answer + + with patch.object(type(llm), "call", side_effect=respond) as call: + result = crew.kickoff(inputs=inputs) + assert call.called + assert inputs == before and tool.api_key == tool_key + return result.raw + + baseline = run() + assert baseline == answer + endpoint, captured = otlp_collector + provider = TracerProvider(shutdown_on_exit=False) + exporter = ( + OTLPSpanExporter( + endpoint=endpoint, + timeout=5, + compression=(Compression.Gzip if compressed else Compression.NoCompression), + ) + if encoding == "protobuf" + else JsonHTTPExporter(endpoint, compressed) + ) + provider.add_span_processor(SimpleSpanProcessor(exporter)) + config = TraceConfig( + hide_inputs=not capture_content, hide_outputs=not capture_content + ) + instrumentor = CrewAIInstrumentor() + instrumentor.instrument(tracer_provider=provider, config=config) + try: + assert run() == baseline + assert provider.force_flush() and len(captured) >= 2 + decoded_spans = [] + for headers, raw in captured: + assert (headers.get("content-encoding") == "gzip") is compressed + assert headers["content-type"] == ( + "application/json" if encoding == "json" else "application/x-protobuf" + ) + body = gzip.decompress(raw) if compressed else raw + for canary in canaries: + assert canary.encode() not in body and canary.encode() not in raw + assert canary not in caplog.text + # Ensure we're checking meaningful parsed OTLP, not empty captures. + decoded = ( + json.loads(body) + if encoding == "json" + else MessageToDict( + ExportTraceServiceRequest.FromString(body), + use_integers_for_enums=True, + ) + ) + decoded_spans.extend( + span + for resource in decoded["resourceSpans"] + for scope in resource["scopeSpans"] + for span in scope["spans"] + ) + assert {span["name"] for span in decoded_spans} >= { + "Task._execute_core", + "Crew.kickoff", + } + wire = b"".join( + gzip.decompress(raw) if compressed else raw for _, raw in captured + ) + assert b"useful local LLM log" in wire + assert (b"useful business query" in wire) is capture_content + assert (answer.encode() in wire) is capture_content + finally: + instrumentor.uninstrument() + provider.shutdown() + + +@pytest.mark.parametrize("encoding", ["protobuf", "json"]) +@pytest.mark.parametrize("compressed", [False, True]) +def test_contextual_payload_policy_business_and_header_shapes_on_wire( + encoding, compressed, otlp_collector, native_tool_usage +): + from traceai_crewai import CrewAIInstrumentor + + payload = { + "topics": ["authentication", "billing"], + "rows": [ + ["authentication", "public-billing"], + ["tokenization", "public-shipping"], + ], + "products": [ + {"name": "Authentication handbook", "value": "public-description"}, + {"key": "tokenization", "value": "public-category"}, + ], + "usage": { + "accepted_prediction_tokens": 1, + "rejected_prediction_tokens": 2, + "candidates_token_count": 3, + "thoughts_token_count": 4, + "cached_content_token_count": 5, + "max_new_tokens": 6, + "input_tokens": 7, + "output_tokens": 8, + "total_tokens": 15, + "prompt_cache_hit_tokens": 9, + "prompt_cache_miss_tokens": 10, + "cacheWriteInputTokens": 11, + "toolUsePromptTokenCount": 12, + }, + "author_email": "writer@example.invalid", + "author_url": "https://example.invalid/writer", + "client": {"api": {"key": SECRET}, "private": {"key": SECRET}}, + "rawHeaders": [ + "Authorization", + SECRET, + "X-API-Key", + SECRET, + "Accept", + "application/json", + ], + "headers": [{"name": "Authorization", "value": [SECRET]}], + # Undocumented names are intentionally outside the application policy. + "custom_vendor_field": "documented application limitation", + } + flat = { + "headers.0.name": "Authorization", + "headers.0.value.0": SECRET, + "headers[1]['key']": "X-API-Key", + "headers[1]['value'][0]": SECRET, + } + before = copy.deepcopy(payload) + + def lookup(data: Any) -> str: + """Emit synthetic application data on the actual native tool span.""" + span = trace.get_current_span() + span.set_attributes(flat) + span.add_event("batch.log", flat) + span.add_event("tool.log", {"log.body": json.dumps(data)}) + return json.dumps(data) + + endpoint, captured = otlp_collector + provider = TracerProvider(shutdown_on_exit=False) + exporter = ( + OTLPSpanExporter( + endpoint=endpoint, + timeout=5, + compression=(Compression.Gzip if compressed else Compression.NoCompression), + ) + if encoding == "protobuf" + else JsonHTTPExporter(endpoint, compressed) + ) + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = CrewAIInstrumentor() + instrumentor.instrument(tracer_provider=provider) + try: + calling = ToolCalling(tool_name="lookup", arguments={"data": payload}) + assert json.loads(native_tool_usage(lookup).use(calling, "lookup")) == before + assert payload == before + assert provider.force_flush() and len(captured) == 1 + _, raw = captured[0] + body = gzip.decompress(raw) if compressed else raw + assert SECRET.encode() not in body + decoded = ( + json.loads(body) + if encoding == "json" + else MessageToDict( + ExportTraceServiceRequest.FromString(body), use_integers_for_enums=True + ) + ) + span = decoded["resourceSpans"][0]["scopeSpans"][0]["spans"][0] + attrs = { + attr["key"]: attr["value"]["stringValue"] for attr in span["attributes"] + } + output = json.loads(attrs["output.value"]) + if isinstance(output, str): + output = json.loads(output) + for field in ( + "topics", + "rows", + "products", + "usage", + "author_email", + "author_url", + "custom_vendor_field", + ): + assert output[field] == before[field] + for event in span["events"]: + if event["name"] == "tool.log": + log = json.loads(event["attributes"][0]["value"]["stringValue"]) + for field in ("topics", "rows", "products", "usage"): + assert log[field] == before[field] + finally: + instrumentor.uninstrument() + provider.shutdown() diff --git a/python/frameworks/crewai/traceai_crewai/__init__.py b/python/frameworks/crewai/traceai_crewai/__init__.py index 08b1dfc4a..612c188d0 100644 --- a/python/frameworks/crewai/traceai_crewai/__init__.py +++ b/python/frameworks/crewai/traceai_crewai/__init__.py @@ -12,7 +12,6 @@ ) from traceai_crewai._privacy import _CrewAITraceConfig from traceai_crewai._tracer import _CrewAITracer -from traceai_crewai._telemetry import _NativeTelemetryGuard from traceai_crewai.version import __version__ from wrapt import wrap_function_wrapper @@ -27,7 +26,6 @@ class CrewAIInstrumentor(BaseInstrumentor): # type: ignore "_original_kickoff", "_original_tool_use", "_tracer", - "_native_telemetry_guard", ) def instrumentation_dependencies(self) -> Collection[str]: @@ -45,9 +43,6 @@ def _instrument(self, **kwargs: Any) -> None: config=_CrewAITraceConfig(config), ) - self._native_telemetry_guard = _NativeTelemetryGuard() - self._native_telemetry_guard.enable() - execute_core_wrapper = _ExecuteCoreWrapper(tracer=self._tracer) self._original_execute_core = getattr( import_module("crewai").Task, "_execute_core", None @@ -77,7 +72,6 @@ def _instrument(self, **kwargs: Any) -> None: ) def _uninstrument(self, **kwargs: Any) -> None: - self._native_telemetry_guard.disable() if self._original_execute_core is not None: task_module = import_module("crewai") task_module.Task._execute_core = self._original_execute_core diff --git a/python/frameworks/crewai/traceai_crewai/_privacy.py b/python/frameworks/crewai/traceai_crewai/_privacy.py index 3983a908b..f49879616 100644 --- a/python/frameworks/crewai/traceai_crewai/_privacy.py +++ b/python/frameworks/crewai/traceai_crewai/_privacy.py @@ -93,6 +93,12 @@ "authors", "authorid", "authorname", + "authoremail", + "authorurl", + "promptcachehittokens", + "promptcachemisstokens", + "cachewriteinputtokens", + "tooluseprompttokencount", "authorship", "authority", "sessioncount", @@ -136,9 +142,7 @@ def _credential_key(key: str) -> bool: # a value/index. Exceptions apply to one complete field, never its children. if _normalized_name(key) in _NON_SECRET_NAMES: return False - parts = [ - _normalized_name(part) for part in re.split(r"[.\[\]]+", key) if part - ] + parts = [_normalized_name(part) for part in re.split(r"[.\[\]]+", key) if part] for normalized in parts: if normalized in _NON_SECRET_NAMES: continue @@ -157,9 +161,7 @@ def _credential_key(key: str) -> bool: def _field_path(key: str) -> tuple: - return tuple( - _normalized_name(part) for part in re.split(r"[.\[\]]+", key) if part - ) + return tuple(_normalized_name(part) for part in re.split(r"[.\[\]]+", key) if part) def _is_headers_field(key: str) -> bool: @@ -167,19 +169,26 @@ def _is_headers_field(key: str) -> bool: return bool(parts and parts[-1].endswith("headers")) -def _redact_header_records(value: dict) -> dict: - """Shallow projection of complete selector/value records, in any order. +def _redact_header_records( + value: dict, *, _header_context: bool = False, _path: tuple = () +) -> dict: + """Project selector/value siblings only in explicitly named header fields. - Leave unrelated values (including core's lazy callbacks) unevaluated. The - caller's per-attribute mask remains responsible for individual values. + Flattened dot/bracket paths are supported, including descendants of a + list-valued value field. No inference from ordinary product records/pairs. + Unrelated values (including core's lazy callbacks) remain unevaluated. """ credential_values = set() for key, item in value.items(): if type(key) is not str or type(item) is not str: continue - parts = _field_path(key) + parts = _path + _field_path(key) + in_headers = _header_context or any( + part.endswith("headers") for part in parts[:-1] + ) if ( - parts + in_headers + and parts and parts[-1] in ("name", "key", "header", "0") and _credential_key(item) ): @@ -188,7 +197,11 @@ def _redact_header_records(value: dict) -> dict: return { key: ( REDACTED - if type(key) is str and _field_path(key) in credential_values + if type(key) is str + and any( + (_path + _field_path(key))[: len(prefix)] == prefix + for prefix in credential_values + ) else item ) for key, item in value.items() @@ -196,13 +209,16 @@ def _redact_header_records(value: dict) -> dict: def _safe_data( - value: Any, depth: int = 0, *, _header_context: bool = False + value: Any, + depth: int = 0, + *, + _header_context: bool = False, + _path: tuple = (), ) -> Any: - """Copy plain data; interpret pairs only inside a header collection. + """Copy plain application data under a documented field-name policy. - Header context comes from a headers field or a collection of two-item pairs, - including serialized collections. A standalone two-string business list is - not a header just because its first element mentions authentication/tokens. + Never inspect, stringify or dump an opaque framework/configuration object. + Header records/pairs/alternating lists require a named headers context. """ if depth > 20: return "[omitted]" @@ -213,13 +229,14 @@ def _safe_data( try: parsed = json.loads(value) except (ValueError, RecursionError): - # Do not export a partially structured payload we cannot safely - # inspect (including valid JSON beyond parser resource limits). return "[omitted unparseable structured data]" if type(parsed) in (dict, list): return json.dumps( _safe_data( - parsed, depth + 1, _header_context=_header_context + parsed, + depth + 1, + _header_context=_header_context, + _path=_path, ) ) return value @@ -229,45 +246,53 @@ def _safe_data( return { key: ( REDACTED - if _credential_key(key) + if _credential_key(".".join(_path + _field_path(key))) else _safe_data( item, depth + 1, - _header_context=_is_headers_field(key), + _header_context=_header_context or _is_headers_field(key), + _path=_path + _field_path(key), ) ) - for key, item in _redact_header_records(value).items() + for key, item in _redact_header_records( + value, _header_context=_header_context, _path=_path + ).items() if type(key) is str } if type(value) in (list, tuple): + # Includes standalone pairs and Node-style rawHeaders, but only in an + # explicit header field. Business tables are never promoted to headers. if ( _header_context - and len(value) == 2 - and type(value[0]) is str - and _credential_key(value[0]) + and value + and len(value) % 2 == 0 + and all(type(value[i]) is str for i in range(0, len(value), 2)) ): - return [value[0], REDACTED] - pair_collection = bool(value) and all( - type(item) in (list, tuple) - and len(item) == 2 - and type(item[0]) is str - for item in value - ) + return [ + REDACTED + if i % 2 and _credential_key(value[i - 1]) + else _safe_data( + item, + depth + 1, + _header_context=_header_context, + _path=_path + (str(i),), + ) + for i, item in enumerate(value) + ] return [ _safe_data( item, depth + 1, - _header_context=pair_collection or _header_context, + _header_context=_header_context, + _path=_path + (str(i),), ) - for item in value + for i, item in enumerate(value) ] return "[omitted runtime object]" def _has_prefix(key: str, prefixes: tuple) -> bool: - return any( - key == prefix or key.startswith(prefix + ".") for prefix in prefixes - ) + return any(key == prefix or key.startswith(prefix + ".") for prefix in prefixes) class _CrewAITraceConfig(TraceConfig): @@ -284,14 +309,10 @@ def __init__(self, config: TraceConfig) -> None: def mask(self, key: str, value: Any) -> Any: hide_input = ( - self.hide_inputs - or self.hide_input_text - or self.hide_input_messages + self.hide_inputs or self.hide_input_text or self.hide_input_messages ) hide_output = ( - self.hide_outputs - or self.hide_output_text - or self.hide_output_messages + self.hide_outputs or self.hide_output_text or self.hide_output_messages ) if (hide_input and _has_prefix(key, _INPUT_ALIASES)) or ( hide_output and _has_prefix(key, _OUTPUT_ALIASES) diff --git a/python/frameworks/crewai/traceai_crewai/_telemetry.py b/python/frameworks/crewai/traceai_crewai/_telemetry.py deleted file mode 100644 index 18603488c..000000000 --- a/python/frameworks/crewai/traceai_crewai/_telemetry.py +++ /dev/null @@ -1,66 +0,0 @@ -"""Disable CrewAI's separate native exporter without disabling the OTel SDK.""" - -from importlib import import_module -from inspect import getattr_static -from typing import Any, List, Tuple - - -class _NativeTelemetryGuard: - """Reversible, version-aware opt-out while traceAI owns CrewAI tracing. - - Instrument before constructing agents/crews. This cannot retract telemetry - queued before instrumentation or stop unrelated application exporters. - """ - - def __init__(self) -> None: - self._originals: List[Tuple[Any, str, Any, bool]] = [] - - def _replace(self, owner: Any, name: str, replacement: Any) -> None: - owned = name in vars(owner) - self._originals.append((owner, name, getattr_static(owner, name), owned)) - setattr(owner, name, replacement) - - def enable(self) -> None: - telemetry = import_module("crewai.telemetry").Telemetry - classes = [telemetry] - # Maintained CrewAI factors the exporter/disable checks into crewai-core. - try: - classes.append(import_module("crewai_core.telemetry").Telemetry) - except ModuleNotFoundError as exc: - if exc.name not in ("crewai_core", "crewai_core.telemetry"): - raise - for cls in classes: - if hasattr(cls, "_is_telemetry_disabled"): - self._replace( - cls, "_is_telemetry_disabled", classmethod(lambda cls: True) - ) - else: - # CrewAI 0.41.1 ignores disable flags and creates an HTTP exporter - # in __init__. Do not construct that exporter in the first place. - def disabled(instance: Any) -> None: - instance.ready = False - instance.trace_set = False - - self._replace(cls, "__init__", disabled) - # Also stop future calls on instances constructed before instrument(). - for name in ( - "set_tracer", - "crew_creation", - "task_started", - "task_ended", - "tool_repeated_usage", - "tool_usage", - "tool_usage_error", - "crew_execution_span", - "end_crew", - ): - if hasattr(cls, name): - self._replace(cls, name, lambda *args, **kwargs: None) - - def disable(self) -> None: - for owner, name, original, owned in reversed(self._originals): - if owned: - setattr(owner, name, original) - else: - delattr(owner, name) - self._originals.clear() diff --git a/python/frameworks/crewai/traceai_crewai/_tracer.py b/python/frameworks/crewai/traceai_crewai/_tracer.py index ee60cebb5..4a0ac1e4b 100644 --- a/python/frameworks/crewai/traceai_crewai/_tracer.py +++ b/python/frameworks/crewai/traceai_crewai/_tracer.py @@ -10,11 +10,7 @@ def _project_attributes(attributes): # Keep selector/value siblings together before core's scalar masking; leave # unrelated values and core's lazy evaluation semantics unchanged. - return ( - None - if attributes is None - else _redact_header_records(dict(attributes)) - ) + return None if attributes is None else _redact_header_records(dict(attributes)) class _CrewAISpan(wrapt.ObjectProxy): @@ -36,9 +32,7 @@ def set_attributes(self, attributes): self.__wrapped__.set_attributes(_project_attributes(attributes)) def add_event(self, name, attributes=None, timestamp=None): - self.__wrapped__.add_event( - name, _project_attributes(attributes), timestamp - ) + self.__wrapped__.add_event(name, _project_attributes(attributes), timestamp) def record_exception( self, exception, attributes=None, timestamp=None, escaped=False diff --git a/python/frameworks/crewai/traceai_crewai/_wrappers.py b/python/frameworks/crewai/traceai_crewai/_wrappers.py index a968d7764..aed1d7e0b 100644 --- a/python/frameworks/crewai/traceai_crewai/_wrappers.py +++ b/python/frameworks/crewai/traceai_crewai/_wrappers.py @@ -3,7 +3,6 @@ from inspect import signature from typing import Any, Callable, Iterator, List, Mapping, Optional, Tuple -from crewai import Agent from crewai.tools.tool_calling import ToolCalling from fi_instrumentation import get_attributes_from_context from fi_instrumentation.fi_types import FiSpanKindValues, SpanAttributes @@ -95,16 +94,11 @@ def _task_input( def _get_input_value(method: Callable[..., Any], *args: Any, **kwargs: Any) -> str: - """ - Parses a method call's inputs into a JSON string. Ensures a consistent - output regardless of whether the those inputs are passed as positional or - keyword arguments. - """ - - # For typical class methods, the corresponding instance of inspect.Signature - # does not include the self parameter. However, the inspect.Signature - # instance for __call__ does include the self parameter. + """Project only ToolCalling's documented application fields. + ToolUsage also receives the runtime tool and parsing state: none of those + arguments may enter a generic bound-arguments/object serialization fallback. + """ bound = _bound_call(method, args, kwargs) calling = bound.get("calling", kwargs.get("calling")) if isinstance(calling, ToolCalling): @@ -114,15 +108,7 @@ def _get_input_value(method: Callable[..., Any], *args: Any, **kwargs: Any) -> s "tool_args": _safe_data(calling.arguments), } ) - if args and isinstance(args[0], Agent): - return json.dumps( - { - "role": _safe_data(args[0].role), - "goal": _safe_data(args[0].goal), - "backstory": _safe_data(args[0].backstory), - } - ) - return json.dumps(_safe_data({k: v for k, v in bound.items() if k != "self"})) + return json.dumps({}) class _ExecuteCoreWrapper: @@ -246,7 +232,6 @@ def __call__( "verbose?": agent.verbose, "max_iter": agent.max_iter, "max_rpm": agent.max_rpm, - "i18n": agent.i18n.prompt_file, "delegation_enabled": agent.allow_delegation, "tools_names": [ tool.name.casefold() diff --git a/python/tests/test_framework_crewai.py b/python/tests/test_framework_crewai.py index 5947b35ad..c21220b3b 100644 --- a/python/tests/test_framework_crewai.py +++ b/python/tests/test_framework_crewai.py @@ -38,10 +38,9 @@ def test_instrumentation_dependencies(self): deps = instrumentor.instrumentation_dependencies() assert "crewai >= 0.41.1" in deps - @patch("traceai_crewai._NativeTelemetryGuard") @patch("traceai_crewai.import_module") @patch("traceai_crewai.wrap_function_wrapper") - def test_instrument_with_defaults(self, mock_wrap, mock_import, mock_guard): + def test_instrument_with_defaults(self, mock_wrap, mock_import): """Test instrumentation with default parameters.""" mock_tracer_provider = Mock() mock_task_module = Mock() @@ -97,7 +96,6 @@ def test_uninstrument(self, mock_import): instrumentor._original_execute_core = original_execute instrumentor._original_kickoff = original_kickoff instrumentor._original_tool_use = original_tool_use - instrumentor._native_telemetry_guard = Mock() instrumentor._uninstrument() @@ -704,24 +702,14 @@ def test_get_input_value_with_tool_calling(self): assert parsed_result["tool_name"] == "search" assert parsed_result["tool_args"] == {"query": "test"} - def test_get_input_value_with_agent(self): - """Test _get_input_value with Agent object.""" - # Mock Agent + def test_get_input_value_does_not_serialize_agent_arguments(self): + """Only explicit ToolCalling content belongs in a ToolUsage input span.""" mock_agent = Mock() - mock_agent.role = "researcher" - mock_agent.goal = "research topics" - mock_agent.backstory = "expert in research" - - # Import the real class for isinstance check - with patch("traceai_crewai._wrappers.Agent", mock_agent.__class__): - mock_method = Mock() - - result = _get_input_value(mock_method, mock_agent) - - parsed_result = json.loads(result) - assert parsed_result["role"] == "researcher" - assert parsed_result["goal"] == "research topics" - assert parsed_result["backstory"] == "expert in research" + mock_agent.api_key = "FAKE_SHARED_AGENT_KEY" + mock_method = Mock() + assert json.loads(_get_input_value(mock_method, mock_agent)) == {} + mock_agent.to_dict.assert_not_called() + mock_agent.model_dump.assert_not_called() def test_projection_nested_containers_does_not_introspect_runtime(self): runtime = Mock() @@ -741,8 +729,5 @@ def method(arg, **kwargs): projected = json.loads( _get_input_value(method, runtime, api_key="FAKE_SHARED_KEY") ) - assert projected == { - "arg": "[omitted runtime object]", - "kwargs": {"api_key": "__REDACTED__"}, - } + assert projected == {} runtime.to_dict.assert_not_called()