From 3c6e9473406f2a54c0b68affcbf66a5e0ff65c7a Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Sat, 31 Oct 2026 09:52:16 -0700 Subject: [PATCH 1/2] fix(open-agent-spec): replace copied exporter with maintained privacy Delete the unnecessary tracing helper and use released register, LangChainInstrumentor and TraceConfig APIs. Preserve content privacy and scoped LangGraph support, with stronger explicit error masking and offline OTLP regressions. Co-Authored-By: Claude Code --- examples/frameworks/open-agent-spec/README.md | 13 +- examples/frameworks/open-agent-spec/app.py | 14 +- .../open-agent-spec/futureagi_tracing.py | 149 --------- .../open-agent-spec/requirements.txt | 8 +- .../open-agent-spec/tests/load_json_probe.py | 12 +- .../open-agent-spec/tests/test_filter.py | 103 +++--- .../tests/test_maintained_setup.py | 50 +++ .../open-agent-spec/tests/test_no_bridge.py | 4 +- .../open-agent-spec/tests/test_page.py | 65 ++-- .../open-agent-spec/tests/test_privacy.py | 5 +- .../open-agent-spec/tests/test_recipe.py | 6 +- .../frameworks/open-agent-spec.mdx | 309 +++++------------- 12 files changed, 246 insertions(+), 492 deletions(-) delete mode 100644 examples/frameworks/open-agent-spec/futureagi_tracing.py create mode 100644 examples/frameworks/open-agent-spec/tests/test_maintained_setup.py diff --git a/examples/frameworks/open-agent-spec/README.md b/examples/frameworks/open-agent-spec/README.md index 63935a343..f357be059 100644 --- a/examples/frameworks/open-agent-spec/README.md +++ b/examples/frameworks/open-agent-spec/README.md @@ -1,7 +1,7 @@ # Open Agent Spec (pyagentspec) tracing example Runnable companion to the docs page -[Open Agent Spec tracing with Future AGI](../../../src/pages/docs/integrations/frameworks/open-agent-spec.mdx). +[Open Agent Spec tracing with Future AGI](../../../../src/pages/docs/integrations/frameworks/open-agent-spec.mdx). An Oracle Open Agent Spec agent (one `OpenAiCompatibleConfig`, one `ServerTool`) is compiled by pyagentspec's LangGraph adapter and traced into Future AGI with traceAI's @@ -9,18 +9,19 @@ LangChain instrumentor. pyagentspec itself has no OpenTelemetry exporter. | File | What it is | | --- | --- | -| `futureagi_tracing.py` | The recipe: `setup_tracing()` = `register()` + an error-detail filter on the exporter + `LangChainInstrumentor`. Shown in full on the page. | | `weather_agent.py` | The Agent Spec definition and the Python implementation of its tool. | -| `app.py` | Runs the agent: `chat`, `async`, `tool-error`, `model-error`. | +| `app.py` | Configures `register()` and `LangChainInstrumentor` with `TraceConfig(hide_error_details=True)`, then runs `chat`, `async`, `tool-error` or `model-error`. No copied tracing helper. | | `run_e2e.sh` | Tests, then every scenario against a real model and a Future AGI collector, content captured and content hidden. | | `scripts/openinference_variant.py` | Comparison only: the same agent with Arize's `openinference-instrumentation-agentspec` (not a dependency of this recipe). | | `scripts/record_fixture.py`, `scripts/replay_fixture.py` | Record one run as OTLP/JSON; post a recorded fixture to a Future AGI project. | | `fixtures/` | Two recorded runs (real model, content hidden or masked): this recipe, and Arize's instrumentor. | | `tests/` | Hermetic tests: a model stand-in (`fake_openai.py`, not a model) and a loopback OTLP recorder. No keys, no network beyond 127.0.0.1. | -Tested 2026-10-16: `pyagentspec` 26.3.1 (`langgraph` 1.2.15, `langchain-core` 1.6.11, -`langchain-openai` 1.2.2), `traceai-langchain` 0.2.0, `fi-instrumentation-otel` 1.1.0, -`opentelemetry-sdk` 1.45.1, Python 3.13.15 and 3.10.17. LangGraph adapter only. +Requires the released `pyagentspec[langgraph]` 26.3.1, `traceai-langchain` 0.2.1 and +`fi-instrumentation-otel` 1.2.2. LangGraph adapter only. Error messages, error status +descriptions and stack traces are omitted independently of input/output capture; +`ERROR` status and exception type remain available. Set both `FI_HIDE_INPUTS=true` +and `FI_HIDE_OUTPUTS=true` to hide prompts, answers and tool payloads as well. ## Run the tests diff --git a/examples/frameworks/open-agent-spec/app.py b/examples/frameworks/open-agent-spec/app.py index 5cf9c49ba..e50ff07fa 100644 --- a/examples/frameworks/open-agent-spec/app.py +++ b/examples/frameworks/open-agent-spec/app.py @@ -25,10 +25,20 @@ import os import sys -from futureagi_tracing import setup_tracing +from fi_instrumentation import TraceConfig, register +from fi_instrumentation.fi_types import ProjectType +from traceai_langchain import LangChainInstrumentor # Tracing first, before the agent is loaded and run. -tracer_provider = setup_tracing(project_name=os.environ.get("FI_PROJECT_NAME", "open-agent-spec")) +tracer_provider = register( + project_name=os.environ.get("FI_PROJECT_NAME", "open-agent-spec"), + project_type=ProjectType.OBSERVE, + verbose=False, +) +LangChainInstrumentor().instrument( + tracer_provider=tracer_provider, + config=TraceConfig(hide_error_details=True), +) import weather_agent # noqa: E402 diff --git a/examples/frameworks/open-agent-spec/futureagi_tracing.py b/examples/frameworks/open-agent-spec/futureagi_tracing.py deleted file mode 100644 index cf86a4ec5..000000000 --- a/examples/frameworks/open-agent-spec/futureagi_tracing.py +++ /dev/null @@ -1,149 +0,0 @@ -"""Send Open Agent Spec (pyagentspec) runs on the LangGraph adapter to Future AGI. - -pyagentspec has no OpenTelemetry exporter. Its tracing package (``pyagentspec.tracing``) -is a span bus: a ``Trace`` hands Agent Spec spans and events to ``SpanProcessor`` objects, -and nothing in pyagentspec 26.3.1 imports OpenTelemetry. Registering a tracer provider -alone exports nothing. - -The LangGraph adapter (``pyagentspec.adapters.langgraph.AgentSpecLoader``) compiles an -Agent Spec ``Agent`` into a LangGraph graph and runs it with LangChain chat models and -tools. traceAI's LangChain instrumentor traces that run: one trace per agent run, the -agent graph (named after the Agent Spec agent) as the root, a span per model call with -model and token counts, and a span per tool call. - -``setup_tracing()`` does three things: - -* ``register()`` builds the Future AGI tracer provider: resource ``project_name`` and - ``project_type=observe``, OTLP/HTTP to ``$FI_BASE_URL/tracer/v1/traces`` (no compression), and - the ``X-Api-Key`` / ``X-Secret-Key`` headers from ``FI_API_KEY`` / ``FI_SECRET_KEY``; -* the exporter is wrapped in ``ErrorDetailFilter``. LangChain writes the whole Python - traceback, with local file paths, into the status of a failed span, and OpenTelemetry - adds an ``exception.stacktrace`` event attribute. Neither is ever exported. With content - hidden, the error message is also reduced to the exception class name; -* ``LangChainInstrumentor`` is instrumented on that provider. - -Content follows traceAI's default: prompts, model outputs, tool arguments and tool -results are captured. Set ``FI_HIDE_INPUTS=true`` and ``FI_HIDE_OUTPUTS=true`` (or pass -``TraceConfig(hide_inputs=True, hide_outputs=True)``) to turn that off. - -Call ``setup_tracing()`` once, before the first agent run. -""" - -from __future__ import annotations - -import re -from typing import Optional, Sequence - -from fi_instrumentation import TraceConfig, register -from fi_instrumentation.fi_types import ProjectType -from fi_instrumentation.otel import HTTPSpanExporter, TracerProvider -from opentelemetry.sdk.trace import Event, ReadableSpan -from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult -from opentelemetry.trace import Status, StatusCode -from traceai_langchain import LangChainInstrumentor - -__all__ = ["ErrorDetailFilter", "setup_tracing", "strip_error_detail"] - -_TRACEBACK = "Traceback (most recent call last)" -_EXCEPTION_CLASS = re.compile(r"^\s*([A-Za-z_][\w.]*)\s*[(:]") - - -def _exception_type(description: str, events: Sequence[Event]) -> Optional[str]: - for event in events: - if event.name == "exception" and event.attributes: - value = event.attributes.get("exception.type") - if value: - return str(value) - match = _EXCEPTION_CLASS.match(description or "") - return match.group(1) if match else None - - -def strip_error_detail(description: Optional[str], hide_error_text: bool, - events: Sequence[Event] = ()) -> Optional[str]: - """Return a span status description that is safe to export. - - LangChain sets ``repr(error)`` followed by the full traceback. The traceback is - always removed. With ``hide_error_text`` only the exception class name is kept. - """ - if not description: - return description - if hide_error_text: - return _exception_type(description, events) or "error" - head = description.split(_TRACEBACK, 1)[0].strip() - return head or (_exception_type(description, events) or "error") - - -def _filter_event(event: Event, hide_error_text: bool) -> Event: - if event.name != "exception": - return event - drop = {"exception.stacktrace"} - if hide_error_text: - drop.add("exception.message") - attributes = {k: v for k, v in (event.attributes or {}).items() if k not in drop} - return Event(name=event.name, attributes=attributes, timestamp=event.timestamp) - - -def _filter_span(span: ReadableSpan, hide_error_text: bool) -> ReadableSpan: - events = tuple(_filter_event(e, hide_error_text) for e in span.events) - status = span.status - if status.status_code == StatusCode.ERROR: - status = Status(StatusCode.ERROR, - strip_error_detail(status.description, hide_error_text, span.events)) - attributes = dict(span.attributes or {}) - attributes.pop("exception.stacktrace", None) - if hide_error_text: - attributes.pop("exception.message", None) - return ReadableSpan( - name=span.name, - context=span.context, - parent=span.parent, - resource=span.resource, - attributes=attributes, - events=events, - links=span.links, - kind=span.kind, - status=status, - start_time=span.start_time, - end_time=span.end_time, - instrumentation_scope=span.instrumentation_scope, - ) - - -class ErrorDetailFilter(SpanExporter): - """Wrap a span exporter so that error details leave the process only as far as allowed. - - * a stack trace is never exported (span status, ``exception`` events, attributes); - * with ``hide_error_text`` the error message is replaced by the exception class name. - """ - - def __init__(self, exporter: SpanExporter, hide_error_text: bool) -> None: - self._exporter = exporter - self._hide_error_text = hide_error_text - - def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult: - return self._exporter.export([_filter_span(s, self._hide_error_text) for s in spans]) - - def shutdown(self) -> None: - self._exporter.shutdown() - - def force_flush(self, timeout_millis: int = 30000) -> bool: - return self._exporter.force_flush(timeout_millis) - - -def setup_tracing(project_name: Optional[str] = None, - config: Optional[TraceConfig] = None) -> TracerProvider: - """Register the Future AGI tracer provider and instrument LangChain/LangGraph. - - ``project_name`` defaults to ``$FI_PROJECT_NAME``. ``config`` defaults to - ``TraceConfig()``, which reads the ``FI_HIDE_*`` environment variables. - """ - config = config or TraceConfig() - hide_error_text = bool(config.hide_inputs or config.hide_outputs) - tracer_provider = register( - project_name=project_name, - project_type=ProjectType.OBSERVE, - span_exporter=ErrorDetailFilter(HTTPSpanExporter(), hide_error_text), - verbose=False, - ) - LangChainInstrumentor().instrument(tracer_provider=tracer_provider, config=config) - return tracer_provider diff --git a/examples/frameworks/open-agent-spec/requirements.txt b/examples/frameworks/open-agent-spec/requirements.txt index d1c6d390d..e47042a8b 100644 --- a/examples/frameworks/open-agent-spec/requirements.txt +++ b/examples/frameworks/open-agent-spec/requirements.txt @@ -1,5 +1,5 @@ -# Tested 2026-10-16 with Python 3.10 and 3.13 (traceAI supports 3.10 to 3.13). +# Released packages for the LangGraph adapter example and offline tests. pyagentspec[langgraph]==26.3.1 -traceai-langchain==0.2.0 -fi-instrumentation-otel==1.1.0 -pytest==9.1.1 +traceai-langchain==0.2.1 +fi-instrumentation-otel==1.2.2 +pytest==9.2.0 diff --git a/examples/frameworks/open-agent-spec/tests/load_json_probe.py b/examples/frameworks/open-agent-spec/tests/load_json_probe.py index 69b276237..bb1da7237 100644 --- a/examples/frameworks/open-agent-spec/tests/load_json_probe.py +++ b/examples/frameworks/open-agent-spec/tests/load_json_probe.py @@ -1,5 +1,5 @@ """Probe for test_load_json.py: the same agent, serialized to Agent Spec JSON and loaded back -with ``AgentSpecLoader.load_json``, traced with ``setup_tracing()``. Prints the JSON's facts.""" +with ``AgentSpecLoader.load_json``, traced with the maintained APIs. Prints the JSON's facts.""" import json import os @@ -8,9 +8,15 @@ sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) -from futureagi_tracing import setup_tracing # noqa: E402 +from fi_instrumentation import TraceConfig, register # noqa: E402 +from fi_instrumentation.fi_types import ProjectType # noqa: E402 +from traceai_langchain import LangChainInstrumentor # noqa: E402 -tracer_provider = setup_tracing() +tracer_provider = register(project_type=ProjectType.OBSERVE, verbose=False) +LangChainInstrumentor().instrument( + tracer_provider=tracer_provider, + config=TraceConfig(hide_error_details=True), +) import weather_agent # noqa: E402 from pyagentspec.adapters.langgraph import AgentSpecLoader # noqa: E402 diff --git a/examples/frameworks/open-agent-spec/tests/test_filter.py b/examples/frameworks/open-agent-spec/tests/test_filter.py index 66da38b90..208edfb23 100644 --- a/examples/frameworks/open-agent-spec/tests/test_filter.py +++ b/examples/frameworks/open-agent-spec/tests/test_filter.py @@ -1,82 +1,59 @@ -"""Unit tests for ErrorDetailFilter / strip_error_detail on SDK spans (no network).""" +"""The released TraceConfig replaces the copied exporter filter (no network).""" from __future__ import annotations +import pytest + +from fi_instrumentation import FITracer, TraceConfig from opentelemetry.sdk.resources import Resource from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import SimpleSpanProcessor from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter from opentelemetry.trace import Status, StatusCode -from futureagi_tracing import ErrorDetailFilter, strip_error_detail - -LANGCHAIN_ERROR = ( - "RuntimeError('weather service unavailable for Lisbon')Traceback (most recent call last):\n" - '\n File "/Users/someone/app/tools.py", line 3, in get_weather\n raise RuntimeError(...)\n' -) - -def make_spans(hide_error_text: bool): +@pytest.mark.parametrize("hide_content", [False, True]) +def test_maintained_config_hides_error_details_and_preserves_metadata(hide_content): memory = InMemorySpanExporter() provider = TracerProvider(resource=Resource({"project_name": "p"})) - provider.add_span_processor(SimpleSpanProcessor(ErrorDetailFilter(memory, hide_error_text))) - tracer = provider.get_tracer("test") - with tracer.start_as_current_span("ok", attributes={"input.value": "hello"}): - pass - with tracer.start_as_current_span("failed") as span: - try: - raise RuntimeError("weather service unavailable for Lisbon") - except RuntimeError as error: - span.record_exception(error) - span.set_status(Status(StatusCode.ERROR, LANGCHAIN_ERROR)) - provider.shutdown() - return {s.name: s for s in memory.get_finished_spans()} - - -def test_strip_keeps_message_without_traceback(): - assert strip_error_detail(LANGCHAIN_ERROR, False) == "RuntimeError('weather service unavailable for Lisbon')" - - -def test_strip_hidden_keeps_only_the_class_name(): - assert strip_error_detail(LANGCHAIN_ERROR, True) == "RuntimeError" - assert strip_error_detail("openai.AuthenticationError: 401", True) == "openai.AuthenticationError" - assert strip_error_detail("no class here", True) == "error" - assert strip_error_detail(None, True) is None - assert strip_error_detail("", False) == "" - - -def test_filter_default_drops_stack_trace_only(): - spans = make_spans(hide_error_text=False) + provider.add_span_processor(SimpleSpanProcessor(memory)) + tracer = FITracer( + provider.get_tracer("test"), + config=TraceConfig( + hide_inputs=hide_content, + hide_outputs=hide_content, + hide_error_details=True, + ), + ) + try: + with tracer.start_as_current_span("ok", attributes={"input.value": "hello"}): + pass + with tracer.start_as_current_span("failed") as span: + try: + raise RuntimeError("private error message /private/tools.py") + except RuntimeError as error: + span.record_exception(error) + span.set_attribute("exception.message", "private error message") + span.set_attribute("exception.stacktrace", "private traceback") + span.set_status(Status(StatusCode.ERROR, "private error description")) + assert provider.force_flush() + finally: + provider.shutdown() + spans = {span.name: span for span in memory.get_finished_spans()} failed = spans["failed"] - assert failed.status.description == "RuntimeError('weather service unavailable for Lisbon')" + assert failed.status.status_code == StatusCode.ERROR + assert failed.status.description is None (event,) = failed.events assert event.name == "exception" - assert "exception.stacktrace" not in event.attributes - assert event.attributes["exception.message"] == "weather service unavailable for Lisbon" assert event.attributes["exception.type"] == "RuntimeError" - assert spans["ok"].attributes["input.value"] == "hello" + assert "exception.message" not in event.attributes + assert "exception.stacktrace" not in event.attributes + assert "exception.message" not in failed.attributes + assert "exception.stacktrace" not in failed.attributes + assert spans["ok"].attributes["input.value"] == ("__REDACTED__" if hide_content else "hello") assert failed.context.trace_id and failed.resource.attributes["project_name"] == "p" -def test_filter_hidden_drops_message_and_stack_trace(): - failed = make_spans(hide_error_text=True)["failed"] - assert failed.status.description == "RuntimeError" - (event,) = failed.events - assert set(event.attributes) <= {"exception.type", "exception.escaped"} - - -def test_filter_forwards_flush_and_shutdown(): - calls = [] - - class Inner(InMemorySpanExporter): - def force_flush(self, timeout_millis=30000): - calls.append(("flush", timeout_millis)) - return True - - def shutdown(self): - calls.append(("shutdown", None)) - - wrapper = ErrorDetailFilter(Inner(), False) - assert wrapper.force_flush(5) is True - wrapper.shutdown() - assert calls == [("flush", 5), ("shutdown", None)] +def test_explicit_error_privacy_overrides_environment(monkeypatch): + monkeypatch.setenv("FI_HIDE_ERROR_DETAILS", "false") + assert TraceConfig(hide_error_details=True).hide_error_details diff --git a/examples/frameworks/open-agent-spec/tests/test_maintained_setup.py b/examples/frameworks/open-agent-spec/tests/test_maintained_setup.py new file mode 100644 index 000000000..3ed09134b --- /dev/null +++ b/examples/frameworks/open-agent-spec/tests/test_maintained_setup.py @@ -0,0 +1,50 @@ +"""Regression contracts for the maintained setup; synthetic model and loopback OTLP only.""" + +from pathlib import Path + +import pytest + +from conftest import by_name + +EXAMPLE = Path(__file__).resolve().parent.parent +PAGE = EXAMPLE.parents[2] / "src/pages/docs/integrations/frameworks/open-agent-spec.mdx" + + +def test_page_uses_released_privacy_api_without_copied_exporter(): + text = PAGE.read_text(encoding="utf-8") + assert "TraceConfig(hide_error_details=True)" in text + assert '"fi-instrumentation-otel>=1.2.2"' in text + assert '"traceai-langchain>=0.2.1"' in text + assert "ErrorDetailFilter" not in text + assert "futureagi_tracing" not in text + assert not (EXAMPLE / "futureagi_tracing.py").exists() + + +@pytest.mark.parametrize("scenario,exception_type", [ + ("tool-error", "RuntimeError"), + ("model-error", "openai.AuthenticationError"), +]) +@pytest.mark.parametrize("hide_content", [False, True]) +def test_error_details_hidden_independently_of_content(run_app, scenario, exception_type, hide_content): + # Explicit application config wins even over an environment opt-out. + run = run_app( + scenario, "What is the weather in Lisbon right now?", + FI_HIDE_INPUTS=str(hide_content).lower(), + FI_HIDE_OUTPUTS=str(hide_content).lower(), + FI_HIDE_ERROR_DETAILS="false", + ) + assert run.returncode == 1, run.stderr[-2000:] + assert run.recorder.requests + raw = run.recorder.raw() + for detail in (b"weather service unavailable", b"Invalid API key", b"Traceback (most recent call last)"): + assert detail not in raw + errors = [span for span in run.spans() if span["status"].get("code") == "STATUS_CODE_ERROR"] + assert errors + assert all(not span["status"].get("message") for span in errors) + events = [event for span in errors for event in span["events"] if event["name"] == "exception"] + assert events + assert exception_type in {event["attributes"]["exception.type"] for event in events} + assert all("exception.message" not in event["attributes"] for event in events) + assert all("exception.stacktrace" not in event["attributes"] for event in events) + if scenario == "tool-error": + assert by_name(run.spans(), "get_weather")[0]["status"]["code"] == "STATUS_CODE_ERROR" diff --git a/examples/frameworks/open-agent-spec/tests/test_no_bridge.py b/examples/frameworks/open-agent-spec/tests/test_no_bridge.py index bbbc11dda..49c610b0e 100644 --- a/examples/frameworks/open-agent-spec/tests/test_no_bridge.py +++ b/examples/frameworks/open-agent-spec/tests/test_no_bridge.py @@ -50,5 +50,5 @@ def test_adapter_emits_on_the_bus_only(): def test_tested_versions(): assert importlib.metadata.version("pyagentspec") == "26.3.1" - assert importlib.metadata.version("traceai-langchain") == "0.2.0" - assert importlib.metadata.version("fi-instrumentation-otel") == "1.1.0" + assert importlib.metadata.version("traceai-langchain") == "0.2.1" + assert importlib.metadata.version("fi-instrumentation-otel") == "1.2.2" diff --git a/examples/frameworks/open-agent-spec/tests/test_page.py b/examples/frameworks/open-agent-spec/tests/test_page.py index 925ef4f18..cec97d8b3 100644 --- a/examples/frameworks/open-agent-spec/tests/test_page.py +++ b/examples/frameworks/open-agent-spec/tests/test_page.py @@ -1,19 +1,10 @@ -"""The docs page shows exactly what this example ships and tests. - -* the ``futureagi_tracing.py`` implementation matches this directory's file (the page has a shorter module docstring); -* the ``app.py`` block on the page runs as written (model stand-in, loopback recorder); -* the Arize snippet on the page is what ``scripts/openinference_variant.py`` runs; -* the install line pins the version in requirements.txt. - -Skipped when the example is used outside the docs repository. -""" +"""Run the page's maintained setup as written against loopback model and OTLP servers.""" from __future__ import annotations import os import re import secrets -import shutil import subprocess import sys from pathlib import Path @@ -36,52 +27,60 @@ def block(title: str) -> str: return match.group(1) -def test_tracing_file_on_the_page_is_the_shipped_file(): - on_page = block("futureagi_tracing.py") - shipped = (EXAMPLE / "futureagi_tracing.py").read_text(encoding="utf-8") - # Only the introductory module docstring differs; all executable code must match exactly. - assert on_page.startswith('"""') and shipped.startswith('"""') - assert on_page.split('"""', 2)[2] == shipped.split('"""', 2)[2] - - def test_self_hosted_origin_is_set_explicitly(): text = PAGE.read_text(encoding="utf-8") assert '\nexport FI_BASE_URL="http://localhost:4318"' in text assert '# export FI_BASE_URL=' not in text -def test_app_on_the_page_runs_as_written(tmp_path): - shutil.copy(EXAMPLE / "futureagi_tracing.py", tmp_path / "futureagi_tracing.py") +@pytest.mark.parametrize("hide_content", [False, True]) +def test_app_on_the_page_runs_as_written(tmp_path, hide_content): (tmp_path / "app.py").write_text(block("app.py"), encoding="utf-8") key = "model-key-" + secrets.token_hex(8) with Recorder() as recorder, FakeOpenAI(key) as fake: env = {k: v for k, v in os.environ.items() if not k.startswith(("FI_", "OTEL_", "MODEL_"))} env.update(FI_API_KEY="fi-a", FI_SECRET_KEY="fi-b", FI_BASE_URL=recorder.origin, FI_PROJECT_NAME="oas-page", MODEL_BASE_URL=fake.url, MODEL_API_KEY=key, - MODEL_ID="fake-model") - out = subprocess.run([sys.executable, "app.py"], cwd=str(tmp_path), env=env, + MODEL_ID="fake-model", FI_HIDE_INPUTS=str(hide_content).lower(), + FI_HIDE_OUTPUTS=str(hide_content).lower()) + out = subprocess.run([sys.executable, "-I", str(tmp_path / "app.py")], env=env, capture_output=True, text=True, timeout=180) spans = recorder.spans() + raw = recorder.raw() assert out.returncode == 0, out.stderr[-2000:] assert "It is sunny" in out.stdout assert [s["name"] for s in spans if not s["parentSpanId"]] == ["weather_assistant"] assert sorted(kind(s) for s in spans) == ["CHAIN"] * 4 + ["LLM"] * 2 + ["TOOL"] assert {s["resource"]["project_name"] for s in spans} == {"oas-page"} + assert key.encode() not in raw + assert (b"What is the weather in Lisbon?" in raw) is not hide_content -def test_arize_snippet_matches_the_variant_script(): - text = PAGE.read_text(encoding="utf-8") - variant = (EXAMPLE / "scripts" / "openinference_variant.py").read_text(encoding="utf-8") - for line in ("tracer_provider = register(project_type=ProjectType.OBSERVE)", - "AgentSpecInstrumentor().instrument(tracer_provider=tracer_provider, mask_sensitive_information="): - assert line in text and line in variant, line +def test_app_on_the_page_flushes_on_failure(tmp_path): + (tmp_path / "app.py").write_text(block("app.py"), encoding="utf-8") + with Recorder() as recorder, FakeOpenAI("expected-key") as fake: + env = {k: v for k, v in os.environ.items() if not k.startswith(("FI_", "OTEL_", "MODEL_"))} + env.update(FI_API_KEY="fi-a", FI_SECRET_KEY="fi-b", FI_BASE_URL=recorder.origin, + FI_PROJECT_NAME="oas-page", MODEL_BASE_URL=fake.url, + MODEL_API_KEY="wrong-synthetic-key", MODEL_ID="fake-model") + out = subprocess.run([sys.executable, "-I", str(tmp_path / "app.py")], env=env, + capture_output=True, text=True, timeout=180) + spans = recorder.spans() + raw = recorder.raw() + assert out.returncode != 0 + assert spans + assert any(s["status"].get("code") == "STATUS_CODE_ERROR" for s in spans) + assert b"Invalid API key" not in raw + assert b"Traceback (most recent call last)" not in raw -def test_page_describes_setup_without_verification_diary(): +def test_page_preserves_scope_without_unqualified_bus_setup(): text = PAGE.read_text(encoding="utf-8") assert "LangGraph adapter" in text assert "FI_HIDE_INPUTS" in text and "FI_HIDE_OUTPUTS" in text - assert "mask_sensitive_information=True" in text + assert "SpanProcessor" in text + assert "AgentSpecInstrumentor().instrument" not in text + assert "Other Agent Spec adapters and `Flow` components are outside" in text assert "How that was checked:" not in text assert "Agent Spec source read" not in text @@ -89,4 +88,8 @@ def test_page_describes_setup_without_verification_diary(): def test_install_line_matches_requirements(): requirements = (EXAMPLE / "requirements.txt").read_text(encoding="utf-8") pin = re.search(r"pyagentspec\[langgraph\]==([\d.]+)", requirements).group(1) - assert f'pip install "pyagentspec[langgraph]=={pin}"' in PAGE.read_text(encoding="utf-8") + text = PAGE.read_text(encoding="utf-8") + assert f'pip install "pyagentspec[langgraph]=={pin}"' in text + for package in ("traceai-langchain", "fi-instrumentation-otel"): + version = re.search(re.escape(package) + r"==([\d.]+)", requirements).group(1) + assert f'"{package}>={version}"' in text diff --git a/examples/frameworks/open-agent-spec/tests/test_privacy.py b/examples/frameworks/open-agent-spec/tests/test_privacy.py index 61e8abafe..819a4ad18 100644 --- a/examples/frameworks/open-agent-spec/tests/test_privacy.py +++ b/examples/frameworks/open-agent-spec/tests/test_privacy.py @@ -1,6 +1,6 @@ """Privacy on the wire. -* default (traceAI's default): content is captured; a stack trace is still never exported; +* content is captured by default; explicit error privacy hides messages and stack traces; * FI_HIDE_INPUTS=true + FI_HIDE_OUTPUTS=true: the marker (in the question, the tool arguments, the tool result, the tool error and the answer) never leaves the process, and neither does the system prompt or the error text. @@ -70,7 +70,8 @@ def test_hidden_keeps_the_metadata(run_app, marker): run = run_app("tool-error", question(marker), **HIDE) spans = run.spans() tool = by_name(spans, "get_weather")[0] - assert tool["status"] == {"code": "STATUS_CODE_ERROR", "message": "RuntimeError"} + assert tool["status"] == {"code": "STATUS_CODE_ERROR"} + assert any(event["attributes"].get("exception.type") == "RuntimeError" for event in tool["events"]) llm = [s for s in spans if kind(s) == "LLM"][0] assert llm["attributes"]["gen_ai.request.model"] == "fake-model" assert llm["attributes"]["gen_ai.usage.input_tokens"] == 17 diff --git a/examples/frameworks/open-agent-spec/tests/test_recipe.py b/examples/frameworks/open-agent-spec/tests/test_recipe.py index bb536be15..dbd8590c5 100644 --- a/examples/frameworks/open-agent-spec/tests/test_recipe.py +++ b/examples/frameworks/open-agent-spec/tests/test_recipe.py @@ -91,7 +91,8 @@ def test_tool_error_is_exported_as_error(run_app): assert len({s["traceId"] for s in spans}) == 1 tool = by_name(spans, "get_weather")[0] assert tool["status"]["code"] == "STATUS_CODE_ERROR" - assert tool["status"]["message"] == "RuntimeError('weather service unavailable for Lisbon')" + assert not tool["status"].get("message") + assert any(event["attributes"].get("exception.type") == "RuntimeError" for event in tool["events"]) root = [s for s in spans if not s["parentSpanId"]][0] assert root["status"]["code"] == "STATUS_CODE_ERROR" @@ -103,5 +104,6 @@ def test_model_error_is_exported_as_error(run_app): spans = run.spans() llm = [s for s in spans if kind(s) == "LLM"][0] assert llm["status"]["code"] == "STATUS_CODE_ERROR" - assert llm["status"]["message"].startswith("AuthenticationError(") + assert not llm["status"].get("message") + assert any(event["attributes"].get("exception.type") == "openai.AuthenticationError" for event in llm["events"]) assert run.env["MODEL_BAD_API_KEY"].encode() not in run.recorder.raw() diff --git a/src/pages/docs/integrations/frameworks/open-agent-spec.mdx b/src/pages/docs/integrations/frameworks/open-agent-spec.mdx index 49b58205a..8ecaa8065 100644 --- a/src/pages/docs/integrations/frameworks/open-agent-spec.mdx +++ b/src/pages/docs/integrations/frameworks/open-agent-spec.mdx @@ -1,178 +1,63 @@ --- title: "Open Agent Spec tracing with Future AGI" -description: "Trace Open Agent Spec agents running on the LangGraph adapter in Future AGI." +description: "Trace Open Agent Spec agents on the LangGraph adapter with maintained traceAI packages, tool-call visibility and configurable content privacy." --- -Trace [Open Agent Spec](https://github.com/oracle/agent-spec) agents running through the LangGraph adapter with traceAI's LangChain instrumentor. This recipe uses `pyagentspec[langgraph]` 26.3.1, an OpenAI-compatible Chat Completions endpoint and a self-hosted Future AGI instance; hosted export is outside this recipe. +Trace [Open Agent Spec](https://github.com/oracle/agent-spec) agents running through the LangGraph adapter with traceAI's LangChain instrumentor. See the agent run, model calls and tool calls in one Future AGI trace. -Use one instrumentor for the LangGraph runtime; do not also instrument it using the [LangGraph](/docs/integrations/traceai/langgraph) or [LangChain](/docs/integrations/traceai/langchain) recipes. The filter below removes Python tracebacks from exported errors. With content capture on, exception messages can still contain local paths or other sensitive text. Other Agent Spec adapters and flows are outside this recipe. +This guide covers `pyagentspec[langgraph]` 26.3.1 agents, an OpenAI-compatible Chat Completions endpoint and a self-hosted Future AGI instance. Other Agent Spec adapters and `Flow` components are outside this guide's scope. + +Use one instrumentor for the runtime. If you already followed the [LangGraph](/docs/integrations/traceai/langgraph) or [LangChain](/docs/integrations/traceai/langchain) guide, reuse that provider and instrumentor rather than registering another. ## 1. Install +Use Python 3.10 to 3.13. For async runs, use Python 3.11 or later so LangChain can propagate parent context across tasks. + ```bash -pip install "pyagentspec[langgraph]==26.3.1" traceai-langchain fi-instrumentation-otel +pip install "pyagentspec[langgraph]==26.3.1" "traceai-langchain>=0.2.1" "fi-instrumentation-otel>=1.2.2" ``` -traceAI supports Python 3.10 to 3.13. The `langgraph` extra installs LangGraph and `langchain-openai`. +The `langgraph` extra installs LangGraph and `langchain-openai`. These traceAI versions provide the maintained error-privacy configuration used below. ## 2. Set environment variables ```bash export FI_API_KEY="your-futureagi-api-key" export FI_SECRET_KEY="your-futureagi-secret-key" -export FI_PROJECT_NAME="agent-spec-weather" # the Observe project the traces land in +export FI_PROJECT_NAME="agent-spec-weather" # Replace with your self-hosted Future AGI origin; traceAI adds /tracer/v1/traces. export FI_BASE_URL="http://localhost:4318" -# Keep prompts, outputs and tool payloads out of the traces (see Privacy): -# export FI_HIDE_INPUTS=true FI_HIDE_OUTPUTS=true -``` -Set `FI_BASE_URL` to the origin only. traceAI adds the path. +export MODEL_BASE_URL="https://your-openai-compatible-endpoint/v1" +export MODEL_API_KEY="your-model-api-key" +export MODEL_ID="your-model-id" -## 3. Add the tracing file +# Optional: keep prompts, outputs and tool payloads out of traces. +# export FI_HIDE_INPUTS=true FI_HIDE_OUTPUTS=true +``` -Save this as `futureagi_tracing.py` next to your agent code. `setup_tracing()` registers the Future AGI tracer provider, wraps its exporter in a filter for error details, and turns on the LangChain instrumentor. +Set `FI_BASE_URL` to the origin only, without `/tracer/v1/traces`. Use your model provider's key for `MODEL_API_KEY`, not your Future AGI keys. -```python title="futureagi_tracing.py" -"""Trace Agent Spec's LangGraph adapter with traceAI's LangChain instrumentor. + + Prompts, answers and tool payloads are captured by default. Set both `FI_HIDE_INPUTS=true` and `FI_HIDE_OUTPUTS=true` before running the application if you do not want that content exported. The configuration below suppresses exception messages and stack traces independently of these content settings. + -Call setup_tracing() once before running the agent. ErrorDetailFilter removes -tracebacks; with either input or output hiding enabled it also hides error text. -Set FI_HIDE_INPUTS=true and FI_HIDE_OUTPUTS=true to suppress prompts and outputs. -""" +## 3. Configure tracing and run the agent -from __future__ import annotations +Save this as `app.py`. Call `register()` and `LangChainInstrumentor().instrument()` once, before loading the agent. No custom exporter or tracing helper is needed. -import re -from typing import Optional, Sequence +```python title="app.py" +import os from fi_instrumentation import TraceConfig, register from fi_instrumentation.fi_types import ProjectType -from fi_instrumentation.otel import HTTPSpanExporter, TracerProvider -from opentelemetry.sdk.trace import Event, ReadableSpan -from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult -from opentelemetry.trace import Status, StatusCode from traceai_langchain import LangChainInstrumentor -__all__ = ["ErrorDetailFilter", "setup_tracing", "strip_error_detail"] - -_TRACEBACK = "Traceback (most recent call last)" -_EXCEPTION_CLASS = re.compile(r"^\s*([A-Za-z_][\w.]*)\s*[(:]") - - -def _exception_type(description: str, events: Sequence[Event]) -> Optional[str]: - for event in events: - if event.name == "exception" and event.attributes: - value = event.attributes.get("exception.type") - if value: - return str(value) - match = _EXCEPTION_CLASS.match(description or "") - return match.group(1) if match else None - - -def strip_error_detail(description: Optional[str], hide_error_text: bool, - events: Sequence[Event] = ()) -> Optional[str]: - """Return a span status description that is safe to export. - - LangChain sets ``repr(error)`` followed by the full traceback. The traceback is - always removed. With ``hide_error_text`` only the exception class name is kept. - """ - if not description: - return description - if hide_error_text: - return _exception_type(description, events) or "error" - head = description.split(_TRACEBACK, 1)[0].strip() - return head or (_exception_type(description, events) or "error") - - -def _filter_event(event: Event, hide_error_text: bool) -> Event: - if event.name != "exception": - return event - drop = {"exception.stacktrace"} - if hide_error_text: - drop.add("exception.message") - attributes = {k: v for k, v in (event.attributes or {}).items() if k not in drop} - return Event(name=event.name, attributes=attributes, timestamp=event.timestamp) - - -def _filter_span(span: ReadableSpan, hide_error_text: bool) -> ReadableSpan: - events = tuple(_filter_event(e, hide_error_text) for e in span.events) - status = span.status - if status.status_code == StatusCode.ERROR: - status = Status(StatusCode.ERROR, - strip_error_detail(status.description, hide_error_text, span.events)) - attributes = dict(span.attributes or {}) - attributes.pop("exception.stacktrace", None) - if hide_error_text: - attributes.pop("exception.message", None) - return ReadableSpan( - name=span.name, - context=span.context, - parent=span.parent, - resource=span.resource, - attributes=attributes, - events=events, - links=span.links, - kind=span.kind, - status=status, - start_time=span.start_time, - end_time=span.end_time, - instrumentation_scope=span.instrumentation_scope, - ) - - -class ErrorDetailFilter(SpanExporter): - """Wrap a span exporter so that error details leave the process only as far as allowed. - - * a stack trace is never exported (span status, ``exception`` events, attributes); - * with ``hide_error_text`` the error message is replaced by the exception class name. - """ - - def __init__(self, exporter: SpanExporter, hide_error_text: bool) -> None: - self._exporter = exporter - self._hide_error_text = hide_error_text - - def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult: - return self._exporter.export([_filter_span(s, self._hide_error_text) for s in spans]) - - def shutdown(self) -> None: - self._exporter.shutdown() - - def force_flush(self, timeout_millis: int = 30000) -> bool: - return self._exporter.force_flush(timeout_millis) - - -def setup_tracing(project_name: Optional[str] = None, - config: Optional[TraceConfig] = None) -> TracerProvider: - """Register the Future AGI tracer provider and instrument LangChain/LangGraph. - - ``project_name`` defaults to ``$FI_PROJECT_NAME``. ``config`` defaults to - ``TraceConfig()``, which reads the ``FI_HIDE_*`` environment variables. - """ - config = config or TraceConfig() - hide_error_text = bool(config.hide_inputs or config.hide_outputs) - tracer_provider = register( - project_name=project_name, - project_type=ProjectType.OBSERVE, - span_exporter=ErrorDetailFilter(HTTPSpanExporter(), hide_error_text), - verbose=False, - ) - LangChainInstrumentor().instrument(tracer_provider=tracer_provider, config=config) - return tracer_provider -``` - -Keep `ErrorDetailFilter` in the export path: it removes Python tracebacks from span status, events and attributes. With input or output hiding enabled, it also reduces error messages to the exception class name. With content capture on, exception text can still contain sensitive data. - -## 4. Call it before you load the agent - -In your entry point, here `app.py`, call `setup_tracing()` first, then build, load and run the agent as usual. This is the example's agent: one LLM config, one server tool. - -```python title="app.py" -from futureagi_tracing import setup_tracing - -tracer_provider = setup_tracing() # project from FI_PROJECT_NAME - -import os +tracer_provider = register(project_type=ProjectType.OBSERVE, verbose=False) +LangChainInstrumentor().instrument( + tracer_provider=tracer_provider, + config=TraceConfig(hide_error_details=True), +) from pyagentspec.adapters.langgraph import AgentSpecLoader from pyagentspec.agent import Agent @@ -185,33 +70,42 @@ def get_weather(city: str) -> str: return f"Sunny, 21 C in {city}." -agent = Agent( - name="weather_assistant", - llm_config=OpenAiCompatibleConfig( - name="chat-model", - url=os.environ["MODEL_BASE_URL"], - model_id=os.environ["MODEL_ID"], - api_key=os.environ["MODEL_API_KEY"], - ), - tools=[ - ServerTool( - name="get_weather", - description="Get the current weather for a city.", - inputs=[StringProperty(title="city")], - outputs=[StringProperty(title="forecast")], - ) - ], - system_prompt="You are a weather assistant. Call get_weather before you answer.", -) +try: + agent = Agent( + name="weather_assistant", + llm_config=OpenAiCompatibleConfig( + name="chat-model", + url=os.environ["MODEL_BASE_URL"], + model_id=os.environ["MODEL_ID"], + api_key=os.environ["MODEL_API_KEY"], + ), + tools=[ + ServerTool( + name="get_weather", + description="Get the current weather for a city.", + inputs=[StringProperty(title="city")], + outputs=[StringProperty(title="forecast")], + ) + ], + system_prompt="You are a weather assistant. Call get_weather before you answer.", + ) -graph = AgentSpecLoader(tool_registry={"get_weather": get_weather}).load_component(agent) -result = graph.invoke({"messages": [{"role": "user", "content": "What is the weather in Lisbon?"}]}) -print(result["messages"][-1].content) + graph = AgentSpecLoader(tool_registry={"get_weather": get_weather}).load_component(agent) + result = graph.invoke({"messages": [{"role": "user", "content": "What is the weather in Lisbon?"}]}) + print(result["messages"][-1].content) +finally: + tracer_provider.force_flush() +``` -tracer_provider.force_flush() # a short script exits before the batch timer fires +```bash +python app.py ``` -An agent stored as Agent Spec JSON is traced the same way. pyagentspec does not write the API key into the JSON; it leaves a reference that you fill in when you load it: +`register()` reads `FI_PROJECT_NAME` and sends traces through its OTLP/HTTP exporter. `force_flush()` sends queued spans before a short script exits, including when an agent run fails. In a long-running service, flush at shutdown rather than after every request. + +### Load an agent from JSON + +To load a serialized definition instead of the in-memory `agent`, replace the `load_component` call with the following. Agent Spec stores a reference in place of the model API key; supply that reference when loading: ```python from pyagentspec.serialization import AgentSpecSerializer @@ -223,79 +117,38 @@ graph = AgentSpecLoader(tool_registry={"get_weather": get_weather}).load_json( ) ``` -## 5. What you see in Future AGI - -One trace per agent run, in the project named by `FI_PROJECT_NAME`: - -```text -weather_assistant chain the Agent Spec agent's name; input = the run's messages -├── model chain LangGraph's model node -│ └── ChatOpenAI llm model, prompt/completion tokens, messages, tool call -├── tools chain LangGraph's tool node -│ └── get_weather tool the Agent Spec ServerTool: arguments and result -└── model chain - └── ChatOpenAI llm model, tokens, final answer -``` - -Two runs in one process are two traces. A failing tool marks the tool span, the `tools` node and the root ERROR. A model call that fails, for example with a rejected API key, marks the `ChatOpenAI` span, the `model` node and the root ERROR. +## 4. Open the trace -### What is captured +Open **Observe** in your self-hosted Future AGI instance, select **agent-spec-weather** and open the latest trace. For a run that calls `get_weather` once, the trace contains: -| Data | Where | Notes | -| --- | --- | --- | -| Agent name | root span name, `lc_agent_name` | The Agent Spec `Agent.name`. | -| Model | `ChatOpenAI` span, model column | From `gen_ai.request.model`. | -| Tokens | `ChatOpenAI` span, token columns | Prompt, completion and total, as the endpoint reported them. | -| Prompt, system prompt, answer | `ChatOpenAI` span messages, input and output | Captured by default. Hidden with `FI_HIDE_INPUTS` / `FI_HIDE_OUTPUTS`. | -| Tool arguments and result | `get_weather` and `tools` spans, input and output | Captured by default. Hidden with the same switches. | -| Errors | span status | `RuntimeError('weather service unavailable for Lisbon')` by default; only the exception class, for example `RuntimeError` or `openai.AuthenticationError`, with content hidden. Never a traceback. | -| Tool schema | `gen_ai.request.parameters` on the `ChatOpenAI` span | The tool name, description and JSON schema from the Agent Spec definition. | +- **weather_assistant**: the root LangGraph chain, named after the Agent Spec agent. +- **model → ChatOpenAI**: one model span per model call, with the model ID and token counts reported by the endpoint. +- **tools → get_weather**: the tool call, including arguments and results when content capture is enabled. -### What is not captured +Separate runs produce separate traces. Failed model and tool calls retain `ERROR` status and an `exception.type` event attribute. Exception messages, status descriptions and stack traces are omitted by this guide's explicit `hide_error_details=True` setting. -- **Agent Spec's own span names and events.** You see LangGraph's tree (`model`, `tools`, `ChatOpenAI`), not `AgentExecution[...]`, `LlmGenerationSpan` or `ToolExecution[...]`. Those exist only on pyagentspec's bus. See [Arize's Agent Spec instrumentor](#arizes-agent-spec-instrumentor) if you need them. -- **An agent span kind.** The root is stored as a `chain`, which is how traceAI's LangChain instrumentor stores the LangGraph graph. -- **Provider.** The provider column stays empty. LangChain reports `ls_provider=openai` as a plain attribute. -- **Session and user.** Not set by the adapter or by this file. -- **Other adapters.** WayFlow, AutoGen, CrewAI, OpenAI Agents and Microsoft Agent Framework run Agent Spec on other frameworks. This recipe is for the LangGraph adapter; the others are outside this guide’s coverage. -- **Flows.** The LangGraph adapter also compiles Agent Spec `Flow` components to LangGraph graphs. Flows are outside this guide’s coverage. +You see LangGraph's runtime tree, not Agent Spec's native span names such as `AgentExecution[...]`. The root uses the `CHAIN` span kind. Session and user identifiers are not set by this example. ## Privacy -Content follows traceAI's default, which is **on**. Prompts, the system prompt, model outputs, tool arguments, tool results and error messages are exported. To turn that off, set both switches: - -```bash -export FI_HIDE_INPUTS=true FI_HIDE_OUTPUTS=true -``` - -or pass a config in code: `setup_tracing(config=TraceConfig(hide_inputs=True, hide_outputs=True))`, with `TraceConfig` from `fi_instrumentation`. Inputs and outputs are then stored as `__REDACTED__`, messages are dropped, and an error keeps only its exception class name. Span names, timing, model, tokens, tool names and ERROR status are kept. - -Never exported, in either mode: Python stack traces and the local file paths in them, the model API key from the Agent Spec `LlmConfig`, and the Future AGI keys (they travel only as the `X-Api-Key` and `X-Secret-Key` headers). With content on, an error message is exported as raised, so it carries whatever the exception text contains. - -## Flushing and shutdown +`TraceConfig(hide_error_details=True)` removes error details even when prompt and response capture is enabled. This is stronger than stripping only the traceback: it also omits exception messages and the error status description, while retaining the exception type and `ERROR` status. The explicit `True` takes precedence over `FI_HIDE_ERROR_DETAILS` in the environment. -`register()` uses a batch span processor, which exports on a timer. A short script should call `tracer_provider.force_flush()` before it exits, as in step 4, so the last run is not left in the queue. Flush on failure as well as on success. - -## Arize's Agent Spec instrumentor - -Arize publishes `openinference-instrumentation-agentspec` ([Arize docs](https://arize.com/docs/ax/integrations/python-agent-frameworks/agentspec/agentspec-tracing)). It is a `SpanProcessor` for pyagentspec's bus that turns Agent Spec spans into OpenTelemetry spans. It is a third-party package, not maintained by Future AGI, and not part of this recipe. It does work with the Future AGI tracer provider: +Input and output privacy remain independent. Set both `FI_HIDE_INPUTS=true` and `FI_HIDE_OUTPUTS=true`, or replace the config in step 3 with: ```python -from fi_instrumentation import register -from fi_instrumentation.fi_types import ProjectType -from openinference.instrumentation.agentspec import AgentSpecInstrumentor - -tracer_provider = register(project_type=ProjectType.OBSERVE) -AgentSpecInstrumentor().instrument(tracer_provider=tracer_provider, mask_sensitive_information=True) +TraceConfig(hide_inputs=True, hide_outputs=True, hide_error_details=True) ``` -Use only one instrumentor: enabling both creates separate traces. Arize's `mask_sensitive_information=True` can still export the agent definition, including the system prompt and model endpoint URL; review that data before using it. With `openinference-instrumentation-agentspec` 0.1.15 and `pyagentspec` 26.3.1, this route reports Agent Spec span names and groups runs under one root exported on `uninstrument()`. The LangGraph adapter leaves token counts empty, and failing calls are not marked `ERROR` as they are in the recipe above. +Input and output values are redacted, and message content is dropped. Span names, timing, model IDs, token counts and tool names remain available. These settings are not a general-purpose secret scrubber: keep sensitive data out of span names, tool descriptions, schemas and custom metadata. Error text that your application returns as normal output is content, not an exception, so use the content switches to hide it. + +### Agent Spec's native tracing interface + +Agent Spec also exposes a `SpanProcessor` interface. [Arize's Agent Spec integration](https://arize.com/docs/ax/integrations/python-agent-frameworks/agentspec/agentspec-tracing) uses that interface instead of LangGraph callbacks. It is a separate third-party route, not a prerequisite for this guide or a guarantee of portable adapter and flow coverage. Do not enable it alongside this LangChain instrumentor: both can observe the same execution and produce duplicate traces. ## Troubleshooting -- **No traces in the project.** Check `FI_API_KEY` and `FI_SECRET_KEY` in the shell that runs the agent, and that `FI_BASE_URL` is the origin only. `setup_tracing()` must run before the agent is loaded. A short script must flush before it exits. -- **`Failed to export spans batch code: 400`.** With a self-hosted collector before v1.56.0, check that `OTEL_EXPORTER_OTLP_COMPRESSION` and `OTEL_EXPORTER_OTLP_TRACES_COMPRESSION` are not set to `gzip`. The exporter honours them. Releases before v1.56.0 answered HTTP 400; current releases accept gzip. -- **Each model call is its own trace (Python 3.10, `ainvoke`).** Before Python 3.11, LangChain cannot carry the run context into asyncio tasks, so the `ChatOpenAI` spans of an `ainvoke` run lose their parent and each becomes its own trace. Plain LangGraph does the same. Use Python 3.11 or later for async runs, or `invoke`. -- **Two traces per run, in two shapes.** Arize's Agent Spec instrumentor is also on. Keep one. -- **The error shows only the exception class.** Content is hidden. Unset `FI_HIDE_INPUTS` and `FI_HIDE_OUTPUTS` to keep the message. The traceback is never exported. -- **No `AgentExecution[...]` spans.** Expected with this recipe. See [Arize's Agent Spec instrumentor](#arizes-agent-spec-instrumentor). +- **No traces in the project.** Check `FI_API_KEY`, `FI_SECRET_KEY` and `FI_BASE_URL` in the shell running the application. Instrument before loading the agent, and flush before the process exits. +- **Unexpected `hide_error_details` argument.** Upgrade `fi-instrumentation-otel` and `traceai-langchain` with the minimum versions in step 1. +- **Each async model call becomes its own trace.** Use Python 3.11 or later for `ainvoke`, or use synchronous `invoke` on Python 3.10. +- **Duplicate traces.** Keep one runtime instrumentor; remove a second LangChain/LangGraph setup or Agent Spec bus instrumentor. +- **The error has no message.** This is intentional. The guide hides error details regardless of input/output capture. Use `exception.type` and `ERROR` status to identify the failure without exporting its text. From 0bbd51d08e491445da6eee461d5cfd2deb5b2e2b Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Sat, 31 Oct 2026 09:52:53 -0700 Subject: [PATCH 2/2] fix(open-agent-spec): retain companion guide link Co-Authored-By: Claude Code --- examples/frameworks/open-agent-spec/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/frameworks/open-agent-spec/README.md b/examples/frameworks/open-agent-spec/README.md index f357be059..a5b8ed418 100644 --- a/examples/frameworks/open-agent-spec/README.md +++ b/examples/frameworks/open-agent-spec/README.md @@ -1,7 +1,7 @@ # Open Agent Spec (pyagentspec) tracing example Runnable companion to the docs page -[Open Agent Spec tracing with Future AGI](../../../../src/pages/docs/integrations/frameworks/open-agent-spec.mdx). +[Open Agent Spec tracing with Future AGI](../../../src/pages/docs/integrations/frameworks/open-agent-spec.mdx). An Oracle Open Agent Spec agent (one `OpenAiCompatibleConfig`, one `ServerTool`) is compiled by pyagentspec's LangGraph adapter and traced into Future AGI with traceAI's