From 5936e0960f01a23e842d3df90b7c101874ed8149 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sat, 3 Oct 2026 18:58:31 -0700 Subject: [PATCH 01/19] test(harness): add shared OTLP harness for TH-8103 contract tests --- python/tests/harness/__init__.py | 184 +++++++++++++++++++++++++++ python/tests/harness/test_harness.py | 101 +++++++++++++++ 2 files changed, 285 insertions(+) create mode 100644 python/tests/harness/__init__.py create mode 100644 python/tests/harness/test_harness.py diff --git a/python/tests/harness/__init__.py b/python/tests/harness/__init__.py new file mode 100644 index 000000000..dfd9198e4 --- /dev/null +++ b/python/tests/harness/__init__.py @@ -0,0 +1,184 @@ +"""Small, loopback-only utilities for OTLP contract tests.""" + +from __future__ import annotations + +import copy +import difflib +import json +import subprocess +import threading +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any, Mapping, Optional, Sequence, Union +from urllib.parse import urlsplit +from urllib.request import Request, urlopen + + +class Receiver: + """An in-memory OTLP/HTTP receiver for tests. + + The receiver only listens on loopback and never forwards received spans. + """ + + def __init__(self) -> None: + self._spans: list[dict[str, Any]] = [] + self._lock = threading.Lock() + owner = self + + class Handler(BaseHTTPRequestHandler): + def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler + if urlsplit(self.path).path != "/v1/traces": + self.send_error(HTTPStatus.NOT_FOUND) + return + + try: + length = int(self.headers.get("Content-Length", "0")) + body = self.rfile.read(length) + content_type = self.headers.get("Content-Type", "").lower() + + if content_type.startswith("application/json"): + request = json.loads(body.decode("utf-8")) + elif content_type.startswith(("application/x-protobuf", "application/protobuf")): + from google.protobuf.json_format import MessageToDict + from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ( + ExportTraceServiceRequest, + ) + + protobuf_request = ExportTraceServiceRequest() + protobuf_request.ParseFromString(body) + request = MessageToDict(protobuf_request) + else: + self.send_error(HTTPStatus.UNSUPPORTED_MEDIA_TYPE) + return + + spans: list[dict[str, Any]] = [] + for resource_spans in request.get( + "resourceSpans", request.get("resource_spans", []) + ): + for scope_spans in resource_spans.get( + "scopeSpans", resource_spans.get("scope_spans", []) + ): + spans.extend(scope_spans.get("spans", [])) + except (ImportError, TypeError, UnicodeDecodeError, ValueError) as error: + self.send_error(HTTPStatus.BAD_REQUEST, str(error)) + return + + with owner._lock: + owner._spans.extend(copy.deepcopy(spans)) + self.send_response(HTTPStatus.OK) + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, _format: str, *args: Any) -> None: + return + + self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.origin = "http://127.0.0.1:{0}".format(self._server.server_port) + self.endpoint = "{0}/v1/traces".format(self.origin) + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + self._thread.start() + + def __enter__(self) -> "Receiver": + return self + + def __exit__(self, _type: Any, _value: Any, _traceback: Any) -> None: + self._server.shutdown() + self._server.server_close() + self._thread.join() + + def spans(self) -> list[dict[str, Any]]: + """Return a copy of the decoded spans received so far.""" + with self._lock: + return copy.deepcopy(self._spans) + + def clear(self) -> None: + """Remove every decoded span received so far.""" + with self._lock: + self._spans.clear() + + +def run( + argv: Sequence[str], + env: Mapping[str, str], + stdin: Optional[bytes], + timeout: float, +) -> subprocess.CompletedProcess[bytes]: + """Run one process and return its captured output, including after a timeout.""" + process = subprocess.Popen( + argv, + env=dict(env), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + timed_out = False + try: + stdout, stderr = process.communicate(input=stdin, timeout=timeout) + except subprocess.TimeoutExpired: + timed_out = True + process.kill() + stdout, stderr = process.communicate() + + result = subprocess.CompletedProcess(argv, process.returncode, stdout, stderr) + result.timed_out = timed_out + return result + + +def post_otlp(spans: dict[str, Any], endpoint: str) -> int: + """Post one JSON-encoded OTLP request to a loopback receiver.""" + parsed = urlsplit(endpoint) + if parsed.scheme != "http" or parsed.hostname != "127.0.0.1": + raise ValueError("post_otlp only sends to a loopback HTTP endpoint") + + request = Request( + endpoint, + data=json.dumps(spans).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urlopen(request, timeout=5) as response: + return response.status + + +def compare(actual: Sequence[dict[str, Any]], golden_path: Union[str, Path]) -> None: + """Fail with a diff unless span names, attributes, and statuses match a golden.""" + with Path(golden_path).open(encoding="utf-8") as golden_file: + expected = json.load(golden_file) + + def canonical(spans: Sequence[dict[str, Any]]) -> list[dict[str, Any]]: + normalized = [] + for span in spans: + attributes = span.get("attributes", []) + if isinstance(attributes, dict): + attributes = [ + {"key": key, "value": value} for key, value in attributes.items() + ] + normalized.append( + { + "name": span.get("name", ""), + "attributes": sorted( + attributes, + key=lambda attribute: json.dumps( + attribute, sort_keys=True, separators=(",", ":") + ), + ), + "status": span.get("status", {}), + } + ) + return normalized + + actual_normalized = canonical(actual) + expected_normalized = canonical(expected) + if actual_normalized != expected_normalized: + diff = "\n".join( + difflib.unified_diff( + json.dumps(expected_normalized, indent=2, sort_keys=True).splitlines(), + json.dumps(actual_normalized, indent=2, sort_keys=True).splitlines(), + fromfile=str(golden_path), + tofile="actual", + lineterm="", + ) + ) + print(diff) + raise AssertionError("spans do not match golden") diff --git a/python/tests/harness/test_harness.py b/python/tests/harness/test_harness.py new file mode 100644 index 000000000..4e76ceabe --- /dev/null +++ b/python/tests/harness/test_harness.py @@ -0,0 +1,101 @@ +"""Contract tests for the shared loopback-only harness.""" + +import json +import sys + +import pytest + +from tests.harness import Receiver, compare, post_otlp, run + + +def test_receiver_stores_posted_span_and_can_clear() -> None: + payload = { + "resourceSpans": [ + { + "scopeSpans": [ + { + "spans": [ + { + "traceId": "trace-id", + "spanId": "span-id", + "name": "harness.receiver", + "attributes": [ + {"key": "component", "value": {"stringValue": "test"}} + ], + "status": {"code": 1}, + } + ] + } + ] + } + ] + } + + with Receiver() as receiver: + assert post_otlp(payload, receiver.endpoint) == 200 + assert receiver.spans() == payload["resourceSpans"][0]["scopeSpans"][0]["spans"] + receiver.clear() + assert receiver.spans() == [] + + +def test_run_captures_output_and_kills_timed_out_process() -> None: + completed = run( + [ + sys.executable, + "-c", + "import sys; print('stdout'); sys.stderr.write('stderr\\n')", + ], + env={}, + stdin=b"", + timeout=1, + ) + assert completed.returncode == 0 + assert completed.stdout == b"stdout\n" + assert completed.stderr == b"stderr\n" + assert completed.timed_out is False + + timed_out = run( + [sys.executable, "-c", "import time; time.sleep(30)"], + env={}, + stdin=b"", + timeout=0.1, + ) + assert timed_out.timed_out is True + assert timed_out.returncode != 0 + + +def test_compare_accepts_normalized_match_and_prints_mismatch_diff(tmp_path, capsys) -> None: + golden = [ + { + "traceId": "golden-trace", + "spanId": "golden-span", + "startTimeUnixNano": "1", + "endTimeUnixNano": "2", + "name": "harness.compare", + "attributes": [ + {"key": "alpha", "value": {"stringValue": "a"}}, + {"key": "beta", "value": {"intValue": "2"}}, + ], + "status": {"code": 1}, + } + ] + golden_path = tmp_path / "spans.golden.json" + golden_path.write_text(json.dumps(golden), encoding="utf-8") + + actual = [ + { + "traceId": "other-trace", + "spanId": "other-span", + "startTimeUnixNano": "3", + "endTimeUnixNano": "4", + "name": "harness.compare", + "attributes": list(reversed(golden[0]["attributes"])), + "status": {"code": 1}, + } + ] + compare(actual, golden_path) + + actual[0]["name"] = "harness.mismatch" + with pytest.raises(AssertionError, match="spans do not match golden"): + compare(actual, golden_path) + assert "---" in capsys.readouterr().out From 32f5c9d2b3b80f97a70e00b3f3ca1b408b14dc5c Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sat, 3 Oct 2026 23:21:29 -0700 Subject: [PATCH 02/19] test(harness): accept the collector /tracer/v1/traces path --- python/tests/harness/__init__.py | 3 ++- python/tests/harness/test_harness.py | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/python/tests/harness/__init__.py b/python/tests/harness/__init__.py index dfd9198e4..cb510ad10 100644 --- a/python/tests/harness/__init__.py +++ b/python/tests/harness/__init__.py @@ -28,7 +28,7 @@ def __init__(self) -> None: class Handler(BaseHTTPRequestHandler): def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler - if urlsplit(self.path).path != "/v1/traces": + if urlsplit(self.path).path not in ("/v1/traces", "/tracer/v1/traces"): self.send_error(HTTPStatus.NOT_FOUND) return @@ -76,6 +76,7 @@ def log_message(self, _format: str, *args: Any) -> None: self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) self.origin = "http://127.0.0.1:{0}".format(self._server.server_port) self.endpoint = "{0}/v1/traces".format(self.origin) + self.collector_endpoint = "{0}/tracer/v1/traces".format(self.origin) self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) self._thread.start() diff --git a/python/tests/harness/test_harness.py b/python/tests/harness/test_harness.py index 4e76ceabe..6a32c2af8 100644 --- a/python/tests/harness/test_harness.py +++ b/python/tests/harness/test_harness.py @@ -38,6 +38,18 @@ def test_receiver_stores_posted_span_and_can_clear() -> None: assert receiver.spans() == [] +def test_receiver_accepts_the_collector_path() -> None: + payload = { + "resourceSpans": [ + {"scopeSpans": [{"spans": [{"name": "collector.path"}]}]} + ] + } + + with Receiver() as receiver: + assert post_otlp(payload, receiver.collector_endpoint) == 200 + assert receiver.spans() == [{"name": "collector.path"}] + + def test_run_captures_output_and_kills_timed_out_process() -> None: completed = run( [ From 3eaadc8bdbf428a472db6779091bbce3d6d89015 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 4 Oct 2026 15:59:06 -0700 Subject: [PATCH 03/19] fix(harness): decode chunked OTLP bodies and record request metadata The Node OTLP exporter streams with Transfer-Encoding: chunked and sends no Content-Length. The Receiver read Content-Length only, so every Node export came back 400 with zero spans; two TH-8103 children had to put a de-chunking relay in front of it. Receiver now accepts both framings. It also records one entry per accepted export (path, lower-cased headers, flattened resource attributes) via requests(), so a contract test can assert X-Api-Key/X-Secret-Key and project_name through the shared harness instead of a private recorder. Verified: 6 harness tests pass, and the TanStack example's real Node exporter delivered 4 chunked exports (4 spans, collector path, both auth headers, project_name/project_type) with no relay. Refs: TH-8339, TH-8103 --- python/tests/harness/__init__.py | 61 ++++++++++++++++++++-- python/tests/harness/test_harness.py | 76 ++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+), 4 deletions(-) diff --git a/python/tests/harness/__init__.py b/python/tests/harness/__init__.py index cb510ad10..cf79047f1 100644 --- a/python/tests/harness/__init__.py +++ b/python/tests/harness/__init__.py @@ -23,18 +23,36 @@ class Receiver: def __init__(self) -> None: self._spans: list[dict[str, Any]] = [] + self._requests: list[dict[str, Any]] = [] self._lock = threading.Lock() owner = self class Handler(BaseHTTPRequestHandler): + def _read_body(self) -> bytes: + # The Node OTLP exporter streams with Transfer-Encoding: chunked + # and sends no Content-Length, so both framings are accepted. + if "chunked" in self.headers.get("Transfer-Encoding", "").lower(): + chunks = [] + while True: + size_line = self.rfile.readline().split(b";", 1)[0].strip() + size = int(size_line, 16) + if size == 0: + while self.rfile.readline() not in (b"\r\n", b"\n", b""): + pass + return b"".join(chunks) + chunks.append(self.rfile.read(size)) + self.rfile.readline() + length = int(self.headers.get("Content-Length", "0")) + return self.rfile.read(length) + def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler - if urlsplit(self.path).path not in ("/v1/traces", "/tracer/v1/traces"): + path = urlsplit(self.path).path + if path not in ("/v1/traces", "/tracer/v1/traces"): self.send_error(HTTPStatus.NOT_FOUND) return try: - length = int(self.headers.get("Content-Length", "0")) - body = self.rfile.read(length) + body = self._read_body() content_type = self.headers.get("Content-Type", "").lower() if content_type.startswith("application/json"): @@ -53,9 +71,15 @@ def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler return spans: list[dict[str, Any]] = [] + resource_attributes: list[dict[str, Any]] = [] for resource_spans in request.get( "resourceSpans", request.get("resource_spans", []) ): + resource_attributes.append( + _flatten_attributes( + resource_spans.get("resource", {}).get("attributes", []) + ) + ) for scope_spans in resource_spans.get( "scopeSpans", resource_spans.get("scope_spans", []) ): @@ -66,6 +90,15 @@ def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler with owner._lock: owner._spans.extend(copy.deepcopy(spans)) + owner._requests.append( + { + "path": path, + "headers": { + key.lower(): value for key, value in self.headers.items() + }, + "resource_attributes": resource_attributes, + } + ) self.send_response(HTTPStatus.OK) self.send_header("Content-Length", "0") self.end_headers() @@ -93,10 +126,30 @@ def spans(self) -> list[dict[str, Any]]: with self._lock: return copy.deepcopy(self._spans) + def requests(self) -> list[dict[str, Any]]: + """Return one record per accepted export: path, lower-cased headers, resource attributes.""" + with self._lock: + return copy.deepcopy(self._requests) + def clear(self) -> None: - """Remove every decoded span received so far.""" + """Remove every decoded span and request record received so far.""" with self._lock: self._spans.clear() + self._requests.clear() + + +def _flatten_attributes(attributes: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + """Turn OTLP key/value attributes into a plain dict of scalar values.""" + flat: dict[str, Any] = {} + for attribute in attributes: + value = attribute.get("value", {}) + for kind in ("stringValue", "boolValue", "intValue", "doubleValue"): + if kind in value: + flat[attribute.get("key", "")] = value[kind] + break + else: + flat[attribute.get("key", "")] = value + return flat def run( diff --git a/python/tests/harness/test_harness.py b/python/tests/harness/test_harness.py index 6a32c2af8..bd61d5adc 100644 --- a/python/tests/harness/test_harness.py +++ b/python/tests/harness/test_harness.py @@ -50,6 +50,82 @@ def test_receiver_accepts_the_collector_path() -> None: assert receiver.spans() == [{"name": "collector.path"}] +def _post_chunked(url: str, body: bytes, headers: dict) -> int: + """Send a body with Transfer-Encoding: chunked, as the Node OTLP exporter does.""" + import http.client + from urllib.parse import urlsplit + + parts = urlsplit(url) + connection = http.client.HTTPConnection(parts.hostname, parts.port, timeout=5) + try: + chunks = [body[: len(body) // 2], body[len(body) // 2 :]] + connection.request( + "POST", + parts.path, + body=iter(chunks), + headers={**headers, "Transfer-Encoding": "chunked"}, + encode_chunked=True, + ) + return connection.getresponse().status + finally: + connection.close() + + +def test_receiver_decodes_a_chunked_body() -> None: + payload = { + "resourceSpans": [ + {"scopeSpans": [{"spans": [{"name": "chunked.body"}]}]} + ] + } + + with Receiver() as receiver: + status = _post_chunked( + receiver.collector_endpoint, + json.dumps(payload).encode("utf-8"), + {"Content-Type": "application/json"}, + ) + assert status == 200 + assert receiver.spans() == [{"name": "chunked.body"}] + + +def test_receiver_records_path_headers_and_resource_attributes() -> None: + payload = { + "resourceSpans": [ + { + "resource": { + "attributes": [ + {"key": "project_name", "value": {"stringValue": "harness"}}, + {"key": "project_type", "value": {"stringValue": "observe"}}, + ] + }, + "scopeSpans": [{"spans": [{"name": "with.resource"}]}], + } + ] + } + + with Receiver() as receiver: + status = _post_chunked( + receiver.collector_endpoint, + json.dumps(payload).encode("utf-8"), + { + "Content-Type": "application/json", + "X-Api-Key": "test-key", + "X-Secret-Key": "test-secret", + }, + ) + assert status == 200 + [request] = receiver.requests() + assert request["path"] == "/tracer/v1/traces" + assert request["headers"]["x-api-key"] == "test-key" + assert request["headers"]["x-secret-key"] == "test-secret" + assert "authorization" not in request["headers"] + assert request["resource_attributes"] == [ + {"project_name": "harness", "project_type": "observe"} + ] + receiver.clear() + assert receiver.requests() == [] + + def test_run_captures_output_and_kills_timed_out_process() -> None: completed = run( [ From e14df89f01d0e09a9819ccc72a82b56e7090017c Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 4 Oct 2026 15:54:59 -0700 Subject: [PATCH 04/19] feat(claude-agent-sdk): add @traceai/claude-agent-sdk for TypeScript New package typescript/packages/traceai_claude_agent_sdk. wrapQuery() wraps the SDK's exported query() and turns its message stream into conversation, assistant_turn, tool_execution, mcp_tool and subagent spans. Each span keeps the Python claude_agent.span_kind string and sets gen_ai.span.kind and fi.span.kind. The conversation span falls back to CHAIN because TS FISpanKind has no CONVERSATION. The attribute names are copied from the Python _attributes.py, and a test fails when one is missing. Tokens and cost come from the result message. Cost is also written to gen_ai.cost.total, the key the collector promotes. The session id goes to session.id. The wrapper also records fork origin, provider (custom when ANTHROPIC_BASE_URL is not an anthropic.com host), abort and errors. Prompts, tool inputs and outputs, and assistant text are hidden unless hideInputs/hideOutputs or FI_HIDE_* opt in. This is stricter than the Python package. @anthropic-ai/claude-agent-sdk (Anthropic Commercial Terms) is a peer dependency, ^0.3.142. It is never imported at runtime or bundled. The contract test packs the tarball and checks it holds no SDK file or native binary. Tests: 40 jest tests, plus shared-harness contract tests. The contract tests send fixture journeys and the real SDK 0.3.289 / 0.3.142 CLI, pointed at a loopback Messages mock through options.env .ANTHROPIC_BASE_URL, through fi-core to the harness Receiver. No Anthropic call. pnpm-lock.yaml gets only this importer and the 15 packages/snapshots it needs. pnpm 10.30.1 also re-resolved other importers' ts-node peers, and those changes were left out. Refs: TH-8235 --- README.md | 1 + .../traceai_claude_agent_sdk/README.md | 124 +++ .../contract/run_fixture.cjs | 55 + .../contract/run_real_sdk.mjs | 137 +++ .../contract/test_harness_contract.py | 424 ++++++++ .../contract/tsconfig.fixtures.json | 18 + .../traceai_claude_agent_sdk/jest.config.js | 22 + .../traceai_claude_agent_sdk/package.json | 75 ++ .../src/__tests__/collector.test.ts | 138 +++ .../src/__tests__/fixtures/fakeQuery.ts | 74 ++ .../src/__tests__/fixtures/messages.ts | 273 +++++ .../src/__tests__/helpers.ts | 79 ++ .../src/__tests__/parity.test.ts | 153 +++ .../src/__tests__/wrapper.test.ts | 586 +++++++++++ .../src/attributes.ts | 225 ++++ .../traceai_claude_agent_sdk/src/index.ts | 23 + .../src/instrumentation.ts | 162 +++ .../src/queryWrapper.ts | 131 +++ .../traceai_claude_agent_sdk/src/spans.ts | 960 ++++++++++++++++++ .../traceai_claude_agent_sdk/src/types.ts | 131 +++ .../traceai_claude_agent_sdk/src/version.ts | 1 + .../tsconfig.esm.json | 17 + .../tsconfig.esnext.json | 17 + .../traceai_claude_agent_sdk/tsconfig.json | 14 + typescript/pnpm-lock.yaml | 182 ++++ 25 files changed, 4022 insertions(+) create mode 100644 typescript/packages/traceai_claude_agent_sdk/README.md create mode 100644 typescript/packages/traceai_claude_agent_sdk/contract/run_fixture.cjs create mode 100644 typescript/packages/traceai_claude_agent_sdk/contract/run_real_sdk.mjs create mode 100644 typescript/packages/traceai_claude_agent_sdk/contract/test_harness_contract.py create mode 100644 typescript/packages/traceai_claude_agent_sdk/contract/tsconfig.fixtures.json create mode 100644 typescript/packages/traceai_claude_agent_sdk/jest.config.js create mode 100644 typescript/packages/traceai_claude_agent_sdk/package.json create mode 100644 typescript/packages/traceai_claude_agent_sdk/src/__tests__/collector.test.ts create mode 100644 typescript/packages/traceai_claude_agent_sdk/src/__tests__/fixtures/fakeQuery.ts create mode 100644 typescript/packages/traceai_claude_agent_sdk/src/__tests__/fixtures/messages.ts create mode 100644 typescript/packages/traceai_claude_agent_sdk/src/__tests__/helpers.ts create mode 100644 typescript/packages/traceai_claude_agent_sdk/src/__tests__/parity.test.ts create mode 100644 typescript/packages/traceai_claude_agent_sdk/src/__tests__/wrapper.test.ts create mode 100644 typescript/packages/traceai_claude_agent_sdk/src/attributes.ts create mode 100644 typescript/packages/traceai_claude_agent_sdk/src/index.ts create mode 100644 typescript/packages/traceai_claude_agent_sdk/src/instrumentation.ts create mode 100644 typescript/packages/traceai_claude_agent_sdk/src/queryWrapper.ts create mode 100644 typescript/packages/traceai_claude_agent_sdk/src/spans.ts create mode 100644 typescript/packages/traceai_claude_agent_sdk/src/types.ts create mode 100644 typescript/packages/traceai_claude_agent_sdk/src/version.ts create mode 100644 typescript/packages/traceai_claude_agent_sdk/tsconfig.esm.json create mode 100644 typescript/packages/traceai_claude_agent_sdk/tsconfig.esnext.json create mode 100644 typescript/packages/traceai_claude_agent_sdk/tsconfig.json diff --git a/README.md b/README.md index 214680868..10add6fa5 100644 --- a/README.md +++ b/README.md @@ -331,6 +331,7 @@ var tracer = TraceAI.Register(opts => | [`@traceai/mastra`](https://www.npmjs.com/package/@traceai/mastra) | Mastra | [![npm](https://img.shields.io/npm/v/@traceai/mastra)](https://www.npmjs.com/package/@traceai/mastra) | | [`@traceai/beeai`](https://www.npmjs.com/package/@traceai/beeai) | IBM BeeAI | [![npm](https://img.shields.io/npm/v/@traceai/beeai)](https://www.npmjs.com/package/@traceai/beeai) | | [`@traceai/strands`](https://www.npmjs.com/package/@traceai/strands) | AWS Strands Agents | [![npm](https://img.shields.io/npm/v/@traceai/strands)](https://www.npmjs.com/package/@traceai/strands) | +| [`@traceai/claude-agent-sdk`](https://www.npmjs.com/package/@traceai/claude-agent-sdk) | Claude Agent SDK (`query()`) | [![npm](https://img.shields.io/npm/v/@traceai/claude-agent-sdk)](https://www.npmjs.com/package/@traceai/claude-agent-sdk) | #### Tools and Libraries diff --git a/typescript/packages/traceai_claude_agent_sdk/README.md b/typescript/packages/traceai_claude_agent_sdk/README.md new file mode 100644 index 000000000..479f617fb --- /dev/null +++ b/typescript/packages/traceai_claude_agent_sdk/README.md @@ -0,0 +1,124 @@ +# @traceai/claude-agent-sdk + +OpenTelemetry tracing for the TypeScript [Claude Agent SDK](https://www.npmjs.com/package/@anthropic-ai/claude-agent-sdk) `query()`, exported to Future AGI through `@traceai/fi-core`. + +Alpha. Tested against `@anthropic-ai/claude-agent-sdk` 0.3.289 with recorded message fixtures and with the real SDK and bundled CLI pointed at a loopback mock of the Messages API. No live Anthropic call was made. + +This is not the Python package (`traceAI-claude-agent-sdk`), not Claude Code CLI session tracing, and not the Anthropic Messages API instrumentation (`@traceai/anthropic`). + +## Licence of the SDK + +`@anthropic-ai/claude-agent-sdk` is under the Anthropic Commercial Terms, not an OSI licence. It is a **peer dependency**: install it yourself. This package never bundles or re-exports the SDK or its CLI binary, and does not import it at runtime. + +| Item | Value | +|---|---| +| Tested pin | `@anthropic-ai/claude-agent-sdk` 0.3.289 (npm shasum `36c710c40a2d9761d6c12c42b469d002e64e9be8`) | +| Peer range | `^0.3.142` (0.3.142 is the oldest 0.3.x on npm; its `query()` and the message fields read here match 0.3.289, and the real-SDK contract test passed against it) | +| Node | 18, 20, 22 (CJS and ESM import checked); the SDK itself declares `node >=18` | + +## Install + +```bash +npm install @traceai/claude-agent-sdk @traceai/fi-core @anthropic-ai/claude-agent-sdk +``` + +## Use + +The SDK is ESM-only and its module namespace cannot be patched, so wrap `query` and call the wrapped function: + +```typescript +import { register, ProjectType } from "@traceai/fi-core"; +import { query } from "@anthropic-ai/claude-agent-sdk"; +import { wrapQuery, shutdown } from "@traceai/claude-agent-sdk"; + +const tracerProvider = register({ + projectType: ProjectType.OBSERVE, + projectName: "my-agent", // or FI_PROJECT_NAME; the collector drops a batch with no project_name +}); + +const tracedQuery = wrapQuery(query, { tracerProvider }); + +for await (const message of tracedQuery({ prompt: "Summarize the README.", options: { maxTurns: 3 } })) { + // the same message objects the SDK yielded +} + +await shutdown(); // flush before a short script exits +``` + +`FI_API_KEY` and `FI_SECRET_KEY` are read by fi-core and sent as the `X-Api-Key` / `X-Secret-Key` headers. They are never copied onto spans. + +`ClaudeAgentSDKInstrumentation` (or `instrumentClaudeAgentSDK()`) does the same with an object you keep: `instrumentation.wrapQuery(query)`, `instrumentation.forceFlush()`. + +### What the wrapper does and does not do + +- Calls the original `query()` with the same params object. Options (`resume`, `forkSession`, `sessionId`, `hooks`, `mcpServers`, `agents`, `abortController`, `allowedTools`, `permissionMode`, `maxTurns`, `env`, ...) are passed through untouched. The wrapper never sets `ANTHROPIC_BASE_URL`. +- Yields the same message objects, in order. Control methods on the returned `Query` (`interrupt()`, `setModel()`, ...) go to the original. +- A tracing error never fails the agent. An exporter or collector failure is logged and does not reject the iterator. +- `startup()` is not wrapped in 0.1.0. + +## Spans + +| Span name | `claude_agent.span_kind` | `gen_ai.span.kind` / `fi.span.kind` | Parent | +|---|---|---|---| +| `claude_agent.conversation` | `conversation` | `CHAIN` (TS fi-semantic-conventions has no `CONVERSATION`) | caller's active span | +| `claude_agent.assistant_turn` | `assistant_turn` | `LLM` | conversation, or the subagent span inside a subagent | +| `tool.` | `tool_execution`, or `mcp_tool` for `mcp____` | `TOOL` | the assistant turn that issued the `tool_use` | +| `claude_agent.subagent.` | `subagent` | `AGENT` | the `tool.Agent` / `tool.Task` span | + +One API response that the SDK splits over several assistant messages (same `message.id`) is one turn. + +| Field | Attributes | Source | +|---|---|---| +| Model | `gen_ai.request.model`, `claude_agent.model` | `options.model`, the init message, each assistant message | +| Provider | `gen_ai.provider.name` = `anthropic`, or `custom` when `ANTHROPIC_BASE_URL` (from `options.env`, else `process.env`) is not an `anthropic.com` host | | +| Tokens | `gen_ai.usage.input_tokens`, `output_tokens`, `total_tokens`; `gen_ai.usage.cache_read_tokens` / `cache_creation_tokens` when present | result message, on the conversation span | +| Cost | `claude_agent.cost.total_usd` and `gen_ai.cost.total` | result `total_cost_usd`; never computed here | +| Session | `session.id`, `claude_agent.session.id`, `claude_agent.session_id`; `claude_agent.session.fork_from` on fork | init / result `session_id`, `options.sessionId`, `options.resume` | +| Tools | `claude_agent.tool.name`, `gen_ai.tool.name`, `use_id`, `source` (`builtin` / `mcp` / `custom`), `is_error`, `duration_ms` | `tool_use` / `tool_result` blocks | +| Errors | span status ERROR; `claude_agent.error.type` / `error.message`; `claude_agent.cancelled=true` on abort | error result, thrown error, `is_error` tool results, `AbortController` | + +The `claude_agent.*` names are the Python package's names (`_attributes.py`); a test fails if one is missing. + +An app-set fi-core `session.id` / `user.id` (fi-core context helpers) is kept; the SDK session id still goes to `claude_agent.session.id`. + +## Privacy: content is off by default + +Prompts, system prompts, tool inputs, tool outputs, assistant text and the final result can contain source code, file contents or secrets. They are **not** recorded unless you opt in: + +```typescript +wrapQuery(query, { tracerProvider, traceConfig: { hideInputs: false, hideOutputs: false } }); +// or: FI_HIDE_INPUTS=false FI_HIDE_OUTPUTS=false +``` + +This is stricter than fi-core's own default and than the Python package, which records the prompt and tool input/output unconditionally. + +## Flushing and batching + +`shutdown()` calls `forceFlush()` on the provider(s) you passed (or the global provider). It does not shut the provider down. The wrapper has no queue of its own. + +fi-core's `register()` exports each span through its default `SimpleSpanProcessor`. At the fi-core version in this repo, `register({ batch: true })` does not replace that processor with OpenTelemetry `sdk-trace-base` 2.x. If you want batching, build a provider with a `BatchSpanProcessor` yourself (`OTEL_BSP_MAX_QUEUE_SIZE`, default 2048). + +## Pointing the SDK at a mock or gateway + +There is no `baseURL` option. `options.env` **replaces** the CLI subprocess environment, so spread `process.env` (or pass `PATH`, `HOME`, ...) and add `ANTHROPIC_BASE_URL`: + +```typescript +options: { env: { ...process.env, ANTHROPIC_BASE_URL: "http://127.0.0.1:8080" } } +``` + +With 0.3.289 the CLI then calls `POST {ANTHROPIC_BASE_URL}/v1/messages?beta=true` with `stream: true` (observed in `contract/run_real_sdk.mjs`). + +## Troubleshooting + +- **No traces.** `register()` needs `projectName` or `FI_PROJECT_NAME`. +- **Zero spans from a short script.** `await shutdown()` before exit. +- **Duplicate model spans.** Do not also run a provider instrumentor on the same calls. +- **A span left open after abort.** Every span for the query is ended with status ERROR and `claude_agent.cancelled=true` when its `AbortController` aborts. Anything else is a bug here. + +## Tests + +```bash +pnpm --filter @traceai/claude-agent-sdk test # jest: wrapper, parity, fi-core export contract +``` + +The shared-harness contract tests (`contract/test_harness_contract.py`) run the built package in Node against the Python OTLP `Receiver`, run the real SDK against a loopback Messages API mock, pack the tarball and check it holds no SDK file or native binary, and import the package as CJS and ESM. See the module docstring for the command. diff --git a/typescript/packages/traceai_claude_agent_sdk/contract/run_fixture.cjs b/typescript/packages/traceai_claude_agent_sdk/contract/run_fixture.cjs new file mode 100644 index 000000000..1b6769466 --- /dev/null +++ b/typescript/packages/traceai_claude_agent_sdk/contract/run_fixture.cjs @@ -0,0 +1,55 @@ +'use strict'; +/** + * Contract script for the shared Python harness (python/tests/harness). + * + * Runs the built package (dist/src, CommonJS) with the real fi-core register() + * and the real OTLP/HTTP exporter. The Claude Agent SDK is replaced by a fake + * query() that yields SDK-typed fixture messages: no CLI, no Anthropic call. + * + * Env: + * FI_BASE_URL, FI_API_KEY, FI_SECRET_KEY, FI_PROJECT_NAME read by fi-core + * CONTRACT_FIXTURES_DIR compiled src/__tests__/fixtures (messages.js, fakeQuery.js) + * JOURNEY simple | subagent | mcp + * + * Prints {"yielded": [...], "expected": [...]} on stdout. + */ +const path = require('path'); + +const pkgDir = path.resolve(__dirname, '..'); +const { register, ProjectType } = require(require.resolve('@traceai/fi-core', { paths: [pkgDir] })); +const { wrapQuery, shutdown } = require(pkgDir); +const fixtures = require(path.join(process.env.CONTRACT_FIXTURES_DIR, 'messages.js')); +const { makeFakeQuery } = require(path.join(process.env.CONTRACT_FIXTURES_DIR, 'fakeQuery.js')); + +const JOURNEYS = { + simple: fixtures.simpleToolJourney, + subagent: fixtures.subagentJourney, + mcp: fixtures.mcpErrorJourney, +}; + +async function main() { + const journey = JOURNEYS[process.env.JOURNEY || 'simple']; + if (!journey) throw new Error(`unknown JOURNEY ${process.env.JOURNEY}`); + + // Project name and collector endpoint come from FI_PROJECT_NAME / FI_BASE_URL. + const provider = register({ projectType: ProjectType.OBSERVE, batch: true }); + + const messages = journey(); + const expected = JSON.parse(JSON.stringify(messages)); + const fake = makeFakeQuery(messages); + const query = wrapQuery(fake.query, { tracerProvider: provider }); + + const yielded = []; + for await (const message of query({ prompt: fixtures.PROMPT, options: { model: fixtures.MODEL } })) { + yielded.push(message); + } + + await shutdown(); + await provider.shutdown(); + process.stdout.write(JSON.stringify({ yielded, expected })); +} + +main().catch((error) => { + process.stderr.write(`${error && error.stack ? error.stack : error}\n`); + process.exit(1); +}); diff --git a/typescript/packages/traceai_claude_agent_sdk/contract/run_real_sdk.mjs b/typescript/packages/traceai_claude_agent_sdk/contract/run_real_sdk.mjs new file mode 100644 index 000000000..d8273c441 --- /dev/null +++ b/typescript/packages/traceai_claude_agent_sdk/contract/run_real_sdk.mjs @@ -0,0 +1,137 @@ +// Contract script: the REAL @anthropic-ai/claude-agent-sdk query() (and its +// bundled Claude Code CLI) against a loopback mock of the Anthropic Messages +// API, reached through options.env.ANTHROPIC_BASE_URL. No Anthropic call: +// the API key is a placeholder, HTTPS_PROXY/HTTP_PROXY point at a dead +// loopback port, and HOME / CLAUDE_CONFIG_DIR are a fresh temp dir. +// +// The mock answers the first /v1/messages call with a Read tool_use for a temp +// file and the second with text, so the CLI runs one real tool. +// +// Env: FI_BASE_URL, FI_API_KEY, FI_SECRET_KEY, FI_PROJECT_NAME (fi-core). +// CLAUDE_AGENT_SDK_ENTRY (optional): absolute path to another SDK build's +// sdk.mjs, to run the same contract against an older 0.3.x. +// Prints {"requests": [...], "messages": [...]} on stdout. +import http from "node:http"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { ProjectType, register } from "@traceai/fi-core"; +import { shutdown, wrapQuery } from "@traceai/claude-agent-sdk"; + +const { query } = process.env.CLAUDE_AGENT_SDK_ENTRY + ? await import(process.env.CLAUDE_AGENT_SDK_ENTRY) + : await import("@anthropic-ai/claude-agent-sdk"); + +const MOCK_MODEL = "claude-sonnet-4-5"; +const PROMPT = "Read README.md and summarize it. SECRET_PROMPT_MARKER"; + +const workdir = fs.mkdtempSync(path.join(os.tmpdir(), "th8235-real-sdk-")); +const readmePath = path.join(workdir, "README.md"); +fs.writeFileSync(readmePath, "# Demo\nSECRET_TOOL_OUTPUT_MARKER\n"); + +const requests = []; +function sendSse(res, model, blocks, stopReason) { + res.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache", "request-id": "req_mock" }); + const send = (event, data) => res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`); + send("message_start", { + type: "message_start", + message: { + id: `msg_mock_${requests.length}`, type: "message", role: "assistant", model, content: [], + stop_reason: null, stop_sequence: null, + usage: { input_tokens: 20, output_tokens: 1, cache_read_input_tokens: 0, cache_creation_input_tokens: 0 }, + }, + }); + blocks.forEach((block, index) => { + if (block.type === "text") { + send("content_block_start", { type: "content_block_start", index, content_block: { type: "text", text: "" } }); + send("content_block_delta", { type: "content_block_delta", index, delta: { type: "text_delta", text: block.text } }); + } else { + send("content_block_start", { type: "content_block_start", index, content_block: { type: "tool_use", id: block.id, name: block.name, input: {} } }); + send("content_block_delta", { type: "content_block_delta", index, delta: { type: "input_json_delta", partial_json: JSON.stringify(block.input) } }); + } + send("content_block_stop", { type: "content_block_stop", index }); + }); + send("message_delta", { type: "message_delta", delta: { stop_reason: stopReason, stop_sequence: null }, usage: { output_tokens: 8 } }); + send("message_stop", { type: "message_stop" }); + res.end(); +} + +const server = http.createServer((req, res) => { + let body = ""; + req.on("data", (chunk) => (body += chunk)); + req.on("end", () => { + let parsed = {}; + try { parsed = JSON.parse(body || "{}"); } catch { /* not JSON */ } + const hasToolResult = JSON.stringify(parsed.messages ?? []).includes('"tool_result"'); + requests.push({ + method: req.method, + url: req.url, + stream: parsed.stream === true, + model: parsed.model, + xApiKey: req.headers["x-api-key"] ?? null, + hasToolResult, + }); + if (req.method === "POST" && req.url.startsWith("/v1/messages") && !req.url.includes("count_tokens")) { + const model = parsed.model || MOCK_MODEL; + if (!hasToolResult) { + return sendSse(res, model, [ + { type: "text", text: "Reading it. SECRET_ASSISTANT_TEXT_MARKER" }, + { type: "tool_use", id: "toolu_mock_read", name: "Read", input: { file_path: readmePath } }, + ], "tool_use"); + } + return sendSse(res, model, [{ type: "text", text: "It is a demo README." }], "end_turn"); + } + res.writeHead(404, { "content-type": "application/json" }); + res.end(JSON.stringify({ type: "error", error: { type: "not_found_error", message: "mock" } })); + }); +}); +await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); +const mockOrigin = `http://127.0.0.1:${server.address().port}`; + +const provider = register({ projectType: ProjectType.OBSERVE }); +const tracedQuery = wrapQuery(query, { tracerProvider: provider }); + +const abortController = new AbortController(); +const timer = setTimeout(() => abortController.abort(), 90_000); +const messages = []; +try { + const stream = tracedQuery({ + prompt: PROMPT, + options: { + model: MOCK_MODEL, + cwd: workdir, + maxTurns: 3, + allowedTools: ["Read"], + permissionMode: "default", + settingSources: [], + persistSession: false, + abortController, + // Options.env REPLACES the subprocess environment: pass everything the CLI needs. + env: { + PATH: process.env.PATH, + HOME: workdir, + CLAUDE_CONFIG_DIR: path.join(workdir, ".claude"), + ANTHROPIC_BASE_URL: mockOrigin, + ANTHROPIC_API_KEY: "sk-ant-PLACEHOLDER-not-a-key", + CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: "1", + DISABLE_TELEMETRY: "1", + DISABLE_ERROR_REPORTING: "1", + DISABLE_AUTOUPDATER: "1", + HTTPS_PROXY: "http://127.0.0.1:9", + HTTP_PROXY: "http://127.0.0.1:9", + NO_PROXY: "127.0.0.1,localhost", + }, + }, + }); + for await (const message of stream) { + messages.push(message); + } +} finally { + clearTimeout(timer); + await shutdown(provider); + await provider.shutdown(); + server.close(); + fs.rmSync(workdir, { recursive: true, force: true }); +} + +process.stdout.write(JSON.stringify({ mockOrigin, requests, messages })); diff --git a/typescript/packages/traceai_claude_agent_sdk/contract/test_harness_contract.py b/typescript/packages/traceai_claude_agent_sdk/contract/test_harness_contract.py new file mode 100644 index 000000000..3c7efd3a8 --- /dev/null +++ b/typescript/packages/traceai_claude_agent_sdk/contract/test_harness_contract.py @@ -0,0 +1,424 @@ +"""Shared-harness contract tests for @traceai/claude-agent-sdk (TH-8235). + +The package is TypeScript, so each test starts the shared loopback Receiver, +runs `node contract/run_fixture.cjs` through `harness.run`, and reads the spans +the real fi-core OTLP/HTTP exporter posted to `{origin}/tracer/v1/traces`. +The Claude Agent SDK is replaced by a fake query() over SDK-typed fixtures: +no Claude Code CLI, no Anthropic call, no network beyond 127.0.0.1. + +The Node OTLP/HTTP exporter streams each body with `Transfer-Encoding: chunked` +and no Content-Length. The shared Receiver decodes that framing and records one +entry per export (path, lower-cased headers, resource attributes), so headers +and `project_name` are asserted through `receiver.requests()`. +src/__tests__/collector.test.ts (jest) asserts them as well. + +Run from the repo root: + PYTHONPATH=python/tests uv run --no-project --python 3.11 --with pytest \ + --with opentelemetry-exporter-otlp-proto-http \ + pytest typescript/packages/traceai_claude_agent_sdk/contract -q -p no:cacheprovider --noconftest -o addopts='' +""" + +from __future__ import annotations + +import hashlib +import json +import os +import shutil +import subprocess +import sys +import tarfile +from pathlib import Path +from typing import Any, Dict, List + +import pytest + +PKG_DIR = Path(__file__).resolve().parents[1] +REPO_ROOT = PKG_DIR.parents[2] +sys.path.insert(0, str(REPO_ROOT / "python" / "tests")) + +from harness import Receiver, run # noqa: E402 + +NODE = os.environ.get("NODE_BINARY") or shutil.which("node") or "node" +PNPM = shutil.which("pnpm") or "pnpm" +TSC = PKG_DIR / "node_modules" / ".bin" / "tsc" +SDK_DIR = PKG_DIR / "node_modules" / "@anthropic-ai" / "claude-agent-sdk" + +SESSION_ID = "11111111-2222-4333-8444-555555555555" +MODEL = "claude-sonnet-4-5" +CONTENT_MARKERS = ( + "SECRET_PROMPT_MARKER", + "SECRET_TOOL_INPUT_MARKER", + "SECRET_TOOL_OUTPUT_MARKER", + "SECRET_ASSISTANT_TEXT_MARKER", + "SECRET_SUBAGENT_PROMPT_MARKER", +) +PLACEHOLDER_API_KEY = "contract-api-key-PLACEHOLDER" +PLACEHOLDER_SECRET_KEY = "contract-secret-key-PLACEHOLDER" + + +def _check(result: subprocess.CompletedProcess, what: str) -> None: + assert not getattr(result, "timed_out", False), f"{what} timed out\n{result.stderr.decode()}" + assert result.returncode == 0, ( + f"{what} exited {result.returncode}\nstdout:\n{result.stdout.decode()}\nstderr:\n{result.stderr.decode()}" + ) + + +@pytest.fixture(scope="session") +def built_package() -> Path: + """Build dist/ (CJS + ESM) from the current sources.""" + result = run([PNPM, "--dir", str(PKG_DIR), "run", "build"], env=_base_env(), stdin=None, timeout=300) + _check(result, "pnpm run build") + assert (PKG_DIR / "dist" / "src" / "index.js").is_file() + assert (PKG_DIR / "dist" / "esm" / "index.js").is_file() + return PKG_DIR + + +@pytest.fixture(scope="session") +def fixtures_dir(tmp_path_factory: pytest.TempPathFactory) -> Path: + """Compile the SDK-typed fixtures; a type error against the pinned sdk.d.ts fails here.""" + out = tmp_path_factory.mktemp("fixtures") + result = run( + [str(TSC), "-p", str(PKG_DIR / "contract" / "tsconfig.fixtures.json"), "--outDir", str(out)], + env=_base_env(), + stdin=None, + timeout=180, + ) + _check(result, "tsc fixtures") + assert (out / "messages.js").is_file() and (out / "fakeQuery.js").is_file() + return out + + +def _base_env() -> Dict[str, str]: + return { + "PATH": os.environ.get("PATH", ""), + "HOME": os.environ.get("HOME", ""), + "NO_PROXY": "127.0.0.1,localhost", + } + + +def _run_journey(receiver: Receiver, fixtures_dir: Path, journey: str, extra_env: Dict[str, str] | None = None) -> Dict[str, Any]: + env = _base_env() + env.update( + { + "FI_BASE_URL": receiver.origin, # fi-core appends /tracer/v1/traces + "FI_API_KEY": PLACEHOLDER_API_KEY, + "FI_SECRET_KEY": PLACEHOLDER_SECRET_KEY, + "FI_PROJECT_NAME": "th8235-harness-contract", + "CONTRACT_FIXTURES_DIR": str(fixtures_dir), + "JOURNEY": journey, + } + ) + env.update(extra_env or {}) + result = run([NODE, str(PKG_DIR / "contract" / "run_fixture.cjs")], env=env, stdin=None, timeout=60) + _check(result, f"node run_fixture.cjs ({journey})") + return json.loads(result.stdout.decode("utf-8")) + + +def _value(value: Dict[str, Any]) -> Any: + for key in ("stringValue", "boolValue", "doubleValue"): + if key in value: + return value[key] + if "intValue" in value: + return int(value["intValue"]) + return value + + +def _attrs(span: Dict[str, Any]) -> Dict[str, Any]: + return {item["key"]: _value(item["value"]) for item in span.get("attributes", [])} + + +def _status_code(span: Dict[str, Any]) -> int: + code = span.get("status", {}).get("code", 0) + names = {"STATUS_CODE_UNSET": 0, "STATUS_CODE_OK": 1, "STATUS_CODE_ERROR": 2} + return names[code] if isinstance(code, str) else int(code) + + +def _by_name(spans: List[Dict[str, Any]], name: str) -> List[Dict[str, Any]]: + return [span for span in spans if span["name"] == name] + + +def _one(spans: List[Dict[str, Any]], name: str) -> Dict[str, Any]: + matches = _by_name(spans, name) + assert len(matches) == 1, f"expected one {name}, got {[s['name'] for s in spans]}" + return matches[0] + + +def test_conversation_turn_tool_reach_collector_path_with_content_off(built_package: Path, fixtures_dir: Path) -> None: + with Receiver() as receiver: + output = _run_journey(receiver, fixtures_dir, "simple") + spans = receiver.spans() + requests = receiver.requests() + + # AC-08: messages are yielded unchanged and in order. + assert output["yielded"] == output["expected"] + + # Collector contract (the Receiver drops headers and resource attributes). + assert requests, "exporter sent nothing" + for request in requests: + assert request["path"] == "/tracer/v1/traces" + headers = request["headers"] + assert headers["x-api-key"] == PLACEHOLDER_API_KEY + assert headers["x-secret-key"] == PLACEHOLDER_SECRET_KEY + assert "authorization" not in headers + assert request["resource_attributes"] + for resource in request["resource_attributes"]: + assert resource["project_name"] == "th8235-harness-contract" + assert resource["project_type"] == "observe" + + assert sorted(span["name"] for span in spans) == sorted( + ["claude_agent.conversation", "claude_agent.assistant_turn", "claude_agent.assistant_turn", "tool.Read"] + ) + assert len({span["traceId"] for span in spans}) == 1 + + conversation = _one(spans, "claude_agent.conversation") + tool = _one(spans, "tool.Read") + turns = _by_name(spans, "claude_agent.assistant_turn") + c, t = _attrs(conversation), _attrs(tool) + + # Span kinds: Python parity string + the Future AGI kind keys. + assert (c["claude_agent.span_kind"], c["gen_ai.span.kind"], c["fi.span.kind"]) == ("conversation", "CHAIN", "CHAIN") + for turn in turns: + a = _attrs(turn) + assert (a["claude_agent.span_kind"], a["gen_ai.span.kind"], a["fi.span.kind"]) == ("assistant_turn", "LLM", "LLM") + assert a["gen_ai.request.model"] == MODEL + assert a["gen_ai.provider.name"] == "anthropic" + assert turn["parentSpanId"] == conversation["spanId"] + assert (t["claude_agent.span_kind"], t["gen_ai.span.kind"], t["fi.span.kind"]) == ("tool_execution", "TOOL", "TOOL") + issuing = [turn for turn in turns if _attrs(turn)["claude_agent.message.has_tool_use"] is True] + assert len(issuing) == 1 and tool["parentSpanId"] == issuing[0]["spanId"] + assert t["gen_ai.tool.name"] == "Read" and t["claude_agent.tool.source"] == "builtin" + + # Model, tokens, cost, session. + assert c["gen_ai.request.model"] == MODEL and c["claude_agent.model"] == MODEL + assert c["gen_ai.usage.input_tokens"] == 120 + assert c["gen_ai.usage.output_tokens"] == 45 + assert c["gen_ai.usage.total_tokens"] == 165 + assert c["gen_ai.cost.total"] == pytest.approx(0.0123) + assert c["claude_agent.cost.total_usd"] == pytest.approx(0.0123) + for span in spans: + assert _attrs(span)["session.id"] == SESSION_ID + assert _status_code(span) == 1 + + # Content capture is off by default. + blob = json.dumps(spans) + for marker in CONTENT_MARKERS: + assert marker not in blob + for span in spans: + for key in ("input.value", "output.value", "claude_agent.prompt", "claude_agent.tool.input", + "claude_agent.tool.output", "claude_agent.message.content"): + assert key not in _attrs(span) + assert PLACEHOLDER_API_KEY not in blob and PLACEHOLDER_SECRET_KEY not in blob + + +def test_subagent_and_mcp_spans_reach_the_receiver(built_package: Path, fixtures_dir: Path) -> None: + with Receiver() as receiver: + _run_journey(receiver, fixtures_dir, "subagent") + subagent_spans = receiver.spans() + receiver.clear() + _run_journey(receiver, fixtures_dir, "mcp") + mcp_spans = receiver.spans() + + agent_tool = _one(subagent_spans, "tool.Agent") + subagent = _one(subagent_spans, "claude_agent.subagent.code-reviewer") + grep = _one(subagent_spans, "tool.Grep") + s = _attrs(subagent) + assert (s["claude_agent.span_kind"], s["gen_ai.span.kind"]) == ("subagent", "AGENT") + assert subagent["parentSpanId"] == agent_tool["spanId"] + grep_turn = [span for span in subagent_spans if span["spanId"] == grep["parentSpanId"]][0] + assert grep_turn["name"] == "claude_agent.assistant_turn" + assert grep_turn["parentSpanId"] == subagent["spanId"] + + mcp_tool = _one(mcp_spans, "tool.mcp__docs__search") + m = _attrs(mcp_tool) + assert (m["claude_agent.span_kind"], m["gen_ai.span.kind"], m["claude_agent.mcp.server_name"]) == ("mcp_tool", "TOOL", "docs") + assert _status_code(mcp_tool) == 2 + conversation = _attrs(_one(mcp_spans, "claude_agent.conversation")) + assert conversation["claude_agent.error.type"] == "error_max_turns" + + +def test_content_is_exported_only_after_opt_in(built_package: Path, fixtures_dir: Path) -> None: + with Receiver() as receiver: + _run_journey(receiver, fixtures_dir, "simple", {"FI_HIDE_INPUTS": "false", "FI_HIDE_OUTPUTS": "false"}) + spans = receiver.spans() + conversation = _attrs(_one(spans, "claude_agent.conversation")) + tool = _attrs(_one(spans, "tool.Read")) + assert "SECRET_PROMPT_MARKER" in conversation["input.value"] + assert "SECRET_ASSISTANT_TEXT_MARKER" in conversation["output.value"] + assert "SECRET_TOOL_INPUT_MARKER" in tool["claude_agent.tool.input"] + assert "SECRET_TOOL_OUTPUT_MARKER" in tool["claude_agent.tool.output"] + + +def _sdk_platform_binary_installed() -> bool: + store = SDK_DIR.resolve().parent if SDK_DIR.exists() else None + return bool(store and list(store.glob("claude-agent-sdk-*"))) + + +@pytest.mark.skipif(not _sdk_platform_binary_installed(), reason="claude-agent-sdk platform binary not installed") +def test_real_sdk_query_through_anthropic_base_url_mock(built_package: Path) -> None: + """The real SDK query() and bundled CLI, wrapped, against a loopback Messages API mock. + + Mechanism from the ADR: Options.env.ANTHROPIC_BASE_URL. No Anthropic call: placeholder + key, dead-port HTTPS_PROXY, temp HOME. Spans go through fi-core to the shared Receiver. + """ + env = _base_env() + env.update( + { + "FI_BASE_URL": "", # replaced below once the Receiver is up + "FI_API_KEY": PLACEHOLDER_API_KEY, + "FI_SECRET_KEY": PLACEHOLDER_SECRET_KEY, + "FI_PROJECT_NAME": "th8235-real-sdk-contract", + } + ) + if os.environ.get("CLAUDE_AGENT_SDK_ENTRY"): + # Run the same contract against another installed SDK build (older 0.3.x matrix). + env["CLAUDE_AGENT_SDK_ENTRY"] = os.environ["CLAUDE_AGENT_SDK_ENTRY"] + with Receiver() as receiver: + env["FI_BASE_URL"] = receiver.origin + result = run([NODE, str(PKG_DIR / "contract" / "run_real_sdk.mjs")], env=env, stdin=None, timeout=180) + _check(result, "node run_real_sdk.mjs") + spans = receiver.spans() + exported = receiver.requests() + output = json.loads(result.stdout.decode("utf-8")) + + # The CLI called the mock (host root + /v1/messages), streaming, with the placeholder key. + mock_requests = output["requests"] + assert len(mock_requests) == 2, mock_requests + for request in mock_requests: + assert request["method"] == "POST" and request["url"].startswith("/v1/messages"), request + assert request["stream"] is True + assert [r["hasToolResult"] for r in mock_requests] == [False, True] + + messages = output["messages"] + init = [m for m in messages if m["type"] == "system" and m.get("subtype") == "init"] + results = [m for m in messages if m["type"] == "result"] + assert len(init) == 1 and len(results) == 1 and results[0]["subtype"] == "success" + session_id = init[0]["session_id"] + + assert sorted(span["name"] for span in spans) == sorted( + ["claude_agent.conversation", "claude_agent.assistant_turn", "claude_agent.assistant_turn", "tool.Read"] + ) + conversation = _one(spans, "claude_agent.conversation") + tool = _one(spans, "tool.Read") + turns = _by_name(spans, "claude_agent.assistant_turn") + issuing = [t for t in turns if _attrs(t)["claude_agent.message.has_tool_use"] is True] + assert len(issuing) == 1 and tool["parentSpanId"] == issuing[0]["spanId"] + for turn in turns: + a = _attrs(turn) + assert turn["parentSpanId"] == conversation["spanId"] + assert (a["gen_ai.span.kind"], a["gen_ai.request.model"], a["gen_ai.provider.name"]) == ("LLM", "claude-sonnet-4-5", "custom") + c = _attrs(conversation) + assert (c["claude_agent.span_kind"], c["gen_ai.span.kind"]) == ("conversation", "CHAIN") + assert c["session.id"] == session_id and c["claude_agent.session.id"] == session_id + assert c["gen_ai.usage.input_tokens"] > 0 and c["gen_ai.usage.output_tokens"] > 0 + assert "gen_ai.cost.total" in c and "claude_agent.cost.total_usd" in c + t = _attrs(tool) + assert (t["claude_agent.span_kind"], t["gen_ai.span.kind"], t["claude_agent.tool.source"]) == ("tool_execution", "TOOL", "builtin") + assert t["claude_agent.tool.is_error"] is False + for span in spans: + assert _status_code(span) == 1 + + blob = json.dumps(spans) + for marker in ("SECRET_PROMPT_MARKER", "SECRET_TOOL_OUTPUT_MARKER", "SECRET_ASSISTANT_TEXT_MARKER"): + assert marker not in blob + assert PLACEHOLDER_API_KEY not in blob and "PLACEHOLDER-not-a-key" not in blob + assert exported, "exporter sent nothing" + for request in exported: + assert request["path"] == "/tracer/v1/traces" + assert request["headers"]["x-api-key"] == PLACEHOLDER_API_KEY + assert request["resource_attributes"] + for resource in request["resource_attributes"]: + assert (resource["project_name"], resource["project_type"]) == ("th8235-real-sdk-contract", "observe") + + +def _read_member(tar: tarfile.TarFile, member: Any) -> bytes: + handle = tar.extractfile(member) + assert handle is not None, member + return handle.read() + + +def _sha256(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def _sdk_file_hashes() -> Dict[str, str]: + """Hash every file of the installed SDK and its platform binary package.""" + hashes: Dict[str, str] = {} + roots = [SDK_DIR.resolve()] + platform_root = SDK_DIR.resolve().parent # node_modules/@anthropic-ai inside the pnpm store entry + roots.extend(p.resolve() for p in platform_root.glob("claude-agent-sdk-*")) + for root in roots: + for file in root.rglob("*"): + if file.is_file(): + hashes[_sha256(file.read_bytes())] = str(file) + return hashes + + +NATIVE_MAGIC = ( + b"\x7fELF", # ELF + b"\xcf\xfa\xed\xfe", # Mach-O 64-bit LE + b"\xce\xfa\xed\xfe", # Mach-O 32-bit LE + b"\xca\xfe\xba\xbe", # Mach-O fat + b"MZ", # PE +) + + +def test_packed_tarball_has_no_sdk_sources_or_native_binary(built_package: Path, tmp_path: Path) -> None: + """AC-09: the SDK (Anthropic Commercial Terms) is a peer dependency, never bundled.""" + result = run( + [PNPM, "--dir", str(PKG_DIR), "pack", "--pack-destination", str(tmp_path)], + env=_base_env(), + stdin=None, + timeout=300, + ) + _check(result, "pnpm pack") + tarballs = list(tmp_path.glob("*.tgz")) + assert len(tarballs) == 1, tarballs + + sdk_hashes = _sdk_file_hashes() + assert len(sdk_hashes) > 10, "installed SDK not found; cannot compare" + with tarfile.open(tarballs[0]) as tar: + members = [m for m in tar.getmembers() if m.isfile()] + names = [m.name for m in members] + manifest = json.loads(_read_member(tar, "package/package.json")) + for member in members: + data = _read_member(tar, member) + assert member.name.startswith("package/"), member.name + relative = member.name[len("package/"):] + assert relative.split("/")[0] in {"dist", "src", "package.json", "README.md", "LICENSE"}, relative + assert "node_modules" not in member.name + assert "__tests__" not in member.name + assert not relative.endswith((".mjs", ".node", ".exe", ".wasm")), relative + assert Path(relative).name not in {"sdk.mjs", "sdk.d.ts", "sdk-tools.d.ts", "bridge.mjs", "claude", "cli.js"} + assert not data.startswith(NATIVE_MAGIC), f"native binary in tarball: {relative}" + assert _sha256(data) not in sdk_hashes, f"{relative} is a copy of {sdk_hashes[_sha256(data)]}" + assert b"Anthropic PBC. All rights reserved" not in data, relative + + assert manifest["peerDependencies"] == {"@anthropic-ai/claude-agent-sdk": "^0.3.142"} + assert "@anthropic-ai/claude-agent-sdk" not in manifest.get("dependencies", {}) + assert not manifest.get("bundledDependencies") and not manifest.get("bundleDependencies") + assert manifest["license"] == "Apache-2.0" + # workspace: ranges must be rewritten to publishable versions by pnpm pack. + assert all(not v.startswith("workspace:") for v in manifest.get("dependencies", {}).values()) + assert sum(1 for n in names if n.startswith("package/dist/")) > 0 + + +@pytest.mark.parametrize("node", [p for p in os.environ.get("TRACEAI_NODE_MATRIX", "").split(os.pathsep) if p] or [NODE]) +def test_esm_and_cjs_entry_points_import(built_package: Path, tmp_path: Path, node: str) -> None: + """AC-09: require() and import() of the package name resolve through package.json exports.""" + scope = tmp_path / "node_modules" / "@traceai" + scope.mkdir(parents=True) + (scope / "claude-agent-sdk").symlink_to(PKG_DIR, target_is_directory=True) + check = ( + "const names=['wrapQuery','shutdown','ClaudeAgentSDKInstrumentation','ClaudeAgentAttributes'];" + "function ok(m,kind){for(const n of names){if(!(n in m))throw new Error(kind+' missing '+n)}}" + ) + cjs = run( + [node, "-e", check + "ok(require('@traceai/claude-agent-sdk'),'cjs');console.log(process.version,'cjs ok')"], + env={**_base_env(), "NODE_PATH": str(tmp_path / "node_modules")}, stdin=None, timeout=60, + ) + script = tmp_path / "check.mjs" + script.write_text(check + "ok(await import('@traceai/claude-agent-sdk'),'esm');console.log(process.version,'esm ok');") + esm = run([node, str(script)], env=_base_env(), stdin=None, timeout=60) + _check(cjs, f"{node} require") + _check(esm, f"{node} import") + assert b"cjs ok" in cjs.stdout and b"esm ok" in esm.stdout diff --git a/typescript/packages/traceai_claude_agent_sdk/contract/tsconfig.fixtures.json b/typescript/packages/traceai_claude_agent_sdk/contract/tsconfig.fixtures.json new file mode 100644 index 000000000..54442154b --- /dev/null +++ b/typescript/packages/traceai_claude_agent_sdk/contract/tsconfig.fixtures.json @@ -0,0 +1,18 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "composite": false, + "incremental": false, + "declaration": false, + "declarationMap": false, + "sourceMap": false, + "module": "commonjs", + "rootDir": "../src/__tests__/fixtures", + "noEmitOnError": true, + "types": ["node"] + }, + "files": [ + "../src/__tests__/fixtures/messages.ts", + "../src/__tests__/fixtures/fakeQuery.ts" + ] +} diff --git a/typescript/packages/traceai_claude_agent_sdk/jest.config.js b/typescript/packages/traceai_claude_agent_sdk/jest.config.js new file mode 100644 index 000000000..1a1a13998 --- /dev/null +++ b/typescript/packages/traceai_claude_agent_sdk/jest.config.js @@ -0,0 +1,22 @@ +module.exports = { + preset: 'ts-jest', + testEnvironment: 'node', + roots: ['/src'], + testMatch: ['**/__tests__/**/*.test.ts'], + testPathIgnorePatterns: ['/node_modules/'], + collectCoverageFrom: ['src/**/*.ts', '!src/**/*.d.ts', '!src/index.ts', '!src/__tests__/**'], + coverageDirectory: 'coverage', + coverageReporters: ['text', 'lcov'], + transform: { + '^.+\\.ts$': ['ts-jest', { + tsconfig: { + module: 'commonjs' + } + }] + }, + moduleNameMapper: { + '^@traceai/fi-core$': '/../fi-core/src', + '^@traceai/fi-semantic-conventions$': '/../fi-semantic-conventions/src' + }, + testTimeout: 15000 +}; diff --git a/typescript/packages/traceai_claude_agent_sdk/package.json b/typescript/packages/traceai_claude_agent_sdk/package.json new file mode 100644 index 000000000..0e90e08ec --- /dev/null +++ b/typescript/packages/traceai_claude_agent_sdk/package.json @@ -0,0 +1,75 @@ +{ + "name": "@traceai/claude-agent-sdk", + "version": "0.1.0", + "description": "OpenTelemetry tracing for the Claude Agent SDK query() message stream", + "main": "./dist/src/index.js", + "module": "./dist/esm/index.js", + "esnext": "./dist/esnext/index.js", + "types": "./dist/src/index.d.ts", + "exports": { + ".": { + "types": "./dist/src/index.d.ts", + "import": "./dist/esm/index.js", + "require": "./dist/src/index.js" + } + }, + "scripts": { + "prebuild": "rimraf dist", + "build": "tsc --build tsconfig.json tsconfig.esm.json tsconfig.esnext.json && tsc-alias -p tsconfig.esm.json && pnpm run postbuild", + "postbuild": "echo '{\"type\": \"module\"}' > ./dist/esm/package.json", + "test": "jest", + "prepare": "pnpm run build" + }, + "keywords": [ + "opentelemetry", + "tracing", + "instrumentation", + "claude", + "claude-agent-sdk", + "agents", + "llm", + "observability" + ], + "author": "Future AGI ", + "license": "Apache-2.0", + "repository": { + "type": "git", + "url": "https://github.com/futureagi/traceai.git", + "directory": "typescript/packages/traceai_claude_agent_sdk" + }, + "bugs": { + "url": "https://github.com/futureagi/traceai/issues" + }, + "homepage": "https://github.com/futureagi/traceai#readme", + "engines": { + "node": ">=18.0.0" + }, + "dependencies": { + "@opentelemetry/api": "^1.9.0", + "@traceai/fi-core": "workspace:^", + "@traceai/fi-semantic-conventions": "workspace:^" + }, + "devDependencies": { + "@anthropic-ai/claude-agent-sdk": "0.3.289", + "@anthropic-ai/sdk": "^0.131.0", + "@opentelemetry/sdk-trace-base": "^2.0.1", + "@types/jest": "^29.5.14", + "@types/node": "^22.10.10", + "jest": "^29.7.0", + "rimraf": "^6.0.1", + "ts-jest": "^29.2.5", + "tsc-alias": "^1.8.10", + "typescript": "^5.7.3" + }, + "peerDependencies": { + "@anthropic-ai/claude-agent-sdk": "^0.3.142" + }, + "files": [ + "dist", + "src", + "!**/__tests__/**" + ], + "publishConfig": { + "access": "public" + } +} diff --git a/typescript/packages/traceai_claude_agent_sdk/src/__tests__/collector.test.ts b/typescript/packages/traceai_claude_agent_sdk/src/__tests__/collector.test.ts new file mode 100644 index 000000000..3fdde4af6 --- /dev/null +++ b/typescript/packages/traceai_claude_agent_sdk/src/__tests__/collector.test.ts @@ -0,0 +1,138 @@ +/** + * Collector contract that the shared Python Receiver cannot check (it drops + * resource attributes and headers): the real fi-core register() + OTLP/HTTP + * exporter posts to {endpoint}/tracer/v1/traces with X-Api-Key / X-Secret-Key, + * and the resource carries project_name and project_type=observe. + * + * Loopback only; placeholder keys; no Anthropic call. + */ +import { createServer, IncomingHttpHeaders, Server } from "http"; +import { AddressInfo } from "net"; +import { ProjectType, register } from "@traceai/fi-core"; +import { shutdown, wrapQuery } from "../index"; +import { makeFakeQuery } from "./fixtures/fakeQuery"; +import { PROMPT, simpleToolJourney } from "./fixtures/messages"; +import { drain } from "./helpers"; + +interface Captured { + path: string; + headers: IncomingHttpHeaders; + body: Buffer; +} + +const API_KEY = "test-api-key-PLACEHOLDER"; +const SECRET_KEY = "test-secret-key-PLACEHOLDER"; + +type OtlpValue = { stringValue?: string; intValue?: string | number; doubleValue?: number; boolValue?: boolean }; +type OtlpAttr = { key: string; value: OtlpValue }; + +function attrMap(attributes: OtlpAttr[] = []): Record { + const out: Record = {}; + for (const { key, value } of attributes) { + out[key] = value.stringValue ?? value.intValue ?? value.doubleValue ?? value.boolValue; + } + return out; +} + +describe("fi-core export contract", () => { + let server: Server; + let origin: string; + let captured: Captured[]; + const savedEnv: Record = {}; + + beforeAll(async () => { + captured = []; + server = createServer((req, res) => { + const chunks: Buffer[] = []; + req.on("data", (chunk) => chunks.push(chunk)); + req.on("end", () => { + captured.push({ path: req.url ?? "", headers: req.headers, body: Buffer.concat(chunks) }); + res.writeHead(200, { "content-type": "application/json" }); + res.end("{}"); + }); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + origin = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; + for (const key of ["FI_API_KEY", "FI_SECRET_KEY", "FI_BASE_URL", "FI_PROJECT_NAME"]) { + savedEnv[key] = process.env[key]; + } + process.env.FI_API_KEY = API_KEY; + process.env.FI_SECRET_KEY = SECRET_KEY; + delete process.env.FI_BASE_URL; + delete process.env.FI_PROJECT_NAME; + }); + + afterAll(async () => { + for (const [key, value] of Object.entries(savedEnv)) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + await new Promise((resolve) => server.close(() => resolve())); + }); + + beforeEach(() => { + captured.length = 0; + }); + + async function runProject(projectName: string) { + const provider = register({ + projectName, + projectType: ProjectType.OBSERVE, + endpoint: origin, + batch: true, + setGlobalTracerProvider: false, + }); + const traced = wrapQuery(makeFakeQuery(simpleToolJourney()).query, { tracerProvider: provider }); + await drain(traced({ prompt: PROMPT })); + await shutdown(provider); + await provider.shutdown(); + } + + function decode(request: Captured) { + expect(String(request.headers["content-type"])).toContain("application/json"); + return JSON.parse(request.body.toString("utf8")) as { + resourceSpans: { resource: { attributes: OtlpAttr[] }; scopeSpans: { spans: { name: string; attributes: OtlpAttr[] }[] }[] }[]; + }; + } + + it("posts to /tracer/v1/traces with X-Api-Key, X-Secret-Key and the project resource", async () => { + await runProject("th8235-contract"); + expect(captured.length).toBeGreaterThan(0); + const names: string[] = []; + for (const request of captured) { + expect(request.path).toBe("/tracer/v1/traces"); + expect(request.headers["x-api-key"]).toBe(API_KEY); + expect(request.headers["x-secret-key"]).toBe(SECRET_KEY); + expect(request.headers["authorization"]).toBeUndefined(); + for (const resourceSpans of decode(request).resourceSpans) { + const resource = attrMap(resourceSpans.resource.attributes); + expect(resource["project_name"]).toBe("th8235-contract"); + expect(resource["project_type"]).toBe("observe"); + expect(resource["openinference.project.name"]).toBeUndefined(); + for (const scope of resourceSpans.scopeSpans) names.push(...scope.spans.map((s) => s.name)); + } + } + expect(names.sort()).toEqual( + ["claude_agent.assistant_turn", "claude_agent.assistant_turn", "claude_agent.conversation", "tool.Read"].sort(), + ); + }); + + it("keeps two projects apart and never puts the API key or secret on a span", async () => { + await runProject("th8235-project-a"); + await runProject("th8235-project-b"); + const projects = new Set(); + for (const request of captured) { + expect(request.body.toString("utf8")).not.toContain(API_KEY); + expect(request.body.toString("utf8")).not.toContain(SECRET_KEY); + for (const resourceSpans of decode(request).resourceSpans) { + projects.add(attrMap(resourceSpans.resource.attributes)["project_name"]); + for (const scope of resourceSpans.scopeSpans) { + for (const span of scope.spans) { + expect(JSON.stringify(span.attributes)).not.toContain("PLACEHOLDER"); + } + } + } + } + expect([...projects].sort()).toEqual(["th8235-project-a", "th8235-project-b"]); + }); +}); diff --git a/typescript/packages/traceai_claude_agent_sdk/src/__tests__/fixtures/fakeQuery.ts b/typescript/packages/traceai_claude_agent_sdk/src/__tests__/fixtures/fakeQuery.ts new file mode 100644 index 000000000..129dc129e --- /dev/null +++ b/typescript/packages/traceai_claude_agent_sdk/src/__tests__/fixtures/fakeQuery.ts @@ -0,0 +1,74 @@ +/** + * A fake `query()` with the SDK's signature. It yields recorded SDK-typed + * messages and never starts the Claude Code CLI or calls the Anthropic API. + */ +import type { Options, Query, SDKMessage, query } from "@anthropic-ai/claude-agent-sdk"; + +export interface FakeQueryControl { + /** Every params object the fake received, by reference. */ + calls: { prompt: unknown; options?: Options }[]; + interruptCalls: number; +} + +export interface FakeQueryBehavior { + /** Throw `error` instead of yielding message `throwAt`. */ + throwAt?: number; + error?: unknown; + /** Before yielding message `waitForAbortAt`, block until options.abortController aborts. */ + waitForAbortAt?: number; +} + +export class FakeAbortError extends Error { + constructor() { + super("Claude Code process aborted by user"); + this.name = "AbortError"; + } +} + +export function makeFakeQuery( + messages: SDKMessage[], + behavior: FakeQueryBehavior = {}, +): { query: typeof query; control: FakeQueryControl } { + const control: FakeQueryControl = { calls: [], interruptCalls: 0 }; + + const fake = (params: { prompt: string | AsyncIterable; options?: Options }): Query => { + control.calls.push(params); + const signal = params.options?.abortController?.signal; + + async function* generate(): AsyncGenerator { + for (let index = 0; index < messages.length; index += 1) { + if (behavior.throwAt === index) { + throw behavior.error ?? new Error("fake query failed"); + } + if (behavior.waitForAbortAt === index) { + await new Promise((_, reject) => { + if (!signal) { + reject(new Error("waitForAbortAt needs options.abortController")); + return; + } + if (signal.aborted) { + reject(new FakeAbortError()); + return; + } + signal.addEventListener("abort", () => reject(new FakeAbortError()), { once: true }); + }); + } + yield messages[index]; + } + } + + const generator = generate(); + const controlMethods = { + interrupt: async () => { + control.interruptCalls += 1; + return undefined; + }, + setPermissionMode: async () => undefined, + setModel: async () => undefined, + close: () => undefined, + }; + return Object.assign(generator, controlMethods) as unknown as Query; + }; + + return { query: fake as unknown as typeof query, control }; +} diff --git a/typescript/packages/traceai_claude_agent_sdk/src/__tests__/fixtures/messages.ts b/typescript/packages/traceai_claude_agent_sdk/src/__tests__/fixtures/messages.ts new file mode 100644 index 000000000..ca57a6a15 --- /dev/null +++ b/typescript/packages/traceai_claude_agent_sdk/src/__tests__/fixtures/messages.ts @@ -0,0 +1,273 @@ +/** + * Recorded-style message sequences for the wrapper tests. + * + * Authored from the `@anthropic-ai/claude-agent-sdk` 0.3.289 `sdk.d.ts` types + * (not from a live call). Every message is type-checked against the installed + * SDK types, so a field rename in the pinned SDK breaks compilation here. + */ +import type { + SDKAssistantMessage, + SDKMessage, + SDKResultError, + SDKResultSuccess, + SDKSystemMessage, + SDKTaskNotificationMessage, + SDKTaskStartedMessage, + SDKUserMessage, +} from "@anthropic-ai/claude-agent-sdk"; + +type UUID = `${string}-${string}-${string}-${string}-${string}`; +type BetaMessage = SDKAssistantMessage["message"]; +type BetaContentBlock = BetaMessage["content"][number]; +type UserContent = Exclude; +type NonNullableUsage = SDKResultSuccess["usage"]; + +export const SESSION_ID = "11111111-2222-4333-8444-555555555555"; +export const MODEL = "claude-sonnet-4-5"; +export const PROMPT = "Summarize README.md. SECRET_PROMPT_MARKER"; + +let uuidCounter = 0; +function uuid(): UUID { + uuidCounter += 1; + const tail = uuidCounter.toString(16).padStart(12, "0"); + return `00000000-0000-4000-8000-${tail}` as UUID; +} + +const betaUsage = (input: number, output: number): BetaMessage["usage"] => ({ + cache_creation: null, + cache_creation_input_tokens: null, + cache_read_input_tokens: null, + fallback_credit: null, + inference_geo: null, + input_tokens: input, + iterations: null, + output_tokens: output, + output_tokens_details: null, + server_tool_use: null, + service_tier: "standard", + speed: null, +}); + +export function text(value: string): BetaContentBlock { + return { type: "text", text: value, citations: null }; +} + +export function toolUse(id: string, name: string, input: unknown): BetaContentBlock { + return { type: "tool_use", id, name, input }; +} + +export function assistant( + messageId: string, + content: BetaContentBlock[], + parentToolUseId: string | null = null, + extra: Partial = {}, +): SDKAssistantMessage { + return { + type: "assistant", + message: { + id: messageId, + type: "message", + role: "assistant", + model: MODEL, + content, + container: null, + context_management: null, + diagnostics: null, + stop_details: null, + stop_reason: "end_turn", + stop_sequence: null, + usage: betaUsage(10, 5), + }, + parent_tool_use_id: parentToolUseId, + uuid: uuid(), + session_id: SESSION_ID, + ...extra, + }; +} + +export function toolResult( + toolUseId: string, + content: string, + parentToolUseId: string | null = null, + isError = false, +): SDKUserMessage { + const blocks: UserContent = [{ type: "tool_result", tool_use_id: toolUseId, content, is_error: isError }]; + return { + type: "user", + message: { role: "user", content: blocks }, + parent_tool_use_id: parentToolUseId, + uuid: uuid(), + session_id: SESSION_ID, + }; +} + +export function init( + overrides: Partial = {}, +): SDKSystemMessage { + return { + type: "system", + subtype: "init", + apiKeySource: "ANTHROPIC_API_KEY", + claude_code_version: "2.1.289", + cwd: "/tmp/project", + tools: ["Read", "Grep", "Agent"], + mcp_servers: [], + model: MODEL, + permissionMode: "default", + slash_commands: [], + output_style: "default", + skills: [], + plugins: [], + uuid: uuid(), + session_id: SESSION_ID, + ...overrides, + }; +} + +const resultUsage: NonNullableUsage = { + cache_creation: { ephemeral_1h_input_tokens: 0, ephemeral_5m_input_tokens: 0 }, + cache_creation_input_tokens: 7, + cache_read_input_tokens: 11, + fallback_credit: null, + inference_geo: "us", + input_tokens: 120, + iterations: [], + output_tokens: 45, + output_tokens_details: { thinking_tokens: 0 }, + server_tool_use: { web_fetch_requests: 0, web_search_requests: 0 }, + service_tier: "standard", + speed: "standard", +}; + +export const TOTAL_COST_USD = 0.0123; + +export function resultSuccess(resultText: string, overrides: Partial = {}): SDKResultSuccess { + return { + type: "result", + subtype: "success", + duration_ms: 1500, + duration_api_ms: 1200, + ttft_ms: 300, + is_error: false, + num_turns: 2, + result: resultText, + stop_reason: "end_turn", + total_cost_usd: TOTAL_COST_USD, + usage: resultUsage, + modelUsage: {}, + permission_denials: [], + uuid: uuid(), + session_id: SESSION_ID, + ...overrides, + }; +} + +export function resultError(overrides: Partial = {}): SDKResultError { + return { + type: "result", + subtype: "error_max_turns", + duration_ms: 900, + duration_api_ms: 800, + is_error: true, + num_turns: 3, + stop_reason: null, + total_cost_usd: 0.002, + usage: resultUsage, + modelUsage: {}, + permission_denials: [], + errors: ["Reached maximum number of turns (3)"], + uuid: uuid(), + session_id: SESSION_ID, + ...overrides, + }; +} + +export function taskStarted(toolUseId: string, overrides: Partial = {}): SDKTaskStartedMessage { + return { + type: "system", + subtype: "task_started", + task_id: "task-1", + tool_use_id: toolUseId, + description: "Review the diff", + subagent_type: "code-reviewer", + uuid: uuid(), + session_id: SESSION_ID, + ...overrides, + }; +} + +export function taskNotification( + toolUseId: string, + status: SDKTaskNotificationMessage["status"] = "completed", +): SDKTaskNotificationMessage { + return { + type: "system", + subtype: "task_notification", + task_id: "task-1", + tool_use_id: toolUseId, + status, + output_file: "/tmp/task-1.md", + summary: "done", + uuid: uuid(), + session_id: SESSION_ID, + }; +} + +// ---------------------------------------------------------------------------- +// Journeys +// ---------------------------------------------------------------------------- + +export const TOOL_INPUT_MARKER = "SECRET_TOOL_INPUT_MARKER.md"; +export const TOOL_OUTPUT_MARKER = "SECRET_TOOL_OUTPUT_MARKER"; +export const ASSISTANT_TEXT_MARKER = "SECRET_ASSISTANT_TEXT_MARKER"; + +/** J1: one built-in tool call, then a final answer. The first API response is split over two SDK messages. */ +export function simpleToolJourney(): SDKMessage[] { + return [ + init(), + assistant("msg_01", [text(`Reading the file. ${ASSISTANT_TEXT_MARKER}`)]), + assistant("msg_01", [toolUse("toolu_read_1", "Read", { file_path: TOOL_INPUT_MARKER })]), + toolResult("toolu_read_1", `# README ${TOOL_OUTPUT_MARKER}`), + assistant("msg_02", [text(`The README describes traceAI. ${ASSISTANT_TEXT_MARKER}`)]), + resultSuccess(`The README describes traceAI. ${ASSISTANT_TEXT_MARKER}`), + ]; +} + +export const AGENT_TOOL_ID = "toolu_agent_1"; +export const SUBAGENT_GREP_ID = "toolu_grep_1"; + +/** J2: the Agent (Task) tool runs a subagent that uses Grep. */ +export function subagentJourney(): SDKMessage[] { + return [ + init(), + assistant("msg_10", [ + toolUse(AGENT_TOOL_ID, "Agent", { + description: "Review the diff", + prompt: "Look for TODOs. SECRET_SUBAGENT_PROMPT_MARKER", + subagent_type: "code-reviewer", + run_in_background: false, + }), + ]), + taskStarted(AGENT_TOOL_ID), + assistant("msg_11", [toolUse(SUBAGENT_GREP_ID, "Grep", { pattern: "TODO" })], AGENT_TOOL_ID), + toolResult(SUBAGENT_GREP_ID, "src/a.ts:3: TODO", AGENT_TOOL_ID), + assistant("msg_12", [text("Found one TODO.")], AGENT_TOOL_ID), + taskNotification(AGENT_TOOL_ID, "completed"), + toolResult(AGENT_TOOL_ID, "Found one TODO."), + assistant("msg_13", [text("The reviewer found one TODO.")]), + resultSuccess("The reviewer found one TODO.", { num_turns: 3 }), + ]; +} + +export const MCP_TOOL_ID = "toolu_mcp_1"; + +/** J3: an MCP tool that errors, then the run stops on max turns. */ +export function mcpErrorJourney(): SDKMessage[] { + return [ + init({ mcp_servers: [{ name: "docs", status: "connected" }] }), + assistant("msg_20", [toolUse(MCP_TOOL_ID, "mcp__docs__search", { query: "tracing" })]), + toolResult(MCP_TOOL_ID, "server unavailable", null, true), + assistant("msg_21", [text("The docs server failed.")]), + resultError(), + ]; +} diff --git a/typescript/packages/traceai_claude_agent_sdk/src/__tests__/helpers.ts b/typescript/packages/traceai_claude_agent_sdk/src/__tests__/helpers.ts new file mode 100644 index 000000000..2476fd5a3 --- /dev/null +++ b/typescript/packages/traceai_claude_agent_sdk/src/__tests__/helpers.ts @@ -0,0 +1,79 @@ +import { Context, ContextManager, ROOT_CONTEXT } from "@opentelemetry/api"; +import { + BasicTracerProvider, + InMemorySpanExporter, + ReadableSpan, + SimpleSpanProcessor, +} from "@opentelemetry/sdk-trace-base"; + +export function memoryProvider(): { provider: BasicTracerProvider; exporter: InMemorySpanExporter } { + const exporter = new InMemorySpanExporter(); + const provider = new BasicTracerProvider({ spanProcessors: [new SimpleSpanProcessor(exporter)] }); + return { provider, exporter }; +} + +export async function drain(iterable: AsyncIterable): Promise { + const out: T[] = []; + for await (const item of iterable) { + out.push(item); + } + return out; +} + +export function byName(spans: ReadableSpan[], name: string): ReadableSpan[] { + return spans.filter((span) => span.name === name); +} + +export function one(spans: ReadableSpan[], name: string): ReadableSpan { + const matches = byName(spans, name); + if (matches.length !== 1) { + throw new Error(`expected one span named ${name}, got ${matches.length}: ${spans.map((s) => s.name).join(", ")}`); + } + return matches[0]; +} + +export function parentId(span: ReadableSpan): string | undefined { + // sdk-trace-base 2.x exposes parentSpanContext; 1.x exposed parentSpanId. + const anySpan = span as unknown as { parentSpanContext?: { spanId: string }; parentSpanId?: string }; + return anySpan.parentSpanContext?.spanId ?? anySpan.parentSpanId; +} + +export function allAttributeText(spans: ReadableSpan[]): string { + return JSON.stringify(spans.map((span) => span.attributes)); +} + +/** Minimal synchronous context manager, so `context.with()` works in tests without extra deps. */ +export class StackContextManager implements ContextManager { + private stack: Context[] = [ROOT_CONTEXT]; + + active(): Context { + return this.stack[this.stack.length - 1]; + } + + with ReturnType>( + ctx: Context, + fn: F, + thisArg?: ThisParameterType, + ...args: A + ): ReturnType { + this.stack.push(ctx); + try { + return fn.call(thisArg, ...args); + } finally { + this.stack.pop(); + } + } + + bind(_ctx: Context, target: T): T { + return target; + } + + enable(): this { + return this; + } + + disable(): this { + this.stack = [ROOT_CONTEXT]; + return this; + } +} diff --git a/typescript/packages/traceai_claude_agent_sdk/src/__tests__/parity.test.ts b/typescript/packages/traceai_claude_agent_sdk/src/__tests__/parity.test.ts new file mode 100644 index 000000000..9b1fbb424 --- /dev/null +++ b/typescript/packages/traceai_claude_agent_sdk/src/__tests__/parity.test.ts @@ -0,0 +1,153 @@ +/** + * AC-01 / AC-02 parity with the Python package. + * + * Reads `python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_attributes.py` + * from this repository and fails when a span kind, attribute name or built-in + * tool from the Python list is missing or has a different value here. + */ +import { readFileSync } from "fs"; +import { join } from "path"; +import { SpanStatusCode } from "@opentelemetry/api"; +import { + BUILTIN_TOOLS, + ClaudeAgentAttributes, + ClaudeAgentSpanKind, + FI_SPAN_KIND_BY_CLAUDE_KIND, + wrapQuery, +} from "../index"; +import { FISpanKind } from "@traceai/fi-semantic-conventions"; +import { makeFakeQuery } from "./fixtures/fakeQuery"; +import { PROMPT, mcpErrorJourney, simpleToolJourney, subagentJourney } from "./fixtures/messages"; +import { drain, memoryProvider } from "./helpers"; + +const PYTHON_ATTRIBUTES = join( + __dirname, + "../../../../../python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_attributes.py", +); + +function pythonClassBody(source: string, className: string): string { + const start = source.indexOf(`class ${className}`); + if (start < 0) throw new Error(`class ${className} not found in ${PYTHON_ATTRIBUTES}`); + const rest = source.slice(start + 1); + const next = rest.search(/\n(class |# Built-in tool names|def )/); + return next < 0 ? rest : rest.slice(0, next); +} + +function pythonAssignments(body: string): Record { + const out: Record = {}; + for (const match of body.matchAll(/^\s+([A-Z_]+)\s*=\s*"([^"]+)"/gm)) { + out[match[1]] = match[2]; + } + return out; +} + +describe("Python parity", () => { + const source = readFileSync(PYTHON_ATTRIBUTES, "utf8"); + + it("has every Python span kind with the same string", () => { + const python = pythonAssignments(pythonClassBody(source, "ClaudeAgentSpanKind")); + expect(Object.keys(python).length).toBe(5); + expect(ClaudeAgentSpanKind).toEqual(python); + }); + + it("has every Python attribute name with the same value (AC-02)", () => { + const python = pythonAssignments(pythonClassBody(source, "ClaudeAgentAttributes")); + expect(Object.keys(python).length).toBeGreaterThan(60); + expect(ClaudeAgentAttributes).toEqual(python); + }); + + it("includes the Python built-in tool list", () => { + const block = source.slice(source.indexOf("BUILTIN_TOOLS = frozenset([")); + const list = block.slice(0, block.indexOf("])")); + const python = [...list.matchAll(/"([A-Za-z]+)"/g)].map((m) => m[1]); + expect(python.length).toBe(13); + for (const name of python) expect(BUILTIN_TOOLS.has(name)).toBe(true); + }); + + it("maps each kind onto a Future AGI span kind that exists in fi-semantic-conventions", () => { + const known = new Set(Object.values(FISpanKind) as string[]); + for (const value of Object.values(FI_SPAN_KIND_BY_CLAUDE_KIND)) { + expect(known.has(value)).toBe(true); + } + // TS FISpanKind has no CONVERSATION: the conversation span falls back to CHAIN. + expect((FISpanKind as Record)["CONVERSATION"]).toBeUndefined(); + expect(FI_SPAN_KIND_BY_CLAUDE_KIND.conversation).toBe("CHAIN"); + expect(FI_SPAN_KIND_BY_CLAUDE_KIND.assistant_turn).toBe("LLM"); + expect(FI_SPAN_KIND_BY_CLAUDE_KIND.tool_execution).toBe("TOOL"); + expect(FI_SPAN_KIND_BY_CLAUDE_KIND.mcp_tool).toBe("TOOL"); + expect(FI_SPAN_KIND_BY_CLAUDE_KIND.subagent).toBe("AGENT"); + }); + + it("emits all five span kinds across J1-J3 and each span carries the names it should (AC-01, AC-02)", async () => { + const { provider, exporter } = memoryProvider(); + for (const journey of [simpleToolJourney(), subagentJourney(), mcpErrorJourney()]) { + await drain(wrapQuery(makeFakeQuery(journey).query, { tracerProvider: provider })({ prompt: PROMPT })); + } + const spans = exporter.getFinishedSpans(); + const kinds = new Set(spans.map((s) => s.attributes[ClaudeAgentAttributes.SPAN_KIND])); + expect([...kinds].sort()).toEqual(Object.values(ClaudeAgentSpanKind).sort()); + + const required: Record = { + conversation: [ + ClaudeAgentAttributes.SPAN_KIND, + ClaudeAgentAttributes.AGENT_MODEL, + ClaudeAgentAttributes.AGENT_SESSION_ID, + ClaudeAgentAttributes.GEN_AI_CONVERSATION_ID, + ClaudeAgentAttributes.AGENT_NUM_TURNS, + ClaudeAgentAttributes.USAGE_INPUT_TOKENS, + ClaudeAgentAttributes.USAGE_OUTPUT_TOKENS, + ClaudeAgentAttributes.USAGE_TOTAL_TOKENS, + ClaudeAgentAttributes.COST_TOTAL_USD, + ClaudeAgentAttributes.DURATION_MS, + ClaudeAgentAttributes.DURATION_API_MS, + ClaudeAgentAttributes.IS_ERROR, + "gen_ai.span.kind", + "gen_ai.cost.total", + "gen_ai.request.model", + "session.id", + ], + assistant_turn: [ + ClaudeAgentAttributes.SPAN_KIND, + ClaudeAgentAttributes.AGENT_NUM_TURNS, + ClaudeAgentAttributes.AGENT_MODEL, + ClaudeAgentAttributes.MESSAGE_HAS_TOOL_USE, + ClaudeAgentAttributes.MESSAGE_TOOL_USE_COUNT, + "gen_ai.span.kind", + "gen_ai.request.model", + "gen_ai.provider.name", + ], + tool_execution: [ + ClaudeAgentAttributes.SPAN_KIND, + ClaudeAgentAttributes.GEN_AI_TOOL_NAME, + ClaudeAgentAttributes.TOOL_USE_ID, + ClaudeAgentAttributes.TOOL_SOURCE, + ClaudeAgentAttributes.TOOL_IS_ERROR, + ClaudeAgentAttributes.TOOL_DURATION_MS, + "gen_ai.span.kind", + "gen_ai.tool.name", + ], + mcp_tool: [ + ClaudeAgentAttributes.SPAN_KIND, + ClaudeAgentAttributes.GEN_AI_TOOL_NAME, + ClaudeAgentAttributes.TOOL_SOURCE, + ClaudeAgentAttributes.MCP_SERVER_NAME, + ClaudeAgentAttributes.TOOL_IS_ERROR, + "gen_ai.span.kind", + ], + subagent: [ + ClaudeAgentAttributes.SPAN_KIND, + ClaudeAgentAttributes.SUBAGENT_TYPE, + ClaudeAgentAttributes.TOOL_USE_ID, + ClaudeAgentAttributes.TOOL_DURATION_MS, + "gen_ai.span.kind", + ], + }; + for (const span of spans) { + const kind = span.attributes[ClaudeAgentAttributes.SPAN_KIND] as string; + for (const key of required[kind]) { + expect([span.name, key, span.attributes[key] !== undefined]).toEqual([span.name, key, true]); + } + expect(span.status.code).not.toBe(SpanStatusCode.UNSET); + } + }); +}); diff --git a/typescript/packages/traceai_claude_agent_sdk/src/__tests__/wrapper.test.ts b/typescript/packages/traceai_claude_agent_sdk/src/__tests__/wrapper.test.ts new file mode 100644 index 000000000..15f9e3c6a --- /dev/null +++ b/typescript/packages/traceai_claude_agent_sdk/src/__tests__/wrapper.test.ts @@ -0,0 +1,586 @@ +import { SpanStatusCode, context, trace } from "@opentelemetry/api"; +import { + BasicTracerProvider, + BatchSpanProcessor, + InMemorySpanExporter, + ReadableSpan, + SpanExporter, +} from "@opentelemetry/sdk-trace-base"; +import { setSession, setUser } from "@traceai/fi-core"; +import { + ClaudeAgentSDKInstrumentation, + isWrappedQuery, + shutdown, + wrapQuery, +} from "../index"; +import { makeFakeQuery } from "./fixtures/fakeQuery"; +import { + AGENT_TOOL_ID, + ASSISTANT_TEXT_MARKER, + MCP_TOOL_ID, + MODEL, + PROMPT, + SESSION_ID, + SUBAGENT_GREP_ID, + TOOL_INPUT_MARKER, + TOOL_OUTPUT_MARKER, + TOTAL_COST_USD, + mcpErrorJourney, + simpleToolJourney, + subagentJourney, +} from "./fixtures/messages"; +import { StackContextManager, allAttributeText, byName, drain, memoryProvider, one, parentId } from "./helpers"; + +const CONTENT_ENV = ["FI_HIDE_INPUTS", "FI_HIDE_OUTPUTS"] as const; + +describe("wrapQuery", () => { + let saved: Record; + + beforeEach(() => { + saved = {}; + for (const key of [...CONTENT_ENV, "ANTHROPIC_BASE_URL"]) { + saved[key] = process.env[key]; + delete process.env[key]; + } + }); + + afterEach(() => { + for (const [key, value] of Object.entries(saved)) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + }); + + async function run( + messages: ReturnType, + options: Record = {}, + traceConfig?: { hideInputs?: boolean; hideOutputs?: boolean }, + ) { + const { provider, exporter } = memoryProvider(); + const fake = makeFakeQuery(messages); + const traced = wrapQuery(fake.query, { tracerProvider: provider, traceConfig }); + const params = { prompt: PROMPT, options }; + const yielded = await drain(traced(params)); + return { spans: exporter.getFinishedSpans(), yielded, fake, params }; + } + + describe("J1: built-in tool (AC-01, AC-03, AC-04, AC-05, AC-08)", () => { + it("emits conversation, assistant_turn and tool_execution spans with both span-kind keys", async () => { + const { spans } = await run(simpleToolJourney()); + + expect(spans.map((s) => s.name).sort()).toEqual( + ["claude_agent.assistant_turn", "claude_agent.assistant_turn", "claude_agent.conversation", "tool.Read"].sort(), + ); + const conversation = one(spans, "claude_agent.conversation"); + const [turn1, turn2] = byName(spans, "claude_agent.assistant_turn").sort( + (a, b) => (a.attributes["claude_agent.num_turns"] as number) - (b.attributes["claude_agent.num_turns"] as number), + ); + const tool = one(spans, "tool.Read"); + + expect(conversation.attributes["claude_agent.span_kind"]).toBe("conversation"); + expect(conversation.attributes["gen_ai.span.kind"]).toBe("CHAIN"); + expect(conversation.attributes["fi.span.kind"]).toBe("CHAIN"); + expect(turn1.attributes["claude_agent.span_kind"]).toBe("assistant_turn"); + expect(turn1.attributes["gen_ai.span.kind"]).toBe("LLM"); + expect(tool.attributes["claude_agent.span_kind"]).toBe("tool_execution"); + expect(tool.attributes["gen_ai.span.kind"]).toBe("TOOL"); + expect(tool.attributes["fi.span.kind"]).toBe("TOOL"); + + // AC-04: one trace, tool parent is the assistant turn that issued it. + expect(new Set(spans.map((s) => s.spanContext().traceId)).size).toBe(1); + expect(parentId(turn1)).toBe(conversation.spanContext().spanId); + expect(parentId(turn2)).toBe(conversation.spanContext().spanId); + expect(parentId(tool)).toBe(turn1.spanContext().spanId); + + // The split API response (two SDK messages, same message.id) is one turn. + expect(turn1.attributes["claude_agent.message.tool_use_count"]).toBe(1); + expect(turn1.attributes["claude_agent.message.has_tool_use"]).toBe(true); + expect(turn2.attributes["claude_agent.message.has_tool_use"]).toBe(false); + + // Model, provider, tool metadata. + expect(conversation.attributes["gen_ai.request.model"]).toBe(MODEL); + expect(conversation.attributes["claude_agent.model"]).toBe(MODEL); + expect(turn1.attributes["gen_ai.request.model"]).toBe(MODEL); + expect(turn1.attributes["gen_ai.provider.name"]).toBe("anthropic"); + expect(tool.attributes["claude_agent.tool.name"]).toBe("Read"); + expect(tool.attributes["gen_ai.tool.name"]).toBe("Read"); + expect(tool.attributes["claude_agent.tool.use_id"]).toBe("toolu_read_1"); + expect(tool.attributes["claude_agent.tool.source"]).toBe("builtin"); + expect(tool.attributes["claude_agent.tool.is_error"]).toBe(false); + expect(typeof tool.attributes["claude_agent.tool.duration_ms"]).toBe("number"); + + // AC-05: session id from the init message on every span. + for (const span of spans) { + expect(span.attributes["session.id"]).toBe(SESSION_ID); + expect(span.attributes["claude_agent.session.id"]).toBe(SESSION_ID); + } + expect(conversation.attributes["claude_agent.session_id"]).toBe(SESSION_ID); + expect(conversation.attributes["claude_agent.session.is_new"]).toBe(true); + + // AC-03: usage and cost from the result message. + expect(conversation.attributes["gen_ai.usage.input_tokens"]).toBe(120); + expect(conversation.attributes["gen_ai.usage.output_tokens"]).toBe(45); + expect(conversation.attributes["gen_ai.usage.total_tokens"]).toBe(165); + expect(conversation.attributes["gen_ai.usage.cache_read_tokens"]).toBe(11); + expect(conversation.attributes["gen_ai.usage.cache_creation_tokens"]).toBe(7); + expect(conversation.attributes["claude_agent.cost.total_usd"]).toBe(TOTAL_COST_USD); + expect(conversation.attributes["gen_ai.cost.total"]).toBe(TOTAL_COST_USD); + expect(conversation.attributes["claude_agent.duration_ms"]).toBe(1500); + expect(conversation.attributes["claude_agent.duration_api_ms"]).toBe(1200); + expect(conversation.attributes["claude_agent.time_to_first_token_ms"]).toBe(300); + expect(conversation.attributes["claude_agent.num_turns"]).toBe(2); + expect(conversation.attributes["claude_agent.is_error"]).toBe(false); + + for (const span of spans) { + expect(span.status.code).toBe(SpanStatusCode.OK); + expect(span.ended).toBe(true); + } + }); + + it("AC-07: keeps prompt, tool input/output and assistant text off spans by default", async () => { + const { spans } = await run(simpleToolJourney(), { systemPrompt: "SECRET_SYSTEM_PROMPT_MARKER" }); + const text = allAttributeText(spans); + for (const marker of [ + "SECRET_PROMPT_MARKER", + "SECRET_SYSTEM_PROMPT_MARKER", + TOOL_INPUT_MARKER, + TOOL_OUTPUT_MARKER, + ASSISTANT_TEXT_MARKER, + ]) { + expect(text).not.toContain(marker); + } + for (const span of spans) { + for (const key of [ + "input.value", + "output.value", + "claude_agent.prompt", + "claude_agent.system_prompt", + "claude_agent.tool.input", + "claude_agent.tool.output", + "claude_agent.message.content", + "claude_agent.tool.file_path", + ]) { + expect(span.attributes[key]).toBeUndefined(); + } + } + }); + + it("writes content only when hideInputs/hideOutputs are false", async () => { + const { spans } = await run( + simpleToolJourney(), + { systemPrompt: "be brief" }, + { hideInputs: false, hideOutputs: false }, + ); + const conversation = one(spans, "claude_agent.conversation"); + const tool = one(spans, "tool.Read"); + expect(conversation.attributes["input.value"]).toBe(PROMPT); + expect(conversation.attributes["claude_agent.prompt"]).toBe(PROMPT); + expect(conversation.attributes["claude_agent.system_prompt"]).toBe("be brief"); + expect(conversation.attributes["output.value"]).toContain(ASSISTANT_TEXT_MARKER); + expect(tool.attributes["claude_agent.tool.input"]).toBe(JSON.stringify({ file_path: TOOL_INPUT_MARKER })); + expect(tool.attributes["input.value"]).toBe(JSON.stringify({ file_path: TOOL_INPUT_MARKER })); + expect(tool.attributes["claude_agent.tool.file_path"]).toBe(TOOL_INPUT_MARKER); + expect(tool.attributes["claude_agent.tool.output"]).toContain(TOOL_OUTPUT_MARKER); + expect(tool.attributes["output.value"]).toContain(TOOL_OUTPUT_MARKER); + const firstTurn = byName(spans, "claude_agent.assistant_turn").find( + (s) => s.attributes["claude_agent.num_turns"] === 1, + )!; + expect(firstTurn.attributes["claude_agent.message.content"]).toBe( + `Reading the file. ${ASSISTANT_TEXT_MARKER} [Tool: Read]`, + ); + }); + + it("hides inputs but keeps outputs when only hideInputs is true", async () => { + const { spans } = await run(simpleToolJourney(), {}, { hideInputs: true, hideOutputs: false }); + const text = allAttributeText(spans); + expect(text).not.toContain("SECRET_PROMPT_MARKER"); + expect(text).not.toContain(TOOL_INPUT_MARKER); + expect(text).toContain(TOOL_OUTPUT_MARKER); + }); + + it("honours FI_HIDE_INPUTS=false / FI_HIDE_OUTPUTS=false as an opt-in", async () => { + process.env.FI_HIDE_INPUTS = "false"; + process.env.FI_HIDE_OUTPUTS = "false"; + const { spans } = await run(simpleToolJourney()); + const text = allAttributeText(spans); + expect(text).toContain("SECRET_PROMPT_MARKER"); + expect(text).toContain(TOOL_OUTPUT_MARKER); + }); + + it("AC-08: yields the same message objects, in order, and passes params through unchanged", async () => { + const messages = simpleToolJourney(); + const expected = JSON.parse(JSON.stringify(messages)); + const abortController = new AbortController(); + const options = { + resume: undefined, + allowedTools: ["Read"], + permissionMode: "default", + maxTurns: 3, + hooks: {}, + mcpServers: {}, + agents: {}, + abortController, + env: { PATH: "/usr/bin" }, + }; + const optionsBefore = { ...options, env: { ...options.env } }; + const { yielded, fake, params } = await run(messages, options); + + expect(yielded).toHaveLength(messages.length); + yielded.forEach((message, index) => expect(message).toBe(messages[index])); + expect(JSON.parse(JSON.stringify(yielded))).toEqual(expected); + + expect(fake.control.calls).toHaveLength(1); + expect(fake.control.calls[0]).toBe(params); + expect(fake.control.calls[0].options).toBe(options); + expect(options).toEqual(optionsBefore); + expect(options.env).not.toHaveProperty("ANTHROPIC_BASE_URL"); + }); + + it("forwards Query control methods to the original object", async () => { + const { provider } = memoryProvider(); + const fake = makeFakeQuery(simpleToolJourney()); + const traced = wrapQuery(fake.query, { tracerProvider: provider }); + const q = traced({ prompt: PROMPT }); + await q.interrupt(); + expect(fake.control.interruptCalls).toBe(1); + await drain(q); + }); + }); + + describe("J2: subagent (AC-01, AC-04)", () => { + it("nests the subagent under the Agent tool span and the subagent's turns and tools under the subagent", async () => { + const { spans } = await run(subagentJourney()); + const conversation = one(spans, "claude_agent.conversation"); + const agentTool = one(spans, "tool.Agent"); + const subagent = one(spans, "claude_agent.subagent.code-reviewer"); + const grep = one(spans, "tool.Grep"); + const turns = byName(spans, "claude_agent.assistant_turn"); + const mainTurns = turns.filter((t) => t.attributes["claude_agent.parent_tool_use_id"] === undefined); + const subTurns = turns.filter((t) => t.attributes["claude_agent.parent_tool_use_id"] === AGENT_TOOL_ID); + + expect(mainTurns).toHaveLength(2); + expect(subTurns).toHaveLength(2); + expect(new Set(spans.map((s) => s.spanContext().traceId)).size).toBe(1); + + const issuingTurn = mainTurns.find((t) => t.attributes["claude_agent.message.has_tool_use"] === true)!; + expect(parentId(agentTool)).toBe(issuingTurn.spanContext().spanId); + expect(parentId(issuingTurn)).toBe(conversation.spanContext().spanId); + expect(parentId(subagent)).toBe(agentTool.spanContext().spanId); + for (const turn of subTurns) { + expect(parentId(turn)).toBe(subagent.spanContext().spanId); + } + const grepTurn = subTurns.find((t) => t.attributes["claude_agent.message.has_tool_use"] === true)!; + expect(parentId(grep)).toBe(grepTurn.spanContext().spanId); + + expect(agentTool.attributes["claude_agent.span_kind"]).toBe("tool_execution"); + expect(agentTool.attributes["claude_agent.tool.source"]).toBe("builtin"); + expect(subagent.attributes["claude_agent.span_kind"]).toBe("subagent"); + expect(subagent.attributes["gen_ai.span.kind"]).toBe("AGENT"); + expect(subagent.attributes["fi.span.kind"]).toBe("AGENT"); + expect(subagent.attributes["claude_agent.subagent.type"]).toBe("code-reviewer"); + expect(subagent.attributes["claude_agent.tool.use_id"]).toBe(AGENT_TOOL_ID); + expect(subagent.attributes["claude_agent.subagent.task_id"]).toBe("task-1"); + expect(subagent.attributes["claude_agent.subagent.status"]).toBe("completed"); + expect(grep.attributes["claude_agent.parent_tool_use_id"]).toBe(AGENT_TOOL_ID); + expect(grep.attributes["claude_agent.tool.use_id"]).toBe(SUBAGENT_GREP_ID); + // Content off by default: the subagent prompt and description are inputs. + expect(subagent.attributes["claude_agent.subagent.prompt"]).toBeUndefined(); + expect(subagent.attributes["claude_agent.subagent.description"]).toBeUndefined(); + expect(allAttributeText(spans)).not.toContain("SECRET_SUBAGENT_PROMPT_MARKER"); + + // No child outlives its parent. + const endMs = (s: ReadableSpan) => s.endTime[0] * 1e3 + s.endTime[1] / 1e6; + expect(endMs(subagent)).toBeLessThanOrEqual(endMs(agentTool)); + expect(endMs(grep)).toBeLessThanOrEqual(endMs(subagent)); + for (const span of spans) expect(span.status.code).toBe(SpanStatusCode.OK); + }); + + it("keeps a backgrounded subagent open after its tool_result until task_notification", async () => { + const journey = subagentJourney(); + // Move the tool_result for the Agent tool before the subagent's work and mark it backgrounded. + const agentResultIndex = journey.findIndex( + (m) => m.type === "user" && JSON.stringify(m).includes(`"tool_use_id":"${AGENT_TOOL_ID}"`), + ); + const [agentResult] = journey.splice(agentResultIndex, 1); + const startedIndex = journey.findIndex((m) => m.type === "system" && m.subtype === "task_started"); + (journey[startedIndex] as { is_backgrounded?: boolean }).is_backgrounded = true; + journey.splice(startedIndex + 1, 0, agentResult); + + const { spans } = await run(journey); + const agentTool = one(spans, "tool.Agent"); + const subagent = one(spans, "claude_agent.subagent.code-reviewer"); + const endMs = (s: ReadableSpan) => s.endTime[0] * 1e3 + s.endTime[1] / 1e6; + expect(endMs(subagent)).toBeGreaterThanOrEqual(endMs(agentTool)); + for (const turn of byName(spans, "claude_agent.assistant_turn").filter( + (t) => t.attributes["claude_agent.parent_tool_use_id"] === AGENT_TOOL_ID, + )) { + expect(parentId(turn)).toBe(subagent.spanContext().spanId); + } + expect(subagent.status.code).toBe(SpanStatusCode.OK); + }); + }); + + describe("J3: MCP tool error and error result", () => { + it("marks the MCP tool span mcp_tool/TOOL with ERROR and the conversation ERROR", async () => { + const { spans } = await run(mcpErrorJourney()); + const tool = one(spans, "tool.mcp__docs__search"); + const conversation = one(spans, "claude_agent.conversation"); + + expect(tool.attributes["claude_agent.span_kind"]).toBe("mcp_tool"); + expect(tool.attributes["gen_ai.span.kind"]).toBe("TOOL"); + expect(tool.attributes["claude_agent.tool.source"]).toBe("mcp"); + expect(tool.attributes["claude_agent.mcp.server_name"]).toBe("docs"); + expect(tool.attributes["claude_agent.mcp.tool_name"]).toBe("search"); + expect(tool.attributes["claude_agent.tool.is_error"]).toBe(true); + expect(tool.status.code).toBe(SpanStatusCode.ERROR); + // Error text is tool output: hidden by default. + expect(tool.attributes["claude_agent.tool.error_message"]).toBeUndefined(); + expect(tool.status.message).toBe("tool error"); + + expect(conversation.status.code).toBe(SpanStatusCode.ERROR); + expect(conversation.attributes["claude_agent.is_error"]).toBe(true); + expect(conversation.attributes["claude_agent.error.type"]).toBe("error_max_turns"); + expect(conversation.attributes["claude_agent.error.message"]).toBe("Reached maximum number of turns (3)"); + expect(conversation.attributes["gen_ai.cost.total"]).toBe(0.002); + }); + + it("records the tool error text when outputs are visible", async () => { + const { spans } = await run(mcpErrorJourney(), {}, { hideOutputs: false }); + const tool = one(spans, "tool.mcp__docs__search"); + expect(tool.attributes["claude_agent.tool.error_message"]).toBe("server unavailable"); + expect(tool.status.message).toBe("server unavailable"); + }); + + it("treats mcp__ names from options.mcpServers as MCP even without an init message", async () => { + const journey = mcpErrorJourney().slice(1); + const { spans } = await run(journey, { mcpServers: { docs: { type: "stdio", command: "docs" } } }); + expect(one(spans, "tool.mcp__docs__search").attributes["claude_agent.span_kind"]).toBe("mcp_tool"); + }); + + it("classifies an unknown tool as custom", async () => { + const journey = mcpErrorJourney().slice(1); // no init, no options: server unknown + const { spans } = await run(journey); + const tool = one(spans, "tool.mcp__docs__search"); + expect(tool.attributes["claude_agent.tool.source"]).toBe("custom"); + expect(tool.attributes["claude_agent.span_kind"]).toBe("tool_execution"); + }); + }); + + describe("sessions (AC-05)", () => { + it("marks a resumed session and keeps the session id", async () => { + const { spans } = await run(simpleToolJourney(), { resume: SESSION_ID }); + const conversation = one(spans, "claude_agent.conversation"); + expect(conversation.attributes["claude_agent.is_resumed"]).toBe(true); + expect(conversation.attributes["claude_agent.resume_session_id"]).toBe(SESSION_ID); + expect(conversation.attributes["claude_agent.session.is_resumed"]).toBe(true); + expect(conversation.attributes["claude_agent.session.is_new"]).toBe(false); + expect(conversation.attributes["claude_agent.session.previous_id"]).toBe(SESSION_ID); + expect(conversation.attributes["session.id"]).toBe(SESSION_ID); + }); + + it("records the fork origin and uses the new session id from init", async () => { + const origin = "99999999-8888-4777-8666-555555555555"; + const { spans } = await run(simpleToolJourney(), { resume: origin, forkSession: true }); + const conversation = one(spans, "claude_agent.conversation"); + expect(conversation.attributes["claude_agent.session.fork_from"]).toBe(origin); + expect(conversation.attributes["claude_agent.session.is_new"]).toBe(true); + expect(conversation.attributes["session.id"]).toBe(SESSION_ID); + expect(conversation.attributes["claude_agent.session.id"]).toBe(SESSION_ID); + }); + + it("uses Options.sessionId before the init message arrives", async () => { + const custom = "abcdefab-cdef-4abc-8def-abcdefabcdef"; + const journey = simpleToolJourney().slice(1); // no init message + for (const message of journey) (message as { session_id: string }).session_id = custom; + const { spans } = await run(journey, { sessionId: custom }); + for (const span of spans) expect(span.attributes["session.id"]).toBe(custom); + }); + + it("leaves an app-set fi-core session.id alone and adds fi-core context attributes", async () => { + const { provider, exporter } = memoryProvider(); + const fake = makeFakeQuery(simpleToolJourney()); + const traced = wrapQuery(fake.query, { tracerProvider: provider }); + context.setGlobalContextManager(new StackContextManager()); + try { + const ctx = setUser(setSession(context.active(), { sessionId: "app-session" }), { userId: "user-1" }); + const stream = context.with(ctx, () => traced({ prompt: PROMPT })); + await drain(stream); + } finally { + context.disable(); + } + const conversation = one(exporter.getFinishedSpans(), "claude_agent.conversation"); + expect(conversation.attributes["session.id"]).toBe("app-session"); + expect(conversation.attributes["user.id"]).toBe("user-1"); + expect(conversation.attributes["claude_agent.session.id"]).toBe(SESSION_ID); + }); + }); + + describe("failure semantics", () => { + it("AC-06: abort ends every open span with ERROR and claude_agent.cancelled=true", async () => { + const { provider, exporter } = memoryProvider(); + const journey = simpleToolJourney(); + // Block right after the tool_use: conversation, turn and tool spans are open. + const fake = makeFakeQuery(journey, { waitForAbortAt: 3 }); + const traced = wrapQuery(fake.query, { tracerProvider: provider }); + const abortController = new AbortController(); + const stream = traced({ prompt: PROMPT, options: { abortController } }); + + const seen: unknown[] = []; + const consumed = (async () => { + for await (const message of stream) { + seen.push(message); + if (seen.length === 3) setTimeout(() => abortController.abort(), 5); + } + })(); + await expect(consumed).rejects.toThrow("aborted"); + + const spans = exporter.getFinishedSpans(); + expect(spans.map((s) => s.name).sort()).toEqual( + ["claude_agent.assistant_turn", "claude_agent.conversation", "tool.Read"].sort(), + ); + for (const span of spans) { + expect(span.ended).toBe(true); + expect(span.status.code).toBe(SpanStatusCode.ERROR); + expect(span.attributes["claude_agent.cancelled"]).toBe(true); + } + }); + + it("ends spans and rethrows the same error when the SDK iterator throws", async () => { + const { provider, exporter } = memoryProvider(); + const boom = new Error("CLI exited with code 1"); + const fake = makeFakeQuery(simpleToolJourney(), { throwAt: 3, error: boom }); + const traced = wrapQuery(fake.query, { tracerProvider: provider }); + await expect(drain(traced({ prompt: PROMPT }))).rejects.toBe(boom); + + const spans = exporter.getFinishedSpans(); + const conversation = one(spans, "claude_agent.conversation"); + expect(conversation.status.code).toBe(SpanStatusCode.ERROR); + expect(conversation.attributes["claude_agent.error.message"]).toBe("CLI exited with code 1"); + expect(conversation.events.some((e) => e.name === "exception")).toBe(true); + expect(conversation.attributes["claude_agent.cancelled"]).toBeUndefined(); + expect(spans).toHaveLength(3); + for (const span of spans) expect(span.ended).toBe(true); + }); + + it("ends open spans when the consumer breaks out early", async () => { + const { provider, exporter } = memoryProvider(); + const fake = makeFakeQuery(simpleToolJourney()); + const traced = wrapQuery(fake.query, { tracerProvider: provider }); + for await (const message of traced({ prompt: PROMPT })) { + if (message.type === "assistant") break; + } + const spans = exporter.getFinishedSpans(); + expect(spans.map((s) => s.name).sort()).toEqual(["claude_agent.assistant_turn", "claude_agent.conversation"]); + }); + + it("does not reject the iterator when the exporter fails (collector down)", async () => { + const failing: SpanExporter = { + // code 1 is ExportResultCode.FAILED (@opentelemetry/core) + export: (_spans, done) => done({ code: 1, error: new Error("ECONNREFUSED") }), + shutdown: async () => undefined, + }; + const provider = new BasicTracerProvider({ spanProcessors: [new BatchSpanProcessor(failing)] }); + const fake = makeFakeQuery(simpleToolJourney()); + const traced = wrapQuery(fake.query, { tracerProvider: provider }); + const messages = await drain(traced({ prompt: PROMPT })); + expect(messages).toHaveLength(simpleToolJourney().length); + await expect(shutdown(provider)).resolves.toBeUndefined(); + }); + + it("AC-10: shutdown() flushes a batch processor before a short script exits", async () => { + const exporter = new InMemorySpanExporter(); + const provider = new BasicTracerProvider({ + spanProcessors: [new BatchSpanProcessor(exporter, { scheduledDelayMillis: 60_000 })], + }); + const instrumentation = new ClaudeAgentSDKInstrumentation({ tracerProvider: provider }); + const traced = instrumentation.wrapQuery(makeFakeQuery(simpleToolJourney()).query); + await drain(traced({ prompt: PROMPT })); + expect(exporter.getFinishedSpans()).toHaveLength(0); + await shutdown(); + expect(exporter.getFinishedSpans()).toHaveLength(4); + }); + }); + + describe("provider name", () => { + it.each([ + [undefined, "anthropic"], + ["https://api.anthropic.com", "anthropic"], + ["http://127.0.0.1:8080", "custom"], + ["https://gateway.example.com/anthropic", "custom"], + ])("ANTHROPIC_BASE_URL=%s in options.env -> %s", async (baseUrl, expected) => { + const env = baseUrl ? { PATH: "/usr/bin", ANTHROPIC_BASE_URL: baseUrl } : { PATH: "/usr/bin" }; + const { spans } = await run(simpleToolJourney(), { env }); + expect(one(spans, "claude_agent.conversation").attributes["gen_ai.provider.name"]).toBe(expected); + }); + + it("reads process.env when options.env is absent", async () => { + process.env.ANTHROPIC_BASE_URL = "http://localhost:4000"; + const { spans } = await run(simpleToolJourney()); + expect(one(spans, "claude_agent.conversation").attributes["gen_ai.provider.name"]).toBe("custom"); + }); + }); + + it("never copies FI_API_KEY / FI_SECRET_KEY onto spans", async () => { + process.env.FI_API_KEY = "fi-api-key-PLACEHOLDER-123"; + process.env.FI_SECRET_KEY = "fi-secret-key-PLACEHOLDER-456"; + try { + const { spans } = await run(subagentJourney(), {}, { hideInputs: false, hideOutputs: false }); + const text = allAttributeText(spans); + expect(text).not.toContain("fi-api-key-PLACEHOLDER-123"); + expect(text).not.toContain("fi-secret-key-PLACEHOLDER-456"); + } finally { + delete process.env.FI_API_KEY; + delete process.env.FI_SECRET_KEY; + } + }); + + it("does not double-wrap and uses the global provider when none is given", async () => { + const { provider, exporter } = memoryProvider(); + trace.setGlobalTracerProvider(provider); + try { + const fake = makeFakeQuery(simpleToolJourney()); + const once = wrapQuery(fake.query); + const twice = wrapQuery(once); + expect(isWrappedQuery(once)).toBe(true); + expect(twice).toBe(once); + await drain(twice({ prompt: PROMPT })); + expect(exporter.getFinishedSpans()).toHaveLength(4); + } finally { + trace.disable(); + } + }); + + it("does not start spans for a query that is never iterated", async () => { + const { provider, exporter } = memoryProvider(); + const traced = wrapQuery(makeFakeQuery(simpleToolJourney()).query, { tracerProvider: provider }); + traced({ prompt: PROMPT }); + expect(exporter.getFinishedSpans()).toHaveLength(0); + }); + + it("does not trace a streaming-input prompt's content", async () => { + const { provider, exporter } = memoryProvider(); + const traced = wrapQuery(makeFakeQuery(simpleToolJourney()).query, { + tracerProvider: provider, + traceConfig: { hideInputs: false }, + }); + async function* input() { + yield { type: "user" as const, message: { role: "user" as const, content: "hi" }, parent_tool_use_id: null }; + } + await drain(traced({ prompt: input() })); + const conversation = one(exporter.getFinishedSpans(), "claude_agent.conversation"); + expect(conversation.attributes["input.value"]).toBeUndefined(); + }); + + it("closes a tool span that never got a result as not completed", async () => { + const journey = simpleToolJourney().filter((m) => m.type !== "user"); + const { spans } = await run(journey); + const tool = one(spans, "tool.Read"); + expect(tool.status.code).toBe(SpanStatusCode.ERROR); + expect(tool.status.message).toBe("Tool span not completed (conversation ended)"); + expect(tool.attributes["claude_agent.cancelled"]).toBeUndefined(); + }); + + it("keeps MCP_TOOL_ID fixture id stable", () => { + expect(MCP_TOOL_ID).toBe("toolu_mcp_1"); + }); +}); diff --git a/typescript/packages/traceai_claude_agent_sdk/src/attributes.ts b/typescript/packages/traceai_claude_agent_sdk/src/attributes.ts new file mode 100644 index 000000000..c56ad24b6 --- /dev/null +++ b/typescript/packages/traceai_claude_agent_sdk/src/attributes.ts @@ -0,0 +1,225 @@ +/** + * Attribute names for Claude Agent SDK spans. + * + * The `claude_agent.*` and `gen_ai.usage.*` names are copied from the Python + * package (`python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/_attributes.py`) + * so that both languages emit the same keys. `parity.test.ts` reads that file + * and fails when a name is missing here. + */ +import { FISpanKind } from "@traceai/fi-semantic-conventions"; + +/** Span kinds, same strings as the Python `ClaudeAgentSpanKind` enum. */ +export const ClaudeAgentSpanKind = { + CONVERSATION: "conversation", + ASSISTANT_TURN: "assistant_turn", + TOOL_EXECUTION: "tool_execution", + SUBAGENT: "subagent", + MCP_TOOL: "mcp_tool", +} as const; + +export type ClaudeAgentSpanKindValue = + (typeof ClaudeAgentSpanKind)[keyof typeof ClaudeAgentSpanKind]; + +/** Same names and values as the Python `ClaudeAgentAttributes` class. */ +export const ClaudeAgentAttributes = { + // Span kind + SPAN_KIND: "claude_agent.span_kind", + + // Agent / conversation + GEN_AI_AGENT_NAME: "claude_agent.name", + AGENT_SESSION_ID: "claude_agent.session_id", + AGENT_PROMPT: "claude_agent.prompt", + AGENT_SYSTEM_PROMPT: "claude_agent.system_prompt", + AGENT_MODEL: "claude_agent.model", + AGENT_PERMISSION_MODE: "claude_agent.permission_mode", + AGENT_ALLOWED_TOOLS: "claude_agent.allowed_tools", + AGENT_NUM_TURNS: "claude_agent.num_turns", + AGENT_IS_RESUMED: "claude_agent.is_resumed", + AGENT_RESUME_SESSION_ID: "claude_agent.resume_session_id", + + // Tool + GEN_AI_TOOL_NAME: "claude_agent.tool.name", + TOOL_USE_ID: "claude_agent.tool.use_id", + TOOL_INPUT: "claude_agent.tool.input", + TOOL_OUTPUT: "claude_agent.tool.output", + TOOL_IS_ERROR: "claude_agent.tool.is_error", + TOOL_ERROR_MESSAGE: "claude_agent.tool.error_message", + TOOL_DURATION_MS: "claude_agent.tool.duration_ms", + TOOL_SOURCE: "claude_agent.tool.source", + + // Built-in tool specific + TOOL_FILE_PATH: "claude_agent.tool.file_path", + TOOL_COMMAND: "claude_agent.tool.command", + TOOL_EXIT_CODE: "claude_agent.tool.exit_code", + TOOL_PATTERN: "claude_agent.tool.pattern", + TOOL_MATCHES_COUNT: "claude_agent.tool.matches_count", + TOOL_URL: "claude_agent.tool.url", + TOOL_SEARCH_QUERY: "claude_agent.tool.search_query", + + // Subagent + SUBAGENT_NAME: "claude_agent.subagent.name", + SUBAGENT_TYPE: "claude_agent.subagent.type", + SUBAGENT_DESCRIPTION: "claude_agent.subagent.description", + SUBAGENT_PROMPT: "claude_agent.subagent.prompt", + SUBAGENT_TOOLS: "claude_agent.subagent.tools", + PARENT_TOOL_USE_ID: "claude_agent.parent_tool_use_id", + + // Message + MESSAGE_TYPE: "claude_agent.message.type", + MESSAGE_ROLE: "claude_agent.message.role", + MESSAGE_CONTENT: "claude_agent.message.content", + MESSAGE_HAS_TOOL_USE: "claude_agent.message.has_tool_use", + MESSAGE_TOOL_USE_COUNT: "claude_agent.message.tool_use_count", + + // MCP server + MCP_SERVER_NAME: "claude_agent.mcp.server_name", + MCP_SERVER_COMMAND: "claude_agent.mcp.server_command", + MCP_SERVER_ARGS: "claude_agent.mcp.server_args", + MCP_TOOL_COUNT: "claude_agent.mcp.tool_count", + + // Session + GEN_AI_CONVERSATION_ID: "claude_agent.session.id", + SESSION_IS_NEW: "claude_agent.session.is_new", + SESSION_IS_RESUMED: "claude_agent.session.is_resumed", + SESSION_PREVIOUS_ID: "claude_agent.session.previous_id", + SESSION_FORK_FROM: "claude_agent.session.fork_from", + + // Usage (Python names; the cache keys are not the dotted semconv spelling) + USAGE_INPUT_TOKENS: "gen_ai.usage.input_tokens", + USAGE_OUTPUT_TOKENS: "gen_ai.usage.output_tokens", + USAGE_TOTAL_TOKENS: "gen_ai.usage.total_tokens", + USAGE_CACHE_READ_TOKENS: "gen_ai.usage.cache_read_tokens", + USAGE_CACHE_CREATION_TOKENS: "gen_ai.usage.cache_creation_tokens", + + // Cost + COST_TOTAL_USD: "claude_agent.cost.total_usd", + COST_INPUT_USD: "claude_agent.cost.input_usd", + COST_OUTPUT_USD: "claude_agent.cost.output_usd", + + // Performance + DURATION_MS: "claude_agent.duration_ms", + DURATION_API_MS: "claude_agent.duration_api_ms", + TIME_TO_FIRST_TOKEN_MS: "claude_agent.time_to_first_token_ms", + + // Error + ERROR_TYPE: "claude_agent.error.type", + ERROR_MESSAGE: "claude_agent.error.message", + IS_ERROR: "claude_agent.is_error", + + // Hook + HOOK_TYPE: "claude_agent.hook.type", + HOOK_MATCHER: "claude_agent.hook.matcher", + HOOK_BLOCKED: "claude_agent.hook.blocked", + HOOK_MODIFIED: "claude_agent.hook.modified", +} as const; + +/** + * Attributes this package adds on top of the Python list. They exist so the + * Future AGI collector and trace UI can read the span (span kind, cost, + * provider, generic tool name) or to record wrapper-only state (cancellation). + */ +export const TraceAIAttributes = { + /** Span kind key the Python fi_instrumentation package and the collector read. */ + GEN_AI_SPAN_KIND: "gen_ai.span.kind", + /** Span kind key in TS fi-semantic-conventions (`SemanticConventions.FI_SPAN_KIND`). */ + FI_SPAN_KIND: "fi.span.kind", + /** Cost key the collector promotes (`gen_ai.cost.total` / `llm.cost.total`). */ + GEN_AI_COST_TOTAL: "gen_ai.cost.total", + GEN_AI_PROVIDER_NAME: "gen_ai.provider.name", + GEN_AI_REQUEST_MODEL: "gen_ai.request.model", + GEN_AI_TOOL_NAME: "gen_ai.tool.name", + SESSION_ID: "session.id", + INPUT_VALUE: "input.value", + INPUT_MIME_TYPE: "input.mime_type", + OUTPUT_VALUE: "output.value", + OUTPUT_MIME_TYPE: "output.mime_type", + /** Set on every span the wrapper closes because the query was aborted. */ + CANCELLED: "claude_agent.cancelled", + /** MCP tool name without the `mcp____` prefix (Python `_mcp_tracker.py`). */ + MCP_TOOL_NAME: "claude_agent.mcp.tool_name", + /** Background task id from `system/task_started`. */ + SUBAGENT_TASK_ID: "claude_agent.subagent.task_id", + /** Final status from `system/task_notification`. */ + SUBAGENT_STATUS: "claude_agent.subagent.status", +} as const; + +/** + * Future AGI span kind for each Claude Agent span kind. + * + * TS `FISpanKind` has no `CONVERSATION` member (Python `FiSpanKindValues` does), + * so the conversation span falls back to `CHAIN` as the architecture requires. + * `claude_agent.span_kind` still carries `conversation`. + */ +export const FI_SPAN_KIND_BY_CLAUDE_KIND: Record = { + [ClaudeAgentSpanKind.CONVERSATION]: + (FISpanKind as Record)["CONVERSATION"] ?? FISpanKind.CHAIN, + [ClaudeAgentSpanKind.ASSISTANT_TURN]: FISpanKind.LLM, + [ClaudeAgentSpanKind.TOOL_EXECUTION]: FISpanKind.TOOL, + [ClaudeAgentSpanKind.MCP_TOOL]: FISpanKind.TOOL, + [ClaudeAgentSpanKind.SUBAGENT]: FISpanKind.AGENT, +}; + +/** Span names (Python `_client_wrapper.py`, `_hooks.py`, `_subagent_tracker.py`). */ +export const SpanNames = { + CONVERSATION: "claude_agent.conversation", + ASSISTANT_TURN: "claude_agent.assistant_turn", + toolSpan: (toolName: string) => `tool.${toolName}`, + subagentSpan: (subagentType: string) => `claude_agent.subagent.${subagentType}`, +} as const; + +/** + * Built-in tool names. Python `BUILTIN_TOOLS` plus `Agent`, the name the + * 0.3.x SDK uses for the subagent tool (sdk.d.ts: "invoked via the Agent tool"). + */ +export const BUILTIN_TOOLS: ReadonlySet = new Set([ + "Read", + "Write", + "Edit", + "Bash", + "Glob", + "Grep", + "WebSearch", + "WebFetch", + "AskUserQuestion", + "Task", + "NotebookEdit", + "TodoRead", + "TodoWrite", + "Agent", +]); + +/** Tool names that start a subagent. Python checks `Task`; 0.3.x names it `Agent`. */ +export const SUBAGENT_TOOLS: ReadonlySet = new Set(["Task", "Agent"]); + +export type ToolSource = "builtin" | "mcp" | "custom"; + +/** + * Parse `mcp____` against the known MCP server names. + * Returns undefined when the tool is not from a known server. + */ +export function parseMcpToolName( + toolName: string, + mcpServerNames: Iterable, +): { server: string; tool: string } | undefined { + for (const server of mcpServerNames) { + const prefix = `mcp__${server}__`; + if (toolName.startsWith(prefix) && toolName.length > prefix.length) { + return { server, tool: toolName.slice(prefix.length) }; + } + } + return undefined; +} + +/** Same rule as Python `get_tool_source`. */ +export function getToolSource( + toolName: string, + mcpServerNames: Iterable = [], +): ToolSource { + if (BUILTIN_TOOLS.has(toolName)) { + return "builtin"; + } + if (parseMcpToolName(toolName, mcpServerNames)) { + return "mcp"; + } + return "custom"; +} diff --git a/typescript/packages/traceai_claude_agent_sdk/src/index.ts b/typescript/packages/traceai_claude_agent_sdk/src/index.ts new file mode 100644 index 000000000..e0d0e77aa --- /dev/null +++ b/typescript/packages/traceai_claude_agent_sdk/src/index.ts @@ -0,0 +1,23 @@ +export { + ClaudeAgentSDKInstrumentation, + INSTRUMENTATION_NAME, + instrumentClaudeAgentSDK, + resolveContentPolicy, + shutdown, + wrapQuery, +} from "./instrumentation"; +export type { ClaudeAgentSDKInstrumentationConfig } from "./instrumentation"; +export { isWrappedQuery } from "./queryWrapper"; +export type { ContentPolicy } from "./spans"; +export { + BUILTIN_TOOLS, + ClaudeAgentAttributes, + ClaudeAgentSpanKind, + FI_SPAN_KIND_BY_CLAUDE_KIND, + SUBAGENT_TOOLS, + SpanNames, + TraceAIAttributes, + getToolSource, +} from "./attributes"; +export type { ClaudeAgentSpanKindValue, ToolSource } from "./attributes"; +export { VERSION } from "./version"; diff --git a/typescript/packages/traceai_claude_agent_sdk/src/instrumentation.ts b/typescript/packages/traceai_claude_agent_sdk/src/instrumentation.ts new file mode 100644 index 000000000..af658f42f --- /dev/null +++ b/typescript/packages/traceai_claude_agent_sdk/src/instrumentation.ts @@ -0,0 +1,162 @@ +/** + * Public entry points: an instrumentation object that wraps `query()`, and a + * `shutdown()` that flushes the tracer provider before a short script exits. + */ +import { TracerProvider, diag, trace } from "@opentelemetry/api"; +import { FITracer, TraceConfigOptions } from "@traceai/fi-core"; +import { ContentPolicy } from "./spans"; +import { QueryTracingContext, wrapQueryFunction } from "./queryWrapper"; +import { QueryFunctionLike } from "./types"; +import { VERSION } from "./version"; + +export const INSTRUMENTATION_NAME = "@traceai/claude-agent-sdk"; + +/** fi-core env keys, read with fi-core's rule (`value.toLowerCase() === "true"`). */ +const FI_HIDE_INPUTS = "FI_HIDE_INPUTS"; +const FI_HIDE_OUTPUTS = "FI_HIDE_OUTPUTS"; + +export interface ClaudeAgentSDKInstrumentationConfig { + /** Provider to create spans with. Defaults to the global provider at call time. */ + tracerProvider?: TracerProvider; + /** + * fi-core trace config. Unlike fi-core, `hideInputs` and `hideOutputs` + * default to `true` here: prompts, tool inputs and tool outputs can contain + * source code, file contents or secrets. Opt in with `false`, or with + * `FI_HIDE_INPUTS=false` / `FI_HIDE_OUTPUTS=false`. + */ + traceConfig?: TraceConfigOptions; +} + +function envFlag(key: string): boolean | undefined { + const value = typeof process !== "undefined" ? process.env?.[key] : undefined; + return value === undefined ? undefined : value.toLowerCase() === "true"; +} + +/** + * Resolve content capture. Precedence: explicit option > fi-core env var > + * this package's default (hidden). + */ +export function resolveContentPolicy(traceConfig?: TraceConfigOptions): ContentPolicy { + return { + hideInputs: traceConfig?.hideInputs ?? envFlag(FI_HIDE_INPUTS) ?? true, + hideOutputs: traceConfig?.hideOutputs ?? envFlag(FI_HIDE_OUTPUTS) ?? true, + }; +} + +const knownProviders = new Set(); + +/** + * Traces Claude Agent SDK `query()` calls. + * + * ```ts + * import { query } from "@anthropic-ai/claude-agent-sdk"; + * const instrumentation = new ClaudeAgentSDKInstrumentation({ tracerProvider }); + * const tracedQuery = instrumentation.wrapQuery(query); + * ``` + * + * The SDK is ESM-only and its module namespace cannot be patched, so the + * application calls the wrapped function instead of the original. + */ +export class ClaudeAgentSDKInstrumentation { + private tracerProvider?: TracerProvider; + private readonly traceConfig?: TraceConfigOptions; + + constructor(config: ClaudeAgentSDKInstrumentationConfig = {}) { + this.traceConfig = config.traceConfig; + if (config.tracerProvider) { + this.setTracerProvider(config.tracerProvider); + } + } + + setTracerProvider(tracerProvider: TracerProvider): void { + this.tracerProvider = tracerProvider; + knownProviders.add(tracerProvider); + } + + /** The content policy this instrumentation applies. */ + get contentPolicy(): ContentPolicy { + return resolveContentPolicy(this.traceConfig); + } + + /** Return a traced `query()` with the same signature as the original. */ + wrapQuery(query: Q): Q { + return wrapQueryFunction(query, () => this.tracingContext()); + } + + /** Flush the provider's span processors. Does not shut the provider down. */ + async forceFlush(): Promise { + await flushProvider(this.tracerProvider ?? trace.getTracerProvider()); + } + + /** Same as `forceFlush()`; named for short scripts (architecture R-10). */ + async shutdown(): Promise { + await this.forceFlush(); + } + + private tracingContext(): QueryTracingContext { + const policy = this.contentPolicy; + const provider = this.tracerProvider ?? trace.getTracerProvider(); + const tracer = new FITracer({ + tracer: provider.getTracer(INSTRUMENTATION_NAME, VERSION), + traceConfig: { ...this.traceConfig, hideInputs: policy.hideInputs, hideOutputs: policy.hideOutputs }, + }); + return { tracer, policy }; + } +} + +/** Python-style convenience constructor (`instrument_claude_agent_sdk`). */ +export function instrumentClaudeAgentSDK( + config: ClaudeAgentSDKInstrumentationConfig = {}, +): ClaudeAgentSDKInstrumentation { + return new ClaudeAgentSDKInstrumentation(config); +} + +/** Wrap `query()` in one call. */ +export function wrapQuery( + query: Q, + config: ClaudeAgentSDKInstrumentationConfig = {}, +): Q { + return new ClaudeAgentSDKInstrumentation(config).wrapQuery(query); +} + +/** + * Flush spans before a short script exits. Flushes `tracerProvider` when + * given; otherwise every provider passed to an instrumentation plus the global + * provider. Export errors are logged, never thrown: export failure does not + * fail the agent. + */ +export async function shutdown(tracerProvider?: TracerProvider): Promise { + const providers = tracerProvider + ? [tracerProvider] + : [...knownProviders, trace.getTracerProvider()]; + const seen = new Set(); + for (const provider of providers) { + const target = resolveDelegate(provider); + if (seen.has(target)) continue; + seen.add(target); + await flushProvider(target); + } +} + +function resolveDelegate(provider: TracerProvider): TracerProvider { + const maybeProxy = provider as TracerProvider & { + getDelegate?: () => TracerProvider; + forceFlush?: () => Promise; + }; + if (typeof maybeProxy.forceFlush !== "function" && typeof maybeProxy.getDelegate === "function") { + return maybeProxy.getDelegate(); + } + return provider; +} + +async function flushProvider(provider: TracerProvider): Promise { + const target = resolveDelegate(provider) as TracerProvider & { forceFlush?: () => Promise }; + if (typeof target.forceFlush !== "function") { + return; + } + try { + await target.forceFlush(); + } catch (error) { + diag.warn(`${INSTRUMENTATION_NAME}: forceFlush failed: ${error}`); + } +} diff --git a/typescript/packages/traceai_claude_agent_sdk/src/queryWrapper.ts b/typescript/packages/traceai_claude_agent_sdk/src/queryWrapper.ts new file mode 100644 index 000000000..564b01799 --- /dev/null +++ b/typescript/packages/traceai_claude_agent_sdk/src/queryWrapper.ts @@ -0,0 +1,131 @@ +/** + * Wrap the SDK's exported `query()` function. + * + * The wrapper calls the original `query()` with the same params object (same + * reference, never copied or modified) and returns a proxy around the `Query` + * it got back. The proxy yields the same message objects the SDK yielded and + * forwards every control method (`interrupt()`, `setModel()`, ...) to the + * original. Only `next`, `return`, `throw` and `Symbol.asyncIterator` are + * intercepted, to drive the span model in `spans.ts`. + */ +import { Tracer, context as otelContext, diag } from "@opentelemetry/api"; +import { ContentPolicy, QueryTracer, clockMs } from "./spans"; +import { QueryFunctionLike, QueryParamsLike } from "./types"; + +const WRAPPED = Symbol.for("@traceai/claude-agent-sdk/wrapped"); + +export interface QueryTracingContext { + tracer: Tracer; + policy: ContentPolicy; +} + +export function isWrappedQuery(fn: unknown): boolean { + return typeof fn === "function" && (fn as unknown as Record)[WRAPPED] === true; +} + +/** + * Return a function with the same signature as `query` that traces every call. + * `getContext` is read per call so a later tracer/config change applies. + */ +export function wrapQueryFunction( + query: Q, + getContext: () => QueryTracingContext, +): Q { + if (typeof query !== "function") { + throw new TypeError("@traceai/claude-agent-sdk: wrapQuery() expects the SDK query function"); + } + if (isWrappedQuery(query)) { + return query; + } + + const wrapped = function tracedQuery(this: unknown, params: QueryParamsLike) { + const startTimeMs = clockMs(); + const parentContext = otelContext.active(); + // The SDK call itself is never wrapped in try/catch: a synchronous throw + // from query() reaches the caller unchanged. + // eslint-disable-next-line prefer-rest-params + const original = query.apply(this, arguments as unknown as [QueryParamsLike]); + + let queryTracer: QueryTracer | undefined; + try { + const { tracer, policy } = getContext(); + queryTracer = new QueryTracer({ tracer, policy, params, parentContext, startTimeMs }); + } catch (error) { + diag.debug(`@traceai/claude-agent-sdk: tracing disabled for this query: ${error}`); + } + if (!queryTracer || !original || typeof original.next !== "function") { + return original; + } + return proxyQuery(original, queryTracer); + }; + + Object.defineProperty(wrapped, WRAPPED, { value: true }); + Object.defineProperty(wrapped, "name", { value: query.name || "query" }); + return wrapped as unknown as Q; +} + +function proxyQuery>( + original: G, + queryTracer: QueryTracer, +): G { + const safe = (fn: () => void) => { + try { + fn(); + } catch (error) { + diag.debug(`@traceai/claude-agent-sdk: ${error}`); + } + }; + + const next = async (...args: [] | [unknown]) => { + safe(() => queryTracer.start()); + let result: IteratorResult; + try { + result = await original.next(...args); + } catch (error) { + safe(() => queryTracer.finish({ kind: "error", error })); + throw error; + } + if (result.done) { + safe(() => queryTracer.finish({ kind: "completed" })); + } else { + safe(() => queryTracer.onMessage(result.value)); + } + return result; + }; + + const ret = async (value?: unknown) => { + try { + return await original.return(value); + } finally { + safe(() => queryTracer.finish({ kind: "returned" })); + } + }; + + const thr = async (error?: unknown) => { + let result: IteratorResult; + try { + result = await original.throw(error); + } catch (thrown) { + safe(() => queryTracer.finish({ kind: "error", error: thrown })); + throw thrown; + } + if (result.done) { + safe(() => queryTracer.finish({ kind: "returned" })); + } else { + safe(() => queryTracer.onMessage(result.value)); + } + return result; + }; + + const proxy: G = new Proxy(original, { + get(target, property) { + if (property === "next") return next; + if (property === "return") return ret; + if (property === "throw") return thr; + if (property === Symbol.asyncIterator) return () => proxy; + const value = Reflect.get(target, property, target); + return typeof value === "function" ? value.bind(target) : value; + }, + }); + return proxy; +} diff --git a/typescript/packages/traceai_claude_agent_sdk/src/spans.ts b/typescript/packages/traceai_claude_agent_sdk/src/spans.ts new file mode 100644 index 000000000..def34d8c2 --- /dev/null +++ b/typescript/packages/traceai_claude_agent_sdk/src/spans.ts @@ -0,0 +1,960 @@ +/** + * Span model for one `query()` call. + * + * The Claude Agent SDK emits a message stream, not spans. `QueryTracer` turns + * that stream into: + * + * claude_agent.conversation (conversation, CHAIN) + * └─ claude_agent.assistant_turn (assistant_turn, LLM) + * └─ tool. (tool_execution | mcp_tool, TOOL) + * └─ claude_agent.subagent. (subagent, AGENT) Task/Agent tool only + * └─ claude_agent.assistant_turn ... (the subagent's own turns) + * + * Every public method swallows its own errors: tracing never fails the agent. + */ +import { + Attributes, + AttributeValue, + Context, + HrTime, + Span, + SpanKind, + SpanStatusCode, + Tracer, + diag, + trace, +} from "@opentelemetry/api"; +import { getAttributesFromContext } from "@traceai/fi-core"; +import { + ClaudeAgentAttributes as A, + ClaudeAgentSpanKind, + ClaudeAgentSpanKindValue, + FI_SPAN_KIND_BY_CLAUDE_KIND, + SUBAGENT_TOOLS, + SpanNames, + TraceAIAttributes as T, + getToolSource, + parseMcpToolName, +} from "./attributes"; +import { + AbortSignalLike, + AssistantMessageLike, + ContentBlockLike, + OptionsLike, + QueryParamsLike, + ResultMessageLike, + SystemMessageLike, + UserMessageLike, +} from "./types"; + +/** Which content the wrapper may copy onto spans. Both default to hidden. */ +export interface ContentPolicy { + hideInputs: boolean; + hideOutputs: boolean; +} + +/** How the wrapped iterator stopped. */ +export type FinishOutcome = + | { kind: "completed" } + | { kind: "returned" } + | { kind: "error"; error: unknown } + | { kind: "aborted"; reason?: unknown }; + +const PROMPT_MAX = 2000; +const SYSTEM_PROMPT_MAX = 1000; +const CONTENT_MAX = 5000; +const MESSAGE_CONTENT_MAX = 2000; +const ERROR_MESSAGE_MAX = 500; +const TOOL_COMMAND_MAX = 500; + +type Scope = string | null; + +/** + * One clock for every start and end time the wrapper sets. OTel falls back to + * millisecond `Date.now()` for the end of a span whose start time was given, + * so mixing explicit and implicit times can order a child after its parent. + */ +export function clockMs(): number { + const perf = (globalThis as { performance?: { now?: () => number; timeOrigin?: number } }).performance; + if (perf && typeof perf.now === "function" && typeof perf.timeOrigin === "number") { + return perf.timeOrigin + perf.now(); + } + return Date.now(); +} + +export function toHrTime(ms: number): HrTime { + let seconds = Math.trunc(ms / 1000); + let nanos = Math.round((ms - seconds * 1000) * 1e6); + if (nanos >= 1e9) { + seconds += 1; + nanos -= 1e9; + } + return [seconds, nanos]; +} + +interface TurnState { + span: Span; + scope: Scope; + messageId?: string; + textParts: string[]; + toolUseCount: number; + openToolIds: Set; + isError: boolean; + ended: boolean; +} + +interface ToolState { + span: Span; + name: string; + startMs: number; + turn?: TurnState; +} + +interface SubagentState { + span: Span; + startMs: number; + backgrounded: boolean; + toolEnded: boolean; + status?: string; + ended: boolean; +} + +interface ScopeState { + currentTurn?: TurnState; + turnCount: number; + nextTurnStartMs?: number; +} + +/** Truncate like the Python package: keep `max` chars, ending in "...". */ +export function truncate(value: string, max: number): string { + return value.length > max ? `${value.slice(0, max - 3)}...` : value; +} + +/** Python `safe_json_serialize`: strings as-is, objects as JSON, truncated. */ +export function safeJson(value: unknown, max = CONTENT_MAX): string { + if (value === undefined || value === null) { + return ""; + } + let text: string; + try { + text = typeof value === "string" ? value : JSON.stringify(value) ?? String(value); + } catch { + text = String(value); + } + return truncate(text, max); +} + +function contentBlocks(content: unknown): ContentBlockLike[] { + return Array.isArray(content) ? (content as ContentBlockLike[]) : []; +} + +/** Flatten a tool_result `content` (string or block list) to text. */ +function flattenToolResult(content: unknown): string { + if (content === undefined || content === null) { + return ""; + } + if (typeof content === "string") { + return content; + } + if (Array.isArray(content)) { + return content + .map((block: ContentBlockLike) => { + if (block && typeof block.text === "string") return block.text; + if (block && block.type === "image") return "[image]"; + return safeJson(block); + }) + .join("\n"); + } + return safeJson(content); +} + +/** Decide `gen_ai.provider.name` from the environment the CLI subprocess sees. */ +export function resolveProviderName(options: OptionsLike | undefined): string { + // Options.env REPLACES the subprocess environment; when it is absent the + // subprocess inherits process.env. + const env = + options?.env ?? + (typeof process !== "undefined" ? (process.env as Record) : {}); + const baseUrl = env?.ANTHROPIC_BASE_URL; + if (!baseUrl) { + return "anthropic"; + } + try { + const host = new URL(baseUrl).hostname.toLowerCase(); + if (host === "anthropic.com" || host.endsWith(".anthropic.com")) { + return "anthropic"; + } + } catch { + // An unparsable base URL is not the Anthropic host. + } + return "custom"; +} + +function isAbortLike(error: unknown, signal?: AbortSignalLike): boolean { + if (signal?.aborted) { + return true; + } + return ( + typeof error === "object" && + error !== null && + (error as { name?: unknown }).name === "AbortError" + ); +} + +function errorParts(error: unknown): { type: string; message: string } { + if (error instanceof Error) { + return { type: error.name || error.constructor?.name || "Error", message: error.message }; + } + return { type: typeof error, message: String(error) }; +} + +export class QueryTracer { + private readonly tracer: Tracer; + private readonly policy: ContentPolicy; + private readonly params: QueryParamsLike; + private readonly options: OptionsLike | undefined; + private readonly parentContext: Context; + private readonly startTimeMs: number; + private readonly contextAttributes: Attributes; + private readonly appSetSession: boolean; + private readonly providerName: string; + private readonly mcpServerNames = new Set(); + + private conversation?: Span; + private conversationContext?: Context; + private conversationIsError = false; + private resultSeen = false; + private sessionId?: string; + private finished = false; + private abortListener?: () => void; + + private readonly scopes = new Map(); + private readonly tools = new Map(); + private readonly subagents = new Map(); + + constructor(args: { + tracer: Tracer; + policy: ContentPolicy; + params: QueryParamsLike; + parentContext: Context; + startTimeMs?: number; + }) { + this.tracer = args.tracer; + this.policy = args.policy; + this.params = args.params ?? ({} as QueryParamsLike); + this.options = this.params.options; + this.parentContext = args.parentContext; + this.startTimeMs = args.startTimeMs ?? clockMs(); + this.contextAttributes = safeContextAttributes(args.parentContext); + this.appSetSession = this.contextAttributes[T.SESSION_ID] != null; + this.providerName = resolveProviderName(this.options); + for (const name of Object.keys(this.options?.mcpServers ?? {})) { + this.mcpServerNames.add(name); + } + if (typeof this.options?.sessionId === "string" && this.options.sessionId) { + this.sessionId = this.options.sessionId; + } else if (typeof this.options?.resume === "string" && this.options.resume && !this.options.forkSession) { + // A plain resume continues the same session id. + this.sessionId = this.options.resume; + } + } + + get isFinished(): boolean { + return this.finished; + } + + /** Open the conversation span. Called on the first `next()`; idempotent. */ + start(): void { + if (this.conversation || this.finished) { + return; + } + try { + const span = this.tracer.startSpan( + SpanNames.CONVERSATION, + { kind: SpanKind.CLIENT, startTime: toHrTime(this.startTimeMs) }, + this.parentContext, + ); + this.conversation = span; + this.conversationContext = trace.setSpan(this.parentContext, span); + span.setAttributes(this.baseAttributes(ClaudeAgentSpanKind.CONVERSATION)); + span.setAttributes(this.conversationStartAttributes()); + this.attachAbortListener(); + } catch (error) { + diag.debug(`@traceai/claude-agent-sdk: failed to start conversation span: ${error}`); + } + } + + /** Record one SDK message. Never throws and never mutates the message. */ + onMessage(message: unknown): void { + if (this.finished || !this.conversation || !message || typeof message !== "object") { + return; + } + try { + const type = (message as { type?: unknown }).type; + if (type === "assistant") { + this.onAssistant(message as AssistantMessageLike); + } else if (type === "user") { + this.onUser(message as UserMessageLike); + } else if (type === "result") { + this.onResult(message as ResultMessageLike); + } else if (type === "system") { + this.onSystem(message as SystemMessageLike); + } + } catch (error) { + diag.debug(`@traceai/claude-agent-sdk: failed to record message: ${error}`); + } + } + + /** End every open span for this query. Idempotent. */ + finish(outcome: FinishOutcome): void { + if (this.finished) { + return; + } + this.finished = true; + this.detachAbortListener(); + if (!this.conversation) { + return; + } + try { + const signal = this.options?.abortController?.signal; + const cancelled = + outcome.kind === "aborted" || + (outcome.kind === "error" && isAbortLike(outcome.error, signal)); + const failed = outcome.kind === "error" && !cancelled; + + // One end time for everything closed here, so no child ends after its parent. + const endMs = clockMs(); + const closeChild = (span: Span, notCompletedMessage: string) => { + if (cancelled) { + span.setAttribute(T.CANCELLED, true); + span.setStatus({ code: SpanStatusCode.ERROR, message: "cancelled" }); + } else { + span.setStatus({ code: SpanStatusCode.ERROR, message: notCompletedMessage }); + } + span.end(toHrTime(endMs)); + }; + + for (const sub of this.subagents.values()) { + if (!sub.ended) { + sub.ended = true; + closeChild(sub.span, "Subagent span not completed"); + } + } + for (const [id, tool] of this.tools) { + this.tools.delete(id); + closeChild(tool.span, "Tool span not completed (conversation ended)"); + } + for (const scope of this.scopes.values()) { + const turn = scope.currentTurn; + if (turn && !turn.ended) { + if (cancelled || failed) { + if (cancelled) turn.span.setAttribute(T.CANCELLED, true); + turn.span.setStatus({ + code: SpanStatusCode.ERROR, + message: cancelled ? "cancelled" : "query failed", + }); + turn.ended = true; + turn.span.end(toHrTime(endMs)); + } else { + this.endTurn(turn, endMs); + } + } + } + + const conversation = this.conversation; + if (cancelled) { + conversation.setAttribute(T.CANCELLED, true); + conversation.setStatus({ code: SpanStatusCode.ERROR, message: "cancelled" }); + } else if (failed) { + const { type, message } = errorParts((outcome as { error: unknown }).error); + conversation.setAttribute(A.ERROR_TYPE, type); + conversation.setAttribute(A.ERROR_MESSAGE, truncate(message, ERROR_MESSAGE_MAX)); + if ((outcome as { error: unknown }).error instanceof Error) { + conversation.recordException((outcome as { error: Error }).error); + } + conversation.setStatus({ code: SpanStatusCode.ERROR, message: truncate(message, ERROR_MESSAGE_MAX) }); + } else if (!this.conversationIsError && (outcome.kind === "completed" || this.resultSeen)) { + conversation.setStatus({ code: SpanStatusCode.OK }); + } + conversation.end(toHrTime(endMs)); + } catch (error) { + diag.debug(`@traceai/claude-agent-sdk: failed to finish spans: ${error}`); + } + } + + // -------------------------------------------------------------------------- + // Message handlers + // -------------------------------------------------------------------------- + + private onAssistant(message: AssistantMessageLike): void { + const scope: Scope = message.parent_tool_use_id ?? null; + const scopeState = this.scope(scope); + const messageId = message.message?.id; + + let turn = scopeState.currentTurn; + if (!turn || turn.ended || !messageId || turn.messageId !== messageId) { + if (turn && !turn.ended) { + this.endTurn(turn); + } + turn = this.startTurn(scope, scopeState, message); + } + + const content = message.message?.content; + const blocks = typeof content === "string" ? [{ type: "text", text: content }] : contentBlocks(content); + for (const block of blocks) { + if (!block || typeof block !== "object") continue; + if (block.type === "text" && typeof block.text === "string") { + turn.textParts.push(block.text); + } else if ( + (block.type === "tool_use" || block.type === "server_tool_use" || block.type === "mcp_tool_use") && + typeof block.id === "string" + ) { + turn.textParts.push(`[Tool: ${block.name ?? "unknown"}]`); + turn.toolUseCount += 1; + this.startTool(block, scope, turn); + } else if ( + typeof block.tool_use_id === "string" && + typeof block.type === "string" && + block.type.endsWith("tool_result") + ) { + // Server-side tool results arrive inside the assistant message. + this.endTool(block.tool_use_id, block.content, block.is_error === true); + } + } + + turn.span.setAttribute(A.MESSAGE_HAS_TOOL_USE, turn.toolUseCount > 0); + turn.span.setAttribute(A.MESSAGE_TOOL_USE_COUNT, turn.toolUseCount); + if (!this.policy.hideOutputs && turn.textParts.length > 0) { + const text = truncate(turn.textParts.join(" "), MESSAGE_CONTENT_MAX); + turn.span.setAttribute(A.MESSAGE_CONTENT, text); + turn.span.setAttribute(T.OUTPUT_VALUE, text); + turn.span.setAttribute(T.OUTPUT_MIME_TYPE, "text/plain"); + } + if (typeof message.error === "string" && message.error) { + turn.isError = true; + turn.span.setAttribute(A.ERROR_TYPE, message.error); + turn.span.setStatus({ code: SpanStatusCode.ERROR, message: message.error }); + } + } + + private onUser(message: UserMessageLike): void { + const scope: Scope = message.parent_tool_use_id ?? null; + const affectedTurns = new Set(); + for (const block of contentBlocks(message.message?.content)) { + if (block && block.type === "tool_result" && typeof block.tool_use_id === "string") { + const turn = this.endTool(block.tool_use_id, block.content, block.is_error === true); + if (turn) affectedTurns.add(turn); + } + } + // The model is called again after a user message: the next turn in this + // scope starts now (Python TurnTracker.mark_next_start). + this.scope(scope).nextTurnStartMs = clockMs(); + for (const turn of affectedTurns) { + if (!turn.ended && turn.openToolIds.size === 0) { + this.endTurn(turn); + } + } + } + + private onResult(message: ResultMessageLike): void { + const conversation = this.conversation!; + this.resultSeen = true; + + const main = this.scopes.get(null)?.currentTurn; + if (main && !main.ended) { + this.endTurn(main); + } + + if (typeof message.session_id === "string" && message.session_id) { + this.setSessionId(message.session_id); + } + + const usage = message.usage; + if (usage && typeof usage === "object") { + const input = typeof usage.input_tokens === "number" ? usage.input_tokens : undefined; + const output = typeof usage.output_tokens === "number" ? usage.output_tokens : undefined; + if (input !== undefined) conversation.setAttribute(A.USAGE_INPUT_TOKENS, input); + if (output !== undefined) conversation.setAttribute(A.USAGE_OUTPUT_TOKENS, output); + if (input !== undefined || output !== undefined) { + conversation.setAttribute(A.USAGE_TOTAL_TOKENS, (input ?? 0) + (output ?? 0)); + } + if (typeof usage.cache_read_input_tokens === "number") { + conversation.setAttribute(A.USAGE_CACHE_READ_TOKENS, usage.cache_read_input_tokens); + } + if (typeof usage.cache_creation_input_tokens === "number") { + conversation.setAttribute(A.USAGE_CACHE_CREATION_TOKENS, usage.cache_creation_input_tokens); + } + } + if (typeof message.total_cost_usd === "number") { + conversation.setAttribute(A.COST_TOTAL_USD, message.total_cost_usd); + conversation.setAttribute(T.GEN_AI_COST_TOTAL, message.total_cost_usd); + } + if (typeof message.duration_ms === "number") { + conversation.setAttribute(A.DURATION_MS, message.duration_ms); + } + if (typeof message.duration_api_ms === "number") { + conversation.setAttribute(A.DURATION_API_MS, message.duration_api_ms); + } + if (typeof message.ttft_ms === "number") { + conversation.setAttribute(A.TIME_TO_FIRST_TOKEN_MS, message.ttft_ms); + } + if (typeof message.num_turns === "number") { + conversation.setAttribute(A.AGENT_NUM_TURNS, message.num_turns); + } + + const subtype = typeof message.subtype === "string" ? message.subtype : undefined; + const isError = message.is_error === true || (subtype !== undefined && subtype.startsWith("error")); + conversation.setAttribute(A.IS_ERROR, isError); + if (isError) { + this.conversationIsError = true; + const errorType = subtype ?? "error"; + conversation.setAttribute(A.ERROR_TYPE, errorType); + if (Array.isArray(message.errors) && message.errors.length > 0) { + conversation.setAttribute( + A.ERROR_MESSAGE, + truncate(message.errors.map(String).join("; "), ERROR_MESSAGE_MAX), + ); + } + conversation.setStatus({ code: SpanStatusCode.ERROR, message: errorType }); + } + + if (!this.policy.hideOutputs && typeof message.result === "string") { + conversation.setAttribute(T.OUTPUT_VALUE, truncate(message.result, CONTENT_MAX)); + conversation.setAttribute(T.OUTPUT_MIME_TYPE, "text/plain"); + } + } + + private onSystem(message: SystemMessageLike): void { + const conversation = this.conversation!; + switch (message.subtype) { + case "init": { + if (typeof message.session_id === "string" && message.session_id) { + this.setSessionId(message.session_id); + } + if (typeof message.model === "string" && message.model) { + conversation.setAttribute(A.AGENT_MODEL, message.model); + conversation.setAttribute(T.GEN_AI_REQUEST_MODEL, message.model); + } + if (typeof message.permissionMode === "string" && message.permissionMode && !this.options?.permissionMode) { + conversation.setAttribute(A.AGENT_PERMISSION_MODE, message.permissionMode); + } + if (Array.isArray(message.mcp_servers)) { + for (const server of message.mcp_servers) { + if (server && typeof server.name === "string") this.mcpServerNames.add(server.name); + } + } + break; + } + case "task_started": { + const sub = message.tool_use_id ? this.subagents.get(message.tool_use_id) : undefined; + if (sub && !sub.ended) { + if (typeof message.task_id === "string") { + sub.span.setAttribute(T.SUBAGENT_TASK_ID, message.task_id); + } + if (message.is_backgrounded === true) { + sub.backgrounded = true; + } + } + break; + } + case "task_notification": { + const id = message.tool_use_id; + const sub = id ? this.subagents.get(id) : undefined; + if (sub && !sub.ended && typeof message.status === "string") { + sub.status = message.status; + sub.span.setAttribute(T.SUBAGENT_STATUS, message.status); + // A backgrounded subagent outlives its tool_result; it ends here. + if (sub.backgrounded || sub.toolEnded) { + this.endSubagent(id!, message.status !== "completed"); + } + } + break; + } + default: + break; + } + } + + // -------------------------------------------------------------------------- + // Span lifecycle + // -------------------------------------------------------------------------- + + private scope(scope: Scope): ScopeState { + let state = this.scopes.get(scope); + if (!state) { + state = { turnCount: 0 }; + this.scopes.set(scope, state); + } + return state; + } + + /** Context the children of `scope` are created in. */ + private scopeParentContext(scope: Scope): Context { + if (scope !== null) { + const parent = this.subagents.get(scope)?.span ?? this.tools.get(scope)?.span; + if (parent) { + return trace.setSpan(this.parentContext, parent); + } + } + return this.conversationContext!; + } + + private startTurn(scope: Scope, scopeState: ScopeState, message: AssistantMessageLike): TurnState { + scopeState.turnCount += 1; + const startTime = + scopeState.nextTurnStartMs ?? + (scope === null ? this.startTimeMs : this.subagents.get(scope)?.startMs) ?? + clockMs(); + scopeState.nextTurnStartMs = undefined; + + const span = this.tracer.startSpan( + SpanNames.ASSISTANT_TURN, + { kind: SpanKind.INTERNAL, startTime: toHrTime(startTime) }, + this.scopeParentContext(scope), + ); + span.setAttributes(this.baseAttributes(ClaudeAgentSpanKind.ASSISTANT_TURN)); + span.setAttribute(A.AGENT_NUM_TURNS, scopeState.turnCount); + span.setAttribute(A.MESSAGE_TYPE, "assistant"); + span.setAttribute(A.MESSAGE_ROLE, "assistant"); + span.setAttribute(T.GEN_AI_PROVIDER_NAME, this.providerName); + const model = message.message?.model; + if (typeof model === "string" && model) { + span.setAttribute(A.AGENT_MODEL, model); + span.setAttribute(T.GEN_AI_REQUEST_MODEL, model); + } + if (scope !== null) { + span.setAttribute(A.PARENT_TOOL_USE_ID, scope); + } + + const turn: TurnState = { + span, + scope, + messageId: message.message?.id, + textParts: [], + toolUseCount: 0, + openToolIds: new Set(), + isError: false, + ended: false, + }; + scopeState.currentTurn = turn; + return turn; + } + + private endTurn(turn: TurnState, endMs: number = clockMs()): void { + if (turn.ended) return; + turn.ended = true; + if (!turn.isError) { + turn.span.setStatus({ code: SpanStatusCode.OK }); + } + turn.span.end(toHrTime(endMs)); + const scopeState = this.scopes.get(turn.scope); + if (scopeState?.currentTurn === turn) { + scopeState.currentTurn = undefined; + } + } + + private startTool(block: ContentBlockLike, scope: Scope, turn: TurnState): void { + const id = block.id as string; + if (this.tools.has(id)) { + return; + } + const name = typeof block.name === "string" && block.name ? block.name : "unknown_tool"; + const mcp = + block.type === "mcp_tool_use" && typeof block.server_name === "string" + ? { server: block.server_name, tool: name } + : parseMcpToolName(name, this.mcpServerNames); + const source = mcp ? "mcp" : getToolSource(name, this.mcpServerNames); + const kind = mcp ? ClaudeAgentSpanKind.MCP_TOOL : ClaudeAgentSpanKind.TOOL_EXECUTION; + + const startMs = clockMs(); + const span = this.tracer.startSpan( + SpanNames.toolSpan(name), + { kind: SpanKind.INTERNAL, startTime: toHrTime(startMs) }, + trace.setSpan(this.parentContext, turn.span), + ); + span.setAttributes(this.baseAttributes(kind)); + span.setAttribute(A.GEN_AI_TOOL_NAME, name); + span.setAttribute(T.GEN_AI_TOOL_NAME, name); + span.setAttribute(A.TOOL_USE_ID, id); + span.setAttribute(A.TOOL_SOURCE, source); + if (mcp) { + span.setAttribute(A.MCP_SERVER_NAME, mcp.server); + span.setAttribute(T.MCP_TOOL_NAME, mcp.tool); + } + if (scope !== null) { + span.setAttribute(A.PARENT_TOOL_USE_ID, scope); + } + const input = block.input; + if (!this.policy.hideInputs && input !== undefined && input !== null) { + const serialized = safeJson(input); + span.setAttribute(A.TOOL_INPUT, serialized); + span.setAttribute(T.INPUT_VALUE, serialized); + span.setAttribute(T.INPUT_MIME_TYPE, typeof input === "string" ? "text/plain" : "application/json"); + setToolSpecificAttributes(span, name, input); + } + + const tool: ToolState = { span, name, startMs, turn }; + this.tools.set(id, tool); + turn.openToolIds.add(id); + + if (!mcp && SUBAGENT_TOOLS.has(name)) { + this.startSubagent(id, input, scope, span); + } + } + + /** End a tool span. Returns the turn that issued the tool, if any. */ + private endTool(id: string, content: unknown, isError: boolean): TurnState | undefined { + const tool = this.tools.get(id); + if (!tool) { + return undefined; + } + this.tools.delete(id); + + const sub = this.subagents.get(id); + if (sub && !sub.ended) { + sub.toolEnded = true; + if (!sub.backgrounded) { + this.endSubagent(id, isError); + } + } + + const span = tool.span; + const endMs = clockMs(); + span.setAttribute(A.TOOL_DURATION_MS, endMs - tool.startMs); + span.setAttribute(A.TOOL_IS_ERROR, isError); + const output = flattenToolResult(content); + if (!this.policy.hideOutputs) { + const serialized = truncate(output, CONTENT_MAX); + span.setAttribute(A.TOOL_OUTPUT, serialized); + span.setAttribute(T.OUTPUT_VALUE, serialized); + span.setAttribute(T.OUTPUT_MIME_TYPE, "text/plain"); + } + if (isError) { + const message = this.policy.hideOutputs ? "tool error" : truncate(output, ERROR_MESSAGE_MAX); + if (!this.policy.hideOutputs) { + span.setAttribute(A.TOOL_ERROR_MESSAGE, message); + } + span.setStatus({ code: SpanStatusCode.ERROR, message }); + } else { + span.setStatus({ code: SpanStatusCode.OK }); + } + span.end(toHrTime(endMs)); + + tool.turn?.openToolIds.delete(id); + return tool.turn; + } + + private startSubagent(toolUseId: string, input: unknown, scope: Scope, toolSpan: Span): void { + const fields = (input && typeof input === "object" ? input : {}) as Record; + const subagentType = + typeof fields.subagent_type === "string" && fields.subagent_type ? fields.subagent_type : "unknown"; + const startMs = clockMs(); + const span = this.tracer.startSpan( + SpanNames.subagentSpan(subagentType), + { kind: SpanKind.INTERNAL, startTime: toHrTime(startMs) }, + trace.setSpan(this.parentContext, toolSpan), + ); + span.setAttributes(this.baseAttributes(ClaudeAgentSpanKind.SUBAGENT)); + span.setAttribute(A.TOOL_USE_ID, toolUseId); + span.setAttribute(A.SUBAGENT_TYPE, subagentType); + if (typeof fields.name === "string" && fields.name) { + span.setAttribute(A.SUBAGENT_NAME, fields.name); + } + if (typeof fields.model === "string" && fields.model) { + span.setAttribute(A.AGENT_MODEL, fields.model); + } + if (scope !== null) { + span.setAttribute(A.PARENT_TOOL_USE_ID, scope); + } + if (!this.policy.hideInputs) { + if (typeof fields.description === "string" && fields.description) { + span.setAttribute(A.SUBAGENT_DESCRIPTION, truncate(fields.description, 500)); + } + if (typeof fields.prompt === "string" && fields.prompt) { + span.setAttribute(A.SUBAGENT_PROMPT, truncate(fields.prompt, PROMPT_MAX)); + span.setAttribute(T.INPUT_VALUE, truncate(fields.prompt, PROMPT_MAX)); + span.setAttribute(T.INPUT_MIME_TYPE, "text/plain"); + } + } + this.subagents.set(toolUseId, { + span, + startMs, + backgrounded: fields.run_in_background === true, + toolEnded: false, + ended: false, + }); + } + + private endSubagent(toolUseId: string, isError: boolean): void { + const sub = this.subagents.get(toolUseId); + if (!sub || sub.ended) return; + // Close the subagent's own open turn first so no child outlives it. + const scopeState = this.scopes.get(toolUseId); + if (scopeState?.currentTurn && !scopeState.currentTurn.ended) { + this.endTurn(scopeState.currentTurn); + } + sub.ended = true; + const endMs = clockMs(); + sub.span.setAttribute(A.TOOL_DURATION_MS, endMs - sub.startMs); + sub.span.setAttribute(A.TOOL_IS_ERROR, isError); + sub.span.setStatus( + isError + ? { code: SpanStatusCode.ERROR, message: sub.status ? `subagent ${sub.status}` : "subagent error" } + : { code: SpanStatusCode.OK }, + ); + sub.span.end(toHrTime(endMs)); + } + + // -------------------------------------------------------------------------- + // Attributes + // -------------------------------------------------------------------------- + + private baseAttributes(kind: ClaudeAgentSpanKindValue): Attributes { + const fiKind = FI_SPAN_KIND_BY_CLAUDE_KIND[kind]; + const attributes: Attributes = { + ...this.contextAttributes, + [A.SPAN_KIND]: kind, + [T.GEN_AI_SPAN_KIND]: fiKind, + [T.FI_SPAN_KIND]: fiKind, + }; + Object.assign(attributes, this.sessionAttributes()); + return attributes; + } + + private sessionAttributes(): Attributes { + if (!this.sessionId) return {}; + const attributes: Attributes = { + [A.GEN_AI_CONVERSATION_ID]: this.sessionId, + [A.AGENT_SESSION_ID]: this.sessionId, + }; + if (!this.appSetSession) { + attributes[T.SESSION_ID] = this.sessionId; + } + return attributes; + } + + private setSessionId(sessionId: string): void { + if (this.sessionId === sessionId) return; + this.sessionId = sessionId; + this.conversation?.setAttributes(this.sessionAttributes()); + } + + private conversationStartAttributes(): Attributes { + const options = this.options ?? {}; + const attributes: Record = { + [T.GEN_AI_PROVIDER_NAME]: this.providerName, + }; + if (typeof options.model === "string" && options.model) { + attributes[A.AGENT_MODEL] = options.model; + attributes[T.GEN_AI_REQUEST_MODEL] = options.model; + } + if (typeof options.permissionMode === "string" && options.permissionMode) { + attributes[A.AGENT_PERMISSION_MODE] = options.permissionMode; + } + if (Array.isArray(options.allowedTools) && options.allowedTools.length > 0) { + attributes[A.AGENT_ALLOWED_TOOLS] = safeJson(options.allowedTools); + } + + const resume = typeof options.resume === "string" && options.resume ? options.resume : undefined; + const fork = resume !== undefined && options.forkSession === true; + if (resume) { + attributes[A.AGENT_IS_RESUMED] = true; + attributes[A.AGENT_RESUME_SESSION_ID] = resume; + } + attributes[A.SESSION_IS_RESUMED] = resume !== undefined && !fork; + attributes[A.SESSION_IS_NEW] = resume === undefined || fork; + if (resume && !fork) { + attributes[A.SESSION_PREVIOUS_ID] = resume; + } + if (fork) { + attributes[A.SESSION_FORK_FROM] = resume!; + } + + if (!this.policy.hideInputs) { + if (typeof this.params.prompt === "string") { + const prompt = truncate(this.params.prompt, PROMPT_MAX); + attributes[A.AGENT_PROMPT] = prompt; + attributes[T.INPUT_VALUE] = prompt; + attributes[T.INPUT_MIME_TYPE] = "text/plain"; + } + const systemPrompt = options.systemPrompt; + if (typeof systemPrompt === "string" && systemPrompt) { + attributes[A.AGENT_SYSTEM_PROMPT] = truncate(systemPrompt, SYSTEM_PROMPT_MAX); + } else if (systemPrompt && typeof systemPrompt === "object") { + attributes[A.AGENT_SYSTEM_PROMPT] = safeJson(systemPrompt, SYSTEM_PROMPT_MAX); + } + } + return attributes; + } + + // -------------------------------------------------------------------------- + // Abort + // -------------------------------------------------------------------------- + + private attachAbortListener(): void { + const signal = this.options?.abortController?.signal; + if (!signal || typeof signal.addEventListener !== "function") return; + if (signal.aborted) { + this.finish({ kind: "aborted", reason: signal.reason }); + return; + } + const listener = () => this.finish({ kind: "aborted", reason: signal.reason }); + this.abortListener = listener; + signal.addEventListener("abort", listener); + } + + private detachAbortListener(): void { + const signal = this.options?.abortController?.signal; + if (this.abortListener && signal && typeof signal.removeEventListener === "function") { + signal.removeEventListener("abort", this.abortListener); + } + this.abortListener = undefined; + } +} + +function safeContextAttributes(ctx: Context): Attributes { + try { + return getAttributesFromContext(ctx); + } catch { + return {}; + } +} + +/** Python `_set_tool_specific_attributes`. Only called when inputs are visible. */ +function setToolSpecificAttributes(span: Span, toolName: string, input: unknown): void { + if (!input || typeof input !== "object") return; + const fields = input as Record; + const str = (key: string) => (typeof fields[key] === "string" ? (fields[key] as string) : undefined); + switch (toolName) { + case "Read": + case "Write": + case "Edit": { + const filePath = str("file_path"); + if (filePath) span.setAttribute(A.TOOL_FILE_PATH, filePath); + break; + } + case "Bash": { + const command = str("command"); + if (command) span.setAttribute(A.TOOL_COMMAND, truncate(command, TOOL_COMMAND_MAX)); + break; + } + case "Glob": + case "Grep": { + const pattern = str("pattern"); + if (pattern) span.setAttribute(A.TOOL_PATTERN, pattern); + break; + } + case "WebSearch": { + const query = str("query"); + if (query) span.setAttribute(A.TOOL_SEARCH_QUERY, query); + break; + } + case "WebFetch": { + const url = str("url"); + if (url) span.setAttribute(A.TOOL_URL, url); + break; + } + default: + break; + } +} diff --git a/typescript/packages/traceai_claude_agent_sdk/src/types.ts b/typescript/packages/traceai_claude_agent_sdk/src/types.ts new file mode 100644 index 000000000..87ebdac64 --- /dev/null +++ b/typescript/packages/traceai_claude_agent_sdk/src/types.ts @@ -0,0 +1,131 @@ +/** + * Structural types for the parts of the Claude Agent SDK surface this package + * reads. They are written from `sdk.d.ts` of `@anthropic-ai/claude-agent-sdk` + * 0.3.289 (and checked against 0.3.142) and deliberately do not import the + * SDK, so the SDK stays a peer dependency with no type or runtime import. + * + * Every field is optional: the wrapper reads defensively and never fails the + * agent because a field moved. + */ + +/** A content block inside `BetaMessage.content` or `MessageParam.content`. */ +export interface ContentBlockLike { + type?: string; + text?: string; + /** tool_use / server_tool_use / mcp_tool_use */ + id?: string; + name?: string; + input?: unknown; + /** mcp_tool_use */ + server_name?: string; + /** tool_result */ + tool_use_id?: string; + content?: unknown; + is_error?: boolean; +} + +/** `SDKAssistantMessage` (type: 'assistant'). */ +export interface AssistantMessageLike { + type: "assistant"; + message?: { + id?: string; + model?: string; + content?: ContentBlockLike[] | string; + stop_reason?: string | null; + }; + parent_tool_use_id?: string | null; + error?: string; + session_id?: string; +} + +/** `SDKUserMessage` / `SDKUserMessageReplay` (type: 'user'). */ +export interface UserMessageLike { + type: "user"; + message?: { + role?: string; + content?: ContentBlockLike[] | string; + }; + parent_tool_use_id?: string | null; + session_id?: string; +} + +/** `SDKResultSuccess` / `SDKResultError` (type: 'result'). */ +export interface ResultMessageLike { + type: "result"; + subtype?: string; + duration_ms?: number; + duration_api_ms?: number; + ttft_ms?: number; + is_error?: boolean; + num_turns?: number; + result?: string; + total_cost_usd?: number; + usage?: { + input_tokens?: number; + output_tokens?: number; + cache_read_input_tokens?: number | null; + cache_creation_input_tokens?: number | null; + }; + errors?: string[]; + session_id?: string; +} + +/** `SDKSystemMessage` and the other `type: 'system'` messages. */ +export interface SystemMessageLike { + type: "system"; + subtype?: string; + session_id?: string; + /** init */ + model?: string; + permissionMode?: string; + mcp_servers?: { name: string; status?: string }[]; + /** task_started / task_notification / task_progress */ + task_id?: string; + tool_use_id?: string; + status?: string; + subagent_type?: string; + is_backgrounded?: boolean; +} + +export type SDKMessageLike = + | AssistantMessageLike + | UserMessageLike + | ResultMessageLike + | SystemMessageLike + | { type: string; [key: string]: unknown }; + +/** The parts of `AbortSignal` the wrapper uses (structural, no DOM/Node lib needed). */ +export interface AbortSignalLike { + readonly aborted: boolean; + readonly reason?: unknown; + addEventListener(type: "abort", listener: () => void): void; + removeEventListener(type: "abort", listener: () => void): void; +} + +/** The subset of `Options` the wrapper reads. It never writes to options. */ +export interface OptionsLike { + abortController?: { readonly signal: AbortSignalLike }; + env?: { [envVar: string]: string | undefined }; + model?: string; + permissionMode?: string; + allowedTools?: string[]; + systemPrompt?: unknown; + resume?: string; + forkSession?: boolean; + sessionId?: string; + mcpServers?: Record; + [key: string]: unknown; +} + +export interface QueryParamsLike { + prompt: string | AsyncIterable; + options?: OptionsLike; +} + +/** + * The `query()` function shape. The SDK's `Query` extends + * `AsyncGenerator` and adds control methods + * (`interrupt()`, `setModel()`, ...); the wrapper forwards those untouched. + */ +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export type QueryFunctionLike = (params: any) => AsyncGenerator; diff --git a/typescript/packages/traceai_claude_agent_sdk/src/version.ts b/typescript/packages/traceai_claude_agent_sdk/src/version.ts new file mode 100644 index 000000000..52c905c11 --- /dev/null +++ b/typescript/packages/traceai_claude_agent_sdk/src/version.ts @@ -0,0 +1 @@ +export const VERSION = "0.1.0"; diff --git a/typescript/packages/traceai_claude_agent_sdk/tsconfig.esm.json b/typescript/packages/traceai_claude_agent_sdk/tsconfig.esm.json new file mode 100644 index 000000000..e5d6f9b6c --- /dev/null +++ b/typescript/packages/traceai_claude_agent_sdk/tsconfig.esm.json @@ -0,0 +1,17 @@ +{ + "extends": "../../tsconfig.base.esm.json", + "compilerOptions": { + "outDir": "dist/esm", + "rootDir": "src", + "tsBuildInfoFile": "dist/esm/tsconfig.esm.tsbuildinfo", + "baseUrl": "./src" + }, + "include": [ + "src/**/*.ts" + ], + "references": [], + "exclude": [ + "src/**/*.test.ts", + "src/**/__tests__/**" + ] +} diff --git a/typescript/packages/traceai_claude_agent_sdk/tsconfig.esnext.json b/typescript/packages/traceai_claude_agent_sdk/tsconfig.esnext.json new file mode 100644 index 000000000..a99f0fe06 --- /dev/null +++ b/typescript/packages/traceai_claude_agent_sdk/tsconfig.esnext.json @@ -0,0 +1,17 @@ +{ + "extends": "../../tsconfig.base.esnext.json", + "compilerOptions": { + "outDir": "dist/esnext", + "rootDir": "src", + "tsBuildInfoFile": "dist/esnext/tsconfig.esnext.tsbuildinfo", + "baseUrl": "./src" + }, + "include": [ + "src/**/*.ts" + ], + "references": [], + "exclude": [ + "src/**/*.test.ts", + "src/**/__tests__/**" + ] +} diff --git a/typescript/packages/traceai_claude_agent_sdk/tsconfig.json b/typescript/packages/traceai_claude_agent_sdk/tsconfig.json new file mode 100644 index 000000000..087f63ee4 --- /dev/null +++ b/typescript/packages/traceai_claude_agent_sdk/tsconfig.json @@ -0,0 +1,14 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": ".", + "baseUrl": "./src" + }, + "include": ["src/**/*.ts"], + "exclude": ["src/**/*.test.ts", "src/**/__tests__/**", "test/**/*.ts"], + "references": [ + { "path": "../fi-core" }, + { "path": "../fi-semantic-conventions" } + ] +} diff --git a/typescript/pnpm-lock.yaml b/typescript/pnpm-lock.yaml index c0f93ce14..5a5c430f6 100644 --- a/typescript/pnpm-lock.yaml +++ b/typescript/pnpm-lock.yaml @@ -486,6 +486,49 @@ importers: specifier: ^1.8.16 version: 1.8.16 + packages/traceai_claude_agent_sdk: + dependencies: + '@opentelemetry/api': + specifier: ^1.9.0 + version: 1.9.1 + '@traceai/fi-core': + specifier: workspace:^ + version: link:../fi-core + '@traceai/fi-semantic-conventions': + specifier: workspace:^ + version: link:../fi-semantic-conventions + devDependencies: + '@anthropic-ai/claude-agent-sdk': + specifier: 0.3.289 + version: 0.3.289(@anthropic-ai/sdk@0.131.0(zod@4.3.6))(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.3.6))(zod@4.3.6) + '@anthropic-ai/sdk': + specifier: ^0.131.0 + version: 0.131.0(zod@4.3.6) + '@opentelemetry/sdk-trace-base': + specifier: ^2.0.1 + version: 2.7.1(@opentelemetry/api@1.9.1) + '@types/jest': + specifier: ^29.5.14 + version: 29.5.14 + '@types/node': + specifier: ^22.10.10 + version: 22.15.33 + jest: + specifier: ^29.7.0 + version: 29.7.0(@types/node@22.15.33)(ts-node@10.9.2(@types/node@22.15.33)(typescript@5.8.3)) + rimraf: + specifier: ^6.0.1 + version: 6.1.2 + ts-jest: + specifier: ^29.2.5 + version: 29.4.6(@babel/core@7.27.1)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.27.1))(jest-util@29.7.0)(jest@29.7.0(@types/node@22.15.33)(ts-node@10.9.2(@types/node@22.15.33)(typescript@5.8.3)))(typescript@5.8.3) + tsc-alias: + specifier: ^1.8.10 + version: 1.8.16 + typescript: + specifier: ^5.7.3 + version: 5.8.3 + packages/traceai_cohere: dependencies: '@opentelemetry/api': @@ -2090,6 +2133,67 @@ packages: resolution: {integrity: sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw==} engines: {node: '>=6.0.0'} + '@anthropic-ai/claude-agent-sdk-darwin-arm64@0.3.289': + resolution: {integrity: sha512-A1iK70JJgj6GhSkGnaVC02B1hX0aPvWTbn3a1Fp9iKhXt1a6eZwAhkqdP+F/rzAVUcQG24ehG/LBF2Ec8tBL0Q==} + cpu: [arm64] + os: [darwin] + + '@anthropic-ai/claude-agent-sdk-darwin-x64@0.3.289': + resolution: {integrity: sha512-8kCv04+eOm26zPta8hym7eNHT/nn44a94Y/XwiR9ExWGGDcBJpNOZkVUxt+ciiKNQN7ujuYqAjdRedIRyD1izw==} + cpu: [x64] + os: [darwin] + + '@anthropic-ai/claude-agent-sdk-linux-arm64-musl@0.3.289': + resolution: {integrity: sha512-nA1iLmY9R1Da042RWEJbJxz2S+jMVk1KcACeAuW2A9vVBqAUZwHXwWG1tvs9OZOBzxtayOxvKfG1pGIjoyStww==} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@anthropic-ai/claude-agent-sdk-linux-arm64@0.3.289': + resolution: {integrity: sha512-u5GKtkDU5ho9g8fveN4PgInHyWOBQ12wSlZAcssrQJx8WmoYTUYQTz5kH4e6VNYfIJv1K5DXOSxPGuqbcSMr6Q==} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@anthropic-ai/claude-agent-sdk-linux-x64-musl@0.3.289': + resolution: {integrity: sha512-/XpyF/cyWNABbgxvtR+v8xz8bjv1lCZKZfBKdm5b/ROlfmyr27Q37JqNUUY6HR9dGvSYf4tCzkqLstk2tQZ6UQ==} + cpu: [x64] + os: [linux] + libc: [musl] + + '@anthropic-ai/claude-agent-sdk-linux-x64@0.3.289': + resolution: {integrity: sha512-d/2f7T/DpnEat78nun+mE26HZ0rO9BGEmoGja1i8iIW62IZX1K3T2IXd1h0Wjgtx1JSGDUK3pxMhQJxMyJdycQ==} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@anthropic-ai/claude-agent-sdk-win32-arm64@0.3.289': + resolution: {integrity: sha512-/99s03mf1X3YB8pqcAsF+yDBo/O6kzZG2A6UU8zAf5Xa0iM/aBgd6kCx5Qz2KozrCkrlnbsWjp3HKEKs0Ls7MA==} + cpu: [arm64] + os: [win32] + + '@anthropic-ai/claude-agent-sdk-win32-x64@0.3.289': + resolution: {integrity: sha512-HBoziJqI6LkEkqZuumOXxXLMZEvnZOQirc3OyZTDJIa7UJcV/pn9EsBXLruMlNztrOWfEqBeYArXSl00H91rjA==} + cpu: [x64] + os: [win32] + + '@anthropic-ai/claude-agent-sdk@0.3.289': + resolution: {integrity: sha512-fQRvZPhf6zxyzj7gwyItbhixmu5+Oy0HMQOyzYz3Phjhxuhj3TnpGPPy0VBnbSqWqYxT67TAFOiF5OzOFMYPLQ==} + engines: {node: '>=18.0.0'} + peerDependencies: + '@anthropic-ai/sdk': '>=0.93.0' + '@modelcontextprotocol/sdk': ^1.29.0 + zod: ^4.0.0 + + '@anthropic-ai/sdk@0.131.0': + resolution: {integrity: sha512-9+PepoU7qVMmM70jSahdnfcMHAvAnzhMOLv2W9WjRLO+OG/MAiBETkwKZqv7KIJ1Cj5h8eZ019bHzZaq+Mbepw==} + hasBin: true + peerDependencies: + zod: ^3.25.0 || ^4.0.0 + peerDependenciesMeta: + zod: + optional: true + '@anthropic-ai/sdk@0.27.3': resolution: {integrity: sha512-IjLt0gd3L4jlOfilxVXTifn42FnVffMgDC04RJK1KDZpmkBWLv0XC92MVVmkxrFZNS/7l3xWgP/I3nqtX1sQHw==} @@ -4361,6 +4465,9 @@ packages: '@so-ric/colorspace@1.1.6': resolution: {integrity: sha512-/KiKkpHNOBgkFJwu9sh48LkHSMYGyuTcSFK/qMBdnOAlrRJzRSXAOFB5qwzaVQuDl8wAvHVMkaASQDReTahxuw==} + '@stablelib/base64@1.0.1': + resolution: {integrity: sha512-1bnPQqSxSuc3Ii6MhBysoWCg58j97aUjuCSZrGSmDxNqtytIi0k8utUenAwTZN4V5mXXYGsVUI9zeBqy+jBOSQ==} + '@standard-community/standard-json@0.3.5': resolution: {integrity: sha512-4+ZPorwDRt47i+O7RjyuaxHRK/37QY/LmgxlGrRrSTLYoFatEOzvqIc85GTlM18SFZ5E91C+v0o/M37wZPpUHA==} peerDependencies: @@ -5493,6 +5600,9 @@ packages: fast-json-stable-stringify@2.1.0: resolution: {integrity: sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==} + fast-sha256@1.3.0: + resolution: {integrity: sha512-n11RGP/lrWEFI/bWdygLxhI+pVeo1ZYIVwvvPkW7azl/rOy+F3HYRZ2K5zeE9mmkhQppyv9sQFx0JM9UabnpPQ==} + fast-uri@3.1.0: resolution: {integrity: sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA==} @@ -6180,6 +6290,10 @@ packages: json-parse-even-better-errors@2.3.1: resolution: {integrity: sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==} + json-schema-to-ts@3.1.1: + resolution: {integrity: sha512-+DWg8jCJG2TEnpy7kOm/7/AxaYoaRbjVB4LFZLySZlWn8exGs3A4OLJR966cVvU26N7X9TWxl+Jsw7dzAqKT6g==} + engines: {node: '>=16'} + json-schema-to-zod@2.8.1: resolution: {integrity: sha512-fRr1mHgZ7hboLKBUdR428gd9dIHUFGivUqOeiDcSmyXkNZCtB1uGaZLvsjZ4GaN5pwBIs+TGIOf6s+Rp5/R/zA==} hasBin: true @@ -7758,6 +7872,9 @@ packages: stackback@0.0.2: resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} + standardwebhooks@1.1.1: + resolution: {integrity: sha512-bCbX9ZEyFkWPsRz7Bl3NuQUJohmwGSev/yhr7vhaGPlc4AfIrspIRa6cPTBuI1ItmrTDJ4d/S2hCsfe4+vQGnQ==} + statuses@2.0.2: resolution: {integrity: sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==} engines: {node: '>= 0.8'} @@ -7935,6 +8052,9 @@ packages: trough@2.2.0: resolution: {integrity: sha512-tmMpK00BjZiUyVyvrBK7knerNgmgvcV/KLVyuma/SC+TQN167GrMRciANTz09+k3zW8L8t60jWO1GpfkZdjTaw==} + ts-algebra@2.0.0: + resolution: {integrity: sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==} + ts-debounce@4.0.0: resolution: {integrity: sha512-+1iDGY6NmOGidq7i7xZGA4cm8DAa6fqdYcvO5Z6yBevH++Bdo9Qt/mN0TzHUgcCcKv1gmh9+W5dHqz8pMWbCbg==} @@ -8493,6 +8613,52 @@ snapshots: '@jridgewell/gen-mapping': 0.3.8 '@jridgewell/trace-mapping': 0.3.25 + '@anthropic-ai/claude-agent-sdk-darwin-arm64@0.3.289': + optional: true + + '@anthropic-ai/claude-agent-sdk-darwin-x64@0.3.289': + optional: true + + '@anthropic-ai/claude-agent-sdk-linux-arm64-musl@0.3.289': + optional: true + + '@anthropic-ai/claude-agent-sdk-linux-arm64@0.3.289': + optional: true + + '@anthropic-ai/claude-agent-sdk-linux-x64-musl@0.3.289': + optional: true + + '@anthropic-ai/claude-agent-sdk-linux-x64@0.3.289': + optional: true + + '@anthropic-ai/claude-agent-sdk-win32-arm64@0.3.289': + optional: true + + '@anthropic-ai/claude-agent-sdk-win32-x64@0.3.289': + optional: true + + '@anthropic-ai/claude-agent-sdk@0.3.289(@anthropic-ai/sdk@0.131.0(zod@4.3.6))(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.3.6))(zod@4.3.6)': + dependencies: + '@anthropic-ai/sdk': 0.131.0(zod@4.3.6) + '@modelcontextprotocol/sdk': 1.29.0(@cfworker/json-schema@4.1.1)(zod@4.3.6) + zod: 4.3.6 + optionalDependencies: + '@anthropic-ai/claude-agent-sdk-darwin-arm64': 0.3.289 + '@anthropic-ai/claude-agent-sdk-darwin-x64': 0.3.289 + '@anthropic-ai/claude-agent-sdk-linux-arm64': 0.3.289 + '@anthropic-ai/claude-agent-sdk-linux-arm64-musl': 0.3.289 + '@anthropic-ai/claude-agent-sdk-linux-x64': 0.3.289 + '@anthropic-ai/claude-agent-sdk-linux-x64-musl': 0.3.289 + '@anthropic-ai/claude-agent-sdk-win32-arm64': 0.3.289 + '@anthropic-ai/claude-agent-sdk-win32-x64': 0.3.289 + + '@anthropic-ai/sdk@0.131.0(zod@4.3.6)': + dependencies: + json-schema-to-ts: 3.1.1 + standardwebhooks: 1.1.1 + optionalDependencies: + zod: 4.3.6 + '@anthropic-ai/sdk@0.27.3': dependencies: '@types/node': 18.19.100 @@ -11570,6 +11736,8 @@ snapshots: color: 5.0.3 text-hex: 1.0.0 + '@stablelib/base64@1.0.1': {} + '@standard-community/standard-json@0.3.5(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(quansync@0.2.11)(zod-to-json-schema@3.25.1(zod@4.3.6))(zod@4.3.6)': dependencies: '@standard-schema/spec': 1.1.0 @@ -12763,6 +12931,8 @@ snapshots: fast-json-stable-stringify@2.1.0: {} + fast-sha256@1.3.0: {} + fast-uri@3.1.0: {} fast-xml-parser@5.2.5: @@ -13822,6 +13992,11 @@ snapshots: json-parse-even-better-errors@2.3.1: {} + json-schema-to-ts@3.1.1: + dependencies: + '@babel/runtime': 7.28.6 + ts-algebra: 2.0.0 + json-schema-to-zod@2.8.1: {} json-schema-traverse@1.0.0: {} @@ -15385,6 +15560,11 @@ snapshots: stackback@0.0.2: {} + standardwebhooks@1.1.1: + dependencies: + '@stablelib/base64': 1.0.1 + fast-sha256: 1.3.0 + statuses@2.0.2: {} std-env@3.10.0: {} @@ -15557,6 +15737,8 @@ snapshots: trough@2.2.0: {} + ts-algebra@2.0.0: {} + ts-debounce@4.0.0: {} ts-error@1.0.6: {} From 5d0efff0a883d041c20cb78ad4a16a1fc8d8d444 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 4 Oct 2026 16:39:37 -0700 Subject: [PATCH 05/19] fix(claude-agent-sdk): keep content hidden unless FI_HIDE_* is exactly false resolveContentPolicy() used fi-core's rule (value.toLowerCase() === "true"), so FI_HIDE_INPUTS=1, =yes or =" true" turned prompt and tool capture ON. Only an explicit "false" (trimmed, any case) now opts in; every other value, including the empty string, keeps the hidden default. The traceConfig option still takes precedence. README documents the rule. approved: Nikhil 2026-10-03 blanket Refs: TH-8235 --- .../traceai_claude_agent_sdk/README.md | 2 ++ .../src/__tests__/wrapper.test.ts | 22 +++++++++++++++++++ .../src/instrumentation.ts | 15 ++++++++----- 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/typescript/packages/traceai_claude_agent_sdk/README.md b/typescript/packages/traceai_claude_agent_sdk/README.md index 479f617fb..248bc4e40 100644 --- a/typescript/packages/traceai_claude_agent_sdk/README.md +++ b/typescript/packages/traceai_claude_agent_sdk/README.md @@ -90,6 +90,8 @@ wrapQuery(query, { tracerProvider, traceConfig: { hideInputs: false, hideOutputs // or: FI_HIDE_INPUTS=false FI_HIDE_OUTPUTS=false ``` +Precedence: `traceConfig` option, then the env var, then hidden. The env vars fail closed: only the value `false` (trimmed, any case) turns capture on. Any other value, including `1`, `yes`, `true`, ` true` or an empty string, keeps content hidden. (fi-core itself treats every value other than `true` as "show", so `FI_HIDE_INPUTS=1` would capture there; this package does not follow that rule.) + This is stricter than fi-core's own default and than the Python package, which records the prompt and tool input/output unconditionally. ## Flushing and batching diff --git a/typescript/packages/traceai_claude_agent_sdk/src/__tests__/wrapper.test.ts b/typescript/packages/traceai_claude_agent_sdk/src/__tests__/wrapper.test.ts index 15f9e3c6a..2b6bde1ff 100644 --- a/typescript/packages/traceai_claude_agent_sdk/src/__tests__/wrapper.test.ts +++ b/typescript/packages/traceai_claude_agent_sdk/src/__tests__/wrapper.test.ts @@ -207,6 +207,28 @@ describe("wrapQuery", () => { expect(text).toContain(TOOL_OUTPUT_MARKER); }); + it.each([" False ", "FALSE"])("treats FI_HIDE_*=%j (trimmed, any case) as the opt-in", async (value) => { + process.env.FI_HIDE_INPUTS = value; + process.env.FI_HIDE_OUTPUTS = value; + const { spans } = await run(simpleToolJourney()); + const text = allAttributeText(spans); + expect(text).toContain("SECRET_PROMPT_MARKER"); + expect(text).toContain(TOOL_OUTPUT_MARKER); + }); + + it.each(["1", "yes", " true", "TRUE", "", "0", "no", "off"])( + "keeps content hidden for FI_HIDE_*=%j: only an explicit 'false' opts in", + async (value) => { + process.env.FI_HIDE_INPUTS = value; + process.env.FI_HIDE_OUTPUTS = value; + const { spans } = await run(simpleToolJourney()); + const text = allAttributeText(spans); + for (const marker of ["SECRET_PROMPT_MARKER", TOOL_INPUT_MARKER, TOOL_OUTPUT_MARKER, ASSISTANT_TEXT_MARKER]) { + expect(text).not.toContain(marker); + } + }, + ); + it("AC-08: yields the same message objects, in order, and passes params through unchanged", async () => { const messages = simpleToolJourney(); const expected = JSON.parse(JSON.stringify(messages)); diff --git a/typescript/packages/traceai_claude_agent_sdk/src/instrumentation.ts b/typescript/packages/traceai_claude_agent_sdk/src/instrumentation.ts index af658f42f..a15acc813 100644 --- a/typescript/packages/traceai_claude_agent_sdk/src/instrumentation.ts +++ b/typescript/packages/traceai_claude_agent_sdk/src/instrumentation.ts @@ -11,7 +11,11 @@ import { VERSION } from "./version"; export const INSTRUMENTATION_NAME = "@traceai/claude-agent-sdk"; -/** fi-core env keys, read with fi-core's rule (`value.toLowerCase() === "true"`). */ +/** + * fi-core env keys. Read fail-closed, unlike fi-core (`value.toLowerCase() === + * "true"`, so `1`, `yes` or ` true` would turn capture ON): only an explicit + * `false` (trimmed, any case) opts in to content capture. + */ const FI_HIDE_INPUTS = "FI_HIDE_INPUTS"; const FI_HIDE_OUTPUTS = "FI_HIDE_OUTPUTS"; @@ -27,9 +31,10 @@ export interface ClaudeAgentSDKInstrumentationConfig { traceConfig?: TraceConfigOptions; } -function envFlag(key: string): boolean | undefined { +/** `false` only for an explicit `false`; any other value (or none) keeps content hidden. */ +function envHide(key: string): boolean { const value = typeof process !== "undefined" ? process.env?.[key] : undefined; - return value === undefined ? undefined : value.toLowerCase() === "true"; + return !(typeof value === "string" && value.trim().toLowerCase() === "false"); } /** @@ -38,8 +43,8 @@ function envFlag(key: string): boolean | undefined { */ export function resolveContentPolicy(traceConfig?: TraceConfigOptions): ContentPolicy { return { - hideInputs: traceConfig?.hideInputs ?? envFlag(FI_HIDE_INPUTS) ?? true, - hideOutputs: traceConfig?.hideOutputs ?? envFlag(FI_HIDE_OUTPUTS) ?? true, + hideInputs: traceConfig?.hideInputs ?? envHide(FI_HIDE_INPUTS), + hideOutputs: traceConfig?.hideOutputs ?? envHide(FI_HIDE_OUTPUTS), }; } From 14e3adf19bec1a54985282f4e53028d1e14e352e Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 4 Oct 2026 16:50:47 -0700 Subject: [PATCH 06/19] fix(claude-agent-sdk): end spans when a Query is closed or disposed The Query proxy only intercepted next/return/throw/asyncIterator. Query.close() (sdk.d.ts:3236-3243, the SDK's abort path) and Symbol.asyncDispose (`await using`) went straight to the original, so a query stopped mid-run never finished its spans and the trace was never exported. Both are now intercepted: the open spans are ended as cancelled (ERROR, claude_agent.cancelled=true), then the call is forwarded. After a completed stream the call changes nothing. Tests: jest close()/asyncDispose cases on the fake Query, and a real-SDK contract scenario (run_real_sdk.mjs SCENARIO=close) that stops two real CLI queries mid-tool. That scenario exported 0 spans before the fix. The real-SDK contract test now also asserts the x-secret-key header. The fake Query gains close/asyncDispose/backgroundTasks and a per-message delay. approved: Nikhil 2026-10-03 blanket Refs: TH-8235 --- .../traceai_claude_agent_sdk/README.md | 4 +- .../contract/run_real_sdk.mjs | 112 ++++++++++++------ .../contract/test_harness_contract.py | 77 +++++++++--- .../src/__tests__/fixtures/fakeQuery.ts | 35 +++++- .../src/__tests__/wrapper.test.ts | 51 ++++++++ .../src/queryWrapper.ts | 30 ++++- 6 files changed, 251 insertions(+), 58 deletions(-) diff --git a/typescript/packages/traceai_claude_agent_sdk/README.md b/typescript/packages/traceai_claude_agent_sdk/README.md index 248bc4e40..e0c8a3af0 100644 --- a/typescript/packages/traceai_claude_agent_sdk/README.md +++ b/typescript/packages/traceai_claude_agent_sdk/README.md @@ -52,7 +52,7 @@ await shutdown(); // flush before a short script exits ### What the wrapper does and does not do - Calls the original `query()` with the same params object. Options (`resume`, `forkSession`, `sessionId`, `hooks`, `mcpServers`, `agents`, `abortController`, `allowedTools`, `permissionMode`, `maxTurns`, `env`, ...) are passed through untouched. The wrapper never sets `ANTHROPIC_BASE_URL`. -- Yields the same message objects, in order. Control methods on the returned `Query` (`interrupt()`, `setModel()`, ...) go to the original. +- Yields the same message objects, in order. Control methods on the returned `Query` (`interrupt()`, `setModel()`, ...) go to the original. `close()` and `Symbol.asyncDispose` first end the query's open spans as cancelled, then go to the original. - A tracing error never fails the agent. An exporter or collector failure is logged and does not reject the iterator. - `startup()` is not wrapped in 0.1.0. @@ -115,7 +115,7 @@ With 0.3.289 the CLI then calls `POST {ANTHROPIC_BASE_URL}/v1/messages?beta=true - **No traces.** `register()` needs `projectName` or `FI_PROJECT_NAME`. - **Zero spans from a short script.** `await shutdown()` before exit. - **Duplicate model spans.** Do not also run a provider instrumentor on the same calls. -- **A span left open after abort.** Every span for the query is ended with status ERROR and `claude_agent.cancelled=true` when its `AbortController` aborts. Anything else is a bug here. +- **A span left open after abort.** Every span for the query is ended with status ERROR and `claude_agent.cancelled=true` when its `AbortController` aborts, when you call `close()` on the returned `Query` (the SDK's abort path), or when `await using` disposes it (`Symbol.asyncDispose`). Calling `close()` after the stream has completed changes nothing. Anything else is a bug here. ## Tests diff --git a/typescript/packages/traceai_claude_agent_sdk/contract/run_real_sdk.mjs b/typescript/packages/traceai_claude_agent_sdk/contract/run_real_sdk.mjs index d8273c441..1540f385b 100644 --- a/typescript/packages/traceai_claude_agent_sdk/contract/run_real_sdk.mjs +++ b/typescript/packages/traceai_claude_agent_sdk/contract/run_real_sdk.mjs @@ -5,12 +5,17 @@ // loopback port, and HOME / CLAUDE_CONFIG_DIR are a fresh temp dir. // // The mock answers the first /v1/messages call with a Read tool_use for a temp -// file and the second with text, so the CLI runs one real tool. +// file and the second with text, so the CLI runs one real tool ("tool" and +// "close" scenarios). Other scenarios get a text reply to every call. // // Env: FI_BASE_URL, FI_API_KEY, FI_SECRET_KEY, FI_PROJECT_NAME (fi-core). // CLAUDE_AGENT_SDK_ENTRY (optional): absolute path to another SDK build's // sdk.mjs, to run the same contract against an older 0.3.x. -// Prints {"requests": [...], "messages": [...]} on stdout. +// SCENARIO (optional, default "tool"): +// tool one query() that runs the Read tool. +// close two query() calls stopped mid-tool: one with close(), one +// with Symbol.asyncDispose. +// Prints {"requests": [...], "messages": [...], "queries": [[...], ...]} on stdout. import http from "node:http"; import fs from "node:fs"; import os from "node:os"; @@ -24,6 +29,8 @@ const { query } = process.env.CLAUDE_AGENT_SDK_ENTRY const MOCK_MODEL = "claude-sonnet-4-5"; const PROMPT = "Read README.md and summarize it. SECRET_PROMPT_MARKER"; +const SCENARIO = process.env.SCENARIO || "tool"; +const TOOL_FLOW = SCENARIO === "tool" || SCENARIO === "close"; const workdir = fs.mkdtempSync(path.join(os.tmpdir(), "th8235-real-sdk-")); const readmePath = path.join(workdir, "README.md"); @@ -73,7 +80,7 @@ const server = http.createServer((req, res) => { }); if (req.method === "POST" && req.url.startsWith("/v1/messages") && !req.url.includes("count_tokens")) { const model = parsed.model || MOCK_MODEL; - if (!hasToolResult) { + if (TOOL_FLOW && !hasToolResult) { return sendSse(res, model, [ { type: "text", text: "Reading it. SECRET_ASSISTANT_TEXT_MARKER" }, { type: "tool_use", id: "toolu_mock_read", name: "Read", input: { file_path: readmePath } }, @@ -93,39 +100,74 @@ const tracedQuery = wrapQuery(query, { tracerProvider: provider }); const abortController = new AbortController(); const timer = setTimeout(() => abortController.abort(), 90_000); -const messages = []; -try { - const stream = tracedQuery({ - prompt: PROMPT, - options: { - model: MOCK_MODEL, - cwd: workdir, - maxTurns: 3, - allowedTools: ["Read"], - permissionMode: "default", - settingSources: [], - persistSession: false, - abortController, - // Options.env REPLACES the subprocess environment: pass everything the CLI needs. - env: { - PATH: process.env.PATH, - HOME: workdir, - CLAUDE_CONFIG_DIR: path.join(workdir, ".claude"), - ANTHROPIC_BASE_URL: mockOrigin, - ANTHROPIC_API_KEY: "sk-ant-PLACEHOLDER-not-a-key", - CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: "1", - DISABLE_TELEMETRY: "1", - DISABLE_ERROR_REPORTING: "1", - DISABLE_AUTOUPDATER: "1", - HTTPS_PROXY: "http://127.0.0.1:9", - HTTP_PROXY: "http://127.0.0.1:9", - NO_PROXY: "127.0.0.1,localhost", - }, + +function options(extra = {}) { + return { + model: MOCK_MODEL, + cwd: workdir, + maxTurns: 3, + allowedTools: ["Read"], + permissionMode: "default", + settingSources: [], + persistSession: false, + abortController, + // Options.env REPLACES the subprocess environment: pass everything the CLI needs. + env: { + PATH: process.env.PATH, + HOME: workdir, + CLAUDE_CONFIG_DIR: path.join(workdir, ".claude"), + ANTHROPIC_BASE_URL: mockOrigin, + ANTHROPIC_API_KEY: "sk-ant-PLACEHOLDER-not-a-key", + CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: "1", + DISABLE_TELEMETRY: "1", + DISABLE_ERROR_REPORTING: "1", + DISABLE_AUTOUPDATER: "1", + HTTPS_PROXY: "http://127.0.0.1:9", + HTTP_PROXY: "http://127.0.0.1:9", + NO_PROXY: "127.0.0.1,localhost", }, - }); - for await (const message of stream) { - messages.push(message); + ...extra, + }; +} + +async function drain(stream) { + const out = []; + for await (const message of stream) out.push(message); + return out; +} + +/** Read until the assistant asks for the Read tool (conversation, turn and tool spans open). */ +async function readUntilToolUse(stream) { + const out = []; + for (;;) { + const { value, done } = await stream.next(); + if (done) return out; + out.push(value); + const content = value.type === "assistant" ? value.message?.content ?? [] : []; + if (content.some((block) => block.type === "tool_use")) return out; } +} + +const SCENARIOS = { + async tool() { + return [await drain(tracedQuery({ prompt: PROMPT, options: options() }))]; + }, + async close() { + const closed = tracedQuery({ prompt: PROMPT, options: options() }); + const first = await readUntilToolUse(closed); + closed.close(); + const disposed = tracedQuery({ prompt: PROMPT, options: options() }); + const second = await readUntilToolUse(disposed); + await disposed[Symbol.asyncDispose](); + return [first, second]; + }, +}; + +const queries = []; +try { + const scenario = SCENARIOS[SCENARIO]; + if (!scenario) throw new Error(`unknown SCENARIO ${SCENARIO}`); + queries.push(...(await scenario())); } finally { clearTimeout(timer); await shutdown(provider); @@ -134,4 +176,4 @@ try { fs.rmSync(workdir, { recursive: true, force: true }); } -process.stdout.write(JSON.stringify({ mockOrigin, requests, messages })); +process.stdout.write(JSON.stringify({ mockOrigin, requests, messages: queries.flat(), queries })); diff --git a/typescript/packages/traceai_claude_agent_sdk/contract/test_harness_contract.py b/typescript/packages/traceai_claude_agent_sdk/contract/test_harness_contract.py index 3c7efd3a8..5399ff160 100644 --- a/typescript/packages/traceai_claude_agent_sdk/contract/test_harness_contract.py +++ b/typescript/packages/traceai_claude_agent_sdk/contract/test_harness_contract.py @@ -253,32 +253,83 @@ def _sdk_platform_binary_installed() -> bool: return bool(store and list(store.glob("claude-agent-sdk-*"))) -@pytest.mark.skipif(not _sdk_platform_binary_installed(), reason="claude-agent-sdk platform binary not installed") -def test_real_sdk_query_through_anthropic_base_url_mock(built_package: Path) -> None: - """The real SDK query() and bundled CLI, wrapped, against a loopback Messages API mock. +requires_real_sdk = pytest.mark.skipif( + not _sdk_platform_binary_installed(), reason="claude-agent-sdk platform binary not installed" +) - Mechanism from the ADR: Options.env.ANTHROPIC_BASE_URL. No Anthropic call: placeholder - key, dead-port HTTPS_PROXY, temp HOME. Spans go through fi-core to the shared Receiver. + +def _run_real_sdk(scenario: str, project: str, extra_env: Dict[str, str] | None = None): + """Run contract/run_real_sdk.mjs with SCENARIO against a fresh Receiver. + + Returns (stdout JSON, received spans, export request records). """ env = _base_env() env.update( { - "FI_BASE_URL": "", # replaced below once the Receiver is up "FI_API_KEY": PLACEHOLDER_API_KEY, "FI_SECRET_KEY": PLACEHOLDER_SECRET_KEY, - "FI_PROJECT_NAME": "th8235-real-sdk-contract", + "FI_PROJECT_NAME": project, + "SCENARIO": scenario, } ) if os.environ.get("CLAUDE_AGENT_SDK_ENTRY"): # Run the same contract against another installed SDK build (older 0.3.x matrix). env["CLAUDE_AGENT_SDK_ENTRY"] = os.environ["CLAUDE_AGENT_SDK_ENTRY"] + env.update(extra_env or {}) with Receiver() as receiver: env["FI_BASE_URL"] = receiver.origin result = run([NODE, str(PKG_DIR / "contract" / "run_real_sdk.mjs")], env=env, stdin=None, timeout=180) - _check(result, "node run_real_sdk.mjs") + _check(result, f"node run_real_sdk.mjs ({scenario})") spans = receiver.spans() exported = receiver.requests() - output = json.loads(result.stdout.decode("utf-8")) + return json.loads(result.stdout.decode("utf-8")), spans, exported + + +def _assert_exported_with_both_keys(exported: List[Dict[str, Any]], project: str) -> None: + assert exported, "exporter sent nothing" + for request in exported: + assert request["path"] == "/tracer/v1/traces" + assert request["headers"]["x-api-key"] == PLACEHOLDER_API_KEY + assert request["headers"]["x-secret-key"] == PLACEHOLDER_SECRET_KEY + assert request["resource_attributes"] + for resource in request["resource_attributes"]: + assert (resource["project_name"], resource["project_type"]) == (project, "observe") + + +@requires_real_sdk +def test_real_sdk_close_and_async_dispose_end_every_span_cancelled(built_package: Path) -> None: + """R3: Query.close() (the SDK abort path) and Symbol.asyncDispose bypass next/return/throw. + + Each query is stopped right after the CLI streamed the Read tool_use, so the + conversation, turn and tool spans are open. All of them must still be exported, + ended ERROR with claude_agent.cancelled=true. + """ + output, spans, exported = _run_real_sdk("close", "th8235-real-sdk-close") + closed, disposed = output["queries"] + for messages in (closed, disposed): + assert not [m for m in messages if m["type"] == "result"], "query finished before it was stopped" + + traces: Dict[str, List[Dict[str, Any]]] = {} + for span in spans: + traces.setdefault(span["traceId"], []).append(span) + assert len(traces) == 2, sorted(span["name"] for span in spans) + for trace_spans in traces.values(): + names = sorted(span["name"] for span in trace_spans) + assert names == sorted(["claude_agent.conversation", "claude_agent.assistant_turn", "tool.Read"]), names + for span in trace_spans: + assert _status_code(span) == 2, span["name"] + assert _attrs(span)["claude_agent.cancelled"] is True, span["name"] + _assert_exported_with_both_keys(exported, "th8235-real-sdk-close") + + +@requires_real_sdk +def test_real_sdk_query_through_anthropic_base_url_mock(built_package: Path) -> None: + """The real SDK query() and bundled CLI, wrapped, against a loopback Messages API mock. + + Mechanism from the ADR: Options.env.ANTHROPIC_BASE_URL. No Anthropic call: placeholder + key, dead-port HTTPS_PROXY, temp HOME. Spans go through fi-core to the shared Receiver. + """ + output, spans, exported = _run_real_sdk("tool", "th8235-real-sdk-contract") # The CLI called the mock (host root + /v1/messages), streaming, with the placeholder key. mock_requests = output["requests"] @@ -321,13 +372,7 @@ def test_real_sdk_query_through_anthropic_base_url_mock(built_package: Path) -> for marker in ("SECRET_PROMPT_MARKER", "SECRET_TOOL_OUTPUT_MARKER", "SECRET_ASSISTANT_TEXT_MARKER"): assert marker not in blob assert PLACEHOLDER_API_KEY not in blob and "PLACEHOLDER-not-a-key" not in blob - assert exported, "exporter sent nothing" - for request in exported: - assert request["path"] == "/tracer/v1/traces" - assert request["headers"]["x-api-key"] == PLACEHOLDER_API_KEY - assert request["resource_attributes"] - for resource in request["resource_attributes"]: - assert (resource["project_name"], resource["project_type"]) == ("th8235-real-sdk-contract", "observe") + _assert_exported_with_both_keys(exported, "th8235-real-sdk-contract") def _read_member(tar: tarfile.TarFile, member: Any) -> bytes: diff --git a/typescript/packages/traceai_claude_agent_sdk/src/__tests__/fixtures/fakeQuery.ts b/typescript/packages/traceai_claude_agent_sdk/src/__tests__/fixtures/fakeQuery.ts index 129dc129e..586e7b95a 100644 --- a/typescript/packages/traceai_claude_agent_sdk/src/__tests__/fixtures/fakeQuery.ts +++ b/typescript/packages/traceai_claude_agent_sdk/src/__tests__/fixtures/fakeQuery.ts @@ -8,6 +8,9 @@ export interface FakeQueryControl { /** Every params object the fake received, by reference. */ calls: { prompt: unknown; options?: Options }[]; interruptCalls: number; + closeCalls: number; + disposeCalls: number; + backgroundTasksCalls: (string | undefined)[]; } export interface FakeQueryBehavior { @@ -16,6 +19,10 @@ export interface FakeQueryBehavior { error?: unknown; /** Before yielding message `waitForAbortAt`, block until options.abortController aborts. */ waitForAbortAt?: number; + /** Wait this long before yielding each message, so span times are ordered. */ + delayMs?: number; + /** What `backgroundTasks()` resolves to (or rejects with, when an Error). */ + backgroundTasksResult?: boolean | Error; } export class FakeAbortError extends Error { @@ -29,7 +36,13 @@ export function makeFakeQuery( messages: SDKMessage[], behavior: FakeQueryBehavior = {}, ): { query: typeof query; control: FakeQueryControl } { - const control: FakeQueryControl = { calls: [], interruptCalls: 0 }; + const control: FakeQueryControl = { + calls: [], + interruptCalls: 0, + closeCalls: 0, + disposeCalls: 0, + backgroundTasksCalls: [], + }; const fake = (params: { prompt: string | AsyncIterable; options?: Options }): Query => { control.calls.push(params); @@ -37,6 +50,9 @@ export function makeFakeQuery( async function* generate(): AsyncGenerator { for (let index = 0; index < messages.length; index += 1) { + if (behavior.delayMs) { + await new Promise((resolve) => setTimeout(resolve, behavior.delayMs)); + } if (behavior.throwAt === index) { throw behavior.error ?? new Error("fake query failed"); } @@ -58,6 +74,7 @@ export function makeFakeQuery( } const generator = generate(); + const asyncDispose = (Symbol as unknown as { asyncDispose?: symbol }).asyncDispose ?? Symbol.for("Symbol.asyncDispose"); const controlMethods = { interrupt: async () => { control.interruptCalls += 1; @@ -65,7 +82,21 @@ export function makeFakeQuery( }, setPermissionMode: async () => undefined, setModel: async () => undefined, - close: () => undefined, + backgroundTasks: async (toolUseId?: string) => { + control.backgroundTasksCalls.push(toolUseId); + const result = behavior.backgroundTasksResult ?? true; + if (result instanceof Error) throw result; + return result; + }, + // Like the SDK: close() and asyncDispose stop the stream; no further messages. + close: () => { + control.closeCalls += 1; + void generator.return(undefined); + }, + [asyncDispose]: async () => { + control.disposeCalls += 1; + await generator.return(undefined); + }, }; return Object.assign(generator, controlMethods) as unknown as Query; }; diff --git a/typescript/packages/traceai_claude_agent_sdk/src/__tests__/wrapper.test.ts b/typescript/packages/traceai_claude_agent_sdk/src/__tests__/wrapper.test.ts index 2b6bde1ff..1f4131fb8 100644 --- a/typescript/packages/traceai_claude_agent_sdk/src/__tests__/wrapper.test.ts +++ b/typescript/packages/traceai_claude_agent_sdk/src/__tests__/wrapper.test.ts @@ -484,6 +484,57 @@ describe("wrapQuery", () => { for (const span of spans) expect(span.ended).toBe(true); }); + const ASYNC_DISPOSE = + (Symbol as unknown as { asyncDispose?: symbol }).asyncDispose ?? Symbol.for("Symbol.asyncDispose"); + + it.each(["close", "asyncDispose"] as const)( + "R3: %s() ends every open span as cancelled, then forwards to the original", + async (method) => { + const { provider, exporter } = memoryProvider(); + const fake = makeFakeQuery(simpleToolJourney()); + const traced = wrapQuery(fake.query, { tracerProvider: provider }); + const q = traced({ prompt: PROMPT }); + // init, text, tool_use: the conversation, turn and tool spans are open. + for (let i = 0; i < 3; i += 1) await q.next(); + expect(exporter.getFinishedSpans()).toHaveLength(0); + + if (method === "close") { + q.close(); + } else { + await (q as unknown as Record Promise>)[ASYNC_DISPOSE](); + } + + expect(method === "close" ? fake.control.closeCalls : fake.control.disposeCalls).toBe(1); + const spans = exporter.getFinishedSpans(); + expect(spans.map((s) => s.name).sort()).toEqual( + ["claude_agent.assistant_turn", "claude_agent.conversation", "tool.Read"].sort(), + ); + for (const span of spans) { + expect(span.ended).toBe(true); + expect(span.status.code).toBe(SpanStatusCode.ERROR); + expect(span.attributes["claude_agent.cancelled"]).toBe(true); + } + // Nothing more is recorded once the query is closed. + await q.next(); + expect(exporter.getFinishedSpans()).toHaveLength(3); + }, + ); + + it("R3: close() after the stream completed leaves the finished spans as they were", async () => { + const { provider, exporter } = memoryProvider(); + const fake = makeFakeQuery(simpleToolJourney()); + const q = wrapQuery(fake.query, { tracerProvider: provider })({ prompt: PROMPT }); + await drain(q); + q.close(); + expect(fake.control.closeCalls).toBe(1); + const spans = exporter.getFinishedSpans(); + expect(spans).toHaveLength(4); + for (const span of spans) { + expect(span.status.code).toBe(SpanStatusCode.OK); + expect(span.attributes["claude_agent.cancelled"]).toBeUndefined(); + } + }); + it("ends open spans when the consumer breaks out early", async () => { const { provider, exporter } = memoryProvider(); const fake = makeFakeQuery(simpleToolJourney()); diff --git a/typescript/packages/traceai_claude_agent_sdk/src/queryWrapper.ts b/typescript/packages/traceai_claude_agent_sdk/src/queryWrapper.ts index 564b01799..5fe753ac9 100644 --- a/typescript/packages/traceai_claude_agent_sdk/src/queryWrapper.ts +++ b/typescript/packages/traceai_claude_agent_sdk/src/queryWrapper.ts @@ -5,8 +5,9 @@ * reference, never copied or modified) and returns a proxy around the `Query` * it got back. The proxy yields the same message objects the SDK yielded and * forwards every control method (`interrupt()`, `setModel()`, ...) to the - * original. Only `next`, `return`, `throw` and `Symbol.asyncIterator` are - * intercepted, to drive the span model in `spans.ts`. + * original. `next`, `return`, `throw` and `Symbol.asyncIterator` are + * intercepted to drive the span model in `spans.ts`; `close()` and + * `Symbol.asyncDispose` end the open spans as cancelled, then forward. */ import { Tracer, context as otelContext, diag } from "@opentelemetry/api"; import { ContentPolicy, QueryTracer, clockMs } from "./spans"; @@ -124,8 +125,31 @@ function proxyQuery>( if (property === "throw") return thr; if (property === Symbol.asyncIterator) return () => proxy; const value = Reflect.get(target, property, target); - return typeof value === "function" ? value.bind(target) : value; + if (typeof value !== "function") return value; + // close() is the SDK's abort path (sdk.d.ts Query.close) and asyncDispose + // runs on `await using`: neither goes through next/return/throw, so end + // every open span as cancelled before forwarding. + if (property === "close") { + return function close(this: unknown, ...args: unknown[]) { + safe(() => queryTracer.finish({ kind: "aborted", reason: "close" })); + return value.apply(target, args); + }; + } + if (isAsyncDisposeKey(property)) { + return function asyncDispose(this: unknown, ...args: unknown[]) { + safe(() => queryTracer.finish({ kind: "aborted", reason: "asyncDispose" })); + return value.apply(target, args); + }; + } + return value.bind(target); }, }); return proxy; } + +/** `Symbol.asyncDispose`, native or the `Symbol.for` polyfill the SDK installs on older Node. */ +function isAsyncDisposeKey(property: string | symbol): boolean { + if (typeof property !== "symbol") return false; + const native = (Symbol as unknown as { asyncDispose?: symbol }).asyncDispose; + return property === native || property === Symbol.for("Symbol.asyncDispose"); +} From 378acc594adea7f92bdabdb10f8c07df492f733f Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 4 Oct 2026 16:53:44 -0700 Subject: [PATCH 07/19] fix(claude-agent-sdk): start streaming-input turns after the previous result In a streaming-input session one query() yields a result per user turn. The next main-loop turn fell back to the query start time (spans.ts startTurn: nextTurnStartMs ?? this.startTimeMs), so the second turn overlapped the first. onResult now sets the main scope's nextTurnStartMs, the same way a user message does. Test: a two-result streaming-input journey (new fixture, numbers taken from the real CLI against the mock) checks turn 2 starts at or after turn 1's end; it failed by 28 ms before the fix. approved: Nikhil 2026-10-03 blanket Refs: TH-8235 --- .../src/__tests__/fixtures/messages.ts | 55 +++++++++++++++++++ .../src/__tests__/wrapper.test.ts | 25 +++++++++ .../traceai_claude_agent_sdk/src/spans.ts | 3 + 3 files changed, 83 insertions(+) diff --git a/typescript/packages/traceai_claude_agent_sdk/src/__tests__/fixtures/messages.ts b/typescript/packages/traceai_claude_agent_sdk/src/__tests__/fixtures/messages.ts index ca57a6a15..3532e5e8f 100644 --- a/typescript/packages/traceai_claude_agent_sdk/src/__tests__/fixtures/messages.ts +++ b/typescript/packages/traceai_claude_agent_sdk/src/__tests__/fixtures/messages.ts @@ -271,3 +271,58 @@ export function mcpErrorJourney(): SDKMessage[] { resultError(), ]; } + +type ModelUsage = SDKResultSuccess["modelUsage"][string]; + +/** One `modelUsage` entry (sdk.d.ts ModelUsage). */ +export function modelUsage( + input: number, + output: number, + costUSD: number, + cacheRead = 0, + cacheCreation = 0, +): ModelUsage { + return { + inputTokens: input, + outputTokens: output, + cacheReadInputTokens: cacheRead, + cacheCreationInputTokens: cacheCreation, + webSearchRequests: 0, + costUSD, + contextWindow: 200000, + maxOutputTokens: 64000, + }; +} + +/** `result.usage` for one main-loop turn (sdk.d.ts:5683: main agent loop only, per turn). */ +export function turnUsage(input: number, output: number): NonNullableUsage { + return { ...resultUsage, input_tokens: input, output_tokens: output, cache_read_input_tokens: 0, cache_creation_input_tokens: 0 }; +} + +/** Cost of one 20-in / 8-out call on claude-sonnet-4-5 list prices, as the real CLI reports it. */ +export const TURN_COST_USD = 0.00018; + +/** + * Streaming input: one query(), two user turns, two results. `total_cost_usd` + * and `modelUsage` are running totals (sdk.d.ts:5679, 5687); `usage` is per + * turn (sdk.d.ts:5683). Same numbers the real CLI reported against the mock. + */ +export function streamingInputJourney(): SDKMessage[] { + return [ + init(), + assistant("msg_s1", [text("First answer.")]), + resultSuccess("First answer.", { + num_turns: 1, + total_cost_usd: TURN_COST_USD, + usage: turnUsage(20, 8), + modelUsage: { [MODEL]: modelUsage(20, 8, TURN_COST_USD) }, + }), + assistant("msg_s2", [text("Second answer.")]), + resultSuccess("Second answer.", { + num_turns: 2, + total_cost_usd: 2 * TURN_COST_USD, + usage: turnUsage(20, 8), + modelUsage: { [MODEL]: modelUsage(40, 16, 2 * TURN_COST_USD) }, + }), + ]; +} diff --git a/typescript/packages/traceai_claude_agent_sdk/src/__tests__/wrapper.test.ts b/typescript/packages/traceai_claude_agent_sdk/src/__tests__/wrapper.test.ts index 1f4131fb8..142fe7e99 100644 --- a/typescript/packages/traceai_claude_agent_sdk/src/__tests__/wrapper.test.ts +++ b/typescript/packages/traceai_claude_agent_sdk/src/__tests__/wrapper.test.ts @@ -27,6 +27,7 @@ import { TOTAL_COST_USD, mcpErrorJourney, simpleToolJourney, + streamingInputJourney, subagentJourney, } from "./fixtures/messages"; import { StackContextManager, allAttributeText, byName, drain, memoryProvider, one, parentId } from "./helpers"; @@ -630,6 +631,30 @@ describe("wrapQuery", () => { expect(exporter.getFinishedSpans()).toHaveLength(0); }); + describe("streaming input (one query(), two user turns)", () => { + async function* userTurns() { + yield { type: "user" as const, message: { role: "user" as const, content: "one" }, parent_tool_use_id: null }; + yield { type: "user" as const, message: { role: "user" as const, content: "two" }, parent_tool_use_id: null }; + } + const hrMs = (t: [number, number]) => t[0] * 1e3 + t[1] / 1e6; + + it("R5: starts the second turn after the first result, so the turns do not overlap", async () => { + const { provider, exporter } = memoryProvider(); + const fake = makeFakeQuery(streamingInputJourney(), { delayMs: 5 }); + await drain(wrapQuery(fake.query, { tracerProvider: provider })({ prompt: userTurns() })); + const spans = exporter.getFinishedSpans(); + const conversation = one(spans, "claude_agent.conversation"); + const [first, second] = byName(spans, "claude_agent.assistant_turn").sort( + (a, b) => hrMs(a.startTime) - hrMs(b.startTime), + ); + expect(first.attributes["claude_agent.num_turns"]).toBe(1); + expect(second.attributes["claude_agent.num_turns"]).toBe(2); + expect(hrMs(first.startTime)).toBe(hrMs(conversation.startTime)); + expect(hrMs(second.startTime)).toBeGreaterThanOrEqual(hrMs(first.endTime)); + expect(hrMs(second.endTime)).toBeLessThanOrEqual(hrMs(conversation.endTime)); + }); + }); + it("does not trace a streaming-input prompt's content", async () => { const { provider, exporter } = memoryProvider(); const traced = wrapQuery(makeFakeQuery(simpleToolJourney()).query, { diff --git a/typescript/packages/traceai_claude_agent_sdk/src/spans.ts b/typescript/packages/traceai_claude_agent_sdk/src/spans.ts index def34d8c2..a5f0aa254 100644 --- a/typescript/packages/traceai_claude_agent_sdk/src/spans.ts +++ b/typescript/packages/traceai_claude_agent_sdk/src/spans.ts @@ -464,6 +464,9 @@ export class QueryTracer { if (main && !main.ended) { this.endTurn(main); } + // Streaming input: the next main-loop turn (if any) answers a user message + // sent after this result, so it cannot start before now. + this.scope(null).nextTurnStartMs = clockMs(); if (typeof message.session_id === "string" && message.session_id) { this.setSessionId(message.session_id); From 8b1ab15880d4fbd56fbddc712499d39bd4870306 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 4 Oct 2026 16:59:58 -0700 Subject: [PATCH 08/19] fix(claude-agent-sdk): keep backgrounded subagent spans open until task_notification A foreground subagent moved to the background arrives as system/task_updated with patch.is_backgrounded (sdk.d.ts:6059; task_id only), or follows an app call to Query.backgroundTasks() (sdk.d.ts:3234). Both were ignored: the subagent span ended at the "running in the background" tool_result and the subagent's later turns were parented to an ended span. - task_started records task_id -> tool_use_id; task_updated with patch.is_backgrounded marks that subagent background. - The Query proxy intercepts backgroundTasks(toolUseId?): matching open foreground subagents are marked before the call is forwarded, and unmarked if it rejects or resolves false. The caller gets the original promise. - A task_notification with status failed/stopped now makes the subagent span ERROR in the foreground case too (it ended OK before when the tool_result itself was not an error). README documents background subagents. approved: Nikhil 2026-10-03 blanket Refs: TH-8235 --- .../traceai_claude_agent_sdk/README.md | 4 + .../src/__tests__/fixtures/messages.ts | 41 ++++++++++ .../src/__tests__/wrapper.test.ts | 82 +++++++++++++++++++ .../src/queryWrapper.ts | 28 +++++++ .../traceai_claude_agent_sdk/src/spans.ts | 75 +++++++++++++++-- .../traceai_claude_agent_sdk/src/types.ts | 4 +- 6 files changed, 228 insertions(+), 6 deletions(-) diff --git a/typescript/packages/traceai_claude_agent_sdk/README.md b/typescript/packages/traceai_claude_agent_sdk/README.md index e0c8a3af0..b0e75f915 100644 --- a/typescript/packages/traceai_claude_agent_sdk/README.md +++ b/typescript/packages/traceai_claude_agent_sdk/README.md @@ -67,6 +67,10 @@ await shutdown(); // flush before a short script exits One API response that the SDK splits over several assistant messages (same `message.id`) is one turn. +### Background subagents + +A foreground subagent span ends with its `tool.Agent` / `tool.Task` result. A subagent that runs in the background stays open after that tool result (which is only the "running in the background" placeholder) until its `system/task_notification`, so its later turns and tools still nest under a live span. A subagent counts as background when the tool input has `run_in_background: true`, `task_started.is_backgrounded` is true, a later `task_updated` carries `patch.is_backgrounded: true`, or the app calls `Query.backgroundTasks()` (all foreground subagents, or the one whose `tool_use_id` it was given; undone if the call rejects or returns `false`). The subagent span is ERROR when the notification status is `failed` or `stopped` (`claude_agent.subagent.status` has the value). A background subagent with no notification by the time the query ends is closed as ERROR "Subagent span not completed". The subagent span can therefore end after its parent tool span. + | Field | Attributes | Source | |---|---|---| | Model | `gen_ai.request.model`, `claude_agent.model` | `options.model`, the init message, each assistant message | diff --git a/typescript/packages/traceai_claude_agent_sdk/src/__tests__/fixtures/messages.ts b/typescript/packages/traceai_claude_agent_sdk/src/__tests__/fixtures/messages.ts index 3532e5e8f..80f85c9d4 100644 --- a/typescript/packages/traceai_claude_agent_sdk/src/__tests__/fixtures/messages.ts +++ b/typescript/packages/traceai_claude_agent_sdk/src/__tests__/fixtures/messages.ts @@ -13,6 +13,7 @@ import type { SDKSystemMessage, SDKTaskNotificationMessage, SDKTaskStartedMessage, + SDKTaskUpdatedMessage, SDKUserMessage, } from "@anthropic-ai/claude-agent-sdk"; @@ -217,6 +218,11 @@ export function taskNotification( // Journeys // ---------------------------------------------------------------------------- +/** `system/task_updated` (sdk.d.ts SDKTaskUpdatedMessage): carries task_id only, no tool_use_id. */ +export function taskUpdated(taskId: string, patch: SDKTaskUpdatedMessage["patch"]): SDKTaskUpdatedMessage { + return { type: "system", subtype: "task_updated", task_id: taskId, patch, uuid: uuid(), session_id: SESSION_ID }; +} + export const TOOL_INPUT_MARKER = "SECRET_TOOL_INPUT_MARKER.md"; export const TOOL_OUTPUT_MARKER = "SECRET_TOOL_OUTPUT_MARKER"; export const ASSISTANT_TEXT_MARKER = "SECRET_ASSISTANT_TEXT_MARKER"; @@ -259,6 +265,41 @@ export function subagentJourney(): SDKMessage[] { ]; } +export const BACKGROUND_PLACEHOLDER = "Agent is running in the background. You will be notified when it completes."; + +/** + * J2b: a foreground subagent moved to the background mid-run, either by + * `system/task_updated` patch.is_backgrounded (sdk.d.ts:6059) or by the app + * calling `Query.backgroundTasks()` (sdk.d.ts:3234). The Agent tool_result is + * the "running in the background" placeholder; the subagent keeps working and + * settles with `task_notification`. + */ +export function backgroundedSubagentJourney( + options: { taskUpdated?: boolean; status?: SDKTaskNotificationMessage["status"] } = {}, +): SDKMessage[] { + const { taskUpdated: withTaskUpdated = true, status = "completed" } = options; + return [ + init(), + assistant("msg_10", [ + toolUse(AGENT_TOOL_ID, "Agent", { + description: "Review the diff", + prompt: "Look for TODOs. SECRET_SUBAGENT_PROMPT_MARKER", + subagent_type: "code-reviewer", + run_in_background: false, + }), + ]), + taskStarted(AGENT_TOOL_ID, { is_backgrounded: false }), + assistant("msg_11", [toolUse(SUBAGENT_GREP_ID, "Grep", { pattern: "TODO" })], AGENT_TOOL_ID), + ...(withTaskUpdated ? [taskUpdated("task-1", { is_backgrounded: true })] : []), + toolResult(AGENT_TOOL_ID, BACKGROUND_PLACEHOLDER), + assistant("msg_13", [text("The reviewer is running in the background.")]), + toolResult(SUBAGENT_GREP_ID, "src/a.ts:3: TODO", AGENT_TOOL_ID), + assistant("msg_12", [text("Found one TODO.")], AGENT_TOOL_ID), + taskNotification(AGENT_TOOL_ID, status), + resultSuccess("The reviewer is running in the background.", { num_turns: 2 }), + ]; +} + export const MCP_TOOL_ID = "toolu_mcp_1"; /** J3: an MCP tool that errors, then the run stops on max turns. */ diff --git a/typescript/packages/traceai_claude_agent_sdk/src/__tests__/wrapper.test.ts b/typescript/packages/traceai_claude_agent_sdk/src/__tests__/wrapper.test.ts index 142fe7e99..9f9c5ee17 100644 --- a/typescript/packages/traceai_claude_agent_sdk/src/__tests__/wrapper.test.ts +++ b/typescript/packages/traceai_claude_agent_sdk/src/__tests__/wrapper.test.ts @@ -25,10 +25,12 @@ import { TOOL_INPUT_MARKER, TOOL_OUTPUT_MARKER, TOTAL_COST_USD, + backgroundedSubagentJourney, mcpErrorJourney, simpleToolJourney, streamingInputJourney, subagentJourney, + taskNotification, } from "./fixtures/messages"; import { StackContextManager, allAttributeText, byName, drain, memoryProvider, one, parentId } from "./helpers"; @@ -341,6 +343,86 @@ describe("wrapQuery", () => { } expect(subagent.status.code).toBe(SpanStatusCode.OK); }); + + const endMsOf = (s: ReadableSpan) => s.endTime[0] * 1e3 + s.endTime[1] / 1e6; + + /** Every subagent turn and tool sits under the subagent span and ends before it. */ + function expectSubagentOutlivesItsWork(spans: ReadableSpan[]) { + const agentTool = one(spans, "tool.Agent"); + const subagent = one(spans, "claude_agent.subagent.code-reviewer"); + const grep = one(spans, "tool.Grep"); + const subTurns = byName(spans, "claude_agent.assistant_turn").filter( + (t) => t.attributes["claude_agent.parent_tool_use_id"] === AGENT_TOOL_ID, + ); + expect(subTurns).toHaveLength(2); + expect(endMsOf(subagent)).toBeGreaterThan(endMsOf(agentTool)); + for (const turn of subTurns) { + expect(parentId(turn)).toBe(subagent.spanContext().spanId); + expect(endMsOf(turn)).toBeLessThanOrEqual(endMsOf(subagent)); + } + expect(endMsOf(grep)).toBeLessThanOrEqual(endMsOf(subagent)); + return subagent; + } + + it("R6: keeps a subagent moved to the background (task_updated patch.is_backgrounded) open until task_notification", async () => { + const { provider, exporter } = memoryProvider(); + const fake = makeFakeQuery(backgroundedSubagentJourney(), { delayMs: 2 }); + await drain(wrapQuery(fake.query, { tracerProvider: provider })({ prompt: PROMPT })); + const subagent = expectSubagentOutlivesItsWork(exporter.getFinishedSpans()); + expect(subagent.status.code).toBe(SpanStatusCode.OK); + expect(subagent.attributes["claude_agent.subagent.status"]).toBe("completed"); + }); + + it("R6: keeps subagents open after the app calls Query.backgroundTasks()", async () => { + const { provider, exporter } = memoryProvider(); + const fake = makeFakeQuery(backgroundedSubagentJourney({ taskUpdated: false }), { delayMs: 2 }); + const q = wrapQuery(fake.query, { tracerProvider: provider })({ prompt: PROMPT }); + for (let i = 0; i < 4; i += 1) await q.next(); // init, Agent tool_use, task_started, subagent Grep turn + await expect(q.backgroundTasks()).resolves.toBe(true); + expect(fake.control.backgroundTasksCalls).toEqual([undefined]); + await drain(q); + const subagent = expectSubagentOutlivesItsWork(exporter.getFinishedSpans()); + expect(subagent.status.code).toBe(SpanStatusCode.OK); + }); + + it("R6: treats a subagent as foreground again when backgroundTasks() rejects", async () => { + const { provider, exporter } = memoryProvider(); + const disabled = new Error("background tasks are disabled"); + const fake = makeFakeQuery(subagentJourney(), { backgroundTasksResult: disabled }); + const q = wrapQuery(fake.query, { tracerProvider: provider })({ prompt: PROMPT }); + for (let i = 0; i < 3; i += 1) await q.next(); + await expect(q.backgroundTasks(AGENT_TOOL_ID)).rejects.toBe(disabled); + await drain(q); + const spans = exporter.getFinishedSpans(); + expect(endMsOf(one(spans, "claude_agent.subagent.code-reviewer"))).toBeLessThanOrEqual( + endMsOf(one(spans, "tool.Agent")), + ); + expect(one(spans, "claude_agent.subagent.code-reviewer").status.code).toBe(SpanStatusCode.OK); + }); + + it.each(["failed", "stopped"] as const)( + "marks a foreground subagent ERROR when task_notification reports %s", + async (status) => { + const journey = subagentJourney().map((m) => + m.type === "system" && m.subtype === "task_notification" ? taskNotification(AGENT_TOOL_ID, status) : m, + ); + const { spans } = await run(journey); + const subagent = one(spans, "claude_agent.subagent.code-reviewer"); + expect(subagent.attributes["claude_agent.subagent.status"]).toBe(status); + expect(subagent.status.code).toBe(SpanStatusCode.ERROR); + expect(subagent.status.message).toBe(`subagent ${status}`); + }, + ); + + it.each(["failed", "stopped"] as const)( + "marks a backgrounded subagent ERROR when task_notification reports %s", + async (status) => { + const { spans } = await run(backgroundedSubagentJourney({ status })); + const subagent = one(spans, "claude_agent.subagent.code-reviewer"); + expect(subagent.attributes["claude_agent.subagent.status"]).toBe(status); + expect(subagent.status.code).toBe(SpanStatusCode.ERROR); + }, + ); }); describe("J3: MCP tool error and error result", () => { diff --git a/typescript/packages/traceai_claude_agent_sdk/src/queryWrapper.ts b/typescript/packages/traceai_claude_agent_sdk/src/queryWrapper.ts index 5fe753ac9..ef663d555 100644 --- a/typescript/packages/traceai_claude_agent_sdk/src/queryWrapper.ts +++ b/typescript/packages/traceai_claude_agent_sdk/src/queryWrapper.ts @@ -141,6 +141,34 @@ function proxyQuery>( return value.apply(target, args); }; } + // backgroundTasks() answers foreground tasks with a "running in the + // background" tool_result: keep their subagent spans open until + // task_notification. Undo if the call rejects or matched nothing. + if (property === "backgroundTasks") { + return function backgroundTasks(this: unknown, ...args: unknown[]) { + let marked: string[] = []; + safe(() => { + marked = queryTracer.markBackgroundRequested(typeof args[0] === "string" ? args[0] : undefined); + }); + let result: unknown; + try { + result = value.apply(target, args); + } catch (error) { + safe(() => queryTracer.cancelBackgroundRequest(marked)); + throw error; + } + if (marked.length > 0 && result && typeof (result as Promise).then === "function") { + // A side branch: the caller still gets the original promise and its rejection. + (result as Promise).then( + (ok) => { + if (ok === false) safe(() => queryTracer.cancelBackgroundRequest(marked)); + }, + () => safe(() => queryTracer.cancelBackgroundRequest(marked)), + ); + } + return result; + }; + } return value.bind(target); }, }); diff --git a/typescript/packages/traceai_claude_agent_sdk/src/spans.ts b/typescript/packages/traceai_claude_agent_sdk/src/spans.ts index a5f0aa254..81d829fd4 100644 --- a/typescript/packages/traceai_claude_agent_sdk/src/spans.ts +++ b/typescript/packages/traceai_claude_agent_sdk/src/spans.ts @@ -113,8 +113,12 @@ interface ToolState { interface SubagentState { span: Span; startMs: number; + /** In the background (run_in_background, task_started, task_updated): ends at task_notification. */ backgrounded: boolean; + /** The app called Query.backgroundTasks() and it has not failed (yet). */ + backgroundRequested: boolean; toolEnded: boolean; + toolIsError: boolean; status?: string; ended: boolean; } @@ -231,6 +235,8 @@ export class QueryTracer { private readonly scopes = new Map(); private readonly tools = new Map(); private readonly subagents = new Map(); + /** task_id -> tool_use_id from task_started; task_updated carries only task_id. */ + private readonly taskToolUseIds = new Map(); constructor(args: { tracer: Tracer; @@ -549,6 +555,9 @@ export class QueryTracer { break; } case "task_started": { + if (typeof message.task_id === "string" && typeof message.tool_use_id === "string") { + this.taskToolUseIds.set(message.task_id, message.tool_use_id); + } const sub = message.tool_use_id ? this.subagents.get(message.tool_use_id) : undefined; if (sub && !sub.ended) { if (typeof message.task_id === "string") { @@ -560,6 +569,17 @@ export class QueryTracer { } break; } + case "task_updated": { + // A foreground task moved to the background (sdk.d.ts:6059). Its tool_result + // becomes a "running in the background" placeholder; the subagent keeps + // running until task_notification. + const id = typeof message.task_id === "string" ? this.taskToolUseIds.get(message.task_id) : undefined; + const sub = id ? this.subagents.get(id) : undefined; + if (sub && !sub.ended && message.patch?.is_backgrounded === true) { + sub.backgrounded = true; + } + break; + } case "task_notification": { const id = message.tool_use_id; const sub = id ? this.subagents.get(id) : undefined; @@ -567,8 +587,8 @@ export class QueryTracer { sub.status = message.status; sub.span.setAttribute(T.SUBAGENT_STATUS, message.status); // A backgrounded subagent outlives its tool_result; it ends here. - if (sub.backgrounded || sub.toolEnded) { - this.endSubagent(id!, message.status !== "completed"); + if (sub.backgrounded || sub.backgroundRequested || sub.toolEnded) { + this.endSubagent(id!, sub.toolIsError); } } break; @@ -716,7 +736,8 @@ export class QueryTracer { const sub = this.subagents.get(id); if (sub && !sub.ended) { sub.toolEnded = true; - if (!sub.backgrounded) { + sub.toolIsError = isError; + if (!sub.backgrounded && !sub.backgroundRequested) { this.endSubagent(id, isError); } } @@ -783,11 +804,52 @@ export class QueryTracer { span, startMs, backgrounded: fields.run_in_background === true, + backgroundRequested: false, toolEnded: false, + toolIsError: false, ended: false, }); } + /** + * The app called `Query.backgroundTasks(toolUseId?)` (sdk.d.ts:3234). Mark the + * matching foreground subagents as background now: their "running in the + * background" tool_result may arrive before any task_updated. Returns the ids + * marked, for `cancelBackgroundRequest` if the call fails. + */ + markBackgroundRequested(toolUseId?: string): string[] { + const marked: string[] = []; + if (this.finished) return marked; + try { + for (const [id, sub] of this.subagents) { + if (sub.ended || sub.toolEnded || sub.backgrounded || sub.backgroundRequested) continue; + if (toolUseId !== undefined && id !== toolUseId) continue; + sub.backgroundRequested = true; + marked.push(id); + } + } catch (error) { + diag.debug(`@traceai/claude-agent-sdk: failed to record backgroundTasks(): ${error}`); + } + return marked; + } + + /** `backgroundTasks()` rejected or matched nothing: those subagents stay foreground. */ + cancelBackgroundRequest(ids: string[]): void { + if (this.finished) return; + try { + for (const id of ids) { + const sub = this.subagents.get(id); + if (!sub || sub.ended || !sub.backgroundRequested) continue; + sub.backgroundRequested = false; + if (sub.toolEnded && !sub.backgrounded) { + this.endSubagent(id, sub.toolIsError); + } + } + } catch (error) { + diag.debug(`@traceai/claude-agent-sdk: failed to record backgroundTasks() failure: ${error}`); + } + } + private endSubagent(toolUseId: string, isError: boolean): void { const sub = this.subagents.get(toolUseId); if (!sub || sub.ended) return; @@ -797,11 +859,14 @@ export class QueryTracer { this.endTurn(scopeState.currentTurn); } sub.ended = true; + // task_notification status failed / stopped is an error even when the + // tool_result itself was not. + const failed = isError || (sub.status !== undefined && sub.status !== "completed"); const endMs = clockMs(); sub.span.setAttribute(A.TOOL_DURATION_MS, endMs - sub.startMs); - sub.span.setAttribute(A.TOOL_IS_ERROR, isError); + sub.span.setAttribute(A.TOOL_IS_ERROR, failed); sub.span.setStatus( - isError + failed ? { code: SpanStatusCode.ERROR, message: sub.status ? `subagent ${sub.status}` : "subagent error" } : { code: SpanStatusCode.OK }, ); diff --git a/typescript/packages/traceai_claude_agent_sdk/src/types.ts b/typescript/packages/traceai_claude_agent_sdk/src/types.ts index 87ebdac64..e0f0a5df5 100644 --- a/typescript/packages/traceai_claude_agent_sdk/src/types.ts +++ b/typescript/packages/traceai_claude_agent_sdk/src/types.ts @@ -79,12 +79,14 @@ export interface SystemMessageLike { model?: string; permissionMode?: string; mcp_servers?: { name: string; status?: string }[]; - /** task_started / task_notification / task_progress */ + /** task_started / task_notification / task_progress / task_updated */ task_id?: string; tool_use_id?: string; status?: string; subagent_type?: string; is_backgrounded?: boolean; + /** task_updated: the TaskState fields that changed (sdk.d.ts SDKTaskUpdatedMessage). */ + patch?: { is_backgrounded?: boolean; status?: string }; } export type SDKMessageLike = From c6fe543bd6d77d82f0471356d5a053795c54bc05 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 4 Oct 2026 19:05:13 -0700 Subject: [PATCH 09/19] fix(claude-agent-sdk): promote only new spend; tokens from modelUsage R1: total_cost_usd is a running total that a resumed, continued or forked session carries forward (sdk.d.ts:5679), and the collector promotes gen_ai.cost.total / gen_ai.usage.* on any span and Observe sums them per trace and per session.id. The wrapper copied the cumulative value, so each resume re-counted earlier spend (real CLI: 0.00018 then 0.00036 promoted for 0.00036 spent). - A process-local LRU (1000 sessions) maps session id -> last running totals. The promoted keys carry the cumulative value minus that baseline: zero for a new session, the session's own totals for resume/continue, the parent's for a fork. A drop within one query (/clear) counts both sides. - No known baseline (resume after restart, continue+forkSession, or a first result below the saved totals): no promoted key and no claude_agent.cost.total_usd; claude_agent.usage.baseline_unknown=true. - Running totals always go on unpromoted claude_agent.cumulative.* keys. - options.continue is now labelled session.is_resumed=true / is_new=false. - A result with nothing usable, or a zeroed crash result, writes no usage key and leaves the baseline alone. R2: tokens came from result.usage, which is main-loop only and per turn in streaming input (sdk.d.ts:5683). They now come from the latest result's modelUsage summed over models (input, output, cache read, cache creation), with the same delta handling. Streaming input stays one conversation span per query() (recorded in the README; Python's ClaudeSDKClient makes one per user turn). Tests: usage.test.ts (21 jest: streaming totals, multi-model modelUsage, no usage on turn/tool/subagent spans, absent usage, resume/continue/fork deltas, unknown baselines, /clear, crash result, LRU). Real-CLI contract scenarios resume, continue_fork, restart (two processes, shared HOME) and streaming assert the promoted keys sum to the final cumulative totals; all four failed before the fix. Fixture results now carry modelUsage. approved: Nikhil 2026-10-03 blanket Refs: TH-8235 --- .../traceai_claude_agent_sdk/README.md | 17 +- .../contract/run_real_sdk.mjs | 70 +++- .../contract/test_harness_contract.py | 156 ++++++++ .../src/__tests__/fixtures/messages.ts | 5 +- .../src/__tests__/usage.test.ts | 343 ++++++++++++++++++ .../src/attributes.ts | 15 + .../traceai_claude_agent_sdk/src/spans.ts | 247 +++++++++++-- .../traceai_claude_agent_sdk/src/types.ts | 13 + 8 files changed, 833 insertions(+), 33 deletions(-) create mode 100644 typescript/packages/traceai_claude_agent_sdk/src/__tests__/usage.test.ts diff --git a/typescript/packages/traceai_claude_agent_sdk/README.md b/typescript/packages/traceai_claude_agent_sdk/README.md index b0e75f915..eb708c580 100644 --- a/typescript/packages/traceai_claude_agent_sdk/README.md +++ b/typescript/packages/traceai_claude_agent_sdk/README.md @@ -75,9 +75,9 @@ A foreground subagent span ends with its `tool.Agent` / `tool.Task` result. A su |---|---|---| | Model | `gen_ai.request.model`, `claude_agent.model` | `options.model`, the init message, each assistant message | | Provider | `gen_ai.provider.name` = `anthropic`, or `custom` when `ANTHROPIC_BASE_URL` (from `options.env`, else `process.env`) is not an `anthropic.com` host | | -| Tokens | `gen_ai.usage.input_tokens`, `output_tokens`, `total_tokens`; `gen_ai.usage.cache_read_tokens` / `cache_creation_tokens` when present | result message, on the conversation span | -| Cost | `claude_agent.cost.total_usd` and `gen_ai.cost.total` | result `total_cost_usd`; never computed here | -| Session | `session.id`, `claude_agent.session.id`, `claude_agent.session_id`; `claude_agent.session.fork_from` on fork | init / result `session_id`, `options.sessionId`, `options.resume` | +| Tokens | `gen_ai.usage.input_tokens`, `output_tokens`, `total_tokens`, `cache_read_tokens`, `cache_creation_tokens` | result `modelUsage` summed over every model, on the conversation span only. See "Usage and cost" | +| Cost | `claude_agent.cost.total_usd` and `gen_ai.cost.total` | result `total_cost_usd`; never computed here. See "Usage and cost" | +| Session | `session.id`, `claude_agent.session.id`, `claude_agent.session_id`; `claude_agent.session.is_new` / `is_resumed` (`resume` or `continue`); `claude_agent.session.fork_from` on fork | init / result `session_id`, `options.sessionId`, `options.resume`, `options.continue` | | Tools | `claude_agent.tool.name`, `gen_ai.tool.name`, `use_id`, `source` (`builtin` / `mcp` / `custom`), `is_error`, `duration_ms` | `tool_use` / `tool_result` blocks | | Errors | span status ERROR; `claude_agent.error.type` / `error.message`; `claude_agent.cancelled=true` on abort | error result, thrown error, `is_error` tool results, `AbortController` | @@ -85,6 +85,17 @@ The `claude_agent.*` names are the Python package's names (`_attributes.py`); a An app-set fi-core `session.id` / `user.id` (fi-core context helpers) is kept; the SDK session id still goes to `claude_agent.session.id`. +## Usage and cost + +The Future AGI collector promotes `gen_ai.usage.*` and `gen_ai.cost.total` on any span, and Observe sums them over a trace and over a `session.id`. So these keys are written once, on the conversation span, and hold only the spend that is new in this `query()` call. Turn, tool and subagent spans carry none of them. + +- Tokens come from the latest result's `modelUsage`, summed over every model (main loop, subagents, compaction). Cost comes from `total_cost_usd`. Both are running totals for the session (`sdk.d.ts:5679`, `5687`). `result.usage` is not read: the SDK documents it as main-loop only and per turn in streaming-input mode (`sdk.d.ts:5683`). This replaces the spec's "result message usage" mapping. +- Streaming input (one `query()`, several user turns) yields one result per turn. The conversation span keeps the latest running totals, so tokens and cost cover every turn. It stays one conversation span per `query()`; the Python `ClaudeSDKClient` path makes one per user turn. +- A resumed (`resume`), continued (`continue`) or forked (`forkSession`) session's first result already carries the earlier turns. The wrapper keeps an in-process map of session id to the last totals it saw (at most 1000 sessions) and writes only the difference. A fork starts from its parent's totals. A drop in the running total within one query (a `/clear`) counts the spend on both sides of it. +- When there is no baseline in this process (resume after a restart, `continue` + `forkSession`, or a first result below the saved totals), no `gen_ai.usage.*` / `gen_ai.cost.total` / `claude_agent.cost.total_usd` is written and `claude_agent.usage.baseline_unknown=true`. +- Every conversation span with a result also carries the running totals on unpromoted keys: `claude_agent.cumulative.cost_usd`, `input_tokens`, `output_tokens`, `cache_read_tokens`, `cache_creation_tokens`. +- A result with no cost and no `modelUsage`, or a zeroed crash result, writes none of these keys and leaves the session's saved totals unchanged. + ## Privacy: content is off by default Prompts, system prompts, tool inputs, tool outputs, assistant text and the final result can contain source code, file contents or secrets. They are **not** recorded unless you opt in: diff --git a/typescript/packages/traceai_claude_agent_sdk/contract/run_real_sdk.mjs b/typescript/packages/traceai_claude_agent_sdk/contract/run_real_sdk.mjs index 1540f385b..04b07d84c 100644 --- a/typescript/packages/traceai_claude_agent_sdk/contract/run_real_sdk.mjs +++ b/typescript/packages/traceai_claude_agent_sdk/contract/run_real_sdk.mjs @@ -15,6 +15,14 @@ // tool one query() that runs the Read tool. // close two query() calls stopped mid-tool: one with close(), one // with Symbol.asyncDispose. +// resume two query() calls; the second resumes the first's session. +// continue_fork three query() calls: a new session, options.continue, +// then resume + forkSession from the first session id. +// restart one query() per process, sharing WORKDIR (HOME): PHASE=first +// starts a session, PHASE=second resumes RESUME_SESSION_ID. +// streaming one query() with an AsyncIterable prompt of two user turns; +// the second turn is sent after the first result. +// WORKDIR (optional): use and keep this directory instead of a temp dir. // Prints {"requests": [...], "messages": [...], "queries": [[...], ...]} on stdout. import http from "node:http"; import fs from "node:fs"; @@ -32,7 +40,9 @@ const PROMPT = "Read README.md and summarize it. SECRET_PROMPT_MARKER"; const SCENARIO = process.env.SCENARIO || "tool"; const TOOL_FLOW = SCENARIO === "tool" || SCENARIO === "close"; -const workdir = fs.mkdtempSync(path.join(os.tmpdir(), "th8235-real-sdk-")); +const keepWorkdir = Boolean(process.env.WORKDIR); +const workdir = keepWorkdir ? process.env.WORKDIR : fs.mkdtempSync(path.join(os.tmpdir(), "th8235-real-sdk-")); +fs.mkdirSync(workdir, { recursive: true }); const readmePath = path.join(workdir, "README.md"); fs.writeFileSync(readmePath, "# Demo\nSECRET_TOOL_OUTPUT_MARKER\n"); @@ -161,8 +171,64 @@ const SCENARIOS = { await disposed[Symbol.asyncDispose](); return [first, second]; }, + async resume() { + const first = await drain(tracedQuery({ prompt: "First question.", options: options({ persistSession: true }) })); + const second = await drain( + tracedQuery({ prompt: "Second question.", options: options({ persistSession: true, resume: sessionOf(first) }) }), + ); + return [first, second]; + }, + async continue_fork() { + const first = await drain(tracedQuery({ prompt: "First question.", options: options({ persistSession: true }) })); + const second = await drain( + tracedQuery({ prompt: "Second question.", options: options({ persistSession: true, continue: true }) }), + ); + const third = await drain( + tracedQuery({ + prompt: "Third question.", + options: options({ persistSession: true, resume: sessionOf(first), forkSession: true }), + }), + ); + return [first, second, third]; + }, + async restart() { + const resume = process.env.PHASE === "second" ? process.env.RESUME_SESSION_ID : undefined; + if (process.env.PHASE === "second" && !resume) throw new Error("PHASE=second needs RESUME_SESSION_ID"); + return [ + await drain( + tracedQuery({ + prompt: resume ? "Second question." : "First question.", + options: options({ persistSession: true, ...(resume ? { resume } : {}) }), + }), + ), + ]; + }, + async streaming() { + let resultSeen; + let waitForResult = new Promise((resolve) => (resultSeen = resolve)); + const userTurn = (content) => ({ type: "user", message: { role: "user", content }, parent_tool_use_id: null }); + async function* prompt() { + yield userTurn("First question."); + await waitForResult; + waitForResult = new Promise((resolve) => (resultSeen = resolve)); + yield userTurn("Second question."); + await waitForResult; + } + const out = []; + for await (const message of tracedQuery({ prompt: prompt(), options: options() })) { + out.push(message); + if (message.type === "result") resultSeen(); + } + return [out]; + }, }; +function sessionOf(messages) { + const init = messages.find((m) => m.type === "system" && m.subtype === "init"); + if (!init) throw new Error("no init message"); + return init.session_id; +} + const queries = []; try { const scenario = SCENARIOS[SCENARIO]; @@ -173,7 +239,7 @@ try { await shutdown(provider); await provider.shutdown(); server.close(); - fs.rmSync(workdir, { recursive: true, force: true }); + if (!keepWorkdir) fs.rmSync(workdir, { recursive: true, force: true }); } process.stdout.write(JSON.stringify({ mockOrigin, requests, messages: queries.flat(), queries })); diff --git a/typescript/packages/traceai_claude_agent_sdk/contract/test_harness_contract.py b/typescript/packages/traceai_claude_agent_sdk/contract/test_harness_contract.py index 5399ff160..c60f64489 100644 --- a/typescript/packages/traceai_claude_agent_sdk/contract/test_harness_contract.py +++ b/typescript/packages/traceai_claude_agent_sdk/contract/test_harness_contract.py @@ -322,6 +322,162 @@ def test_real_sdk_close_and_async_dispose_end_every_span_cancelled(built_package _assert_exported_with_both_keys(exported, "th8235-real-sdk-close") +# Keys fi-collector promotes into hot columns on ANY span; Observe sums them per +# trace and per session.id, so they must equal new spend exactly once. +PROMOTED_PREFIXES = ("gen_ai.usage.", "llm.token_count.") +PROMOTED_KEYS = ("gen_ai.cost.total", "llm.cost.total") + + +def _promoted(attributes: Dict[str, Any]) -> Dict[str, Any]: + return {k: v for k, v in attributes.items() if k.startswith(PROMOTED_PREFIXES) or k in PROMOTED_KEYS} + + +def _results(messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + return [m for m in messages if m["type"] == "result"] + + +def _init_session(messages: List[Dict[str, Any]]) -> str: + return [m for m in messages if m["type"] == "system" and m.get("subtype") == "init"][0]["session_id"] + + +def _cumulative(result: Dict[str, Any]) -> Dict[str, Any]: + """The running totals a result carries: total_cost_usd and modelUsage summed over models.""" + usage = result["modelUsage"].values() + return { + "cost": result["total_cost_usd"], + "input": sum(u["inputTokens"] for u in usage), + "output": sum(u["outputTokens"] for u in usage), + } + + +def _conversations_in_query_order(spans: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + conversations = _by_name(spans, "claude_agent.conversation") + return sorted(conversations, key=lambda span: int(span["startTimeUnixNano"])) + + +def _assert_only_conversations_carry_usage(spans: List[Dict[str, Any]]) -> None: + for span in spans: + if span["name"] != "claude_agent.conversation": + assert not _promoted(_attrs(span)), (span["name"], _promoted(_attrs(span))) + + +def _assert_promoted_sum_equals(conversations: List[Dict[str, Any]], final: Dict[str, Any]) -> None: + attrs = [_attrs(c) for c in conversations] + assert sum(a["gen_ai.cost.total"] for a in attrs) == pytest.approx(final["cost"], rel=1e-9) + assert sum(a["gen_ai.usage.input_tokens"] for a in attrs) == final["input"] + assert sum(a["gen_ai.usage.output_tokens"] for a in attrs) == final["output"] + assert sum(a["gen_ai.usage.total_tokens"] for a in attrs) == final["input"] + final["output"] + + +@requires_real_sdk +def test_real_sdk_resume_counts_cost_and_tokens_once(built_package: Path) -> None: + """R1: a resumed session's first result already carries the earlier turns (sdk.d.ts:5679). + + Two query() calls, the second with options.resume. The promoted cost/tokens summed + over both traces must equal the final cumulative totals, not exceed them. + """ + output, spans, exported = _run_real_sdk("resume", "th8235-real-sdk-resume") + first, second = output["queries"] + (r1,), (r2,) = _results(first), _results(second) + session_id = _init_session(first) + assert _init_session(second) == session_id + final = _cumulative(r2) + # What the reviewer saw: the resumed result is cumulative over both queries. + assert final["cost"] > _cumulative(r1)["cost"] > 0 and final["input"] > _cumulative(r1)["input"] + + conversations = _conversations_in_query_order(spans) + assert len(conversations) == 2 and len({c["traceId"] for c in conversations}) == 2 + _assert_promoted_sum_equals(conversations, final) + c1, c2 = (_attrs(c) for c in conversations) + assert c1["gen_ai.cost.total"] == pytest.approx(r1["total_cost_usd"]) + assert (c2["session.id"], c2["claude_agent.session.is_new"], c2["claude_agent.session.is_resumed"]) == ( + session_id, False, True, + ) + assert c2["claude_agent.cumulative.cost_usd"] == pytest.approx(final["cost"]) + assert (c2["claude_agent.cumulative.input_tokens"], c2["claude_agent.cumulative.output_tokens"]) == ( + final["input"], final["output"], + ) + assert c2["claude_agent.usage.baseline_unknown"] is False + _assert_only_conversations_carry_usage(spans) + _assert_exported_with_both_keys(exported, "th8235-real-sdk-resume") + + +@requires_real_sdk +def test_real_sdk_continue_and_fork_count_cost_and_tokens_once(built_package: Path) -> None: + """R1: options.continue keeps the session id and its totals; a fork gets a new id but + starts from the parent's saved totals. Sum of promoted keys == the fork's final totals.""" + output, spans, _ = _run_real_sdk("continue_fork", "th8235-real-sdk-continue-fork") + first, continued, forked = output["queries"] + parent = _init_session(first) + assert _init_session(continued) == parent + fork_id = _init_session(forked) + assert fork_id != parent + final = _cumulative(_results(forked)[-1]) + + conversations = _conversations_in_query_order(spans) + assert len(conversations) == 3 + _assert_promoted_sum_equals(conversations, final) + _, c_continue, c_fork = (_attrs(c) for c in conversations) + assert (c_continue["claude_agent.session.is_new"], c_continue["claude_agent.session.is_resumed"]) == (False, True) + assert c_continue["claude_agent.is_resumed"] is True + assert (c_fork["session.id"], c_fork["claude_agent.session.fork_from"]) == (fork_id, parent) + assert c_fork["claude_agent.session.is_new"] is True + _assert_only_conversations_carry_usage(spans) + + +@requires_real_sdk +def test_real_sdk_resume_after_process_restart_marks_baseline_unknown(built_package: Path, tmp_path: Path) -> None: + """R1: a resume with no baseline in this process (restart) must not put promoted keys: + the result is cumulative and the earlier share is unknown. Cumulative values go on + claude_agent.cumulative.* and claude_agent.usage.baseline_unknown=true.""" + workdir = tmp_path / "home" + first_out, first_spans, _ = _run_real_sdk("restart", "th8235-real-sdk-restart", {"WORKDIR": str(workdir), "PHASE": "first"}) + session_id = _init_session(first_out["queries"][0]) + second_out, second_spans, _ = _run_real_sdk( + "restart", "th8235-real-sdk-restart", + {"WORKDIR": str(workdir), "PHASE": "second", "RESUME_SESSION_ID": session_id}, + ) + (r1,), (r2,) = _results(first_out["queries"][0]), _results(second_out["queries"][0]) + final = _cumulative(r2) + assert final["cost"] > r1["total_cost_usd"] + + c1 = _attrs(_one(first_spans, "claude_agent.conversation")) + c2 = _attrs(_one(second_spans, "claude_agent.conversation")) + assert c1["gen_ai.cost.total"] == pytest.approx(r1["total_cost_usd"]) # a new session: baseline 0 + assert c1["claude_agent.usage.baseline_unknown"] is False + assert _promoted(c2) == {}, _promoted(c2) + assert "claude_agent.cost.total_usd" not in c2 + assert c2["claude_agent.usage.baseline_unknown"] is True + assert c2["claude_agent.cumulative.cost_usd"] == pytest.approx(final["cost"]) + assert (c2["claude_agent.cumulative.input_tokens"], c2["claude_agent.cumulative.output_tokens"]) == ( + final["input"], final["output"], + ) + assert c2["session.id"] == session_id and c2["claude_agent.session.is_new"] is False + _assert_only_conversations_carry_usage(first_spans + second_spans) + + +@requires_real_sdk +def test_real_sdk_streaming_input_counts_tokens_once(built_package: Path) -> None: + """R2 + R5: one query() with two user turns yields two results. result.usage is per turn + (sdk.d.ts:5683); modelUsage and total_cost_usd are running totals (sdk.d.ts:5679, 5687). + The conversation span must carry the latest running totals, and turn 2 must start + after turn 1 ended.""" + output, spans, _ = _run_real_sdk("streaming", "th8235-real-sdk-streaming") + (messages,) = output["queries"] + results = _results(messages) + assert len(results) == 2 + final = _cumulative(results[-1]) + # What the reviewer saw: usage on the last result is one turn only. + assert results[-1]["usage"]["input_tokens"] < final["input"] + + conversation = _one(spans, "claude_agent.conversation") + _assert_promoted_sum_equals([conversation], final) + turns = sorted(_by_name(spans, "claude_agent.assistant_turn"), key=lambda s: int(s["startTimeUnixNano"])) + assert len(turns) == 2 + assert int(turns[1]["startTimeUnixNano"]) >= int(turns[0]["endTimeUnixNano"]) + _assert_only_conversations_carry_usage(spans) + + @requires_real_sdk def test_real_sdk_query_through_anthropic_base_url_mock(built_package: Path) -> None: """The real SDK query() and bundled CLI, wrapped, against a loopback Messages API mock. diff --git a/typescript/packages/traceai_claude_agent_sdk/src/__tests__/fixtures/messages.ts b/typescript/packages/traceai_claude_agent_sdk/src/__tests__/fixtures/messages.ts index 80f85c9d4..35571c0e3 100644 --- a/typescript/packages/traceai_claude_agent_sdk/src/__tests__/fixtures/messages.ts +++ b/typescript/packages/traceai_claude_agent_sdk/src/__tests__/fixtures/messages.ts @@ -155,7 +155,8 @@ export function resultSuccess(resultText: string, overrides: Partial = {}): SDKResultE stop_reason: null, total_cost_usd: 0.002, usage: resultUsage, - modelUsage: {}, + modelUsage: { [MODEL]: modelUsage(120, 45, 0.002, 11, 7) }, permission_denials: [], errors: ["Reached maximum number of turns (3)"], uuid: uuid(), diff --git a/typescript/packages/traceai_claude_agent_sdk/src/__tests__/usage.test.ts b/typescript/packages/traceai_claude_agent_sdk/src/__tests__/usage.test.ts new file mode 100644 index 000000000..21c490bd0 --- /dev/null +++ b/typescript/packages/traceai_claude_agent_sdk/src/__tests__/usage.test.ts @@ -0,0 +1,343 @@ +/** + * R1 / R2: usage and cost are promoted exactly once. + * + * fi-collector promotes gen_ai.usage.*, llm.token_count.*, gen_ai.cost.total and + * llm.cost.total into hot columns on ANY span, and Observe sums them over every + * span of a trace and per session.id. So the promoted keys on a conversation + * span must equal the spend new in that query() call: + * + * - tokens come from the latest result's `modelUsage` summed over models; cost + * from `total_cost_usd`. Both are running totals for the session (sdk.d.ts:5679, + * 5687). `result.usage` is main-loop only and per turn (sdk.d.ts:5683). + * - a resumed, continued or forked session starts from the totals its + * transcript saved, so the promoted value is cumulative minus the baseline + * this process last saw for that session (the parent session for a fork). + * - with no known baseline (e.g. after a restart) nothing is promoted; the + * cumulative values go on claude_agent.cumulative.* and + * claude_agent.usage.baseline_unknown=true. + */ +import type { SDKMessage } from "@anthropic-ai/claude-agent-sdk"; +import { ReadableSpan } from "@opentelemetry/sdk-trace-base"; +import { wrapQuery } from "../index"; +import { SessionUsageStore, sessionUsageStore } from "../spans"; +import { makeFakeQuery } from "./fixtures/fakeQuery"; +import { + MODEL, + PROMPT, + TURN_COST_USD, + assistant, + backgroundedSubagentJourney, + init, + mcpErrorJourney, + modelUsage, + resultError, + resultSuccess, + simpleToolJourney, + streamingInputJourney, + subagentJourney, + text, + turnUsage, +} from "./fixtures/messages"; +import { byName, drain, memoryProvider, one } from "./helpers"; + +const PROMOTED = /^(gen_ai\.usage\.|llm\.token_count\.)|^(gen_ai\.cost\.total|llm\.cost\.total)$/; +const USAGE_KEYS = /^(gen_ai\.usage\.|llm\.token_count\.|claude_agent\.cumulative\.|claude_agent\.cost\.|claude_agent\.usage\.)|^(gen_ai\.cost\.total|llm\.cost\.total)$/; + +function promoted(span: ReadableSpan): Record { + return Object.fromEntries(Object.entries(span.attributes).filter(([key]) => PROMOTED.test(key))); +} + +function usageKeys(span: ReadableSpan): string[] { + return Object.keys(span.attributes).filter((key) => USAGE_KEYS.test(key)); +} + +interface Totals { + cost: number; + input: number; + output: number; + cacheRead?: number; + cacheCreation?: number; +} + +/** One query against session `sessionId` whose result carries running totals `totals`. */ +function sessionQuery(sessionId: string, totals: Totals | null, extra: Partial[1]> = {}) { + const usageFields = totals + ? { + total_cost_usd: totals.cost, + usage: turnUsage(20, 8), + modelUsage: { + [MODEL]: modelUsage(totals.input, totals.output, totals.cost, totals.cacheRead ?? 0, totals.cacheCreation ?? 0), + }, + } + : {}; + return [ + init({ session_id: sessionId }), + assistant("msg_a", [text("Answer.")], null, { session_id: sessionId }), + resultSuccess("Answer.", { num_turns: 1, session_id: sessionId, ...usageFields, ...extra }), + ] as SDKMessage[]; +} + +async function runQuery(messages: SDKMessage[], options: Record = {}) { + const { provider, exporter } = memoryProvider(); + await drain(wrapQuery(makeFakeQuery(messages).query, { tracerProvider: provider })({ prompt: PROMPT, options })); + const spans = exporter.getFinishedSpans(); + return { spans, conversation: one(spans, "claude_agent.conversation") }; +} + +let counter = 0; +function newSessionId(): string { + counter += 1; + return `aaaaaaaa-bbbb-4ccc-8ddd-${counter.toString(16).padStart(12, "0")}`; +} + +beforeEach(() => sessionUsageStore().clear()); + +describe("R2: tokens come from modelUsage, cost from total_cost_usd", () => { + it("streaming input: the conversation carries the latest running totals, not the last turn's usage", async () => { + const { conversation } = await runQuery(streamingInputJourney()); + expect(conversation.attributes["gen_ai.usage.input_tokens"]).toBe(40); + expect(conversation.attributes["gen_ai.usage.output_tokens"]).toBe(16); + expect(conversation.attributes["gen_ai.usage.total_tokens"]).toBe(56); + expect(conversation.attributes["gen_ai.cost.total"]).toBeCloseTo(2 * TURN_COST_USD, 12); + expect(conversation.attributes["claude_agent.cost.total_usd"]).toBeCloseTo(2 * TURN_COST_USD, 12); + expect(conversation.attributes["claude_agent.cumulative.input_tokens"]).toBe(40); + expect(conversation.attributes["claude_agent.cumulative.output_tokens"]).toBe(16); + expect(conversation.attributes["claude_agent.cumulative.cost_usd"]).toBeCloseTo(2 * TURN_COST_USD, 12); + expect(conversation.attributes["claude_agent.usage.baseline_unknown"]).toBe(false); + }); + + it("subagent journey: sums modelUsage over every model when it exceeds the main-loop usage", async () => { + const journey = subagentJourney(); + const result = journey[journey.length - 1] as Extract; + // usage (main loop only) says 120 in / 45 out; modelUsage adds the subagent's haiku calls. + result.modelUsage = { + [MODEL]: modelUsage(120, 45, 0.01, 11, 7), + "claude-haiku-4-5": modelUsage(300, 80, 0.0023, 5, 3), + }; + result.total_cost_usd = 0.0123; + const { conversation } = await runQuery(journey); + expect(conversation.attributes["gen_ai.usage.input_tokens"]).toBe(420); + expect(conversation.attributes["gen_ai.usage.output_tokens"]).toBe(125); + expect(conversation.attributes["gen_ai.usage.total_tokens"]).toBe(545); + expect(conversation.attributes["gen_ai.usage.cache_read_tokens"]).toBe(16); + expect(conversation.attributes["gen_ai.usage.cache_creation_tokens"]).toBe(10); + expect(conversation.attributes["gen_ai.cost.total"]).toBe(0.0123); + }); + + it("turn, tool and subagent spans carry no usage or cost keys", async () => { + const all: ReadableSpan[] = []; + for (const journey of [ + simpleToolJourney(), + subagentJourney(), + backgroundedSubagentJourney(), + mcpErrorJourney(), + streamingInputJourney(), + ]) { + all.push(...(await runQuery(journey)).spans); + } + const others = all.filter((span) => span.name !== "claude_agent.conversation"); + expect(new Set(others.map((s) => s.attributes["claude_agent.span_kind"]))).toEqual( + new Set(["assistant_turn", "tool_execution", "mcp_tool", "subagent"]), + ); + for (const span of others) { + expect([span.name, usageKeys(span)]).toEqual([span.name, []]); + } + for (const conversation of byName(all, "claude_agent.conversation")) { + expect(Object.keys(promoted(conversation)).length).toBeGreaterThan(0); + } + }); + + it("a result with no usage or cost leaves every usage and cost attribute out", async () => { + const journey = simpleToolJourney(); + const result = journey[journey.length - 1] as Record; + delete result.usage; + delete result.modelUsage; + delete result.total_cost_usd; + const { spans } = await runQuery(journey); + for (const span of spans) { + expect([span.name, usageKeys(span)]).toEqual([span.name, []]); + } + }); + + it("does not fall back to result.usage when modelUsage is missing", async () => { + const journey = simpleToolJourney(); + delete (journey[journey.length - 1] as Record).modelUsage; + const { conversation } = await runQuery(journey); + expect(conversation.attributes["gen_ai.usage.input_tokens"]).toBeUndefined(); + expect(conversation.attributes["gen_ai.cost.total"]).toBe(0.0123); + }); +}); + +describe("R1: resumed, continued and forked sessions are counted once", () => { + const FIRST: Totals = { cost: TURN_COST_USD, input: 20, output: 8 }; + const BOTH: Totals = { cost: 2 * TURN_COST_USD, input: 40, output: 16 }; + const ALL_THREE: Totals = { cost: 3 * TURN_COST_USD, input: 60, output: 24 }; + + function expectPromotedSumsTo(conversations: ReadableSpan[], final: Totals) { + const sum = (key: string) => conversations.reduce((acc, span) => acc + ((span.attributes[key] as number) ?? 0), 0); + expect(sum("gen_ai.cost.total")).toBeCloseTo(final.cost, 12); + expect(sum("gen_ai.usage.input_tokens")).toBe(final.input); + expect(sum("gen_ai.usage.output_tokens")).toBe(final.output); + expect(sum("gen_ai.usage.total_tokens")).toBe(final.input + final.output); + } + + it("resume: promotes cumulative minus the session's last totals (the reviewer's 0.00018 / 0.00036 repro)", async () => { + const id = newSessionId(); + const q1 = await runQuery(sessionQuery(id, FIRST)); + const q2 = await runQuery(sessionQuery(id, BOTH), { resume: id }); + expect(q2.conversation.attributes["gen_ai.cost.total"]).toBeCloseTo(TURN_COST_USD, 12); + expect(q2.conversation.attributes["claude_agent.cost.total_usd"]).toBeCloseTo(TURN_COST_USD, 12); + expect(q2.conversation.attributes["gen_ai.usage.input_tokens"]).toBe(20); + expect(q2.conversation.attributes["gen_ai.usage.output_tokens"]).toBe(8); + expect(q2.conversation.attributes["claude_agent.cumulative.cost_usd"]).toBeCloseTo(2 * TURN_COST_USD, 12); + expect(q2.conversation.attributes["claude_agent.cumulative.input_tokens"]).toBe(40); + expect(q2.conversation.attributes["claude_agent.usage.baseline_unknown"]).toBe(false); + expect(q2.conversation.attributes["claude_agent.session.is_new"]).toBe(false); + expectPromotedSumsTo([q1.conversation, q2.conversation], BOTH); + }); + + it("resume: cache tokens are deltas too", async () => { + const id = newSessionId(); + await runQuery(sessionQuery(id, { ...FIRST, cacheRead: 100, cacheCreation: 50 })); + const q2 = await runQuery(sessionQuery(id, { ...BOTH, cacheRead: 250, cacheCreation: 50 }), { resume: id }); + expect(q2.conversation.attributes["gen_ai.usage.cache_read_tokens"]).toBe(150); + expect(q2.conversation.attributes["gen_ai.usage.cache_creation_tokens"]).toBe(0); + expect(q2.conversation.attributes["claude_agent.cumulative.cache_read_tokens"]).toBe(250); + expect(q2.conversation.attributes["claude_agent.cumulative.cache_creation_tokens"]).toBe(50); + }); + + it("continue: handled like resume (same session id from init) and labelled not new", async () => { + const id = newSessionId(); + const q1 = await runQuery(sessionQuery(id, FIRST)); + const q2 = await runQuery(sessionQuery(id, BOTH), { continue: true }); + expect(q2.conversation.attributes["claude_agent.session.is_new"]).toBe(false); + expect(q2.conversation.attributes["claude_agent.session.is_resumed"]).toBe(true); + expect(q2.conversation.attributes["claude_agent.is_resumed"]).toBe(true); + expect(q2.conversation.attributes["gen_ai.cost.total"]).toBeCloseTo(TURN_COST_USD, 12); + expectPromotedSumsTo([q1.conversation, q2.conversation], BOTH); + }); + + it("fork: the new session starts from the parent's last totals", async () => { + const parent = newSessionId(); + const fork = newSessionId(); + const q1 = await runQuery(sessionQuery(parent, FIRST)); + const q2 = await runQuery(sessionQuery(parent, BOTH), { resume: parent }); + const q3 = await runQuery(sessionQuery(fork, ALL_THREE), { resume: parent, forkSession: true }); + expect(q3.conversation.attributes["session.id"]).toBe(fork); + expect(q3.conversation.attributes["claude_agent.session.fork_from"]).toBe(parent); + expect(q3.conversation.attributes["gen_ai.cost.total"]).toBeCloseTo(TURN_COST_USD, 12); + expectPromotedSumsTo([q1.conversation, q2.conversation, q3.conversation], ALL_THREE); + // The fork's own totals are now its baseline. + expect(sessionUsageStore().get(fork)?.costUsd).toBeCloseTo(3 * TURN_COST_USD, 12); + }); + + it.each([ + ["resume", (id: string) => ({ resume: id })], + ["continue", () => ({ continue: true })], + ["resume + forkSession", (id: string) => ({ resume: id, forkSession: true })], + ["continue + forkSession", () => ({ continue: true, forkSession: true })], + ])("%s with no known baseline: no promoted keys, cumulative on claude_agent.cumulative.*", async (_name, opts) => { + const id = newSessionId(); // never seen in this process, as after a restart + const { conversation } = await runQuery(sessionQuery(id, BOTH), opts(id)); + expect(promoted(conversation)).toEqual({}); + expect(conversation.attributes["claude_agent.cost.total_usd"]).toBeUndefined(); + expect(conversation.attributes["claude_agent.usage.baseline_unknown"]).toBe(true); + expect(conversation.attributes["claude_agent.cumulative.cost_usd"]).toBeCloseTo(2 * TURN_COST_USD, 12); + expect(conversation.attributes["claude_agent.cumulative.input_tokens"]).toBe(40); + expect(conversation.attributes["claude_agent.cumulative.output_tokens"]).toBe(16); + expect(conversation.attributes["claude_agent.cumulative.cache_read_tokens"]).toBe(0); + expect(conversation.attributes["claude_agent.cumulative.cache_creation_tokens"]).toBe(0); + // The result still seeds the baseline for the next resume in this process. + const next = await runQuery(sessionQuery(id, ALL_THREE), { resume: id }); + expect(next.conversation.attributes["gen_ai.cost.total"]).toBeCloseTo(TURN_COST_USD, 12); + }); + + it("a new session with Options.sessionId starts from zero even if the id is in the store", async () => { + const id = newSessionId(); + sessionUsageStore().set(id, { costUsd: 1, inputTokens: 1000, outputTokens: 1000, cacheReadTokens: 0, cacheCreationTokens: 0 }); + const { conversation } = await runQuery(sessionQuery(id, FIRST), { sessionId: id }); + expect(conversation.attributes["gen_ai.cost.total"]).toBeCloseTo(TURN_COST_USD, 12); + expect(conversation.attributes["gen_ai.usage.input_tokens"]).toBe(20); + }); + + it("a resume whose totals are below the baseline marks the baseline unknown", async () => { + const id = newSessionId(); + await runQuery(sessionQuery(id, BOTH)); + const { conversation } = await runQuery(sessionQuery(id, FIRST), { resume: id }); // e.g. resumeSessionAt an earlier message + expect(promoted(conversation)).toEqual({}); + expect(conversation.attributes["claude_agent.usage.baseline_unknown"]).toBe(true); + }); + + it("a mid-query /clear (running total drops) counts the spend before and after it", async () => { + const id = newSessionId(); + const journey: SDKMessage[] = [ + init({ session_id: id }), + assistant("msg_c1", [text("one")], null, { session_id: id }), + resultSuccess("one", { session_id: id, total_cost_usd: 0.0003, modelUsage: { [MODEL]: modelUsage(30, 10, 0.0003) } }), + assistant("msg_c2", [text("two")], null, { session_id: id }), + resultSuccess("two", { session_id: id, total_cost_usd: 0.0001, modelUsage: { [MODEL]: modelUsage(10, 4, 0.0001) } }), + ]; + const { conversation } = await runQuery(journey); + expect(conversation.attributes["gen_ai.cost.total"]).toBeCloseTo(0.0004, 12); + expect(conversation.attributes["gen_ai.usage.input_tokens"]).toBe(40); + expect(conversation.attributes["gen_ai.usage.output_tokens"]).toBe(14); + expect(conversation.attributes["claude_agent.cumulative.cost_usd"]).toBeCloseTo(0.0001, 12); + }); + + it("a zeroed crash result neither counts nor resets the session baseline", async () => { + const id = newSessionId(); + const q1 = await runQuery(sessionQuery(id, FIRST)); + const crash: SDKMessage[] = [ + init({ session_id: id }), + resultError({ + subtype: "error_during_execution", + session_id: id, + total_cost_usd: 0, + modelUsage: {}, + usage: turnUsage(0, 0), + errors: ["startup failed"], + }), + ]; + const q2 = await runQuery(crash, { resume: id }); + expect(usageKeys(q2.conversation)).toEqual([]); + const q3 = await runQuery(sessionQuery(id, BOTH), { resume: id }); + expect(q3.conversation.attributes["gen_ai.cost.total"]).toBeCloseTo(TURN_COST_USD, 12); + expectPromotedSumsTo([q1.conversation, q3.conversation], BOTH); + }); + + it("an aborted query with no result promotes nothing", async () => { + const id = newSessionId(); + const { conversation } = await runQuery(sessionQuery(id, BOTH).slice(0, 2)); + expect(usageKeys(conversation)).toEqual([]); + }); +}); + +describe("SessionUsageStore", () => { + it("is a bounded LRU keyed by session id", () => { + const store = new SessionUsageStore(2); + store.set("a", { costUsd: 1 }); + store.set("b", { costUsd: 2 }); + expect(store.get("a")?.costUsd).toBe(1); // a is now the most recent + store.set("c", { costUsd: 3 }); + expect(store.get("b")).toBeUndefined(); + expect(store.get("a")?.costUsd).toBe(1); + expect(store.get("c")?.costUsd).toBe(3); + expect(store.size).toBe(2); + }); + + it("the process-wide store keeps at most 1000 sessions", () => { + const store = sessionUsageStore(); + for (let i = 0; i < 1005; i += 1) store.set(`s-${i}`, { costUsd: i }); + expect(store.size).toBe(1000); + expect(store.get("s-0")).toBeUndefined(); + expect(store.get("s-1004")?.costUsd).toBe(1004); + }); + + it("stores a copy, so a later change to the caller's object does not move the baseline", () => { + const store = new SessionUsageStore(10); + const totals = { costUsd: 1 }; + store.set("x", totals); + totals.costUsd = 99; + expect(store.get("x")?.costUsd).toBe(1); + }); +}); diff --git a/typescript/packages/traceai_claude_agent_sdk/src/attributes.ts b/typescript/packages/traceai_claude_agent_sdk/src/attributes.ts index c56ad24b6..c77c154ee 100644 --- a/typescript/packages/traceai_claude_agent_sdk/src/attributes.ts +++ b/typescript/packages/traceai_claude_agent_sdk/src/attributes.ts @@ -141,6 +141,21 @@ export const TraceAIAttributes = { SUBAGENT_TASK_ID: "claude_agent.subagent.task_id", /** Final status from `system/task_notification`. */ SUBAGENT_STATUS: "claude_agent.subagent.status", + /** + * Session running totals from the latest result (`total_cost_usd`, `modelUsage` + * summed over models). Not promoted by the collector; the promoted + * `gen_ai.usage.*` / `gen_ai.cost.total` carry only this query's new spend. + */ + CUMULATIVE_COST_USD: "claude_agent.cumulative.cost_usd", + CUMULATIVE_INPUT_TOKENS: "claude_agent.cumulative.input_tokens", + CUMULATIVE_OUTPUT_TOKENS: "claude_agent.cumulative.output_tokens", + CUMULATIVE_CACHE_READ_TOKENS: "claude_agent.cumulative.cache_read_tokens", + CUMULATIVE_CACHE_CREATION_TOKENS: "claude_agent.cumulative.cache_creation_tokens", + /** + * True when a resumed / continued / forked session's earlier totals are not + * known in this process, so no promoted usage or cost key was written. + */ + USAGE_BASELINE_UNKNOWN: "claude_agent.usage.baseline_unknown", } as const; /** diff --git a/typescript/packages/traceai_claude_agent_sdk/src/spans.ts b/typescript/packages/traceai_claude_agent_sdk/src/spans.ts index 81d829fd4..f65251a6d 100644 --- a/typescript/packages/traceai_claude_agent_sdk/src/spans.ts +++ b/typescript/packages/traceai_claude_agent_sdk/src/spans.ts @@ -129,6 +129,65 @@ interface ScopeState { nextTurnStartMs?: number; } +/** + * Running totals a result message carries for its session: `total_cost_usd` + * and `modelUsage` summed over models (sdk.d.ts:5679, 5687). A field is absent + * when the result did not carry it. + */ +export interface UsageTotals { + costUsd?: number; + inputTokens?: number; + outputTokens?: number; + cacheReadTokens?: number; + cacheCreationTokens?: number; +} + +const USAGE_FIELDS = ["costUsd", "inputTokens", "outputTokens", "cacheReadTokens", "cacheCreationTokens"] as const; + +/** + * Process-local map of session id -> the last running totals seen for it, so a + * resumed, continued or forked session promotes only its new spend. Bounded + * LRU: the oldest session is dropped past `capacity`. + */ +export class SessionUsageStore { + private readonly entries = new Map(); + + constructor(private readonly capacity: number) {} + + get size(): number { + return this.entries.size; + } + + get(sessionId: string): UsageTotals | undefined { + const totals = this.entries.get(sessionId); + if (!totals) return undefined; + this.entries.delete(sessionId); + this.entries.set(sessionId, totals); + return { ...totals }; + } + + set(sessionId: string, totals: UsageTotals): void { + this.entries.delete(sessionId); + this.entries.set(sessionId, { ...totals }); + while (this.entries.size > this.capacity) { + const oldest = this.entries.keys().next().value as string; + this.entries.delete(oldest); + } + } + + clear(): void { + this.entries.clear(); + } +} + +const SESSION_USAGE_CAPACITY = 1000; +const sharedSessionUsage = new SessionUsageStore(SESSION_USAGE_CAPACITY); + +/** The store every QueryTracer in this process shares. */ +export function sessionUsageStore(): SessionUsageStore { + return sharedSessionUsage; +} + /** Truncate like the Python package: keep `max` chars, ending in "...". */ export function truncate(value: string, max: number): string { return value.length > max ? `${value.slice(0, max - 3)}...` : value; @@ -238,6 +297,14 @@ export class QueryTracer { /** task_id -> tool_use_id from task_started; task_updated carries only task_id. */ private readonly taskToolUseIds = new Map(); + /** How this query relates to an earlier session, for the usage baseline. */ + private readonly usageMode: "new" | "resume" | "fork" | "continue" | "continue-fork"; + private readonly resumeId?: string; + /** undefined: no counted result yet; null: baseline unknown for this query. */ + private usagePrev?: UsageTotals | null; + /** New spend counted in this query (sum of deltas), written to the promoted keys. */ + private readonly usageCounted: UsageTotals = {}; + constructor(args: { tracer: Tracer; policy: ContentPolicy; @@ -263,6 +330,18 @@ export class QueryTracer { // A plain resume continues the same session id. this.sessionId = this.options.resume; } + const resume = typeof this.options?.resume === "string" && this.options.resume ? this.options.resume : undefined; + const fork = this.options?.forkSession === true; + this.resumeId = resume; + this.usageMode = resume + ? fork + ? "fork" + : "resume" + : this.options?.continue === true + ? fork + ? "continue-fork" + : "continue" + : "new"; } get isFinished(): boolean { @@ -478,26 +557,7 @@ export class QueryTracer { this.setSessionId(message.session_id); } - const usage = message.usage; - if (usage && typeof usage === "object") { - const input = typeof usage.input_tokens === "number" ? usage.input_tokens : undefined; - const output = typeof usage.output_tokens === "number" ? usage.output_tokens : undefined; - if (input !== undefined) conversation.setAttribute(A.USAGE_INPUT_TOKENS, input); - if (output !== undefined) conversation.setAttribute(A.USAGE_OUTPUT_TOKENS, output); - if (input !== undefined || output !== undefined) { - conversation.setAttribute(A.USAGE_TOTAL_TOKENS, (input ?? 0) + (output ?? 0)); - } - if (typeof usage.cache_read_input_tokens === "number") { - conversation.setAttribute(A.USAGE_CACHE_READ_TOKENS, usage.cache_read_input_tokens); - } - if (typeof usage.cache_creation_input_tokens === "number") { - conversation.setAttribute(A.USAGE_CACHE_CREATION_TOKENS, usage.cache_creation_input_tokens); - } - } - if (typeof message.total_cost_usd === "number") { - conversation.setAttribute(A.COST_TOTAL_USD, message.total_cost_usd); - conversation.setAttribute(T.GEN_AI_COST_TOTAL, message.total_cost_usd); - } + this.recordUsage(cumulativeUsage(message)); if (typeof message.duration_ms === "number") { conversation.setAttribute(A.DURATION_MS, message.duration_ms); } @@ -907,6 +967,89 @@ export class QueryTracer { this.conversation?.setAttributes(this.sessionAttributes()); } + /** + * Record one result's running totals. The collector promotes gen_ai.usage.* + * and gen_ai.cost.total on any span and Observe sums them per trace and per + * session, so the promoted keys carry only the spend new since the session's + * last known totals. Without a known baseline nothing is promoted. + */ + private recordUsage(cumulative: UsageTotals | undefined): void { + if (!cumulative) return; + const conversation = this.conversation!; + const store = sessionUsageStore(); + + const first = this.usagePrev === undefined; + if (first) { + this.usagePrev = this.resolveUsageBaseline(); + } + let prev = this.usagePrev; + if (prev) { + const dropped = USAGE_FIELDS.some( + (f) => cumulative[f] !== undefined && prev![f] !== undefined && cumulative[f]! < prev![f]!, + ); + if (dropped && first) { + // Below the saved baseline on the first result (e.g. resumeSessionAt an + // earlier message): the earlier share is unknown. + prev = null; + } else { + // Within one query a drop is a /clear: the running total restarted at 0. + for (const f of USAGE_FIELDS) { + const value = cumulative[f]; + const base = dropped ? 0 : prev[f]; + if (value === undefined || base === undefined) continue; + this.usageCounted[f] = (this.usageCounted[f] ?? 0) + (value - base); + } + prev = { ...prev, ...definedUsage(cumulative) }; + } + this.usagePrev = prev; + } + if (this.sessionId) { + const stored = prev ?? { ...(store.get(this.sessionId) ?? {}), ...definedUsage(cumulative) }; + store.set(this.sessionId, stored); + } + + const setIf = (key: string, value: number | undefined) => { + if (value !== undefined) conversation.setAttribute(key, value); + }; + setIf(T.CUMULATIVE_COST_USD, cumulative.costUsd); + setIf(T.CUMULATIVE_INPUT_TOKENS, cumulative.inputTokens); + setIf(T.CUMULATIVE_OUTPUT_TOKENS, cumulative.outputTokens); + setIf(T.CUMULATIVE_CACHE_READ_TOKENS, cumulative.cacheReadTokens); + setIf(T.CUMULATIVE_CACHE_CREATION_TOKENS, cumulative.cacheCreationTokens); + conversation.setAttribute(T.USAGE_BASELINE_UNKNOWN, prev === null); + if (prev === null) return; + + const counted = this.usageCounted; + setIf(A.USAGE_INPUT_TOKENS, counted.inputTokens); + setIf(A.USAGE_OUTPUT_TOKENS, counted.outputTokens); + if (counted.inputTokens !== undefined || counted.outputTokens !== undefined) { + conversation.setAttribute(A.USAGE_TOTAL_TOKENS, (counted.inputTokens ?? 0) + (counted.outputTokens ?? 0)); + } + setIf(A.USAGE_CACHE_READ_TOKENS, counted.cacheReadTokens); + setIf(A.USAGE_CACHE_CREATION_TOKENS, counted.cacheCreationTokens); + setIf(A.COST_TOTAL_USD, counted.costUsd); + setIf(T.GEN_AI_COST_TOTAL, counted.costUsd); + } + + /** The session totals this query starts from, or null when they are not known here. */ + private resolveUsageBaseline(): UsageTotals | null { + const store = sessionUsageStore(); + switch (this.usageMode) { + case "new": + return { costUsd: 0, inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheCreationTokens: 0 }; + case "resume": + case "fork": + // A fork starts from the parent's saved totals under a new session id. + return (this.resumeId && store.get(this.resumeId)) || null; + case "continue": + return (this.sessionId && store.get(this.sessionId)) || null; + case "continue-fork": + default: + // The parent is "the most recent session in cwd": not identifiable here. + return null; + } + } + private conversationStartAttributes(): Attributes { const options = this.options ?? {}; const attributes: Record = { @@ -924,18 +1067,22 @@ export class QueryTracer { } const resume = typeof options.resume === "string" && options.resume ? options.resume : undefined; - const fork = resume !== undefined && options.forkSession === true; - if (resume) { + // options.continue (sdk.d.ts:1594) resumes the most recent session in cwd. + const continuing = resume !== undefined || options.continue === true; + const fork = continuing && options.forkSession === true; + if (continuing) { attributes[A.AGENT_IS_RESUMED] = true; + } + if (resume) { attributes[A.AGENT_RESUME_SESSION_ID] = resume; } - attributes[A.SESSION_IS_RESUMED] = resume !== undefined && !fork; - attributes[A.SESSION_IS_NEW] = resume === undefined || fork; + attributes[A.SESSION_IS_RESUMED] = continuing && !fork; + attributes[A.SESSION_IS_NEW] = !continuing || fork; if (resume && !fork) { attributes[A.SESSION_PREVIOUS_ID] = resume; } - if (fork) { - attributes[A.SESSION_FORK_FROM] = resume!; + if (fork && resume) { + attributes[A.SESSION_FORK_FROM] = resume; } if (!this.policy.hideInputs) { @@ -988,6 +1135,54 @@ function safeContextAttributes(ctx: Context): Attributes { } } +function finiteNumber(value: unknown): number | undefined { + return typeof value === "number" && Number.isFinite(value) ? value : undefined; +} + +function definedUsage(totals: UsageTotals): UsageTotals { + const out: UsageTotals = {}; + for (const f of USAGE_FIELDS) { + if (totals[f] !== undefined) out[f] = totals[f]; + } + return out; +} + +/** + * Running totals from a result: cost from `total_cost_usd`, tokens from + * `modelUsage` summed over models (sdk.d.ts:5679, 5687). `result.usage` is not + * read: it is main-loop only and per turn (sdk.d.ts:5683). Returns undefined + * when the result carries nothing, or only zeros (a crash/startup-error result). + */ +function cumulativeUsage(message: ResultMessageLike): UsageTotals | undefined { + const totals: UsageTotals = {}; + const cost = finiteNumber(message.total_cost_usd); + if (cost !== undefined) totals.costUsd = cost; + const models = message.modelUsage; + if (models && typeof models === "object") { + let seen = false; + let input = 0; + let output = 0; + let cacheRead = 0; + let cacheCreation = 0; + for (const entry of Object.values(models)) { + if (!entry || typeof entry !== "object") continue; + seen = true; + input += finiteNumber(entry.inputTokens) ?? 0; + output += finiteNumber(entry.outputTokens) ?? 0; + cacheRead += finiteNumber(entry.cacheReadInputTokens) ?? 0; + cacheCreation += finiteNumber(entry.cacheCreationInputTokens) ?? 0; + } + if (seen) { + totals.inputTokens = input; + totals.outputTokens = output; + totals.cacheReadTokens = cacheRead; + totals.cacheCreationTokens = cacheCreation; + } + } + const values = USAGE_FIELDS.map((f) => totals[f]).filter((v): v is number => v !== undefined); + return values.some((v) => v !== 0) ? totals : undefined; +} + /** Python `_set_tool_specific_attributes`. Only called when inputs are visible. */ function setToolSpecificAttributes(span: Span, toolName: string, input: unknown): void { if (!input || typeof input !== "object") return; diff --git a/typescript/packages/traceai_claude_agent_sdk/src/types.ts b/typescript/packages/traceai_claude_agent_sdk/src/types.ts index e0f0a5df5..4a5f1bd76 100644 --- a/typescript/packages/traceai_claude_agent_sdk/src/types.ts +++ b/typescript/packages/traceai_claude_agent_sdk/src/types.ts @@ -66,6 +66,17 @@ export interface ResultMessageLike { cache_read_input_tokens?: number | null; cache_creation_input_tokens?: number | null; }; + /** Running per-model totals for the session (sdk.d.ts ModelUsage). */ + modelUsage?: Record< + string, + { + inputTokens?: number; + outputTokens?: number; + cacheReadInputTokens?: number; + cacheCreationInputTokens?: number; + costUSD?: number; + } + >; errors?: string[]; session_id?: string; } @@ -113,6 +124,8 @@ export interface OptionsLike { allowedTools?: string[]; systemPrompt?: unknown; resume?: string; + /** Continue the most recent conversation in cwd (sdk.d.ts:1594). */ + continue?: boolean; forkSession?: boolean; sessionId?: string; mcpServers?: Record; From 06a3a20b0705fb94e5444cdace13f79eccdb44be Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 4 Oct 2026 19:11:56 -0700 Subject: [PATCH 10/19] chore(claude-agent-sdk): fix repo metadata, hold providers weakly, drop tsbuildinfo R7 review nits: - package.json repository/bugs/homepage pointed at github.com/futureagi/traceai (copied from strands). They now match traceai_anthropic: future-agi/traceAI, with this package's directory. - The module-level knownProviders Set kept every provider passed to an instrumentation alive forever. It now holds WeakRefs, pruned on read and add, and dedupes a provider set twice. shutdown() with no argument still flushes every live provider. - The tarball shipped dist/**/*.tsbuildinfo; files now excludes it. Tests: package.test.ts (metadata against the sibling; 20 dropped providers are released after a forced GC, failed with 20 held before; a live provider is tracked once). The pack contract test rejects *.tsbuildinfo (failed on dist/esm/tsconfig.esm.tsbuildinfo before). approved: Nikhil 2026-10-03 blanket Refs: TH-8235 --- .../contract/test_harness_contract.py | 1 + .../traceai_claude_agent_sdk/package.json | 9 +-- .../src/__tests__/package.test.ts | 67 +++++++++++++++++++ .../src/instrumentation.ts | 32 ++++++++- 4 files changed, 102 insertions(+), 7 deletions(-) create mode 100644 typescript/packages/traceai_claude_agent_sdk/src/__tests__/package.test.ts diff --git a/typescript/packages/traceai_claude_agent_sdk/contract/test_harness_contract.py b/typescript/packages/traceai_claude_agent_sdk/contract/test_harness_contract.py index c60f64489..6826c8068 100644 --- a/typescript/packages/traceai_claude_agent_sdk/contract/test_harness_contract.py +++ b/typescript/packages/traceai_claude_agent_sdk/contract/test_harness_contract.py @@ -589,6 +589,7 @@ def test_packed_tarball_has_no_sdk_sources_or_native_binary(built_package: Path, assert "node_modules" not in member.name assert "__tests__" not in member.name assert not relative.endswith((".mjs", ".node", ".exe", ".wasm")), relative + assert not relative.endswith(".tsbuildinfo"), f"build cache in tarball: {relative}" assert Path(relative).name not in {"sdk.mjs", "sdk.d.ts", "sdk-tools.d.ts", "bridge.mjs", "claude", "cli.js"} assert not data.startswith(NATIVE_MAGIC), f"native binary in tarball: {relative}" assert _sha256(data) not in sdk_hashes, f"{relative} is a copy of {sdk_hashes[_sha256(data)]}" diff --git a/typescript/packages/traceai_claude_agent_sdk/package.json b/typescript/packages/traceai_claude_agent_sdk/package.json index 0e90e08ec..a09f26b22 100644 --- a/typescript/packages/traceai_claude_agent_sdk/package.json +++ b/typescript/packages/traceai_claude_agent_sdk/package.json @@ -34,13 +34,13 @@ "license": "Apache-2.0", "repository": { "type": "git", - "url": "https://github.com/futureagi/traceai.git", + "url": "git+https://github.com/future-agi/traceAI.git", "directory": "typescript/packages/traceai_claude_agent_sdk" }, "bugs": { - "url": "https://github.com/futureagi/traceai/issues" + "url": "https://github.com/future-agi/traceAI/issues" }, - "homepage": "https://github.com/futureagi/traceai#readme", + "homepage": "https://github.com/future-agi/traceAI/tree/main/typescript/packages/traceai_claude_agent_sdk#readme", "engines": { "node": ">=18.0.0" }, @@ -67,7 +67,8 @@ "files": [ "dist", "src", - "!**/__tests__/**" + "!**/__tests__/**", + "!**/*.tsbuildinfo" ], "publishConfig": { "access": "public" diff --git a/typescript/packages/traceai_claude_agent_sdk/src/__tests__/package.test.ts b/typescript/packages/traceai_claude_agent_sdk/src/__tests__/package.test.ts new file mode 100644 index 000000000..8ce59d955 --- /dev/null +++ b/typescript/packages/traceai_claude_agent_sdk/src/__tests__/package.test.ts @@ -0,0 +1,67 @@ +/** + * R7 nits: package metadata points at the real repository, and the + * instrumentation does not keep tracer providers alive forever. + */ +import { readFileSync } from "fs"; +import { join } from "path"; +import { setFlagsFromString } from "v8"; +import { runInNewContext } from "vm"; +import { BasicTracerProvider } from "@opentelemetry/sdk-trace-base"; +import { ClaudeAgentSDKInstrumentation } from "../index"; +import { trackedProviderCount } from "../instrumentation"; + +const PKG_DIR = join(__dirname, "../.."); + +function manifest(dir: string): Record { + return JSON.parse(readFileSync(join(dir, "package.json"), "utf8")); +} + +describe("package.json", () => { + it("points repository, bugs and homepage at future-agi/traceAI, as traceai_anthropic does", () => { + const ours = manifest(PKG_DIR); + const sibling = manifest(join(PKG_DIR, "../traceai_anthropic")); + expect(ours.repository).toEqual({ + ...(sibling.repository as object), + directory: "typescript/packages/traceai_claude_agent_sdk", + }); + expect(ours.bugs).toEqual(sibling.bugs); + expect(ours.homepage).toBe( + (sibling.homepage as string).replace("traceai_anthropic", "traceai_claude_agent_sdk"), + ); + expect(JSON.stringify([ours.repository, ours.bugs, ours.homepage])).not.toContain("github.com/futureagi/"); + }); +}); + +describe("tracer provider registry", () => { + setFlagsFromString("--expose_gc"); + const gc = runInNewContext("gc") as () => void; + const tick = () => new Promise((resolve) => setImmediate(resolve)); + + it("does not keep a provider alive after the app drops it and its instrumentation", async () => { + const before = trackedProviderCount(); + (() => { + for (let i = 0; i < 20; i += 1) { + new ClaudeAgentSDKInstrumentation({ tracerProvider: new BasicTracerProvider() }); + } + })(); + expect(trackedProviderCount()).toBe(before + 20); + for (let attempt = 0; attempt < 10 && trackedProviderCount() > before; attempt += 1) { + await tick(); + gc(); + await tick(); + } + expect(trackedProviderCount()).toBe(before); + }); + + it("still tracks a provider while its instrumentation is alive, once per provider", async () => { + const provider = new BasicTracerProvider(); + const kept = [new ClaudeAgentSDKInstrumentation({ tracerProvider: provider })]; + kept[0].setTracerProvider(provider); + const before = trackedProviderCount(); + await tick(); + gc(); + await tick(); + expect(trackedProviderCount()).toBe(before); + expect(kept).toHaveLength(1); + }); +}); diff --git a/typescript/packages/traceai_claude_agent_sdk/src/instrumentation.ts b/typescript/packages/traceai_claude_agent_sdk/src/instrumentation.ts index a15acc813..1aef030da 100644 --- a/typescript/packages/traceai_claude_agent_sdk/src/instrumentation.ts +++ b/typescript/packages/traceai_claude_agent_sdk/src/instrumentation.ts @@ -48,7 +48,33 @@ export function resolveContentPolicy(traceConfig?: TraceConfigOptions): ContentP }; } -const knownProviders = new Set(); +/** + * Providers passed to an instrumentation, for `shutdown()` with no argument. + * Held weakly: an app that drops a provider and its instrumentation lets both + * be collected. Dead entries are pruned whenever the set is read or added to. + */ +const knownProviders = new Set>(); + +function liveProviders(): TracerProvider[] { + const live: TracerProvider[] = []; + for (const ref of knownProviders) { + const provider = ref.deref(); + if (provider) live.push(provider); + else knownProviders.delete(ref); + } + return live; +} + +function trackProvider(provider: TracerProvider): void { + if (!liveProviders().includes(provider)) { + knownProviders.add(new WeakRef(provider)); + } +} + +/** Providers currently tracked for `shutdown()`. */ +export function trackedProviderCount(): number { + return liveProviders().length; +} /** * Traces Claude Agent SDK `query()` calls. @@ -75,7 +101,7 @@ export class ClaudeAgentSDKInstrumentation { setTracerProvider(tracerProvider: TracerProvider): void { this.tracerProvider = tracerProvider; - knownProviders.add(tracerProvider); + trackProvider(tracerProvider); } /** The content policy this instrumentation applies. */ @@ -133,7 +159,7 @@ export function wrapQuery( export async function shutdown(tracerProvider?: TracerProvider): Promise { const providers = tracerProvider ? [tracerProvider] - : [...knownProviders, trace.getTracerProvider()]; + : [...liveProviders(), trace.getTracerProvider()]; const seen = new Set(); for (const provider of providers) { const target = resolveDelegate(provider); From cc1a5818ee0c5d79096362851080b1a564f4c914 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 4 Oct 2026 22:35:10 -0700 Subject: [PATCH 11/19] fix(claude-agent-sdk): review-r2 N1-N4: clean close after a result, task_id notifications, partial baselines N1: close()/asyncDispose/abort after a result with no open turn, tool or subagent ends the query as returned (conversation OK), the normal end of a streaming-input session. A close with work still in flight stays cancelled. N2: task_notification without tool_use_id (optional, sdk.d.ts:5997) is matched through the task_started task_id mapping. N3: an unknown usage baseline covers only the first result; later results in the same query promote their delta against it (flag stays true). N4: resume/fork with resumeSessionAt has no known baseline. README notes the /clear undercount edge. Tests first: 8 jest cases and the new real-CLI streaming_close contract test failed on 06a3a20 and pass now. approved: Nikhil 2026-10-03 blanket Refs: TH-8235 --- .../traceai_claude_agent_sdk/README.md | 9 ++- .../contract/run_real_sdk.mjs | 23 ++++++ .../contract/test_harness_contract.py | 21 ++++++ .../src/__tests__/usage.test.ts | 44 +++++++++++ .../src/__tests__/wrapper.test.ts | 74 +++++++++++++++++++ .../traceai_claude_agent_sdk/src/spans.ts | 39 +++++++++- .../traceai_claude_agent_sdk/src/types.ts | 2 + 7 files changed, 204 insertions(+), 8 deletions(-) diff --git a/typescript/packages/traceai_claude_agent_sdk/README.md b/typescript/packages/traceai_claude_agent_sdk/README.md index eb708c580..879b76b72 100644 --- a/typescript/packages/traceai_claude_agent_sdk/README.md +++ b/typescript/packages/traceai_claude_agent_sdk/README.md @@ -52,7 +52,7 @@ await shutdown(); // flush before a short script exits ### What the wrapper does and does not do - Calls the original `query()` with the same params object. Options (`resume`, `forkSession`, `sessionId`, `hooks`, `mcpServers`, `agents`, `abortController`, `allowedTools`, `permissionMode`, `maxTurns`, `env`, ...) are passed through untouched. The wrapper never sets `ANTHROPIC_BASE_URL`. -- Yields the same message objects, in order. Control methods on the returned `Query` (`interrupt()`, `setModel()`, ...) go to the original. `close()` and `Symbol.asyncDispose` first end the query's open spans as cancelled, then go to the original. +- Yields the same message objects, in order. Control methods on the returned `Query` (`interrupt()`, `setModel()`, ...) go to the original. `close()` and `Symbol.asyncDispose` first end the query's open spans, then go to the original: as cancelled if a turn, tool or subagent was still running, or as a normal end if a result had arrived and nothing was running (the usual way to end a streaming-input session). - A tracing error never fails the agent. An exporter or collector failure is logged and does not reject the iterator. - `startup()` is not wrapped in 0.1.0. @@ -69,7 +69,7 @@ One API response that the SDK splits over several assistant messages (same `mess ### Background subagents -A foreground subagent span ends with its `tool.Agent` / `tool.Task` result. A subagent that runs in the background stays open after that tool result (which is only the "running in the background" placeholder) until its `system/task_notification`, so its later turns and tools still nest under a live span. A subagent counts as background when the tool input has `run_in_background: true`, `task_started.is_backgrounded` is true, a later `task_updated` carries `patch.is_backgrounded: true`, or the app calls `Query.backgroundTasks()` (all foreground subagents, or the one whose `tool_use_id` it was given; undone if the call rejects or returns `false`). The subagent span is ERROR when the notification status is `failed` or `stopped` (`claude_agent.subagent.status` has the value). A background subagent with no notification by the time the query ends is closed as ERROR "Subagent span not completed". The subagent span can therefore end after its parent tool span. +A foreground subagent span ends with its `tool.Agent` / `tool.Task` result. A subagent that runs in the background stays open after that tool result (which is only the "running in the background" placeholder) until its `system/task_notification` (matched by `tool_use_id`, or by `task_id` through `task_started` when the notification has no `tool_use_id`), so its later turns and tools still nest under a live span. A subagent counts as background when the tool input has `run_in_background: true`, `task_started.is_backgrounded` is true, a later `task_updated` carries `patch.is_backgrounded: true`, or the app calls `Query.backgroundTasks()` (all foreground subagents, or the one whose `tool_use_id` it was given; undone if the call rejects or returns `false`). The subagent span is ERROR when the notification status is `failed` or `stopped` (`claude_agent.subagent.status` has the value). A background subagent with no notification by the time the query ends is closed as ERROR "Subagent span not completed". The subagent span can therefore end after its parent tool span. | Field | Attributes | Source | |---|---|---| @@ -92,7 +92,8 @@ The Future AGI collector promotes `gen_ai.usage.*` and `gen_ai.cost.total` on an - Tokens come from the latest result's `modelUsage`, summed over every model (main loop, subagents, compaction). Cost comes from `total_cost_usd`. Both are running totals for the session (`sdk.d.ts:5679`, `5687`). `result.usage` is not read: the SDK documents it as main-loop only and per turn in streaming-input mode (`sdk.d.ts:5683`). This replaces the spec's "result message usage" mapping. - Streaming input (one `query()`, several user turns) yields one result per turn. The conversation span keeps the latest running totals, so tokens and cost cover every turn. It stays one conversation span per `query()`; the Python `ClaudeSDKClient` path makes one per user turn. - A resumed (`resume`), continued (`continue`) or forked (`forkSession`) session's first result already carries the earlier turns. The wrapper keeps an in-process map of session id to the last totals it saw (at most 1000 sessions) and writes only the difference. A fork starts from its parent's totals. A drop in the running total within one query (a `/clear`) counts the spend on both sides of it. -- When there is no baseline in this process (resume after a restart, `continue` + `forkSession`, or a first result below the saved totals), no `gen_ai.usage.*` / `gen_ai.cost.total` / `claude_agent.cost.total_usd` is written and `claude_agent.usage.baseline_unknown=true`. +- When there is no baseline in this process (resume after a restart, `continue` + `forkSession`, `resume` with `resumeSessionAt`, or a first result below the saved totals), the first result writes no `gen_ai.usage.*` / `gen_ai.cost.total` / `claude_agent.cost.total_usd` and the span has `claude_agent.usage.baseline_unknown=true`. Later results in the same `query()` (streaming input) use that first result's totals as their baseline, so the promoted keys then hold the spend after the first result and the flag stays `true`. +- Known undercount: a `/clear` followed by more spend than the session had before it is not seen as a drop, so the pre-clear share of that query is missed. Nothing here can count spend twice. - Every conversation span with a result also carries the running totals on unpromoted keys: `claude_agent.cumulative.cost_usd`, `input_tokens`, `output_tokens`, `cache_read_tokens`, `cache_creation_tokens`. - A result with no cost and no `modelUsage`, or a zeroed crash result, writes none of these keys and leaves the session's saved totals unchanged. @@ -130,7 +131,7 @@ With 0.3.289 the CLI then calls `POST {ANTHROPIC_BASE_URL}/v1/messages?beta=true - **No traces.** `register()` needs `projectName` or `FI_PROJECT_NAME`. - **Zero spans from a short script.** `await shutdown()` before exit. - **Duplicate model spans.** Do not also run a provider instrumentor on the same calls. -- **A span left open after abort.** Every span for the query is ended with status ERROR and `claude_agent.cancelled=true` when its `AbortController` aborts, when you call `close()` on the returned `Query` (the SDK's abort path), or when `await using` disposes it (`Symbol.asyncDispose`). Calling `close()` after the stream has completed changes nothing. Anything else is a bug here. +- **A span left open after abort.** Every span for the query is ended with status ERROR and `claude_agent.cancelled=true` when its `AbortController` aborts, when you call `close()` on the returned `Query` (the SDK's abort path), or when `await using` disposes it (`Symbol.asyncDispose`). Calling `close()` after the stream has completed changes nothing, and `close()` after a result with nothing still running ends the conversation OK. Anything else is a bug here. ## Tests diff --git a/typescript/packages/traceai_claude_agent_sdk/contract/run_real_sdk.mjs b/typescript/packages/traceai_claude_agent_sdk/contract/run_real_sdk.mjs index 04b07d84c..20b8fc9ed 100644 --- a/typescript/packages/traceai_claude_agent_sdk/contract/run_real_sdk.mjs +++ b/typescript/packages/traceai_claude_agent_sdk/contract/run_real_sdk.mjs @@ -22,6 +22,9 @@ // starts a session, PHASE=second resumes RESUME_SESSION_ID. // streaming one query() with an AsyncIterable prompt of two user turns; // the second turn is sent after the first result. +// streaming_close one streaming-input query(); after its result the app +// calls close() while the prompt iterable is still open (the +// usual way to end a streaming-input session). // WORKDIR (optional): use and keep this directory instead of a temp dir. // Prints {"requests": [...], "messages": [...], "queries": [[...], ...]} on stdout. import http from "node:http"; @@ -221,6 +224,26 @@ const SCENARIOS = { } return [out]; }, + async streaming_close() { + let release; + const held = new Promise((resolve) => (release = resolve)); + const userTurn = (content) => ({ type: "user", message: { role: "user", content }, parent_tool_use_id: null }); + async function* prompt() { + yield userTurn("First question."); + await held; // more input could still come: the stream stays open + } + const q = tracedQuery({ prompt: prompt(), options: options() }); + const out = []; + for (;;) { + const { value, done } = await q.next(); + if (done) break; + out.push(value); + if (value.type === "result") break; + } + q.close(); + release(); + return [out]; + }, }; function sessionOf(messages) { diff --git a/typescript/packages/traceai_claude_agent_sdk/contract/test_harness_contract.py b/typescript/packages/traceai_claude_agent_sdk/contract/test_harness_contract.py index 6826c8068..6f8ae4f80 100644 --- a/typescript/packages/traceai_claude_agent_sdk/contract/test_harness_contract.py +++ b/typescript/packages/traceai_claude_agent_sdk/contract/test_harness_contract.py @@ -478,6 +478,27 @@ def test_real_sdk_streaming_input_counts_tokens_once(built_package: Path) -> Non _assert_only_conversations_carry_usage(spans) +@requires_real_sdk +def test_real_sdk_close_after_result_ends_streaming_session_ok(built_package: Path) -> None: + """N1: close() after a result, with nothing still running, is the normal end of a + streaming-input session (the prompt iterable stays open). The conversation must end + OK, not cancelled, and still carry the session's spend once.""" + output, spans, exported = _run_real_sdk("streaming_close", "th8235-real-sdk-streaming-close") + (messages,) = output["queries"] + results = _results(messages) + assert len(results) == 1 and messages[-1]["type"] == "result" + + names = sorted(span["name"] for span in spans) + assert names == sorted(["claude_agent.conversation", "claude_agent.assistant_turn"]), names + for span in spans: + assert "claude_agent.cancelled" not in _attrs(span), span["name"] + conversation = _one(spans, "claude_agent.conversation") + assert _status_code(conversation) == 1 + _assert_promoted_sum_equals([conversation], _cumulative(results[0])) + _assert_only_conversations_carry_usage(spans) + _assert_exported_with_both_keys(exported, "th8235-real-sdk-streaming-close") + + @requires_real_sdk def test_real_sdk_query_through_anthropic_base_url_mock(built_package: Path) -> None: """The real SDK query() and bundled CLI, wrapped, against a loopback Messages API mock. diff --git a/typescript/packages/traceai_claude_agent_sdk/src/__tests__/usage.test.ts b/typescript/packages/traceai_claude_agent_sdk/src/__tests__/usage.test.ts index 21c490bd0..b756e5f45 100644 --- a/typescript/packages/traceai_claude_agent_sdk/src/__tests__/usage.test.ts +++ b/typescript/packages/traceai_claude_agent_sdk/src/__tests__/usage.test.ts @@ -268,6 +268,50 @@ describe("R1: resumed, continued and forked sessions are counted once", () => { expect(conversation.attributes["claude_agent.usage.baseline_unknown"]).toBe(true); }); + it("N3: an unknown baseline covers only the first result; later results in the query promote their delta", async () => { + const id = newSessionId(); // resumed, but this process never saw the session (e.g. after a restart) + const journey: SDKMessage[] = [ + init({ session_id: id }), + assistant("msg_u1", [text("one")], null, { session_id: id }), + resultSuccess("one", { session_id: id, total_cost_usd: 0.001, modelUsage: { [MODEL]: modelUsage(100, 40, 0.001) } }), + assistant("msg_u2", [text("two")], null, { session_id: id }), + resultSuccess("two", { session_id: id, total_cost_usd: 0.0014, modelUsage: { [MODEL]: modelUsage(130, 52, 0.0014) } }), + ]; + const { conversation } = await runQuery(journey, { resume: id }); + expect(conversation.attributes["gen_ai.cost.total"]).toBeCloseTo(0.0004, 12); + expect(conversation.attributes["claude_agent.cost.total_usd"]).toBeCloseTo(0.0004, 12); + expect(conversation.attributes["gen_ai.usage.input_tokens"]).toBe(30); + expect(conversation.attributes["gen_ai.usage.output_tokens"]).toBe(12); + expect(conversation.attributes["gen_ai.usage.total_tokens"]).toBe(42); + // Still flagged: the spend before the first result is not in the promoted keys. + expect(conversation.attributes["claude_agent.usage.baseline_unknown"]).toBe(true); + expect(conversation.attributes["claude_agent.cumulative.cost_usd"]).toBeCloseTo(0.0014, 12); + // The next resume in this process starts from the latest totals. + const next = await runQuery(sessionQuery(id, { cost: 0.002, input: 160, output: 64 }), { resume: id }); + expect(next.conversation.attributes["gen_ai.cost.total"]).toBeCloseTo(0.0006, 12); + expect(next.conversation.attributes["claude_agent.usage.baseline_unknown"]).toBe(false); + }); + + it.each([ + ["resume", false], + ["fork", true], + ] as const)( + "N4: %s with resumeSessionAt has no known baseline (the saved totals are for a later message)", + async (_mode, forkSession) => { + const id = newSessionId(); + await runQuery(sessionQuery(id, { cost: 0.001, input: 100, output: 40 })); + const target = forkSession ? newSessionId() : id; + const { conversation } = await runQuery(sessionQuery(target, { cost: 0.0015, input: 150, output: 60 }), { + resume: id, + resumeSessionAt: "msg-uuid-earlier", + ...(forkSession ? { forkSession: true } : {}), + }); + expect(promoted(conversation)).toEqual({}); + expect(conversation.attributes["claude_agent.cost.total_usd"]).toBeUndefined(); + expect(conversation.attributes["claude_agent.usage.baseline_unknown"]).toBe(true); + }, + ); + it("a mid-query /clear (running total drops) counts the spend before and after it", async () => { const id = newSessionId(); const journey: SDKMessage[] = [ diff --git a/typescript/packages/traceai_claude_agent_sdk/src/__tests__/wrapper.test.ts b/typescript/packages/traceai_claude_agent_sdk/src/__tests__/wrapper.test.ts index 9f9c5ee17..ddb4e37e3 100644 --- a/typescript/packages/traceai_claude_agent_sdk/src/__tests__/wrapper.test.ts +++ b/typescript/packages/traceai_claude_agent_sdk/src/__tests__/wrapper.test.ts @@ -1,3 +1,4 @@ +import type { SDKMessage } from "@anthropic-ai/claude-agent-sdk"; import { SpanStatusCode, context, trace } from "@opentelemetry/api"; import { BasicTracerProvider, @@ -423,6 +424,20 @@ describe("wrapQuery", () => { expect(subagent.status.code).toBe(SpanStatusCode.ERROR); }, ); + + it("N2: matches a task_notification without tool_use_id (optional, sdk.d.ts:5997) by its task_id", async () => { + const journey = backgroundedSubagentJourney().map((m) => { + if (m.type === "system" && m.subtype === "task_notification") { + const { tool_use_id: _dropped, ...rest } = m as typeof m & { tool_use_id?: string }; + return rest as SDKMessage; + } + return m; + }); + const { spans } = await run(journey); + const subagent = one(spans, "claude_agent.subagent.code-reviewer"); + expect(subagent.attributes["claude_agent.subagent.status"]).toBe("completed"); + expect(subagent.status.code).toBe(SpanStatusCode.OK); + }); }); describe("J3: MCP tool error and error result", () => { @@ -618,6 +633,65 @@ describe("wrapQuery", () => { } }); + /** Read until the stream has yielded a result message (done is still false). */ + async function readThroughResult(q: AsyncGenerator): Promise { + for (;;) { + const step = await q.next(); + if (step.done) throw new Error("stream ended before a result"); + if (step.value.type === "result") return; + } + } + + it.each([ + ["close", "simpleToolJourney"], + ["asyncDispose", "simpleToolJourney"], + ["close", "streamingInputJourney"], + ["asyncDispose", "streamingInputJourney"], + ] as const)( + "N1: %s() after a result with nothing in flight ends the conversation OK, not cancelled (%s)", + async (method, journeyName) => { + const { provider, exporter } = memoryProvider(); + const journey = journeyName === "simpleToolJourney" ? simpleToolJourney() : streamingInputJourney(); + const fake = makeFakeQuery(journey); + const q = wrapQuery(fake.query, { tracerProvider: provider })({ prompt: PROMPT }); + await readThroughResult(q); + expect(exporter.getFinishedSpans().some((s) => s.name === "claude_agent.conversation")).toBe(false); + + if (method === "close") { + q.close(); + } else { + await (q as unknown as Record Promise>)[ASYNC_DISPOSE](); + } + + const spans = exporter.getFinishedSpans(); + const conversation = one(spans, "claude_agent.conversation"); + expect(conversation.status.code).toBe(SpanStatusCode.OK); + expect(conversation.attributes["claude_agent.cancelled"]).toBeUndefined(); + for (const span of spans) { + expect(span.ended).toBe(true); + expect(span.attributes["claude_agent.cancelled"]).toBeUndefined(); + } + }, + ); + + it("N1: close() after a result while a background subagent still runs stays cancelled", async () => { + const { provider, exporter } = memoryProvider(); + // The result arrives before the background subagent's task_notification. + const journey = backgroundedSubagentJourney().filter( + (m) => !(m.type === "system" && m.subtype === "task_notification"), + ); + const fake = makeFakeQuery(journey); + const q = wrapQuery(fake.query, { tracerProvider: provider })({ prompt: PROMPT }); + await readThroughResult(q); + q.close(); + const spans = exporter.getFinishedSpans(); + const subagent = one(spans, "claude_agent.subagent.code-reviewer"); + expect(subagent.attributes["claude_agent.cancelled"]).toBe(true); + const conversation = one(spans, "claude_agent.conversation"); + expect(conversation.status.code).toBe(SpanStatusCode.ERROR); + expect(conversation.attributes["claude_agent.cancelled"]).toBe(true); + }); + it("ends open spans when the consumer breaks out early", async () => { const { provider, exporter } = memoryProvider(); const fake = makeFakeQuery(simpleToolJourney()); diff --git a/typescript/packages/traceai_claude_agent_sdk/src/spans.ts b/typescript/packages/traceai_claude_agent_sdk/src/spans.ts index f65251a6d..26b766715 100644 --- a/typescript/packages/traceai_claude_agent_sdk/src/spans.ts +++ b/typescript/packages/traceai_claude_agent_sdk/src/spans.ts @@ -302,6 +302,8 @@ export class QueryTracer { private readonly resumeId?: string; /** undefined: no counted result yet; null: baseline unknown for this query. */ private usagePrev?: UsageTotals | null; + /** Some spend before this query's first result is not in the promoted keys. */ + private usageBaselineUnknown = false; /** New spend counted in this query (sum of deltas), written to the promoted keys. */ private readonly usageCounted: UsageTotals = {}; @@ -400,6 +402,11 @@ export class QueryTracer { if (!this.conversation) { return; } + // close()/asyncDispose/abort after a result with nothing still running + // (no open turn, tool or subagent) cancels nothing: end it as returned. + if (outcome.kind === "aborted" && this.resultSeen && this.nothingInFlight()) { + outcome = { kind: "returned" }; + } try { const signal = this.options?.abortController?.signal; const cancelled = @@ -467,6 +474,18 @@ export class QueryTracer { } } + /** True when no turn, tool or subagent span of this query is still open. */ + private nothingInFlight(): boolean { + if (this.tools.size > 0) return false; + for (const sub of this.subagents.values()) { + if (!sub.ended) return false; + } + for (const scope of this.scopes.values()) { + if (scope.currentTurn && !scope.currentTurn.ended) return false; + } + return true; + } + // -------------------------------------------------------------------------- // Message handlers // -------------------------------------------------------------------------- @@ -641,7 +660,10 @@ export class QueryTracer { break; } case "task_notification": { - const id = message.tool_use_id; + // tool_use_id is optional (sdk.d.ts:5997); fall back to the task_started mapping. + const id = + message.tool_use_id ?? + (typeof message.task_id === "string" ? this.taskToolUseIds.get(message.task_id) : undefined); const sub = id ? this.subagents.get(id) : undefined; if (sub && !sub.ended && typeof message.status === "string") { sub.status = message.status; @@ -988,8 +1010,8 @@ export class QueryTracer { (f) => cumulative[f] !== undefined && prev![f] !== undefined && cumulative[f]! < prev![f]!, ); if (dropped && first) { - // Below the saved baseline on the first result (e.g. resumeSessionAt an - // earlier message): the earlier share is unknown. + // Below the saved baseline on the first result (the transcript no longer + // matches the saved totals): the earlier share is unknown. prev = null; } else { // Within one query a drop is a /clear: the running total restarted at 0. @@ -1003,6 +1025,12 @@ export class QueryTracer { } this.usagePrev = prev; } + if (prev === null) { + // No baseline for the spend up to this result, but its totals are an exact + // baseline for any later result in the same query (streaming input). + this.usageBaselineUnknown = true; + this.usagePrev = { ...definedUsage(cumulative) }; + } if (this.sessionId) { const stored = prev ?? { ...(store.get(this.sessionId) ?? {}), ...definedUsage(cumulative) }; store.set(this.sessionId, stored); @@ -1016,7 +1044,7 @@ export class QueryTracer { setIf(T.CUMULATIVE_OUTPUT_TOKENS, cumulative.outputTokens); setIf(T.CUMULATIVE_CACHE_READ_TOKENS, cumulative.cacheReadTokens); setIf(T.CUMULATIVE_CACHE_CREATION_TOKENS, cumulative.cacheCreationTokens); - conversation.setAttribute(T.USAGE_BASELINE_UNKNOWN, prev === null); + conversation.setAttribute(T.USAGE_BASELINE_UNKNOWN, this.usageBaselineUnknown); if (prev === null) return; const counted = this.usageCounted; @@ -1039,6 +1067,9 @@ export class QueryTracer { return { costUsd: 0, inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheCreationTokens: 0 }; case "resume": case "fork": + // resumeSessionAt rewinds to an earlier message: the saved totals are for + // a later point in the transcript, so the share already spent is unknown. + if (typeof this.options?.resumeSessionAt === "string" && this.options.resumeSessionAt) return null; // A fork starts from the parent's saved totals under a new session id. return (this.resumeId && store.get(this.resumeId)) || null; case "continue": diff --git a/typescript/packages/traceai_claude_agent_sdk/src/types.ts b/typescript/packages/traceai_claude_agent_sdk/src/types.ts index 4a5f1bd76..5b9ddb0c6 100644 --- a/typescript/packages/traceai_claude_agent_sdk/src/types.ts +++ b/typescript/packages/traceai_claude_agent_sdk/src/types.ts @@ -127,6 +127,8 @@ export interface OptionsLike { /** Continue the most recent conversation in cwd (sdk.d.ts:1594). */ continue?: boolean; forkSession?: boolean; + /** Resume the transcript only up to this message uuid (sdk.d.ts:2101). */ + resumeSessionAt?: string; sessionId?: string; mcpServers?: Record; [key: string]: unknown; From a8831d69cf5ec6cd64927d20b51d7c59730a92da Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 4 Oct 2026 22:36:46 -0700 Subject: [PATCH 12/19] docs(claude-agent-sdk): V1: promoted usage is exact only while a session stays in one process The usage baseline is process-local. Resuming one session from several processes can promote spend made in another process again. Documented, with claude_agent.cumulative.* as the exact source for those deployments. Rejected: a cross-process store (shared state and a new dependency) and dropping promotion on every resume (loses resumed spend in the common single-process case). approved: Nikhil 2026-10-03 blanket Refs: TH-8235 --- typescript/packages/traceai_claude_agent_sdk/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/typescript/packages/traceai_claude_agent_sdk/README.md b/typescript/packages/traceai_claude_agent_sdk/README.md index 879b76b72..899303047 100644 --- a/typescript/packages/traceai_claude_agent_sdk/README.md +++ b/typescript/packages/traceai_claude_agent_sdk/README.md @@ -93,7 +93,8 @@ The Future AGI collector promotes `gen_ai.usage.*` and `gen_ai.cost.total` on an - Streaming input (one `query()`, several user turns) yields one result per turn. The conversation span keeps the latest running totals, so tokens and cost cover every turn. It stays one conversation span per `query()`; the Python `ClaudeSDKClient` path makes one per user turn. - A resumed (`resume`), continued (`continue`) or forked (`forkSession`) session's first result already carries the earlier turns. The wrapper keeps an in-process map of session id to the last totals it saw (at most 1000 sessions) and writes only the difference. A fork starts from its parent's totals. A drop in the running total within one query (a `/clear`) counts the spend on both sides of it. - When there is no baseline in this process (resume after a restart, `continue` + `forkSession`, `resume` with `resumeSessionAt`, or a first result below the saved totals), the first result writes no `gen_ai.usage.*` / `gen_ai.cost.total` / `claude_agent.cost.total_usd` and the span has `claude_agent.usage.baseline_unknown=true`. Later results in the same `query()` (streaming input) use that first result's totals as their baseline, so the promoted keys then hold the spend after the first result and the flag stays `true`. -- Known undercount: a `/clear` followed by more spend than the session had before it is not seen as a drop, so the pre-clear share of that query is missed. Nothing here can count spend twice. +- Known undercount: a `/clear` followed by more spend than the session had before it is not seen as a drop, so the pre-clear share of that query is missed. +- **One process per session.** The baseline lives in the process that ran the session's last query. Within one process nothing is counted twice. If the same session is resumed from more than one process (for example workers behind a load balancer with no session affinity), a worker's saved totals can be older than the transcript, and the spend made in between is promoted again. Keep a session on one worker, or use `claude_agent.cumulative.*` (always the session's running totals) instead of the promoted keys for those sessions. - Every conversation span with a result also carries the running totals on unpromoted keys: `claude_agent.cumulative.cost_usd`, `input_tokens`, `output_tokens`, `cache_read_tokens`, `cache_creation_tokens`. - A result with no cost and no `modelUsage`, or a zeroed crash result, writes none of these keys and leaves the session's saved totals unchanged. From a1dfe627104654867d4b35dee37564dfe910a308 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 4 Oct 2026 22:45:23 -0700 Subject: [PATCH 13/19] fix(claude-agent-sdk): review-r3 M1: only an idle query right after a result ends OK on close The N1 shortcut keyed off resultSeen, which never resets, so an abort in a later streaming turn between model steps (turn span ended, next assistant message not yet out) was recorded as OK. Track idleAfterResult instead: a result sets it, any later assistant/user message clears it. close(), asyncDispose and AbortController aborts mid-turn are cancelled again. README names the residual gap (abort after a new prompt is sent, before the SDK yields anything for it). Comments in queryWrapper.ts refreshed. M2 (reset the in-query baseline on conversation_reset) not taken: whether the CLI's running totals restart there is unverified; the README keeps the documented /clear undercount. Test first: 3 new jest cases (close, asyncDispose, abort) failed on a8831d6. approved: Nikhil 2026-10-03 blanket Refs: TH-8235 --- .../traceai_claude_agent_sdk/README.md | 4 +- .../src/__tests__/wrapper.test.ts | 42 +++++++++++++++++++ .../src/queryWrapper.ts | 6 ++- .../traceai_claude_agent_sdk/src/spans.ts | 14 +++++-- 4 files changed, 59 insertions(+), 7 deletions(-) diff --git a/typescript/packages/traceai_claude_agent_sdk/README.md b/typescript/packages/traceai_claude_agent_sdk/README.md index 899303047..17d483bfb 100644 --- a/typescript/packages/traceai_claude_agent_sdk/README.md +++ b/typescript/packages/traceai_claude_agent_sdk/README.md @@ -52,7 +52,7 @@ await shutdown(); // flush before a short script exits ### What the wrapper does and does not do - Calls the original `query()` with the same params object. Options (`resume`, `forkSession`, `sessionId`, `hooks`, `mcpServers`, `agents`, `abortController`, `allowedTools`, `permissionMode`, `maxTurns`, `env`, ...) are passed through untouched. The wrapper never sets `ANTHROPIC_BASE_URL`. -- Yields the same message objects, in order. Control methods on the returned `Query` (`interrupt()`, `setModel()`, ...) go to the original. `close()` and `Symbol.asyncDispose` first end the query's open spans, then go to the original: as cancelled if a turn, tool or subagent was still running, or as a normal end if a result had arrived and nothing was running (the usual way to end a streaming-input session). +- Yields the same message objects, in order. Control methods on the returned `Query` (`interrupt()`, `setModel()`, ...) go to the original. `close()` and `Symbol.asyncDispose` first end the query's open spans, then go to the original: as cancelled while the query is working, or as a normal end when it is idle right after a result (the usual way to end a streaming-input session). - A tracing error never fails the agent. An exporter or collector failure is logged and does not reject the iterator. - `startup()` is not wrapped in 0.1.0. @@ -132,7 +132,7 @@ With 0.3.289 the CLI then calls `POST {ANTHROPIC_BASE_URL}/v1/messages?beta=true - **No traces.** `register()` needs `projectName` or `FI_PROJECT_NAME`. - **Zero spans from a short script.** `await shutdown()` before exit. - **Duplicate model spans.** Do not also run a provider instrumentor on the same calls. -- **A span left open after abort.** Every span for the query is ended with status ERROR and `claude_agent.cancelled=true` when its `AbortController` aborts, when you call `close()` on the returned `Query` (the SDK's abort path), or when `await using` disposes it (`Symbol.asyncDispose`). Calling `close()` after the stream has completed changes nothing, and `close()` after a result with nothing still running ends the conversation OK. Anything else is a bug here. +- **A span left open after abort.** Every span for the query is ended with status ERROR and `claude_agent.cancelled=true` when its `AbortController` aborts, when you call `close()` on the returned `Query` (the SDK's abort path), or when `await using` disposes it (`Symbol.asyncDispose`). Calling `close()` after the stream has completed changes nothing. Right after a result, before any further assistant or user message and with no background subagent running, `close()`, `Symbol.asyncDispose` and an abort end the conversation OK instead, because nothing was cancelled. One gap remains: an abort after the app sends a new streaming-input prompt but before the SDK yields any message for it is recorded as OK. Anything else is a bug here. ## Tests diff --git a/typescript/packages/traceai_claude_agent_sdk/src/__tests__/wrapper.test.ts b/typescript/packages/traceai_claude_agent_sdk/src/__tests__/wrapper.test.ts index ddb4e37e3..c8513412f 100644 --- a/typescript/packages/traceai_claude_agent_sdk/src/__tests__/wrapper.test.ts +++ b/typescript/packages/traceai_claude_agent_sdk/src/__tests__/wrapper.test.ts @@ -26,12 +26,18 @@ import { TOOL_INPUT_MARKER, TOOL_OUTPUT_MARKER, TOTAL_COST_USD, + assistant, backgroundedSubagentJourney, + init, mcpErrorJourney, + resultSuccess, simpleToolJourney, streamingInputJourney, subagentJourney, taskNotification, + text, + toolResult, + toolUse, } from "./fixtures/messages"; import { StackContextManager, allAttributeText, byName, drain, memoryProvider, one, parentId } from "./helpers"; @@ -674,6 +680,42 @@ describe("wrapQuery", () => { }, ); + /** Streaming input: turn 1 ends with a result, turn 2 is mid-tool-loop (its turn span already ended). */ + function secondTurnBetweenModelSteps(): SDKMessage[] { + return [ + init(), + assistant("msg_s1", [text("First answer.")]), + resultSuccess("First answer.", { num_turns: 1 }), + assistant("msg_s2", [toolUse("toolu_read_2", "Read", { file_path: "/tmp/b.md" })]), + toolResult("toolu_read_2", "# b"), + assistant("msg_s3", [text("Second answer.")]), + resultSuccess("Second answer.", { num_turns: 2 }), + ]; + } + + it.each(["close", "asyncDispose", "abort"] as const)( + "M1: %s() in a later streaming turn, between model steps, stays cancelled", + async (method) => { + const { provider, exporter } = memoryProvider(); + const abortController = new AbortController(); + const fake = makeFakeQuery(secondTurnBetweenModelSteps()); + const q = wrapQuery(fake.query, { tracerProvider: provider })({ prompt: PROMPT, options: { abortController } }); + // Read through turn 2's tool_result: the turn span has ended, the next model step has no span yet. + for (;;) { + const step = await q.next(); + if (step.done) throw new Error("stream ended early"); + if (step.value.type === "user") break; + } + if (method === "close") q.close(); + else if (method === "asyncDispose") await (q as unknown as Record Promise>)[ASYNC_DISPOSE](); + else abortController.abort(); + + const conversation = one(exporter.getFinishedSpans(), "claude_agent.conversation"); + expect(conversation.status.code).toBe(SpanStatusCode.ERROR); + expect(conversation.attributes["claude_agent.cancelled"]).toBe(true); + }, + ); + it("N1: close() after a result while a background subagent still runs stays cancelled", async () => { const { provider, exporter } = memoryProvider(); // The result arrives before the background subagent's task_notification. diff --git a/typescript/packages/traceai_claude_agent_sdk/src/queryWrapper.ts b/typescript/packages/traceai_claude_agent_sdk/src/queryWrapper.ts index ef663d555..bb689fad8 100644 --- a/typescript/packages/traceai_claude_agent_sdk/src/queryWrapper.ts +++ b/typescript/packages/traceai_claude_agent_sdk/src/queryWrapper.ts @@ -7,7 +7,8 @@ * forwards every control method (`interrupt()`, `setModel()`, ...) to the * original. `next`, `return`, `throw` and `Symbol.asyncIterator` are * intercepted to drive the span model in `spans.ts`; `close()` and - * `Symbol.asyncDispose` end the open spans as cancelled, then forward. + * `Symbol.asyncDispose` end the open spans (as cancelled unless the query was + * idle right after a result), then forward. */ import { Tracer, context as otelContext, diag } from "@opentelemetry/api"; import { ContentPolicy, QueryTracer, clockMs } from "./spans"; @@ -128,7 +129,8 @@ function proxyQuery>( if (typeof value !== "function") return value; // close() is the SDK's abort path (sdk.d.ts Query.close) and asyncDispose // runs on `await using`: neither goes through next/return/throw, so end - // every open span as cancelled before forwarding. + // every open span before forwarding (cancelled, unless the query was idle + // right after a result: see QueryTracer.finish). if (property === "close") { return function close(this: unknown, ...args: unknown[]) { safe(() => queryTracer.finish({ kind: "aborted", reason: "close" })); diff --git a/typescript/packages/traceai_claude_agent_sdk/src/spans.ts b/typescript/packages/traceai_claude_agent_sdk/src/spans.ts index 26b766715..57973fff5 100644 --- a/typescript/packages/traceai_claude_agent_sdk/src/spans.ts +++ b/typescript/packages/traceai_claude_agent_sdk/src/spans.ts @@ -287,6 +287,8 @@ export class QueryTracer { private conversationContext?: Context; private conversationIsError = false; private resultSeen = false; + /** A result arrived and no assistant/user message has followed it yet. */ + private idleAfterResult = false; private sessionId?: string; private finished = false; private abortListener?: () => void; @@ -378,6 +380,9 @@ export class QueryTracer { } try { const type = (message as { type?: unknown }).type; + if (type === "assistant" || type === "user") { + this.idleAfterResult = false; + } if (type === "assistant") { this.onAssistant(message as AssistantMessageLike); } else if (type === "user") { @@ -402,9 +407,11 @@ export class QueryTracer { if (!this.conversation) { return; } - // close()/asyncDispose/abort after a result with nothing still running - // (no open turn, tool or subagent) cancels nothing: end it as returned. - if (outcome.kind === "aborted" && this.resultSeen && this.nothingInFlight()) { + // close()/asyncDispose/abort right after a result, with nothing still + // running (no open turn, tool or subagent), cancels nothing: end it as + // returned. Any assistant/user message after the result means a new turn + // is under way, even between model steps when no span is open. + if (outcome.kind === "aborted" && this.idleAfterResult && this.nothingInFlight()) { outcome = { kind: "returned" }; } try { @@ -576,6 +583,7 @@ export class QueryTracer { this.setSessionId(message.session_id); } + this.idleAfterResult = true; this.recordUsage(cumulativeUsage(message)); if (typeof message.duration_ms === "number") { conversation.setAttribute(A.DURATION_MS, message.duration_ms); From 005a70f15834348d8005b1832d462f8a61c07e69 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 4 Oct 2026 22:47:58 -0700 Subject: [PATCH 14/19] fix(claude-agent-sdk): verify-r3 M1 note: only main-loop messages end the idle-after-result state Background subagent messages (parent_tool_use_id set) that arrive after the main loop's result no longer clear idleAfterResult, so close() after that subagent's task_notification ends the conversation OK; a subagent still open stays cancelled through nothingInFlight(). Test first: the new post-result background subagent case failed on a1dfe62. approved: Nikhil 2026-10-03 blanket Refs: TH-8235 --- .../traceai_claude_agent_sdk/README.md | 2 +- .../src/__tests__/wrapper.test.ts | 23 +++++++++++++++++++ .../traceai_claude_agent_sdk/src/spans.ts | 14 +++++++---- 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/typescript/packages/traceai_claude_agent_sdk/README.md b/typescript/packages/traceai_claude_agent_sdk/README.md index 17d483bfb..75fce8a77 100644 --- a/typescript/packages/traceai_claude_agent_sdk/README.md +++ b/typescript/packages/traceai_claude_agent_sdk/README.md @@ -132,7 +132,7 @@ With 0.3.289 the CLI then calls `POST {ANTHROPIC_BASE_URL}/v1/messages?beta=true - **No traces.** `register()` needs `projectName` or `FI_PROJECT_NAME`. - **Zero spans from a short script.** `await shutdown()` before exit. - **Duplicate model spans.** Do not also run a provider instrumentor on the same calls. -- **A span left open after abort.** Every span for the query is ended with status ERROR and `claude_agent.cancelled=true` when its `AbortController` aborts, when you call `close()` on the returned `Query` (the SDK's abort path), or when `await using` disposes it (`Symbol.asyncDispose`). Calling `close()` after the stream has completed changes nothing. Right after a result, before any further assistant or user message and with no background subagent running, `close()`, `Symbol.asyncDispose` and an abort end the conversation OK instead, because nothing was cancelled. One gap remains: an abort after the app sends a new streaming-input prompt but before the SDK yields any message for it is recorded as OK. Anything else is a bug here. +- **A span left open after abort.** Every span for the query is ended with status ERROR and `claude_agent.cancelled=true` when its `AbortController` aborts, when you call `close()` on the returned `Query` (the SDK's abort path), or when `await using` disposes it (`Symbol.asyncDispose`). Calling `close()` after the stream has completed changes nothing. Right after a result, before any further main-loop assistant or user message and with no background subagent still running, `close()`, `Symbol.asyncDispose` and an abort end the conversation OK instead, because nothing was cancelled. One gap remains: an abort after the app sends a new streaming-input prompt but before the SDK yields any message for it is recorded as OK. Anything else is a bug here. ## Tests diff --git a/typescript/packages/traceai_claude_agent_sdk/src/__tests__/wrapper.test.ts b/typescript/packages/traceai_claude_agent_sdk/src/__tests__/wrapper.test.ts index c8513412f..c9dd630a7 100644 --- a/typescript/packages/traceai_claude_agent_sdk/src/__tests__/wrapper.test.ts +++ b/typescript/packages/traceai_claude_agent_sdk/src/__tests__/wrapper.test.ts @@ -716,6 +716,29 @@ describe("wrapQuery", () => { }, ); + it("M1: close() after a background subagent finishes post-result (its messages arrive after the result) ends OK", async () => { + const { provider, exporter } = memoryProvider(); + const base = backgroundedSubagentJourney(); + const result = base[base.length - 1]; + const idx = base.findIndex((m) => m.type === "assistant" && m.message.id === "msg_13"); + // Main loop answers and yields its result; the background subagent's tool_result, + // final turn and task_notification arrive afterwards. + const journey = [...base.slice(0, idx + 1), result, ...base.slice(idx + 1, base.length - 1)]; + const fake = makeFakeQuery(journey); + const q = wrapQuery(fake.query, { tracerProvider: provider })({ prompt: PROMPT }); + for (;;) { + const step = await q.next(); + if (step.done) throw new Error("stream ended early"); + if (step.value.type === "system" && step.value.subtype === "task_notification") break; + } + q.close(); + const spans = exporter.getFinishedSpans(); + const conversation = one(spans, "claude_agent.conversation"); + expect(conversation.status.code).toBe(SpanStatusCode.OK); + expect(conversation.attributes["claude_agent.cancelled"]).toBeUndefined(); + expect(one(spans, "claude_agent.subagent.code-reviewer").status.code).toBe(SpanStatusCode.OK); + }); + it("N1: close() after a result while a background subagent still runs stays cancelled", async () => { const { provider, exporter } = memoryProvider(); // The result arrives before the background subagent's task_notification. diff --git a/typescript/packages/traceai_claude_agent_sdk/src/spans.ts b/typescript/packages/traceai_claude_agent_sdk/src/spans.ts index 57973fff5..ceece9ff2 100644 --- a/typescript/packages/traceai_claude_agent_sdk/src/spans.ts +++ b/typescript/packages/traceai_claude_agent_sdk/src/spans.ts @@ -287,7 +287,7 @@ export class QueryTracer { private conversationContext?: Context; private conversationIsError = false; private resultSeen = false; - /** A result arrived and no assistant/user message has followed it yet. */ + /** A result arrived and no main-loop assistant/user message has followed it yet. */ private idleAfterResult = false; private sessionId?: string; private finished = false; @@ -380,7 +380,13 @@ export class QueryTracer { } try { const type = (message as { type?: unknown }).type; - if (type === "assistant" || type === "user") { + if ( + (type === "assistant" || type === "user") && + ((message as { parent_tool_use_id?: unknown }).parent_tool_use_id ?? null) === null + ) { + // Main-loop activity after a result starts a new turn. Background + // subagent messages (parent_tool_use_id set) do not: nothingInFlight() + // already covers subagents that are still open. this.idleAfterResult = false; } if (type === "assistant") { @@ -409,8 +415,8 @@ export class QueryTracer { } // close()/asyncDispose/abort right after a result, with nothing still // running (no open turn, tool or subagent), cancels nothing: end it as - // returned. Any assistant/user message after the result means a new turn - // is under way, even between model steps when no span is open. + // returned. Any main-loop assistant/user message after the result means a + // new turn is under way, even between model steps when no span is open. if (outcome.kind === "aborted" && this.idleAfterResult && this.nothingInFlight()) { outcome = { kind: "returned" }; } From 4cf737f9d2d100118da08cda40c3242ea70ee51b Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 4 Oct 2026 23:00:36 -0700 Subject: [PATCH 15/19] fix(claude-agent-sdk): verify-r4: notifications, stream events and session state drive the idle-after-result state R4-1: a background task_notification is queued as a main-thread command that runs its own turn (sdk.d.ts:5440, 4603), so close() after one is a cancellation again (supersedes 005a70f's expectation). R4-2: a main-loop stream_event (includePartialMessages) after a result also starts a turn. system/session_state_changed is authoritative when sent (sdk.d.ts:5858): 'idle' after a result ends OK, 'running' cancelled. README updated. Test first: 3 new jest cases failed on 005a70f. approved: Nikhil 2026-10-03 blanket Refs: TH-8235 --- .../traceai_claude_agent_sdk/README.md | 2 +- .../src/__tests__/wrapper.test.ts | 71 ++++++++++++++++--- .../traceai_claude_agent_sdk/src/spans.ts | 36 +++++++--- 3 files changed, 88 insertions(+), 21 deletions(-) diff --git a/typescript/packages/traceai_claude_agent_sdk/README.md b/typescript/packages/traceai_claude_agent_sdk/README.md index 75fce8a77..8c90e66ca 100644 --- a/typescript/packages/traceai_claude_agent_sdk/README.md +++ b/typescript/packages/traceai_claude_agent_sdk/README.md @@ -132,7 +132,7 @@ With 0.3.289 the CLI then calls `POST {ANTHROPIC_BASE_URL}/v1/messages?beta=true - **No traces.** `register()` needs `projectName` or `FI_PROJECT_NAME`. - **Zero spans from a short script.** `await shutdown()` before exit. - **Duplicate model spans.** Do not also run a provider instrumentor on the same calls. -- **A span left open after abort.** Every span for the query is ended with status ERROR and `claude_agent.cancelled=true` when its `AbortController` aborts, when you call `close()` on the returned `Query` (the SDK's abort path), or when `await using` disposes it (`Symbol.asyncDispose`). Calling `close()` after the stream has completed changes nothing. Right after a result, before any further main-loop assistant or user message and with no background subagent still running, `close()`, `Symbol.asyncDispose` and an abort end the conversation OK instead, because nothing was cancelled. One gap remains: an abort after the app sends a new streaming-input prompt but before the SDK yields any message for it is recorded as OK. Anything else is a bug here. +- **A span left open after abort.** Every span for the query is ended with status ERROR and `claude_agent.cancelled=true` when its `AbortController` aborts, when you call `close()` on the returned `Query` (the SDK's abort path), or when `await using` disposes it (`Symbol.asyncDispose`). Calling `close()` after the stream has completed changes nothing. Right after a result, before the main loop starts another turn (a main-loop assistant, user or `stream_event` frame, or a background `task_notification`, which the CLI runs as its own turn) and with no background subagent still running, `close()`, `Symbol.asyncDispose` and an abort end the conversation OK instead, because nothing was cancelled. A `system/session_state_changed` frame, when the CLI sends one, decides it: `idle` after a result ends OK, `running` ends cancelled. One gap remains: an abort after the app sends a new streaming-input prompt but before the SDK yields any message for it is recorded as OK. Anything else is a bug here. ## Tests diff --git a/typescript/packages/traceai_claude_agent_sdk/src/__tests__/wrapper.test.ts b/typescript/packages/traceai_claude_agent_sdk/src/__tests__/wrapper.test.ts index c9dd630a7..1afd1b328 100644 --- a/typescript/packages/traceai_claude_agent_sdk/src/__tests__/wrapper.test.ts +++ b/typescript/packages/traceai_claude_agent_sdk/src/__tests__/wrapper.test.ts @@ -716,29 +716,80 @@ describe("wrapQuery", () => { }, ); - it("M1: close() after a background subagent finishes post-result (its messages arrive after the result) ends OK", async () => { - const { provider, exporter } = memoryProvider(); + /** Background subagent journey reordered: the main loop's result comes before the subagent's tail. */ + function resultBeforeBackgroundTail(): SDKMessage[] { const base = backgroundedSubagentJourney(); const result = base[base.length - 1]; const idx = base.findIndex((m) => m.type === "assistant" && m.message.id === "msg_13"); - // Main loop answers and yields its result; the background subagent's tool_result, - // final turn and task_notification arrive afterwards. - const journey = [...base.slice(0, idx + 1), result, ...base.slice(idx + 1, base.length - 1)]; - const fake = makeFakeQuery(journey); - const q = wrapQuery(fake.query, { tracerProvider: provider })({ prompt: PROMPT }); + return [...base.slice(0, idx + 1), result, ...base.slice(idx + 1, base.length - 1)]; + } + + async function readThrough(q: AsyncGenerator, stop: (m: SDKMessage) => boolean): Promise { for (;;) { const step = await q.next(); if (step.done) throw new Error("stream ended early"); - if (step.value.type === "system" && step.value.subtype === "task_notification") break; + if (stop(step.value)) return; } + } + + const isTaskNotification = (m: SDKMessage) => m.type === "system" && m.subtype === "task_notification"; + + it("R4-1: close() after a post-result task_notification stays cancelled (the notification starts a main-loop turn)", async () => { + const { provider, exporter } = memoryProvider(); + const fake = makeFakeQuery(resultBeforeBackgroundTail()); + const q = wrapQuery(fake.query, { tracerProvider: provider })({ prompt: PROMPT }); + await readThrough(q, isTaskNotification); q.close(); const spans = exporter.getFinishedSpans(); const conversation = one(spans, "claude_agent.conversation"); - expect(conversation.status.code).toBe(SpanStatusCode.OK); - expect(conversation.attributes["claude_agent.cancelled"]).toBeUndefined(); + expect(conversation.status.code).toBe(SpanStatusCode.ERROR); + expect(conversation.attributes["claude_agent.cancelled"]).toBe(true); + // The subagent itself finished at its notification. expect(one(spans, "claude_agent.subagent.code-reviewer").status.code).toBe(SpanStatusCode.OK); }); + it.each([ + ["idle", SpanStatusCode.OK], + ["running", SpanStatusCode.ERROR], + ] as const)( + "R4-1: session_state_changed %s is authoritative for close() after a result", + async (state, expected) => { + const { provider, exporter } = memoryProvider(); + const stateMessage = { + type: "system", + subtype: "session_state_changed", + state, + uuid: "00000000-0000-4000-8000-0000000000aa", + session_id: SESSION_ID, + } as unknown as SDKMessage; + const fake = makeFakeQuery([...resultBeforeBackgroundTail(), stateMessage]); + const q = wrapQuery(fake.query, { tracerProvider: provider })({ prompt: PROMPT }); + await readThrough(q, (m) => m.type === "system" && (m as { subtype?: string }).subtype === "session_state_changed"); + q.close(); + const conversation = one(exporter.getFinishedSpans(), "claude_agent.conversation"); + expect(conversation.status.code).toBe(expected); + }, + ); + + it("R4-2: a main-loop stream_event after a result starts a new turn, so close() stays cancelled", async () => { + const { provider, exporter } = memoryProvider(); + const partial = { + type: "stream_event", + event: { type: "message_start", message: { id: "msg_p2" } }, + parent_tool_use_id: null, + uuid: "00000000-0000-4000-8000-0000000000bb", + session_id: SESSION_ID, + } as unknown as SDKMessage; + const journey = [init(), assistant("msg_s1", [text("First answer.")]), resultSuccess("First answer.", { num_turns: 1 }), partial]; + const fake = makeFakeQuery(journey); + const q = wrapQuery(fake.query, { tracerProvider: provider })({ prompt: PROMPT }); + await readThrough(q, (m) => m.type === "stream_event"); + q.close(); + const conversation = one(exporter.getFinishedSpans(), "claude_agent.conversation"); + expect(conversation.status.code).toBe(SpanStatusCode.ERROR); + expect(conversation.attributes["claude_agent.cancelled"]).toBe(true); + }); + it("N1: close() after a result while a background subagent still runs stays cancelled", async () => { const { provider, exporter } = memoryProvider(); // The result arrives before the background subagent's task_notification. diff --git a/typescript/packages/traceai_claude_agent_sdk/src/spans.ts b/typescript/packages/traceai_claude_agent_sdk/src/spans.ts index ceece9ff2..5df5a5002 100644 --- a/typescript/packages/traceai_claude_agent_sdk/src/spans.ts +++ b/typescript/packages/traceai_claude_agent_sdk/src/spans.ts @@ -287,7 +287,7 @@ export class QueryTracer { private conversationContext?: Context; private conversationIsError = false; private resultSeen = false; - /** A result arrived and no main-loop assistant/user message has followed it yet. */ + /** A result arrived and the main loop has not started another turn since (see trackIdleAfterResult). */ private idleAfterResult = false; private sessionId?: string; private finished = false; @@ -380,15 +380,7 @@ export class QueryTracer { } try { const type = (message as { type?: unknown }).type; - if ( - (type === "assistant" || type === "user") && - ((message as { parent_tool_use_id?: unknown }).parent_tool_use_id ?? null) === null - ) { - // Main-loop activity after a result starts a new turn. Background - // subagent messages (parent_tool_use_id set) do not: nothingInFlight() - // already covers subagents that are still open. - this.idleAfterResult = false; - } + this.trackIdleAfterResult(type, message as { parent_tool_use_id?: unknown; subtype?: unknown; state?: unknown }); if (type === "assistant") { this.onAssistant(message as AssistantMessageLike); } else if (type === "user") { @@ -487,6 +479,30 @@ export class QueryTracer { } } + /** + * Keep idleAfterResult in step with the main loop. Main-loop assistant, user + * and stream_event frames after a result start a new turn, and so does a + * task_notification: the CLI queues it as a main-thread command that runs + * its own turn (sdk.d.ts:5440, 4603). Background subagent frames + * (parent_tool_use_id set) do not; nothingInFlight() covers subagents still + * open. system/session_state_changed is authoritative when the CLI sends it + * (sdk.d.ts:5858): 'idle' after a result means the turn is over. + */ + private trackIdleAfterResult( + type: unknown, + message: { parent_tool_use_id?: unknown; subtype?: unknown; state?: unknown }, + ): void { + if (type === "assistant" || type === "user" || type === "stream_event") { + if ((message.parent_tool_use_id ?? null) === null) this.idleAfterResult = false; + } else if (type === "system") { + if (message.subtype === "task_notification") { + this.idleAfterResult = false; + } else if (message.subtype === "session_state_changed") { + this.idleAfterResult = message.state === "idle" && this.resultSeen; + } + } + } + /** True when no turn, tool or subagent span of this query is still open. */ private nothingInFlight(): boolean { if (this.tools.size > 0) return false; From 53ba9fc4bf5e9ef999e819dc548aeb947b5e6eb1 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 11 Oct 2026 20:39:25 -0700 Subject: [PATCH 16/19] fix(claude-agent-sdk): record the requested model on a synthetic API-error turn On an API error the Claude Code CLI yields an assistant message it wrote itself, with message.model "" (seen live with 0.3.289 and an unknown model). That turn's LLM span carried gen_ai.request.model and claude_agent.model "", so Observe listed a model that does not exist. The turn now carries the requested model (options.model, else the init message's model) on both keys, and keeps the CLI's value on the new claude_agent.message.model. Real model replies are unchanged. The key lives in TraceAIAttributes, so the Python attribute parity list stays exact. Live: unknown model via CLIProxy -> turn model no-such-model-th8103, claude_agent.message.model "", status ERROR (was model ""). Co-Authored-By: Claude Opus 5.5 --- .../traceai_claude_agent_sdk/README.md | 2 +- .../src/__tests__/wrapper.test.ts | 36 +++++++++++++++++++ .../src/attributes.ts | 5 +++ .../traceai_claude_agent_sdk/src/spans.ts | 22 ++++++++++-- 4 files changed, 62 insertions(+), 3 deletions(-) diff --git a/typescript/packages/traceai_claude_agent_sdk/README.md b/typescript/packages/traceai_claude_agent_sdk/README.md index 8c90e66ca..4832db34d 100644 --- a/typescript/packages/traceai_claude_agent_sdk/README.md +++ b/typescript/packages/traceai_claude_agent_sdk/README.md @@ -73,7 +73,7 @@ A foreground subagent span ends with its `tool.Agent` / `tool.Task` result. A su | Field | Attributes | Source | |---|---|---| -| Model | `gen_ai.request.model`, `claude_agent.model` | `options.model`, the init message, each assistant message | +| Model | `gen_ai.request.model`, `claude_agent.model` | `options.model`, the init message, each assistant message. On an API error the CLI yields an assistant message it wrote itself, with model ``: that turn carries the requested model (`options.model`, else the init message's) and `claude_agent.message.model` = `` | | Provider | `gen_ai.provider.name` = `anthropic`, or `custom` when `ANTHROPIC_BASE_URL` (from `options.env`, else `process.env`) is not an `anthropic.com` host | | | Tokens | `gen_ai.usage.input_tokens`, `output_tokens`, `total_tokens`, `cache_read_tokens`, `cache_creation_tokens` | result `modelUsage` summed over every model, on the conversation span only. See "Usage and cost" | | Cost | `claude_agent.cost.total_usd` and `gen_ai.cost.total` | result `total_cost_usd`; never computed here. See "Usage and cost" | diff --git a/typescript/packages/traceai_claude_agent_sdk/src/__tests__/wrapper.test.ts b/typescript/packages/traceai_claude_agent_sdk/src/__tests__/wrapper.test.ts index 1afd1b328..10298ac6f 100644 --- a/typescript/packages/traceai_claude_agent_sdk/src/__tests__/wrapper.test.ts +++ b/typescript/packages/traceai_claude_agent_sdk/src/__tests__/wrapper.test.ts @@ -866,6 +866,42 @@ describe("wrapQuery", () => { }); }); + describe("synthetic API-error turn", () => { + // On an API error (e.g. an unknown model) the CLI yields an assistant message it wrote itself: + // message.model "", error "unknown", the API error as text (seen live with 0.3.289). + function syntheticErrorJourney(initModel: string): SDKMessage[] { + const turn = assistant("msg_synthetic", [text("API Error: 400 unknown provider for model x")], null, { + error: "unknown", + } as Partial>); + (turn.message as { model: string }).model = ""; + return [init({ model: initModel }), turn, resultSuccess("API Error: 400 unknown provider for model x", { is_error: true } as never)]; + } + + it("records the requested model on the turn and keeps in claude_agent.message.model", async () => { + const { spans } = await run(syntheticErrorJourney("no-such-model"), { model: "no-such-model" }); + const turn = one(spans, "claude_agent.assistant_turn"); + expect(turn.attributes["gen_ai.request.model"]).toBe("no-such-model"); + expect(turn.attributes["claude_agent.model"]).toBe("no-such-model"); + expect(turn.attributes["claude_agent.message.model"]).toBe(""); + expect(turn.status.code).toBe(SpanStatusCode.ERROR); + }); + + it("falls back to the init message's model when options.model is not set", async () => { + const { spans } = await run(syntheticErrorJourney("claude-from-init")); + const turn = one(spans, "claude_agent.assistant_turn"); + expect(turn.attributes["gen_ai.request.model"]).toBe("claude-from-init"); + expect(turn.attributes["claude_agent.message.model"]).toBe(""); + }); + + it("leaves a real model reply unchanged", async () => { + const { spans } = await run(simpleToolJourney(), { model: "requested-alias" }); + for (const turn of byName(spans, "claude_agent.assistant_turn")) { + expect(turn.attributes["gen_ai.request.model"]).toBe(MODEL); + expect(turn.attributes["claude_agent.message.model"]).toBeUndefined(); + } + }); + }); + it("never copies FI_API_KEY / FI_SECRET_KEY onto spans", async () => { process.env.FI_API_KEY = "fi-api-key-PLACEHOLDER-123"; process.env.FI_SECRET_KEY = "fi-secret-key-PLACEHOLDER-456"; diff --git a/typescript/packages/traceai_claude_agent_sdk/src/attributes.ts b/typescript/packages/traceai_claude_agent_sdk/src/attributes.ts index c77c154ee..bc01af55c 100644 --- a/typescript/packages/traceai_claude_agent_sdk/src/attributes.ts +++ b/typescript/packages/traceai_claude_agent_sdk/src/attributes.ts @@ -156,6 +156,11 @@ export const TraceAIAttributes = { * known in this process, so no promoted usage or cost key was written. */ USAGE_BASELINE_UNKNOWN: "claude_agent.usage.baseline_unknown", + /** + * The model the CLI put on an assistant message it synthesized itself ("", on an + * API error). The turn's model keys then carry the requested model instead. + */ + MESSAGE_MODEL: "claude_agent.message.model", } as const; /** diff --git a/typescript/packages/traceai_claude_agent_sdk/src/spans.ts b/typescript/packages/traceai_claude_agent_sdk/src/spans.ts index 5df5a5002..3dfd6a8a5 100644 --- a/typescript/packages/traceai_claude_agent_sdk/src/spans.ts +++ b/typescript/packages/traceai_claude_agent_sdk/src/spans.ts @@ -60,6 +60,8 @@ export type FinishOutcome = | { kind: "error"; error: unknown } | { kind: "aborted"; reason?: unknown }; +/** `message.model` on an assistant message the CLI synthesized itself (API errors). */ +const SYNTHETIC_MODEL = ""; const PROMPT_MAX = 2000; const SYSTEM_PROMPT_MAX = 1000; const CONTENT_MAX = 5000; @@ -290,6 +292,8 @@ export class QueryTracer { /** A result arrived and the main loop has not started another turn since (see trackIdleAfterResult). */ private idleAfterResult = false; private sessionId?: string; + /** The model the CLI's init message reports for the main loop. */ + private initModel?: string; private finished = false; private abortListener?: () => void; @@ -650,6 +654,7 @@ export class QueryTracer { this.setSessionId(message.session_id); } if (typeof message.model === "string" && message.model) { + this.initModel = message.model; conversation.setAttribute(A.AGENT_MODEL, message.model); conversation.setAttribute(T.GEN_AI_REQUEST_MODEL, message.model); } @@ -754,8 +759,15 @@ export class QueryTracer { span.setAttribute(T.GEN_AI_PROVIDER_NAME, this.providerName); const model = message.message?.model; if (typeof model === "string" && model) { - span.setAttribute(A.AGENT_MODEL, model); - span.setAttribute(T.GEN_AI_REQUEST_MODEL, model); + // The CLI writes its own assistant message for an API error, with model "". + // Record the model that was asked for instead; keep the CLI's value on its own key. + const synthetic = model === SYNTHETIC_MODEL; + if (synthetic) span.setAttribute(T.MESSAGE_MODEL, model); + const recorded = synthetic ? this.requestedModel() : model; + if (recorded) { + span.setAttribute(A.AGENT_MODEL, recorded); + span.setAttribute(T.GEN_AI_REQUEST_MODEL, recorded); + } } if (scope !== null) { span.setAttribute(A.PARENT_TOOL_USE_ID, scope); @@ -1111,6 +1123,12 @@ export class QueryTracer { } } + /** The model this query asked for: options.model, else the init message's model. */ + private requestedModel(): string | undefined { + const model = this.options?.model; + return typeof model === "string" && model ? model : this.initModel; + } + private conversationStartAttributes(): Attributes { const options = this.options ?? {}; const attributes: Record = { From 3588c8d23d69696e86a3ca62c807111e1385a8eb Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 11 Oct 2026 21:21:12 -0700 Subject: [PATCH 17/19] docs: move the README catalog row to the TH-8103 catalog PR The catalog rows for all TH-8103 integrations now live in one PR (docs/th-8103-integration-catalog), so this PR no longer conflicts with its siblings in README.md and can merge in any order. Co-Authored-By: Claude Opus 5.5 --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 10add6fa5..214680868 100644 --- a/README.md +++ b/README.md @@ -331,7 +331,6 @@ var tracer = TraceAI.Register(opts => | [`@traceai/mastra`](https://www.npmjs.com/package/@traceai/mastra) | Mastra | [![npm](https://img.shields.io/npm/v/@traceai/mastra)](https://www.npmjs.com/package/@traceai/mastra) | | [`@traceai/beeai`](https://www.npmjs.com/package/@traceai/beeai) | IBM BeeAI | [![npm](https://img.shields.io/npm/v/@traceai/beeai)](https://www.npmjs.com/package/@traceai/beeai) | | [`@traceai/strands`](https://www.npmjs.com/package/@traceai/strands) | AWS Strands Agents | [![npm](https://img.shields.io/npm/v/@traceai/strands)](https://www.npmjs.com/package/@traceai/strands) | -| [`@traceai/claude-agent-sdk`](https://www.npmjs.com/package/@traceai/claude-agent-sdk) | Claude Agent SDK (`query()`) | [![npm](https://img.shields.io/npm/v/@traceai/claude-agent-sdk)](https://www.npmjs.com/package/@traceai/claude-agent-sdk) | #### Tools and Libraries From 7c11a1d1196e8e636d1b89387353c29928f6b3e9 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Fri, 16 Oct 2026 19:59:42 -0700 Subject: [PATCH 18/19] fix(claude-agent-sdk): TraceConfig privacy default, no local paths or stack traces, peer >=0.3.277 (TH-8235) Re-verification 2026-10-16 against local Future AGI with the real SDK (0.3.302 latest, 0.3.289 pin) and a real model through an Anthropic-compatible gateway. Privacy default (TH-8103 coordinator decision 2026-10-16): follow the traceAI TraceConfig convention instead of a package-specific switch. Content is captured by default (fi-core DEFAULT_HIDE_* = false, same as the Python package); FI_HIDE_INPUTS / FI_HIDE_OUTPUTS (and FI_HIDE_INPUT_MESSAGES / _INPUT_TEXT / _OUTPUT_MESSAGES / _OUTPUT_TEXT) hide it, with fi-core's parsing (only "true" hides) and precedence (traceConfig > env > default). The env vars now apply even when no traceConfig is passed (fi-core only reads them when one is), and the full resolved config reaches FITracer so FI_PII_REDACTION also works. Never exported, in either mode: - stack traces: the conversation span no longer calls recordException(); it adds an exception event with exception.type (plus the scrubbed message when outputs are visible). With hideOutputs the error text and the result's error strings are dropped and the status carries the type. - local paths: new scrub.ts rewrites the cwd to [cwd], the home dir to ~, other absolute local paths / file URLs / Windows paths to [path], and cuts Python tracebacks and V8 frames in every content attribute; error text also loses emails and this machine's user and host names. The real run had leaked /Users//... through tool.Read input and a /private/tmp/... output_file through the Agent tool result. Peer range ^0.3.142 -> ^0.3.277: before 0.3.277 the SDK restarted a resumed or forked session's total_cost_usd / modelUsage at zero (SDK CHANGELOG 0.3.277), so the per-query usage delta undercounts. The real-SDK resume, continue/fork and restart contract tests fail on 0.3.142, 0.3.215, 0.3.250, 0.3.272, 0.3.276 and the whole suite passes on 0.3.277, 0.3.289 and 0.3.302. Tests: jest 117 passed (6 suites, new scrub.test.ts and privacy cases); contract 13 passed on 0.3.277, 0.3.289, 0.3.302, 14 on the Node 22 + 26 import matrix. Co-Authored-By: Claude Opus 5.5 --- .../traceai_claude_agent_sdk/README.md | 31 +++- .../contract/test_harness_contract.py | 27 ++- .../traceai_claude_agent_sdk/package.json | 2 +- .../src/__tests__/scrub.test.ts | 92 ++++++++++ .../src/__tests__/wrapper.test.ts | 171 ++++++++++++++---- .../src/instrumentation.ts | 83 ++++++--- .../traceai_claude_agent_sdk/src/scrub.ts | 130 +++++++++++++ .../traceai_claude_agent_sdk/src/spans.ts | 96 +++++++--- .../traceai_claude_agent_sdk/src/types.ts | 4 +- 9 files changed, 526 insertions(+), 110 deletions(-) create mode 100644 typescript/packages/traceai_claude_agent_sdk/src/__tests__/scrub.test.ts create mode 100644 typescript/packages/traceai_claude_agent_sdk/src/scrub.ts diff --git a/typescript/packages/traceai_claude_agent_sdk/README.md b/typescript/packages/traceai_claude_agent_sdk/README.md index 4832db34d..05626f9ff 100644 --- a/typescript/packages/traceai_claude_agent_sdk/README.md +++ b/typescript/packages/traceai_claude_agent_sdk/README.md @@ -2,7 +2,7 @@ OpenTelemetry tracing for the TypeScript [Claude Agent SDK](https://www.npmjs.com/package/@anthropic-ai/claude-agent-sdk) `query()`, exported to Future AGI through `@traceai/fi-core`. -Alpha. Tested against `@anthropic-ai/claude-agent-sdk` 0.3.289 with recorded message fixtures and with the real SDK and bundled CLI pointed at a loopback mock of the Messages API. No live Anthropic call was made. +Alpha. Tested against `@anthropic-ai/claude-agent-sdk` 0.3.289 (the devDependency pin) and 0.3.302 (latest on 2026-10-16) with recorded message fixtures, with the real SDK and bundled CLI pointed at a loopback mock of the Messages API, and end to end against a local Future AGI with the real CLI calling a real model through an Anthropic-compatible gateway (`ANTHROPIC_BASE_URL`). No direct `api.anthropic.com` call was made. This is not the Python package (`traceAI-claude-agent-sdk`), not Claude Code CLI session tracing, and not the Anthropic Messages API instrumentation (`@traceai/anthropic`). @@ -12,8 +12,8 @@ This is not the Python package (`traceAI-claude-agent-sdk`), not Claude Code CLI | Item | Value | |---|---| -| Tested pin | `@anthropic-ai/claude-agent-sdk` 0.3.289 (npm shasum `36c710c40a2d9761d6c12c42b469d002e64e9be8`) | -| Peer range | `^0.3.142` (0.3.142 is the oldest 0.3.x on npm; its `query()` and the message fields read here match 0.3.289, and the real-SDK contract test passed against it) | +| Tested versions | `@anthropic-ai/claude-agent-sdk` 0.3.289 (devDependency pin, npm shasum `36c710c40a2d9761d6c12c42b469d002e64e9be8`) and 0.3.302 (latest, 2026-10-16: contract suite + end-to-end run) | +| Peer range | `^0.3.277`. Before 0.3.277 a resumed or forked session's `total_cost_usd` / `modelUsage` restarted at zero (SDK CHANGELOG 0.3.277), so the per-query usage below would be wrong; the real-SDK resume, continue/fork and restart contract tests fail on 0.3.142, 0.3.215, 0.3.250, 0.3.272 and 0.3.276 and the whole contract suite passes on 0.3.277, 0.3.289 and 0.3.302 | | Node | 18, 20, 22 (CJS and ESM import checked); the SDK itself declares `node >=18` | ## Install @@ -79,7 +79,7 @@ A foreground subagent span ends with its `tool.Agent` / `tool.Task` result. A su | Cost | `claude_agent.cost.total_usd` and `gen_ai.cost.total` | result `total_cost_usd`; never computed here. See "Usage and cost" | | Session | `session.id`, `claude_agent.session.id`, `claude_agent.session_id`; `claude_agent.session.is_new` / `is_resumed` (`resume` or `continue`); `claude_agent.session.fork_from` on fork | init / result `session_id`, `options.sessionId`, `options.resume`, `options.continue` | | Tools | `claude_agent.tool.name`, `gen_ai.tool.name`, `use_id`, `source` (`builtin` / `mcp` / `custom`), `is_error`, `duration_ms` | `tool_use` / `tool_result` blocks | -| Errors | span status ERROR; `claude_agent.error.type` / `error.message`; `claude_agent.cancelled=true` on abort | error result, thrown error, `is_error` tool results, `AbortController` | +| Errors | span status ERROR; `claude_agent.error.type` / `error.message`; an `exception` event with `exception.type` (and `exception.message` when outputs are visible); `claude_agent.cancelled=true` on abort. No stack trace is ever recorded, and error text has local paths, emails and this machine's user and host names replaced | error result, thrown error, `is_error` tool results, `AbortController` | The `claude_agent.*` names are the Python package's names (`_attributes.py`); a test fails if one is missing. @@ -98,18 +98,29 @@ The Future AGI collector promotes `gen_ai.usage.*` and `gen_ai.cost.total` on an - Every conversation span with a result also carries the running totals on unpromoted keys: `claude_agent.cumulative.cost_usd`, `input_tokens`, `output_tokens`, `cache_read_tokens`, `cache_creation_tokens`. - A result with no cost and no `modelUsage`, or a zeroed crash result, writes none of these keys and leaves the session's saved totals unchanged. -## Privacy: content is off by default +## Privacy: standard traceAI `TraceConfig` -Prompts, system prompts, tool inputs, tool outputs, assistant text and the final result can contain source code, file contents or secrets. They are **not** recorded unless you opt in: +Like every traceAI instrumentor (and the Python package), content is **captured by default**: the prompt, system prompt, tool inputs, tool outputs, assistant text, the final result and error text. Prompts and tool I/O can contain source code, file contents or secrets, so hide them in production when you do not want that text in Future AGI: ```typescript -wrapQuery(query, { tracerProvider, traceConfig: { hideInputs: false, hideOutputs: false } }); -// or: FI_HIDE_INPUTS=false FI_HIDE_OUTPUTS=false +wrapQuery(query, { tracerProvider, traceConfig: { hideInputs: true, hideOutputs: true } }); +// or: FI_HIDE_INPUTS=true FI_HIDE_OUTPUTS=true ``` -Precedence: `traceConfig` option, then the env var, then hidden. The env vars fail closed: only the value `false` (trimmed, any case) turns capture on. Any other value, including `1`, `yes`, `true`, ` true` or an empty string, keeps content hidden. (fi-core itself treats every value other than `true` as "show", so `FI_HIDE_INPUTS=1` would capture there; this package does not follow that rule.) +| fi-core flag (env var) | Hides | +|---|---| +| `hideInputs` (`FI_HIDE_INPUTS`) | prompt, system prompt, tool input (`input.value`, `claude_agent.tool.input`, `file_path`, `command`, `pattern`, `url`, `search_query`), subagent prompt and description | +| `hideOutputs` (`FI_HIDE_OUTPUTS`) | assistant text, tool output, tool error text, final result, error text (only the error type is kept) | +| `hideInputMessages` / `hideInputText` (`FI_HIDE_INPUT_MESSAGES` / `FI_HIDE_INPUT_TEXT`) | prompt, system prompt and subagent prompt text only | +| `hideOutputMessages` / `hideOutputText` (`FI_HIDE_OUTPUT_MESSAGES` / `FI_HIDE_OUTPUT_TEXT`) | assistant text and final result only | + +Precedence and parsing are fi-core's: the `traceConfig` option, then the env var (hidden only when the value is `true`, any case), then the fi-core default (`false`). The env vars apply even when you pass no `traceConfig`. `FI_PII_REDACTION=true` applies fi-core's regex redaction on top. + +Structure is always kept: span names and kinds, model, provider, tokens, cost, session, tool names and ids, durations, status. + +Never exported, in either mode: local file paths, stack traces, and (in error text) emails and this machine's user and host names. In captured content the query's working directory (`options.cwd`, else `process.cwd()`) is written as `[cwd]`, the home directory as `~`, and any other absolute local path (`/Users/...`, `/home/...`, `/private/...`, `/tmp/...`, `C:\...`, `file://...`) as `[path]`; Python tracebacks and V8 `at ...` frames become `[stack trace removed]`. So a Read of `/src/a.ts` shows `claude_agent.tool.file_path = [cwd]/src/a.ts`. A thrown error is recorded as an `exception` event with `exception.type` (plus the scrubbed `exception.message` when outputs are visible), never with `recordException()`'s stack trace. -This is stricter than fi-core's own default and than the Python package, which records the prompt and tool input/output unconditionally. +A streaming-input prompt (an async iterable) is not recorded: the wrapper does not read or copy the app's input stream. ## Flushing and batching diff --git a/typescript/packages/traceai_claude_agent_sdk/contract/test_harness_contract.py b/typescript/packages/traceai_claude_agent_sdk/contract/test_harness_contract.py index 6f8ae4f80..555da8f81 100644 --- a/typescript/packages/traceai_claude_agent_sdk/contract/test_harness_contract.py +++ b/typescript/packages/traceai_claude_agent_sdk/contract/test_harness_contract.py @@ -52,6 +52,7 @@ "SECRET_ASSISTANT_TEXT_MARKER", "SECRET_SUBAGENT_PROMPT_MARKER", ) +HIDE_CONTENT = {"FI_HIDE_INPUTS": "true", "FI_HIDE_OUTPUTS": "true"} PLACEHOLDER_API_KEY = "contract-api-key-PLACEHOLDER" PLACEHOLDER_SECRET_KEY = "contract-secret-key-PLACEHOLDER" @@ -143,9 +144,9 @@ def _one(spans: List[Dict[str, Any]], name: str) -> Dict[str, Any]: return matches[0] -def test_conversation_turn_tool_reach_collector_path_with_content_off(built_package: Path, fixtures_dir: Path) -> None: +def test_conversation_turn_tool_reach_collector_path_with_content_hidden(built_package: Path, fixtures_dir: Path) -> None: with Receiver() as receiver: - output = _run_journey(receiver, fixtures_dir, "simple") + output = _run_journey(receiver, fixtures_dir, "simple", HIDE_CONTENT) spans = receiver.spans() requests = receiver.requests() @@ -199,7 +200,7 @@ def test_conversation_turn_tool_reach_collector_path_with_content_off(built_pack assert _attrs(span)["session.id"] == SESSION_ID assert _status_code(span) == 1 - # Content capture is off by default. + # FI_HIDE_INPUTS=true / FI_HIDE_OUTPUTS=true: no content anywhere in the export. blob = json.dumps(spans) for marker in CONTENT_MARKERS: assert marker not in blob @@ -236,9 +237,10 @@ def test_subagent_and_mcp_spans_reach_the_receiver(built_package: Path, fixtures assert conversation["claude_agent.error.type"] == "error_max_turns" -def test_content_is_exported_only_after_opt_in(built_package: Path, fixtures_dir: Path) -> None: +def test_content_is_exported_by_default(built_package: Path, fixtures_dir: Path) -> None: + """traceAI TraceConfig convention: content on unless FI_HIDE_* / traceConfig hides it.""" with Receiver() as receiver: - _run_journey(receiver, fixtures_dir, "simple", {"FI_HIDE_INPUTS": "false", "FI_HIDE_OUTPUTS": "false"}) + _run_journey(receiver, fixtures_dir, "simple") spans = receiver.spans() conversation = _attrs(_one(spans, "claude_agent.conversation")) tool = _attrs(_one(spans, "tool.Read")) @@ -506,7 +508,7 @@ def test_real_sdk_query_through_anthropic_base_url_mock(built_package: Path) -> Mechanism from the ADR: Options.env.ANTHROPIC_BASE_URL. No Anthropic call: placeholder key, dead-port HTTPS_PROXY, temp HOME. Spans go through fi-core to the shared Receiver. """ - output, spans, exported = _run_real_sdk("tool", "th8235-real-sdk-contract") + output, spans, exported = _run_real_sdk("tool", "th8235-real-sdk-contract", HIDE_CONTENT) # The CLI called the mock (host root + /v1/messages), streaming, with the placeholder key. mock_requests = output["requests"] @@ -552,6 +554,17 @@ def test_real_sdk_query_through_anthropic_base_url_mock(built_package: Path) -> _assert_exported_with_both_keys(exported, "th8235-real-sdk-contract") +@requires_real_sdk +def test_real_sdk_captures_content_by_default(built_package: Path) -> None: + """Same real-SDK run with no FI_HIDE_* set: prompt, tool output and assistant text are exported.""" + _, spans, _ = _run_real_sdk("tool", "th8235-real-sdk-contract-content") + conversation = _attrs(_one(spans, "claude_agent.conversation")) + tool = _attrs(_one(spans, "tool.Read")) + assert "SECRET_PROMPT_MARKER" in conversation["input.value"] + assert "SECRET_TOOL_OUTPUT_MARKER" in tool["claude_agent.tool.output"] + assert "SECRET_ASSISTANT_TEXT_MARKER" in json.dumps(spans) + + def _read_member(tar: tarfile.TarFile, member: Any) -> bytes: handle = tar.extractfile(member) assert handle is not None, member @@ -616,7 +629,7 @@ def test_packed_tarball_has_no_sdk_sources_or_native_binary(built_package: Path, assert _sha256(data) not in sdk_hashes, f"{relative} is a copy of {sdk_hashes[_sha256(data)]}" assert b"Anthropic PBC. All rights reserved" not in data, relative - assert manifest["peerDependencies"] == {"@anthropic-ai/claude-agent-sdk": "^0.3.142"} + assert manifest["peerDependencies"] == {"@anthropic-ai/claude-agent-sdk": "^0.3.277"} assert "@anthropic-ai/claude-agent-sdk" not in manifest.get("dependencies", {}) assert not manifest.get("bundledDependencies") and not manifest.get("bundleDependencies") assert manifest["license"] == "Apache-2.0" diff --git a/typescript/packages/traceai_claude_agent_sdk/package.json b/typescript/packages/traceai_claude_agent_sdk/package.json index a09f26b22..3f924e0bb 100644 --- a/typescript/packages/traceai_claude_agent_sdk/package.json +++ b/typescript/packages/traceai_claude_agent_sdk/package.json @@ -62,7 +62,7 @@ "typescript": "^5.7.3" }, "peerDependencies": { - "@anthropic-ai/claude-agent-sdk": "^0.3.142" + "@anthropic-ai/claude-agent-sdk": "^0.3.277" }, "files": [ "dist", diff --git a/typescript/packages/traceai_claude_agent_sdk/src/__tests__/scrub.test.ts b/typescript/packages/traceai_claude_agent_sdk/src/__tests__/scrub.test.ts new file mode 100644 index 000000000..a755aa336 --- /dev/null +++ b/typescript/packages/traceai_claude_agent_sdk/src/__tests__/scrub.test.ts @@ -0,0 +1,92 @@ +import * as os from "os"; +import { SpanStatusCode } from "@opentelemetry/api"; +import { wrapQuery } from "../index"; +import { CWD_TOKEN, PATH_REMOVED, STACK_TRACE_REMOVED, scrubContent, scrubErrorText } from "../scrub"; +import { makeFakeQuery } from "./fixtures/fakeQuery"; +import { MODEL, SESSION_ID, assistant, init, resultSuccess, text, toolResult, toolUse } from "./fixtures/messages"; +import { allAttributeText, drain, memoryProvider, one } from "./helpers"; + +const ctx = { cwd: "/Users/alice/work/proj", home: "/Users/alice" }; + +describe("scrubContent", () => { + it("rewrites the cwd to [cwd] and the home directory to ~", () => { + expect(scrubContent('{"file_path":"/Users/alice/work/proj/src/a.ts"}', ctx)).toBe( + `{"file_path":"${CWD_TOKEN}/src/a.ts"}`, + ); + expect(scrubContent("see /Users/alice/.config/x.json", ctx)).toBe("see ~/.config/x.json"); + expect(scrubContent("cwd is /Users/alice/work/proj", ctx)).toBe(`cwd is ${CWD_TOKEN}`); + }); + + it("does not treat a sibling directory as the cwd", () => { + expect(scrubContent("/Users/alice/work/proj2/a.ts", ctx)).toBe("~/work/proj2/a.ts"); + }); + + it("replaces other absolute local paths, file URLs and Windows paths", () => { + expect(scrubContent("output_file: /private/tmp/claude-502/-Users-bob-x/tasks/a.output", ctx)).toBe( + `output_file: ${PATH_REMOVED}`, + ); + expect(scrubContent("open /home/bob/notes.txt now", ctx)).toBe(`open ${PATH_REMOVED} now`); + expect(scrubContent("at file:///Users/bob/app/index.mjs:3:9", ctx)).toBe(`at ${PATH_REMOVED}`); + expect(scrubContent('{"p":"C:\\\\Users\\\\bob\\\\a.txt"}', {})).toBe(`{"p":"${PATH_REMOVED}"}`); + }); + + it("leaves URLs, API routes and relative paths alone", () => { + for (const value of [ + "POST /v1/messages?beta=true", + "https://example.com/Users/docs", + "src/index.ts and ./README.md", + "http://127.0.0.1:8317/v1/messages", + ]) { + expect(scrubContent(value, ctx)).toBe(value); + } + }); + + it("cuts Python tracebacks and V8 stack frames", () => { + const py = 'failed\nTraceback (most recent call last):\n File "/home/bob/x.py", line 1, in \nValueError: x'; + expect(scrubContent(py, ctx)).toBe(`failed\n${STACK_TRACE_REMOVED}`); + const js = "Error: boom\n at run (/Users/bob/app/x.js:1:2)\n at main (node:internal/x:3:4)"; + expect(scrubContent(js, ctx)).toBe(`Error: boom\n${STACK_TRACE_REMOVED}`); + }); +}); + +describe("scrubErrorText", () => { + it("also removes emails and this machine's user and host names", () => { + const user = os.userInfo().username; + const host = os.hostname(); + const out = scrubErrorText(`denied for ${user} on ${host} (mail ops@example.com)`, ctx); + expect(out).not.toContain(user); + expect(out).not.toContain(host); + expect(out).not.toContain("ops@example.com"); + expect(out).toContain(""); + }); +}); + +describe("exported content never carries local paths or stack traces", () => { + it("rewrites paths in prompt, tool input, tool output and assistant text", async () => { + const home = os.homedir(); + const cwd = `${home}/th8235-scrub-proj`; + const toolId = "toolu_scrub_1"; + const messages = [ + init(), + assistant("msg_s1", [text(`Reading ${cwd}/notes.txt`), toolUse(toolId, "Read", { file_path: `${cwd}/notes.txt` })]), + toolResult(toolId, `contents of ${home}/secret/notes.txt\nError: x\n at f (${home}/app/x.js:1:2)`), + assistant("msg_s2", [text("Done.")]), + resultSuccess("Done."), + ]; + const { provider, exporter } = memoryProvider(); + const traced = wrapQuery(makeFakeQuery(messages as never).query, { tracerProvider: provider }); + await drain(traced({ prompt: `Summarize ${cwd}/notes.txt`, options: { cwd } })); + const spans = exporter.getFinishedSpans(); + const blob = allAttributeText(spans); + expect(blob).not.toContain(home); + expect(blob).not.toContain(os.userInfo().username); + expect(blob).not.toContain(" at "); + const tool = one(spans, "tool.Read"); + expect(tool.attributes["claude_agent.tool.file_path"]).toBe(`${CWD_TOKEN}/notes.txt`); + expect(tool.attributes["claude_agent.tool.output"]).toBe(`contents of ~/secret/notes.txt\nError: x\n${STACK_TRACE_REMOVED}`); + expect(one(spans, "claude_agent.conversation").attributes["input.value"]).toBe(`Summarize ${CWD_TOKEN}/notes.txt`); + expect(one(spans, "claude_agent.conversation").attributes["session.id"]).toBe(SESSION_ID); + expect(one(spans, "claude_agent.conversation").attributes["gen_ai.request.model"]).toBe(MODEL); + for (const span of spans) expect(span.status.code).toBe(SpanStatusCode.OK); + }); +}); diff --git a/typescript/packages/traceai_claude_agent_sdk/src/__tests__/wrapper.test.ts b/typescript/packages/traceai_claude_agent_sdk/src/__tests__/wrapper.test.ts index 10298ac6f..42d75d8a0 100644 --- a/typescript/packages/traceai_claude_agent_sdk/src/__tests__/wrapper.test.ts +++ b/typescript/packages/traceai_claude_agent_sdk/src/__tests__/wrapper.test.ts @@ -41,7 +41,15 @@ import { } from "./fixtures/messages"; import { StackContextManager, allAttributeText, byName, drain, memoryProvider, one, parentId } from "./helpers"; -const CONTENT_ENV = ["FI_HIDE_INPUTS", "FI_HIDE_OUTPUTS"] as const; +const CONTENT_ENV = [ + "FI_HIDE_INPUTS", + "FI_HIDE_OUTPUTS", + "FI_HIDE_INPUT_MESSAGES", + "FI_HIDE_OUTPUT_MESSAGES", + "FI_HIDE_INPUT_TEXT", + "FI_HIDE_OUTPUT_TEXT", +] as const; +const HIDE_ALL = { hideInputs: true, hideOutputs: true }; describe("wrapQuery", () => { let saved: Record; @@ -64,7 +72,7 @@ describe("wrapQuery", () => { async function run( messages: ReturnType, options: Record = {}, - traceConfig?: { hideInputs?: boolean; hideOutputs?: boolean }, + traceConfig?: Record, ) { const { provider, exporter } = memoryProvider(); const fake = makeFakeQuery(messages); @@ -147,8 +155,8 @@ describe("wrapQuery", () => { } }); - it("AC-07: keeps prompt, tool input/output and assistant text off spans by default", async () => { - const { spans } = await run(simpleToolJourney(), { systemPrompt: "SECRET_SYSTEM_PROMPT_MARKER" }); + it("AC-07: keeps prompt, tool input/output and assistant text off spans with hideInputs/hideOutputs", async () => { + const { spans } = await run(simpleToolJourney(), { systemPrompt: "SECRET_SYSTEM_PROMPT_MARKER" }, HIDE_ALL); const text = allAttributeText(spans); for (const marker of [ "SECRET_PROMPT_MARKER", @@ -175,12 +183,8 @@ describe("wrapQuery", () => { } }); - it("writes content only when hideInputs/hideOutputs are false", async () => { - const { spans } = await run( - simpleToolJourney(), - { systemPrompt: "be brief" }, - { hideInputs: false, hideOutputs: false }, - ); + it("captures content by default (fi-core DEFAULT_HIDE_* = false, same as the Python package)", async () => { + const { spans } = await run(simpleToolJourney(), { systemPrompt: "be brief" }); const conversation = one(spans, "claude_agent.conversation"); const tool = one(spans, "tool.Read"); expect(conversation.attributes["input.value"]).toBe(PROMPT); @@ -206,38 +210,75 @@ describe("wrapQuery", () => { expect(text).not.toContain("SECRET_PROMPT_MARKER"); expect(text).not.toContain(TOOL_INPUT_MARKER); expect(text).toContain(TOOL_OUTPUT_MARKER); + expect(text).toContain(ASSISTANT_TEXT_MARKER); }); - it("honours FI_HIDE_INPUTS=false / FI_HIDE_OUTPUTS=false as an opt-in", async () => { - process.env.FI_HIDE_INPUTS = "false"; - process.env.FI_HIDE_OUTPUTS = "false"; - const { spans } = await run(simpleToolJourney()); + it.each(["true", "TRUE", "True"])( + "hides every content value for FI_HIDE_INPUTS=FI_HIDE_OUTPUTS=%j with no traceConfig", + async (value) => { + process.env.FI_HIDE_INPUTS = value; + process.env.FI_HIDE_OUTPUTS = value; + const { spans } = await run(simpleToolJourney(), { systemPrompt: "SECRET_SYSTEM_PROMPT_MARKER" }); + const text = allAttributeText(spans); + for (const marker of [ + "SECRET_PROMPT_MARKER", + "SECRET_SYSTEM_PROMPT_MARKER", + TOOL_INPUT_MARKER, + TOOL_OUTPUT_MARKER, + ASSISTANT_TEXT_MARKER, + ]) { + expect(text).not.toContain(marker); + } + }, + ); + + it.each(["false", "1", "yes", " true", "", "0"])( + "follows fi-core's env rule: FI_HIDE_*=%j is not 'true', so content stays captured", + async (value) => { + process.env.FI_HIDE_INPUTS = value; + process.env.FI_HIDE_OUTPUTS = value; + const { spans } = await run(simpleToolJourney()); + const text = allAttributeText(spans); + expect(text).toContain("SECRET_PROMPT_MARKER"); + expect(text).toContain(TOOL_OUTPUT_MARKER); + }, + ); + + it("lets an explicit traceConfig override the env vars", async () => { + process.env.FI_HIDE_INPUTS = "true"; + process.env.FI_HIDE_OUTPUTS = "true"; + const { spans } = await run(simpleToolJourney(), {}, { hideInputs: false, hideOutputs: false }); const text = allAttributeText(spans); expect(text).toContain("SECRET_PROMPT_MARKER"); expect(text).toContain(TOOL_OUTPUT_MARKER); }); - it.each([" False ", "FALSE"])("treats FI_HIDE_*=%j (trimmed, any case) as the opt-in", async (value) => { - process.env.FI_HIDE_INPUTS = value; - process.env.FI_HIDE_OUTPUTS = value; - const { spans } = await run(simpleToolJourney()); + it.each([ + ["FI_HIDE_INPUT_MESSAGES"], + ["FI_HIDE_INPUT_TEXT"], + ])("%s=true hides the prompt and system prompt but keeps tool input and outputs", async (key) => { + process.env[key] = "true"; + const { spans } = await run(simpleToolJourney(), { systemPrompt: "SECRET_SYSTEM_PROMPT_MARKER" }); const text = allAttributeText(spans); - expect(text).toContain("SECRET_PROMPT_MARKER"); + expect(text).not.toContain("SECRET_PROMPT_MARKER"); + expect(text).not.toContain("SECRET_SYSTEM_PROMPT_MARKER"); + expect(text).toContain(TOOL_INPUT_MARKER); expect(text).toContain(TOOL_OUTPUT_MARKER); + expect(text).toContain(ASSISTANT_TEXT_MARKER); }); - it.each(["1", "yes", " true", "TRUE", "", "0", "no", "off"])( - "keeps content hidden for FI_HIDE_*=%j: only an explicit 'false' opts in", - async (value) => { - process.env.FI_HIDE_INPUTS = value; - process.env.FI_HIDE_OUTPUTS = value; - const { spans } = await run(simpleToolJourney()); - const text = allAttributeText(spans); - for (const marker of ["SECRET_PROMPT_MARKER", TOOL_INPUT_MARKER, TOOL_OUTPUT_MARKER, ASSISTANT_TEXT_MARKER]) { - expect(text).not.toContain(marker); - } - }, - ); + it.each([ + ["FI_HIDE_OUTPUT_MESSAGES"], + ["FI_HIDE_OUTPUT_TEXT"], + ])("%s=true hides assistant text and the final result but keeps tool output and inputs", async (key) => { + process.env[key] = "true"; + const { spans } = await run(simpleToolJourney()); + const text = allAttributeText(spans); + expect(text).not.toContain(ASSISTANT_TEXT_MARKER); + expect(one(spans, "claude_agent.conversation").attributes["output.value"]).toBeUndefined(); + expect(text).toContain(TOOL_OUTPUT_MARKER); + expect(text).toContain("SECRET_PROMPT_MARKER"); + }); it("AC-08: yields the same message objects, in order, and passes params through unchanged", async () => { const messages = simpleToolJourney(); @@ -281,7 +322,7 @@ describe("wrapQuery", () => { describe("J2: subagent (AC-01, AC-04)", () => { it("nests the subagent under the Agent tool span and the subagent's turns and tools under the subagent", async () => { - const { spans } = await run(subagentJourney()); + const { spans } = await run(subagentJourney(), {}, HIDE_ALL); const conversation = one(spans, "claude_agent.conversation"); const agentTool = one(spans, "tool.Agent"); const subagent = one(spans, "claude_agent.subagent.code-reviewer"); @@ -315,7 +356,7 @@ describe("wrapQuery", () => { expect(subagent.attributes["claude_agent.subagent.status"]).toBe("completed"); expect(grep.attributes["claude_agent.parent_tool_use_id"]).toBe(AGENT_TOOL_ID); expect(grep.attributes["claude_agent.tool.use_id"]).toBe(SUBAGENT_GREP_ID); - // Content off by default: the subagent prompt and description are inputs. + // hideInputs: the subagent prompt and description are inputs. expect(subagent.attributes["claude_agent.subagent.prompt"]).toBeUndefined(); expect(subagent.attributes["claude_agent.subagent.description"]).toBeUndefined(); expect(allAttributeText(spans)).not.toContain("SECRET_SUBAGENT_PROMPT_MARKER"); @@ -448,7 +489,7 @@ describe("wrapQuery", () => { describe("J3: MCP tool error and error result", () => { it("marks the MCP tool span mcp_tool/TOOL with ERROR and the conversation ERROR", async () => { - const { spans } = await run(mcpErrorJourney()); + const { spans } = await run(mcpErrorJourney(), {}, HIDE_ALL); const tool = one(spans, "tool.mcp__docs__search"); const conversation = one(spans, "claude_agent.conversation"); @@ -459,19 +500,27 @@ describe("wrapQuery", () => { expect(tool.attributes["claude_agent.mcp.tool_name"]).toBe("search"); expect(tool.attributes["claude_agent.tool.is_error"]).toBe(true); expect(tool.status.code).toBe(SpanStatusCode.ERROR); - // Error text is tool output: hidden by default. + // Error text is tool output: hidden with hideOutputs. expect(tool.attributes["claude_agent.tool.error_message"]).toBeUndefined(); expect(tool.status.message).toBe("tool error"); expect(conversation.status.code).toBe(SpanStatusCode.ERROR); expect(conversation.attributes["claude_agent.is_error"]).toBe(true); expect(conversation.attributes["claude_agent.error.type"]).toBe("error_max_turns"); - expect(conversation.attributes["claude_agent.error.message"]).toBe("Reached maximum number of turns (3)"); + // The result's error strings are raw error text: hidden with hideOutputs. + expect(conversation.attributes["claude_agent.error.message"]).toBeUndefined(); + expect(conversation.status.message).toBe("error_max_turns"); expect(conversation.attributes["gen_ai.cost.total"]).toBe(0.002); }); - it("records the tool error text when outputs are visible", async () => { - const { spans } = await run(mcpErrorJourney(), {}, { hideOutputs: false }); + it("records the result's error text by default", async () => { + const { spans } = await run(mcpErrorJourney()); + const conversation = one(spans, "claude_agent.conversation"); + expect(conversation.attributes["claude_agent.error.message"]).toBe("Reached maximum number of turns (3)"); + }); + + it("records the tool error text when outputs are visible (the default)", async () => { + const { spans } = await run(mcpErrorJourney()); const tool = one(spans, "tool.mcp__docs__search"); expect(tool.attributes["claude_agent.tool.error_message"]).toBe("server unavailable"); expect(tool.status.message).toBe("server unavailable"); @@ -582,12 +631,56 @@ describe("wrapQuery", () => { const conversation = one(spans, "claude_agent.conversation"); expect(conversation.status.code).toBe(SpanStatusCode.ERROR); expect(conversation.attributes["claude_agent.error.message"]).toBe("CLI exited with code 1"); - expect(conversation.events.some((e) => e.name === "exception")).toBe(true); + const exception = conversation.events.find((e) => e.name === "exception"); + expect(exception?.attributes).toEqual({ "exception.type": "Error", "exception.message": "CLI exited with code 1" }); expect(conversation.attributes["claude_agent.cancelled"]).toBeUndefined(); expect(spans).toHaveLength(3); for (const span of spans) expect(span.ended).toBe(true); }); + it("never exports a stack trace, local path, user name or host name from a thrown error", async () => { + const os = await import("os"); + const { provider, exporter } = memoryProvider(); + const home = os.homedir(); + const boom = new Error( + `spawn ${home}/proj/node_modules/@anthropic-ai/claude-agent-sdk/cli.js ENOENT (user ${os.userInfo().username}` + + ` on ${os.hostname()}, contact dev@example.com, C:\\Users\\someone\\app.js)`, + ); + const fake = makeFakeQuery(simpleToolJourney(), { throwAt: 3, error: boom }); + await expect(drain(wrapQuery(fake.query, { tracerProvider: provider })({ prompt: PROMPT }))).rejects.toBe(boom); + const conversation = one(exporter.getFinishedSpans(), "claude_agent.conversation"); + const exported = JSON.stringify({ + attributes: conversation.attributes, + events: conversation.events, + status: conversation.status, + }); + expect(exported).not.toContain("exception.stacktrace"); + expect(exported).not.toContain(home); + expect(exported).not.toContain(os.userInfo().username); + expect(exported).not.toContain(os.hostname()); + expect(exported).not.toContain("dev@example.com"); + expect(exported).not.toContain("someone"); + expect(exported).not.toContain("wrapper.test"); + expect(conversation.attributes["claude_agent.error.message"]).toContain( + "spawn ~/proj/node_modules/@anthropic-ai/claude-agent-sdk/cli.js ENOENT", + ); + }); + + it("keeps the thrown error's text off the span with hideOutputs (type only)", async () => { + const { provider, exporter } = memoryProvider(); + const boom = new TypeError("SECRET_ERROR_TEXT_MARKER"); + const fake = makeFakeQuery(simpleToolJourney(), { throwAt: 3, error: boom }); + const traced = wrapQuery(fake.query, { tracerProvider: provider, traceConfig: HIDE_ALL }); + await expect(drain(traced({ prompt: PROMPT }))).rejects.toBe(boom); + const conversation = one(exporter.getFinishedSpans(), "claude_agent.conversation"); + expect(JSON.stringify({ a: conversation.attributes, e: conversation.events, s: conversation.status })).not.toContain( + "SECRET_ERROR_TEXT_MARKER", + ); + expect(conversation.status.message).toBe("TypeError"); + expect(conversation.attributes["claude_agent.error.type"]).toBe("TypeError"); + expect(conversation.events.find((e) => e.name === "exception")?.attributes).toEqual({ "exception.type": "TypeError" }); + }); + const ASYNC_DISPOSE = (Symbol as unknown as { asyncDispose?: symbol }).asyncDispose ?? Symbol.for("Symbol.asyncDispose"); diff --git a/typescript/packages/traceai_claude_agent_sdk/src/instrumentation.ts b/typescript/packages/traceai_claude_agent_sdk/src/instrumentation.ts index 1aef030da..01b96b5f0 100644 --- a/typescript/packages/traceai_claude_agent_sdk/src/instrumentation.ts +++ b/typescript/packages/traceai_claude_agent_sdk/src/instrumentation.ts @@ -3,7 +3,22 @@ * `shutdown()` that flushes the tracer provider before a short script exits. */ import { TracerProvider, diag, trace } from "@opentelemetry/api"; -import { FITracer, TraceConfigOptions } from "@traceai/fi-core"; +import { + DEFAULT_HIDE_INPUTS, + DEFAULT_HIDE_INPUT_MESSAGES, + DEFAULT_HIDE_INPUT_TEXT, + DEFAULT_HIDE_OUTPUTS, + DEFAULT_HIDE_OUTPUT_MESSAGES, + DEFAULT_HIDE_OUTPUT_TEXT, + FITracer, + FI_HIDE_INPUTS, + FI_HIDE_INPUT_MESSAGES, + FI_HIDE_INPUT_TEXT, + FI_HIDE_OUTPUTS, + FI_HIDE_OUTPUT_MESSAGES, + FI_HIDE_OUTPUT_TEXT, + TraceConfigOptions, +} from "@traceai/fi-core"; import { ContentPolicy } from "./spans"; import { QueryTracingContext, wrapQueryFunction } from "./queryWrapper"; import { QueryFunctionLike } from "./types"; @@ -11,40 +26,62 @@ import { VERSION } from "./version"; export const INSTRUMENTATION_NAME = "@traceai/claude-agent-sdk"; -/** - * fi-core env keys. Read fail-closed, unlike fi-core (`value.toLowerCase() === - * "true"`, so `1`, `yes` or ` true` would turn capture ON): only an explicit - * `false` (trimmed, any case) opts in to content capture. - */ -const FI_HIDE_INPUTS = "FI_HIDE_INPUTS"; -const FI_HIDE_OUTPUTS = "FI_HIDE_OUTPUTS"; - export interface ClaudeAgentSDKInstrumentationConfig { /** Provider to create spans with. Defaults to the global provider at call time. */ tracerProvider?: TracerProvider; /** - * fi-core trace config. Unlike fi-core, `hideInputs` and `hideOutputs` - * default to `true` here: prompts, tool inputs and tool outputs can contain - * source code, file contents or secrets. Opt in with `false`, or with - * `FI_HIDE_INPUTS=false` / `FI_HIDE_OUTPUTS=false`. + * fi-core trace config, same defaults as every traceAI instrumentor: content + * is captured unless you hide it. `hideInputs` / `hideOutputs` (or + * `FI_HIDE_INPUTS=true` / `FI_HIDE_OUTPUTS=true`) hide every input / output + * value; `hideInputMessages` / `hideInputText` and `hideOutputMessages` / + * `hideOutputText` hide the prompt-side and assistant-side text only. */ traceConfig?: TraceConfigOptions; } -/** `false` only for an explicit `false`; any other value (or none) keeps content hidden. */ -function envHide(key: string): boolean { - const value = typeof process !== "undefined" ? process.env?.[key] : undefined; - return !(typeof value === "string" && value.trim().toLowerCase() === "false"); +/** + * One fi-core boolean flag: option > env var > fi-core default, with fi-core's + * own env rule (`value.toLowerCase() === "true"` hides). Same precedence as + * fi-core `generateTraceConfig`, which is not exported; read here so the env + * vars apply even when no `traceConfig` is passed (fi-core only reads them + * when one is). + */ +function flag(option: boolean | undefined, envKey: string, fallback: boolean): boolean { + if (option !== undefined) return option; + const value = typeof process !== "undefined" ? process.env?.[envKey] : undefined; + if (value !== undefined) return value.toLowerCase() === "true"; + return fallback; +} + +/** The fi-core flags this package reads, resolved (option > env > default). */ +export function resolveHideFlags(traceConfig?: TraceConfigOptions): Required< + Pick< + TraceConfigOptions, + "hideInputs" | "hideOutputs" | "hideInputMessages" | "hideOutputMessages" | "hideInputText" | "hideOutputText" + > +> { + return { + hideInputs: flag(traceConfig?.hideInputs, FI_HIDE_INPUTS, DEFAULT_HIDE_INPUTS), + hideOutputs: flag(traceConfig?.hideOutputs, FI_HIDE_OUTPUTS, DEFAULT_HIDE_OUTPUTS), + hideInputMessages: flag(traceConfig?.hideInputMessages, FI_HIDE_INPUT_MESSAGES, DEFAULT_HIDE_INPUT_MESSAGES), + hideOutputMessages: flag(traceConfig?.hideOutputMessages, FI_HIDE_OUTPUT_MESSAGES, DEFAULT_HIDE_OUTPUT_MESSAGES), + hideInputText: flag(traceConfig?.hideInputText, FI_HIDE_INPUT_TEXT, DEFAULT_HIDE_INPUT_TEXT), + hideOutputText: flag(traceConfig?.hideOutputText, FI_HIDE_OUTPUT_TEXT, DEFAULT_HIDE_OUTPUT_TEXT), + }; } /** - * Resolve content capture. Precedence: explicit option > fi-core env var > - * this package's default (hidden). + * Resolve content capture from the standard fi-core flags. Content is + * captured by default (fi-core `DEFAULT_HIDE_* = false`, like every traceAI + * instrumentor and the Python package). */ export function resolveContentPolicy(traceConfig?: TraceConfigOptions): ContentPolicy { + const f = resolveHideFlags(traceConfig); return { - hideInputs: traceConfig?.hideInputs ?? envHide(FI_HIDE_INPUTS), - hideOutputs: traceConfig?.hideOutputs ?? envHide(FI_HIDE_OUTPUTS), + hideInputs: f.hideInputs, + hideOutputs: f.hideOutputs, + hideInputMessages: f.hideInputs || f.hideInputMessages || f.hideInputText, + hideOutputMessages: f.hideOutputs || f.hideOutputMessages || f.hideOutputText, }; } @@ -127,9 +164,11 @@ export class ClaudeAgentSDKInstrumentation { private tracingContext(): QueryTracingContext { const policy = this.contentPolicy; const provider = this.tracerProvider ?? trace.getTracerProvider(); + // Always pass an object, so FITracer also reads the other fi-core env vars + // (FI_PII_REDACTION, ...) when the app gave no traceConfig. const tracer = new FITracer({ tracer: provider.getTracer(INSTRUMENTATION_NAME, VERSION), - traceConfig: { ...this.traceConfig, hideInputs: policy.hideInputs, hideOutputs: policy.hideOutputs }, + traceConfig: { ...(this.traceConfig ?? {}), ...resolveHideFlags(this.traceConfig) }, }); return { tracer, policy }; } diff --git a/typescript/packages/traceai_claude_agent_sdk/src/scrub.ts b/typescript/packages/traceai_claude_agent_sdk/src/scrub.ts new file mode 100644 index 000000000..ad0728354 --- /dev/null +++ b/typescript/packages/traceai_claude_agent_sdk/src/scrub.ts @@ -0,0 +1,130 @@ +/** + * What never leaves the process, in either privacy mode: local file paths + * (they carry the user name), stack traces, and, in error text, emails and + * this machine's user and host names. + * + * Content (prompt, tool input/output, assistant text) is otherwise exported + * as the agent saw it when capture is on. Paths inside it are rewritten: + * the query's working directory becomes `[cwd]`, the home directory `~`, and + * any other absolute local path `[path]`, so a Read of `/src/a.ts` is + * still readable as `[cwd]/src/a.ts`. + */ +import * as os from "os"; + +export const PATH_REMOVED = "[path]"; +export const CWD_TOKEN = "[cwd]"; +export const STACK_TRACE_REMOVED = "[stack trace removed]"; + +/** Where the paths to rewrite come from. */ +export interface ScrubContext { + /** The query's working directory (options.cwd, else process.cwd()). */ + cwd?: string; + /** The user's home directory (os.homedir()). */ + home?: string; +} + +const PYTHON_TRACEBACK = /Traceback \(most recent call last\):[\s\S]*/g; +/** A V8 stack: "\n at fn (file:line:col)" frames, from the first frame on. */ +const JS_STACK = /(?:\r?\n[ \t]+at [^\r\n]+)+/g; +const FILE_URL = /file:\/\/[^\s"'`<>]+/g; +const WINDOWS_PATH = /(?|*?]+/g; +const UNC_PATH = /(?|*?\\]+\\[^\s"'`<>|*?]+/g; +/** Absolute paths under the usual local roots; API routes such as /v1/messages are left alone. */ +const LOCAL_PATH = + /(?|*?,;)\]}]+/g; +const EMAIL = /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g; + +function escapeRegExp(text: string): string { + return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +function replacePrefix(text: string, prefix: string | undefined, token: string): string { + if (!prefix || prefix.length < 2 || prefix === "/" || !(prefix.startsWith("/") || /^[A-Za-z]:\\/.test(prefix))) { + return text; + } + const trimmed = prefix.replace(/[\\/]+$/, ""); + if (trimmed.length < 2) return text; + // The prefix itself, then a separator or a non-path character (not /Users/a/proj2 for /Users/a/proj). + return text.replace(new RegExp(`${escapeRegExp(trimmed)}(?=[\\\\/]|$|[^\\w.-])`, "g"), token); +} + +/** Default context: process.cwd() and os.homedir(), each only if readable. */ +export function defaultScrubContext(cwd?: unknown): ScrubContext { + const ctx: ScrubContext = {}; + if (typeof cwd === "string" && cwd) { + ctx.cwd = cwd; + } else { + try { + ctx.cwd = process.cwd(); + } catch { + // no cwd (deleted directory) + } + } + try { + ctx.home = os.homedir(); + } catch { + // no home directory + } + return ctx; +} + +/** Remove stack traces and rewrite local paths in exported content. */ +export function scrubContent(text: string, ctx: ScrubContext = {}): string { + if (!text) return text; + let out = text; + if (out.includes("Traceback (most recent call last)")) { + out = out.replace(PYTHON_TRACEBACK, STACK_TRACE_REMOVED); + } + if (/\n[ \t]+at /.test(out)) { + out = out.replace(JS_STACK, `\n${STACK_TRACE_REMOVED}`); + } + out = out.replace(FILE_URL, PATH_REMOVED); + // Longest first: the cwd usually sits under the home directory. + const prefixes: Array<[string | undefined, string]> = [ + [ctx.cwd, CWD_TOKEN], + [ctx.home, "~"], + ]; + prefixes.sort((a, b) => (b[0]?.length ?? 0) - (a[0]?.length ?? 0)); + for (const [prefix, token] of prefixes) { + out = replacePrefix(out, prefix, token); + } + out = out.replace(WINDOWS_PATH, PATH_REMOVED); + out = out.replace(UNC_PATH, PATH_REMOVED); + out = out.replace(LOCAL_PATH, PATH_REMOVED); + return out; +} + +function localIdentity(): string[] { + const out: string[] = []; + try { + const user = os.userInfo().username; + if (user && user.length >= 3) out.push(user); + } catch { + // no user info (sandboxed runtime) + } + try { + const host = os.hostname(); + if (host && host.length >= 3) { + out.push(host); + const short = host.split(".")[0]; + if (short && short.length >= 3 && short !== host) out.push(short); + } + } catch { + // no hostname + } + return out.sort((a, b) => b.length - a.length); +} + +/** + * Error text as exported: `scrubContent`, then emails and this machine's user + * and host names replaced. An SDK/CLI error such as "spawn /Users//... + * ENOENT" keeps its meaning without the path. + */ +export function scrubErrorText(text: string, ctx: ScrubContext = {}): string { + let out = scrubContent(text, ctx); + out = out.replace(EMAIL, ""); + for (const value of localIdentity()) { + out = out.split(value).join(""); + } + return out; +} diff --git a/typescript/packages/traceai_claude_agent_sdk/src/spans.ts b/typescript/packages/traceai_claude_agent_sdk/src/spans.ts index 3dfd6a8a5..08721f34d 100644 --- a/typescript/packages/traceai_claude_agent_sdk/src/spans.ts +++ b/typescript/packages/traceai_claude_agent_sdk/src/spans.ts @@ -36,6 +36,7 @@ import { getToolSource, parseMcpToolName, } from "./attributes"; +import { ScrubContext, defaultScrubContext, scrubContent, scrubErrorText } from "./scrub"; import { AbortSignalLike, AssistantMessageLike, @@ -47,10 +48,19 @@ import { UserMessageLike, } from "./types"; -/** Which content the wrapper may copy onto spans. Both default to hidden. */ +/** + * Which content the wrapper may copy onto spans, resolved from the standard + * fi-core flags (all default to false: content is captured). + */ export interface ContentPolicy { + /** FI_HIDE_INPUTS: every input-side value (prompt, system prompt, tool input, subagent prompt). */ hideInputs: boolean; + /** FI_HIDE_OUTPUTS: every output-side value (assistant text, tool output, final result, error text). */ hideOutputs: boolean; + /** hideInputs, FI_HIDE_INPUT_MESSAGES or FI_HIDE_INPUT_TEXT: prompt, system prompt and subagent prompt text. */ + hideInputMessages: boolean; + /** hideOutputs, FI_HIDE_OUTPUT_MESSAGES or FI_HIDE_OUTPUT_TEXT: assistant text and the final result. */ + hideOutputMessages: boolean; } /** How the wrapped iterator stopped. */ @@ -284,6 +294,8 @@ export class QueryTracer { private readonly appSetSession: boolean; private readonly providerName: string; private readonly mcpServerNames = new Set(); + /** cwd and home directory, rewritten in exported content (see scrub.ts). */ + private readonly scrubContext: ScrubContext; private conversation?: Span; private conversationContext?: Context; @@ -329,6 +341,7 @@ export class QueryTracer { this.contextAttributes = safeContextAttributes(args.parentContext); this.appSetSession = this.contextAttributes[T.SESSION_ID] != null; this.providerName = resolveProviderName(this.options); + this.scrubContext = defaultScrubContext(this.options?.cwd); for (const name of Object.keys(this.options?.mcpServers ?? {})) { this.mcpServerNames.add(name); } @@ -469,11 +482,18 @@ export class QueryTracer { } else if (failed) { const { type, message } = errorParts((outcome as { error: unknown }).error); conversation.setAttribute(A.ERROR_TYPE, type); - conversation.setAttribute(A.ERROR_MESSAGE, truncate(message, ERROR_MESSAGE_MAX)); - if ((outcome as { error: unknown }).error instanceof Error) { - conversation.recordException((outcome as { error: Error }).error); + // No recordException(): it would export the stack trace (local paths). + // The event carries the type, and the scrubbed message only when + // outputs are visible. + const event: Attributes = { "exception.type": type }; + let statusMessage = type; + if (!this.policy.hideOutputs) { + statusMessage = truncate(scrubErrorText(message, this.scrubContext), ERROR_MESSAGE_MAX); + conversation.setAttribute(A.ERROR_MESSAGE, statusMessage); + event["exception.message"] = statusMessage; } - conversation.setStatus({ code: SpanStatusCode.ERROR, message: truncate(message, ERROR_MESSAGE_MAX) }); + conversation.addEvent("exception", event, toHrTime(endMs)); + conversation.setStatus({ code: SpanStatusCode.ERROR, message: statusMessage }); } else if (!this.conversationIsError && (outcome.kind === "completed" || this.resultSeen)) { conversation.setStatus({ code: SpanStatusCode.OK }); } @@ -561,8 +581,8 @@ export class QueryTracer { turn.span.setAttribute(A.MESSAGE_HAS_TOOL_USE, turn.toolUseCount > 0); turn.span.setAttribute(A.MESSAGE_TOOL_USE_COUNT, turn.toolUseCount); - if (!this.policy.hideOutputs && turn.textParts.length > 0) { - const text = truncate(turn.textParts.join(" "), MESSAGE_CONTENT_MAX); + if (!this.policy.hideOutputMessages && turn.textParts.length > 0) { + const text = this.clean(turn.textParts.join(" "), MESSAGE_CONTENT_MAX); turn.span.setAttribute(A.MESSAGE_CONTENT, text); turn.span.setAttribute(T.OUTPUT_VALUE, text); turn.span.setAttribute(T.OUTPUT_MIME_TYPE, "text/plain"); @@ -631,17 +651,17 @@ export class QueryTracer { this.conversationIsError = true; const errorType = subtype ?? "error"; conversation.setAttribute(A.ERROR_TYPE, errorType); - if (Array.isArray(message.errors) && message.errors.length > 0) { + if (!this.policy.hideOutputs && Array.isArray(message.errors) && message.errors.length > 0) { conversation.setAttribute( A.ERROR_MESSAGE, - truncate(message.errors.map(String).join("; "), ERROR_MESSAGE_MAX), + truncate(scrubErrorText(message.errors.map(String).join("; "), this.scrubContext), ERROR_MESSAGE_MAX), ); } conversation.setStatus({ code: SpanStatusCode.ERROR, message: errorType }); } - if (!this.policy.hideOutputs && typeof message.result === "string") { - conversation.setAttribute(T.OUTPUT_VALUE, truncate(message.result, CONTENT_MAX)); + if (!this.policy.hideOutputMessages && typeof message.result === "string") { + conversation.setAttribute(T.OUTPUT_VALUE, this.clean(message.result, CONTENT_MAX)); conversation.setAttribute(T.OUTPUT_MIME_TYPE, "text/plain"); } } @@ -833,11 +853,11 @@ export class QueryTracer { } const input = block.input; if (!this.policy.hideInputs && input !== undefined && input !== null) { - const serialized = safeJson(input); + const serialized = this.clean(safeJson(input), CONTENT_MAX); span.setAttribute(A.TOOL_INPUT, serialized); span.setAttribute(T.INPUT_VALUE, serialized); span.setAttribute(T.INPUT_MIME_TYPE, typeof input === "string" ? "text/plain" : "application/json"); - setToolSpecificAttributes(span, name, input); + setToolSpecificAttributes(span, name, input, (value, max) => this.clean(value, max)); } const tool: ToolState = { span, name, startMs, turn }; @@ -870,9 +890,9 @@ export class QueryTracer { const endMs = clockMs(); span.setAttribute(A.TOOL_DURATION_MS, endMs - tool.startMs); span.setAttribute(A.TOOL_IS_ERROR, isError); - const output = flattenToolResult(content); + const output = this.policy.hideOutputs ? "" : this.clean(flattenToolResult(content), CONTENT_MAX); if (!this.policy.hideOutputs) { - const serialized = truncate(output, CONTENT_MAX); + const serialized = output; span.setAttribute(A.TOOL_OUTPUT, serialized); span.setAttribute(T.OUTPUT_VALUE, serialized); span.setAttribute(T.OUTPUT_MIME_TYPE, "text/plain"); @@ -914,13 +934,14 @@ export class QueryTracer { if (scope !== null) { span.setAttribute(A.PARENT_TOOL_USE_ID, scope); } - if (!this.policy.hideInputs) { + if (!this.policy.hideInputMessages) { if (typeof fields.description === "string" && fields.description) { - span.setAttribute(A.SUBAGENT_DESCRIPTION, truncate(fields.description, 500)); + span.setAttribute(A.SUBAGENT_DESCRIPTION, this.clean(fields.description, 500)); } if (typeof fields.prompt === "string" && fields.prompt) { - span.setAttribute(A.SUBAGENT_PROMPT, truncate(fields.prompt, PROMPT_MAX)); - span.setAttribute(T.INPUT_VALUE, truncate(fields.prompt, PROMPT_MAX)); + const prompt = this.clean(fields.prompt, PROMPT_MAX); + span.setAttribute(A.SUBAGENT_PROMPT, prompt); + span.setAttribute(T.INPUT_VALUE, prompt); span.setAttribute(T.INPUT_MIME_TYPE, "text/plain"); } } @@ -1123,6 +1144,16 @@ export class QueryTracer { } } + /** + * Content as exported: stack traces cut and local paths rewritten + * (scrub.ts), then truncated like the Python package. Bounded before the + * scrub so a huge tool result is not scanned in full. + */ + private clean(value: string, max: number): string { + const bounded = value.length > max * 2 ? value.slice(0, max * 2) : value; + return truncate(scrubContent(bounded, this.scrubContext), max); + } + /** The model this query asked for: options.model, else the init message's model. */ private requestedModel(): string | undefined { const model = this.options?.model; @@ -1164,18 +1195,18 @@ export class QueryTracer { attributes[A.SESSION_FORK_FROM] = resume; } - if (!this.policy.hideInputs) { + if (!this.policy.hideInputMessages) { if (typeof this.params.prompt === "string") { - const prompt = truncate(this.params.prompt, PROMPT_MAX); + const prompt = this.clean(this.params.prompt, PROMPT_MAX); attributes[A.AGENT_PROMPT] = prompt; attributes[T.INPUT_VALUE] = prompt; attributes[T.INPUT_MIME_TYPE] = "text/plain"; } const systemPrompt = options.systemPrompt; if (typeof systemPrompt === "string" && systemPrompt) { - attributes[A.AGENT_SYSTEM_PROMPT] = truncate(systemPrompt, SYSTEM_PROMPT_MAX); + attributes[A.AGENT_SYSTEM_PROMPT] = this.clean(systemPrompt, SYSTEM_PROMPT_MAX); } else if (systemPrompt && typeof systemPrompt === "object") { - attributes[A.AGENT_SYSTEM_PROMPT] = safeJson(systemPrompt, SYSTEM_PROMPT_MAX); + attributes[A.AGENT_SYSTEM_PROMPT] = this.clean(safeJson(systemPrompt, SYSTEM_PROMPT_MAX), SYSTEM_PROMPT_MAX); } } return attributes; @@ -1262,8 +1293,13 @@ function cumulativeUsage(message: ResultMessageLike): UsageTotals | undefined { return values.some((v) => v !== 0) ? totals : undefined; } -/** Python `_set_tool_specific_attributes`. Only called when inputs are visible. */ -function setToolSpecificAttributes(span: Span, toolName: string, input: unknown): void { +/** Python `_set_tool_specific_attributes`. Only called when inputs are visible; values go through `clean`. */ +function setToolSpecificAttributes( + span: Span, + toolName: string, + input: unknown, + clean: (value: string, max: number) => string, +): void { if (!input || typeof input !== "object") return; const fields = input as Record; const str = (key: string) => (typeof fields[key] === "string" ? (fields[key] as string) : undefined); @@ -1272,28 +1308,28 @@ function setToolSpecificAttributes(span: Span, toolName: string, input: unknown) case "Write": case "Edit": { const filePath = str("file_path"); - if (filePath) span.setAttribute(A.TOOL_FILE_PATH, filePath); + if (filePath) span.setAttribute(A.TOOL_FILE_PATH, clean(filePath, CONTENT_MAX)); break; } case "Bash": { const command = str("command"); - if (command) span.setAttribute(A.TOOL_COMMAND, truncate(command, TOOL_COMMAND_MAX)); + if (command) span.setAttribute(A.TOOL_COMMAND, clean(command, TOOL_COMMAND_MAX)); break; } case "Glob": case "Grep": { const pattern = str("pattern"); - if (pattern) span.setAttribute(A.TOOL_PATTERN, pattern); + if (pattern) span.setAttribute(A.TOOL_PATTERN, clean(pattern, CONTENT_MAX)); break; } case "WebSearch": { const query = str("query"); - if (query) span.setAttribute(A.TOOL_SEARCH_QUERY, query); + if (query) span.setAttribute(A.TOOL_SEARCH_QUERY, clean(query, CONTENT_MAX)); break; } case "WebFetch": { const url = str("url"); - if (url) span.setAttribute(A.TOOL_URL, url); + if (url) span.setAttribute(A.TOOL_URL, clean(url, CONTENT_MAX)); break; } default: diff --git a/typescript/packages/traceai_claude_agent_sdk/src/types.ts b/typescript/packages/traceai_claude_agent_sdk/src/types.ts index 5b9ddb0c6..abc8c0c9a 100644 --- a/typescript/packages/traceai_claude_agent_sdk/src/types.ts +++ b/typescript/packages/traceai_claude_agent_sdk/src/types.ts @@ -1,7 +1,7 @@ /** * Structural types for the parts of the Claude Agent SDK surface this package * reads. They are written from `sdk.d.ts` of `@anthropic-ai/claude-agent-sdk` - * 0.3.289 (and checked against 0.3.142) and deliberately do not import the + * 0.3.289 (and checked against 0.3.277 and 0.3.302) and deliberately do not import the * SDK, so the SDK stays a peer dependency with no type or runtime import. * * Every field is optional: the wrapper reads defensively and never fails the @@ -131,6 +131,8 @@ export interface OptionsLike { resumeSessionAt?: string; sessionId?: string; mcpServers?: Record; + /** The CLI's working directory; rewritten to `[cwd]` in exported content. */ + cwd?: string; [key: string]: unknown; } From 832fdecd6a2c54d5154056986f3e65786c5dbe58 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Wed, 21 Oct 2026 15:33:33 -0700 Subject: [PATCH 19/19] test(claude-agent-sdk): neutral loopback port in the URL scrub test Co-Authored-By: Claude Opus 5.5 --- .../traceai_claude_agent_sdk/src/__tests__/scrub.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/typescript/packages/traceai_claude_agent_sdk/src/__tests__/scrub.test.ts b/typescript/packages/traceai_claude_agent_sdk/src/__tests__/scrub.test.ts index a755aa336..f80c4b6ac 100644 --- a/typescript/packages/traceai_claude_agent_sdk/src/__tests__/scrub.test.ts +++ b/typescript/packages/traceai_claude_agent_sdk/src/__tests__/scrub.test.ts @@ -35,7 +35,7 @@ describe("scrubContent", () => { "POST /v1/messages?beta=true", "https://example.com/Users/docs", "src/index.ts and ./README.md", - "http://127.0.0.1:8317/v1/messages", + "http://127.0.0.1:8080/v1/messages", ]) { expect(scrubContent(value, ctx)).toBe(value); }