diff --git a/python/frameworks/anthropic/poetry.lock b/python/frameworks/anthropic/poetry.lock index 65ee3277e..1ab16753a 100644 --- a/python/frameworks/anthropic/poetry.lock +++ b/python/frameworks/anthropic/poetry.lock @@ -226,14 +226,14 @@ test = ["pytest (>=6)"] [[package]] name = "fi-instrumentation-otel" -version = "1.1.0" -description = "OpenTelemetry instrumentation" +version = "1.2.2" +description = "OpenTelemetry tracing for AI applications: tracer setup, span helpers, exporters, and semantic conventions for Future AGI." optional = false python-versions = "<3.14,>3.9" groups = ["main"] files = [ - {file = "fi_instrumentation_otel-1.1.0-py3-none-any.whl", hash = "sha256:4fa2bf663e0ce09447aca92ae798cd7b0dee35c5c92c2e033f1d89d158a9e62c"}, - {file = "fi_instrumentation_otel-1.1.0.tar.gz", hash = "sha256:7ec324c3c9242aa87a88cf23a21c11919b8736c6451c5fb50917f592a853b059"}, + {file = "fi_instrumentation_otel-1.2.2-py3-none-any.whl", hash = "sha256:25cf88f93e33aa928d4fae0f631afe8a32565639490862b715869e0633becbaf"}, + {file = "fi_instrumentation_otel-1.2.2.tar.gz", hash = "sha256:b675df47d7a03c506fbd2e4b0420fe4f74d12a6353a88b219bfd97797b276d4c"}, ] [package.dependencies] @@ -1135,4 +1135,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = ">3.9, <3.14" -content-hash = "239c305494e211c7f0af947af955f6163a90bad1795aba8ce185de3a5846d963" +content-hash = "212f97a32a59ae314872c8a3eab2de167f6372210b691eb867b40f7620b933f4" diff --git a/python/frameworks/anthropic/pyproject.toml b/python/frameworks/anthropic/pyproject.toml index 7f73c2c0b..e1930f4ca 100644 --- a/python/frameworks/anthropic/pyproject.toml +++ b/python/frameworks/anthropic/pyproject.toml @@ -30,7 +30,7 @@ classifiers = [ [tool.poetry.dependencies] python = ">3.9, <3.14" anthropic = ">=0.41.0" -fi-instrumentation-otel = ">=0.1.7" +fi-instrumentation-otel = ">=1.2.2" [tool.poetry.urls] diff --git a/python/frameworks/anthropic/tests/test_r2_native_regression.py b/python/frameworks/anthropic/tests/test_r2_native_regression.py new file mode 100644 index 000000000..d55670c93 --- /dev/null +++ b/python/frameworks/anthropic/tests/test_r2_native_regression.py @@ -0,0 +1,165 @@ +"""Anthropic SDK offline regressions; no vendor calls are made.""" +import asyncio +import json + +import anthropic +import pytest +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from traceai_anthropic import AnthropicInstrumentor + +try: + import httpx2 as httpx +except ImportError: + import httpx + +MESSAGE = {"id": "msg_local", "type": "message", "role": "assistant", "content": [{"type": "text", "text": "answer"}], "model": "local", "stop_reason": "end_turn", "stop_sequence": None, "usage": {"input_tokens": 4, "output_tokens": 2}} +START = {"id": "msg_local", "type": "message", "role": "assistant", "content": [], "model": "local", "stop_reason": None, "stop_sequence": None, "usage": {"input_tokens": 4, "output_tokens": 0}} +EVENTS = [{"type": "message_start", "message": START}, {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "hello"}}, {"type": "content_block_stop", "index": 0}, {"type": "message_delta", "delta": {"stop_reason": "end_turn", "stop_sequence": None}, "usage": {"output_tokens": 2}}, {"type": "message_stop"}] +BODY = "".join("event: " + event["type"] + "\ndata: " + json.dumps(event) + "\n\n" for event in EVENTS) +PARAMS = {"model": "local", "max_tokens": 16, "messages": [{"role": "user", "content": "hello"}]} +TOOLS = [{"role": "assistant", "content": [{"type": "tool_use", "id": "tool1", "name": "lookup", "input": {"query": "abc"}}]}, {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "tool1", "content": "ok"}]}] + + +def setup(): + p = TracerProvider() + x = InMemorySpanExporter() + p.add_span_processor(SimpleSpanProcessor(x)) + i = AnthropicInstrumentor() + i.instrument(tracer_provider=p) + return p, x, i + + +def handler(req): + if json.loads(req.content).get("stream"): + return httpx.Response(200, headers={"content-type": "text/event-stream"}, text=BODY) + return httpx.Response(200, json=MESSAGE) + + +def test_native_tool_mapping_sync(): + c = anthropic.Anthropic(api_key="local", http_client=httpx.Client(transport=httpx.MockTransport(handler))) + params = dict(PARAMS, messages=TOOLS) + expected = c.messages.create(**params).model_dump() + p, x, i = setup() + try: + assert c.messages.create(**params).model_dump() == expected + assert len(x.get_finished_spans()) == 1 + finally: + i.uninstrument(); c.close(); p.shutdown() + + +def test_native_tool_mapping_async(): + async def run(): + c = anthropic.AsyncAnthropic(api_key="local", http_client=httpx.AsyncClient(transport=httpx.MockTransport(handler))) + params = dict(PARAMS, messages=TOOLS) + expected = (await c.messages.create(**params)).model_dump() + p, x, i = setup() + try: + assert (await c.messages.create(**params)).model_dump() == expected + assert len(x.get_finished_spans()) == 1 + finally: + i.uninstrument(); await c.close(); p.shutdown() + asyncio.run(run()) + + +@pytest.mark.parametrize("mode", ["full", "close", "context"]) +def test_native_sync_stream_lifecycle(mode): + p, x, i = setup() + c = anthropic.Anthropic(api_key="local", http_client=httpx.Client(transport=httpx.MockTransport(handler))) + try: + s = c.messages.create(**PARAMS, stream=True) + if mode == "full": + assert len(list(s)) == len(EVENTS) + elif mode == "close": + assert s.close() is None + assert s.close() is None + else: + with s as entered: + assert entered is s + assert next(iter(entered)).type == "message_start" + assert len(x.get_finished_spans()) == 1 + finally: + i.uninstrument(); c.close(); p.shutdown() + + +@pytest.mark.parametrize("mode", ["full", "close", "context"]) +def test_native_async_stream_lifecycle(mode): + async def run(): + p, x, i = setup() + c = anthropic.AsyncAnthropic(api_key="local", http_client=httpx.AsyncClient(transport=httpx.MockTransport(handler))) + try: + s = await c.messages.create(**PARAMS, stream=True) + if mode == "full": + assert len([e async for e in s]) == len(EVENTS) + elif mode == "close": + assert await s.close() is None + assert await s.close() is None + else: + async with s as entered: + assert entered is s + assert (await entered.__aiter__().__anext__()).type == "message_start" + assert len(x.get_finished_spans()) == 1 + finally: + i.uninstrument(); await c.close(); p.shutdown() + asyncio.run(run()) + + +def test_native_typed_tool_use(): + block = anthropic.types.ToolUseBlock(id="tool1", name="lookup", input={"query": "abc"}, type="tool_use") + params = dict(PARAMS, messages=[{"role": "assistant", "content": [block]}, TOOLS[1]]) + c = anthropic.Anthropic(api_key="local", http_client=httpx.Client(transport=httpx.MockTransport(handler))) + expected = c.messages.create(**params).model_dump() + p, x, i = setup() + try: + assert c.messages.create(**params).model_dump() == expected + assert len(x.get_finished_spans()) == 1 + finally: + i.uninstrument(); c.close(); p.shutdown() + + +def test_native_stream_error_is_hidden(monkeypatch, caplog): + secret = "R2_ERROR_SECRET_831" + monkeypatch.setenv("FI_HIDE_ERROR_DETAILS", "true") + body = 'event: error\ndata: ' + json.dumps({"type": "error", "error": {"type": "api_error", "message": secret}}) + '\n\n' + c = anthropic.Anthropic(api_key="local", http_client=httpx.Client(transport=httpx.MockTransport(lambda req: httpx.Response(200, headers={"content-type": "text/event-stream"}, text=body)))) + p, x, i = setup() + try: + with pytest.raises(Exception, match=secret): + list(c.messages.create(**PARAMS, stream=True)) + spans = x.get_finished_spans() + assert len(spans) == 1 + serialized = str(spans[0].status) + str(spans[0].attributes) + str([(e.name, dict(e.attributes)) for e in spans[0].events]) + caplog.text + assert secret not in serialized + finally: + i.uninstrument(); c.close(); p.shutdown() + + +def test_native_async_stream_cancel(): + async def run(): + entered = asyncio.Event() + class WaitingStream(httpx.AsyncByteStream): + async def __aiter__(self): + entered.set() + await asyncio.Event().wait() + yield b"" + def slow(req): + return httpx.Response(200, headers={"content-type": "text/event-stream"}, stream=WaitingStream()) + c = anthropic.AsyncAnthropic(api_key="local", http_client=httpx.AsyncClient(transport=httpx.MockTransport(slow))) + p, x, i = setup() + try: + s = await c.messages.create(**PARAMS, stream=True) + async def consume(): + async for _ in s: + pass + task = asyncio.create_task(consume()) + await entered.wait() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert len(x.get_finished_spans()) == 1 + await s.close() + assert len(x.get_finished_spans()) == 1 + finally: + i.uninstrument(); await c.close(); p.shutdown() + asyncio.run(run()) diff --git a/python/frameworks/anthropic/tests/test_r3_stream_resume.py b/python/frameworks/anthropic/tests/test_r3_stream_resume.py new file mode 100644 index 000000000..a046f94d0 --- /dev/null +++ b/python/frameworks/anthropic/tests/test_r3_stream_resume.py @@ -0,0 +1,249 @@ +"""Native SDK stream ownership regressions, using local mock HTTP only.""" +import asyncio +import gc +import json +from importlib.metadata import version + +import anthropic +import pytest +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import StatusCode +from traceai_anthropic import AnthropicInstrumentor + +try: + import httpx2 as httpx +except ImportError: + import httpx + +START = {'id': 'msg_1', 'type': 'message', 'role': 'assistant', 'content': [], 'model': 'local', 'stop_reason': None, 'stop_sequence': None, 'usage': {'input_tokens': 4, 'output_tokens': 0}} +EVENTS = [{'type': 'message_start', 'message': START}, {'type': 'content_block_start', 'index': 0, 'content_block': {'type': 'text', 'text': '', 'citations': []}}, {'type': 'content_block_delta', 'index': 0, 'delta': {'type': 'text_delta', 'text': 'hello'}}, {'type': 'content_block_stop', 'index': 0}, {'type': 'message_delta', 'delta': {'stop_reason': 'end_turn', 'stop_sequence': None}, 'usage': {'output_tokens': 2}}, {'type': 'message_stop'}] +CITATION = {'type': 'content_block_delta', 'index': 0, 'delta': {'type': 'citations_delta', 'citation': {'type': 'char_location', 'cited_text': 'fact', 'document_index': 0, 'document_title': 'doc', 'start_char_index': 0, 'end_char_index': 4}}} +PARAMS = dict(model='local', max_tokens=16, messages=[{'role': 'user', 'content': 'hi'}], stream=True) +SECRET = 'r3_hidden_error_sentinel' + + +def body(events): + return ''.join('event: ' + e['type'] + '\ndata: ' + json.dumps(e) + '\n\n' for e in events) + + +def make_client(events, asynchronous=False): + handler = lambda req: httpx.Response(200, headers={'content-type': 'text/event-stream'}, text=body(events)) + if asynchronous: + return anthropic.AsyncAnthropic(api_key='local', http_client=httpx.AsyncClient(transport=httpx.MockTransport(handler))) + return anthropic.Anthropic(api_key='local', http_client=httpx.Client(transport=httpx.MockTransport(handler))) + + +def setup(): + p = TracerProvider(); exporter = InMemorySpanExporter(); p.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = AnthropicInstrumentor(); instrumentor.instrument(tracer_provider=p) + return p, exporter, instrumentor + + +def assert_span(exporter, error=False, full=True): + spans = exporter.get_finished_spans() + assert len(spans) == 1 + span = spans[0] + assert span.status.status_code == (StatusCode.ERROR if error else StatusCode.OK) + if full: + attrs = dict(span.attributes) + assert 'hello' in str(attrs) + assert attrs['gen_ai.usage.input_tokens'] == 4 + assert attrs['gen_ai.usage.output_tokens'] == 2 + assert SECRET not in str(span.status) + str(span.attributes) + str([(e.name, dict(e.attributes)) for e in span.events]) + + +@pytest.mark.parametrize('mode', ['temporary', 'break', 'iterator_close', 'stream_close']) +def test_sync_stream_lifetime(mode): + c = make_client(EVENTS); p, x, i = setup() + try: + stream = c.messages.create(**PARAMS) + if mode == 'temporary': + assert next(iter(stream)).type == 'message_start'; gc.collect() + elif mode == 'break': + for event in stream: + assert event.type == 'message_start'; break + gc.collect() + else: + it = iter(stream); assert next(it).type == 'message_start'; it.close() + assert len(x.get_finished_spans()) == 0 + if mode == 'stream_close': + stream.close(); stream.close(); assert_span(x, full=False) + else: + assert list(stream)[-1].type == 'message_stop'; assert_span(x) + stream.close(); assert_span(x) + finally: + i.uninstrument(); c.close(); p.shutdown() + + +@pytest.mark.parametrize('mode', ['break', 'iterator_close', 'stream_close']) +def test_async_stream_lifetime(mode): + async def run(): + c = make_client(EVENTS, True); p, x, i = setup() + try: + stream = await c.messages.create(**PARAMS) + if mode == 'break': + async for event in stream: + assert event.type == 'message_start'; break + await asyncio.sleep(0); gc.collect(); await asyncio.sleep(0) + else: + it = stream.__aiter__(); assert (await it.__anext__()).type == 'message_start'; await it.aclose() + assert len(x.get_finished_spans()) == 0 + if mode == 'stream_close': + await stream.close(); await stream.close(); assert_span(x, full=False) + else: + assert [e async for e in stream][-1].type == 'message_stop'; assert_span(x) + await stream.close(); assert_span(x) + finally: + i.uninstrument(); await c.close(); p.shutdown() + asyncio.run(run()) + + +@pytest.mark.parametrize('asynchronous', [False, True]) +def test_resumed_api_error_preserves_partial_output_and_privacy(asynchronous, monkeypatch, caplog): + monkeypatch.setenv('FI_HIDE_ERROR_DETAILS', 'true') + events = EVENTS[:-1] + [{'type': 'error', 'error': {'type': 'api_error', 'message': SECRET}}] + async def run_async(): + c = make_client(events, True); p, x, i = setup() + try: + stream = await c.messages.create(**PARAMS) + it = stream.__aiter__(); await it.__anext__(); await it.aclose() + assert not x.get_finished_spans() + with pytest.raises(anthropic.APIStatusError, match=SECRET): + async for _ in stream: pass + assert_span(x, error=True); assert SECRET not in caplog.text + await stream.close(); assert_span(x, error=True) + finally: + i.uninstrument(); await c.close(); p.shutdown() + if asynchronous: + asyncio.run(run_async()); return + c = make_client(events); p, x, i = setup() + try: + stream = c.messages.create(**PARAMS); assert next(iter(stream)).type == 'message_start' + assert not x.get_finished_spans() + with pytest.raises(anthropic.APIStatusError, match=SECRET): list(stream) + assert_span(x, error=True); assert SECRET not in caplog.text + stream.close(); assert_span(x, error=True) + finally: + i.uninstrument(); c.close(); p.shutdown() + + +@pytest.mark.parametrize('asynchronous', [False, True]) +def test_native_citation_parity(asynchronous): + if version('anthropic') == '0.41.0': + pytest.skip('NA: 0.41.0 has no citations delta in its SDK type contract') + events = EVENTS[:3] + [CITATION] + EVENTS[3:] + async def run_async(): + c = make_client(events, True) + expected = [e.model_dump() async for e in await c.messages.create(**PARAMS)] + p, x, i = setup() + try: + actual = [e.model_dump() async for e in await c.messages.create(**PARAMS)] + assert actual == expected; assert_span(x) + finally: + i.uninstrument(); await c.close(); p.shutdown() + if asynchronous: + asyncio.run(run_async()); return + c = make_client(events); expected = [e.model_dump() for e in c.messages.create(**PARAMS)] + p, x, i = setup() + try: + assert [e.model_dump() for e in c.messages.create(**PARAMS)] == expected; assert_span(x) + finally: + i.uninstrument(); c.close(); p.shutdown() + + +@pytest.mark.parametrize('asynchronous', [False, True]) +def test_native_thinking_tool_and_unknown_delta_parity(asynchronous): + if version('anthropic') == '0.41.0': + pytest.skip('NA: thinking/unknown delta fixture is for current SDK') + extra = [ + {'type': 'content_block_start', 'index': 1, 'content_block': {'type': 'thinking', 'thinking': '', 'signature': ''}}, + {'type': 'content_block_delta', 'index': 1, 'delta': {'type': 'thinking_delta', 'thinking': 'reason'}}, + {'type': 'content_block_delta', 'index': 1, 'delta': {'type': 'signature_delta', 'signature': 'signed'}}, + {'type': 'content_block_stop', 'index': 1}, + {'type': 'content_block_start', 'index': 2, 'content_block': {'type': 'tool_use', 'id': 'tool1', 'name': 'lookup', 'input': {}}}, + {'type': 'content_block_delta', 'index': 2, 'delta': {'type': 'input_json_delta', 'partial_json': '{"x":1}'}}, + {'type': 'content_block_delta', 'index': 2, 'delta': {'type': 'future_nontext_delta', 'value': 'future'}}, + {'type': 'content_block_stop', 'index': 2}, + ] + events = EVENTS[:4] + extra + EVENTS[4:] + # Compare native models, not model_dump(): SDK 0.57.1 accepts unknown + # deltas but its Pydantic serializer cannot dump their fallback model. + async def run_async(): + c = make_client(events, True) + expected = [e async for e in await c.messages.create(**PARAMS)] + p, x, i = setup() + try: + assert [e async for e in await c.messages.create(**PARAMS)] == expected + assert_span(x) + assert 'lookup' in str(x.get_finished_spans()[0].attributes) + finally: + i.uninstrument(); await c.close(); p.shutdown() + if asynchronous: + asyncio.run(run_async()); return + c = make_client(events) + expected = list(c.messages.create(**PARAMS)) + p, x, i = setup() + try: + assert list(c.messages.create(**PARAMS)) == expected + assert_span(x) + assert 'lookup' in str(x.get_finished_spans()[0].attributes) + finally: + i.uninstrument(); c.close(); p.shutdown() + + +@pytest.mark.parametrize('asynchronous', [False, True]) +def test_resumed_stream_content_privacy_and_usage(asynchronous, monkeypatch): + monkeypatch.setenv('FI_HIDE_INPUTS', 'true') + monkeypatch.setenv('FI_HIDE_OUTPUTS', 'true') + params = dict(PARAMS, messages=[{'role': 'user', 'content': 'r3_private_prompt_sentinel'}]) + def check(x): + spans = x.get_finished_spans(); assert len(spans) == 1 + span = spans[0]; attrs = dict(span.attributes) + assert span.status.status_code == StatusCode.OK + assert 'r3_private_prompt_sentinel' not in str(attrs) + assert 'hello' not in str(attrs) + assert attrs['gen_ai.usage.input_tokens'] == 4 + assert attrs['gen_ai.usage.output_tokens'] == 2 + async def run_async(): + c = make_client(EVENTS, True); p, x, i = setup() + try: + s = await c.messages.create(**params); it = s.__aiter__(); await it.__anext__(); await it.aclose() + assert not x.get_finished_spans() + assert [event async for event in s][-1].type == 'message_stop'; check(x) + finally: + i.uninstrument(); await c.close(); p.shutdown() + if asynchronous: + asyncio.run(run_async()); return + c = make_client(EVENTS); p, x, i = setup() + try: + s = c.messages.create(**params); next(iter(s)); assert not x.get_finished_spans() + assert list(s)[-1].type == 'message_stop'; check(x) + finally: + i.uninstrument(); c.close(); p.shutdown() + + +@pytest.mark.parametrize('asynchronous', [False, True]) +def test_consumer_injected_exception_does_not_finish_stream(asynchronous): + class ConsumerError(Exception): + pass + async def run_async(): + c = make_client(EVENTS, True); p, x, i = setup() + try: + s = await c.messages.create(**PARAMS); it = s.__aiter__(); await it.__anext__() + with pytest.raises(ConsumerError): await it.athrow(ConsumerError('consumer')) + assert not x.get_finished_spans() + assert [event async for event in s][-1].type == 'message_stop'; assert_span(x) + finally: + i.uninstrument(); await c.close(); p.shutdown() + if asynchronous: + asyncio.run(run_async()); return + c = make_client(EVENTS); p, x, i = setup() + try: + s = c.messages.create(**PARAMS); it = iter(s); next(it) + with pytest.raises(ConsumerError): it.throw(ConsumerError('consumer')) + assert not x.get_finished_spans() + assert list(s)[-1].type == 'message_stop'; assert_span(x) + finally: + i.uninstrument(); c.close(); p.shutdown() diff --git a/python/frameworks/anthropic/tests/test_r4_direct_lifecycle.py b/python/frameworks/anthropic/tests/test_r4_direct_lifecycle.py new file mode 100644 index 000000000..97ea105d8 --- /dev/null +++ b/python/frameworks/anthropic/tests/test_r4_direct_lifecycle.py @@ -0,0 +1,225 @@ +"""Native SDK direct/mixed stream advancement and in-flight close ownership.""" +import asyncio +import json + +import anthropic +import pytest +from opentelemetry.trace import StatusCode + +from test_r3_stream_resume import EVENTS, PARAMS, SECRET, assert_span, httpx, make_client, setup + + +def test_mixed_sync_direct_iter_resume(): + client = make_client(EVENTS) + provider, exporter, instrumentor = setup() + try: + stream = client.messages.create(**PARAMS) + assert next(stream).type == 'message_start' + iterator = iter(stream) + assert next(iterator).type == 'content_block_start' + assert next(stream).type == 'content_block_delta' + iterator.close() + assert not exporter.get_finished_spans() + assert [event.type for event in stream] == ['content_block_stop', 'message_delta', 'message_stop'] + assert_span(exporter) + with pytest.raises(StopIteration): + next(stream) + assert_span(exporter) + finally: + instrumentor.uninstrument(); client.close(); provider.shutdown() + + +def test_mixed_async_direct_iter_resume(): + async def run(): + client = make_client(EVENTS, True) + provider, exporter, instrumentor = setup() + try: + stream = await client.messages.create(**PARAMS) + assert (await stream.__anext__()).type == 'message_start' + iterator = stream.__aiter__() + assert (await iterator.__anext__()).type == 'content_block_start' + assert (await stream.__anext__()).type == 'content_block_delta' + await iterator.aclose() + assert not exporter.get_finished_spans() + assert [event.type async for event in stream] == ['content_block_stop', 'message_delta', 'message_stop'] + assert_span(exporter) + with pytest.raises(StopAsyncIteration): + await stream.__anext__() + assert_span(exporter) + finally: + instrumentor.uninstrument(); await client.close(); provider.shutdown() + asyncio.run(run()) + + +@pytest.mark.parametrize('asynchronous', [False, True]) +def test_direct_partial_close(asynchronous): + async def run(): + client = make_client(EVENTS, True) + provider, exporter, instrumentor = setup() + try: + stream = await client.messages.create(**PARAMS) + for _ in range(3): + await stream.__anext__() + assert not exporter.get_finished_spans() + assert await stream.close() is None + assert_span(exporter, full=False) + attrs = exporter.get_finished_spans()[0].attributes + assert 'hello' in str(attrs) + assert attrs['gen_ai.usage.input_tokens'] == 4 + finally: + instrumentor.uninstrument(); await client.close(); provider.shutdown() + if asynchronous: + asyncio.run(run()); return + client = make_client(EVENTS) + provider, exporter, instrumentor = setup() + try: + stream = client.messages.create(**PARAMS) + for _ in range(3): + next(stream) + assert not exporter.get_finished_spans() + assert stream.close() is None + assert_span(exporter, full=False) + attrs = exporter.get_finished_spans()[0].attributes + assert 'hello' in str(attrs) + assert attrs['gen_ai.usage.input_tokens'] == 4 + finally: + instrumentor.uninstrument(); client.close(); provider.shutdown() + + +@pytest.mark.parametrize('traced', [False, True]) +@pytest.mark.parametrize('outcome', ['error', 'cancel']) +def test_close_returns_before_pending_error_and_multiple_closers(traced, outcome, monkeypatch): + monkeypatch.setenv('FI_HIDE_ERROR_DETAILS', 'true') + class Wire(httpx.AsyncByteStream): + def __init__(self): + self.pending = asyncio.Event(); self.release = asyncio.Event() + async def __aiter__(self): + yield ('event: message_start\ndata: ' + json.dumps(EVENTS[0]) + '\n\n').encode() + self.pending.set() + await self.release.wait() + raise RuntimeError(SECRET) + async def aclose(self): + pass + async def run(): + wire = Wire() + client = anthropic.AsyncAnthropic(api_key='local', http_client=httpx.AsyncClient(transport=httpx.MockTransport(lambda req: httpx.Response(200, headers={'content-type': 'text/event-stream'}, stream=wire)))) + if traced: + provider, exporter, instrumentor = setup() + try: + stream = await client.messages.create(**PARAMS) + assert (await stream.__anext__()).type == 'message_start' + pending = asyncio.create_task(stream.__anext__()) + await asyncio.wait_for(wire.pending.wait(), 2) + assert await asyncio.wait_for(asyncio.gather(stream.close(), stream.close()), 2) == [None, None] + if traced: + assert not exporter.get_finished_spans() + if outcome == 'cancel': + pending.cancel() + with pytest.raises(asyncio.CancelledError): + await pending + else: + wire.release.set() + with pytest.raises(RuntimeError, match=SECRET): + await pending + if traced: + assert_span(exporter, error=True, full=False) + finally: + if traced: + instrumentor.uninstrument(); provider.shutdown() + await client.close() + asyncio.run(run()) + + +def test_close_exception_preserves_native_error_and_return(): + from traceai_anthropic._stream import _MessagesStream + from traceai_anthropic._with_span import _WithSpan + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import SimpleSpanProcessor + from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + class Native: + def close(self): + raise ValueError('native close failure') + provider = TracerProvider(); exporter = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + stream = _MessagesStream(Native(), _WithSpan(provider.get_tracer(__name__).start_span('close'))) + try: + with pytest.raises(ValueError, match='native close failure'): + stream.close() + assert len(exporter.get_finished_spans()) == 1 + assert exporter.get_finished_spans()[0].status.status_code == StatusCode.ERROR + finally: + provider.shutdown() + + +@pytest.mark.parametrize('traced', [False, True]) +def test_sync_close_does_not_finish_before_pending_error(traced, monkeypatch): + import threading + monkeypatch.setenv('FI_HIDE_ERROR_DETAILS', 'true') + class Wire(httpx.SyncByteStream): + def __init__(self): + self.pending = threading.Event(); self.release = threading.Event() + def __iter__(self): + yield ('event: message_start\ndata: ' + json.dumps(EVENTS[0]) + '\n\n').encode() + self.pending.set() + if not self.release.wait(3): + raise TimeoutError('test release timed out') + raise RuntimeError(SECRET) + def close(self): + pass + wire = Wire() + client = anthropic.Anthropic(api_key='local', http_client=httpx.Client(transport=httpx.MockTransport(lambda req: httpx.Response(200, headers={'content-type': 'text/event-stream'}, stream=wire)))) + if traced: + provider, exporter, instrumentor = setup() + try: + stream = client.messages.create(**PARAMS) + assert next(stream).type == 'message_start' + errors = [] + def advance(): + try: + next(stream) + except BaseException as exc: + errors.append(exc) + thread = threading.Thread(target=advance) + thread.start() + assert wire.pending.wait(2) + assert stream.close() is None + assert stream.close() is None + if traced: + assert not exporter.get_finished_spans() + wire.release.set(); thread.join(2) + assert not thread.is_alive() + assert len(errors) == 1 and isinstance(errors[0], RuntimeError) + assert str(errors[0]) == SECRET + if traced: + assert_span(exporter, error=True, full=False) + finally: + wire.release.set() + if traced: + instrumentor.uninstrument(); provider.shutdown() + client.close() + + +@pytest.mark.parametrize('asynchronous', [False, True]) +def test_direct_advance_context_exception(asynchronous, monkeypatch): + monkeypatch.setenv('FI_HIDE_ERROR_DETAILS', 'true') + async def run(): + client = make_client(EVENTS, True); provider, exporter, instrumentor = setup() + try: + with pytest.raises(ValueError, match=SECRET): + async with await client.messages.create(**PARAMS) as stream: + assert (await stream.__anext__()).type == 'message_start' + raise ValueError(SECRET) + assert_span(exporter, error=True, full=False) + finally: + instrumentor.uninstrument(); await client.close(); provider.shutdown() + if asynchronous: + asyncio.run(run()); return + client = make_client(EVENTS); provider, exporter, instrumentor = setup() + try: + with pytest.raises(ValueError, match=SECRET): + with client.messages.create(**PARAMS) as stream: + assert next(stream).type == 'message_start' + raise ValueError(SECRET) + assert_span(exporter, error=True, full=False) + finally: + instrumentor.uninstrument(); client.close(); provider.shutdown() diff --git a/python/frameworks/anthropic/tests/test_r5_native_lifecycle.py b/python/frameworks/anthropic/tests/test_r5_native_lifecycle.py new file mode 100644 index 000000000..0258133ad --- /dev/null +++ b/python/frameworks/anthropic/tests/test_r5_native_lifecycle.py @@ -0,0 +1,270 @@ +import asyncio, json +import anthropic, pytest +try: + import httpx2 as httpx +except ImportError: + import httpx +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from traceai_anthropic import AnthropicInstrumentor + +START={'type':'message_start','message':{'id':'msg_1','type':'message','role':'assistant','content':[],'model':'local','stop_reason':None,'stop_sequence':None,'usage':{'input_tokens':4,'output_tokens':0}}} +DELTA={'type':'message_delta','delta':{'stop_reason':'end_turn','stop_sequence':None},'usage':{'output_tokens':2}} +PARAMS=dict(model='local',max_tokens=16,messages=[{'role':'user','content':'hi'}],stream=True) +def event(data): return ('event: '+data['type']+'\ndata: '+json.dumps(data)+'\n\n').encode() +def setup(): + p=TracerProvider(); x=InMemorySpanExporter(); p.add_span_processor(SimpleSpanProcessor(x)); i=AnthropicInstrumentor(); i.instrument(tracer_provider=p); return p,x,i + +@pytest.mark.parametrize('traced',[False,True]) +def test_close_during_advance_preserves_late_result_and_usage(traced): + class Wire(httpx.AsyncByteStream): + def __init__(self): self.pending=asyncio.Event(); self.release=asyncio.Event() + async def __aiter__(self): + yield event(START) + self.pending.set(); await self.release.wait() + yield event(DELTA) + async def aclose(self): pass + async def run(): + wire=Wire(); c=anthropic.AsyncAnthropic(api_key='local',http_client=httpx.AsyncClient(transport=httpx.MockTransport(lambda req:httpx.Response(200,headers={'content-type':'text/event-stream'},stream=wire)))) + if traced: p,x,i=setup() + try: + s=await c.messages.create(**PARAMS); assert (await s.__anext__()).type=='message_start' + pending=asyncio.create_task(s.__anext__()); await asyncio.wait_for(wire.pending.wait(),2) + assert await asyncio.wait_for(s.close(),2) is None + if traced: assert len(x.get_finished_spans())==0 + wire.release.set(); result=await asyncio.wait_for(pending,2); assert result.type=='message_delta' and result.usage.output_tokens==2 + if traced: + spans=x.get_finished_spans(); assert len(spans)==1 + assert spans[0].attributes['gen_ai.usage.input_tokens']==4 + assert spans[0].attributes['gen_ai.usage.output_tokens']==2 + assert spans[0].status.status_code.name=='OK' + await s.close(); assert len(x.get_finished_spans())==1 + finally: + wire.release.set() + if traced: i.uninstrument(); p.shutdown() + await c.close() + asyncio.run(run()) + +class NonStringifiableError(Exception): + def __str__(self): raise RuntimeError('str failed') + +@pytest.mark.parametrize('traced',[False,True]) +def test_advance_preserves_exact_exception_even_if_stringification_fails(traced, monkeypatch): + monkeypatch.setenv('FI_HIDE_ERROR_DETAILS','true') + err=NonStringifiableError() + class Wire(httpx.SyncByteStream): + def __iter__(self): + yield event(START) + raise err + def close(self): pass + c=anthropic.Anthropic(api_key='local',http_client=httpx.Client(transport=httpx.MockTransport(lambda req:httpx.Response(200,headers={'content-type':'text/event-stream'},stream=Wire())))) + if traced: p,x,i=setup() + try: + s=c.messages.create(**PARAMS); assert next(s).type=='message_start' + try: next(s) + except BaseException as got: assert got is err, f'expected original exception identity, got {type(got).__name__}' + else: pytest.fail('expected transport error') + finally: + if traced: i.uninstrument(); p.shutdown() + c.close() + +@pytest.mark.parametrize('traced',[False,True]) +def test_close_preserves_exact_exception_even_if_stringification_fails(traced, monkeypatch): + monkeypatch.setenv('FI_HIDE_ERROR_DETAILS','true') + err=NonStringifiableError() + class Wire(httpx.SyncByteStream): + def __iter__(self): yield event(START) + def close(self): raise err + c=anthropic.Anthropic(api_key='local',http_client=httpx.Client(transport=httpx.MockTransport(lambda req:httpx.Response(200,headers={'content-type':'text/event-stream'},stream=Wire())))) + if traced: p,x,i=setup() + try: + s=c.messages.create(**PARAMS) + try: s.close() + except BaseException as got: assert got is err, f'expected original exception identity, got {type(got).__name__}' + else: pytest.fail('expected transport error') + finally: + if traced: i.uninstrument(); p.shutdown() + c.close() + +@pytest.mark.parametrize('traced',[False,True]) +def test_close_is_not_blocked_by_other_thread_export(traced): + import threading + from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult + exporter_entered=threading.Event(); release=threading.Event(); close_done=threading.Event(); reader_errors=[]; close_errors=[] + class BlockingExporter(SpanExporter): + def export(self, spans): + exporter_entered.set() + if not release.wait(5): raise TimeoutError('test exporter timeout') + return SpanExportResult.SUCCESS + def shutdown(self): pass + body=event(START)+event(DELTA) + c=anthropic.Anthropic(api_key='local',http_client=httpx.Client(transport=httpx.MockTransport(lambda req:httpx.Response(200,headers={'content-type':'text/event-stream'},content=body)))) + if traced: + p=TracerProvider(); p.add_span_processor(SimpleSpanProcessor(BlockingExporter())); i=AnthropicInstrumentor(); i.instrument(tracer_provider=p) + try: + s=c.messages.create(**PARAMS) + def read(): + try: list(s) + except BaseException as exc: reader_errors.append(exc) + finally: + if not traced: exporter_entered.set() + def close(): + try: s.close() + except BaseException as exc: close_errors.append(exc) + finally: close_done.set() + t=threading.Thread(target=read); t.start(); assert exporter_entered.wait(2) + closer=threading.Thread(target=close); closer.start() + returned=close_done.wait(0.5) + release.set(); t.join(2); closer.join(2) + assert not t.is_alive() and not closer.is_alive() + assert not reader_errors and not close_errors + assert returned, 'native close blocked behind tracing export' + finally: + release.set() + if traced: i.uninstrument(); p.shutdown() + c.close() + +@pytest.mark.parametrize('hidden', [False, True]) +@pytest.mark.parametrize('traced', [False, True]) +@pytest.mark.parametrize('asynchronous', [False, True]) +def test_context_body_exception_identity_and_status(hidden, traced, asynchronous, monkeypatch): + monkeypatch.setenv('FI_HIDE_ERROR_DETAILS', str(hidden).lower()) + err = NonStringifiableError() + if asynchronous: + async def run(): + c = anthropic.AsyncAnthropic(api_key='local', http_client=httpx.AsyncClient(transport=httpx.MockTransport(lambda req: httpx.Response(200, headers={'content-type':'text/event-stream'}, content=event(START))))) + if traced: p,x,i=setup() + try: + s = await c.messages.create(**PARAMS) + try: + async with s: + raise err + except BaseException as got: assert got is err + else: pytest.fail('expected body error') + if traced: + spans=x.get_finished_spans(); assert len(spans)==1 + assert spans[0].status.status_code.name == 'ERROR' + if hidden: assert 'NonStringifiableError' not in str(spans[0].status.description) + finally: + if traced: i.uninstrument(); p.shutdown() + await c.close() + asyncio.run(run()); return + c = anthropic.Anthropic(api_key='local', http_client=httpx.Client(transport=httpx.MockTransport(lambda req: httpx.Response(200, headers={'content-type':'text/event-stream'}, content=event(START))))) + if traced: p,x,i=setup() + try: + s=c.messages.create(**PARAMS) + try: + with s: raise err + except BaseException as got: assert got is err + else: pytest.fail('expected body error') + if traced: + spans=x.get_finished_spans(); assert len(spans)==1 + assert spans[0].status.status_code.name == 'ERROR' + if hidden: assert 'NonStringifiableError' not in str(spans[0].status.description) + finally: + if traced: i.uninstrument(); p.shutdown() + c.close() + +@pytest.mark.parametrize('hidden', [False, True]) +@pytest.mark.parametrize('traced', [False, True]) +def test_request_error_identity(hidden, traced, monkeypatch): + monkeypatch.setenv('FI_HIDE_ERROR_DETAILS', str(hidden).lower()) + err=NonStringifiableError() + def transport(req): raise err + c=anthropic.Anthropic(api_key='local', max_retries=0, http_client=httpx.Client(transport=httpx.MockTransport(transport))) + if traced: p,x,i=setup() + try: + try: c.messages.create(**PARAMS) + except BaseException as got: + # The SDK wraps transport exceptions; compare its cause identity. + assert got.__cause__ is err + else: pytest.fail('expected transport error') + if traced: + spans=x.get_finished_spans(); assert len(spans)==1 + assert spans[0].status.status_code.name == 'ERROR' + finally: + if traced: i.uninstrument(); p.shutdown() + c.close() + +@pytest.mark.parametrize('hidden', [False, True]) +@pytest.mark.parametrize('where', ['advance', 'close']) +@pytest.mark.parametrize('traced', [False, True]) +def test_async_transport_exception_identity(hidden, where, traced, monkeypatch): + monkeypatch.setenv('FI_HIDE_ERROR_DETAILS', str(hidden).lower()) + err = NonStringifiableError() + class Wire(httpx.AsyncByteStream): + async def __aiter__(self): + yield event(START) + if where == 'advance': raise err + async def aclose(self): + if where == 'close': raise err + async def run(): + c=anthropic.AsyncAnthropic(api_key='local', http_client=httpx.AsyncClient(transport=httpx.MockTransport(lambda req:httpx.Response(200,headers={'content-type':'text/event-stream'},stream=Wire())))) + if traced: p,x,i=setup() + try: + s=await c.messages.create(**PARAMS) + if where == 'advance': assert (await s.__anext__()).type == 'message_start' + try: + if where == 'advance': await s.__anext__() + else: await s.close() + except BaseException as got: assert got is err + else: pytest.fail('expected transport error') + if traced: + spans=x.get_finished_spans(); assert len(spans)==1 + assert spans[0].status.status_code.name=='ERROR' + if hidden: assert 'NonStringifiableError' not in str(spans[0].status.description) + finally: + if traced: i.uninstrument(); p.shutdown() + await c.close() + asyncio.run(run()) + +@pytest.mark.parametrize('hidden', [False, True]) +def test_ordinary_error_diagnostics_and_redaction(hidden, monkeypatch): + monkeypatch.setenv('FI_HIDE_ERROR_DETAILS', str(hidden).lower()) + secret='private-error-diagnostic-r5' + err=ValueError(secret) + class Wire(httpx.SyncByteStream): + def __iter__(self): + yield event(START) + raise err + def close(self): pass + c=anthropic.Anthropic(api_key='local',http_client=httpx.Client(transport=httpx.MockTransport(lambda req:httpx.Response(200,headers={'content-type':'text/event-stream'},stream=Wire())))) + p,x,i=setup() + try: + s=c.messages.create(**PARAMS); assert next(s).type=='message_start' + with pytest.raises(ValueError) as caught: next(s) + assert caught.value is err + spans=x.get_finished_spans(); assert len(spans)==1 + span=spans[0]; assert span.status.status_code.name=='ERROR' + data=str(span.status.description)+str(span.events)+str(span.attributes) + assert (secret not in data) if hidden else (secret in data) + finally: + i.uninstrument(); p.shutdown(); c.close() + +def test_failing_exporter_does_not_replace_native_error(): + from opentelemetry import context as context_api + from opentelemetry.sdk.trace.export import SpanExporter + err = NonStringifiableError() + class Wire(httpx.SyncByteStream): + def __iter__(self): + yield event(START) + raise err + def close(self): pass + class BrokenExporter(SpanExporter): + def export(self, spans): raise KeyboardInterrupt('export callback') + def shutdown(self): pass + p=TracerProvider(); p.add_span_processor(SimpleSpanProcessor(BrokenExporter())) + i=AnthropicInstrumentor(); i.instrument(tracer_provider=p) + c=anthropic.Anthropic(api_key='local',http_client=httpx.Client(transport=httpx.MockTransport(lambda req:httpx.Response(200,headers={'content-type':'text/event-stream'},stream=Wire())))) + # Older OTel processors leave instrumentation suppressed if an exporter + # raises BaseException. Isolate this deliberate failure from later tests. + token = context_api.attach(context_api.get_current()) + try: + s=c.messages.create(**PARAMS); assert next(s).type=='message_start' + try: next(s) + except BaseException as got: assert got is err + else: pytest.fail('expected native error') + finally: + context_api.detach(token) + i.uninstrument(); c.close(); p.shutdown() diff --git a/python/frameworks/anthropic/tests/test_r6_telemetry_isolation.py b/python/frameworks/anthropic/tests/test_r6_telemetry_isolation.py new file mode 100644 index 000000000..b6adf5cbd --- /dev/null +++ b/python/frameworks/anthropic/tests/test_r6_telemetry_isolation.py @@ -0,0 +1,90 @@ +import json, threading +import anthropic, pytest +try: + import httpx2 as httpx +except ImportError: + import httpx +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from traceai_anthropic import AnthropicInstrumentor + +START={'type':'message_start','message':{'id':'msg_1','type':'message','role':'assistant','content':[],'model':'local','stop_reason':None,'stop_sequence':None,'usage':{'input_tokens':4,'output_tokens':0}}} +BLOCK={'type':'content_block_start','index':0,'content_block':{'type':'text','text':''}} +TEXT={'type':'content_block_delta','index':0,'delta':{'type':'text_delta','text':'hello'}} +PARAMS=dict(model='local',max_tokens=16,messages=[{'role':'user','content':'hi'}],stream=True) +def event(data): return ('event: '+data['type']+'\ndata: '+json.dumps(data)+'\n\n').encode() +def client(body): return anthropic.Anthropic(api_key='local',http_client=httpx.Client(transport=httpx.MockTransport(lambda req:httpx.Response(200,headers={'content-type':'text/event-stream'},content=body)))) +def setup(): + p=TracerProvider(); x=InMemorySpanExporter(); p.add_span_processor(SimpleSpanProcessor(x)); i=AnthropicInstrumentor(); i.instrument(tracer_provider=p); return p,x,i + +@pytest.mark.parametrize('traced',[False,True]) +def test_event_with_null_usage_reaches_caller_and_rest_is_traced(traced): + # The SDK accepts a message_start whose usage is null; telemetry must not replace it. + start=json.loads(json.dumps(START)); start['message']['usage']=None + c=client(event(start)+event(BLOCK)+event(TEXT)) + if traced: p,x,i=setup() + try: + s=c.messages.create(**PARAMS) + first=next(s); assert first.type=='message_start' and first.message.usage is None + assert [e.type for e in s]==['content_block_start','content_block_delta'] + assert s.close() is None + if traced: + spans=x.get_finished_spans(); assert len(spans)==1 + assert spans[0].status.status_code.name!='ERROR' + assert spans[0].attributes['gen_ai.output.messages.0.message.role']=='assistant' + assert spans[0].attributes['gen_ai.output.messages.0.message.content']=='hello' + assert spans[0].attributes['gen_ai.usage.input_tokens']==0 + finally: + if traced: i.uninstrument(); p.shutdown() + c.close() + +def test_unreadable_event_keeps_native_item_and_later_telemetry(monkeypatch): + from anthropic.types.raw_message_start_event import RawMessageStartEvent + class Broken(Exception): pass + def role(self): raise Broken() + c=client(event(START)+event(BLOCK)+event(TEXT)) + p,x,i=setup() + try: + s=c.messages.create(**PARAMS) + monkeypatch.setattr(RawMessageStartEvent,'message',property(role),raising=False) + first=next(s) + monkeypatch.undo() + assert first.type=='message_start' + assert [e.type for e in s]==['content_block_start','content_block_delta'] + assert s.close() is None + spans=x.get_finished_spans(); assert len(spans)==1 + assert spans[0].status.status_code.name!='ERROR' + assert 'hello' in spans[0].attributes['output.value'] + finally: + i.uninstrument(); p.shutdown(); c.close() + +@pytest.mark.parametrize('traced',[False,True]) +def test_sdk_serializer_runs_outside_lifecycle_lock(monkeypatch,traced): + try: + from anthropic.types import Completion + except ImportError: + pytest.skip('native SDK removed completions') + c=client(b'event: completion\ndata: {"id":"c1","type":"completion","model":"local","completion":"hello","stop_reason":"stop_sequence"}\n\n') + if not hasattr(c,'completions'): + c.close(); pytest.skip('native SDK removed completions') + original=Completion.model_dump; acquired=[]; holder={} + def model_dump(self,*args,**kwargs): + def probe(): + got=holder['stream']._self_lock.acquire(timeout=0.5); acquired.append(got) + if got: holder['stream']._self_lock.release() + t=threading.Thread(target=probe); t.start(); t.join(2) + return original(self,*args,**kwargs) + monkeypatch.setattr(Completion,'model_dump',model_dump) + if traced: p,x,i=setup() + try: + s=c.completions.create(model='local',prompt='Hi',max_tokens_to_sample=8,stream=True); holder['stream']=s + assert next(s).completion=='hello' + assert acquired==([True] if traced else []) + assert s.close() is None + if traced: + spans=x.get_finished_spans(); assert len(spans)==1 + assert 'hello' in spans[0].attributes['output.value'] + finally: + if traced: i.uninstrument(); p.shutdown() + c.close() diff --git a/python/frameworks/anthropic/tests/test_r7_merge_order.py b/python/frameworks/anthropic/tests/test_r7_merge_order.py new file mode 100644 index 000000000..e7f34ba11 --- /dev/null +++ b/python/frameworks/anthropic/tests/test_r7_merge_order.py @@ -0,0 +1,57 @@ +import threading +from traceai_anthropic._stream import _Stream + + +class Native: + """A thread-safe native iterator, so two threads can advance one stream.""" + def __init__(self, items): self.items = iter(items); self.lock = threading.Lock() + def __next__(self): + with self.lock: return next(self.items) + def close(self): return None + + +def lock_free(stream, seen): + # Record whether another thread can take the lifecycle lock right now. + def probe(): + got = stream._self_lock.acquire(timeout=0.5); seen.append(got) + if got: stream._self_lock.release() + t = threading.Thread(target=probe); t.start(); t.join(2) + + +def test_concurrent_advances_merge_in_arrival_order(): + ready, release = threading.Event(), threading.Event() + class Slow: + def model_dump(self, **kwargs): + ready.set(); release.wait(3); return {'completion': 'A'} + class Fast: + def model_dump(self, **kwargs): return {'completion': 'B'} + s = _Stream(Native([Slow(), Fast()]), None) + first = threading.Thread(target=lambda: next(s)); first.start(); assert ready.wait(2) + second = threading.Thread(target=lambda: next(s)); second.start(); second.join(2) + assert not second.is_alive() # the later item finished extracting first + release.set(); first.join(3) + assert s._response_accumulator._result() == {'completion': 'AB'} + assert not s._self_pending and not s._self_extracting + + +def test_serializer_result_runs_no_foreign_code_under_lock(): + seen = []; holder = {} + class Foreign: + def __deepcopy__(self, memo): lock_free(holder['s'], seen); return 'copied' + def __str__(self): lock_free(holder['s'], seen); return 'foreign' + class Result(dict): + def get(self, *a): lock_free(holder['s'], seen); return super().get(*a) + def items(self): lock_free(holder['s'], seen); return super().items() + def keys(self): lock_free(holder['s'], seen); return super().keys() + def __iter__(self): lock_free(holder['s'], seen); return super().__iter__() + def __getitem__(self, k): lock_free(holder['s'], seen); return super().__getitem__(k) + class Text(str): + def __add__(self, other): lock_free(holder['s'], seen); return str(self) + other + class Item: + def model_dump(self, **kwargs): return Result(completion=Text('hello'), extra=Foreign()) + s = _Stream(Native([Item()]), None); holder['s'] = s + next(s) + assert seen and all(seen), seen + result = s._response_accumulator._result() + assert result['completion'] == 'hello' and type(result['completion']) is str + assert result['extra'] == 'foreign' diff --git a/python/frameworks/anthropic/tests/test_r8_ticket_recovery.py b/python/frameworks/anthropic/tests/test_r8_ticket_recovery.py new file mode 100644 index 000000000..fdf864240 --- /dev/null +++ b/python/frameworks/anthropic/tests/test_r8_ticket_recovery.py @@ -0,0 +1,54 @@ +import threading +import pytest +from traceai_anthropic import _stream +from traceai_anthropic._stream import _Stream + + +class Native: + """A thread-safe native iterator, so two threads can advance one stream.""" + def __init__(self, items): self.items = iter(items); self.lock = threading.Lock() + def __next__(self): + with self.lock: return next(self.items) + def close(self): return None + + +class Item: + def __init__(self, text='x', ready=None, release=None): self.text = text; self.ready = ready; self.release = release + def model_dump(self, **kwargs): + if self.ready: + self.ready.set(); self.release.wait(5) + return {'completion': self.text} + + +MAX_PENDING = getattr(_stream, '_MAX_PENDING', 1024) + + +def test_interrupt_during_merge_leaves_nothing_stranded(monkeypatch): + s = _Stream(Native([Item('a')] + [Item() for _ in range(1000)]), None) + merge = _stream._ResponseAccumulator.merge + calls = [] + def interrupted(self, values): + calls.append(values) + if len(calls) == 1: + raise KeyboardInterrupt('async interrupt while merging') + return merge(self, values) + monkeypatch.setattr(_stream._ResponseAccumulator, 'merge', interrupted) + with pytest.raises(KeyboardInterrupt): + next(s) + for _ in range(1000): + next(s) + assert not s._self_pending and not getattr(s, '_self_extracting', ()) and s._self_active == 0 + assert s._response_accumulator._result() == {'completion': 'x' * 1000} + + +def test_stalled_extraction_keeps_the_queue_bounded(): + ready, release = threading.Event(), threading.Event() + s = _Stream(Native([Item('slow', ready, release)] + [Item() for _ in range(2 * MAX_PENDING)]), None) + slow = threading.Thread(target=lambda: next(s)); slow.start(); assert ready.wait(2) + for _ in range(2 * MAX_PENDING): + next(s) + assert len(s._self_pending) <= MAX_PENDING + 1 + release.set(); slow.join(5); assert not slow.is_alive() + assert not s._self_pending and not getattr(s, '_self_extracting', ()) and not getattr(s, '_self_skipped', ()) + # The stalled item lost its place, so its telemetry is dropped; every later item is kept. + assert s._response_accumulator._result() == {'completion': 'x' * (2 * MAX_PENDING)} diff --git a/python/frameworks/anthropic/tests/test_sdk_native_compat.py b/python/frameworks/anthropic/tests/test_sdk_native_compat.py new file mode 100644 index 000000000..cca377b9e --- /dev/null +++ b/python/frameworks/anthropic/tests/test_sdk_native_compat.py @@ -0,0 +1,108 @@ +"""Offline tests exercise the native SDK over its real HTTP transport interface.""" +import asyncio +import json + +import anthropic +import pytest +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from traceai_anthropic import AnthropicInstrumentor + +try: + import httpx2 as httpx +except ImportError: + import httpx + + +MESSAGE = { + "id": "msg_local", "type": "message", "role": "assistant", + "content": [{"type": "text", "text": "offline result"}], + "model": "local-test", "stop_reason": "end_turn", "stop_sequence": None, + "usage": {"input_tokens": 4, "output_tokens": 2}, +} + + +def fixture_provider(): + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + return provider, exporter + + +def transport_handler(request): + assert json.loads(request.content)["model"] == "local-test" + return httpx.Response(200, json=MESSAGE) + + +def test_real_sync_sdk_matches_untraced_and_exports_one_span(): + provider, exporter = fixture_provider() + client = anthropic.Anthropic(api_key="local-test", http_client=httpx.Client(transport=httpx.MockTransport(transport_handler))) + params = dict(model="local-test", max_tokens=16, messages=[{"role": "user", "content": "hello"}]) + expected = client.messages.create(**params).model_dump() + instrumentor = AnthropicInstrumentor() + instrumentor.instrument(tracer_provider=provider) + try: + actual = client.messages.create(**params).model_dump() + assert actual == expected + spans = exporter.get_finished_spans() + assert len(spans) == 1 + finally: + instrumentor.uninstrument() + client.close() + provider.shutdown() + + +def test_real_async_sdk_matches_untraced_and_exports_one_span(): + async def run(): + provider, exporter = fixture_provider() + client = anthropic.AsyncAnthropic(api_key="local-test", http_client=httpx.AsyncClient(transport=httpx.MockTransport(transport_handler))) + params = dict(model="local-test", max_tokens=16, messages=[{"role": "user", "content": "hello"}]) + expected = (await client.messages.create(**params)).model_dump() + instrumentor = AnthropicInstrumentor() + instrumentor.instrument(tracer_provider=provider) + try: + actual = (await client.messages.create(**params)).model_dump() + assert actual == expected + assert len(exporter.get_finished_spans()) == 1 + finally: + instrumentor.uninstrument() + await client.close() + provider.shutdown() + asyncio.run(run()) + + +def test_real_sdk_error_is_not_swallowed(): + def fail(request): + return httpx.Response(400, json={"type": "error", "error": {"type": "invalid_request_error", "message": "offline error"}}) + provider, exporter = fixture_provider() + client = anthropic.Anthropic(api_key="local-test", max_retries=0, http_client=httpx.Client(transport=httpx.MockTransport(fail))) + instrumentor = AnthropicInstrumentor() + instrumentor.instrument(tracer_provider=provider) + try: + with pytest.raises(anthropic.BadRequestError, match="offline error"): + client.messages.create(model="local-test", max_tokens=16, messages=[{"role": "user", "content": "hello"}]) + assert len(exporter.get_finished_spans()) == 1 + finally: + instrumentor.uninstrument() + client.close() + provider.shutdown() + + +def test_real_async_sdk_error_is_not_swallowed(): + async def run(): + def fail(request): + return httpx.Response(400, json={"type": "error", "error": {"type": "invalid_request_error", "message": "offline error"}}) + provider, exporter = fixture_provider() + client = anthropic.AsyncAnthropic(api_key="local-test", max_retries=0, http_client=httpx.AsyncClient(transport=httpx.MockTransport(fail))) + instrumentor = AnthropicInstrumentor() + instrumentor.instrument(tracer_provider=provider) + try: + with pytest.raises(anthropic.BadRequestError, match="offline error"): + await client.messages.create(model="local-test", max_tokens=16, messages=[{"role": "user", "content": "hello"}]) + assert len(exporter.get_finished_spans()) == 1 + finally: + instrumentor.uninstrument() + await client.close() + provider.shutdown() + asyncio.run(run()) diff --git a/python/frameworks/anthropic/traceai_anthropic/__init__.py b/python/frameworks/anthropic/traceai_anthropic/__init__.py index 7e1c13cca..e60118782 100644 --- a/python/frameworks/anthropic/traceai_anthropic/__init__.py +++ b/python/frameworks/anthropic/traceai_anthropic/__init__.py @@ -30,6 +30,7 @@ class AnthropicInstrumentor(BaseInstrumentor): # type: ignore[misc] "_original_completions_create", "_original_async_completions_create", "_original_messages_create", + "_original_messages_count_tokens", "_original_async_messages_create", "_original_protect", "_instruments", @@ -40,8 +41,16 @@ def instrumentation_dependencies(self) -> Collection[str]: return _instruments def _instrument(self, **kwargs: Any) -> None: - from anthropic.resources.completions import AsyncCompletions, Completions from anthropic.resources.messages import AsyncMessages, Messages + + try: + # anthropic>=1.0 removed the legacy Completions API (`/v1/complete`) + # entirely - see anthropic-sdk-python's MIGRATION.md, "Removed: the + # legacy Text Completions API". Messages is unaffected, so degrade to + # instrumenting only Messages instead of failing to instrument anything. + from anthropic.resources.completions import AsyncCompletions, Completions + except ImportError: + AsyncCompletions = Completions = None try: from fi.evals import Protect except ImportError: @@ -60,19 +69,25 @@ def _instrument(self, **kwargs: Any) -> None: config=config, ) - self._original_completions_create = Completions.create - wrap_function_wrapper( - module="anthropic.resources.completions", - name="Completions.create", - wrapper=_CompletionsWrapper(tracer=self._tracer), - ) + if Completions is not None: + self._original_completions_create = Completions.create + wrap_function_wrapper( + module="anthropic.resources.completions", + name="Completions.create", + wrapper=_CompletionsWrapper(tracer=self._tracer), + ) + else: + self._original_completions_create = None - self._original_async_completions_create = AsyncCompletions.create - wrap_function_wrapper( - module="anthropic.resources.completions", - name="AsyncCompletions.create", - wrapper=_AsyncCompletionsWrapper(tracer=self._tracer), - ) + if AsyncCompletions is not None: + self._original_async_completions_create = AsyncCompletions.create + wrap_function_wrapper( + module="anthropic.resources.completions", + name="AsyncCompletions.create", + wrapper=_AsyncCompletionsWrapper(tracer=self._tracer), + ) + else: + self._original_async_completions_create = None self._original_messages_create = Messages.create wrap_function_wrapper( @@ -88,6 +103,7 @@ def _instrument(self, **kwargs: Any) -> None: wrapper=_AsyncMessagesWrapper(tracer=self._tracer), ) + self._original_messages_count_tokens = Messages.count_tokens wrap_function_wrapper( module="anthropic.resources.messages", name="Messages.count_tokens", @@ -105,21 +121,30 @@ def _instrument(self, **kwargs: Any) -> None: self._original_protect = None def _uninstrument(self, **kwargs: Any) -> None: - from anthropic.resources.completions import AsyncCompletions, Completions from anthropic.resources.messages import AsyncMessages, Messages + + try: + from anthropic.resources.completions import AsyncCompletions, Completions + except ImportError: + AsyncCompletions = Completions = None try: from fi.evals import Protect except ImportError: logger.warning("ai-evaluation is not installed, please install it to trace protect") Protect = None - if self._original_completions_create is not None: + if Completions is not None and self._original_completions_create is not None: Completions.create = self._original_completions_create # type: ignore[method-assign] - if self._original_async_completions_create is not None: + if ( + AsyncCompletions is not None + and self._original_async_completions_create is not None + ): AsyncCompletions.create = self._original_async_completions_create # type: ignore[method-assign] if self._original_messages_create is not None: Messages.create = self._original_messages_create # type: ignore[method-assign] + if self._original_messages_count_tokens is not None: + Messages.count_tokens = self._original_messages_count_tokens # type: ignore[method-assign] if self._original_async_messages_create is not None: AsyncMessages.create = self._original_async_messages_create # type: ignore[method-assign] diff --git a/python/frameworks/anthropic/traceai_anthropic/_stream.py b/python/frameworks/anthropic/traceai_anthropic/_stream.py index d73e58f88..98f3fc212 100644 --- a/python/frameworks/anthropic/traceai_anthropic/_stream.py +++ b/python/frameworks/anthropic/traceai_anthropic/_stream.py @@ -1,5 +1,7 @@ from collections import defaultdict from copy import deepcopy +import inspect +import threading from typing import ( TYPE_CHECKING, Any, @@ -13,7 +15,6 @@ Mapping, Optional, Tuple, - Union, ) from fi_instrumentation import safe_json_dumps @@ -27,6 +28,7 @@ from opentelemetry.util.types import AttributeValue from traceai_anthropic._utils import ( _as_output_attributes, + _error_status, _finish_tracing, _ValueAndType, ) @@ -36,12 +38,224 @@ if TYPE_CHECKING: from anthropic import Stream from anthropic.types import Completion, RawMessageStreamEvent - from anthropic.types.raw_content_block_delta_event import RawContentBlockDeltaEvent - from anthropic.types.text_block import TextBlock - from anthropic.types.tool_use_block import ToolUseBlock -class _Stream(ObjectProxy): # type: ignore +# Settled items held back by a stalled earlier extraction before ordering is +# given up for that extraction. +_MAX_PENDING = 1024 + + +class _StreamLifecycle(ObjectProxy): + """Keep the tracing lifetime coupled to the SDK stream's native lifetime.""" + + def __init__(self, stream): + super().__init__(stream) + self._self_lock = threading.RLock() + self._self_active = 0 + self._self_terminal = False + self._self_error = None + self._self_finalizing = False + # Items are extracted outside the lock, possibly concurrently; tickets + # taken on arrival keep their merges in the order the stream produced them. + # A settled item waits only for earlier tickets still being extracted. + self._self_next_ticket = 0 + self._self_extracting = set() + self._self_pending = {} + self._self_skipped = set() + + def _begin_operation(self): + with self._self_lock: + self._self_active += 1 + + def _end_operation(self, *, terminal=False, error=None, item=None, has_item=False): + ticket = values = None + try: + if has_item: + with self._self_lock: + ticket = self._self_next_ticket + self._self_next_ticket += 1 + self._self_extracting.add(ticket) + values = self._extract(item) + finally: + self._settle(terminal, error, ticket, values) + + def _extract(self, item): + # SDK model methods and the values they return are external code, so read + # and copy the item before taking the lifecycle lock. A telemetry failure + # must not replace the native item. + try: + return _plain(self._response_accumulator.extract(item)) + except Exception: + return None + + def _merge(self, values): + # Once finalization is claimed, a subsequent native operation + # must not mutate the accumulator being read by the exporter. + if values is None or self._self_finalizing: + return + try: + self._response_accumulator.merge(values) + except Exception: + # Losing one item's telemetry must not change the native operation. + pass + + def _drain(self, everything=False): + # Merge settled items in ticket order, up to the earliest ticket still + # being extracted. Recomputed from state on every call, so an interrupt + # part-way through leaves nothing stranded. + floor = None + if not everything and self._self_extracting: + floor = min(self._self_extracting) + for ticket in sorted(self._self_pending): + if floor is not None and ticket > floor: + break + self._merge(self._self_pending.pop(ticket)) + + def _settle(self, terminal, error, ticket, values): + finalize = False + final_error = None + try: + with self._self_lock: + try: + if ticket is not None: + self._self_extracting.discard(ticket) + if ticket in self._self_skipped: + # Later items were already merged without it. + self._self_skipped.discard(ticket) + else: + self._self_pending[ticket] = values + if len(self._self_pending) > _MAX_PENDING: + # An earlier extraction is stalled; stop holding later + # items for it so the queue stays bounded. + latest = max(self._self_pending) + stalled = {t for t in self._self_extracting if t < latest} + self._self_skipped |= stalled + self._self_extracting -= stalled + self._drain() + finally: + if terminal: + self._self_terminal = True + if error is not None and self._self_error is None: + self._self_error = error + self._self_active -= 1 + if (self._self_terminal and not self._self_active + and not self._self_finalizing): + # Every operation has settled; merge whatever is left. + self._drain(everything=True) + self._self_finalizing = True + finalize = True + final_error = self._self_error + finally: + if finalize: + # Span processors/exporters are arbitrary synchronous user code. + # Claim exactly once under the lock, then release it before calling + # them so another thread can always reach the native close. + try: + if final_error is not None: + self._finish_error(final_error) + else: + self._finish_tracing(trace_api.Status(trace_api.StatusCode.OK)) + except BaseException: + # Telemetry must not replace the native operation's outcome. + pass + + def _advance(self, iterator): + self._begin_operation() + try: + item = next(iterator) + except StopIteration: + self._end_operation(terminal=True) + raise + except BaseException as exc: + self._end_operation(terminal=True, error=exc) + raise + self._end_operation(item=item, has_item=True) + return item + + async def _advance_async(self, iterator): + self._begin_operation() + try: + item = await iterator.__anext__() + except StopAsyncIteration: + self._end_operation(terminal=True) + raise + except BaseException as exc: + self._end_operation(terminal=True, error=exc) + raise + self._end_operation(item=item, has_item=True) + return item + + def close(self): + # Calling an async native close merely creates a coroutine; ownership + # starts when it is awaited, just as the underlying close does. + if inspect.iscoroutinefunction(self.__wrapped__.close): + async def close_async(): + self._begin_operation() + try: + result = await self.__wrapped__.close() + except BaseException as exc: + self._end_operation(terminal=True, error=exc) + raise + self._end_operation(terminal=True) + return result + return close_async() + self._begin_operation() + try: + result = self.__wrapped__.close() + except BaseException as exc: + self._end_operation(terminal=True, error=exc) + raise + if inspect.isawaitable(result): + async def wait_for_close(): + try: + value = await result + except BaseException as exc: + self._end_operation(terminal=True, error=exc) + raise + self._end_operation(terminal=True) + return value + return wait_for_close() + self._end_operation(terminal=True) + return result + + def __enter__(self): + self.__wrapped__.__enter__() + return self + + def __exit__(self, exc_type, exc, tb): + self._begin_operation() + try: + result = self.__wrapped__.__exit__(exc_type, exc, tb) + except BaseException as error: + self._end_operation(terminal=True, error=error) + raise + self._end_operation(terminal=True, error=exc) + return result + + async def __aenter__(self): + await self.__wrapped__.__aenter__() + return self + + async def __aexit__(self, exc_type, exc, tb): + self._begin_operation() + try: + result = await self.__wrapped__.__aexit__(exc_type, exc, tb) + except BaseException as error: + self._end_operation(terminal=True, error=error) + raise + self._end_operation(terminal=True, error=exc) + return result + + def _finish_error(self, exception): + status = _error_status(exception) + try: + self._with_span.record_exception(exception) + except BaseException: + pass + self._finish_tracing(status) + + +class _Stream(_StreamLifecycle): # type: ignore __slots__ = ( "_response_accumulator", "_with_span", @@ -57,42 +271,30 @@ def __init__( self._response_accumulator = _ResponseAccumulator() self._with_span = with_span + def __next__(self): + return self._advance(self.__wrapped__) + + async def __anext__(self): + return await self._advance_async(self.__wrapped__) + def __iter__(self) -> Iterator["Completion"]: - try: - for item in self.__wrapped__: - self._response_accumulator.process_chunk(item) - yield item - except Exception as exception: - status = trace_api.Status( - status_code=trace_api.StatusCode.ERROR, - description=f"{type(exception).__name__}: {exception}", - ) - self._with_span.record_exception(exception) - self._finish_tracing(status=status) - raise - status = trace_api.Status( - status_code=trace_api.StatusCode.OK, - ) - self._finish_tracing(status=status) + iterator = iter(self.__wrapped__) + while True: + try: + item = self._advance(iterator) + except StopIteration: + return + # Closing this iterator does not close the native stream. + yield item async def __aiter__(self) -> AsyncIterator["Completion"]: - try: - async for item in self.__wrapped__: - self._response_accumulator.process_chunk(item) - yield item - except Exception as exception: - status = trace_api.Status( - status_code=trace_api.StatusCode.ERROR, - description=f"{type(exception).__name__}: {exception}", - ) - self._with_span.record_exception(exception) - self._finish_tracing(status=status) - raise - # completed without exception - status = trace_api.Status( - status_code=trace_api.StatusCode.OK, - ) - self._finish_tracing(status=status) + iterator = self.__wrapped__.__aiter__() + while True: + try: + item = await self._advance_async(iterator) + except StopAsyncIteration: + return + yield item def _finish_tracing( self, @@ -121,9 +323,11 @@ def __init__(self) -> None: stop_reason=_SimpleStringReplace(), ) - def process_chunk(self, chunk: "Completion") -> None: + def extract(self, chunk: "Completion") -> Dict[str, Any]: + return chunk.model_dump(exclude_unset=True, warnings=False) + + def merge(self, values: Mapping[str, Any]) -> None: self._is_null = False - values = chunk.model_dump(exclude_unset=True, warnings=False) self._values += values def _result(self) -> Optional[Dict[str, Any]]: @@ -159,13 +363,21 @@ def get_extra_attributes(self) -> Iterator[Tuple[str, AttributeValue]]: yield SpanAttributes.GEN_AI_OUTPUT_MESSAGES, completion -class _MessagesStream(ObjectProxy): # type: ignore +class _MessagesStream(_StreamLifecycle): # type: ignore __slots__ = ( "_response_accumulator", "_with_span", "_is_finished", ) + def __del__(self): + # Preserve the prior best-effort cleanup for abandoned message streams. + try: + self._begin_operation() + self._end_operation(terminal=True) + except Exception: + pass + def __init__( self, stream: "Stream[RawMessageStreamEvent]", @@ -175,55 +387,30 @@ def __init__( self._response_accumulator = _MessageResponseAccumulator() self._with_span = with_span - def __iter__(self) -> Iterator["RawMessageStreamEvent"]: - try: - for item in self.__wrapped__: - self._response_accumulator.process_chunk(item) - yield item - except Exception as exception: - status = trace_api.Status( - status_code=trace_api.StatusCode.ERROR, - description=f"{type(exception).__name__}: {exception}", - ) - self._with_span.record_exception(exception) - self._finish_tracing(status=status) - raise - status = trace_api.Status( - status_code=trace_api.StatusCode.OK, - ) - self._finish_tracing(status=status) + def __next__(self): + return self._advance(self.__wrapped__) - async def __aiter__(self) -> AsyncIterator["RawMessageStreamEvent"]: - try: - async for item in self.__wrapped__: - self._response_accumulator.process_chunk(item) - yield item - except Exception as exception: - status = trace_api.Status( - status_code=trace_api.StatusCode.ERROR, - description=f"{type(exception).__name__}: {exception}", - ) - self._with_span.record_exception(exception) - self._finish_tracing(status=status) - raise - # completed without exception - status = trace_api.Status( - status_code=trace_api.StatusCode.OK, - ) - self._finish_tracing(status=status) + async def __anext__(self): + return await self._advance_async(self.__wrapped__) - async def aclose(self): - await self.__wrapped__.aclose() - if not self._is_finished: - self._is_finished = True - status = trace_api.Status(status_code=trace_api.StatusCode.OK) - self._finish_tracing(status=status) + def __iter__(self) -> Iterator["RawMessageStreamEvent"]: + iterator = iter(self.__wrapped__) + while True: + try: + item = self._advance(iterator) + except StopIteration: + return + # Closing this iterator does not close the native stream. + yield item - def __del__(self): - if not getattr(self, "_is_finished", False): - self._is_finished = True - status = trace_api.Status(status_code=trace_api.StatusCode.OK) - self._finish_tracing(status=status) + async def __aiter__(self) -> AsyncIterator["RawMessageStreamEvent"]: + iterator = self.__wrapped__.__aiter__() + while True: + try: + item = await self._advance_async(iterator) + except StopAsyncIteration: + return + yield item def _finish_tracing( self, @@ -238,19 +425,44 @@ def _finish_tracing( ) +def _token_count(value: Any) -> Optional[str]: + return None if value is None else str(value) + + +def _plain(value: Any) -> Any: + """Copy extracted values into built-in types, so merging runs no foreign code.""" + if value is None or type(value) in (bool, int, float, str): + return value + if isinstance(value, str): + return str.__str__(value) + if isinstance(value, int): + return int.__int__(value) + if isinstance(value, float): + return float.__float__(value) + if isinstance(value, Mapping): + return {_plain(key): _plain(item) for key, item in value.items()} + if isinstance(value, tuple): + return tuple(_plain(item) for item in value) + if isinstance(value, list): + return [_plain(item) for item in value] + return str(value) + + class _MessageResponseAccumulator: __slots__ = ( "_is_null", "_values", "_current_message_idx", - "_current_content_block_type", + "_current_block", ) def __init__(self) -> None: self._is_null = True self._current_message_idx = -1 - self._current_content_block_type: Union["TextBlock", "ToolUseBlock", None] = ( - None + # (kind, type, tool name) of the open content block; kind is "text", + # "tool_use" or None for blocks whose deltas are not recorded. + self._current_block: Tuple[Optional[str], Optional[str], Optional[str]] = ( + None, None, None ) self._values = _ValuesAccumulator( messages=_IndexedAccumulator( @@ -271,7 +483,8 @@ def __init__(self) -> None: ), ) - def process_chunk(self, chunk: "RawContentBlockDeltaEvent") -> None: + def extract(self, chunk: "RawMessageStreamEvent") -> Tuple[Any, ...]: + """Read an SDK event into plain values; merge applies them in order.""" from anthropic.types.raw_content_block_delta_event import ( RawContentBlockDeltaEvent, ) @@ -283,54 +496,75 @@ def process_chunk(self, chunk: "RawContentBlockDeltaEvent") -> None: from anthropic.types.text_block import TextBlock from anthropic.types.tool_use_block import ToolUseBlock - self._is_null = False if isinstance(chunk, RawMessageStartEvent): + # The SDK accepts events whose usage is null; keep the rest of the event. + usage = getattr(chunk.message, "usage", None) + return ("message_start", chunk.message.role, + _token_count(getattr(usage, "input_tokens", None))) + if isinstance(chunk, RawContentBlockStartEvent): + block = chunk.content_block + if isinstance(block, TextBlock): + return ("block_start", "text", block.type, None) + if isinstance(block, ToolUseBlock): + return ("block_start", "tool_use", block.type, block.name) + return ("block_start", None, None, None) + if isinstance(chunk, RawContentBlockDeltaEvent): + delta_type = getattr(chunk.delta, "type", None) + if delta_type == "text_delta": + return ("block_delta", chunk.index, delta_type, chunk.delta.text) # type: ignore + if delta_type == "input_json_delta": + return ("block_delta", chunk.index, delta_type, chunk.delta.partial_json) # type: ignore + return ("other",) + if isinstance(chunk, RawMessageDeltaEvent): + usage = getattr(chunk, "usage", None) + return ("message_delta", chunk.delta.stop_reason, + _token_count(getattr(usage, "output_tokens", None))) + return ("other",) + + def merge(self, event: Tuple[Any, ...]) -> None: + self._is_null = False + kind = event[0] + if kind == "message_start": + _, role, input_tokens = event self._current_message_idx += 1 - value = { + self._values += { "messages": { "index": str(self._current_message_idx), - "role": chunk.message.role, - "input_tokens": str(chunk.message.usage.input_tokens), + "role": role, + "input_tokens": input_tokens, } } - self._values += value - elif isinstance(chunk, RawContentBlockStartEvent): - self._current_content_block_type = chunk.content_block - elif isinstance(chunk, RawContentBlockDeltaEvent): - if isinstance(self._current_content_block_type, TextBlock): - value = { - "messages": { - "index": str(self._current_message_idx), - "content": { - "index": chunk.index, - "type": self._current_content_block_type.type, - "text": chunk.delta.text, # type: ignore - }, - } + elif kind == "block_start": + self._current_block = event[1:] + elif kind == "block_delta": + _, index, delta_type, fragment = event + block_kind, block_type, tool_name = self._current_block + if block_kind == "text" and delta_type == "text_delta": + content = {"index": index, "type": block_type, "text": fragment} + elif block_kind == "tool_use" and delta_type == "input_json_delta": + content = { + "index": index, + "type": block_type, + "tool_name": tool_name, + "tool_input": fragment, } - self._values += value - elif isinstance(self._current_content_block_type, ToolUseBlock): - value = { - "messages": { - "index": str(self._current_message_idx), - "content": { - "index": chunk.index, - "type": self._current_content_block_type.type, - "tool_name": self._current_content_block_type.name, - "tool_input": chunk.delta.partial_json, # type: ignore - }, - } + else: + return + self._values += { + "messages": { + "index": str(self._current_message_idx), + "content": content, } - self._values += value - elif isinstance(chunk, RawMessageDeltaEvent): - value = { + } + elif kind == "message_delta": + _, stop_reason, output_tokens = event + self._values += { "messages": { "index": str(self._current_message_idx), - "stop_reason": chunk.delta.stop_reason, - "output_tokens": str(chunk.usage.output_tokens), + "stop_reason": stop_reason, + "output_tokens": output_tokens, } } - self._values += value def _result(self) -> Optional[Dict[str, Any]]: if self._is_null: diff --git a/python/frameworks/anthropic/traceai_anthropic/_utils.py b/python/frameworks/anthropic/traceai_anthropic/_utils.py index 3e44de6e0..d49eae795 100644 --- a/python/frameworks/anthropic/traceai_anthropic/_utils.py +++ b/python/frameworks/anthropic/traceai_anthropic/_utils.py @@ -11,6 +11,30 @@ logger.addHandler(logging.NullHandler()) +def _error_status(exception: BaseException) -> trace_api.Status: + """Format useful diagnostics without trusting an exception's __str__.""" + name = type(exception).__name__ + try: + detail = str(exception) + except BaseException: + detail = "" + return trace_api.Status(trace_api.StatusCode.ERROR, f"{name}: {detail}") + + +def _finish_request_error(span: _WithSpan, exception: BaseException) -> None: + # Each operation is isolated: even a broken processor must not obscure the + # original SDK error or prevent a best-effort attempt to end the span. + for action in ( + lambda: span.set_status(_error_status(exception)), + lambda: span.record_exception(exception), + span.finish_tracing, + ): + try: + action() + except BaseException: + pass + + class _ValueAndType(NamedTuple): value: str type: FiMimeTypeValues @@ -30,13 +54,13 @@ def _finish_tracing( try: attributes: Attributes = dict(has_attributes.get_attributes()) except Exception as e: - logger.exception("Failed to get attributes") + logger.warning("Failed to get attributes") attributes = None try: extra_attributes: Attributes = dict(has_attributes.get_extra_attributes()) except Exception as e: - logger.exception("Failed to get extra attributes") + logger.warning("Failed to get extra attributes") extra_attributes = None try: @@ -46,8 +70,7 @@ def _finish_tracing( extra_attributes=extra_attributes, ) except Exception as e: - print(f"Failed to finish tracing: {e}") - logger.exception("Failed to finish tracing") + logger.warning("Failed to finish tracing") raise @@ -56,8 +79,7 @@ def _io_value_and_type(obj: Any) -> _ValueAndType: json_value = safe_json_dumps(obj) return _ValueAndType(json_value, FiMimeTypeValues.JSON) except Exception as e: - print(f"Failed to serialize as JSON: {e}") - logger.exception("Failed to get input attributes from request parameters.") + logger.warning("Failed to get input attributes from request parameters.") return _ValueAndType(str(obj), FiMimeTypeValues.TEXT) diff --git a/python/frameworks/anthropic/traceai_anthropic/_with_span.py b/python/frameworks/anthropic/traceai_anthropic/_with_span.py index f81d4e7ad..4b354ff4d 100644 --- a/python/frameworks/anthropic/traceai_anthropic/_with_span.py +++ b/python/frameworks/anthropic/traceai_anthropic/_with_span.py @@ -22,7 +22,7 @@ def __init__( try: self._is_finished = not self._span.is_recording() except Exception: - logger.exception("Failed to check if span is recording") + logger.warning("Failed to check if span is recording") print("Error checking span recording status, defaulting to finished") self._is_finished = True @@ -42,7 +42,7 @@ def record_exception(self, exception: Exception) -> None: try: self._span.record_exception(exception) except Exception: - logger.exception("Failed to record exception on span") + logger.warning("Failed to record exception on span") print("Error recording exception on span") def set_status(self, status: Union[trace_api.Status, trace_api.StatusCode]) -> None: @@ -51,7 +51,7 @@ def set_status(self, status: Union[trace_api.Status, trace_api.StatusCode]) -> N try: self._span.set_status(status=status) except Exception: - logger.exception("Failed to set status on span") + logger.warning("Failed to set status on span") print("Error setting status on span") def add_event(self, name: str) -> None: @@ -60,7 +60,7 @@ def add_event(self, name: str) -> None: try: self._span.add_event(name) except Exception: - logger.exception("Failed to add event to span") + logger.warning("Failed to add event to span") print("Error adding event to span") def finish_tracing( @@ -81,20 +81,20 @@ def finish_tracing( try: self._span.set_attribute(key, value) except Exception: - logger.exception("Failed to set attribute on span") + logger.warning("Failed to set attribute on span") print(f"Error setting attribute: {key}") if status is not None: try: self._span.set_status(status=status) except Exception: - logger.exception("Failed to set status code on span") + logger.warning("Failed to set status code on span") print("Error setting final status") try: self._span.end() except Exception: - logger.exception("Failed to end span") + logger.warning("Failed to end span") print("Error ending span") self._is_finished = True diff --git a/python/frameworks/anthropic/traceai_anthropic/_wrappers.py b/python/frameworks/anthropic/traceai_anthropic/_wrappers.py index 15b6aa3a2..c21251a9f 100644 --- a/python/frameworks/anthropic/traceai_anthropic/_wrappers.py +++ b/python/frameworks/anthropic/traceai_anthropic/_wrappers.py @@ -33,6 +33,7 @@ from opentelemetry import trace as trace_api from opentelemetry.trace import INVALID_SPAN from traceai_anthropic._stream import _MessagesStream, _Stream +from traceai_anthropic._utils import _finish_request_error from traceai_anthropic._with_span import _WithSpan if TYPE_CHECKING: @@ -64,7 +65,7 @@ def _start_as_current_span( attributes=attributes, ) except Exception as e: - print(f"Error creating span: {e}") + logger.warning("Failed to create span") span = INVALID_SPAN with trace_api.use_span( span, @@ -151,12 +152,8 @@ def __call__( ), ) - except Exception as exception: - print(f"Error in completion: {exception}") - span.set_status( - trace_api.Status(trace_api.StatusCode.ERROR, str(exception)) - ) - span.record_exception(exception) + except BaseException as exception: + _finish_request_error(span, exception) raise streaming = kwargs.get("stream", False) @@ -219,11 +216,8 @@ async def __call__( try: response = await wrapped(*args, **kwargs) span.set_attributes(dict(_get_raw_output(response))) - except Exception as exception: - span.set_status( - trace_api.Status(trace_api.StatusCode.ERROR, str(exception)) - ) - span.record_exception(exception) + except BaseException as exception: + _finish_request_error(span, exception) raise span.set_status(trace_api.StatusCode.OK) streaming = kwargs.get("stream", False) @@ -305,11 +299,8 @@ def __call__( # Add I/O to span attributes instead of sending separately self._add_io_to_span_attributes(span=span, output_data=output_content) - except Exception as exception: - span.set_status( - trace_api.Status(trace_api.StatusCode.ERROR, str(exception)) - ) - span.record_exception(exception) + except BaseException as exception: + _finish_request_error(span, exception) raise streaming = kwargs.get("stream", False) @@ -382,11 +373,8 @@ async def __call__( try: response = await wrapped(*args, **kwargs) span.set_attributes(dict(_get_raw_output(response))) - except Exception as exception: - span.set_status( - trace_api.Status(trace_api.StatusCode.ERROR, str(exception)) - ) - span.record_exception(exception) + except BaseException as exception: + _finish_request_error(span, exception) raise streaming = kwargs.get("stream", False) if streaming: @@ -454,11 +442,8 @@ def __call__( try: response = wrapped(*args, **kwargs) span.set_attributes(dict(_get_raw_output(response))) - except Exception as exception: - span.set_status( - trace_api.Status(trace_api.StatusCode.ERROR, str(exception)) - ) - span.record_exception(exception) + except BaseException as exception: + _finish_request_error(span, exception) raise span.set_status(trace_api.StatusCode.OK) @@ -569,7 +554,7 @@ def process_message_content(content): "query": str(eval_input[0]) if eval_input else None, } except Exception as e: - logger.exception(f"Error in _extract_image_data: {e}") + logger.warning("Failed to extract image data") return { "input_images": None, "filtered_messages": messages, @@ -699,17 +684,17 @@ def _get_llm_input_messages(messages: List[Dict[str, str]]) -> Any: def _get_tool_use_block( block: ToolUseBlock, msg_prefix: str, tool_index: int ) -> Iterator[Tuple[str, Any]]: - if tool_call_id := block.id: + if tool_call_id := (block.get("id") if isinstance(block, Mapping) else block.id): yield ( f"{msg_prefix}.{MESSAGE_TOOL_CALLS}.{tool_index}.{TOOL_CALL_ID}", tool_call_id, ) - if block_name := block.name: + if block_name := (block.get("name") if isinstance(block, Mapping) else block.name): yield ( f"{msg_prefix}.{MESSAGE_TOOL_CALLS}.{tool_index}.{TOOL_CALL_FUNCTION_NAME}", block_name, ) - if block_input := block.input: + if block_input := (block.get("input") if isinstance(block, Mapping) else block.input): yield ( f"{msg_prefix}.{MESSAGE_TOOL_CALLS}.{tool_index}.{TOOL_CALL_FUNCTION_ARGUMENTS_JSON}", safe_json_dumps(block_input), diff --git a/python/tests/test_framework_anthropic.py b/python/tests/test_framework_anthropic.py index 23e09b62e..9403c09c6 100644 --- a/python/tests/test_framework_anthropic.py +++ b/python/tests/test_framework_anthropic.py @@ -18,7 +18,7 @@ from fi_instrumentation.otel import register from fi_instrumentation.fi_types import ( EvalName, - EvalSpanKind, + EvalSpanKind, EvalTag, EvalTagType, ProjectType, @@ -26,11 +26,19 @@ ) from fi_instrumentation.instrumentation.context_attributes import using_attributes +try: + import anthropic.resources.completions # noqa: F401 + + _HAS_LEGACY_COMPLETIONS = True +except ImportError: + # anthropic>=1.0 removed the legacy Completions API entirely. + _HAS_LEGACY_COMPLETIONS = False + class TestAnthropicFramework: """Test Anthropic framework instrumentation.""" - @pytest.fixture(autouse=True) + @pytest.fixture(autouse=True) def setup_trace_provider(self): """Set up trace provider for Anthropic testing.""" eval_tags = [ @@ -43,7 +51,7 @@ def setup_trace_provider(self): custom_eval_name="anthropic_test_eval" ) ] - + with patch('fi_instrumentation.otel.check_custom_eval_config_exists') as mock_check: mock_check.return_value = False self.trace_provider = register( @@ -54,12 +62,12 @@ def setup_trace_provider(self): verbose=False ) yield - + @pytest.fixture def mock_anthropic_requests(self): """Mock Anthropic HTTP requests to avoid real API calls.""" from anthropic.types import Message, Usage, TextBlock - + # Create a proper Message object to return mock_usage = Usage(input_tokens=10, output_tokens=12) mock_content = [TextBlock(type="text", text="This is a test response from Claude.")] @@ -73,32 +81,55 @@ def mock_anthropic_requests(self): stop_sequence=None, usage=mock_usage ) - + with patch('anthropic.resources.messages.Messages.create') as mock_create: mock_create.return_value = mock_message yield mock_create - + def test_anthropic_import(self): """Test that we can import the Anthropic instrumentor.""" assert AnthropicInstrumentor is not None - + # Test basic instantiation instrumentor = AnthropicInstrumentor() assert instrumentor is not None - + + def test_instrument_without_legacy_completions_api(self): + """anthropic>=1.0 removed anthropic.resources.completions entirely + (the legacy Text Completions API). instrument() must degrade to + wrapping only Messages instead of raising ImportError and + instrumenting nothing - regression test for that failure mode. + """ + instrumentor = AnthropicInstrumentor() + + with patch.dict(sys.modules, {"anthropic.resources.completions": None}): + instrumentor.instrument(tracer_provider=self.trace_provider) + + try: + from anthropic.resources.messages import Messages + + assert type(Messages.create).__name__ == 'BoundFunctionWrapper' + assert instrumentor._original_messages_create is not None + assert instrumentor._original_completions_create is None + assert instrumentor._original_async_completions_create is None + finally: + instrumentor.uninstrument() + + assert type(Messages.create).__name__ != 'BoundFunctionWrapper' + def test_anthropic_basic_instrumentation(self, mock_anthropic_requests): """Test basic Anthropic messages instrumentation.""" # Initialize instrumentor instrumentor = AnthropicInstrumentor() instrumentor.instrument(tracer_provider=self.trace_provider) - + try: # Create client and make request client = anthropic.Anthropic(api_key="test-key") - + with using_attributes( session_id="test-session-123", - user_id="test-user-456", + user_id="test-user-456", metadata={"test_type": "instrumentation", "framework": "anthropic"}, tags=["anthropic", "test", "claude-3-sonnet"] ): @@ -109,24 +140,24 @@ def test_anthropic_basic_instrumentation(self, mock_anthropic_requests): {"role": "user", "content": "Write a short poem about AI testing."} ] ) - + # Verify the create method was called assert mock_anthropic_requests.called - + # Verify response structure (directly from mock Message object) assert response.content[0].text == "This is a test response from Claude." assert response.model == "claude-3-7-sonnet-latest" assert response.usage.input_tokens == 10 assert response.usage.output_tokens == 12 - + finally: # Clean up instrumentor.uninstrument() - + def test_anthropic_tool_use(self, mock_anthropic_requests): """Test Anthropic instrumentation with tool use.""" from anthropic.types import Message, Usage, ToolUseBlock - + # Set up tool use response mock_usage = Usage(input_tokens=15, output_tokens=8) mock_content = [ToolUseBlock( @@ -146,13 +177,13 @@ def test_anthropic_tool_use(self, mock_anthropic_requests): usage=mock_usage ) mock_anthropic_requests.return_value = mock_tool_message - + instrumentor = AnthropicInstrumentor() instrumentor.instrument(tracer_provider=self.trace_provider) - + try: client = anthropic.Anthropic(api_key="test-key") - + tools = [{ "name": "get_weather", "description": "Get current weather for a location", @@ -164,7 +195,7 @@ def test_anthropic_tool_use(self, mock_anthropic_requests): "required": ["location"] } }] - + with using_attributes(session_id="tool-test-session"): response = client.messages.create( model="claude-3-7-sonnet-latest", @@ -172,116 +203,116 @@ def test_anthropic_tool_use(self, mock_anthropic_requests): tools=tools, messages=[{"role": "user", "content": "What's the weather in San Francisco?"}] ) - + # Verify tool use was captured (directly from mock Message object) assert response.content[0].name == "get_weather" assert response.content[0].input["location"] == "San Francisco" assert mock_anthropic_requests.called - + finally: instrumentor.uninstrument() - + @pytest.mark.asyncio async def test_anthropic_async_instrumentation(self, mock_anthropic_requests): """Test Anthropic async instrumentation setup.""" # For async, let's just test that the instrumentor can handle async calls instrumentor = AnthropicInstrumentor() instrumentor.instrument(tracer_provider=self.trace_provider) - + try: client = anthropic.AsyncAnthropic(api_key="test-key") - + # Test that the async client is properly instrumented assert hasattr(client.messages, 'create') - + # Test context attributes work with using_attributes(session_id="async-test-session"): # Just verify the context is set up correctly without making a real call pass - + finally: instrumentor.uninstrument() - + + @pytest.mark.skipif( + not _HAS_LEGACY_COMPLETIONS, + reason="anthropic>=1.0 removed the legacy Completions API entirely", + ) def test_anthropic_completions_legacy(self, mock_anthropic_requests): """Test Anthropic completions (legacy) instrumentation setup.""" # Don't need to modify mock for this test - just testing setup - + instrumentor = AnthropicInstrumentor() instrumentor.instrument(tracer_provider=self.trace_provider) - + try: client = anthropic.Anthropic(api_key="test-key") - + # Verify the client is properly instrumented for completions assert hasattr(client.completions, 'create') - + # Test context attributes work with completions with using_attributes(session_id="completions-test-session"): # Just verify completions parameter handling without making real calls pass - + finally: instrumentor.uninstrument() - + def test_anthropic_streaming_setup(self, mock_anthropic_requests): """Test Anthropic streaming instrumentation setup.""" instrumentor = AnthropicInstrumentor() instrumentor.instrument(tracer_provider=self.trace_provider) - + try: client = anthropic.Anthropic(api_key="test-key") - + # Verify the client is properly instrumented for streaming assert hasattr(client.messages, 'create') - + # Test context attributes work with streaming with using_attributes(session_id="streaming-test-session"): # Just verify streaming parameter handling without making real calls pass - + finally: instrumentor.uninstrument() - + def test_anthropic_error_handling(self, mock_anthropic_requests): """Test Anthropic instrumentation with error scenarios.""" instrumentor = AnthropicInstrumentor() instrumentor.instrument(tracer_provider=self.trace_provider) - + try: client = anthropic.Anthropic(api_key="test-key") - + # Verify the client is properly instrumented even for error cases assert hasattr(client.messages, 'create') - + # Test context attributes work in error scenarios with using_attributes(session_id="error-test-session"): # Just verify the instrumentation is set up correctly pass - + finally: instrumentor.uninstrument() - + def test_instrumentor_uninstrumentation(self): - """Test that uninstrumentation properly restores original behavior.""" + """Messages must be restored on every SDK, including without completions.""" from anthropic.resources.messages import Messages - from anthropic.resources.completions import Completions - + if _HAS_LEGACY_COMPLETIONS: + from anthropic.resources.completions import Completions + original_completions_create = Completions.create + instrumentor = AnthropicInstrumentor() - - # Store original methods original_messages_create = Messages.create - original_completions_create = Completions.create - - # Instrument + original_count_tokens = Messages.count_tokens instrumentor.instrument(tracer_provider=self.trace_provider) - - # Methods should be wrapped (different types) assert type(Messages.create).__name__ == 'BoundFunctionWrapper' - assert type(Completions.create).__name__ == 'BoundFunctionWrapper' - - # Uninstrument + assert type(Messages.count_tokens).__name__ == 'BoundFunctionWrapper' + if _HAS_LEGACY_COMPLETIONS: + assert type(Completions.create).__name__ == 'BoundFunctionWrapper' instrumentor.uninstrument() - - # Methods should be restored (back to functions) assert Messages.create == original_messages_create - assert Completions.create == original_completions_create \ No newline at end of file + assert Messages.count_tokens == original_count_tokens + if _HAS_LEGACY_COMPLETIONS: + assert Completions.create == original_completions_create