Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 5 additions & 21 deletions python/frameworks/anthropic/traceai_anthropic/_wrappers.py
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,6 @@ def __call__(
llm_filtered_input_messages = extracted_data["filtered_messages"]
input_images = extracted_data["input_images"]
eval_input = extracted_data["eval_input"]
query = extracted_data["query"]

with self._start_as_current_span(
span_name="Completions",
Expand All @@ -166,9 +165,8 @@ def __call__(
_get_llm_invocation_parameters(llm_invocation_parameters),
_get_llm_tools(llm_invocation_parameters),
_get_image_inputs(input_images),
_get_eval_input(eval_input),
_get_raw_input(arguments),
_get_query(query),
_get_eval_input(eval_input),
)
),
) as span:
Expand Down Expand Up @@ -223,7 +221,6 @@ async def __call__(
filtered_llm_input_messages = extracted_data["filtered_messages"]
input_images = extracted_data["input_images"]
eval_input = extracted_data["eval_input"]
query = extracted_data["query"]

with self._start_as_current_span(
span_name="AsyncCompletions",
Expand All @@ -239,9 +236,8 @@ async def __call__(
_get_llm_invocation_parameters(invocation_parameters),
_get_llm_tools(invocation_parameters),
_get_image_inputs(input_images),
_get_eval_input(eval_input),
_get_raw_input(arguments),
_get_query(query),
_get_eval_input(eval_input),
)
),
) as span:
Expand Down Expand Up @@ -288,7 +284,6 @@ def __call__(
llm_filtered_input_messages = extracted_data["filtered_messages"]
input_images = extracted_data["input_images"]
eval_input = extracted_data["eval_input"]
query = extracted_data["query"]

with self._start_as_current_span(
span_name="Messages",
Expand All @@ -305,9 +300,8 @@ def __call__(
_get_llm_tools(invocation_parameters),
_get_inputs(llm_filtered_input_messages),
_get_image_inputs(input_images),
_get_eval_input(eval_input),
_get_raw_input(arguments),
_get_query(query),
_get_eval_input(eval_input),
)
),
) as span:
Expand Down Expand Up @@ -346,7 +340,6 @@ async def __call__(
llm_filtered_input_messages = extracted_data["filtered_messages"]
input_images = extracted_data["input_images"]
eval_input = extracted_data["eval_input"]
query = extracted_data["query"]

with self._start_as_current_span(
span_name="AsyncMessages",
Expand All @@ -363,9 +356,8 @@ async def __call__(
_get_llm_tools(invocation_parameters),
_get_inputs(llm_filtered_input_messages),
_get_image_inputs(input_images),
_get_eval_input(eval_input),
_get_raw_input(arguments),
_get_query(query),
_get_eval_input(eval_input),
)
),
) as span:
Expand Down Expand Up @@ -427,7 +419,6 @@ def __call__(
llm_filtered_input_messages = extracted_data["filtered_messages"]
input_images = extracted_data["input_images"]
eval_input = extracted_data["eval_input"]
query = extracted_data["query"]

with self._start_as_current_span(
span_name="MessagesCountTokens",
Expand All @@ -442,9 +433,8 @@ def __call__(
_get_llm_input_messages(llm_input_messages),
_get_inputs(llm_filtered_input_messages),
_get_image_inputs(input_images),
_get_eval_input(eval_input),
_get_raw_input(arguments),
_get_query(query),
_get_eval_input(eval_input),
)
),
) as span:
Expand Down Expand Up @@ -485,10 +475,6 @@ def _get_raw_output(response) -> Iterator[Tuple[str, Any]]:
yield OUTPUT_VALUE, safe_json_dumps(response)


def _get_query(query: str) -> Iterator[Tuple[str, Any]]:
yield INPUT_VALUE, query


def _get_response(response: Any) -> Iterator[Tuple[str, Any]]:
yield OUTPUT_VALUE, response

Expand Down Expand Up @@ -560,15 +546,13 @@ def process_message_content(content):
"input_images": input_images if input_images else None,
"filtered_messages": filtered_messages if filtered_messages else None,
"eval_input": "\n".join(eval_input),
"query": str(eval_input[0]) if eval_input else None,
}
except Exception as e:
logger.warning("Failed to extract image data")
return {
"input_images": None,
"filtered_messages": messages,
"eval_input": None,
"query": None,
}


Expand Down
72 changes: 72 additions & 0 deletions python/tests/test_framework_anthropic.py
Original file line number Diff line number Diff line change
Expand Up @@ -316,3 +316,75 @@ def test_instrumentor_uninstrumentation(self):
assert Messages.count_tokens == original_count_tokens
if _HAS_LEGACY_COMPLETIONS:
assert Completions.create == original_completions_create


class TestInputValueMultiMessage:
"""Regression for the same first-message overwrite fixed in openai/litellm.

The wrappers' attribute chain ended with _get_query(eval_input[0]), which
overwrote the joined multi-message INPUT_VALUE. The join must now win.
"""

def test_extract_image_data_joins_all_messages(self):
from traceai_anthropic._wrappers import _extract_image_data

result = _extract_image_data(
[
{"role": "user", "content": "first"},
{"role": "assistant", "content": "second"},
{"role": "user", "content": "third"},
]
)
assert "query" not in result
for text in ("first", "second", "third"):
assert text in result["eval_input"]

def test_extract_image_data_embeds_image_placeholders(self):
from fi_instrumentation.fi_types import SpanAttributes
from traceai_anthropic._wrappers import _extract_image_data

result = _extract_image_data(
[
{
"role": "user",
"content": [
{"type": "text", "text": "describe"},
{
"type": "image",
"source": {"type": "base64", "data": "QUJD"},
},
{"type": "text", "text": "closely"},
],
}
]
)
assert "describe" in result["eval_input"]
assert "closely" in result["eval_input"]
assert f"{{{SpanAttributes.INPUT_IMAGES}.0}}" in result["eval_input"]
assert result["input_images"] == ["QUJD"]

def test_eval_input_wins_over_raw_arguments(self):
"""dict(chain(...)) keeps the last INPUT_VALUE — the join, not kwargs."""
from itertools import chain

from fi_instrumentation.fi_types import SpanAttributes
from traceai_anthropic._wrappers import (
_get_eval_input,
_get_raw_input,
)
import json

arguments = {
"model": "claude-3",
"messages": [
{"role": "user", "content": "one"},
{"role": "user", "content": "two"},
],
}
attrs = dict(
chain(
_get_raw_input(arguments),
_get_eval_input("one\ntwo"),
)
)
assert json.loads(attrs[SpanAttributes.INPUT_VALUE]) == "one\ntwo"
Loading