diff --git a/python/frameworks/anthropic/traceai_anthropic/_wrappers.py b/python/frameworks/anthropic/traceai_anthropic/_wrappers.py index 9815f9042..c8ee93615 100644 --- a/python/frameworks/anthropic/traceai_anthropic/_wrappers.py +++ b/python/frameworks/anthropic/traceai_anthropic/_wrappers.py @@ -150,7 +150,6 @@ def __call__( llm_filtered_input_messages = extracted_data["filtered_messages"] input_images = extracted_data["input_images"] eval_input = extracted_data["eval_input"] - query = extracted_data["query"] with self._start_as_current_span( span_name="Completions", @@ -166,9 +165,8 @@ def __call__( _get_llm_invocation_parameters(llm_invocation_parameters), _get_llm_tools(llm_invocation_parameters), _get_image_inputs(input_images), - _get_eval_input(eval_input), _get_raw_input(arguments), - _get_query(query), + _get_eval_input(eval_input), ) ), ) as span: @@ -223,7 +221,6 @@ async def __call__( filtered_llm_input_messages = extracted_data["filtered_messages"] input_images = extracted_data["input_images"] eval_input = extracted_data["eval_input"] - query = extracted_data["query"] with self._start_as_current_span( span_name="AsyncCompletions", @@ -239,9 +236,8 @@ async def __call__( _get_llm_invocation_parameters(invocation_parameters), _get_llm_tools(invocation_parameters), _get_image_inputs(input_images), - _get_eval_input(eval_input), _get_raw_input(arguments), - _get_query(query), + _get_eval_input(eval_input), ) ), ) as span: @@ -288,7 +284,6 @@ def __call__( llm_filtered_input_messages = extracted_data["filtered_messages"] input_images = extracted_data["input_images"] eval_input = extracted_data["eval_input"] - query = extracted_data["query"] with self._start_as_current_span( span_name="Messages", @@ -305,9 +300,8 @@ def __call__( _get_llm_tools(invocation_parameters), _get_inputs(llm_filtered_input_messages), _get_image_inputs(input_images), - _get_eval_input(eval_input), _get_raw_input(arguments), - _get_query(query), + _get_eval_input(eval_input), ) ), ) as span: @@ -346,7 +340,6 @@ async def __call__( llm_filtered_input_messages = extracted_data["filtered_messages"] input_images = extracted_data["input_images"] eval_input = extracted_data["eval_input"] - query = extracted_data["query"] with self._start_as_current_span( span_name="AsyncMessages", @@ -363,9 +356,8 @@ async def __call__( _get_llm_tools(invocation_parameters), _get_inputs(llm_filtered_input_messages), _get_image_inputs(input_images), - _get_eval_input(eval_input), _get_raw_input(arguments), - _get_query(query), + _get_eval_input(eval_input), ) ), ) as span: @@ -427,7 +419,6 @@ def __call__( llm_filtered_input_messages = extracted_data["filtered_messages"] input_images = extracted_data["input_images"] eval_input = extracted_data["eval_input"] - query = extracted_data["query"] with self._start_as_current_span( span_name="MessagesCountTokens", @@ -442,9 +433,8 @@ def __call__( _get_llm_input_messages(llm_input_messages), _get_inputs(llm_filtered_input_messages), _get_image_inputs(input_images), - _get_eval_input(eval_input), _get_raw_input(arguments), - _get_query(query), + _get_eval_input(eval_input), ) ), ) as span: @@ -485,10 +475,6 @@ def _get_raw_output(response) -> Iterator[Tuple[str, Any]]: yield OUTPUT_VALUE, safe_json_dumps(response) -def _get_query(query: str) -> Iterator[Tuple[str, Any]]: - yield INPUT_VALUE, query - - def _get_response(response: Any) -> Iterator[Tuple[str, Any]]: yield OUTPUT_VALUE, response @@ -560,7 +546,6 @@ def process_message_content(content): "input_images": input_images if input_images else None, "filtered_messages": filtered_messages if filtered_messages else None, "eval_input": "\n".join(eval_input), - "query": str(eval_input[0]) if eval_input else None, } except Exception as e: logger.warning("Failed to extract image data") @@ -568,7 +553,6 @@ def process_message_content(content): "input_images": None, "filtered_messages": messages, "eval_input": None, - "query": None, } diff --git a/python/tests/test_framework_anthropic.py b/python/tests/test_framework_anthropic.py index 9403c09c6..3e5f430c3 100644 --- a/python/tests/test_framework_anthropic.py +++ b/python/tests/test_framework_anthropic.py @@ -316,3 +316,75 @@ def test_instrumentor_uninstrumentation(self): assert Messages.count_tokens == original_count_tokens if _HAS_LEGACY_COMPLETIONS: assert Completions.create == original_completions_create + + +class TestInputValueMultiMessage: + """Regression for the same first-message overwrite fixed in openai/litellm. + + The wrappers' attribute chain ended with _get_query(eval_input[0]), which + overwrote the joined multi-message INPUT_VALUE. The join must now win. + """ + + def test_extract_image_data_joins_all_messages(self): + from traceai_anthropic._wrappers import _extract_image_data + + result = _extract_image_data( + [ + {"role": "user", "content": "first"}, + {"role": "assistant", "content": "second"}, + {"role": "user", "content": "third"}, + ] + ) + assert "query" not in result + for text in ("first", "second", "third"): + assert text in result["eval_input"] + + def test_extract_image_data_embeds_image_placeholders(self): + from fi_instrumentation.fi_types import SpanAttributes + from traceai_anthropic._wrappers import _extract_image_data + + result = _extract_image_data( + [ + { + "role": "user", + "content": [ + {"type": "text", "text": "describe"}, + { + "type": "image", + "source": {"type": "base64", "data": "QUJD"}, + }, + {"type": "text", "text": "closely"}, + ], + } + ] + ) + assert "describe" in result["eval_input"] + assert "closely" in result["eval_input"] + assert f"{{{SpanAttributes.INPUT_IMAGES}.0}}" in result["eval_input"] + assert result["input_images"] == ["QUJD"] + + def test_eval_input_wins_over_raw_arguments(self): + """dict(chain(...)) keeps the last INPUT_VALUE — the join, not kwargs.""" + from itertools import chain + + from fi_instrumentation.fi_types import SpanAttributes + from traceai_anthropic._wrappers import ( + _get_eval_input, + _get_raw_input, + ) + import json + + arguments = { + "model": "claude-3", + "messages": [ + {"role": "user", "content": "one"}, + {"role": "user", "content": "two"}, + ], + } + attrs = dict( + chain( + _get_raw_input(arguments), + _get_eval_input("one\ntwo"), + ) + ) + assert json.loads(attrs[SpanAttributes.INPUT_VALUE]) == "one\ntwo"