Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 4 additions & 13 deletions python/frameworks/langchain/traceai_langchain/_tracer.py
Original file line number Diff line number Diff line change
Expand Up @@ -525,7 +525,7 @@ def _update_span(
)
span.set_attribute(GEN_AI_SPAN_KIND, span_kind.value)
span.set_attributes(dict(captured_context))
filtered_data, images, eval_input, query = _filter_images(
filtered_data, images, eval_input = _filter_images(
run.inputs.get("messages", [])
)

Expand Down Expand Up @@ -561,7 +561,6 @@ def _update_span(
_metadata(run),
_session_id_from_thread_id(run, captured_context),
_as_raw_input_output(run, outputs),
_as_query(query),
)
)
)
Expand Down Expand Up @@ -651,11 +650,6 @@ def _as_raw_input_output(
yield OUTPUT_VALUE, safe_json_dumps(raw_output.__dict__)


def _as_query(values: Iterable[str]) -> Iterator[Tuple[str, str]]:
if values:
yield INPUT_VALUE, safe_json_dumps(values)


def _convert_io(obj: Optional[Mapping[str, Any]]) -> Iterator[str]:
if not obj:
return
Expand Down Expand Up @@ -1230,14 +1224,13 @@ def _get_attributes_from_image(
yield f"{IMAGE_URL}", url


def _filter_images(messages: Any) -> Tuple[Any, List[str], str, str]:
def _filter_images(messages: Any) -> Tuple[Any, List[str], str]:
filtered_data = []
images = []
eval_input = []
query = ""

if not messages or not isinstance(messages, list):
return messages, [], "", ""
return messages, [], ""

for message_group in messages:
if not isinstance(message_group, list):
Expand Down Expand Up @@ -1271,10 +1264,8 @@ def _filter_images(messages: Any) -> Tuple[Any, List[str], str, str]:
{"type": "text", "text": f"[Image: {image_url}]"}
)

if eval_input and len(eval_input) > 0:
query = eval_input[0]
eval_input = " | ".join(map(str, eval_input))
return filtered_data, images, eval_input, query
return filtered_data, images, eval_input


LANGCHAIN_SESSION_ID = "session_id"
Expand Down
1 change: 0 additions & 1 deletion python/frameworks/mistralai/traceai_mistralai/_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,6 @@ def _process_input_messages(messages: list) -> Iterator[Tuple[str, AttributeValu
yield SpanAttributes.INPUT_MIME_TYPE, FiMimeTypeValues.JSON.value
if eval_input:
yield SpanAttributes.INPUT_VALUE, safe_json_dumps("\n".join(eval_input))
yield SpanAttributes.INPUT_VALUE, safe_json_dumps(eval_input[0])

except Exception as e:
print(f"Error in _process_input_messages: {str(e)}")
Expand Down
7 changes: 0 additions & 7 deletions python/frameworks/vertexai/traceai_vertexai/_wrapper.py
Original file line number Diff line number Diff line change
Expand Up @@ -150,8 +150,6 @@ def __call__(
span.set_attribute(INPUT_IMAGES, safe_json_dumps(input_images))
if eval_input := extracted_data.get("eval_input"):
span.set_attribute(INPUT_VALUE, safe_json_dumps(eval_input))
if query := extracted_data.get("query"):
span.set_attribute(INPUT_VALUE, safe_json_dumps(query))

else:
span.set_attribute(INPUT_VALUE, safe_json_dumps(request))
Expand Down Expand Up @@ -641,7 +639,6 @@ def _extract_image_data(messages):
input_images = []
filtered_messages = []
eval_input = []
query = ""

if isinstance(messages, list):
for message in messages:
Expand Down Expand Up @@ -678,22 +675,18 @@ def _extract_image_data(messages):
}
filtered_messages.append(filtered_message)

if eval_input and len(eval_input) > 0:
query = eval_input[0]

return {
"input_images": input_images if input_images else None,
"filtered_messages": filtered_messages if filtered_messages else None,
"eval_input": " | ".join(eval_input),
"query": query,
}
except Exception as e:
print(f"Error in _extract_image_data: {e}")
return {
"input_images": input_images,
"filtered_messages": messages,
"eval_input": "",
"query": query,
}


Expand Down
17 changes: 17 additions & 0 deletions python/tests/test_framework_langchain.py
Original file line number Diff line number Diff line change
Expand Up @@ -774,5 +774,22 @@ def test_error_handling_scenarios(self, fi_tracer_instance):
tracer._send_span_data_to_api(mock_span, {"input": "test"}, {"output": "test"})


class TestInputValueMultiMessage:
"""Regression: eval_input must keep every message; no query (#151 family)."""

def test_filter_images_joins_all_messages(self):
from traceai_langchain._tracer import _filter_images

messages = [
[
{"kwargs": {"content": "first"}},
{"kwargs": {"content": "second"}},
]
]
filtered_data, images, eval_input = _filter_images(messages)
assert "first" in eval_input
assert "second" in eval_input


if __name__ == "__main__":
pytest.main([__file__, "-v"])
23 changes: 23 additions & 0 deletions python/tests/test_framework_mistralai.py
Original file line number Diff line number Diff line change
Expand Up @@ -778,5 +778,28 @@ def test_attribute_extraction_integration(self, mock_mistralai_module, sample_re
assert hasattr(response_extractor, 'get_attributes_from_response')


class TestInputValueMultiMessage:
"""Regression: last INPUT_VALUE yield must keep every message (#151 family)."""

def test_joined_eval_input_wins(self):
import json as _json

from fi_instrumentation.fi_types import SpanAttributes

messages = [
{"role": "system", "content": "You are helpful."},
{"role": "user", "content": "What is 2+2?"},
{"role": "assistant", "content": "4"},
{"role": "user", "content": "And 3+3?"},
]
attributes = list(_process_input_messages(messages))
input_values = [
v for k, v in attributes if k == SpanAttributes.INPUT_VALUE
]
final = _json.loads(input_values[-1])
for text in ("You are helpful.", "What is 2+2?", "4", "And 3+3?"):
assert text in final


if __name__ == "__main__":
pytest.main([__file__, "-v"])
16 changes: 15 additions & 1 deletion python/tests/test_framework_vertexai.py
Original file line number Diff line number Diff line change
Expand Up @@ -487,7 +487,6 @@ def test_extract_image_data_empty(self):
"filtered_messages": None,
"input_images": None,
"eval_input": "",
"query": ""
}

def test_extract_image_data_with_text(self):
Expand Down Expand Up @@ -796,5 +795,20 @@ def test_proxy_with_none_objects(self):
assert result == "test"


class TestInputValueMultiMessage:
"""Regression: eval_input must keep every message; no query (#151 family)."""

def test_extract_image_data_joins_all_contents(self):
messages = [
{"role": "user", "parts": [{"text": "first"}]},
{"role": "model", "parts": [{"text": "second"}]},
{"role": "user", "parts": [{"text": "third"}]},
]
result = _extract_image_data(messages)
assert "query" not in result
for text in ("first", "second", "third"):
assert text in result["eval_input"]


if __name__ == "__main__":
pytest.main([__file__])
Loading