diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/.changelog/511.fixed b/instrumentation/opentelemetry-instrumentation-genai-langchain/.changelog/511.fixed new file mode 100644 index 000000000..ca6b58e10 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/.changelog/511.fixed @@ -0,0 +1 @@ +Divert `SystemMessage` inputs to `gen_ai.system_instructions` instead of emitting them as `role="system"` inside `gen_ai.input.messages`. diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py index 39181ae9a..180649d9c 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py @@ -34,7 +34,7 @@ make_last_output_message, normalize_provider, prepare_tool_definitions, - to_input_messages, + split_system_and_input_messages, ) from opentelemetry.util.genai.handler import TelemetryHandler from opentelemetry.util.genai.invocation import ( @@ -285,14 +285,15 @@ def on_chat_model_start( if "ls_max_tokens" in metadata: max_tokens = metadata.get("ls_max_tokens") - # ``messages`` from on_chat_model_start is ``list[list[BaseMessage]]`` - # (one inner list per generation request). Flatten and let - # :func:`to_input_messages` produce spec-conformant ``InputMessage`` s - # with proper roles, tool-call requests, tool results, and reasoning. + # Flatten ``list[list[BaseMessage]]`` (one inner list per generation + # request) before splitting into system / input. flattened: list[BaseMessage] = [msg for sub in messages for msg in sub] + system_instruction: list[MessagePart] = [] input_messages: list[InputMessage] = [] if self._telemetry_handler.should_capture_content(): - input_messages = to_input_messages(flattened) + system_instruction, input_messages = ( + split_system_and_input_messages(flattened) + ) llm_invocation = self._telemetry_handler.inference( provider, @@ -300,6 +301,8 @@ def on_chat_model_start( ) llm_invocation.conversation_id = _conversation_id(metadata) llm_invocation.input_messages = input_messages + if system_instruction: + llm_invocation.system_instruction = system_instruction llm_invocation.top_p = top_p llm_invocation.frequency_penalty = frequency_penalty llm_invocation.presence_penalty = presence_penalty diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/utils.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/utils.py index 10d0ba90a..fdd3b4e10 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/utils.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/utils.py @@ -11,6 +11,7 @@ from langchain_core.messages import ( AIMessage, BaseMessage, + SystemMessage, ToolMessage, convert_to_messages, ) @@ -327,13 +328,14 @@ def to_input_messages( Called only when content capture is enabled (``TelemetryHandler.should_capture_content()``). """ + materialized = list(messages) try: normalized_messages: Iterable[BaseMessage] = convert_to_messages( - list(messages) + materialized ) except Exception: # pylint: disable=broad-except normalized_messages = [ - m for m in messages if isinstance(m, BaseMessage) + m for m in materialized if isinstance(m, BaseMessage) ] result: list[InputMessage] = [] for message in normalized_messages: @@ -344,6 +346,36 @@ def to_input_messages( return result +def split_system_and_input_messages( + messages: Iterable[Any], +) -> tuple[list[MessagePart], list[InputMessage]]: + """Split ``messages`` into ``system_instruction`` parts and ``InputMessage`` s. + + Called only when content capture is enabled + (``TelemetryHandler.should_capture_content()``). + """ + materialized = list(messages) + try: + normalized: Iterable[BaseMessage] = convert_to_messages(materialized) + except Exception: # pylint: disable=broad-except + normalized = [m for m in materialized if isinstance(m, BaseMessage)] + + system_parts: list[MessagePart] = [] + input_messages: list[InputMessage] = [] + + for message in normalized: + if isinstance(message, SystemMessage): + system_parts.extend(_content_to_parts(message.content)) + else: + parts = _message_parts(message) + if parts: + input_messages.append( + InputMessage(role=_normalize_role(message), parts=parts) + ) + + return system_parts, input_messages + + def to_output_messages( messages: Iterable[BaseMessage], *, diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/conformance/_shared.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/conformance/_shared.py new file mode 100644 index 000000000..1564bf9d2 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/conformance/_shared.py @@ -0,0 +1,21 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Helpers shared across langchain conformance scenarios.""" + +from __future__ import annotations + +from typing import Any + +from opentelemetry.test.weaver_live_check import LiveCheckReport + + +def span_attribute_values(report: LiveCheckReport, name: str) -> list[Any]: + """Return every value of ``name`` across all span samples in ``report``.""" + return [ + attr["value"] + for entry in report["samples"] + if "span" in entry + for attr in entry["span"]["attributes"] + if attr["name"] == name + ] diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/conformance/inference.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/conformance/inference.py index 90c0c109e..16e7976e8 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/conformance/inference.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/conformance/inference.py @@ -16,12 +16,15 @@ from opentelemetry.sdk._logs import LoggerProvider from opentelemetry.sdk.metrics import MeterProvider from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.test.weaver_live_check import LiveCheckReport from opentelemetry.test_util_genai.conformance import ( ExpectedViolation, Scenario, ) from opentelemetry.test_util_genai.instrumentor import instrument +from ._shared import span_attribute_values + class InferenceScenario(Scenario): expected_spans = {"chat": 1} @@ -37,6 +40,20 @@ class InferenceScenario(Scenario): ), ) + def validate(self, report: LiveCheckReport) -> None: + super().validate(report) + system_instructions = span_attribute_values( + report, "gen_ai.system_instructions" + ) + assert len(system_instructions) == 1, ( + "chat span with a SystemMessage input should set " + f"gen_ai.system_instructions once; saw {system_instructions}" + ) + assert "You are a helpful assistant!" in system_instructions[0], ( + "gen_ai.system_instructions should carry the SystemMessage " + f"content; got {system_instructions[0]}" + ) + def run( self, *, diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/conformance/inference_streaming.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/conformance/inference_streaming.py index 41f4cd2af..ef62c6715 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/conformance/inference_streaming.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/conformance/inference_streaming.py @@ -28,6 +28,8 @@ ) from opentelemetry.test_util_genai.instrumentor import instrument +from ._shared import span_attribute_values + class InferenceStreamingScenario(Scenario): expected_spans = {"chat": 1} @@ -57,17 +59,22 @@ class InferenceStreamingScenario(Scenario): def validate(self, report: LiveCheckReport) -> None: super().validate(report) - stream_values = [ - attr["value"] - for entry in report["samples"] - if "span" in entry - for attr in entry["span"]["attributes"] - if attr["name"] == "gen_ai.request.stream" - ] + stream_values = span_attribute_values(report, "gen_ai.request.stream") assert stream_values == [True], ( "streaming chat should set gen_ai.request.stream=true on the chat " f"span; saw {stream_values}" ) + system_instructions = span_attribute_values( + report, "gen_ai.system_instructions" + ) + assert len(system_instructions) == 1, ( + "streaming chat span with a SystemMessage input should set " + f"gen_ai.system_instructions once; saw {system_instructions}" + ) + assert "You are a helpful assistant!" in system_instructions[0], ( + "gen_ai.system_instructions should carry the SystemMessage " + f"content; got {system_instructions[0]}" + ) def run( self, diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py index 1de782f07..0bb63c92e 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py @@ -7,9 +7,11 @@ import pytest from langchain_anthropic import ChatAnthropic from langchain_core.messages import ( + AIMessage, FunctionMessage, HumanMessage, SystemMessage, + SystemMessageChunk, ) from langchain_core.tools import tool from openai import AuthenticationError @@ -18,6 +20,7 @@ LangChainInstrumentor, ) from opentelemetry.instrumentation.genai.langchain.utils import ( + split_system_and_input_messages, to_input_messages, ) from opentelemetry.sdk.trace import ReadableSpan @@ -145,6 +148,48 @@ def test_chat_openai_gpt_3_5_turbo_model_llm_call( assert len(logs) == 0 +@pytest.mark.parametrize( + "capture_content", + ["NO_CONTENT", "EVENT_ONLY"], +) +def test_system_instructions_absent_from_span_without_content_capture( + span_exporter, + start_instrumentation, + chat_openai_gpt_3_5_turbo_model, + monkeypatch, + capture_content, + vcr, +): + """``gen_ai.system_instructions`` is a sensitive attribute and must be + gated by ``OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT``: when + content capture excludes ``SPAN`` (``NO_CONTENT`` / ``EVENT_ONLY``), the + span attribute must not appear even though the input contained a + ``SystemMessage``. + """ + monkeypatch.setenv( + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", capture_content + ) + + messages = [ + SystemMessage(content="You are a helpful assistant!"), + HumanMessage(content="What is the capital of France?"), + ] + + with vcr.use_cassette( + _openai_cassette_name( + chat_openai_gpt_3_5_turbo_model, + "test_chat_openai_gpt_3_5_turbo_model_llm_call", + ) + ): + chat_openai_gpt_3_5_turbo_model.invoke(messages) + + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + assert ( + gen_ai_attributes.GEN_AI_SYSTEM_INSTRUCTIONS not in spans[0].attributes + ) + + # span_exporter, metric_reader, log_exporter, start_instrumentation, chat_openai_gpt_3_5_turbo_model are coming from fixtures defined in conftest.py @pytest.mark.parametrize( "capture_content", @@ -715,6 +760,79 @@ def test_function_message_role_maps_to_tool(): assert result[0].role == "tool" +def test_split_system_and_input_messages_diverts_system_instructions(): + system, inputs = split_system_and_input_messages( + [ + SystemMessage(content="You are helpful."), + HumanMessage(content="Hi"), + ] + ) + assert len(system) == 1 + assert system[0].content == "You are helpful." + assert system[0].type == "text" + assert len(inputs) == 1 + assert inputs[0].role == "user" + + +def test_split_system_and_input_messages_position_independent(): + # A ``SystemMessage`` interleaved between non-system messages still lands + # in ``system_instruction`` — semconv treats it as a top-level list rather + # than an interleaved role, and combining multiple system chunks in order + # matches how providers deliver them. + system, inputs = split_system_and_input_messages( + [ + SystemMessage(content="First guidance."), + HumanMessage(content="Hi"), + AIMessage(content="Hello"), + SystemMessage(content="Second guidance."), + HumanMessage(content="Follow-up"), + ] + ) + assert [part.content for part in system] == [ + "First guidance.", + "Second guidance.", + ] + assert [msg.role for msg in inputs] == ["user", "assistant", "user"] + + +def test_split_system_and_input_messages_handles_system_message_chunk(): + # ``SystemMessageChunk`` (streaming) subclasses ``SystemMessage`` — the + # helper matches by ``isinstance`` so streamed system content is diverted + # too instead of leaking into ``gen_ai.input.messages``. + system, inputs = split_system_and_input_messages( + [ + SystemMessageChunk(content="Streamed system prompt."), + HumanMessage(content="Hi"), + ] + ) + assert len(system) == 1 + assert system[0].content == "Streamed system prompt." + assert [msg.role for msg in inputs] == ["user"] + + +def test_split_system_and_input_messages_empty_when_no_system_message(): + system, inputs = split_system_and_input_messages( + [HumanMessage(content="Hi")] + ) + assert system == [] + assert [msg.role for msg in inputs] == ["user"] + + +def test_split_system_and_input_messages_normalizes_shorthand_inputs(): + # Short-hand tuple / dict forms are normalized via ``convert_to_messages`` + # before the ``isinstance`` partition so that a short-hand system entry is + # diverted to ``system_instruction`` instead of leaking into + # ``gen_ai.input.messages`` as ``role: "system"``. + system, inputs = split_system_and_input_messages( + [ + ("system", "You are helpful."), + {"role": "user", "content": "Hi"}, + ] + ) + assert [part.content for part in system] == ["You are helpful."] + assert [msg.role for msg in inputs] == ["user"] + + def assert_openai_completion_attributes( span: ReadableSpan, response: Optional, verify_content: bool = True ): @@ -778,11 +896,18 @@ def assert_openai_completion_attributes( if verify_content: input_message = attributes[gen_ai_attributes.GEN_AI_INPUT_MESSAGES] assert input_message is not None - assert '"role":"system"' in input_message - assert '"content":"You are a helpful assistant!"' in input_message + assert '"role":"system"' not in input_message assert '"role":"user"' in input_message assert '"content":"What is the capital of France?"' in input_message + system_instructions = attributes[ + gen_ai_attributes.GEN_AI_SYSTEM_INSTRUCTIONS + ] + assert system_instructions is not None + assert ( + '"content":"You are a helpful assistant!"' in system_instructions + ) + # Assert output message output_message = attributes[gen_ai_attributes.GEN_AI_OUTPUT_MESSAGES] assert output_message is not None @@ -792,6 +917,7 @@ def assert_openai_completion_attributes( else: assert gen_ai_attributes.GEN_AI_INPUT_MESSAGES not in attributes assert gen_ai_attributes.GEN_AI_OUTPUT_MESSAGES not in attributes + assert gen_ai_attributes.GEN_AI_SYSTEM_INSTRUCTIONS not in attributes def assert_openai_completion_attributes_with_error( @@ -828,16 +954,24 @@ def assert_openai_completion_attributes_with_error( if verify_content: input_message = attributes[gen_ai_attributes.GEN_AI_INPUT_MESSAGES] assert input_message is not None - assert '"role":"system"' in input_message - assert '"content":"You are a helpful assistant!"' in input_message + assert '"role":"system"' not in input_message assert '"role":"user"' in input_message assert '"content":"What is the capital of France?"' in input_message + system_instructions = attributes[ + gen_ai_attributes.GEN_AI_SYSTEM_INSTRUCTIONS + ] + assert system_instructions is not None + assert ( + '"content":"You are a helpful assistant!"' in system_instructions + ) + # Assert output message assert gen_ai_attributes.GEN_AI_OUTPUT_MESSAGES not in attributes else: assert gen_ai_attributes.GEN_AI_INPUT_MESSAGES not in attributes assert gen_ai_attributes.GEN_AI_OUTPUT_MESSAGES not in attributes + assert gen_ai_attributes.GEN_AI_SYSTEM_INSTRUCTIONS not in attributes def assert_bedrock_completion_attributes( @@ -1077,12 +1211,6 @@ def assert_log_record(log_record, parent_span, response=None): attrs.get(gen_ai_attributes.GEN_AI_INPUT_MESSAGES, []) ) expected_input = [ - { - "parts": [ - {"content": "You are a helpful assistant!", "type": "text"} - ], - "role": "system", - }, { "parts": [ {"content": "What is the capital of France?", "type": "text"} @@ -1090,12 +1218,19 @@ def assert_log_record(log_record, parent_span, response=None): "role": "user", }, ] - assert len(input_msgs) == 2 + assert len(input_msgs) == 1 for i, exp in enumerate(expected_input): got = _normalize_to_dict(input_msgs[i]) assert got["role"] == exp["role"] assert _normalize_to_list(got["parts"]) == exp["parts"] + system_instructions = _normalize_to_list( + attrs.get(gen_ai_attributes.GEN_AI_SYSTEM_INSTRUCTIONS, []) + ) + assert system_instructions == [ + {"content": "You are a helpful assistant!", "type": "text"} + ] + output_msgs = _normalize_to_list( attrs.get(gen_ai_attributes.GEN_AI_OUTPUT_MESSAGES, []) ) @@ -1141,12 +1276,6 @@ def assert_log_record_when_error(log_record, parent_span): attrs.get(gen_ai_attributes.GEN_AI_INPUT_MESSAGES, []) ) expected_input = [ - { - "parts": [ - {"content": "You are a helpful assistant!", "type": "text"} - ], - "role": "system", - }, { "parts": [ {"content": "What is the capital of France?", "type": "text"} @@ -1154,12 +1283,19 @@ def assert_log_record_when_error(log_record, parent_span): "role": "user", }, ] - assert len(input_msgs) == 2 + assert len(input_msgs) == 1 for i, exp in enumerate(expected_input): got = _normalize_to_dict(input_msgs[i]) assert got["role"] == exp["role"] assert _normalize_to_list(got["parts"]) == exp["parts"] + system_instructions = _normalize_to_list( + attrs.get(gen_ai_attributes.GEN_AI_SYSTEM_INSTRUCTIONS, []) + ) + assert system_instructions == [ + {"content": "You are a helpful assistant!", "type": "text"} + ] + assert gen_ai_attributes.GEN_AI_OUTPUT_MESSAGES not in attrs assert_log_parent(log_record, parent_span)