From 3ba482ba91eda4419468143087ffad394f4a5b1d Mon Sep 17 00:00:00 2001 From: Zening Chen Date: Mon, 31 Aug 2026 08:26:43 +0000 Subject: [PATCH 1/3] langchain: divert SystemMessage to gen_ai.system_instructions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `on_chat_model_start` used to fold every ``SystemMessage`` into ``gen_ai.input.messages`` as ``role="system"`` while leaving ``gen_ai.system_instructions`` empty, so the semconv system-instruction attribute never appeared on chat spans. Split the flattened message list once — system messages populate ``InferenceInvocation.system_instruction`` via a new ``split_system_and_input_messages`` helper, non-system messages pass through ``to_input_messages`` unchanged. ``SystemMessageChunk`` is handled via ``isinstance``. --- .../genai/langchain/callback_handler.py | 14 +- .../instrumentation/genai/langchain/utils.py | 38 ++++ .../tests/test_llm_call.py | 164 ++++++++++++++++-- 3 files changed, 192 insertions(+), 24 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py index c41f3a386..23f0d402c 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/callback_handler.py @@ -29,7 +29,7 @@ make_last_output_message, normalize_provider, prepare_tool_definitions, - to_input_messages, + split_system_and_input_messages, ) from opentelemetry.util.genai.handler import TelemetryHandler from opentelemetry.util.genai.invocation import ( @@ -265,18 +265,20 @@ def on_chat_model_start( if "ls_max_tokens" in metadata: max_tokens = metadata.get("ls_max_tokens") - # ``messages`` from on_chat_model_start is ``list[list[BaseMessage]]`` - # (one inner list per generation request). Flatten and let - # :func:`to_input_messages` produce spec-conformant ``InputMessage`` s - # with proper roles, tool-call requests, tool results, and reasoning. + # Flatten ``list[list[BaseMessage]]`` (one inner list per generation + # request) before splitting into system / input. flattened: list[BaseMessage] = [msg for sub in messages for msg in sub] - input_messages = to_input_messages(flattened) + system_instruction, input_messages = split_system_and_input_messages( + flattened + ) llm_invocation = self._telemetry_handler.inference( provider, request_model=request_model, ) llm_invocation.input_messages = input_messages + if system_instruction: + llm_invocation.system_instruction = system_instruction llm_invocation.top_p = top_p llm_invocation.frequency_penalty = frequency_penalty llm_invocation.presence_penalty = presence_penalty diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/utils.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/utils.py index ef726f1ac..b1b560e3f 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/utils.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/utils.py @@ -11,6 +11,7 @@ from langchain_core.messages import ( AIMessage, BaseMessage, + SystemMessage, ToolMessage, convert_to_messages, ) @@ -205,6 +206,43 @@ def to_input_messages( return result +def to_system_instruction( + messages: Iterable[Any], +) -> list[MessagePart]: + """Extract ``MessagePart`` s from ``SystemMessage`` s for ``gen_ai.system_instructions``.""" + try: + normalized_messages: Iterable[BaseMessage] = convert_to_messages( + list(messages) + ) + except Exception: # pylint: disable=broad-except + normalized_messages = [ + m for m in messages if isinstance(m, BaseMessage) + ] + parts: list[MessagePart] = [] + for message in normalized_messages: + if isinstance(message, SystemMessage): + parts.extend(_content_to_parts(message.content)) + return parts + + +def split_system_and_input_messages( + messages: Iterable[Any], +) -> tuple[list[MessagePart], list[InputMessage]]: + """Split ``messages`` into ``system_instruction`` parts and ``InputMessage`` s.""" + try: + normalized: Iterable[BaseMessage] = convert_to_messages(list(messages)) + except Exception: # pylint: disable=broad-except + normalized = [m for m in messages if isinstance(m, BaseMessage)] + system: list[BaseMessage] = [] + non_system: list[BaseMessage] = [] + for message in normalized: + if isinstance(message, SystemMessage): + system.append(message) + else: + non_system.append(message) + return to_system_instruction(system), to_input_messages(non_system) + + def to_output_messages( messages: Iterable[BaseMessage], *, diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py index 8529a061b..c21b5bfea 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/tests/test_llm_call.py @@ -7,15 +7,19 @@ import pytest from langchain_anthropic import ChatAnthropic from langchain_core.messages import ( + AIMessage, FunctionMessage, HumanMessage, SystemMessage, + SystemMessageChunk, ) from langchain_core.tools import tool from openai import AuthenticationError from opentelemetry.instrumentation.genai.langchain.utils import ( + split_system_and_input_messages, to_input_messages, + to_system_instruction, ) from opentelemetry.sdk.trace import ReadableSpan from opentelemetry.semconv._incubating.attributes import ( @@ -402,6 +406,112 @@ def test_function_message_role_maps_to_tool(): assert result[0].role == "tool" +def test_split_system_and_input_messages_diverts_system_instructions(): + system, inputs = split_system_and_input_messages( + [ + SystemMessage(content="You are helpful."), + HumanMessage(content="Hi"), + ] + ) + assert len(system) == 1 + assert system[0].content == "You are helpful." + assert system[0].type == "text" + assert len(inputs) == 1 + assert inputs[0].role == "user" + + +def test_split_system_and_input_messages_position_independent(): + # A ``SystemMessage`` interleaved between non-system messages still lands + # in ``system_instruction`` — semconv treats it as a top-level list rather + # than an interleaved role, and combining multiple system chunks in order + # matches how providers deliver them. + system, inputs = split_system_and_input_messages( + [ + SystemMessage(content="First guidance."), + HumanMessage(content="Hi"), + AIMessage(content="Hello"), + SystemMessage(content="Second guidance."), + HumanMessage(content="Follow-up"), + ] + ) + assert [part.content for part in system] == [ + "First guidance.", + "Second guidance.", + ] + assert [msg.role for msg in inputs] == ["user", "assistant", "user"] + + +def test_split_system_and_input_messages_handles_system_message_chunk(): + # ``SystemMessageChunk`` (streaming) subclasses ``SystemMessage`` — the + # helper matches by ``isinstance`` so streamed system content is diverted + # too instead of leaking into ``gen_ai.input.messages``. + system, inputs = split_system_and_input_messages( + [ + SystemMessageChunk(content="Streamed system prompt."), + HumanMessage(content="Hi"), + ] + ) + assert len(system) == 1 + assert system[0].content == "Streamed system prompt." + assert [msg.role for msg in inputs] == ["user"] + + +def test_split_system_and_input_messages_empty_when_no_system_message(): + system, inputs = split_system_and_input_messages( + [HumanMessage(content="Hi")] + ) + assert system == [] + assert [msg.role for msg in inputs] == ["user"] + + +def test_split_system_and_input_messages_normalizes_shorthand_inputs(): + # Short-hand tuple / dict forms are normalized via ``convert_to_messages`` + # before the ``isinstance`` partition so that a short-hand system entry is + # diverted to ``system_instruction`` instead of leaking into + # ``gen_ai.input.messages`` as ``role: "system"``. + system, inputs = split_system_and_input_messages( + [ + ("system", "You are helpful."), + {"role": "user", "content": "Hi"}, + ] + ) + assert [part.content for part in system] == ["You are helpful."] + assert [msg.role for msg in inputs] == ["user"] + + +def test_to_system_instruction_ignores_non_system_messages(): + # Symmetric to ``to_input_messages``: ``to_system_instruction`` only + # emits parts from ``SystemMessage`` s and silently drops everything else. + parts = to_system_instruction( + [ + HumanMessage(content="Hi"), + SystemMessage(content="You are helpful."), + AIMessage(content="Hello"), + SystemMessageChunk(content="More guidance."), + ] + ) + assert [part.content for part in parts] == [ + "You are helpful.", + "More guidance.", + ] + + +def test_to_system_instruction_normalizes_shorthand_inputs(): + # Accepts ``("system", "…")`` tuples and ``{"role": "system", …}`` dicts + # via ``convert_to_messages``, matching ``to_input_messages`` behavior. + parts = to_system_instruction( + [ + ("system", "First guidance."), + {"role": "user", "content": "Hi"}, + {"role": "system", "content": "Second guidance."}, + ] + ) + assert [part.content for part in parts] == [ + "First guidance.", + "Second guidance.", + ] + + def assert_openai_completion_attributes( span: ReadableSpan, response: Optional, verify_content: bool = True ): @@ -465,11 +575,18 @@ def assert_openai_completion_attributes( if verify_content: input_message = attributes[gen_ai_attributes.GEN_AI_INPUT_MESSAGES] assert input_message is not None - assert '"role":"system"' in input_message - assert '"content":"You are a helpful assistant!"' in input_message + assert '"role":"system"' not in input_message assert '"role":"user"' in input_message assert '"content":"What is the capital of France?"' in input_message + system_instructions = attributes[ + gen_ai_attributes.GEN_AI_SYSTEM_INSTRUCTIONS + ] + assert system_instructions is not None + assert ( + '"content":"You are a helpful assistant!"' in system_instructions + ) + # Assert output message output_message = attributes[gen_ai_attributes.GEN_AI_OUTPUT_MESSAGES] assert output_message is not None @@ -479,6 +596,7 @@ def assert_openai_completion_attributes( else: assert gen_ai_attributes.GEN_AI_INPUT_MESSAGES not in attributes assert gen_ai_attributes.GEN_AI_OUTPUT_MESSAGES not in attributes + assert gen_ai_attributes.GEN_AI_SYSTEM_INSTRUCTIONS not in attributes def assert_openai_completion_attributes_with_error( @@ -517,16 +635,24 @@ def assert_openai_completion_attributes_with_error( if verify_content: input_message = attributes[gen_ai_attributes.GEN_AI_INPUT_MESSAGES] assert input_message is not None - assert '"role":"system"' in input_message - assert '"content":"You are a helpful assistant!"' in input_message + assert '"role":"system"' not in input_message assert '"role":"user"' in input_message assert '"content":"What is the capital of France?"' in input_message + system_instructions = attributes[ + gen_ai_attributes.GEN_AI_SYSTEM_INSTRUCTIONS + ] + assert system_instructions is not None + assert ( + '"content":"You are a helpful assistant!"' in system_instructions + ) + # Assert output message assert gen_ai_attributes.GEN_AI_OUTPUT_MESSAGES not in attributes else: assert gen_ai_attributes.GEN_AI_INPUT_MESSAGES not in attributes assert gen_ai_attributes.GEN_AI_OUTPUT_MESSAGES not in attributes + assert gen_ai_attributes.GEN_AI_SYSTEM_INSTRUCTIONS not in attributes def assert_bedrock_completion_attributes( @@ -765,12 +891,6 @@ def assert_log_record(log_record, parent_span, response=None): attrs.get(gen_ai_attributes.GEN_AI_INPUT_MESSAGES, []) ) expected_input = [ - { - "parts": [ - {"content": "You are a helpful assistant!", "type": "text"} - ], - "role": "system", - }, { "parts": [ {"content": "What is the capital of France?", "type": "text"} @@ -778,12 +898,19 @@ def assert_log_record(log_record, parent_span, response=None): "role": "user", }, ] - assert len(input_msgs) == 2 + assert len(input_msgs) == 1 for i, exp in enumerate(expected_input): got = _normalize_to_dict(input_msgs[i]) assert got["role"] == exp["role"] assert _normalize_to_list(got["parts"]) == exp["parts"] + system_instructions = _normalize_to_list( + attrs.get(gen_ai_attributes.GEN_AI_SYSTEM_INSTRUCTIONS, []) + ) + assert system_instructions == [ + {"content": "You are a helpful assistant!", "type": "text"} + ] + output_msgs = _normalize_to_list( attrs.get(gen_ai_attributes.GEN_AI_OUTPUT_MESSAGES, []) ) @@ -829,12 +956,6 @@ def assert_log_record_when_error(log_record, parent_span): attrs.get(gen_ai_attributes.GEN_AI_INPUT_MESSAGES, []) ) expected_input = [ - { - "parts": [ - {"content": "You are a helpful assistant!", "type": "text"} - ], - "role": "system", - }, { "parts": [ {"content": "What is the capital of France?", "type": "text"} @@ -842,12 +963,19 @@ def assert_log_record_when_error(log_record, parent_span): "role": "user", }, ] - assert len(input_msgs) == 2 + assert len(input_msgs) == 1 for i, exp in enumerate(expected_input): got = _normalize_to_dict(input_msgs[i]) assert got["role"] == exp["role"] assert _normalize_to_list(got["parts"]) == exp["parts"] + system_instructions = _normalize_to_list( + attrs.get(gen_ai_attributes.GEN_AI_SYSTEM_INSTRUCTIONS, []) + ) + assert system_instructions == [ + {"content": "You are a helpful assistant!", "type": "text"} + ] + assert gen_ai_attributes.GEN_AI_OUTPUT_MESSAGES not in attrs assert_log_parent(log_record, parent_span) From 60333abad48fc5b79c230950e26d3d05b6d8ffff Mon Sep 17 00:00:00 2001 From: Zening Chen Date: Mon, 31 Aug 2026 08:27:22 +0000 Subject: [PATCH 2/3] langchain: add changelog fragment for #511 --- .../.changelog/511.fixed | 1 + 1 file changed, 1 insertion(+) create mode 100644 instrumentation/opentelemetry-instrumentation-genai-langchain/.changelog/511.fixed diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/.changelog/511.fixed b/instrumentation/opentelemetry-instrumentation-genai-langchain/.changelog/511.fixed new file mode 100644 index 000000000..ca6b58e10 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/.changelog/511.fixed @@ -0,0 +1 @@ +Divert `SystemMessage` inputs to `gen_ai.system_instructions` instead of emitting them as `role="system"` inside `gen_ai.input.messages`. From 0f350581b75c7ee6339ea4e5af9035540165cb9b Mon Sep 17 00:00:00 2001 From: Zening Chen Date: Mon, 31 Aug 2026 09:49:21 +0000 Subject: [PATCH 3/3] langchain: materialize messages before convert_to_messages try/except Move ``list(messages)`` outside the try so the fallback in ``except`` still sees the input if ``convert_to_messages`` raises. Without this, a one-shot iterator would be consumed by ``list(messages)`` inside the try and the except branch would silently drop all content. --- .../instrumentation/genai/langchain/utils.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/utils.py b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/utils.py index 31344c73a..4281c2628 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/utils.py +++ b/instrumentation/opentelemetry-instrumentation-genai-langchain/src/opentelemetry/instrumentation/genai/langchain/utils.py @@ -222,13 +222,14 @@ def to_system_instruction( messages: Iterable[Any], ) -> list[MessagePart]: """Extract ``MessagePart`` s from ``SystemMessage`` s for ``gen_ai.system_instructions``.""" + materialized = list(messages) try: normalized_messages: Iterable[BaseMessage] = convert_to_messages( - list(messages) + materialized ) except Exception: # pylint: disable=broad-except normalized_messages = [ - m for m in messages if isinstance(m, BaseMessage) + m for m in materialized if isinstance(m, BaseMessage) ] parts: list[MessagePart] = [] for message in normalized_messages: @@ -241,10 +242,11 @@ def split_system_and_input_messages( messages: Iterable[Any], ) -> tuple[list[MessagePart], list[InputMessage]]: """Split ``messages`` into ``system_instruction`` parts and ``InputMessage`` s.""" + materialized = list(messages) try: - normalized: Iterable[BaseMessage] = convert_to_messages(list(messages)) + normalized: Iterable[BaseMessage] = convert_to_messages(materialized) except Exception: # pylint: disable=broad-except - normalized = [m for m in messages if isinstance(m, BaseMessage)] + normalized = [m for m in materialized if isinstance(m, BaseMessage)] system: list[BaseMessage] = [] non_system: list[BaseMessage] = [] for message in normalized: