fix(evals): include agent output in AgentRunHistory [AE-2147]

packages/uipath/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "uipath"
-version = "2.14.14"
+version = "2.14.15"
 description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools."
 readme = { file = "README.md", content-type = "text/markdown" }
 requires-python = ">=3.11"

packages/uipath/src/uipath/eval/_helpers/evaluators_helpers.py

@@ -634,14 +634,34 @@
     ), justifications
 
 
-def trace_to_str(workload_trace: Sequence[ReadableSpan]) -> str:
+def _format_workload_output(workload_output: dict[str, Any] | str) -> str:
+    """Render a workload's own output for the run-history string."""
+    if isinstance(workload_output, str):
+        return workload_output
+    try:
+        return json.dumps(workload_output)
+    except (TypeError, ValueError):
+        return str(workload_output)
+
+
+def trace_to_str(
+    workload_trace: Sequence[ReadableSpan],
+    workload_output: dict[str, Any] | str | None = None,
+) -> str:
     """Convert OTEL spans to a platform-style workload run history string.
 
     Creates a similar structure to LangChain message processing but using OTEL spans.
-    Only processes tool spans (spans with 'tool.name' attribute).
+    Tool spans (spans with 'tool.name' attribute) are rendered as the trajectory;
+    the workload's own output is appended as a final "Agent Output" block.
+
+    LLM spans are deliberately not rendered — they carry the full system prompt
+    and would swamp the judge's context. `workload_output` is the workload's
+    answer, so it is passed in rather than recovered from the trace.
 
     Args:
         workload_trace: List of ReadableSpan objects from the workload execution
+        workload_output: The workload's own output. Omit it to render the tool
+            calls alone.
 
     Returns:
         String representation of the workload run history in platform format
@@ -705,4 +725,9 @@
             platform_history.append(f"{tool_result}")
             platform_history.append("")
 
+    if workload_output is not None:
+        platform_history.append("Agent Output:")
+        platform_history.append(_format_workload_output(workload_output))
+        platform_history.append("")
+
     return "\n".join(platform_history)

packages/uipath/src/uipath/eval/evaluators/legacy_trajectory_evaluator.py

@@ -101,6 +101,7 @@ async def evaluate(
         evaluation_prompt = self._create_evaluation_prompt(
             expected_agent_behavior=workload_execution.expected_agent_behavior,
             agent_run_history=workload_execution.workload_trace,
+            workload_output=workload_execution.workload_output,
         )
         llm_response = await self._get_llm_response(evaluation_prompt)
 
@@ -113,6 +114,7 @@ def _create_evaluation_prompt(
         self,
         expected_agent_behavior: Any,
         agent_run_history: Any,
+        workload_output: dict[str, Any] | str | None = None,
     ) -> str:
         """Create the evaluation prompt for the LLM."""
         # Validate that expected agent behavior is not empty
@@ -134,12 +136,13 @@ def _create_evaluation_prompt(
         )
 
         # Trim extra properties from the spans (such as timestamps which are not relevant to the eval)
-        if (
-            isinstance(agent_run_history, list)
-            and agent_run_history
-            and isinstance(agent_run_history[0], ReadableSpan)
-        ):
-            agent_run_history = trace_to_str(agent_run_history)
+        is_span_trace = isinstance(agent_run_history, list) and (
+            not agent_run_history or isinstance(agent_run_history[0], ReadableSpan)
+        )
+        # A run with no tool spans still has an output to grade, so an empty
+        # trace goes through trace_to_str whenever there is one to append.
+        if is_span_trace and (agent_run_history or workload_output is not None):
+            agent_run_history = trace_to_str(agent_run_history, workload_output)
         else:
             agent_run_history = str(agent_run_history)
 

packages/uipath/src/uipath/eval/evaluators/llm_judge_trajectory_evaluator.py

@@ -80,7 +80,10 @@ async def evaluate(
 
     def _get_actual_output(self, workload_execution: WorkloadExecution) -> Any:
         """Get the actual output from the workload execution."""
-        return trace_to_str(workload_execution.workload_trace)
+        return trace_to_str(
+            workload_execution.workload_trace,
+            workload_execution.workload_output,
+        )
 
     def _get_expected_output(
         self, evaluation_criteria: TrajectoryEvaluationCriteria

packages/uipath/tests/evaluators/test_evaluator_helpers.py

@@ -22,6 +22,7 @@
     tool_calls_count_score,
     tool_calls_order_score,
     tool_calls_output_score,
+    trace_to_str,
 )
 from uipath.eval.models.models import ToolCall, ToolOutput
 
@@ -1198,3 +1199,72 @@ def test_count_score_id_keyed_expected_still_wins(self) -> None:
         expected = {"webSearch1": (">=", 1)}
         score, _ = tool_calls_count_score(actual, expected)
         assert score == 1.0
+
+
+class TestTraceToStrAgentOutput:
+    """``trace_to_str`` must carry the workload's own output, not just its tool calls.
+
+    Regression guard for AE-2147: the trajectory judge reads the run through
+    ``{{AgentRunHistory}}`` only, so an output missing from this string is an
+    output the judge cannot grade.
+    """
+
+    @staticmethod
+    def _web_search_span() -> Any:
+        from opentelemetry.sdk.trace import ReadableSpan
+
+        return ReadableSpan(
+            name="Web_Search",
+            start_time=1_756_233_000_000_000_000,
+            end_time=1_756_233_007_000_000_000,
+            attributes={
+                "openinference.span.kind": "TOOL",
+                "tool.name": "Web_Search",
+                "input.value": "{}",
+                "output.value": "{}",
+            },
+        )
+
+    def test_includes_workload_output(self) -> None:
+        history = trace_to_str(
+            [self._web_search_span()],
+            workload_output={
+                "search_results_answer": "Argentina won the most recent FIFA World Cup."
+            },
+        )
+
+        assert "Tool Call Response - Web_Search" in history
+        assert "Agent Output:" in history
+        assert "Argentina won the most recent FIFA World Cup." in history
+
+    def test_output_comes_after_the_tool_calls(self) -> None:
+        history = trace_to_str(
+            [self._web_search_span()],
+            workload_output={"answer": "Argentina"},
+        )
+
+        assert history.index("Tool Call Response") < history.index("Agent Output:")
+
+    def test_string_output_is_rendered_verbatim(self) -> None:
+        history = trace_to_str([], workload_output="Argentina")
+
+        assert history.strip() == "Agent Output:\nArgentina"
+
+    def test_output_omitted_when_not_supplied(self) -> None:
+        history = trace_to_str([self._web_search_span()])
+
+        assert "Agent Output:" not in history
+
+    def test_empty_output_is_still_reported(self) -> None:
+        history = trace_to_str([self._web_search_span()], workload_output={})
+
+        assert "Agent Output:\n{}" in history
+
+    def test_unserialisable_output_falls_back_to_repr(self) -> None:
+        """A judge run must not die because an output holds a non-JSON value."""
+        sentinel = object()
+
+        history = trace_to_str([], workload_output={"handle": sentinel})
+
+        assert "Agent Output:" in history
+        assert repr(sentinel) in history

packages/uipath/tests/evaluators/test_trajectory_agent_output.py

@@ -0,0 +1,196 @@
+"""Regression tests for AE-2147.
+
+The trajectory judges hand the run to the LLM through a single
+``{{AgentRunHistory}}`` placeholder. That string was built from tool spans
+alone, so the workload's own output never reached the judge and runs were
+graded as if the agent had answered nothing.
+"""
+
+import uuid
+from typing import Any
+
+import pytest
+from opentelemetry.sdk.trace import ReadableSpan
+
+from uipath.eval.evaluators import LegacyTrajectoryEvaluator
+from uipath.eval.evaluators.base_legacy_evaluator import LegacyEvaluationCriteria
+from uipath.eval.evaluators.legacy_trajectory_evaluator import (
+    LegacyTrajectoryEvaluatorConfig,
+)
+from uipath.eval.evaluators.llm_judge_trajectory_evaluator import (
+    LLMJudgeTrajectoryEvaluator,
+    TrajectoryEvaluationCriteria,
+)
+from uipath.eval.models.models import (
+    LegacyEvaluatorCategory,
+    LegacyEvaluatorType,
+    WorkloadExecution,
+)
+
+AGENT_ANSWER = "Argentina won the most recent FIFA World Cup (Qatar 2022)."
+
+
+def _web_search_span() -> ReadableSpan:
+    """The one tool call from the AE-2147 repro run."""
+    return ReadableSpan(
+        name="Web_Search",
+        start_time=1_756_233_000_000_000_000,
+        end_time=1_756_233_007_000_000_000,
+        attributes={
+            "openinference.span.kind": "TOOL",
+            "tool.name": "Web_Search",
+            "input.value": "{}",
+            "output.value": "{}",
+        },
+    )
+
+
+def _workload_execution() -> WorkloadExecution:
+    return WorkloadExecution(
+        agent_input={"query": "Who won the most recent soccer World Cup?"},
+        workload_output={"search_results_answer": AGENT_ANSWER},
+        workload_trace=[_web_search_span()],
+        expected_agent_behavior=(
+            "The agent should perform a web search and return a clear answer "
+            "identifying Argentina as the most recent winner."
+        ),
+    )
+
+
+def test_llm_judge_trajectory_prompt_contains_the_agent_output() -> None:
+    evaluator = LLMJudgeTrajectoryEvaluator.model_validate(
+        {
+            "id": str(uuid.uuid4()),
+            "evaluatorConfig": {
+                "name": "Default Trajectory Evaluator",
+                "prompt": (
+                    "ExpectedAgentBehavior:\n{{ExpectedAgentBehavior}}\n"
+                    "AgentRunHistory:\n{{AgentRunHistory}}"
+                ),
+                "model": "gpt-4",
+            },
+        }
+    )
+    workload_execution = _workload_execution()
+
+    prompt = evaluator._create_evaluation_prompt(
+        workload_execution,
+        TrajectoryEvaluationCriteria(
+            expected_agent_behavior=workload_execution.expected_agent_behavior or ""
+        ),
+    )
+
+    assert "Tool Call Response - Web_Search" in prompt
+    assert AGENT_ANSWER in prompt
+
+
+def test_legacy_trajectory_prompt_contains_the_agent_output() -> None:
+    evaluator = LegacyTrajectoryEvaluator(
+        id=str(uuid.uuid4()),
+        name="Legacy trajectory",
+        config_type=LegacyTrajectoryEvaluatorConfig,
+        evaluation_criteria_type=LegacyEvaluationCriteria,
+        justification_type=str,
+        category=LegacyEvaluatorCategory.Trajectory,
+        type=LegacyEvaluatorType.Trajectory,
+        prompt="History:\n{{AgentRunHistory}}\nExpected:\n{{ExpectedAgentBehavior}}",
+        createdAt="2026-05-14T00:00:00Z",
+        updatedAt="2026-05-14T00:00:00Z",
+    )
+    workload_execution = _workload_execution()
+
+    prompt = evaluator._create_evaluation_prompt(
+        expected_agent_behavior=workload_execution.expected_agent_behavior,
+        agent_run_history=workload_execution.workload_trace,
+        workload_output=workload_execution.workload_output,
+    )
+
+    assert "Tool Call Response - Web_Search" in prompt
+    assert AGENT_ANSWER in prompt
+
+
+@pytest.mark.asyncio
+async def test_legacy_trajectory_evaluate_sends_the_agent_output_to_the_llm(
+    mocker: Any,
+) -> None:
+    """The output has to survive the real ``evaluate`` path, not just the helper."""
+    evaluator = LegacyTrajectoryEvaluator(
+        id=str(uuid.uuid4()),
+        name="Legacy trajectory",
+        config_type=LegacyTrajectoryEvaluatorConfig,
+        evaluation_criteria_type=LegacyEvaluationCriteria,
+        justification_type=str,
+        category=LegacyEvaluatorCategory.Trajectory,
+        type=LegacyEvaluatorType.Trajectory,
+        prompt="History:\n{{AgentRunHistory}}\nExpected:\n{{ExpectedAgentBehavior}}",
+        createdAt="2026-05-14T00:00:00Z",
+        updatedAt="2026-05-14T00:00:00Z",
+    )
+
+    sent_prompts: list[str] = []
+    tool_call = mocker.MagicMock()
+    tool_call.arguments = {"score": 90, "justification": "answered correctly"}
+    response = mocker.MagicMock()
+    response.choices = [
+        mocker.MagicMock(message=mocker.MagicMock(tool_calls=[tool_call]))
+    ]
+
+    async def chat_completions(**kwargs: Any) -> Any:
+        sent_prompts.append(kwargs["messages"][0]["content"])
+        return response
+
+    evaluator.llm = mocker.MagicMock(chat_completions=chat_completions)
+
+    result = await evaluator.evaluate(
+        _workload_execution(),
+        LegacyEvaluationCriteria.model_validate(
+            {
+                "expectedOutput": {},
+                "expectedAgentBehavior": "The agent should identify Argentina.",
+            }
+        ),
+    )
+
+    assert result.score == 90
+    assert AGENT_ANSWER in sent_prompts[0]
+
+
+def _legacy_evaluator() -> LegacyTrajectoryEvaluator:
+    return LegacyTrajectoryEvaluator(
+        id=str(uuid.uuid4()),
+        name="Legacy trajectory",
+        config_type=LegacyTrajectoryEvaluatorConfig,
+        evaluation_criteria_type=LegacyEvaluationCriteria,
+        justification_type=str,
+        category=LegacyEvaluatorCategory.Trajectory,
+        type=LegacyEvaluatorType.Trajectory,
+        prompt="History:\n{{AgentRunHistory}}\nExpected:\n{{ExpectedAgentBehavior}}",
+        createdAt="2026-05-14T00:00:00Z",
+        updatedAt="2026-05-14T00:00:00Z",
+    )
+
+
+def test_legacy_trajectory_keeps_the_output_when_the_run_called_no_tools() -> None:
+    """An agent that answers without calling a tool has an empty span list.
+
+    The span-list check used to require a first element, so those runs fell to
+    ``str([])`` and lost the output all over again.
+    """
+    prompt = _legacy_evaluator()._create_evaluation_prompt(
+        expected_agent_behavior="The agent should identify Argentina.",
+        agent_run_history=[],
+        workload_output={"search_results_answer": AGENT_ANSWER},
+    )
+
+    assert AGENT_ANSWER in prompt
+
+
+def test_legacy_trajectory_empty_trace_without_output_stays_empty() -> None:
+    """No trace and no output is still rendered the way it always was."""
+    prompt = _legacy_evaluator()._create_evaluation_prompt(
+        expected_agent_behavior="The agent should identify Argentina.",
+        agent_run_history=[],
+        workload_output=None,
+    )
+
+    assert "History:\n[]\n" in prompt