diff --git a/packages/uipath/pyproject.toml b/packages/uipath/pyproject.toml index 86bb4d18f..c4496ad9e 100644 --- a/packages/uipath/pyproject.toml +++ b/packages/uipath/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "uipath" -version = "2.14.14" +version = "2.14.15" description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools." readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.11" diff --git a/packages/uipath/src/uipath/eval/_helpers/evaluators_helpers.py b/packages/uipath/src/uipath/eval/_helpers/evaluators_helpers.py index 577c61ce7..f4a445048 100644 --- a/packages/uipath/src/uipath/eval/_helpers/evaluators_helpers.py +++ b/packages/uipath/src/uipath/eval/_helpers/evaluators_helpers.py @@ -634,14 +634,34 @@ def tool_calls_output_score( ), justifications -def trace_to_str(workload_trace: Sequence[ReadableSpan]) -> str: +def _format_workload_output(workload_output: dict[str, Any] | str) -> str: + """Render a workload's own output for the run-history string.""" + if isinstance(workload_output, str): + return workload_output + try: + return json.dumps(workload_output) + except (TypeError, ValueError): + return str(workload_output) + + +def trace_to_str( + workload_trace: Sequence[ReadableSpan], + workload_output: dict[str, Any] | str | None = None, +) -> str: """Convert OTEL spans to a platform-style workload run history string. Creates a similar structure to LangChain message processing but using OTEL spans. - Only processes tool spans (spans with 'tool.name' attribute). + Tool spans (spans with 'tool.name' attribute) are rendered as the trajectory; + the workload's own output is appended as a final "Agent Output" block. + + LLM spans are deliberately not rendered — they carry the full system prompt + and would swamp the judge's context. `workload_output` is the workload's + answer, so it is passed in rather than recovered from the trace. Args: workload_trace: List of ReadableSpan objects from the workload execution + workload_output: The workload's own output. Omit it to render the tool + calls alone. Returns: String representation of the workload run history in platform format @@ -705,4 +725,9 @@ def trace_to_str(workload_trace: Sequence[ReadableSpan]) -> str: platform_history.append(f"{tool_result}") platform_history.append("") + if workload_output is not None: + platform_history.append("Agent Output:") + platform_history.append(_format_workload_output(workload_output)) + platform_history.append("") + return "\n".join(platform_history) diff --git a/packages/uipath/src/uipath/eval/evaluators/legacy_trajectory_evaluator.py b/packages/uipath/src/uipath/eval/evaluators/legacy_trajectory_evaluator.py index 1368d9339..c7b531967 100644 --- a/packages/uipath/src/uipath/eval/evaluators/legacy_trajectory_evaluator.py +++ b/packages/uipath/src/uipath/eval/evaluators/legacy_trajectory_evaluator.py @@ -101,6 +101,7 @@ async def evaluate( evaluation_prompt = self._create_evaluation_prompt( expected_agent_behavior=workload_execution.expected_agent_behavior, agent_run_history=workload_execution.workload_trace, + workload_output=workload_execution.workload_output, ) llm_response = await self._get_llm_response(evaluation_prompt) @@ -113,6 +114,7 @@ def _create_evaluation_prompt( self, expected_agent_behavior: Any, agent_run_history: Any, + workload_output: dict[str, Any] | str | None = None, ) -> str: """Create the evaluation prompt for the LLM.""" # Validate that expected agent behavior is not empty @@ -134,12 +136,13 @@ def _create_evaluation_prompt( ) # Trim extra properties from the spans (such as timestamps which are not relevant to the eval) - if ( - isinstance(agent_run_history, list) - and agent_run_history - and isinstance(agent_run_history[0], ReadableSpan) - ): - agent_run_history = trace_to_str(agent_run_history) + is_span_trace = isinstance(agent_run_history, list) and ( + not agent_run_history or isinstance(agent_run_history[0], ReadableSpan) + ) + # A run with no tool spans still has an output to grade, so an empty + # trace goes through trace_to_str whenever there is one to append. + if is_span_trace and (agent_run_history or workload_output is not None): + agent_run_history = trace_to_str(agent_run_history, workload_output) else: agent_run_history = str(agent_run_history) diff --git a/packages/uipath/src/uipath/eval/evaluators/llm_judge_trajectory_evaluator.py b/packages/uipath/src/uipath/eval/evaluators/llm_judge_trajectory_evaluator.py index d8b16ea90..b6084f04f 100644 --- a/packages/uipath/src/uipath/eval/evaluators/llm_judge_trajectory_evaluator.py +++ b/packages/uipath/src/uipath/eval/evaluators/llm_judge_trajectory_evaluator.py @@ -80,7 +80,10 @@ async def evaluate( def _get_actual_output(self, workload_execution: WorkloadExecution) -> Any: """Get the actual output from the workload execution.""" - return trace_to_str(workload_execution.workload_trace) + return trace_to_str( + workload_execution.workload_trace, + workload_execution.workload_output, + ) def _get_expected_output( self, evaluation_criteria: TrajectoryEvaluationCriteria diff --git a/packages/uipath/tests/evaluators/test_evaluator_helpers.py b/packages/uipath/tests/evaluators/test_evaluator_helpers.py index 6064381cb..fa604ed3e 100644 --- a/packages/uipath/tests/evaluators/test_evaluator_helpers.py +++ b/packages/uipath/tests/evaluators/test_evaluator_helpers.py @@ -22,6 +22,7 @@ tool_calls_count_score, tool_calls_order_score, tool_calls_output_score, + trace_to_str, ) from uipath.eval.models.models import ToolCall, ToolOutput @@ -1198,3 +1199,72 @@ def test_count_score_id_keyed_expected_still_wins(self) -> None: expected = {"webSearch1": (">=", 1)} score, _ = tool_calls_count_score(actual, expected) assert score == 1.0 + + +class TestTraceToStrAgentOutput: + """``trace_to_str`` must carry the workload's own output, not just its tool calls. + + Regression guard for AE-2147: the trajectory judge reads the run through + ``{{AgentRunHistory}}`` only, so an output missing from this string is an + output the judge cannot grade. + """ + + @staticmethod + def _web_search_span() -> Any: + from opentelemetry.sdk.trace import ReadableSpan + + return ReadableSpan( + name="Web_Search", + start_time=1_756_233_000_000_000_000, + end_time=1_756_233_007_000_000_000, + attributes={ + "openinference.span.kind": "TOOL", + "tool.name": "Web_Search", + "input.value": "{}", + "output.value": "{}", + }, + ) + + def test_includes_workload_output(self) -> None: + history = trace_to_str( + [self._web_search_span()], + workload_output={ + "search_results_answer": "Argentina won the most recent FIFA World Cup." + }, + ) + + assert "Tool Call Response - Web_Search" in history + assert "Agent Output:" in history + assert "Argentina won the most recent FIFA World Cup." in history + + def test_output_comes_after_the_tool_calls(self) -> None: + history = trace_to_str( + [self._web_search_span()], + workload_output={"answer": "Argentina"}, + ) + + assert history.index("Tool Call Response") < history.index("Agent Output:") + + def test_string_output_is_rendered_verbatim(self) -> None: + history = trace_to_str([], workload_output="Argentina") + + assert history.strip() == "Agent Output:\nArgentina" + + def test_output_omitted_when_not_supplied(self) -> None: + history = trace_to_str([self._web_search_span()]) + + assert "Agent Output:" not in history + + def test_empty_output_is_still_reported(self) -> None: + history = trace_to_str([self._web_search_span()], workload_output={}) + + assert "Agent Output:\n{}" in history + + def test_unserialisable_output_falls_back_to_repr(self) -> None: + """A judge run must not die because an output holds a non-JSON value.""" + sentinel = object() + + history = trace_to_str([], workload_output={"handle": sentinel}) + + assert "Agent Output:" in history + assert repr(sentinel) in history diff --git a/packages/uipath/tests/evaluators/test_trajectory_agent_output.py b/packages/uipath/tests/evaluators/test_trajectory_agent_output.py new file mode 100644 index 000000000..5e53fe20b --- /dev/null +++ b/packages/uipath/tests/evaluators/test_trajectory_agent_output.py @@ -0,0 +1,196 @@ +"""Regression tests for AE-2147. + +The trajectory judges hand the run to the LLM through a single +``{{AgentRunHistory}}`` placeholder. That string was built from tool spans +alone, so the workload's own output never reached the judge and runs were +graded as if the agent had answered nothing. +""" + +import uuid +from typing import Any + +import pytest +from opentelemetry.sdk.trace import ReadableSpan + +from uipath.eval.evaluators import LegacyTrajectoryEvaluator +from uipath.eval.evaluators.base_legacy_evaluator import LegacyEvaluationCriteria +from uipath.eval.evaluators.legacy_trajectory_evaluator import ( + LegacyTrajectoryEvaluatorConfig, +) +from uipath.eval.evaluators.llm_judge_trajectory_evaluator import ( + LLMJudgeTrajectoryEvaluator, + TrajectoryEvaluationCriteria, +) +from uipath.eval.models.models import ( + LegacyEvaluatorCategory, + LegacyEvaluatorType, + WorkloadExecution, +) + +AGENT_ANSWER = "Argentina won the most recent FIFA World Cup (Qatar 2022)." + + +def _web_search_span() -> ReadableSpan: + """The one tool call from the AE-2147 repro run.""" + return ReadableSpan( + name="Web_Search", + start_time=1_756_233_000_000_000_000, + end_time=1_756_233_007_000_000_000, + attributes={ + "openinference.span.kind": "TOOL", + "tool.name": "Web_Search", + "input.value": "{}", + "output.value": "{}", + }, + ) + + +def _workload_execution() -> WorkloadExecution: + return WorkloadExecution( + agent_input={"query": "Who won the most recent soccer World Cup?"}, + workload_output={"search_results_answer": AGENT_ANSWER}, + workload_trace=[_web_search_span()], + expected_agent_behavior=( + "The agent should perform a web search and return a clear answer " + "identifying Argentina as the most recent winner." + ), + ) + + +def test_llm_judge_trajectory_prompt_contains_the_agent_output() -> None: + evaluator = LLMJudgeTrajectoryEvaluator.model_validate( + { + "id": str(uuid.uuid4()), + "evaluatorConfig": { + "name": "Default Trajectory Evaluator", + "prompt": ( + "ExpectedAgentBehavior:\n{{ExpectedAgentBehavior}}\n" + "AgentRunHistory:\n{{AgentRunHistory}}" + ), + "model": "gpt-4", + }, + } + ) + workload_execution = _workload_execution() + + prompt = evaluator._create_evaluation_prompt( + workload_execution, + TrajectoryEvaluationCriteria( + expected_agent_behavior=workload_execution.expected_agent_behavior or "" + ), + ) + + assert "Tool Call Response - Web_Search" in prompt + assert AGENT_ANSWER in prompt + + +def test_legacy_trajectory_prompt_contains_the_agent_output() -> None: + evaluator = LegacyTrajectoryEvaluator( + id=str(uuid.uuid4()), + name="Legacy trajectory", + config_type=LegacyTrajectoryEvaluatorConfig, + evaluation_criteria_type=LegacyEvaluationCriteria, + justification_type=str, + category=LegacyEvaluatorCategory.Trajectory, + type=LegacyEvaluatorType.Trajectory, + prompt="History:\n{{AgentRunHistory}}\nExpected:\n{{ExpectedAgentBehavior}}", + createdAt="2026-05-14T00:00:00Z", + updatedAt="2026-05-14T00:00:00Z", + ) + workload_execution = _workload_execution() + + prompt = evaluator._create_evaluation_prompt( + expected_agent_behavior=workload_execution.expected_agent_behavior, + agent_run_history=workload_execution.workload_trace, + workload_output=workload_execution.workload_output, + ) + + assert "Tool Call Response - Web_Search" in prompt + assert AGENT_ANSWER in prompt + + +@pytest.mark.asyncio +async def test_legacy_trajectory_evaluate_sends_the_agent_output_to_the_llm( + mocker: Any, +) -> None: + """The output has to survive the real ``evaluate`` path, not just the helper.""" + evaluator = LegacyTrajectoryEvaluator( + id=str(uuid.uuid4()), + name="Legacy trajectory", + config_type=LegacyTrajectoryEvaluatorConfig, + evaluation_criteria_type=LegacyEvaluationCriteria, + justification_type=str, + category=LegacyEvaluatorCategory.Trajectory, + type=LegacyEvaluatorType.Trajectory, + prompt="History:\n{{AgentRunHistory}}\nExpected:\n{{ExpectedAgentBehavior}}", + createdAt="2026-05-14T00:00:00Z", + updatedAt="2026-05-14T00:00:00Z", + ) + + sent_prompts: list[str] = [] + tool_call = mocker.MagicMock() + tool_call.arguments = {"score": 90, "justification": "answered correctly"} + response = mocker.MagicMock() + response.choices = [ + mocker.MagicMock(message=mocker.MagicMock(tool_calls=[tool_call])) + ] + + async def chat_completions(**kwargs: Any) -> Any: + sent_prompts.append(kwargs["messages"][0]["content"]) + return response + + evaluator.llm = mocker.MagicMock(chat_completions=chat_completions) + + result = await evaluator.evaluate( + _workload_execution(), + LegacyEvaluationCriteria.model_validate( + { + "expectedOutput": {}, + "expectedAgentBehavior": "The agent should identify Argentina.", + } + ), + ) + + assert result.score == 90 + assert AGENT_ANSWER in sent_prompts[0] + + +def _legacy_evaluator() -> LegacyTrajectoryEvaluator: + return LegacyTrajectoryEvaluator( + id=str(uuid.uuid4()), + name="Legacy trajectory", + config_type=LegacyTrajectoryEvaluatorConfig, + evaluation_criteria_type=LegacyEvaluationCriteria, + justification_type=str, + category=LegacyEvaluatorCategory.Trajectory, + type=LegacyEvaluatorType.Trajectory, + prompt="History:\n{{AgentRunHistory}}\nExpected:\n{{ExpectedAgentBehavior}}", + createdAt="2026-05-14T00:00:00Z", + updatedAt="2026-05-14T00:00:00Z", + ) + + +def test_legacy_trajectory_keeps_the_output_when_the_run_called_no_tools() -> None: + """An agent that answers without calling a tool has an empty span list. + + The span-list check used to require a first element, so those runs fell to + ``str([])`` and lost the output all over again. + """ + prompt = _legacy_evaluator()._create_evaluation_prompt( + expected_agent_behavior="The agent should identify Argentina.", + agent_run_history=[], + workload_output={"search_results_answer": AGENT_ANSWER}, + ) + + assert AGENT_ANSWER in prompt + + +def test_legacy_trajectory_empty_trace_without_output_stays_empty() -> None: + """No trace and no output is still rendered the way it always was.""" + prompt = _legacy_evaluator()._create_evaluation_prompt( + expected_agent_behavior="The agent should identify Argentina.", + agent_run_history=[], + workload_output=None, + ) + + assert "History:\n[]\n" in prompt diff --git a/packages/uipath/uv.lock b/packages/uipath/uv.lock index d62b3c81c..c26a79388 100644 --- a/packages/uipath/uv.lock +++ b/packages/uipath/uv.lock @@ -3,7 +3,7 @@ revision = 3 requires-python = ">=3.11" [options] -exclude-newer = "0001-01-01T00:00:00Z" # This has no effect and is included for backwards compatibility when using relative exclude-newer values. +exclude-newer = "2026-09-08T15:37:23.989122Z" exclude-newer-span = "P2D" [options.exclude-newer-package] @@ -2599,7 +2599,7 @@ wheels = [ [[package]] name = "uipath" -version = "2.14.14" +version = "2.14.15" source = { editable = "." } dependencies = [ { name = "applicationinsights" },