-
Notifications
You must be signed in to change notification settings - Fork 4.7k
fix: add run_in_parallel support to OutputGuardrail for sequential ex… #4791
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -1660,26 +1660,54 @@ async def _run_output_guardrails( | |
|
|
||
| triggered_results = [] | ||
|
|
||
| for guardrail in output_guardrails: | ||
| sequential_guardrails = [g for g in output_guardrails if not getattr(g, "run_in_parallel", True)] | ||
| parallel_guardrails = [g for g in output_guardrails if getattr(g, "run_in_parallel", True)] | ||
|
|
||
| async def _run_single(guardrail: Any) -> Any: | ||
| try: | ||
| result = await guardrail.run( | ||
| # TODO (rm) Remove this cast, it's wrong | ||
| self._context_wrapper, | ||
| cast(Agent[Any], source_agent), | ||
| text, | ||
| ) | ||
| if self._closing or self._closed: | ||
| return False | ||
| if result.output.tripwire_triggered: | ||
| triggered_results.append(result) | ||
| return result | ||
| except Exception as exc: | ||
| log_model_and_tool_action_warning( | ||
| logger, | ||
| "Output guardrail raised an exception; skipping it", | ||
| exc, | ||
| diagnostic_extra=partial(_guardrail_diagnostic_extra, guardrail), | ||
| ) | ||
| continue | ||
| return None | ||
|
|
||
| # Run sequential guardrails first | ||
| for guardrail in sequential_guardrails: | ||
| result = await _run_single(guardrail) | ||
| if self._closing or self._closed: | ||
| return False | ||
| if result and result.output.tripwire_triggered: | ||
| triggered_results.append(result) | ||
| break | ||
|
|
||
| # Run parallel guardrails only if sequential didn't trip | ||
| if not triggered_results and parallel_guardrails: | ||
| tasks = [asyncio.create_task(_run_single(g)) for g in parallel_guardrails] | ||
| try: | ||
| for done in asyncio.as_completed(tasks): | ||
| result = await done | ||
| if self._closing or self._closed: | ||
| return False | ||
| if result and result.output.tripwire_triggered: | ||
| triggered_results.append(result) | ||
| for t in tasks: | ||
| t.cancel() | ||
| break | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
When two default-parallel Realtime output guardrails both trip, this AGENTS.md reference: AGENTS.md:L147-L147 Useful? React with 👍 / 👎. |
||
| finally: | ||
| for t in tasks: | ||
| if not t.done(): | ||
| t.cancel() | ||
| await asyncio.gather(*tasks, return_exceptions=True) | ||
|
|
||
| if triggered_results: | ||
| # Double-check: bail if already interrupted for this response | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,48 @@ | ||
| import asyncio | ||
|
|
||
| import pytest | ||
|
|
||
| from agents.agent import Agent | ||
| from agents.guardrail import GuardrailFunctionOutput, output_guardrail | ||
| from agents.run_context import RunContextWrapper | ||
| from agents.run_internal.guardrails import run_output_guardrails | ||
|
|
||
|
|
||
| @pytest.mark.asyncio | ||
| async def test_output_guardrail_sequential_execution(): | ||
| execution_order = [] | ||
|
|
||
| @output_guardrail(run_in_parallel=False, name="seq1") | ||
| async def seq_guardrail_1(ctx, agent, output): | ||
| await asyncio.sleep(0.05) | ||
| execution_order.append("seq1") | ||
| return GuardrailFunctionOutput(tripwire_triggered=False, output_info=None) | ||
|
|
||
| @output_guardrail(run_in_parallel=False, name="seq2") | ||
| async def seq_guardrail_2(ctx, agent, output): | ||
| execution_order.append("seq2") | ||
| return GuardrailFunctionOutput(tripwire_triggered=False, output_info=None) | ||
|
|
||
| @output_guardrail(run_in_parallel=True, name="par1") | ||
| async def par_guardrail_1(ctx, agent, output): | ||
| await asyncio.sleep(0.02) | ||
| execution_order.append("par1") | ||
| return GuardrailFunctionOutput(tripwire_triggered=False, output_info=None) | ||
|
|
||
| @output_guardrail(run_in_parallel=True, name="par2") | ||
| async def par_guardrail_2(ctx, agent, output): | ||
| execution_order.append("par2") | ||
| return GuardrailFunctionOutput(tripwire_triggered=False, output_info=None) | ||
|
|
||
| agent = Agent(name="test") | ||
| ctx = RunContextWrapper(context=None) | ||
| guardrails = [par_guardrail_1, seq_guardrail_1, par_guardrail_2, seq_guardrail_2] | ||
|
|
||
| results = await run_output_guardrails( | ||
| guardrails=guardrails, agent=agent, agent_output="test_output", context=ctx, results_sink=[] | ||
| ) | ||
|
|
||
| assert len(results) == 4 | ||
| # Sequential ones should run first and in order, then parallel ones run concurrently | ||
| # par2 will finish before par1 because par1 sleeps | ||
| assert execution_order == ["seq1", "seq2", "par2", "par1"] |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
When these public guardrails are supplied through
RealtimeAgent.output_guardrailsorRealtimeRunConfig.output_guardrails,RealtimeSession._run_output_guardrailsstill awaits every guardrail in list order without consultingrun_in_parallel(src/agents/realtime/session.py:1663-1669). Consequently, with an expensive default-parallel guardrail listed before a cheaprun_in_parallel=Falseguardrail, the expensive check runs first and cannot be skipped when the cheap check trips—the exact cost-saving behavior this option promises works inRunnerbut is silently ignored on the supported Realtime path. Partition or otherwise honor the flag in that consumer as well.AGENTS.md reference: AGENTS.md:L147-L147
Useful? React with 👍 / 👎.