From 8fa86033bc555d8ed02b77b4e09f3eec42e1392e Mon Sep 17 00:00:00 2001 From: "nick.yi" Date: Wed, 28 Oct 2026 15:33:44 +0800 Subject: [PATCH] feat(agent-testing): let output assertions and llmJudge read rich content - Some replies keep their substance only in the rich message (e.g. an upload card's photo and video lists), so text-only assertions could not check them. The driver now also records the rich message as JSON. - Opt-in through target "richContent"; the default still reads the reply text, so button titles never leak into existing outputNotContains checks. - An unknown target fails the assertion instead of falling back to the reply text. Co-Authored-By: Claude Opus 5.5 --- .../Models/AgentTestCaseResult.cs | 4 ++ .../Services/AgentTestCaseRunner.cs | 9 ++- .../Services/AssertionContext.cs | 4 ++ .../Services/AssertionEvaluator.cs | 58 +++++++++++++++++-- .../BotSharpAgentConversationDriver.cs | 34 ++++++++++- .../Services/IAgentConversationDriver.cs | 11 +++- .../Services/LlmAgentTestJudge.cs | 16 +++-- .../AgentTesting/AgentTestCaseRunnerTests.cs | 4 +- .../AgentTesting/AgentTestJudgeTests.cs | 4 +- .../AgentTesting/AssertionEvaluatorTests.cs | 29 ++++++++++ 10 files changed, 152 insertions(+), 21 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.AgentTesting/Models/AgentTestCaseResult.cs b/src/Plugins/BotSharp.Plugin.AgentTesting/Models/AgentTestCaseResult.cs index 2b04b3684..d5ebdcce7 100644 --- a/src/Plugins/BotSharp.Plugin.AgentTesting/Models/AgentTestCaseResult.cs +++ b/src/Plugins/BotSharp.Plugin.AgentTesting/Models/AgentTestCaseResult.cs @@ -97,6 +97,10 @@ public class TurnResult public int Index { get; set; } public string UserMessage { get; set; } = default!; public string? Output { get; set; } + + /// The reply's rich message as JSON, null for a plain reply. What richContent assertions read. + public string? RichContent { get; set; } + public List Assertions { get; set; } = []; /// diff --git a/src/Plugins/BotSharp.Plugin.AgentTesting/Services/AgentTestCaseRunner.cs b/src/Plugins/BotSharp.Plugin.AgentTesting/Services/AgentTestCaseRunner.cs index ad36f2641..7390f7343 100644 --- a/src/Plugins/BotSharp.Plugin.AgentTesting/Services/AgentTestCaseRunner.cs +++ b/src/Plugins/BotSharp.Plugin.AgentTesting/Services/AgentTestCaseRunner.cs @@ -213,7 +213,7 @@ async Task AwaitOrHandOffAsync(Task driverTask) // canary, the mock lookups and the conversation reads, and on a fast case that // overhead is a big enough share to move a latency percentile. var turnTimer = Stopwatch.StartNew(); - var output = await AwaitOrHandOffAsync( + var reply = await AwaitOrHandOffAsync( _driver.SendAsync(conversationId, entryAgentId, turn.UserMessage, timeout.Token)); turnTimer.Stop(); modelDurationMs += turnTimer.ElapsedMilliseconds; @@ -226,7 +226,8 @@ async Task AwaitOrHandOffAsync(Task driverTask) { Index = turn.Index, UserMessage = turn.UserMessage, - Output = output, + Output = reply.Text, + RichContent = reply.RichContent, ModelDurationMs = turnTimer.ElapsedMilliseconds, // Names, not hops: a result is read by a person, and the ids are only needed // while an assertion is being evaluated. @@ -239,7 +240,8 @@ async Task AwaitOrHandOffAsync(Task driverTask) // assertion about routing that never happened. var turnContext = new AssertionContext { - Output = output, + Output = reply.Text, + RichContent = reply.RichContent, ToolCalls = active.ObservedCalls.Where(c => c.TurnIndex == turn.Index).ToList(), States = await _driver.ReadStatesAsync(conversationId), AgentChain = turnChain @@ -265,6 +267,7 @@ async Task AwaitOrHandOffAsync(Task driverTask) var finalContext = new AssertionContext { Output = result.Turns.LastOrDefault()?.Output, + RichContent = result.Turns.LastOrDefault()?.RichContent, ToolCalls = active.ObservedCalls, States = await _driver.ReadStatesAsync(conversationId), AgentChain = caseChain diff --git a/src/Plugins/BotSharp.Plugin.AgentTesting/Services/AssertionContext.cs b/src/Plugins/BotSharp.Plugin.AgentTesting/Services/AssertionContext.cs index 0f34f6541..619f21fb5 100644 --- a/src/Plugins/BotSharp.Plugin.AgentTesting/Services/AssertionContext.cs +++ b/src/Plugins/BotSharp.Plugin.AgentTesting/Services/AssertionContext.cs @@ -7,6 +7,10 @@ namespace BotSharp.Plugin.AgentTesting.Services; public class AssertionContext { public string? Output { get; set; } + + /// Read only by assertions targeting . + public string? RichContent { get; set; } + public IReadOnlyList ToolCalls { get; set; } = []; public IReadOnlyDictionary States { get; set; } = new Dictionary(); diff --git a/src/Plugins/BotSharp.Plugin.AgentTesting/Services/AssertionEvaluator.cs b/src/Plugins/BotSharp.Plugin.AgentTesting/Services/AssertionEvaluator.cs index ac783dc23..2395f0dce 100644 --- a/src/Plugins/BotSharp.Plugin.AgentTesting/Services/AssertionEvaluator.cs +++ b/src/Plugins/BotSharp.Plugin.AgentTesting/Services/AssertionEvaluator.cs @@ -41,10 +41,23 @@ public static AssertionResult Evaluate(TestAssertion assertion, AssertionContext Expected = assertion.Expected }; + string? output = null; + if (assertion.Type is AssertionTypes.OutputContains or AssertionTypes.OutputNotContains or AssertionTypes.OutputRegex) + { + var source = OutputSources.Normalize(assertion.Target); + if (source == null) + { + result.Passed = false; + result.Message = OutputSources.InvalidTargetMessage(assertion); + return result; + } + output = OutputSources.Read(source, context); + } + switch (assertion.Type) { case AssertionTypes.OutputContains: - result.Actual = context.Output; + result.Actual = output; if (string.IsNullOrEmpty(assertion.Expected)) { // Contains("") is vacuously true for any non-null string -- a blank/omitted @@ -55,21 +68,21 @@ public static AssertionResult Evaluate(TestAssertion assertion, AssertionContext } else { - result.Passed = context.Output?.Contains(assertion.Expected, + result.Passed = output?.Contains(assertion.Expected, StringComparison.OrdinalIgnoreCase) == true; if (!result.Passed) result.Message = "output does not contain the expected text"; } break; case AssertionTypes.OutputNotContains: - result.Actual = context.Output; - result.Passed = context.Output?.Contains(assertion.Expected ?? string.Empty, + result.Actual = output; + result.Passed = output?.Contains(assertion.Expected ?? string.Empty, StringComparison.OrdinalIgnoreCase) != true; if (!result.Passed) result.Message = "output contains text that should not appear"; break; case AssertionTypes.OutputRegex: - result.Actual = context.Output; + result.Actual = output; if (string.IsNullOrEmpty(assertion.Expected)) { // An empty pattern matches everything -- a blank/omitted Expected must not @@ -80,7 +93,7 @@ public static AssertionResult Evaluate(TestAssertion assertion, AssertionContext } try { - result.Passed = Regex.IsMatch(context.Output ?? string.Empty, assertion.Expected, + result.Passed = Regex.IsMatch(output ?? string.Empty, assertion.Expected, RegexOptions.IgnoreCase, TimeSpan.FromSeconds(1)); if (!result.Passed) result.Message = "output does not match the pattern"; } @@ -338,6 +351,39 @@ public static class AgentChainModes : All.FirstOrDefault(m => string.Equals(m, value.Trim(), StringComparison.OrdinalIgnoreCase)); } +/// +/// Which text outputContains / outputNotContains / outputRegex and llmJudge read, selected by +/// . +/// +/// Reply -- the text a user reads. The default. +/// RichContent -- the reply's rich message as JSON, for content that exists only there, such as an +/// upload card's photo and video lists. Opt-in, so button titles and other structure +/// never leak into the reply text every existing assertion was written against. +/// +public static class OutputSources +{ + public const string Reply = "reply"; + public const string RichContent = "richContent"; + + public static readonly string[] All = [Reply, RichContent]; + + /// + /// Canonical source for any casing, with blank meaning ; null for an + /// unrecognised value, so a typo fails loudly instead of silently reading the reply text. + /// + public static string? Normalize(string? value) + => string.IsNullOrWhiteSpace(value) + ? Reply + : All.FirstOrDefault(s => string.Equals(s, value.Trim(), StringComparison.OrdinalIgnoreCase)); + + public static string? Read(string source, AssertionContext context) + => source == RichContent ? context.RichContent : context.Output; + + public static string InvalidTargetMessage(TestAssertion assertion) + => $"{assertion.Type} 'target' must be one of {string.Join(", ", All)} " + + $"(or empty for {Reply}), not '{assertion.Target}'"; +} + /// /// Save-time counterpart to the four fixed branches above /// (outputContains/outputRegex/toolNotCalled/routedToAgent): an assertion missing the one field diff --git a/src/Plugins/BotSharp.Plugin.AgentTesting/Services/BotSharpAgentConversationDriver.cs b/src/Plugins/BotSharp.Plugin.AgentTesting/Services/BotSharpAgentConversationDriver.cs index ee3198c97..e38e83f76 100644 --- a/src/Plugins/BotSharp.Plugin.AgentTesting/Services/BotSharpAgentConversationDriver.cs +++ b/src/Plugins/BotSharp.Plugin.AgentTesting/Services/BotSharpAgentConversationDriver.cs @@ -1,6 +1,9 @@ +using System.Text.Encodings.Web; +using System.Text.Json; using BotSharp.Abstraction.Agents; using BotSharp.Abstraction.Agents.Enums; using BotSharp.Abstraction.Conversations.Enums; +using BotSharp.Abstraction.Messaging; using BotSharp.Abstraction.Models; using BotSharp.Abstraction.Repositories; using BotSharp.Abstraction.Routing; @@ -126,7 +129,7 @@ public async Task InjectHistoryAsync( return stored.Count(d => d.MetaData?.MessageId != null && written.Contains(d.MetaData.MessageId)); } - public async Task SendAsync(string conversationId, string agentId, string userMessage, CancellationToken ct) + public async Task SendAsync(string conversationId, string agentId, string userMessage, CancellationToken ct) { // Fail fast only if the case had already timed out before we ever got here (e.g. a prior // turn ran long). Once the real call below starts, there is no cancellation hook left to @@ -152,7 +155,34 @@ await _conversations.SendMessage( // label -- are what decide pass/fail. await EnsureConversationTaggedAsync(conversationId); - return last?.RichContent?.Message?.Text ?? last?.Content; + var message = last?.RichContent?.Message; + return new AgentReply(message?.Text ?? last?.Content, SerializeRichMessage(message)); + } + + // Unescaped so apostrophes and dashes stay literal: assertions match against this text and a + // judge reads it, and neither should have to see "it's". + private static readonly JsonSerializerOptions RichMessageJsonOptions = new() + { + Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping + }; + + private string? SerializeRichMessage(IRichMessage? message) + { + if (message == null) return null; + + try + { + // By runtime type: serializing as IRichMessage would drop every property the concrete + // message adds, which is exactly the content a richContent assertion is after. + return JsonSerializer.Serialize(message, message.GetType(), RichMessageJsonOptions); + } + catch (Exception ex) when (ex is NotSupportedException or JsonException) + { + // Only richContent assertions read this; a message that cannot be serialized must not + // fail a case whose assertions never look at it. + _logger.LogWarning(ex, "Could not serialize rich message {Type} for agent-test output.", message.GetType().Name); + return null; + } } private async Task EnsureConversationTaggedAsync(string conversationId) diff --git a/src/Plugins/BotSharp.Plugin.AgentTesting/Services/IAgentConversationDriver.cs b/src/Plugins/BotSharp.Plugin.AgentTesting/Services/IAgentConversationDriver.cs index de6fce7cb..14113d6af 100644 --- a/src/Plugins/BotSharp.Plugin.AgentTesting/Services/IAgentConversationDriver.cs +++ b/src/Plugins/BotSharp.Plugin.AgentTesting/Services/IAgentConversationDriver.cs @@ -22,8 +22,8 @@ public interface IAgentConversationDriver Task InjectHistoryAsync( string conversationId, string agentId, IReadOnlyList history); - /// Drives one turn and returns that turn's output text. - Task SendAsync(string conversationId, string agentId, string userMessage, CancellationToken ct); + /// Drives one turn and returns what that turn produced. + Task SendAsync(string conversationId, string agentId, string userMessage, CancellationToken ct); /// Calls the canary function once; returns whether the mock seam took it over. Task RunCanaryAsync(string conversationId, string agentId, CancellationToken ct); @@ -43,3 +43,10 @@ Task InjectHistoryAsync( /// Task> ReadAssistantAgentSequenceAsync(string conversationId); } + +/// +/// One turn's reply. Text is what a user reads: the rich message's text, else the plain content. +/// RichContent is the rich message serialized as JSON, null for a plain reply -- structured content +/// such as an upload card's photo list exists only there. +/// +public record AgentReply(string? Text, string? RichContent); diff --git a/src/Plugins/BotSharp.Plugin.AgentTesting/Services/LlmAgentTestJudge.cs b/src/Plugins/BotSharp.Plugin.AgentTesting/Services/LlmAgentTestJudge.cs index 5525df4c1..5dcb42ed3 100644 --- a/src/Plugins/BotSharp.Plugin.AgentTesting/Services/LlmAgentTestJudge.cs +++ b/src/Plugins/BotSharp.Plugin.AgentTesting/Services/LlmAgentTestJudge.cs @@ -62,13 +62,21 @@ public async Task JudgeAsync( + "Set the suite's judgeProvider and judgeModel, or remove the llmJudge assertion."); } - if (string.IsNullOrWhiteSpace(context.Output)) + var source = OutputSources.Normalize(assertion.Target); + if (source == null) + { + throw new AgentTestJudgeUnavailableException(OutputSources.InvalidTargetMessage(assertion)); + } + + var output = OutputSources.Read(source, context); + if (string.IsNullOrWhiteSpace(output)) { // Not a failing verdict: the agent produced no text at all, so there is nothing for the // judge to grade. Whatever went wrong upstream is the real finding, and the other // assertions on this case will say so far more usefully than a fabricated score would. - throw new AgentTestJudgeUnavailableException( - "the agent produced no reply text, so there is nothing for llmJudge to score"); + throw new AgentTestJudgeUnavailableException(source == OutputSources.RichContent + ? "the agent produced no rich content, so there is nothing for llmJudge to score" + : "the agent produced no reply text, so there is nothing for llmJudge to score"); } ct.ThrowIfCancellationRequested(); @@ -101,7 +109,7 @@ public async Task JudgeAsync( { var response = await completion.GetChatCompletions( promptAgent, - [new RoleDialogModel(AgentRole.User, BuildPrompt(criterion, context.Output))]); + [new RoleDialogModel(AgentRole.User, BuildPrompt(criterion, output))]); raw = response?.Content ?? string.Empty; } diff --git a/tests/BotSharp.Core.UnitTests/AgentTesting/AgentTestCaseRunnerTests.cs b/tests/BotSharp.Core.UnitTests/AgentTesting/AgentTestCaseRunnerTests.cs index 529c8b5d9..57a4ec5c7 100644 --- a/tests/BotSharp.Core.UnitTests/AgentTesting/AgentTestCaseRunnerTests.cs +++ b/tests/BotSharp.Core.UnitTests/AgentTesting/AgentTestCaseRunnerTests.cs @@ -115,7 +115,7 @@ public Task PrepareAsync(string conversationId, string agentId, IReadOnlyList BlockedOnFirstSend { get; } = []; - public async Task SendAsync(string conversationId, string agentId, string userMessage, CancellationToken ct) + public async Task SendAsync(string conversationId, string agentId, string userMessage, CancellationToken ct) { Sent.Add(userMessage); SentAgentIds.Add(agentId); @@ -171,7 +171,7 @@ public Task PrepareAsync(string conversationId, string agentId, IReadOnlyList 0 ? Replies.Dequeue() : string.Empty; + return new AgentReply(Replies.Count > 0 ? Replies.Dequeue() : string.Empty, null); } public Task InjectHistoryAsync( diff --git a/tests/BotSharp.Core.UnitTests/AgentTesting/AgentTestJudgeTests.cs b/tests/BotSharp.Core.UnitTests/AgentTesting/AgentTestJudgeTests.cs index 84592a82e..78ab435b2 100644 --- a/tests/BotSharp.Core.UnitTests/AgentTesting/AgentTestJudgeTests.cs +++ b/tests/BotSharp.Core.UnitTests/AgentTesting/AgentTestJudgeTests.cs @@ -212,8 +212,8 @@ private sealed class StubDriver : IAgentConversationDriver public Task PrepareAsync(string conversationId, string agentId, IReadOnlyList initialStates) => Task.CompletedTask; - public Task SendAsync(string conversationId, string agentId, string userMessage, CancellationToken ct) - => Task.FromResult(_reply); + public Task SendAsync(string conversationId, string agentId, string userMessage, CancellationToken ct) + => Task.FromResult(new AgentReply(_reply, null)); public Task RunCanaryAsync(string conversationId, string agentId, CancellationToken ct) => Task.FromResult(true); diff --git a/tests/BotSharp.Core.UnitTests/AgentTesting/AssertionEvaluatorTests.cs b/tests/BotSharp.Core.UnitTests/AgentTesting/AssertionEvaluatorTests.cs index 88b911bcf..fe3c11eb1 100644 --- a/tests/BotSharp.Core.UnitTests/AgentTesting/AssertionEvaluatorTests.cs +++ b/tests/BotSharp.Core.UnitTests/AgentTesting/AssertionEvaluatorTests.cs @@ -72,6 +72,35 @@ public void Output_regex_reports_a_bad_pattern_as_a_failed_assertion_not_an_exce Assert.Contains("invalid regular expression", result.Message!); } + [Fact] + public void Output_assertions_read_the_rich_content_only_when_targeted() + { + var context = Context(output: "One photo would help."); + context.RichContent = """{"upload_guidance":{"photos":["Photograph the cracked pipe"]},"buttons":[{"title":"Skip upload"}]}"""; + + var targeted = AssertionEvaluator.Evaluate( + new TestAssertion { Type = AssertionTypes.OutputContains, Target = "richcontent", Expected = "cracked pipe" }, + context); + var untargeted = AssertionEvaluator.Evaluate( + new TestAssertion { Type = AssertionTypes.OutputNotContains, Expected = "skip" }, + context); + + Assert.True(targeted.Passed); + Assert.Equal(context.RichContent, targeted.Actual); + Assert.True(untargeted.Passed); + } + + [Fact] + public void Output_assertions_fail_on_an_unknown_target_instead_of_reading_the_reply() + { + var result = AssertionEvaluator.Evaluate( + new TestAssertion { Type = AssertionTypes.OutputContains, Target = "rich", Expected = "photo" }, + Context(output: "One photo would help.")); + + Assert.False(result.Passed); + Assert.Contains("'target' must be one of", result.Message!); + } + [Fact] public void Tool_called_matches_arguments_as_a_subset() {