diff --git a/src/Plugins/BotSharp.Plugin.AgentTesting/Controllers/AgentTestController.cs b/src/Plugins/BotSharp.Plugin.AgentTesting/Controllers/AgentTestController.cs index d9d097d2c..dbbf7ee3e 100644 --- a/src/Plugins/BotSharp.Plugin.AgentTesting/Controllers/AgentTestController.cs +++ b/src/Plugins/BotSharp.Plugin.AgentTesting/Controllers/AgentTestController.cs @@ -413,6 +413,7 @@ public async Task> DeleteRuns( continue; } + run = await FailIfAbandonedAsync(run); if (!IsTerminalStatus(run.Status)) { // Deleting a run that is still executing does not stop it: the queue keeps driving @@ -637,7 +638,14 @@ public async Task> TriggerRun( [HttpGet("runs")] public async Task> ListRuns([FromQuery] string? suiteId) - => await _repo.ListRunsAsync(suiteId); + { + var runs = await _repo.ListRunsAsync(suiteId); + for (var i = 0; i < runs.Count; i++) + { + runs[i] = await FailIfAbandonedAsync(runs[i]); + } + return runs; + } [HttpGet("runs/{id}")] public async Task> GetRun(string id) @@ -648,6 +656,7 @@ public async Task> GetRun(string id) return NotFound($"agent test run {id} not found"); } + run = await FailIfAbandonedAsync(run); var results = await _repo.ListCaseResultsAsync(id); return new AgentTestRunDetailDto { Run = run, Results = results }; } @@ -669,8 +678,13 @@ public async Task CancelRun(string id) return Conflict($"agent test run {id} has already finished ({run.Status}) and cannot be cancelled"); } - run.CancelRequested = true; - await _repo.UpdateRunAsync(run); + // A single-field update, not a write-back of `run`: the executor keeps writing this document + // while the run goes, and replacing it with the copy read above would revert its progress. + if (!await _repo.RequestRunCancelAsync(id)) + { + return Conflict($"agent test run {id} finished before it could be cancelled"); + } + return Ok(); } @@ -765,13 +779,16 @@ private static void ApplyCase(AgentTestCase testCase, AgentTestCaseUpsertRequest { testCase.SuiteId = request.SuiteId; testCase.Name = request.Name; - testCase.Enabled = request.Enabled; + // Null keeps the case's current value, as ApplySuite does: a partial PUT must not enable a + // recorded draft nobody has reviewed yet. + testCase.Enabled = request.Enabled ?? testCase.Enabled; // Normalised, not taken verbatim: CaseValidation.Validate has already rejected anything that is // neither blank nor a known type, so this only fixes casing ("routing" -> "Routing") and // maps blank onto the default. Storing the caller's casing would break every later // Ordinal comparison against CaseTypes.Routing. testCase.CaseType = CaseTypes.Normalize(request.CaseType) ?? CaseTypes.Agent; testCase.EntryAgentId = string.IsNullOrWhiteSpace(request.EntryAgentId) ? null : request.EntryAgentId.Trim(); + testCase.TemplateName = string.IsNullOrWhiteSpace(request.TemplateName) ? null : request.TemplateName.Trim(); testCase.Turns = request.Turns ?? []; testCase.Assertions = request.Assertions ?? []; testCase.InitialStates = request.InitialStates ?? []; @@ -822,7 +839,63 @@ private static void ApplyCase(AgentTestCase testCase, AgentTestCaseUpsertRequest } var agent = await _agents.GetAgent(request.EntryAgentId.Trim()); - return agent == null ? $"entry agent {request.EntryAgentId} not found" : null; + if (agent == null) + { + return $"entry agent {request.EntryAgentId} not found"; + } + + var isTemplateCase = string.Equals( + CaseTypes.Normalize(request.CaseType), CaseTypes.Template, StringComparison.Ordinal); + if (isTemplateCase + && !string.IsNullOrWhiteSpace(request.TemplateName) + // Ordinal, as AgentService.RenderTemplate looks templates up with ==: a name that only + // matches ignoring case would save fine and then render an empty instruction. + && agent.Templates?.Any(t => string.Equals(t.Name, request.TemplateName.Trim(), StringComparison.Ordinal)) != true) + { + return $"agent {agent.Name} has no template '{request.TemplateName}'"; + } + + return null; + } + + /// + /// The run queue is in-process, so a run whose host died stays Running in the database. A live + /// executor stamps UpdateDate every AgentTestRunExecutor.HeartbeatInterval (30s), so this long + /// without one means nothing is executing it any more. + /// + private static readonly TimeSpan AbandonedAfter = TimeSpan.FromMinutes(3); + + /// + /// Fails a Running run whose heartbeat stopped, and returns the run as it now stands. Checked + /// when a run is read rather than swept on host startup: hosts share one database, and a sweep + /// cannot tell another host's live run from a dead one. + /// + private async Task FailIfAbandonedAsync(AgentTestRun run) + { + if (run.Status != AgentTestStatus.Running) + { + return run; + } + + var lastProgress = run.UpdateDate ?? run.StartedAt ?? run.CreateDate; + if (DateTime.UtcNow - lastProgress <= AbandonedAfter) + { + return run; + } + + var error = $"The run stopped reporting progress for over {AbandonedAfter.TotalMinutes:0} minutes, so " + + "the host running it has stopped. The queue is in-process and does not survive a restart -- " + + "trigger the run again."; + if (!await _repo.TryAbandonRunAsync(run.Id, run.UpdateDate, error)) + { + // The executor wrote since this copy was read: it is alive, or already finished. + return await _repo.GetRunAsync(run.Id) ?? run; + } + + run.Status = AgentTestStatus.Error; + run.Error = error; + run.CompletedAt = DateTime.UtcNow; + return run; } private static bool IsTerminalStatus(string status) => diff --git a/src/Plugins/BotSharp.Plugin.AgentTesting/Models/AgentTestAuthorDtos.cs b/src/Plugins/BotSharp.Plugin.AgentTesting/Models/AgentTestAuthorDtos.cs index 8f636fbb2..58f238fa7 100644 --- a/src/Plugins/BotSharp.Plugin.AgentTesting/Models/AgentTestAuthorDtos.cs +++ b/src/Plugins/BotSharp.Plugin.AgentTesting/Models/AgentTestAuthorDtos.cs @@ -135,6 +135,7 @@ public static class AuthorFields public const string Enabled = "enabled"; public const string CaseType = "caseType"; public const string EntryAgentId = "entryAgentId"; + public const string TemplateName = "templateName"; public const string Turns = "turns"; public const string Assertions = "assertions"; public const string InitialStates = "initialStates"; @@ -150,7 +151,7 @@ public static class AuthorFields public static readonly string[] All = [ - Name, Enabled, CaseType, EntryAgentId, Turns, Assertions, InitialStates, History, Mocks, + Name, Enabled, CaseType, EntryAgentId, TemplateName, Turns, Assertions, InitialStates, History, Mocks, Priority, Severity, Batch, CrossCutting, InvolvedAgents, BusinessDomain, ExpectedOutcome ]; diff --git a/src/Plugins/BotSharp.Plugin.AgentTesting/Models/AgentTestCase.cs b/src/Plugins/BotSharp.Plugin.AgentTesting/Models/AgentTestCase.cs index 7cfabd6da..56aada7f4 100644 --- a/src/Plugins/BotSharp.Plugin.AgentTesting/Models/AgentTestCase.cs +++ b/src/Plugins/BotSharp.Plugin.AgentTesting/Models/AgentTestCase.cs @@ -32,6 +32,11 @@ public class AgentTestCase : MongoBase /// public string? EntryAgentId { get; set; } + /// + /// The entry agent's template a Template case calls directly; ignored for every other type. + /// + public string? TemplateName { get; set; } + /// A length of 1 is a single-turn case. public List Turns { get; set; } = []; @@ -139,9 +144,25 @@ public class TestTurn { public int Index { get; set; } public string UserMessage { get; set; } = default!; + + /// Attached to the user message, e.g. photos for a vision template. + public List Files { get; set; } = []; + public List Assertions { get; set; } = []; } +[BsonIgnoreExtraElements(Inherited = true)] +public class TestFile +{ + public string FileName { get; set; } = string.Empty; + + /// + /// Must stay reachable for as long as the case exists: a signed URL that expires turns every + /// later run into a model looking at nothing. + /// + public string FileUrl { get; set; } = string.Empty; +} + [BsonIgnoreExtraElements(Inherited = true)] public class TestState { @@ -321,13 +342,17 @@ public static int Effective(AgentTestCase testCase) /// Agent -- one agent's own behaviour, normally entered directly so the router is not part of what /// is being measured. Multi-agent journeys are Agent cases too; the agentChain assertion /// is what describes their hand-offs. +/// Template -- one of the entry agent's templates called directly, the way code such as a photo +/// check calls it outside any conversation. Each turn is an independent call: no routing, +/// no function calls, no dialog history. /// public static class CaseTypes { public const string Routing = "Routing"; public const string Agent = "Agent"; + public const string Template = "Template"; - public static readonly string[] All = [Routing, Agent]; + public static readonly string[] All = [Routing, Agent, Template]; /// /// Maps any casing of a known type onto its canonical constant; null for anything unknown, so diff --git a/src/Plugins/BotSharp.Plugin.AgentTesting/Models/AgentTestCaseResult.cs b/src/Plugins/BotSharp.Plugin.AgentTesting/Models/AgentTestCaseResult.cs index 76a448ee4..b5c7f9080 100644 --- a/src/Plugins/BotSharp.Plugin.AgentTesting/Models/AgentTestCaseResult.cs +++ b/src/Plugins/BotSharp.Plugin.AgentTesting/Models/AgentTestCaseResult.cs @@ -71,6 +71,13 @@ public class AgentTestCaseResult : MongoBase /// public double Cost { get; set; } + /// + /// What this case's llmJudge assertions spent, kept out of and + /// so those describe the model under test alone. Priced by the judge model. + /// + public long JudgeTokens { get; set; } + public double JudgeCost { get; set; } + /// /// Infrastructure-level reason for failure (a timeout, a dead canary), kept distinct from an /// assertion failure. diff --git a/src/Plugins/BotSharp.Plugin.AgentTesting/Models/AgentTestDtos.cs b/src/Plugins/BotSharp.Plugin.AgentTesting/Models/AgentTestDtos.cs index 972dbce5c..8ad6f75cf 100644 --- a/src/Plugins/BotSharp.Plugin.AgentTesting/Models/AgentTestDtos.cs +++ b/src/Plugins/BotSharp.Plugin.AgentTesting/Models/AgentTestDtos.cs @@ -43,7 +43,9 @@ public class AgentTestCaseUpsertRequest { public string SuiteId { get; set; } = string.Empty; public string Name { get; set; } = string.Empty; - public bool Enabled { get; set; } = true; + + /// Null keeps the current value on update; a new case defaults to enabled. + public bool? Enabled { get; set; } /// /// See . Blank or omitted means Agent, so an existing client that never @@ -67,6 +69,9 @@ public class AgentTestCaseUpsertRequest /// public string? EntryAgentId { get; set; } + /// See . Required for a Template case. + public string? TemplateName { get; set; } + public List Turns { get; set; } = []; public List Assertions { get; set; } = []; public List InitialStates { get; set; } = []; diff --git a/src/Plugins/BotSharp.Plugin.AgentTesting/Models/AgentTestRun.cs b/src/Plugins/BotSharp.Plugin.AgentTesting/Models/AgentTestRun.cs index 25116fa0b..4c0a7f759 100644 --- a/src/Plugins/BotSharp.Plugin.AgentTesting/Models/AgentTestRun.cs +++ b/src/Plugins/BotSharp.Plugin.AgentTesting/Models/AgentTestRun.cs @@ -54,6 +54,12 @@ public class AgentTestRun : MongoBase /// public int MaxParallelCases { get; set; } + /// + /// How many case x model executions the run set out to do, set when it starts Running. + /// TotalCount is how many of them have finished. Zero on runs written before this field existed. + /// + public int PlannedCount { get; set; } + public int TotalCount { get; set; } public int PassedCount { get; set; } public int FailedCount { get; set; } @@ -90,6 +96,13 @@ public class AgentTestRun : MongoBase /// public List ModelPricing { get; set; } = []; + /// + /// The judge model llmJudge assertions were scored with, and its unit costs. Taken from the suite + /// this run executed against, because the suite's setting can change afterwards and the run report + /// would then name a judge that never scored it. Null when the suite had no judge configured. + /// + public ModelPricingSnapshot? JudgePricing { get; set; } + /// /// Why a run ended as -- an infrastructure stop that /// happened before or instead of executing cases (suite gone, suite disabled, the CaseIds @@ -106,6 +119,13 @@ public class AgentTestRun : MongoBase public DateTime? StartedAt { get; set; } public DateTime? CompletedAt { get; set; } public DateTime CreateDate { get; set; } = DateTime.UtcNow; + + /// + /// Set on every run write and by the executor's heartbeat while cases run, so a Running run whose + /// UpdateDate falls far behind no longer has a live executor. Null on runs written before this + /// field existed. + /// + public DateTime? UpdateDate { get; set; } } /// @@ -132,6 +152,10 @@ public class PerformanceSummary public long TotalTokens { get; set; } public double TotalCost { get; set; } + + /// llmJudge spend for this model's cases, not included in TotalTokens/TotalCost. + public long JudgeTokens { get; set; } + public double JudgeCost { get; set; } } /// diff --git a/src/Plugins/BotSharp.Plugin.AgentTesting/Repositories/AgentTestRepository.cs b/src/Plugins/BotSharp.Plugin.AgentTesting/Repositories/AgentTestRepository.cs index 4773ebde9..3467ab85c 100644 --- a/src/Plugins/BotSharp.Plugin.AgentTesting/Repositories/AgentTestRepository.cs +++ b/src/Plugins/BotSharp.Plugin.AgentTesting/Repositories/AgentTestRepository.cs @@ -24,16 +24,32 @@ public interface IAgentTestRepository Task> ListRunsAsync(string? suiteId); /// - /// Bounded counterpart to for the startup reconciliation sweep - /// (AgentTestRunQueue.ReconcileStaleRunningRunsAsync), which only ever cares about one status - /// (AgentTestStatus.Running) across every suite -- ListRunsAsync(null) would otherwise pull - /// the entire, ever-growing AgentTestRuns collection into memory on every host startup just to - /// filter it down to a handful of rows client-side. + /// Whole-document replace that never reverts CancelRequested: if a cancel landed after + /// was read, the write keeps it and sets it on . + /// Stamps UpdateDate. /// - Task> ListRunsByStatusAsync(string status); - Task UpdateRunAsync(AgentTestRun run); + /// + /// Sets CancelRequested on a Pending or Running run and touches no other field. False when the + /// run does not exist or has already finished. + /// + Task RequestRunCancelAsync(string id); + + /// + /// Stamps UpdateDate on a Running run and touches no other field. False when the run is gone or + /// no longer Running. + /// + Task TouchRunAsync(string id); + + /// + /// Marks a Running run Error only while its UpdateDate still equals + /// , setting just Status, Error and CompletedAt. False when + /// the executor wrote anything since it was observed, so a stale copy never overwrites newer + /// progress or a terminal status. + /// + Task TryAbandonRunAsync(string id, DateTime? observedUpdateDate, string error); + /// /// Removes a run and every case result belonging to it, and reports how many results went with /// it. @@ -151,13 +167,52 @@ public async Task> ListRunsAsync(string? suiteId) .ToListAsync(); } - public async Task> ListRunsByStatusAsync(string status) - => await _mongoDbContext.AgentTestRuns - .Find(x => x.Status == status) - .ToListAsync(); - public async Task UpdateRunAsync(AgentTestRun run) - => await _mongoDbContext.AgentTestRuns.ReplaceOneAsync(x => x.Id == run.Id, run); + { + run.UpdateDate = DateTime.UtcNow; + + // CancelRequested is the one field written outside the executor, and it only ever goes + // false -> true. Replacing only while the stored flag still matches ours means a cancel that + // arrived after this copy was read is never overwritten; on a miss, carry it over and write. + var replaced = await _mongoDbContext.AgentTestRuns.ReplaceOneAsync( + x => x.Id == run.Id && x.CancelRequested == run.CancelRequested, run); + + if (replaced.MatchedCount == 0 && !run.CancelRequested) + { + run.CancelRequested = true; + await _mongoDbContext.AgentTestRuns.ReplaceOneAsync(x => x.Id == run.Id, run); + } + } + + public async Task RequestRunCancelAsync(string id) + { + var updated = await _mongoDbContext.AgentTestRuns.UpdateOneAsync( + x => x.Id == id && (x.Status == AgentTestStatus.Pending || x.Status == AgentTestStatus.Running), + Builders.Update.Set(x => x.CancelRequested, true)); + return updated.MatchedCount > 0; + } + + public async Task TouchRunAsync(string id) + { + var updated = await _mongoDbContext.AgentTestRuns.UpdateOneAsync( + x => x.Id == id && x.Status == AgentTestStatus.Running, + Builders.Update.Set(x => x.UpdateDate, DateTime.UtcNow)); + return updated.MatchedCount > 0; + } + + public async Task TryAbandonRunAsync(string id, DateTime? observedUpdateDate, string error) + { + // observedUpdateDate was read back from this collection, so it carries the stored + // millisecond precision and compares equal. Null matches a run written before UpdateDate + // existed. + var updated = await _mongoDbContext.AgentTestRuns.UpdateOneAsync( + x => x.Id == id && x.Status == AgentTestStatus.Running && x.UpdateDate == observedUpdateDate, + Builders.Update + .Set(x => x.Status, AgentTestStatus.Error) + .Set(x => x.Error, error) + .Set(x => x.CompletedAt, DateTime.UtcNow)); + return updated.MatchedCount > 0; + } public async Task DeleteRunAsync(string id) { diff --git a/src/Plugins/BotSharp.Plugin.AgentTesting/Services/AgentTestCaseRunner.cs b/src/Plugins/BotSharp.Plugin.AgentTesting/Services/AgentTestCaseRunner.cs index c7c413619..c9f6cb3bd 100644 --- a/src/Plugins/BotSharp.Plugin.AgentTesting/Services/AgentTestCaseRunner.cs +++ b/src/Plugins/BotSharp.Plugin.AgentTesting/Services/AgentTestCaseRunner.cs @@ -11,6 +11,12 @@ public class AgentTestCaseRunner : ICaseRunner private readonly ITokenStatistics? _tokens; private readonly ILogger _logger; + // What llmJudge calls added to _tokens. The judge resolves its completion provider from this + // scope, so TokenStatsConversationHook counts it alongside the model under test; it is taken + // back out of the case's figures so a model comparison is not skewed by the judge. + private long _judgeTokens; + private float _judgeCost; + public AgentTestCaseRunner( IAgentTestRunRegistry registry, IAgentConversationDriver driver, @@ -96,6 +102,8 @@ public async Task RunAsync( // the delta is simply the final reading. var tokensBefore = _tokens?.Total ?? 0; var costBefore = _tokens?.AccumulatedCost ?? 0f; + var judgeTokensBefore = _judgeTokens; + var judgeCostBefore = _judgeCost; long modelDurationMs = 0; using var timeout = CancellationTokenSource.CreateLinkedTokenSource(ct); @@ -213,8 +221,11 @@ async Task AwaitOrHandOffAsync(Task driverTask) // canary, the mock lookups and the conversation reads, and on a fast case that // overhead is a big enough share to move a latency percentile. var turnTimer = Stopwatch.StartNew(); + var files = turn.Files ?? []; var reply = await AwaitOrHandOffAsync( - _driver.SendAsync(conversationId, entryAgentId, turn.UserMessage, timeout.Token)); + string.Equals(testCase.CaseType, CaseTypes.Template, StringComparison.Ordinal) + ? _driver.InstructAsync(entryAgentId, testCase.TemplateName!, turn.UserMessage, files, timeout.Token) + : _driver.SendAsync(conversationId, entryAgentId, turn.UserMessage, files, timeout.Token)); turnTimer.Stop(); modelDurationMs += turnTimer.ElapsedMilliseconds; @@ -345,8 +356,10 @@ async Task AwaitOrHandOffAsync(Task driverTask) // In the finally block so a timed-out or crashed case still reports what it spent. A run // that fell over having burned the budget is exactly the run whose cost matters. - result.TotalTokens = Math.Max(0, (_tokens?.Total ?? 0) - tokensBefore); - result.Cost = Math.Max(0, (_tokens?.AccumulatedCost ?? 0f) - costBefore); + result.JudgeTokens = _judgeTokens - judgeTokensBefore; + result.JudgeCost = _judgeCost - judgeCostBefore; + result.TotalTokens = Math.Max(0, (_tokens?.Total ?? 0) - tokensBefore - result.JudgeTokens); + result.Cost = Math.Max(0, (_tokens?.AccumulatedCost ?? 0f) - costBefore - result.JudgeCost); if (result.TotalTokens == 0 && result.Turns.Count > 0 && _tokens != null) { @@ -414,7 +427,19 @@ private async Task EvaluateAsync( "no IAgentTestJudge is registered, so llmJudge assertions cannot be scored"); } - return await _judge.JudgeAsync(assertion, context, suite, ct); + // Turns and assertions run one after another, so nothing else reports into _tokens while + // the judge call is in flight and the difference is the judge's alone. + var tokensBefore = _tokens?.Total ?? 0; + var costBefore = _tokens?.AccumulatedCost ?? 0f; + try + { + return await _judge.JudgeAsync(assertion, context, suite, ct); + } + finally + { + _judgeTokens += (_tokens?.Total ?? 0) - tokensBefore; + _judgeCost += (_tokens?.AccumulatedCost ?? 0f) - costBefore; + } } /// diff --git a/src/Plugins/BotSharp.Plugin.AgentTesting/Services/AgentTestRunExecutor.cs b/src/Plugins/BotSharp.Plugin.AgentTesting/Services/AgentTestRunExecutor.cs index 62802326a..94261cd1b 100644 --- a/src/Plugins/BotSharp.Plugin.AgentTesting/Services/AgentTestRunExecutor.cs +++ b/src/Plugins/BotSharp.Plugin.AgentTesting/Services/AgentTestRunExecutor.cs @@ -59,6 +59,14 @@ public AgentTestRunExecutor( _logger = logger; } + /// + /// How often a Running run stamps UpdateDate while its cases execute. Case completions alone are + /// no proof of life: PrepareAsync and the judge's provider call are not cut off by the case + /// timeout, so a live case can go quiet for longer than it. AgentTestController fails a Running + /// run whose heartbeat stopped. + /// + public TimeSpan HeartbeatInterval { get; init; } = TimeSpan.FromSeconds(30); + public async Task ExecuteAsync(string runId, CancellationToken ct) { var run = await _repo.GetRunAsync(runId); @@ -139,22 +147,6 @@ public async Task ExecuteAsync(string runId, CancellationToken ct) // it stands today. var parallelism = AgentTestConcurrency.Resolve(run.MaxParallelCases, suite.MaxParallelCases); - run.MaxParallelCases = parallelism; - run.Status = AgentTestStatus.Running; - run.StartedAt = DateTime.UtcNow; - run.TotalCount = 0; - run.PassedCount = 0; - run.FailedCount = 0; - run.ErrorCount = 0; - await _repo.UpdateRunAsync(run); - - // "Stop starting new work" (cancelRequested) is a different fact from "the run ended - // early because of it" (cancelSkipped). A cancel that lands while the LAST case is - // executing stops nothing -- the run finished everything it was ever going to run -- so - // only a case that was actually held back turns the terminal status into Cancelled. - var cancelRequested = false; - var cancelSkipped = 0; - // The model dimension. An absent/empty Models list means "one pass, using each agent's own // LlmConfig" -- byte-for-byte the behavior from before multi-model existed, which is what // keeps every historical run document and every caller that omits the field working. @@ -171,6 +163,26 @@ public async Task ExecuteAsync(string runId, CancellationToken ct) .SelectMany(testCase => models.Select(model => (Case: testCase, Model: model))) .ToList(); + run.MaxParallelCases = parallelism; + run.Status = AgentTestStatus.Running; + run.StartedAt = DateTime.UtcNow; + run.PlannedCount = workItems.Count; + run.TotalCount = 0; + run.PassedCount = 0; + run.FailedCount = 0; + run.ErrorCount = 0; + await _repo.UpdateRunAsync(run); + + // "Stop starting new work" (cancelRequested) is a different fact from "the run ended + // early because of it" (cancelSkipped). A cancel that lands while the LAST case is + // executing stops nothing -- the run finished everything it was ever going to run -- so + // only a case that was actually held back turns the terminal status into Cancelled. + // Seeded from the run, not false: a cancel sent while the run was still Pending is already + // on it (UpdateRunAsync above carries over one that raced the Running write), and no case + // should start after it. + var cancelRequested = run.CancelRequested; + var cancelSkipped = 0; + // Serialises everything that touches the run document or the result collection, so the // only thing that actually overlaps is the case executions. Each pass below is a // read-modify-write of one document (GetRunAsync, then UpdateRunAsync -- a whole-document @@ -187,6 +199,9 @@ public async Task ExecuteAsync(string runId, CancellationToken ct) // through a disappearing process. var hostStopping = false; + var heartbeatStop = CancellationTokenSource.CreateLinkedTokenSource(ct); + var heartbeat = BeatAsync(runId, heartbeatStop.Token); + // MaxDegreeOfParallelism, not the CancellationToken: passing `ct` here would surface host // shutdown as an OperationCanceledException out of this method, and this run's row has to // be left exactly as it is on that path (see the hostStopping check inside). At @@ -203,9 +218,8 @@ await Parallel.ForEachAsync( { // The host itself is shutting down (BackgroundService's stoppingToken), not a // user-requested cancel of THIS run. Leave the row as Running rather than racing - // the shutdown grace period to persist a different terminal status here -- the - // queue's own startup reconciliation step sweeps any Running row a killed process - // left behind into Error the next time it boots. + // the shutdown grace period to persist a different terminal status here -- once + // the heartbeat stops, the next read of the run fails it (see HeartbeatInterval). Volatile.Write(ref hostStopping, true); return; } @@ -288,11 +302,6 @@ await Parallel.ForEachAsync( break; } - if (run.CancelRequested) - { - Volatile.Write(ref cancelRequested, true); - } - // Progress, one line per finished case. Serial execution used to make progress // readable from the order of a case's own log lines; several cases interleaved do // not, so the run says where it is itself. @@ -302,9 +311,17 @@ await Parallel.ForEachAsync( // Real-time accumulation, per case -- not just at the very end. Without this, GET // /agent-test/runs/{id} shows 0/0/0/0 for the run's entire duration, and a process - // death mid-run leaves the startup sweep's Error row claiming zero cases ran even + // death mid-run leaves the abandoned-run Error row claiming zero cases ran even // though N AgentTestCaseResult rows already exist for it. + await SummarisePerformanceAsync(run); await _repo.UpdateRunAsync(run); + + // After the write, not before: UpdateRunAsync sets CancelRequested on `run` when a + // cancel landed between the re-read above and this write. + if (run.CancelRequested) + { + Volatile.Write(ref cancelRequested, true); + } } finally { @@ -312,6 +329,10 @@ await Parallel.ForEachAsync( } }); + heartbeatStop.Cancel(); + await heartbeat; + heartbeatStop.Dispose(); + if (hostStopping) { return; @@ -325,17 +346,51 @@ await Parallel.ForEachAsync( run.CompletedAt = DateTime.UtcNow; await SummarisePerformanceAsync(run); + run.JudgePricing = SnapshotJudgePricing(run, suite); await _repo.UpdateRunAsync(run); } + /// + /// Stamps UpdateDate every HeartbeatInterval until stopped, or until the run is no longer + /// Running -- e.g. the queue marked it Error after a crash out of the case loop. A failed write + /// is logged and retried on the next tick rather than ending the heartbeat, which would get a + /// live run failed as abandoned. + /// + private async Task BeatAsync(string runId, CancellationToken ct) + { + using var timer = new PeriodicTimer(HeartbeatInterval); + try + { + while (await timer.WaitForNextTickAsync(ct)) + { + try + { + if (!await _repo.TouchRunAsync(runId)) + { + return; + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Could not record the heartbeat of agent test run {RunId}.", runId); + } + } + } + catch (OperationCanceledException) + { + } + } + /// /// Fills in the run's per-model latency, token and cost figures, plus the pricing that produced /// the cost. /// - /// Once, at the end, reading the results back -- not accumulated per case like the counts are. - /// A percentile is not incrementally computable: it needs every value at once, and keeping the - /// whole list on the run document to update it in place would store the same numbers twice. + /// Recomputed from the stored results after every case and once more at the end -- not + /// accumulated like the counts are. A percentile is not incrementally computable: it needs every + /// value at once, and keeping the whole list on the run document to update it in place would + /// store the same numbers twice. Per case so the run page shows spend while the run goes, and a + /// run whose host dies keeps the figures for the cases that finished. /// private async Task SummarisePerformanceAsync(AgentTestRun run) { @@ -383,7 +438,9 @@ private static PerformanceSummary BuildSummary( // Tokens and cost come from EVERY result, unlike latency: a case that errored still spent // whatever it spent, and hiding that would understate the run's real cost. TotalTokens = results.Sum(r => r.TotalTokens), - TotalCost = results.Sum(r => r.Cost) + TotalCost = results.Sum(r => r.Cost), + JudgeTokens = results.Sum(r => r.JudgeTokens), + JudgeCost = results.Sum(r => r.JudgeCost) }; } @@ -421,31 +478,47 @@ private List SnapshotPricing(AgentTestRun run) } return run.Models - .Select(m => - { - LlmModelSetting? setting = null; - try - { - setting = _llmProviders.GetSetting(m.Provider, m.Model); - } - catch (Exception ex) - { - _logger.LogWarning(ex, - "Could not read pricing for {Provider}/{Model} while summarising run {RunId}.", - m.Provider, m.Model, run.Id); - } - - return new ModelPricingSnapshot - { - Provider = m.Provider, - Model = m.Model, - TextInputCost = setting?.Cost?.TextInputCost, - TextOutputCost = setting?.Cost?.TextOutputCost - }; - }) + .Select(m => SnapshotModelPricing(m.Provider, m.Model, run.Id)) .ToList(); } + /// + /// The suite's judge model and its unit costs. The same suite instance the cases were judged + /// with, so it names the judge that actually scored this run. + /// + private ModelPricingSnapshot? SnapshotJudgePricing(AgentTestRun run, AgentTestSuite suite) + { + if (string.IsNullOrWhiteSpace(suite.JudgeProvider) || string.IsNullOrWhiteSpace(suite.JudgeModel)) + { + return null; + } + + return SnapshotModelPricing(suite.JudgeProvider, suite.JudgeModel, run.Id); + } + + private ModelPricingSnapshot SnapshotModelPricing(string provider, string model, string runId) + { + LlmModelSetting? setting = null; + try + { + setting = _llmProviders?.GetSetting(provider, model); + } + catch (Exception ex) + { + _logger.LogWarning(ex, + "Could not read pricing for {Provider}/{Model} while summarising run {RunId}.", + provider, model, runId); + } + + return new ModelPricingSnapshot + { + Provider = provider, + Model = model, + TextInputCost = setting?.Cost?.TextInputCost, + TextOutputCost = setting?.Cost?.TextOutputCost + }; + } + /// /// Folds one case result into the run's per-model routing accuracy. Only Routing cases count: /// the evaluation framework gates routing accuracy separately from the agent pass rate, so diff --git a/src/Plugins/BotSharp.Plugin.AgentTesting/Services/AgentTestRunQueue.cs b/src/Plugins/BotSharp.Plugin.AgentTesting/Services/AgentTestRunQueue.cs index d21c690ea..6497331b1 100644 --- a/src/Plugins/BotSharp.Plugin.AgentTesting/Services/AgentTestRunQueue.cs +++ b/src/Plugins/BotSharp.Plugin.AgentTesting/Services/AgentTestRunQueue.cs @@ -1,4 +1,5 @@ using System.Threading.Channels; +using BotSharp.Abstraction.MLTasks; using Microsoft.Extensions.Hosting; using BotSharp.Plugin.AgentTesting.Repositories; @@ -48,11 +49,9 @@ public void Enqueue(string runId) protected override async Task ExecuteAsync(CancellationToken stoppingToken) { - // The queue is in-process, so a restart wipes out every run still in flight. Without - // sweeping them to Error they stay Running forever and the admin page keeps claiming they - // are still going. Runs once per host start. - await ReconcileStaleRunningRunsAsync(); - + // The queue is in-process, so a restart loses every run still in flight. Those are failed + // when read (AgentTestController.FailIfAbandonedAsync), not swept here on startup: hosts + // share one database, and a sweep cannot tell another host's live run from a dead one. while (!stoppingToken.IsCancellationRequested) { try @@ -71,35 +70,6 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) } } - private async Task ReconcileStaleRunningRunsAsync() - { - try - { - using var scope = _serviceProvider.CreateScope(); - var repo = scope.ServiceProvider.GetRequiredService(); - - // Status-filtered at the query level, not ListRunsAsync(null) filtered in memory -- - // this collection only ever grows, and every host restart used to pull the entire - // thing into memory just to find a handful of stale Running rows. - var runs = await repo.ListRunsByStatusAsync(AgentTestStatus.Running); - foreach (var run in runs) - { - run.Status = AgentTestStatus.Error; - run.Error = "The host restarted while this run was still going, so it was abandoned. " - + "The queue is in-process and does not survive a restart -- trigger the run again."; - run.CompletedAt = DateTime.UtcNow; - await repo.UpdateRunAsync(run); - _logger.LogWarning( - "Agent test run {RunId} was left Running by a previous process; marked Error on startup.", - run.Id); - } - } - catch (Exception ex) - { - _logger.LogError(ex, "Failed to reconcile leftover Running agent test runs on startup."); - } - } - private async Task ProcessAsync(string runId, CancellationToken ct) { try @@ -108,7 +78,8 @@ private async Task ProcessAsync(string runId, CancellationToken ct) var repo = scope.ServiceProvider.GetRequiredService(); var executorLogger = scope.ServiceProvider.GetRequiredService>(); - var executor = new AgentTestRunExecutor(repo, new ScopedCaseRunner(_serviceProvider), executorLogger); + var executor = new AgentTestRunExecutor(repo, new ScopedCaseRunner(_serviceProvider), executorLogger, + scope.ServiceProvider.GetService()); await executor.ExecuteAsync(runId, ct); } catch (Exception ex) diff --git a/src/Plugins/BotSharp.Plugin.AgentTesting/Services/BotSharpAgentConversationDriver.cs b/src/Plugins/BotSharp.Plugin.AgentTesting/Services/BotSharpAgentConversationDriver.cs index e38e83f76..fe9e208b8 100644 --- a/src/Plugins/BotSharp.Plugin.AgentTesting/Services/BotSharpAgentConversationDriver.cs +++ b/src/Plugins/BotSharp.Plugin.AgentTesting/Services/BotSharpAgentConversationDriver.cs @@ -3,6 +3,8 @@ using BotSharp.Abstraction.Agents; using BotSharp.Abstraction.Agents.Enums; using BotSharp.Abstraction.Conversations.Enums; +using BotSharp.Abstraction.Files.Models; +using BotSharp.Abstraction.Instructs; using BotSharp.Abstraction.Messaging; using BotSharp.Abstraction.Models; using BotSharp.Abstraction.Repositories; @@ -35,6 +37,7 @@ public class BotSharpAgentConversationDriver : IAgentConversationDriver private readonly IRoutingService _routing; private readonly IBotSharpRepository _repository; private readonly IAgentService _agents; + private readonly IInstructService _instructs; private readonly ILogger _logger; // Set at most once per instance: this driver is scoped per-case (a fresh DI scope per case, @@ -49,12 +52,14 @@ public BotSharpAgentConversationDriver( IRoutingService routing, IBotSharpRepository repository, IAgentService agents, + IInstructService instructs, ILogger logger) { _conversations = conversations; _routing = routing; _repository = repository; _agents = agents; + _instructs = instructs; _logger = logger; } @@ -129,7 +134,8 @@ public async Task InjectHistoryAsync( return stored.Count(d => d.MetaData?.MessageId != null && written.Contains(d.MetaData.MessageId)); } - public async Task SendAsync(string conversationId, string agentId, string userMessage, CancellationToken ct) + public async Task SendAsync( + string conversationId, string agentId, string userMessage, IReadOnlyList files, CancellationToken ct) { // Fail fast only if the case had already timed out before we ever got here (e.g. a prior // turn ran long). Once the real call below starts, there is no cancellation hook left to @@ -140,7 +146,10 @@ public async Task SendAsync(string conversationId, string agentId, s await _conversations.SendMessage( agentId, - new RoleDialogModel(AgentRole.User, userMessage), + new RoleDialogModel(AgentRole.User, userMessage) + { + Files = files.Select(f => new BotSharpFile { FileName = f.FileName, FileUrl = f.FileUrl }).ToList() + }, replyMessage: null, onResponseReceived: r => { @@ -159,6 +168,25 @@ await _conversations.SendMessage( return new AgentReply(message?.Text ?? last?.Content, SerializeRichMessage(message)); } + public async Task InstructAsync( + string agentId, string templateName, string userMessage, IReadOnlyList files, CancellationToken ct) + { + // Same fail-fast-only cancellation as SendAsync: Execute takes no token either. + ct.ThrowIfCancellationRequested(); + + // "#TEMPLATE#" makes the rendered template the system instruction and userMessage the user + // message. Without it InstructService sends the template as the user message and drops + // userMessage entirely, which is not how code that calls a template in production uses it. + var result = await _instructs.Execute( + agentId, + new RoleDialogModel(AgentRole.User, userMessage), + instruction: "#TEMPLATE#", + templateName: templateName, + files: files.Select(f => new InstructFileModel { FileName = f.FileName, FileUrl = f.FileUrl }).ToList()); + + return new AgentReply(result.Text, null); + } + // Unescaped so apostrophes and dashes stay literal: assertions match against this text and a // judge reads it, and neither should have to see "it's". private static readonly JsonSerializerOptions RichMessageJsonOptions = new() diff --git a/src/Plugins/BotSharp.Plugin.AgentTesting/Services/CaseValidation.cs b/src/Plugins/BotSharp.Plugin.AgentTesting/Services/CaseValidation.cs index 6a413e44b..56381f668 100644 --- a/src/Plugins/BotSharp.Plugin.AgentTesting/Services/CaseValidation.cs +++ b/src/Plugins/BotSharp.Plugin.AgentTesting/Services/CaseValidation.cs @@ -80,7 +80,39 @@ public static class CaseValidation } } - return ValidateRoutingCase(caseType ?? CaseTypes.Agent, request, allAssertions); + return ValidateRoutingCase(caseType ?? CaseTypes.Agent, request, allAssertions) + ?? ValidateTemplateCase(caseType ?? CaseTypes.Agent, request); + } + + /// + /// The entry agent is required rather than falling back to the suite's, so the controller can + /// check the template exists on a known agent at save time instead of every run failing on it. + /// History is rejected because a template call never reads the dialog: it would be silently + /// ignored and the case would test a context it never had. + /// + private static string? ValidateTemplateCase(string caseType, AgentTestCaseUpsertRequest request) + { + if (!string.Equals(caseType, CaseTypes.Template, StringComparison.Ordinal)) + { + return null; + } + + if (string.IsNullOrWhiteSpace(request.EntryAgentId)) + { + return "a Template case needs an entryAgentId: the agent that owns the template"; + } + + if (string.IsNullOrWhiteSpace(request.TemplateName)) + { + return "a Template case needs a templateName"; + } + + if ((request.History ?? []).Count > 0) + { + return "a Template case cannot have history: a template call does not read the conversation"; + } + + return null; } /// diff --git a/src/Plugins/BotSharp.Plugin.AgentTesting/Services/IAgentConversationDriver.cs b/src/Plugins/BotSharp.Plugin.AgentTesting/Services/IAgentConversationDriver.cs index 14113d6af..262c8d633 100644 --- a/src/Plugins/BotSharp.Plugin.AgentTesting/Services/IAgentConversationDriver.cs +++ b/src/Plugins/BotSharp.Plugin.AgentTesting/Services/IAgentConversationDriver.cs @@ -23,7 +23,16 @@ Task InjectHistoryAsync( string conversationId, string agentId, IReadOnlyList history); /// Drives one turn and returns what that turn produced. - Task SendAsync(string conversationId, string agentId, string userMessage, CancellationToken ct); + Task SendAsync( + string conversationId, string agentId, string userMessage, IReadOnlyList files, CancellationToken ct); + + /// + /// Calls one of the agent's templates directly, outside the conversation: the template is the + /// system instruction and the user message, with + /// attached to it. Rich content is always null. + /// + Task InstructAsync( + string agentId, string templateName, string userMessage, IReadOnlyList files, CancellationToken ct); /// Calls the canary function once; returns whether the mock seam took it over. Task RunCanaryAsync(string conversationId, string agentId, CancellationToken ct); diff --git a/src/Plugins/BotSharp.Plugin.AgentTesting/Services/LlmCaseAuthor.cs b/src/Plugins/BotSharp.Plugin.AgentTesting/Services/LlmCaseAuthor.cs index 06b25a464..75805ee13 100644 --- a/src/Plugins/BotSharp.Plugin.AgentTesting/Services/LlmCaseAuthor.cs +++ b/src/Plugins/BotSharp.Plugin.AgentTesting/Services/LlmCaseAuthor.cs @@ -415,6 +415,11 @@ public static AgentTestAuthorResponse Assemble( { Apply(merged, attempt.Draft, field); } + + if (attempt.DeclaredFields.Contains(AuthorFields.Turns)) + { + RestoreTurnFiles(baseline.Turns, merged.Turns, warnings); + } } } @@ -451,6 +456,7 @@ private static void Apply(AgentTestCaseUpsertRequest target, AgentTestCaseUpsert case AuthorFields.Enabled: target.Enabled = source.Enabled; break; case AuthorFields.CaseType: target.CaseType = source.CaseType; break; case AuthorFields.EntryAgentId: target.EntryAgentId = source.EntryAgentId; break; + case AuthorFields.TemplateName: target.TemplateName = source.TemplateName; break; case AuthorFields.Turns: target.Turns = source.Turns ?? []; break; case AuthorFields.Assertions: target.Assertions = source.Assertions ?? []; break; case AuthorFields.InitialStates: target.InitialStates = source.InitialStates ?? []; break; @@ -466,11 +472,45 @@ private static void Apply(AgentTestCaseUpsertRequest target, AgentTestCaseUpsert } } + /// + /// A model rewriting turns tends to copy back only what the prompt talks about, and the files a + /// turn attaches are easy to leave out -- the turn then runs without them and nothing says so. + /// With the same number of turns a turn is the same turn, so an empty file list is filled from + /// the baseline; once turns were added or removed the positions no longer line up, so a loss is + /// only reported. + /// + private static void RestoreTurnFiles(List? baseline, List? merged, List warnings) + { + baseline ??= []; + merged ??= []; + + if (baseline.Count == merged.Count) + { + for (var i = 0; i < merged.Count; i++) + { + if ((merged[i].Files ?? []).Count == 0 && (baseline[i].Files ?? []).Count > 0) + { + merged[i].Files = baseline[i].Files; + } + } + return; + } + + var before = baseline.Sum(t => (t.Files ?? []).Count); + var after = merged.Sum(t => (t.Files ?? []).Count); + if (after < before) + { + warnings.Add($"the turns now attach {after} file(s) where they attached {before}; " + + "check each turn's files"); + } + } + /// /// Corrects what is provably wrong and flags what is merely suspect. /// - /// The split matters. A function name is authoritative -- it comes from the agent definition, so a - /// mock naming something else can never match and is dropped. A state key is not: nothing in this + /// The split matters. Only a mock with no function name is provably wrong and dropped. A function + /// the entry agent lacks is merely suspect: a routed-to agent may call it, and mocks match across + /// every agent in the conversation. A state key is the same: nothing in this /// system enumerates the keys an agent writes (a state value records the message it was written /// on and a coarse source, never a function name), so the only key list available is the one /// other cases in this suite happen to use. Dropping a key for being absent from an admittedly @@ -488,13 +528,20 @@ private static void Sanitise( var keptMocks = new List(); foreach (var mock in draft.Mocks ?? []) { - if (string.IsNullOrWhiteSpace(mock.FunctionName) || !callable.Contains(mock.FunctionName)) + if (string.IsNullOrWhiteSpace(mock.FunctionName)) { - warnings.Add($"dropped a mock for '{mock.FunctionName}': this agent has no such function, " - + "so the mock could never match at run time"); + warnings.Add("dropped a mock with no function name: it could never match at run time"); continue; } + // Kept, not dropped: mocks match by function name for every agent in the conversation, + // so a mock for a function of an agent the entry agent routes to is valid and needed. + if (!callable.Contains(mock.FunctionName)) + { + warnings.Add($"the entry agent has no function '{mock.FunctionName}'; the mock was kept " + + "because a routed-to agent may call it -- check the name"); + } + keptMocks.Add(mock); } draft.Mocks = keptMocks; @@ -577,11 +624,12 @@ private static List FilterAssertions( string.Equals(assertion.Type, AssertionTypes.ToolCalled, StringComparison.Ordinal) || string.Equals(assertion.Type, AssertionTypes.ToolNotCalled, StringComparison.Ordinal); + // Warned, not dropped, for the same reason as mocks in Sanitise: a routed-to agent's + // calls are observed too. if (isToolAssertion && !string.IsNullOrWhiteSpace(assertion.Target) && !callable.Contains(assertion.Target!)) { - warnings.Add($"dropped a {assertion.Type} assertion on {where}: this agent has no function " - + $"called '{assertion.Target}'"); - continue; + warnings.Add($"the {assertion.Type} assertion on {where} names '{assertion.Target}', which the " + + "entry agent has no function for; kept because a routed-to agent may call it -- check the name"); } kept.Add(assertion); @@ -620,6 +668,7 @@ private static List Diff(AgentTestCaseUpsertRequest before, AgentT AuthorFields.Enabled => draft.Enabled, AuthorFields.CaseType => draft.CaseType, AuthorFields.EntryAgentId => draft.EntryAgentId, + AuthorFields.TemplateName => draft.TemplateName, AuthorFields.Turns => draft.Turns, AuthorFields.Assertions => draft.Assertions, AuthorFields.InitialStates => draft.InitialStates, @@ -851,6 +900,8 @@ CASE TYPE RULES - caseType "Routing" means "did the router pick the right agent": exactly one turn, at least one routedToAgent or agentChain assertion, and no llmJudge. - caseType "Agent" is everything else, including multi-agent journeys. + - caseType "Template" calls one template of the entry agent directly: it needs + entryAgentId and templateName, and no history. - priority P0/P1/P2 decides which batch runs first; severity S0/S1/S2 says what a failure means. Leave them at P1/S1 unless the user says otherwise. diff --git a/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs index b9437120f..da9c72aed 100644 --- a/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs @@ -634,6 +634,19 @@ private void CollectMessageContentParts(List contentParts, List BlockedOnFirstSend { get; } = []; - public async Task SendAsync(string conversationId, string agentId, string userMessage, CancellationToken ct) + /// Every InstructAsync call, so a test can tell a template call from a conversation turn. + public List<(string AgentId, string TemplateName, string UserMessage, IReadOnlyList Files)> Instructed { get; } = []; + public List> SentFiles { get; } = []; + + public Task InstructAsync( + string agentId, string templateName, string userMessage, IReadOnlyList files, CancellationToken ct) { + Instructed.Add((agentId, templateName, userMessage, files)); + return Task.FromResult(new AgentReply(Replies.Count > 0 ? Replies.Dequeue() : string.Empty, null)); + } + + public async Task SendAsync( + string conversationId, string agentId, string userMessage, IReadOnlyList files, CancellationToken ct) + { + SentFiles.Add(files); Sent.Add(userMessage); SentAgentIds.Add(agentId); OnSend?.Invoke(); @@ -240,6 +253,60 @@ private static AgentTestCaseRunner Build( CaseTimeoutSeconds = timeoutSeconds }; + [Fact] + public async Task A_template_case_calls_the_template_with_the_turn_files_and_never_sends_a_message() + { + var driver = new FakeDriver(); + driver.Replies.Enqueue("{\"issue_photos\":[]}"); + var runner = Build(driver, out _); + var file = new TestFile { FileName = "dark.jpg", FileUrl = "https://example.com/dark.jpg" }; + + var result = await runner.RunAsync(Suite(), new AgentTestCase + { + Id = "case-1", + Name = "photo check", + CaseType = CaseTypes.Template, + EntryAgentId = "wo-creator", + TemplateName = "photo_upload_check", + Turns = + [ + new TestTurn + { + Index = 0, + UserMessage = "{\"issue_description\":\"leak\"}", + Files = [file], + Assertions = [new TestAssertion { Type = AssertionTypes.OutputContains, Expected = "issue_photos" }] + } + ] + }, "run-1", model: null, CancellationToken.None); + + Assert.Equal(AgentTestStatus.Passed, result.Status); + Assert.Empty(driver.Sent); + var call = Assert.Single(driver.Instructed); + Assert.Equal("wo-creator", call.AgentId); + Assert.Equal("photo_upload_check", call.TemplateName); + Assert.Same(file, Assert.Single(call.Files)); + } + + [Fact] + public async Task An_agent_case_passes_the_turn_files_to_the_conversation() + { + var driver = new FakeDriver(); + driver.Replies.Enqueue("ok"); + var runner = Build(driver, out _); + var file = new TestFile { FileName = "sink.jpg", FileUrl = "https://example.com/sink.jpg" }; + + await runner.RunAsync(Suite(), new AgentTestCase + { + Id = "case-1", + Name = "photo in a conversation", + Turns = [new TestTurn { Index = 0, UserMessage = "here is the sink", Files = [file] }] + }, "run-1", model: null, CancellationToken.None); + + Assert.Empty(driver.Instructed); + Assert.Same(file, Assert.Single(Assert.Single(driver.SentFiles))); + } + [Fact] public async Task A_blocked_tool_fails_the_case_even_when_every_authored_assertion_passed() { @@ -976,6 +1043,51 @@ public async Task Token_usage_is_recorded_as_a_delta_not_an_absolute_reading() Assert.Equal(0.03, result.Cost, precision: 4); } + /// Spends tokens the way the real judge does: into the case's own ITokenStatistics. + private sealed class SpendingJudge(FakeTokens tokens) : IAgentTestJudge + { + public Task JudgeAsync(TestAssertion assertion, AssertionContext context, AgentTestSuite suite, CancellationToken ct) + { + tokens.Total += 1000; + tokens.AccumulatedCost += 0.5f; + return Task.FromResult(new AssertionResult { Type = assertion.Type, Passed = true }); + } + } + + [Fact] + public async Task Judge_usage_is_not_billed_to_the_model_under_test() + { + var driver = new FakeDriver(); + driver.Replies.Enqueue("done"); + var tokens = new FakeTokens(); + driver.OnSend = () => { tokens.Total += 300; tokens.AccumulatedCost += 0.03f; }; + var runner = new AgentTestCaseRunner( + new AgentTestRunRegistry(), driver, NullLogger.Instance, + judge: new SpendingJudge(tokens), tokens: tokens); + + var result = await runner.RunAsync(Suite(), new AgentTestCase + { + Id = "case-1", + Name = "c", + Turns = + [ + new TestTurn + { + Index = 0, + UserMessage = "a", + Assertions = [new TestAssertion { Type = AssertionTypes.LlmJudge, Expected = "anything" }] + } + ], + Assertions = [new TestAssertion { Type = AssertionTypes.LlmJudge, Expected = "anything" }] + }, "run-1", model: null, CancellationToken.None); + + Assert.Equal(AgentTestStatus.Passed, result.Status); + Assert.Equal(300, result.TotalTokens); + Assert.Equal(0.03, result.Cost, precision: 4); + Assert.Equal(2000, result.JudgeTokens); + Assert.Equal(1.0, result.JudgeCost, precision: 4); + } + [Fact] public async Task Usage_is_still_recorded_when_the_case_times_out() { diff --git a/tests/BotSharp.Core.UnitTests/AgentTesting/AgentTestControllerTests.cs b/tests/BotSharp.Core.UnitTests/AgentTesting/AgentTestControllerTests.cs index a1060a33b..4b1b8da4c 100644 --- a/tests/BotSharp.Core.UnitTests/AgentTesting/AgentTestControllerTests.cs +++ b/tests/BotSharp.Core.UnitTests/AgentTesting/AgentTestControllerTests.cs @@ -82,9 +82,52 @@ public Task CreateRunAsync(AgentTestRun run) } public Task GetRunAsync(string id) => Task.FromResult(Runs.TryGetValue(id, out var r) ? r : null); public Task> ListRunsAsync(string? suiteId) => Task.FromResult(Runs.Values.ToList()); - public Task> ListRunsByStatusAsync(string status) - => Task.FromResult(Runs.Values.Where(r => r.Status == status).ToList()); - public Task UpdateRunAsync(AgentTestRun run) { Runs[run.Id] = run; return Task.CompletedTask; } + public Task UpdateRunAsync(AgentTestRun run) + { + run.UpdateDate = DateTime.UtcNow; + Runs[run.Id] = run; + return Task.CompletedTask; + } + public Task RequestRunCancelAsync(string id) + { + if (!Runs.TryGetValue(id, out var run) + || (run.Status != AgentTestStatus.Pending && run.Status != AgentTestStatus.Running)) + { + return Task.FromResult(false); + } + run.CancelRequested = true; + return Task.FromResult(true); + } + + public Task TouchRunAsync(string id) + { + if (!Runs.TryGetValue(id, out var run) || run.Status != AgentTestStatus.Running) + { + return Task.FromResult(false); + } + run.UpdateDate = DateTime.UtcNow; + return Task.FromResult(true); + } + + /// Runs first inside TryAbandonRunAsync: an executor write landing after the read. + public Action? BeforeAbandon { get; set; } + + public Task TryAbandonRunAsync(string id, DateTime? observedUpdateDate, string error) + { + if (!Runs.TryGetValue(id, out var run)) + { + return Task.FromResult(false); + } + BeforeAbandon?.Invoke(run); + if (run.Status != AgentTestStatus.Running || run.UpdateDate != observedUpdateDate) + { + return Task.FromResult(false); + } + run.Status = AgentTestStatus.Error; + run.Error = error; + run.CompletedAt = DateTime.UtcNow; + return Task.FromResult(true); + } /// /// Cascades exactly as the real repository does. A fake that dropped only the run would let @@ -745,6 +788,112 @@ public async Task CancelRun_still_accepts_a_run_that_has_not_finished(string liv Assert.True(repo.Runs["run-1"].CancelRequested); } + // ---- Template cases: the template name must match exactly, as the runtime looks it up ----- + + [Theory] + [InlineData("photo_check", true)] + [InlineData("PHOTO_CHECK", false)] + public async Task A_template_case_needs_the_template_name_spelled_exactly(string templateName, bool accepted) + { + var repo = new InMemoryRepo(); + repo.Suites["suite-1"] = EnabledSuite(); + var agents = new Mock(); + agents.Setup(x => x.GetAgent("agent-1")).ReturnsAsync(new Agent + { + Id = "agent-1", Name = "agent-1", Templates = [new AgentTemplate("photo_check", "check the photo")] + }); + var controller = BuildController(repo, out _, agents: agents.Object); + + var request = ValidRequest(); + request.CaseType = CaseTypes.Template; + request.EntryAgentId = "agent-1"; + request.TemplateName = templateName; + + var result = await controller.CreateCase(request); + + Assert.Equal(accepted, result.Result is not BadRequestObjectResult); + Assert.Equal(accepted ? 1 : 0, repo.Cases.Count); + } + + // ---- Abandoned runs: a Running run whose heartbeat stopped for 3 minutes ------------------- + + [Fact] + public async Task GetRun_fails_a_running_run_whose_heartbeat_stopped() + { + var repo = new InMemoryRepo(); + repo.Runs["run-1"] = new AgentTestRun + { + Id = "run-1", SuiteId = "suite-1", Status = AgentTestStatus.Running, + UpdateDate = DateTime.UtcNow.AddMinutes(-4) + }; + var controller = BuildController(repo, out _); + + var result = await controller.GetRun("run-1"); + + Assert.Equal(AgentTestStatus.Error, result.Value!.Run.Status); + Assert.Equal(AgentTestStatus.Error, repo.Runs["run-1"].Status); + Assert.NotNull(repo.Runs["run-1"].CompletedAt); + } + + [Fact] + public async Task ListRuns_leaves_a_running_run_alone_while_its_heartbeat_is_recent() + { + // Another host may be executing it: a long run is fine as long as its heartbeat goes on. + var repo = new InMemoryRepo(); + repo.Runs["run-1"] = new AgentTestRun + { + Id = "run-1", SuiteId = "suite-1", Status = AgentTestStatus.Running, + StartedAt = DateTime.UtcNow.AddHours(-1), + UpdateDate = DateTime.UtcNow.AddMinutes(-2) + }; + var controller = BuildController(repo, out _); + + var runs = await controller.ListRuns(null); + + Assert.Equal(AgentTestStatus.Running, Assert.Single(runs).Status); + } + + [Fact] + public async Task A_stale_read_does_not_fail_a_run_the_executor_wrote_to_since() + { + var repo = new InMemoryRepo(); + repo.Runs["run-1"] = new AgentTestRun + { + Id = "run-1", SuiteId = "suite-1", Status = AgentTestStatus.Running, + UpdateDate = DateTime.UtcNow.AddMinutes(-4) + }; + repo.BeforeAbandon = run => + { + run.TotalCount = 5; + run.UpdateDate = DateTime.UtcNow; + }; + var controller = BuildController(repo, out _); + + var result = await controller.GetRun("run-1"); + + Assert.Equal(AgentTestStatus.Running, result.Value!.Run.Status); + Assert.Equal(AgentTestStatus.Running, repo.Runs["run-1"].Status); + Assert.Equal(5, repo.Runs["run-1"].TotalCount); + } + + [Fact] + public async Task An_abandoned_running_run_can_be_deleted() + { + // A run written before UpdateDate existed falls back to StartedAt. + var repo = new InMemoryRepo(); + repo.Runs["run-1"] = new AgentTestRun + { + Id = "run-1", SuiteId = "suite-1", Status = AgentTestStatus.Running, + StartedAt = DateTime.UtcNow.AddMinutes(-5) + }; + var controller = BuildController(repo, out _); + + var result = await controller.DeleteRuns(new AgentTestRunDeleteRequest { RunIds = ["run-1"] }); + + Assert.Empty(result.Value!.Skipped); + Assert.False(repo.Runs.ContainsKey("run-1")); + } + private static AgentTestCaseUpsertRequest RoutingCaseRequest( int turns = 1, params TestAssertion[] assertions) => new() { diff --git a/tests/BotSharp.Core.UnitTests/AgentTesting/AgentTestJudgeTests.cs b/tests/BotSharp.Core.UnitTests/AgentTesting/AgentTestJudgeTests.cs index 78ab435b2..9ab952319 100644 --- a/tests/BotSharp.Core.UnitTests/AgentTesting/AgentTestJudgeTests.cs +++ b/tests/BotSharp.Core.UnitTests/AgentTesting/AgentTestJudgeTests.cs @@ -212,7 +212,12 @@ private sealed class StubDriver : IAgentConversationDriver public Task PrepareAsync(string conversationId, string agentId, IReadOnlyList initialStates) => Task.CompletedTask; - public Task SendAsync(string conversationId, string agentId, string userMessage, CancellationToken ct) + public Task SendAsync( + string conversationId, string agentId, string userMessage, IReadOnlyList files, CancellationToken ct) + => Task.FromResult(new AgentReply(_reply, null)); + + public Task InstructAsync( + string agentId, string templateName, string userMessage, IReadOnlyList files, CancellationToken ct) => Task.FromResult(new AgentReply(_reply, null)); public Task RunCanaryAsync(string conversationId, string agentId, CancellationToken ct) diff --git a/tests/BotSharp.Core.UnitTests/AgentTesting/AgentTestRunExecutorTests.cs b/tests/BotSharp.Core.UnitTests/AgentTesting/AgentTestRunExecutorTests.cs index be85a6e82..c2e8056ee 100644 --- a/tests/BotSharp.Core.UnitTests/AgentTesting/AgentTestRunExecutorTests.cs +++ b/tests/BotSharp.Core.UnitTests/AgentTesting/AgentTestRunExecutorTests.cs @@ -45,9 +45,15 @@ private sealed class InMemoryRepo : IAgentTestRepository public Task CreateRunAsync(AgentTestRun run) => Task.FromResult(run); public Task GetRunAsync(string id) => Task.FromResult(Run); public Task> ListRunsAsync(string? suiteId) => Task.FromResult(new List { Run }); - public Task> ListRunsByStatusAsync(string status) - => Task.FromResult(Run.Status == status ? new List { Run } : []); public Task UpdateRunAsync(AgentTestRun run) { Run = run; return Task.CompletedTask; } + public Task RequestRunCancelAsync(string id) { Run.CancelRequested = true; return Task.FromResult(true); } + public int Heartbeats; + public Task TouchRunAsync(string id) + { + Interlocked.Increment(ref Heartbeats); + return Task.FromResult(Run.Status == AgentTestStatus.Running); + } + public Task TryAbandonRunAsync(string id, DateTime? observedUpdateDate, string error) => Task.FromResult(false); public Task DeleteRunAsync(string id) { var removed = Results.RemoveAll(r => r.RunId == id); @@ -92,9 +98,25 @@ private sealed class CloningRunRepo : IAgentTestRepository public Task CreateRunAsync(AgentTestRun run) => Task.FromResult(run); public Task GetRunAsync(string id) => Task.FromResult(Clone(_stored)); public Task> ListRunsAsync(string? suiteId) => Task.FromResult(new List { Clone(_stored) }); - public Task> ListRunsByStatusAsync(string status) - => Task.FromResult(_stored.Status == status ? new List { Clone(_stored) } : []); - public Task UpdateRunAsync(AgentTestRun run) { _stored = Clone(run); return Task.CompletedTask; } + /// + /// A cancel landing just before the next write: like the real repository, the write keeps + /// it and sets it on the caller's run. + /// + public bool CancelBeforeNextWrite; + + public Task UpdateRunAsync(AgentTestRun run) + { + if (CancelBeforeNextWrite) + { + CancelBeforeNextWrite = false; + run.CancelRequested = true; + } + _stored = Clone(run); + return Task.CompletedTask; + } + public Task RequestRunCancelAsync(string id) { _stored.CancelRequested = true; return Task.FromResult(true); } + public Task TouchRunAsync(string id) => Task.FromResult(_stored.Status == AgentTestStatus.Running); + public Task TryAbandonRunAsync(string id, DateTime? observedUpdateDate, string error) => Task.FromResult(false); public Task DeleteRunAsync(string id) { var removed = Results.RemoveAll(r => r.RunId == id); @@ -111,11 +133,13 @@ public Task DeleteRunAsync(string id) TriggeredBy = source.TriggeredBy, CaseIds = source.CaseIds, MaxParallelCases = source.MaxParallelCases, + PlannedCount = source.PlannedCount, TotalCount = source.TotalCount, PassedCount = source.PassedCount, FailedCount = source.FailedCount, ErrorCount = source.ErrorCount, CancelRequested = source.CancelRequested, + PerformanceSummaries = source.PerformanceSummaries, StartedAt = source.StartedAt, CompletedAt = source.CompletedAt, CreateDate = source.CreateDate @@ -345,6 +369,24 @@ public async Task Stops_between_cases_once_cancellation_is_requested() Assert.Equal(AgentTestStatus.Cancelled, repo.Run.Status); } + [Fact] + public async Task A_cancel_sent_while_the_run_was_pending_starts_no_case() + { + var repo = new InMemoryRepo { Cases = [CaseNamed("a"), CaseNamed("b")] }; + repo.Run.CancelRequested = true; + var started = 0; + var executor = Build(repo, c => + { + started++; + return new AgentTestCaseResult { CaseId = c.Id, Status = AgentTestStatus.Passed }; + }); + + await executor.ExecuteAsync("run-1", CancellationToken.None); + + Assert.Equal(0, started); + Assert.Equal(AgentTestStatus.Cancelled, repo.Run.Status); + } + /// /// An ICaseRunner that records how many cases were in flight at once, and holds each one until /// it is released -- the only way to observe overlap rather than infer it from timing. @@ -590,7 +632,7 @@ public async Task Persists_the_running_totals_after_each_case_not_only_at_the_en // Important 1: GET /agent-test/runs/{id} is the only progress surface while a run is in // flight. If counters are only written once at the very end, a 10-case suite at the // default 120s-per-case timeout would show 0/0/0/0 for up to 20 minutes, and a process - // death mid-run would leave the startup sweep's Error row claiming zero cases ran even + // death mid-run would leave the abandoned-run Error row claiming zero cases ran even // though N AgentTestCaseResult rows already exist. // // This has to use CloningRunRepo, not InMemoryRepo: InMemoryRepo's GetRunAsync/Run field @@ -622,6 +664,87 @@ public async Task Persists_the_running_totals_after_each_case_not_only_at_the_en Assert.Equal(2, repo.Stored.TotalCount); } + [Fact] + public async Task Persists_the_planned_count_and_the_spend_so_far_while_the_run_goes() + { + // The run page shows "1 / 2" and the cost table mid-run, and a run whose host dies keeps + // both for the cases that finished. + var repo = new CloningRunRepo { Cases = [CaseNamed("a"), CaseNamed("b")] }; + AgentTestRun? storedBeforeSecondCase = null; + var executor = new AgentTestRunExecutor( + repo, + new DelegatingCaseRunner(c => + { + if (c.Id == "b") + { + storedBeforeSecondCase = repo.Stored; + } + return new AgentTestCaseResult + { + CaseId = c.Id, Status = AgentTestStatus.Passed, + ModelDurationMs = 10, TotalTokens = 100, Cost = 0.5 + }; + }), + NullLogger.Instance); + + await executor.ExecuteAsync("run-1", CancellationToken.None); + + Assert.Equal(2, storedBeforeSecondCase!.PlannedCount); + Assert.Equal(1, storedBeforeSecondCase.TotalCount); + var summary = Assert.Single(storedBeforeSecondCase.PerformanceSummaries); + Assert.Equal(100, summary.TotalTokens); + Assert.Equal(200, Assert.Single(repo.Stored.PerformanceSummaries).TotalTokens); + } + + [Fact] + public async Task A_cancel_carried_over_by_the_per_case_write_stops_the_next_case() + { + // The cancel lands after the executor re-read the run but before it wrote: only the write + // sees it. + var repo = new CloningRunRepo { Cases = [CaseNamed("a"), CaseNamed("b")] }; + var ran = new List(); + var executor = new AgentTestRunExecutor( + repo, + new DelegatingCaseRunner(c => + { + ran.Add(c.Id); + repo.CancelBeforeNextWrite = true; + return new AgentTestCaseResult { CaseId = c.Id, Status = AgentTestStatus.Passed }; + }), + NullLogger.Instance); + + await executor.ExecuteAsync("run-1", CancellationToken.None); + + Assert.Equal(["a"], ran); + Assert.Equal(AgentTestStatus.Cancelled, repo.Stored.Status); + } + + [Fact] + public async Task Beats_while_a_case_runs_and_stops_once_the_run_ends() + { + // A case can go quiet for longer than its timeout, so liveness comes from the heartbeat, + // not from case completions. + var repo = new InMemoryRepo { Cases = [CaseNamed("a")] }; + var executor = new AgentTestRunExecutor( + repo, + new DelegatingCaseRunner(c => + { + Thread.Sleep(300); + return new AgentTestCaseResult { CaseId = c.Id, Status = AgentTestStatus.Passed }; + }), + NullLogger.Instance) + { + HeartbeatInterval = TimeSpan.FromMilliseconds(20) + }; + + await executor.ExecuteAsync("run-1", CancellationToken.None); + var beatsAtEnd = Volatile.Read(ref repo.Heartbeats); + await Task.Delay(100); + + Assert.True(beatsAtEnd > 0); + Assert.Equal(beatsAtEnd, Volatile.Read(ref repo.Heartbeats)); + } + [Fact] public async Task A_cancel_that_arrives_while_the_only_case_is_executing_still_survives_the_terminal_write() { @@ -827,6 +950,26 @@ public async Task A_case_that_never_reached_the_model_is_left_out_of_the_percent Assert.Equal(0.0015, summary.TotalCost, precision: 6); } + [Fact] + public async Task Judge_spend_is_summarised_apart_from_the_model_under_test() + { + var repo = new InMemoryRepo { Cases = [CaseNamed("a"), CaseNamed("b")] }; + var executor = Build(repo, c => + { + var result = Timed(c.Id, modelMs: 100, tokens: 10, cost: 0.001); + result.JudgeTokens = 40; + result.JudgeCost = 0.002; + return result; + }); + + await executor.ExecuteAsync("run-1", CancellationToken.None); + + var summary = Assert.Single(repo.Run.PerformanceSummaries); + Assert.Equal(20, summary.TotalTokens); + Assert.Equal(80, summary.JudgeTokens); + Assert.Equal(0.004, summary.JudgeCost, precision: 6); + } + [Fact] public async Task Performance_is_summarised_per_model() { diff --git a/tests/BotSharp.Core.UnitTests/AgentTesting/CaseAuthoringTests.cs b/tests/BotSharp.Core.UnitTests/AgentTesting/CaseAuthoringTests.cs index 342283938..108d6879e 100644 --- a/tests/BotSharp.Core.UnitTests/AgentTesting/CaseAuthoringTests.cs +++ b/tests/BotSharp.Core.UnitTests/AgentTesting/CaseAuthoringTests.cs @@ -13,7 +13,8 @@ namespace BotSharp.Core.UnitTests.AgentTesting; /// , and this is where they are pinned: /// /// 1. a field the model does not declare cannot be deleted by being left out of its answer; -/// 2. a mock or tool assertion naming a function the agent cannot call is removed, not stored; +/// 2. a mock or tool assertion naming a function the entry agent lacks is kept with a warning -- +/// mocks match across every agent in the conversation, so a routed-to agent may call it; /// 3. a draft that would fail on save comes back labelled, not presented as progress; /// 4. the diff shown to the user is computed from the two drafts, never read off the model's own /// account of what it did. @@ -66,6 +67,23 @@ private static AgentTestAuthorResponse Run( // ---- 1. A field the model does not declare survives ------------------------------------- + [Fact] + public void A_declared_template_name_is_taken_from_the_model() + { + var modelDraft = new AgentTestCaseUpsertRequest + { + CaseType = CaseTypes.Template, + EntryAgentId = "agent-1", + TemplateName = "photo_check" + }; + + var response = Run(Attempt(["caseType", "entryAgentId", "templateName"], modelDraft)); + + Assert.Equal(AuthorFields.TemplateName, AuthorFields.Normalize("templateName")); + Assert.Equal("photo_check", response.Draft.TemplateName); + Assert.Contains(response.Changes, c => c.Field == AuthorFields.TemplateName); + } + [Fact] public void An_undeclared_field_is_kept_even_when_the_model_returns_it_empty() { @@ -150,10 +168,10 @@ public void Declaring_a_field_without_returning_a_draft_changes_nothing() Assert.Contains(response.Warnings, w => w.Contains("returned no draft")); } - // ---- 2. Never a function the agent cannot call ------------------------------------------ + // ---- 2. A function the entry agent lacks is flagged, not removed ------------------------ [Fact] - public void A_mock_for_an_unknown_function_is_dropped() + public void A_mock_for_a_function_the_entry_agent_lacks_is_kept_with_a_warning() { var modelDraft = new AgentTestCaseUpsertRequest { @@ -166,16 +184,33 @@ public void A_mock_for_an_unknown_function_is_dropped() var response = Run(Attempt(["mocks"], modelDraft)); - Assert.Single(response.Draft.Mocks); - Assert.Equal("get_eta", response.Draft.Mocks[0].FunctionName); + Assert.Equal(["get_eta", "send_sms_to_resident"], response.Draft.Mocks.Select(m => m.FunctionName)); Assert.Contains(response.Warnings, w => w.Contains("send_sms_to_resident")); } [Fact] - public void A_tool_assertion_on_an_unknown_function_is_dropped() + public void A_mock_with_no_function_name_is_dropped() { - // Kept, it would never match at run time, so the case would fail forever for a reason that - // reads as an agent regression. + var modelDraft = new AgentTestCaseUpsertRequest + { + Mocks = + [ + new TestToolMock { FunctionName = "get_eta", ResultContent = "{}" }, + new TestToolMock { FunctionName = " ", ResultContent = "{}" } + ] + }; + + var response = Run(Attempt(["mocks"], modelDraft)); + + Assert.Single(response.Draft.Mocks); + Assert.Contains(response.Warnings, w => w.Contains("no function name")); + } + + [Fact] + public void A_tool_assertion_on_a_function_the_entry_agent_lacks_is_kept_with_a_warning() + { + // Kept: a routed-to agent's calls are observed too. The warning is what catches a name the + // model made up. var modelDraft = new AgentTestCaseUpsertRequest { Assertions = @@ -187,11 +222,67 @@ public void A_tool_assertion_on_an_unknown_function_is_dropped() var response = Run(Attempt(["assertions"], modelDraft)); - Assert.Single(response.Draft.Assertions); - Assert.Equal("get_eta", response.Draft.Assertions[0].Target); + Assert.Equal(["get_eta", "get_estimated_arrival"], response.Draft.Assertions.Select(a => a.Target)); Assert.Contains(response.Warnings, w => w.Contains("get_estimated_arrival")); } + // ---- Turn files survive a rewrite that leaves them out ----------------------------------- + + private static AgentTestCaseUpsertRequest BaselineWithFile() => new() + { + SuiteId = "suite-1", + Name = "dark photo", + Turns = + [ + new TestTurn + { + Index = 0, + UserMessage = "my toilet keeps running", + Files = [new TestFile { FileName = "photo_1.jpg", FileUrl = "https://example.com/photo_1.jpg" }] + } + ] + }; + + [Fact] + public void A_rewritten_turn_that_drops_its_files_gets_them_back() + { + var modelDraft = new AgentTestCaseUpsertRequest + { + Turns = + [ + new TestTurn + { + Index = 0, + UserMessage = "my toilet keeps running", + Assertions = [new TestAssertion { Type = AssertionTypes.OutputContains, Expected = "dark" }] + } + ] + }; + + var response = Run(Attempt(["turns"], modelDraft), baseline: BaselineWithFile()); + + Assert.Equal("photo_1.jpg", Assert.Single(response.Draft.Turns[0].Files).FileName); + Assert.Single(response.Draft.Turns[0].Assertions); + } + + [Fact] + public void Files_are_not_guessed_back_when_the_turn_count_changed() + { + var modelDraft = new AgentTestCaseUpsertRequest + { + Turns = + [ + new TestTurn { Index = 0, UserMessage = "hi" }, + new TestTurn { Index = 1, UserMessage = "my toilet keeps running" } + ] + }; + + var response = Run(Attempt(["turns"], modelDraft), baseline: BaselineWithFile()); + + Assert.All(response.Draft.Turns, t => Assert.Empty(t.Files)); + Assert.Contains(response.Warnings, w => w.Contains("file(s)")); + } + [Fact] public void A_non_tool_assertion_is_never_dropped_for_its_target() {