From d6c05cac477545a5e5227d76df92f2a9ed6819d7 Mon Sep 17 00:00:00 2001 From: "nick.yi" Date: Thu, 29 Oct 2026 14:03:01 +0800 Subject: [PATCH 1/2] feat(agent-testing): show used models, rerun failures on their model, render rich content - Re-run Failures re-runs each failed case on the model it failed with - Label the agent-default row and case results with the models actually used - Show a turn's rich content as its own block on the run page --- src/lib/helpers/types/agentTestTypes.js | 2 + src/lib/helpers/utils/agent-test.js | 14 +++++ .../page/agent-test/run/[runId]/+page.svelte | 55 ++++++++++++++++++- 3 files changed, 69 insertions(+), 2 deletions(-) diff --git a/src/lib/helpers/types/agentTestTypes.js b/src/lib/helpers/types/agentTestTypes.js index 7def88b1..063b0dae 100644 --- a/src/lib/helpers/types/agentTestTypes.js +++ b/src/lib/helpers/types/agentTestTypes.js @@ -317,6 +317,8 @@ * LlmConfig was used. * @property {string?} model - As above. In a multi-model run the same `caseId` yields several * results, told apart by these two fields. + * @property {string[]?} usedModels - "provider/model" of every completion the agents under test + * actually ran on, filled even when `model` is null. Can hold several (router + hand-off target). * @property {number} durationMs - Wall clock for the whole case, including the canary and the * conversation reads. Comparable between models, but not a model-latency measurement. * @property {number} modelDurationMs - The agent calls alone, summed over the turns. This is what the diff --git a/src/lib/helpers/utils/agent-test.js b/src/lib/helpers/utils/agent-test.js index 67a078bd..04060d0d 100644 --- a/src/lib/helpers/utils/agent-test.js +++ b/src/lib/helpers/utils/agent-test.js @@ -302,6 +302,20 @@ export function formatAgentChain(chain) { return chain?.length ? chain.join(' -> ') : '--'; } +/** + * A turn's rich content, pretty-printed. Stored as a JSON string; anything that does not parse is + * shown as-is rather than hidden. + * @param {string} richContent + * @returns {string} + */ +export function formatRichContent(richContent) { + try { + return JSON.stringify(JSON.parse(richContent), null, 2); + } catch { + return richContent; + } +} + /** * @param {string} text * @returns {boolean} diff --git a/src/routes/page/agent-test/run/[runId]/+page.svelte b/src/routes/page/agent-test/run/[runId]/+page.svelte index 41062336..d0f59f1d 100644 --- a/src/routes/page/agent-test/run/[runId]/+page.svelte +++ b/src/routes/page/agent-test/run/[runId]/+page.svelte @@ -14,6 +14,7 @@ formatDuration, formatDateTime, formatAgentChain, + formatRichContent, t } from '$lib/helpers/utils/agent-test.js'; @@ -58,6 +59,28 @@ ...new Set(results.filter(r => r.status === 'Failed' || r.status === 'Error').map(r => r.caseId)) ]); + /** + * The models the failures ran on, so a re-run reproduces them on the same model instead of + * falling back to the agent's own (possibly since changed) LlmConfig. An "agent default" failure + * counts only when it ran on a single model: forcing one model onto a case whose router and + * hand-off target used different ones would re-run a different conversation. Null = re-run on + * the agents' own LlmConfig. + */ + let failedModels = $derived.by(() => { + /** @type {Map} */ + const models = new Map(); + for (const r of results) { + if (r.status !== 'Failed' && r.status !== 'Error') continue; + if (r.model) { + models.set(`${r.provider}/${r.model}`, { provider: r.provider, model: r.model }); + } else if (r.usedModels?.length === 1) { + const [provider, ...rest] = r.usedModels[0].split('/'); + models.set(r.usedModels[0], { provider, model: rest.join('/') }); + } + } + return models.size > 0 ? [...models.values()] : null; + }); + /** Column order for the comparison grid; follows the run's own model order. */ let modelColumns = $derived((run?.models || []).map(m => ({ ...m, key: `${m.provider}/${m.model}` }))); @@ -77,9 +100,20 @@ * Latency, token and cost figures per model, as stored on the run. A table rather than tiles * because there is one row per model and the whole point is comparing them side by side. */ + /** + * A run that named no models still ran on something: each agent's own LlmConfig as it stood at + * run time. Named here because that config may have changed since. + */ + let defaultLabel = $derived.by(() => { + const used = [...new Set(results.filter(r => !r.model).flatMap(r => r.usedModels || []))]; + return used.length > 0 + ? `${$_('agent default')} (${used.map(u => u.split('/').slice(1).join('/')).join(', ')})` + : $_('agent default'); + }); + let performance = $derived((run?.performanceSummaries || []).map(s => ({ ...s, - label: s.model || $_('agent default') + label: s.model || defaultLabel }))); /** @@ -212,7 +246,7 @@ function handleRerunFailed() { if (isBusy || !suiteId || failedCaseIds.length === 0) return; isBusy = true; - triggerRun(suiteId, failedCaseIds).then(newRun => { + triggerRun(suiteId, failedCaseIds, failedModels).then(newRun => { notifySuccess(t('Re-run queued!')); goto(`/page/agent-test/run/${newRun.id}`); }).catch(err => { @@ -557,6 +591,14 @@ {result.model} + {:else} + + {#each result.usedModels || [] as used (used)} + + {used.split('/').slice(1).join('/')} + + {/each} {/if} {formatDuration(result.durationMs)} @@ -618,6 +660,15 @@
{$_('Agent output')}
{turn.output || '--'}
+ {#if turn.richContent} + +
+
{$_('Rich content')}
+
{formatRichContent(turn.richContent)}
+
+ {/if} {#if turn.agentChain?.length > 0}
{$_('Answered by')}
From e61cb20bef571879c7be3c6932aa6d7e5cb29083 Mon Sep 17 00:00:00 2001 From: "nick.yi" Date: Thu, 29 Oct 2026 15:48:15 +0800 Subject: [PATCH 2/2] feat(agent-testing): pick an output assertion's target from content or richContent - A free-text target let typos through, and the backend fails the assertion on an unknown source instead of falling back to the reply text - "content" is stored as "reply", the value the backend expects; blank still means reply --- src/lib/helpers/utils/agent-test.js | 14 ++++++++++++++ src/lib/langs/en.json | 1 + src/lib/langs/zh.json | 1 + .../[suiteId]/case/[caseId]/+page.svelte | 19 ++++++++++++++++++- 4 files changed, 34 insertions(+), 1 deletion(-) diff --git a/src/lib/helpers/utils/agent-test.js b/src/lib/helpers/utils/agent-test.js index 04060d0d..a6ff1fc3 100644 --- a/src/lib/helpers/utils/agent-test.js +++ b/src/lib/helpers/utils/agent-test.js @@ -59,6 +59,13 @@ const TARGET_REQUIRED = ['toolCalled', 'toolNotCalled', 'stateEquals']; */ export const AGENT_CHAIN_MODES = ['contains', 'ordered', 'exact']; +/** + * Which text an output assertion or llmJudge reads (OutputSources in + * AssertionEvaluator.cs), carried in `target`. Blank means `reply`. + */ +export const OUTPUT_SOURCES = ['reply', 'richContent']; +export const OUTPUT_SOURCE_TYPES = ['outputContains', 'outputNotContains', 'outputRegex', 'llmJudge']; + /** * What a case is verifying (CaseTypes in AgentTestCase.cs). Routing is checked * more strictly -- see `validateCaseType` -- because it is the only type counted @@ -241,6 +248,13 @@ export function validateAssertion(assertion) { return t('Assertion "agentChain" mode must be one of {modes}.', { modes: AGENT_CHAIN_MODES.join(', ') }); } + // Same for the output source: an unrecognised value fails the assertion rather + // than falling back to the reply text. Blank is legal and means `reply`. + if (OUTPUT_SOURCE_TYPES.includes(type) && assertion.target?.trim() + && !OUTPUT_SOURCES.some(s => s.toLowerCase() === assertion.target.trim().toLowerCase())) { + return t('Assertion "{type}" target must be one of {sources}.', { type, sources: OUTPUT_SOURCES.join(', ') }); + } + if (assertion.argsMatchJson?.trim() && !isParsableJson(assertion.argsMatchJson)) { return t('The args match on an assertion is not valid JSON.'); } diff --git a/src/lib/langs/en.json b/src/lib/langs/en.json index 75430d0d..9c694915 100644 --- a/src/lib/langs/en.json +++ b/src/lib/langs/en.json @@ -558,6 +558,7 @@ "A Routing case cannot use llmJudge: routing is judged only by which agent handled the conversation.": "A Routing case cannot use llmJudge: routing is judged only by which agent handled the conversation.", "Mode": "Mode", "Assertion \"agentChain\" mode must be one of {modes}.": "Assertion \"agentChain\" mode must be one of {modes}.", + "Assertion \"{type}\" target must be one of {sources}.": "Assertion \"{type}\" target must be one of {sources}.", "Copilot, Work Order Creator": "Copilot, Work Order Creator", "These agents appear somewhere in the chain, in any order.": "These agents appear somewhere in the chain, in any order.", "These agents appear in this order; other agents may come in between. This is the hand-off assertion.": "These agents appear in this order; other agents may come in between. This is the hand-off assertion.", diff --git a/src/lib/langs/zh.json b/src/lib/langs/zh.json index 574713a3..9b55c231 100644 --- a/src/lib/langs/zh.json +++ b/src/lib/langs/zh.json @@ -752,6 +752,7 @@ "A Routing case cannot use llmJudge: routing is judged only by which agent handled the conversation.": "路由用例不能用 llmJudge：路由只按由哪个 agent 处理了对话来判定。", "Mode": "比较方式", "Assertion \"agentChain\" mode must be one of {modes}.": "agentChain 断言的比较方式只能是 {modes} 之一。", + "Assertion \"{type}\" target must be one of {sources}.": "{type} 断言的 Target 只能是 {sources} 之一。", "Copilot, Work Order Creator": "Copilot, Work Order Creator", "These agents appear somewhere in the chain, in any order.": "这些 agent 出现在链路中的任意位置，不论顺序。", "These agents appear in this order; other agents may come in between. This is the hand-off assertion.": "这些 agent 按此顺序出现，中间允许有其他 agent。这条就是交接断言。", diff --git a/src/routes/page/agent-test/[suiteId]/case/[caseId]/+page.svelte b/src/routes/page/agent-test/[suiteId]/case/[caseId]/+page.svelte index c67e4205..ea6db9f3 100644 --- a/src/routes/page/agent-test/[suiteId]/case/[caseId]/+page.svelte +++ b/src/routes/page/agent-test/[suiteId]/case/[caseId]/+page.svelte @@ -14,6 +14,8 @@ import { ASSERTION_TYPES, AGENT_CHAIN_MODES, + OUTPUT_SOURCES, + OUTPUT_SOURCE_TYPES, HISTORY_ROLES, CASE_PRIORITIES, CASE_SEVERITIES, @@ -32,6 +34,8 @@ /** Assertion types as Select options. Constant -- the list never changes at runtime. */ const ASSERTION_TYPE_OPTIONS = ASSERTION_TYPES.map(name => ({ label: name, value: name })); const AGENT_CHAIN_MODE_OPTIONS = AGENT_CHAIN_MODES.map(name => ({ label: name, value: name })); + /** `reply` is shown as "content": it is the reply text, as opposed to the rich content. */ + const OUTPUT_SOURCE_OPTIONS = OUTPUT_SOURCES.map(name => ({ label: name === 'reply' ? 'content' : name, value: name })); let isLoading = $state(false); let isComplete = $state(false); @@ -665,6 +669,12 @@ assertion.target = ''; } + // Output types carry the output source in `target` the same way. + const isSource = OUTPUT_SOURCES.some(s => s.toLowerCase() === (assertion.target || '').trim().toLowerCase()); + if (OUTPUT_SOURCE_TYPES.includes(assertion.type) ? !isSource : isSource) { + assertion.target = ''; + } + // Both fields mean something different under the new type, so an explicit "let me type it" from // the old one does not carry over. A value the new picker cannot show still opens in the text // box on its own -- see isManualEntry. @@ -905,7 +915,7 @@