feat(agent-testing): show used models, rerun failures on their model, render rich content

src/lib/helpers/types/agentTestTypes.js

@@ -317,6 +317,8 @@
  *   LlmConfig was used.
  * @property {string?} model - As above. In a multi-model run the same `caseId` yields several
  *   results, told apart by these two fields.
+ * @property {string[]?} usedModels - "provider/model" of every completion the agents under test
+ *   actually ran on, filled even when `model` is null. Can hold several (router + hand-off target).
  * @property {number} durationMs - Wall clock for the whole case, including the canary and the
  *   conversation reads. Comparable between models, but not a model-latency measurement.
  * @property {number} modelDurationMs - The agent calls alone, summed over the turns. This is what the

src/lib/helpers/utils/agent-test.js

@@ -59,6 +59,13 @@ const TARGET_REQUIRED = ['toolCalled', 'toolNotCalled', 'stateEquals'];
  */
 export const AGENT_CHAIN_MODES = ['contains', 'ordered', 'exact'];
 
+/**
+ * Which text an output assertion or llmJudge reads (OutputSources in
+ * AssertionEvaluator.cs), carried in `target`. Blank means `reply`.
+ */
+export const OUTPUT_SOURCES = ['reply', 'richContent'];
+export const OUTPUT_SOURCE_TYPES = ['outputContains', 'outputNotContains', 'outputRegex', 'llmJudge'];
+
 /**
  * What a case is verifying (CaseTypes in AgentTestCase.cs). Routing is checked
  * more strictly -- see `validateCaseType` -- because it is the only type counted
@@ -241,6 +248,13 @@ export function validateAssertion(assertion) {
 		return t('Assertion "agentChain" mode must be one of {modes}.', { modes: AGENT_CHAIN_MODES.join(', ') });
 	}
 
+	// Same for the output source: an unrecognised value fails the assertion rather
+	// than falling back to the reply text. Blank is legal and means `reply`.
+	if (OUTPUT_SOURCE_TYPES.includes(type) && assertion.target?.trim()
+		&& !OUTPUT_SOURCES.some(s => s.toLowerCase() === assertion.target.trim().toLowerCase())) {
+		return t('Assertion "{type}" target must be one of {sources}.', { type, sources: OUTPUT_SOURCES.join(', ') });
+	}
+
 	if (assertion.argsMatchJson?.trim() && !isParsableJson(assertion.argsMatchJson)) {
 		return t('The args match on an assertion is not valid JSON.');
 	}
@@ -302,6 +316,20 @@ export function formatAgentChain(chain) {
 	return chain?.length ? chain.join(' -> ') : '--';
 }
 
+/**
+ * A turn's rich content, pretty-printed. Stored as a JSON string; anything that does not parse is
+ * shown as-is rather than hidden.
+ * @param {string} richContent
+ * @returns {string}
+ */
+export function formatRichContent(richContent) {
+	try {
+		return JSON.stringify(JSON.parse(richContent), null, 2);
+	} catch {
+		return richContent;
+	}
+}
+
 /**
  * @param {string} text
  * @returns {boolean}

src/lib/langs/en.json

@@ -558,6 +558,7 @@
     "A Routing case cannot use llmJudge: routing is judged only by which agent handled the conversation.": "A Routing case cannot use llmJudge: routing is judged only by which agent handled the conversation.",
     "Mode": "Mode",
     "Assertion \"agentChain\" mode must be one of {modes}.": "Assertion \"agentChain\" mode must be one of {modes}.",
+    "Assertion \"{type}\" target must be one of {sources}.": "Assertion \"{type}\" target must be one of {sources}.",
     "Copilot, Work Order Creator": "Copilot, Work Order Creator",
     "These agents appear somewhere in the chain, in any order.": "These agents appear somewhere in the chain, in any order.",
     "These agents appear in this order; other agents may come in between. This is the hand-off assertion.": "These agents appear in this order; other agents may come in between. This is the hand-off assertion.",

src/lib/langs/zh.json

@@ -752,6 +752,7 @@
     "A Routing case cannot use llmJudge: routing is judged only by which agent handled the conversation.": "路由用例不能用 llmJudge：路由只按由哪个 agent 处理了对话来判定。",
     "Mode": "比较方式",
     "Assertion \"agentChain\" mode must be one of {modes}.": "agentChain 断言的比较方式只能是 {modes} 之一。",
+    "Assertion \"{type}\" target must be one of {sources}.": "{type} 断言的 Target 只能是 {sources} 之一。",
     "Copilot, Work Order Creator": "Copilot, Work Order Creator",
     "These agents appear somewhere in the chain, in any order.": "这些 agent 出现在链路中的任意位置，不论顺序。",
     "These agents appear in this order; other agents may come in between. This is the hand-off assertion.": "这些 agent 按此顺序出现，中间允许有其他 agent。这条就是交接断言。",

src/routes/page/agent-test/[suiteId]/case/[caseId]/+page.svelte

@@ -14,6 +14,8 @@
 	import {
 		ASSERTION_TYPES,
 		AGENT_CHAIN_MODES,
+		OUTPUT_SOURCES,
+		OUTPUT_SOURCE_TYPES,
 		HISTORY_ROLES,
 		CASE_PRIORITIES,
 		CASE_SEVERITIES,
@@ -32,6 +34,8 @@
 	/** Assertion types as Select options. Constant -- the list never changes at runtime. */
 	const ASSERTION_TYPE_OPTIONS = ASSERTION_TYPES.map(name => ({ label: name, value: name }));
 	const AGENT_CHAIN_MODE_OPTIONS = AGENT_CHAIN_MODES.map(name => ({ label: name, value: name }));
+	/** `reply` is shown as "content": it is the reply text, as opposed to the rich content. */
+	const OUTPUT_SOURCE_OPTIONS = OUTPUT_SOURCES.map(name => ({ label: name === 'reply' ? 'content' : name, value: name }));
 
 	let isLoading = $state(false);
 	let isComplete = $state(false);
@@ -665,6 +669,12 @@
 			assertion.target = '';
 		}
 
+		// Output types carry the output source in `target` the same way.
+		const isSource = OUTPUT_SOURCES.some(s => s.toLowerCase() === (assertion.target || '').trim().toLowerCase());
+		if (OUTPUT_SOURCE_TYPES.includes(assertion.type) ? !isSource : isSource) {
+			assertion.target = '';
+		}
+
 		// Both fields mean something different under the new type, so an explicit "let me type it" from
 		// the old one does not carry over. A value the new picker cannot show still opens in the text
 		// box on its own -- see isManualEntry.
@@ -905,7 +915,7 @@
 				<div class="md:col-span-3">
 					<label
 						class="ats-label"
-						for={assertion.type === 'agentChain' || toolPicked ? undefined : `${idPrefix}-target-${i}`}
+						for={assertion.type === 'agentChain' || OUTPUT_SOURCE_TYPES.includes(assertion.type) || toolPicked ? undefined : `${idPrefix}-target-${i}`}
 					>
 						{assertion.type === 'agentChain' ? $_('Mode') : $_('Target')}
 						{#if ['toolCalled', 'toolNotCalled', 'stateEquals'].includes(assertion.type)}
@@ -922,6 +932,13 @@
 							options={AGENT_CHAIN_MODE_OPTIONS}
 							onselect={e => changeAgentChainMode(assertion, e)}
 						/>
+					{:else if OUTPUT_SOURCE_TYPES.includes(assertion.type)}
+						<Select
+							tag={`${idPrefix}-source-${i}`}
+							selectedValues={[assertion.target || 'reply']}
+							options={OUTPUT_SOURCE_OPTIONS}
+							onselect={e => pickValue(assertion, 'target', e)}
+						/>
 					{:else if assertion.type === 'toolCalled' || assertion.type === 'toolNotCalled'}
 						{#if toolPicked}
 							<Select

src/routes/page/agent-test/run/[runId]/+page.svelte

@@ -14,6 +14,7 @@
 		formatDuration,
 		formatDateTime,
 		formatAgentChain,
+		formatRichContent,
 		t
 	} from '$lib/helpers/utils/agent-test.js';
 
@@ -58,6 +59,28 @@
 		...new Set(results.filter(r => r.status === 'Failed' || r.status === 'Error').map(r => r.caseId))
 	]);
 
+	/**
+	 * The models the failures ran on, so a re-run reproduces them on the same model instead of
+	 * falling back to the agent's own (possibly since changed) LlmConfig. An "agent default" failure
+	 * counts only when it ran on a single model: forcing one model onto a case whose router and
+	 * hand-off target used different ones would re-run a different conversation. Null = re-run on
+	 * the agents' own LlmConfig.
+	 */
+	let failedModels = $derived.by(() => {
+		/** @type {Map<string, import('$agentTestTypes').TestModel>} */
+		const models = new Map();
+		for (const r of results) {
+			if (r.status !== 'Failed' && r.status !== 'Error') continue;
+			if (r.model) {
+				models.set(`${r.provider}/${r.model}`, { provider: r.provider, model: r.model });
+			} else if (r.usedModels?.length === 1) {
+				const [provider, ...rest] = r.usedModels[0].split('/');
+				models.set(r.usedModels[0], { provider, model: rest.join('/') });
+			}
+		}
+		return models.size > 0 ? [...models.values()] : null;
+	});
+
 	/** Column order for the comparison grid; follows the run's own model order. */
 	let modelColumns = $derived((run?.models || []).map(m => ({ ...m, key: `${m.provider}/${m.model}` })));
 
@@ -77,9 +100,20 @@
 	 * Latency, token and cost figures per model, as stored on the run. A table rather than tiles
 	 * because there is one row per model and the whole point is comparing them side by side.
 	 */
+	/**
+	 * A run that named no models still ran on something: each agent's own LlmConfig as it stood at
+	 * run time. Named here because that config may have changed since.
+	 */
+	let defaultLabel = $derived.by(() => {
+		const used = [...new Set(results.filter(r => !r.model).flatMap(r => r.usedModels || []))];
+		return used.length > 0
+			? `${$_('agent default')} (${used.map(u => u.split('/').slice(1).join('/')).join(', ')})`
+			: $_('agent default');
+	});
+
 	let performance = $derived((run?.performanceSummaries || []).map(s => ({
 		...s,
-		label: s.model || $_('agent default')
+		label: s.model || defaultLabel
 	})));
 
 	/**
@@ -212,7 +246,7 @@
 	function handleRerunFailed() {
 		if (isBusy || !suiteId || failedCaseIds.length === 0) return;
 		isBusy = true;
-		triggerRun(suiteId, failedCaseIds).then(newRun => {
+		triggerRun(suiteId, failedCaseIds, failedModels).then(newRun => {
 			notifySuccess(t('Re-run queued!'));
 			goto(`/page/agent-test/run/${newRun.id}`);
 		}).catch(err => {
@@ -557,6 +591,14 @@
 										<span class="ats-badge ats-tone-primary font-code" title={result.provider}>
 											{result.model}
 										</span>
+									{:else}
+										<!-- "Agent default" run: what each agent's LlmConfig resolved to at run
+										     time. Several when a router and its hand-off target differ. -->
+										{#each result.usedModels || [] as used (used)}
+											<span class="ats-badge ats-tone-primary font-code" title={used}>
+												{used.split('/').slice(1).join('/')}
+											</span>
+										{/each}
 									{/if}
 									<span class="text-xs text-muted" title={$_('Wall clock for the whole case')}>
 										{formatDuration(result.durationMs)}
@@ -618,6 +660,15 @@
 												<div class="ats-meta-label">{$_('Agent output')}</div>
 												<div class="ats-code mt-1">{turn.output || '--'}</div>
 											</div>
+											{#if turn.richContent}
+												<!-- Its own block, not appended to the output: assertions target the
+												     two separately (output vs. richContent), so each is shown where
+												     its assertions point. -->
+												<div class="mb-2">
+													<div class="ats-meta-label">{$_('Rich content')}</div>
+													<div class="ats-code mt-1 max-h-80 overflow-auto">{formatRichContent(turn.richContent)}</div>
+												</div>
+											{/if}
 											{#if turn.agentChain?.length > 0}
 												<div class="mb-2">
 													<div class="ats-meta-label">{$_('Answered by')}</div>