diff --git a/src/lib/helpers/types/agentTestTypes.js b/src/lib/helpers/types/agentTestTypes.js index 9c9cf950..7def88b1 100644 --- a/src/lib/helpers/types/agentTestTypes.js +++ b/src/lib/helpers/types/agentTestTypes.js @@ -5,8 +5,9 @@ * @property {string} name * @property {string?} description * @property {boolean} enabled - * @property {string?} judgeProvider - Provider for llmJudge assertions. In P1 llmJudge always fails, - * whether or not this is configured. + * @property {string?} judgeProvider - Provider for llmJudge assertions. Both this and judgeModel are + * required for llmJudge to be scored at all; with either missing, an llmJudge assertion records a + * case-level Error rather than being scored with some default model. * @property {string?} judgeModel * @property {string[]} extraAllowedFunctions - Functions let through on top of the default * control-flow allow list. @@ -79,9 +80,13 @@ * null/empty on every run and can never pass. It fails rather than errors, so nothing points at it. * The form is deliberately stricter and treats `expected` as required there too. * - * `llmJudge` always fails in P1 (the backend returns "llmJudge is not available in P1"), regardless - * of minScore or whether the suite configured judgeProvider/judgeModel. The form may offer it, but - * says so next to it. + * `llmJudge` is scored 1-5 by the suite's judge model, and `minScore` is the pass mark (default 4). + * It is the one assertion type that is not reproducible, and the one that can end a case in Error + * rather than Failed: an unconfigured judge model, an unregistered provider, a vendor failure or a + * reply the backend cannot read as a score all mean "no verdict", which is Error. Those are + * deliberately not failures -- a vendor timeout is not an agent regression. Note also that the judge + * only ever sees the criterion and the agent's reply, never the user's message, so a criterion has + * to be self-contained. * * @typedef {Object} TestAssertion * @property {string} type - outputContains | outputNotContains | outputRegex | toolCalled | toolNotCalled | stateEquals | routedToAgent | llmJudge @@ -99,15 +104,41 @@ * @property {string} name * @property {boolean} enabled - Recorded drafts land as false; a human has to review and enable them * before they join a normal run. + * @property {string} caseType - Routing | Agent. Defaults to Agent, which is also what every case + * stored before the field existed reads back as. Routing is validated more strictly (one turn, + * must assert a routing outcome, no llmJudge) and is the only type counted towards a run's routing + * accuracy. A journey across several agents is an Agent case whose `agentChain` assertion + * describes the hand-offs. + * @property {string?} entryAgentId - Agent the conversation opens on, overriding the suite's; null + * uses the suite's. This is the switch between testing routing and testing one agent alone: + * BotSharp sends a routing-type agent through the router (which can hand off) and any other agent + * straight into itself (the router never runs). * @property {TestTurn[]} turns - Length 1 is a single-turn case. * @property {TestAssertion[]} assertions - Case-level assertions, evaluated after every turn has run. * @property {TestState[]} initialStates - Injected before the conversation starts; maps to BotSharp's * MessageState. + * @property {TestHistoryMessage[]} history - Prior turns written into the conversation before the + * case's own turns run, so a real question-and-answer exchange becomes the fixed starting context. + * Not driven through the model (no token cost, no flakiness) and never counted in `agentChain`. * @property {TestToolMock[]} mocks * @property {string} unmockedToolPolicy - P1 accepts only "Block"; sending "Passthrough" is a 400 * ("Passthrough is not supported in P1"). The form should not offer it. * @property {string?} sourceConversationId - The conversation this was recorded from, for * traceability; null for a hand-written case. + * @property {string} priority - P0 | P1 | P2. Decides the batch, and therefore whether a failure + * stops the evaluation. Defaults to P1, which is also what a case stored before the field existed + * reads back as. + * @property {string} severity - S0 | S1 | S2. What a failure MEANS, as opposed to how urgent the case + * is to run. Defaults to S1. + * @property {number?} batch - Explicit override; null derives it from priority and crossCutting. + * @property {boolean} crossCutting - Runs in every scope whatever changed, and always in batch 1. + * @property {string[]} involvedAgents - Agent ids. Empty falls back to the case's entry agent, which + * is right for an Agent case and only a starting point for a Routing case, where the agents that + * matter are downstream of the router. + * @property {string?} businessDomain + * @property {string?} expectedOutcome - For whoever reviews the result; never evaluated. + * @property {string?} lastReviewedDate - ISO date. Never set by the server: editing a case is not + * reviewing it. * @property {string} createDate - ISO date string. * @property {string} updateDate - ISO date string. */ @@ -125,6 +156,20 @@ * @property {string} suiteId * @property {string} name * @property {boolean} [enabled] - Defaults to true. + * @property {string} [caseType] - Routing | Agent; blank or omitted means Agent. Any other value is + * a 400 rather than a silent fallback. + * @property {string?} [entryAgentId] - Send null rather than "" to mean "use the suite's agent"; + * validated to exist at save time. + * @property {TestHistoryMessage[]} [history] - Authored prior turns; every message needs a + * user/assistant role and non-empty content. + * @property {string} [priority] - P0 | P1 | P2; blank keeps P1. Any other value is a 400. + * @property {string} [severity] - S0 | S1 | S2; blank keeps S1. + * @property {number?} [batch] - 1, 2 or 3; null derives it. Out of range is a 400, not a clamp. + * @property {boolean} [crossCutting] + * @property {string[]} [involvedAgents] + * @property {string?} [businessDomain] + * @property {string?} [expectedOutcome] + * @property {string?} [lastReviewedDate] - ISO date; send it only when a human actually reviewed. * @property {TestTurn[]} turns * @property {TestAssertion[]} assertions * @property {TestState[]} initialStates @@ -133,6 +178,16 @@ * @property {string?} [sourceConversationId] */ +/** + * One authored message in a case's history. Just a role and text: a mocked tool call belongs in + * `mocks`, and a fabricated function-call dialog would let a case claim a tool ran when nothing did. + * @typedef {Object} TestHistoryMessage + * @property {string} role - user | assistant. Nothing else: `system` would compete with the agent's + * own instruction and `function` would fake a tool call. + * @property {string} content - Rejected as empty; BotSharp's dialog storage drops blank elements, so + * an empty message would silently not be in the conversation at run time. + */ + /** * One model to run against. * @typedef {Object} TestModel @@ -163,12 +218,57 @@ * @property {number} passedCount * @property {number} failedCount * @property {number} errorCount + * @property {RoutingAccuracy[]} routingAccuracies - One row per model swept, counting only Routing + * cases; empty when the run contained none. + * @property {PerformanceSummary[]} performanceSummaries - Latency, token and cost figures, one row per + * model. Computed when the run finishes, because a percentile needs every value at once. + * @property {ModelPricingSnapshot[]} modelPricing - The unit costs in force when the run executed. A + * cost figure is not comparable with another run's unless these match. Kept per model because a single run-wide figure would + * average a candidate model together with the baseline and hide the difference the run exists to + * measure. * @property {boolean} cancelRequested * @property {string?} startedAt - ISO date string; null until it starts. * @property {string?} completedAt - ISO date string; null until it finishes. * @property {string} createDate - ISO date string. Note AgentTestRun has no updateDate field. */ +/** + * Latency, tokens and cost for one model within a run. Averages are absent on purpose: an average is + * Total/CaseCount, and a stored copy is one more thing that can disagree with the rows it came from. + * @typedef {Object} PerformanceSummary + * @property {string?} provider + * @property {string?} model + * @property {number} caseCount - Only the cases that reached the model. A case that failed before its + * first turn would drag a latency percentile towards zero and make a broken run look fast. + * @property {number} latencyP50Ms - Nearest-rank median of the agent-call time, so it is always a + * duration some case actually took rather than an interpolated one that none did. + * @property {number} latencyP95Ms + * @property {number} totalTokens - Over EVERY result, unlike latency: a case that errored still spent + * what it spent. + * @property {number} totalCost + */ + +/** + * One model's configured text-token unit costs at the moment a run executed. + * @typedef {Object} ModelPricingSnapshot + * @property {string?} provider + * @property {string?} model + * @property {number?} textInputCost - Null means the settings could not be read. Never render it as + * 0, which would read as "this model is free". + * @property {number?} textOutputCost + */ + +/** + * Routing accuracy for one model within a run. Counts, never a stored percentage: "3/4" says how + * much the figure is worth trusting and "75%" does not. + * @typedef {Object} RoutingAccuracy + * @property {string?} provider - Null for both when the run swept no models. + * @property {string?} model + * @property {number} caseCount - Routing cases executed under this model, whatever their outcome -- + * Error rows included, because "could not tell" is not "routed correctly". + * @property {number} passedCount + */ + /** * @typedef {Object} AssertionResult * @property {string} type @@ -185,6 +285,11 @@ * @property {string} userMessage * @property {string?} output * @property {AssertionResult[]} assertions + * @property {number} modelDurationMs - Time the agent call for this turn took, excluding the + * assertion evaluation and conversation reads that follow it. + * @property {string[]} agentChain - The agents that answered during THIS turn, in order, with + * consecutive repeats collapsed. Not derivable from the case-level chain, which collapses across + * turn boundaries too. */ /** @@ -203,6 +308,8 @@ * @property {string} runId * @property {string} caseId * @property {string} caseName + * @property {string} caseType - Copied off the case so a result is self-describing; routing + * accuracy is aggregated from these rows rather than from cases that may since have been edited. * @property {string} status - Passed | Failed | Error | Cancelled (never Pending/Running). * @property {string?} conversationId - The conversation this execution created; live conversations * are never reused. @@ -210,16 +317,67 @@ * LlmConfig was used. * @property {string?} model - As above. In a multi-model run the same `caseId` yields several * results, told apart by these two fields. - * @property {number} durationMs + * @property {number} durationMs - Wall clock for the whole case, including the canary and the + * conversation reads. Comparable between models, but not a model-latency measurement. + * @property {number} modelDurationMs - The agent calls alone, summed over the turns. This is what the + * run's latency percentiles are built from. + * @property {number} totalTokens - Measured as a delta across this case's own execution. Total only: + * the input/output split is not reachable through ITokenStatistics. + * @property {number} cost * @property {string?} error - Infrastructure-level reason for failure (timeout, a dead mock seam, a * case with no turns), as distinct from an assertion failure. Show this text whenever `status` is * `Error`. * @property {TurnResult[]} turns * @property {AssertionResult[]} assertions - Case-level assertion results. * @property {ObservedToolCall[]} observedToolCalls + * @property {string[]} agentChain - Every agent that answered over the whole case, in order, + * consecutive repeats collapsed. The only record of the hand-offs: route_to_agent is allowed + * through the mock seam untouched, so it never appears in `observedToolCalls`. * @property {string} createDate - ISO date string. */ +/** + * One case's place in a scope, carrying the metadata the decision was made from rather than just the + * verdict -- a scope nobody can explain is a scope nobody can review. + * @typedef {Object} ScopedCase + * @property {string} caseId + * @property {string} caseName + * @property {string} suiteId + * @property {string} suiteName + * @property {string} caseType + * @property {string} priority + * @property {string} severity + * @property {boolean} crossCutting + * @property {boolean} enabled + * @property {number} batch - The effective batch, after the priority and cross-cutting derivation. + * @property {string[]} involvedAgentIds - Authored, or derived from the entry agent. + * @property {string} reason - crossCutting | fullPlatform | targetAgent | unknownAgents | + * notInvolved | disabled | otherBatch. + */ + +/** + * The answer to "what will this change actually test". Both halves are returned on purpose: an + * excluded case produces no result to notice, so the exclusions and their reasons are the half worth + * reading. + * @typedef {Object} ScopeSelection + * @property {string[]} targetAgentIds + * @property {boolean} fullPlatform + * @property {number?} batch + * @property {number} totalCases - Every registered case, whatever its state: the coverage denominator. + * @property {ScopedCase[]} included + * @property {ScopedCase[]} excluded + */ + +/** + * What clearing run history actually did. The skipped half matters: a still-running run is left + * alone, and the caller has to be able to say why one survived rather than leaving the user to + * notice a row that quietly stayed. + * @typedef {Object} RunDeleteResult + * @property {string[]} deletedRunIds + * @property {number} deletedResultCount - Case results removed along with those runs. + * @property {{ runId: string, reason: string }[]} skipped + */ + /** * Body of GET /agent-test/runs/{id} (the camelCase projection of the backend's * AgentTestRunDetailDto). @@ -228,4 +386,44 @@ * @property {AgentTestCaseResult[]} results */ +/** + * Body of POST /agent-test/author. Carries the whole conversation and the whole draft because the + * server keeps no authoring session -- there is no draft row to orphan when a tab is closed. + * @typedef {Object} AgentTestAuthorRequest + * @property {string} suiteId + * @property {string?} [caseId] - The case being edited; null while creating. Supplied so the backend + * can ground the model in that case's most recent run -- what the agent really replied and what + * arguments it really passed -- instead of letting it invent both. + * @property {AuthorChatMessage[]} messages - Oldest first; the last one is the new instruction. + * @property {AgentTestCaseUpsertRequest?} [draft] - The draft as it stands, or null to start empty. + * @property {TestModel?} [model] - Omit to use the suite's judge model. + */ + +/** + * @typedef {Object} AuthorChatMessage + * @property {string} role - user | assistant. + * @property {string} content + */ + +/** + * One changed field of a draft. + * @typedef {Object} AuthorChange + * @property {string} field - camelCase draft field name, e.g. "turns". + * @property {string} detail - Short human summary, e.g. "3 -> 4 item(s)". + */ + +/** + * Result of one authoring turn. + * @typedef {Object} AgentTestAuthorResult + * @property {string} reply - What to show in the chat. + * @property {AgentTestCaseUpsertRequest} draft - Always populated, so the client can assign it + * unconditionally. + * @property {boolean} draftChanged + * @property {AuthorChange[]} changes - Computed by the backend by diffing the two drafts, never read + * off the model's own account of what it did. + * @property {string[]} validationErrors - Non-empty means the draft cannot be saved as it stands. + * @property {string[]} warnings - Silently wrong things that were corrected (a mock for a function + * this agent cannot call) or are merely suspect and were left alone (an unfamiliar state key). + */ + export default {}; diff --git a/src/lib/helpers/utils/agent-test.js b/src/lib/helpers/utils/agent-test.js index 41c06c92..67a078bd 100644 --- a/src/lib/helpers/utils/agent-test.js +++ b/src/lib/helpers/utils/agent-test.js @@ -35,15 +35,115 @@ export const ASSERTION_TYPES = [ 'toolNotCalled', 'stateEquals', 'routedToAgent', + 'agentChain', 'llmJudge' ]; /** Types whose `expected` the backend rejects as empty. */ -const EXPECTED_REQUIRED = ['outputContains', 'outputNotContains', 'outputRegex', 'routedToAgent', 'llmJudge']; +const EXPECTED_REQUIRED = [ + 'outputContains', + 'outputNotContains', + 'outputRegex', + 'routedToAgent', + 'agentChain', + 'llmJudge' +]; /** Types whose `target` the backend rejects as empty. */ const TARGET_REQUIRED = ['toolCalled', 'toolNotCalled', 'stateEquals']; +/** + * How an `agentChain` assertion compares its expected list (AgentChainModes in + * AssertionEvaluator.cs). On this assertion type `target` carries the mode + * rather than a function name or state key. + */ +export const AGENT_CHAIN_MODES = ['contains', 'ordered', 'exact']; + +/** + * What a case is verifying (CaseTypes in AgentTestCase.cs). Routing is checked + * more strictly -- see `validateCaseType` -- because it is the only type counted + * towards a run's routing accuracy. A multi-agent journey is an Agent case whose + * `agentChain` assertion describes the hand-offs. + */ +export const CASE_TYPES = ['Routing', 'Agent']; + +/** + * Roles an authored history message may take (HistoryRoles in AgentTestCase.cs). + * Only these two: `system` would compete with the agent's own instruction and + * `function` would fake a tool call, letting a case claim a tool ran when + * nothing did. + */ +export const HISTORY_ROLES = ['user', 'assistant']; + +/** + * How urgent it is to run a case, which is what decides its batch (CasePriorities + * in AgentTestCase.cs). Distinct from severity: priority is scheduling, + * severity is consequence. + */ +export const CASE_PRIORITIES = ['P0', 'P1', 'P2']; + +/** + * What a failure means (CaseSeverities in AgentTestCase.cs). S0 is a stop rather + * than a statistic; S2 must never be able to mask an S0 or S1. + */ +export const CASE_SEVERITIES = ['S0', 'S1', 'S2']; + +/** Batches run in order: 1 is the stop-loss batch, 3 does not block a release. */ +export const CASE_BATCHES = [1, 2, 3]; + +/** + * Tone for a priority badge. P0 is the stop-loss batch -- one failure there halts the + * whole evaluation -- so it reads as urgent; P2 does not block a release and reads as + * quiet. The middle stays neutral rather than warning-coloured, because P1 is the + * default every untriaged case carries and a wall of amber would say nothing. + * @param {string?} priority + */ +export function priorityTone(priority) { + switch (priority) { + case 'P0': return 'danger'; + case 'P2': return 'secondary'; + default: return 'primary'; + } +} + +/** + * Tone for a severity badge. S0 is zero-tolerance -- data leakage, an unauthorised + * action, a missed escalation -- and has to be visible at a glance in a list, which is + * the whole reason severity is worth a column. S2 is phrasing and must never look as + * loud as the other two. + * @param {string?} severity + */ +export function severityTone(severity) { + switch (severity) { + case 'S0': return 'danger'; + case 'S1': return 'warning'; + default: return 'secondary'; + } +} + +/** + * The batch a case will actually run in, mirroring CaseBatches.Effective. An + * explicit batch wins; a cross-cutting case is batch 1 whatever its priority, + * because a safety case that only runs after everything else has passed cannot + * stop anything. + * @param {{ batch?: number | null, crossCutting?: boolean, priority?: string }} testCase + * @returns {number} + */ +export function effectiveBatch(testCase) { + if (CASE_BATCHES.includes(Number(testCase?.batch))) { + return Number(testCase.batch); + } + if (testCase?.crossCutting) { + return 1; + } + if (testCase?.priority === 'P0') return 1; + if (testCase?.priority === 'P2') return 3; + return 2; +} + +/** Assertion types that establish a routing outcome, so a Routing case needs one. */ +const ROUTING_ASSERTION_TYPES = ['routedToAgent', 'agentChain']; + /** * Bootstrap contextual class for a run/case status. * @param {string?} status @@ -132,6 +232,15 @@ export function validateAssertion(assertion) { } } + // agentChain puts the comparison mode in `target`. An unrecognised value fails + // at evaluation time rather than falling back to the loosest mode, so catching + // it here just moves that failure to where the author can see it. Blank is + // legal and means `contains`. + if (type === 'agentChain' && assertion.target?.trim() + && !AGENT_CHAIN_MODES.includes(assertion.target.trim().toLowerCase())) { + return t('Assertion "agentChain" mode must be one of {modes}.', { modes: AGENT_CHAIN_MODES.join(', ') }); + } + if (assertion.argsMatchJson?.trim() && !isParsableJson(assertion.argsMatchJson)) { return t('The args match on an assertion is not valid JSON.'); } @@ -139,6 +248,60 @@ export function validateAssertion(assertion) { return null; } +/** + * The extra rules the backend applies to a Routing case, checked here so the + * author sees them while editing instead of as a 400 on save. Kept in step with + * AgentTestController.ValidateRoutingCase -- if these two ever disagree, the + * backend wins and the save fails, which is the safe direction. + * @param {string} caseType + * @param {import('$agentTestTypes').TestTurn[]} turns + * @param {import('$agentTestTypes').TestAssertion[]} caseAssertions + * @returns {string[]} one message per problem, empty when the case is acceptable + */ +export function validateCaseType(caseType, turns, caseAssertions) { + if (caseType !== 'Routing') { + return []; + } + + const problems = []; + const all = [...(turns || []).flatMap(turn => turn.assertions || []), ...(caseAssertions || [])]; + + // Routing asks a single-turn question: which agent picks this message up. A + // second turn asks something else, and its verdict would still land in the + // routing accuracy figure. + // Authored history is deliberately not counted: replaying a prior exchange and + // then asking one question is still a single routing decision, and it is the most + // realistic way to test routing that depends on context. + if ((turns || []).length !== 1) { + problems.push(t('A Routing case must have exactly one turn. Use an Agent case for a multi-turn case.')); + } + + // Without one of these the case asserts nothing about routing yet still counts + // towards routing accuracy -- it reports Passed for having said anything at all. + if (!all.some(a => ROUTING_ASSERTION_TYPES.includes(a?.type))) { + problems.push(t('A Routing case needs at least one routedToAgent or agentChain assertion.')); + } + + // Routing is scored purely as expected agent == actual agent. An llmJudge would + // also make the figure depend on a vendor call, so a vendor outage would read + // as a routing regression. + if (all.some(a => a?.type === 'llmJudge')) { + problems.push(t('A Routing case cannot use llmJudge: routing is judged only by which agent handled the conversation.')); + } + + return problems; +} + +/** + * An agent chain rendered the way the backend reports it in an assertion's + * `actual`, so the two read the same on screen. + * @param {string[] | null | undefined} chain + * @returns {string} + */ +export function formatAgentChain(chain) { + return chain?.length ? chain.join(' -> ') : '--'; +} + /** * @param {string} text * @returns {boolean} diff --git a/src/lib/langs/en.json b/src/lib/langs/en.json index be11ddbe..084812fa 100644 --- a/src/lib/langs/en.json +++ b/src/lib/langs/en.json @@ -453,8 +453,8 @@ "No mocks. Every tool call this case makes will be blocked.": "No mocks. Every tool call this case makes will be blocked.", "A disabled suite cannot be run at all.": "A disabled suite cannot be run at all.", "This suite is disabled. Triggering a run is rejected by the server until you enable it in Settings.": "This suite is disabled. Triggering a run is rejected by the server until you enable it in Settings.", - "llmJudge assertions always fail in P1 regardless of these two fields.": "llmJudge assertions always fail in P1 regardless of these two fields.", - "Always fails in P1.": "Always fails in P1.", + "Required by llmJudge assertions. Without both, an llmJudge assertion records an Error rather than a score -- it is never scored with a default model.": "Required by llmJudge assertions. Without both, an llmJudge assertion records an Error rather than a score -- it is never scored with a default model.", + "Scored 1-5 by the judge model configured on this suite. Pass mark defaults to 4.": "Scored 1-5 by the judge model configured on this suite. Pass mark defaults to 4.", "One function name per line. These run for real during a test -- only list functions with no side effects.": "One function name per line. These run for real during a test -- only list functions with no side effects.", "One per line. Blocked even if allow-listed elsewhere.": "One per line. Blocked even if allow-listed elsewhere.", "Assertions checked right after this turn": "Assertions checked right after this turn", @@ -546,5 +546,124 @@ "This run could not complete": "This run could not complete", "Nothing ran -- see the reason above.": "Nothing ran -- see the reason above.", "Nothing here can run -- every case you picked is disabled. Enable at least one first.": "Nothing here can run -- every case you picked is disabled. Enable at least one first.", - "{count} of the selected case(s) are disabled and will be skipped.": "{count} of the selected case(s) are disabled and will be skipped." + "{count} of the selected case(s) are disabled and will be skipped.": "{count} of the selected case(s) are disabled and will be skipped.", + "Case type": "Case type", + "Routing case": "Routing case", + "Agent case": "Agent case", + "One turn, entered on the Copilot entry agent, asserting only which agent took the conversation. The only type counted towards a run's routing accuracy.": "One turn, entered on the Copilot entry agent, asserting only which agent took the conversation. The only type counted towards a run's routing accuracy.", + "One agent's own behaviour. Enter directly on that agent to keep the router out of what is measured.": "One agent's own behaviour. Enter directly on that agent to keep the router out of what is measured.", + "Entry agent": "Entry agent", + "A routing agent runs the router and can hand off; any other agent is entered directly, so the router never runs.": "A routing agent runs the router and can hand off; any other agent is entered directly, so the router never runs.", + "A Routing case needs at least one routedToAgent or agentChain assertion.": "A Routing case needs at least one routedToAgent or agentChain assertion.", + "A Routing case cannot use llmJudge: routing is judged only by which agent handled the conversation.": "A Routing case cannot use llmJudge: routing is judged only by which agent handled the conversation.", + "Mode": "Mode", + "Assertion \"agentChain\" mode must be one of {modes}.": "Assertion \"agentChain\" mode must be one of {modes}.", + "Copilot, Work Order Creator": "Copilot, Work Order Creator", + "These agents appear somewhere in the chain, in any order.": "These agents appear somewhere in the chain, in any order.", + "These agents appear in this order; other agents may come in between. This is the hand-off assertion.": "These agents appear in this order; other agents may come in between. This is the hand-off assertion.", + "The chain is exactly these agents and nothing else. One name asserts that nothing routed away.": "The chain is exactly these agents and nothing else. One name asserts that nothing routed away.", + "Agent chain": "Agent chain", + "Answered by": "Answered by", + "Routing accuracy": "Routing accuracy", + "agent default": "agent default", + "A Routing case must have exactly one turn. Use an Agent case for a multi-turn case.": "A Routing case must have exactly one turn. Use an Agent case for a multi-turn case.", + "One agent's own behaviour. Enter directly on that agent to keep the router out of what is measured. A journey across several agents is an Agent case too -- use an agentChain assertion for the hand-offs.": "One agent's own behaviour. Enter directly on that agent to keep the router out of what is measured. A journey across several agents is an Agent case too -- use an agentChain assertion for the hand-offs.", + "Use the suite's agent": "Use the suite's agent", + "The agent the conversation opens on. Leave it unset to use the suite's.": "The agent the conversation opens on. Leave it unset to use the suite's.", + "History": "History", + "Add Message": "Add Message", + "Written into the conversation before the first turn runs, so an existing question-and-answer exchange becomes the fixed starting context for this case.": "Written into the conversation before the first turn runs, so an existing question-and-answer exchange becomes the fixed starting context for this case.", + "These messages are not driven through the model, cost no tokens, and never appear in the agent chain.": "These messages are not driven through the model, cost no tokens, and never appear in the agent chain.", + "Role": "Role", + "Content": "Content", + "Move up": "Move up", + "Move down": "Move down", + "Remove message": "Remove message", + "No history. The case starts from an empty conversation.": "No history. The case starts from an empty conversation.", + "History message {n} has no content.": "History message {n} has no content.", + "Assistant": "Assistant", + "Copy": "Copy", + "Copy case": "Copy case", + "Copied to \"{name}\". It is disabled until you enable it.": "Copied to \"{name}\". It is disabled until you enable it.", + "Failed to copy the case.": "Failed to copy the case.", + "Registration": "Registration", + "Decides when this case runs and what a failure of it means. Used to work out which cases a change actually needs to run.": "Decides when this case runs and what a failure of it means. Used to work out which cases a change actually needs to run.", + "Priority": "Priority", + "P0 runs in the stop-loss batch, P2 does not block a release.": "P0 runs in the stop-loss batch, P2 does not block a release.", + "Severity": "Severity", + "S0 is a stop, not a statistic. S2 is phrasing and experience.": "S0 is a stop, not a statistic. S2 is phrasing and experience.", + "Batch": "Batch", + "Derive from priority": "Derive from priority", + "Runs in batch {n}.": "Runs in batch {n}.", + "Cross-cutting": "Cross-cutting", + "Runs in every scope, whatever changed, and always in batch 1.": "Runs in every scope, whatever changed, and always in batch 1.", + "Involved agents": "Involved agents", + "Derived from the entry agent": "Derived from the entry agent", + "Leave empty and the entry agent is used. Worth filling in for a routing case, where the agents that matter are the ones downstream of the router.": "Leave empty and the entry agent is used. Worth filling in for a routing case, where the agents that matter are the ones downstream of the router.", + "Business domain": "Business domain", + "Last reviewed": "Last reviewed", + "Today": "Today", + "Never set automatically -- editing a case is not reviewing it.": "Never set automatically -- editing a case is not reviewing it.", + "Expected outcome": "Expected outcome", + "For whoever reviews the result. Never evaluated -- an expected outcome a machine can check is an assertion, and belongs below where it will be enforced.": "For whoever reviews the result. Never evaluated -- an expected outcome a machine can check is an assertion, and belongs below where it will be enforced.", + "Latency, tokens and cost": "Latency, tokens and cost", + "Model": "Model", + "Cases": "Cases", + "Latency P50": "Latency P50", + "Latency P95": "Latency P95", + "Tokens": "Tokens", + "Cost": "Cost", + "Unit cost (in / out)": "Unit cost (in / out)", + "unknown": "unknown", + "Percentiles are nearest-rank over the agent-call time of the cases that reached the model, so every figure is a duration some case actually took. Cases that failed before their first turn are excluded from the percentiles but still counted in tokens and cost.": "Percentiles are nearest-rank over the agent-call time of the cases that reached the model, so every figure is a duration some case actually took. Cases that failed before their first turn are excluded from the percentiles but still counted in tokens and cost.", + "Wall clock for the whole case": "Wall clock for the whole case", + "Agent call time only": "Agent call time only", + "Tokens / cost": "Tokens / cost", + "Plan a scope": "Plan a scope", + "Which cases a change needs to run, and which it does not. Read-only -- it plans a run, it does not start one.": "Which cases a change needs to run, and which it does not. Read-only -- it plans a run, it does not start one.", + "Changed agents": "Changed agents", + "All batches": "All batches", + "Platform-wide change": "Platform-wide change", + "Narrowing is switched off: no agent is demonstrably untouched.": "Narrowing is switched off: no agent is demonstrably untouched.", + "Plan": "Plan", + "Failed to work out the scope.": "Failed to work out the scope.", + "included": "included", + "excluded": "excluded", + "of {n} registered": "of {n} registered", + "In scope": "In scope", + "Suite": "Suite", + "Reason": "Reason", + "Yes": "Yes", + "No": "No", + "No involved agents known, so it was included to be safe.": "No involved agents known, so it was included to be safe.", + "Select all runs": "Select all runs", + "Select run": "Select run", + "Delete selected": "Delete selected", + "{n} still running and cannot be deleted": "{n} still running and cannot be deleted", + "Delete run": "Delete run", + "Delete this run and its case results? You won't be able to revert this!": "Delete this run and its case results? You won't be able to revert this!", + "Delete {n} runs and all their case results? You won't be able to revert this!": "Delete {n} runs and all their case results? You won't be able to revert this!", + "Deleted {runs} run(s) and {results} result(s).": "Deleted {runs} run(s) and {results} result(s).", + "Run {id} was kept: {reason}": "Run {id} was kept: {reason}", + "Failed to delete the runs.": "Failed to delete the runs.", + "Evaluation scope": "Evaluation scope", + "No test cases are registered yet.": "No test cases are registered yet.", + "batch {n}": "batch {n}", + "Write by Chat": "Write by Chat", + "Show": "Show", + "Hide": "Hide", + "Undo": "Undo", + "Say what the case should cover, in your own words. Every reply edits the draft on this page -- nothing is stored until you press Save.": "Say what the case should cover, in your own words. Every reply edits the draft on this page -- nothing is stored until you press Save.", + "For example: the resident says the fridge is leaking and asks when someone will come; it should reach the work order agent and look the order up.": "For example: the resident says the fridge is leaking and asks when someone will come; it should reach the work order agent and look the order up.", + "Working on it...": "Working on it...", + "Your instruction": "Your instruction", + "Add a turn, change an assertion, ask what a field means...": "Add a turn, change an assertion, ask what a field means...", + "Enter sends. Shift+Enter starts a new line.": "Enter sends. Shift+Enter starts a new line.", + "Send": "Send", + "Changed in this reply": "Changed in this reply", + "The assistant could not produce a valid draft:": "The assistant could not produce a valid draft:", + "Worth checking:": "Worth checking:", + "changed by chat": "changed by chat", + "New Case by Chat": "New Case by Chat", + "This suite has no judge model configured, so chat authoring has no default to fall back on -- pick one above.": "This suite has no judge model configured, so chat authoring has no default to fall back on -- pick one above." } diff --git a/src/lib/langs/zh.json b/src/lib/langs/zh.json index 0f2e8187..ab7f6f34 100644 --- a/src/lib/langs/zh.json +++ b/src/lib/langs/zh.json @@ -647,8 +647,8 @@ "No mocks. Every tool call this case makes will be blocked.": "没有 mock。这条用例发起的每一次工具调用都会被阻断。", "A disabled suite cannot be run at all.": "停用的套件完全无法运行。", "This suite is disabled. Triggering a run is rejected by the server until you enable it in Settings.": "这个套件已停用。在设置里重新启用之前，服务端会拒绝任何运行请求。", - "llmJudge assertions always fail in P1 regardless of these two fields.": "P1 阶段 llmJudge 断言恒定判失败，与这两个字段是否配置无关。", - "Always fails in P1.": "P1 阶段恒定判失败。", + "Required by llmJudge assertions. Without both, an llmJudge assertion records an Error rather than a score -- it is never scored with a default model.": "llmJudge 断言需要这两项。两项缺一，llmJudge 断言会记为 Error 而不是给分 —— 不会用默认模型代跑。", + "Scored 1-5 by the judge model configured on this suite. Pass mark defaults to 4.": "由测试集的判定模型按 1-5 打分，及格线默认 4。", "One function name per line. These run for real during a test -- only list functions with no side effects.": "每行一个函数名。这些函数在测试期间会真实执行——只填没有副作用的函数。", "One per line. Blocked even if allow-listed elsewhere.": "每行一个。即使在别处被放行也照样阻断。", "Assertions checked right after this turn": "本轮结束后立即求值的断言", @@ -740,5 +740,124 @@ "This run could not complete": "这次运行没能执行", "Nothing ran -- see the reason above.": "什么都没跑，原因见上方。", "Nothing here can run -- every case you picked is disabled. Enable at least one first.": "这样跑不起来——你勾选的用例全部处于停用状态。请先至少启用一条。", - "{count} of the selected case(s) are disabled and will be skipped.": "勾选的用例里有 {count} 条处于停用状态，将被跳过。" + "{count} of the selected case(s) are disabled and will be skipped.": "勾选的用例里有 {count} 条处于停用状态，将被跳过。", + "Case type": "用例类型", + "Routing case": "路由用例", + "Agent case": "Agent 用例", + "One turn, entered on the Copilot entry agent, asserting only which agent took the conversation. The only type counted towards a run's routing accuracy.": "单轮，从 Copilot 入口 agent 进入，只断言由哪个 agent 接走了这段对话。只有这一类会计入跑批的路由准确率。", + "One agent's own behaviour. Enter directly on that agent to keep the router out of what is measured.": "单个 agent 自身的行为。直接从该 agent 进入，把 Router 排除在被测范围之外。", + "Entry agent": "入口 agent", + "A routing agent runs the router and can hand off; any other agent is entered directly, so the router never runs.": "路由类型的 agent 会走 Router 并可能交接；其他 agent 是直接进入的，Router 完全不参与。", + "A Routing case needs at least one routedToAgent or agentChain assertion.": "路由用例至少需要一条 routedToAgent 或 agentChain 断言。", + "A Routing case cannot use llmJudge: routing is judged only by which agent handled the conversation.": "路由用例不能用 llmJudge：路由只按由哪个 agent 处理了对话来判定。", + "Mode": "比较方式", + "Assertion \"agentChain\" mode must be one of {modes}.": "agentChain 断言的比较方式只能是 {modes} 之一。", + "Copilot, Work Order Creator": "Copilot, Work Order Creator", + "These agents appear somewhere in the chain, in any order.": "这些 agent 出现在链路中的任意位置，不论顺序。", + "These agents appear in this order; other agents may come in between. This is the hand-off assertion.": "这些 agent 按此顺序出现，中间允许有其他 agent。这条就是交接断言。", + "The chain is exactly these agents and nothing else. One name asserts that nothing routed away.": "链路恰好是这些 agent，不多不少。只写一个名字就等于断言没有路由出去。", + "Agent chain": "Agent 链路", + "Answered by": "由谁回答", + "Routing accuracy": "路由准确率", + "agent default": "agent 默认模型", + "A Routing case must have exactly one turn. Use an Agent case for a multi-turn case.": "路由用例必须只有一轮。多轮请用 Agent 用例。", + "One agent's own behaviour. Enter directly on that agent to keep the router out of what is measured. A journey across several agents is an Agent case too -- use an agentChain assertion for the hand-offs.": "单个 agent 自身的行为。直接从该 agent 进入，把 Router 排除在被测范围之外。跨多个 agent 的旅程也算 Agent 用例 —— 交接用 agentChain 断言来表达。", + "Use the suite's agent": "使用测试集的 agent", + "The agent the conversation opens on. Leave it unset to use the suite's.": "对话从哪个 agent 开始。不选则使用测试集的。", + "History": "历史消息", + "Add Message": "新增消息", + "Written into the conversation before the first turn runs, so an existing question-and-answer exchange becomes the fixed starting context for this case.": "在第一轮开始前写入对话，把已有的一段问答变成这条用例固定的起始上下文。", + "These messages are not driven through the model, cost no tokens, and never appear in the agent chain.": "这些消息不经过模型驱动，不消耗 token，也不会出现在 Agent 链路里。", + "Role": "角色", + "Content": "内容", + "Move up": "上移", + "Move down": "下移", + "Remove message": "删除消息", + "No history. The case starts from an empty conversation.": "没有历史消息，用例从空对话开始。", + "History message {n} has no content.": "第 {n} 条历史消息没有内容。", + "Assistant": "助手", + "Copy": "复制", + "Copy case": "复制用例", + "Copied to \"{name}\". It is disabled until you enable it.": "已复制为“{name}”，处于停用状态，需要你手动启用。", + "Failed to copy the case.": "复制用例失败。", + "Registration": "登记信息", + "Decides when this case runs and what a failure of it means. Used to work out which cases a change actually needs to run.": "决定这条用例什么时候跑、以及它失败意味着什么。用于推算一次变更实际需要跑哪些用例。", + "Priority": "优先级", + "P0 runs in the stop-loss batch, P2 does not block a release.": "P0 在止损批次里跑，P2 不阻塞发布。", + "Severity": "严重级别", + "S0 is a stop, not a statistic. S2 is phrasing and experience.": "S0 是止损，不是统计项。S2 是措辞和体验问题。", + "Batch": "批次", + "Derive from priority": "按优先级推导", + "Runs in batch {n}.": "将在第 {n} 批运行。", + "Cross-cutting": "跨切面", + "Runs in every scope, whatever changed, and always in batch 1.": "任何变更范围都会跑，且固定在第 1 批。", + "Involved agents": "涉及的 agent", + "Derived from the entry agent": "由入口 agent 推导", + "Leave empty and the entry agent is used. Worth filling in for a routing case, where the agents that matter are the ones downstream of the router.": "留空则使用入口 agent。路由用例值得填 —— 真正要紧的是 Router 下游的那些 agent。", + "Business domain": "业务域", + "Last reviewed": "最近复核", + "Today": "今天", + "Never set automatically -- editing a case is not reviewing it.": "永不自动填写 —— 改用例不等于复核用例。", + "Expected outcome": "期望结果", + "For whoever reviews the result. Never evaluated -- an expected outcome a machine can check is an assertion, and belongs below where it will be enforced.": "给看结果的人读的，永不参与判定 —— 机器能检查的期望结果就是断言，应该写在下面那些会被真正执行的地方。", + "Latency, tokens and cost": "延迟、Token 与成本", + "Model": "模型", + "Cases": "用例数", + "Latency P50": "延迟 P50", + "Latency P95": "延迟 P95", + "Tokens": "Token", + "Cost": "成本", + "Unit cost (in / out)": "单价（输入 / 输出）", + "unknown": "未知", + "Percentiles are nearest-rank over the agent-call time of the cases that reached the model, so every figure is a duration some case actually took. Cases that failed before their first turn are excluded from the percentiles but still counted in tokens and cost.": "百分位取最近秩，基于真正调用到模型的那些用例的 agent 调用耗时，所以每个数字都是某条用例真实花掉的时间。第一轮之前就失败的用例不计入百分位，但仍计入 Token 与成本。", + "Wall clock for the whole case": "整条用例的墙钟时间", + "Agent call time only": "仅 agent 调用耗时", + "Tokens / cost": "Token / 成本", + "Plan a scope": "规划评估范围", + "Which cases a change needs to run, and which it does not. Read-only -- it plans a run, it does not start one.": "一次变更需要跑哪些用例、以及哪些不需要。只读 —— 它只做规划，不会启动跑批。", + "Changed agents": "变更涉及的 agent", + "All batches": "全部批次", + "Platform-wide change": "平台级变更", + "Narrowing is switched off: no agent is demonstrably untouched.": "关闭裁剪：没有哪个 agent 能被证明不受影响。", + "Plan": "规划", + "Failed to work out the scope.": "计算评估范围失败。", + "included": "纳入", + "excluded": "排除", + "of {n} registered": "共登记 {n} 条", + "In scope": "是否纳入", + "Suite": "测试集", + "Reason": "依据", + "Yes": "是", + "No": "否", + "No involved agents known, so it was included to be safe.": "涉及的 agent 未知，出于安全默认纳入。", + "Select all runs": "全选跑批记录", + "Select run": "选择这条跑批", + "Delete selected": "删除所选", + "{n} still running and cannot be deleted": "{n} 条仍在运行，无法删除", + "Delete run": "删除跑批记录", + "Delete this run and its case results? You won't be able to revert this!": "删除这条跑批记录及其全部用例结果？此操作不可撤销！", + "Delete {n} runs and all their case results? You won't be able to revert this!": "删除 {n} 条跑批记录及其全部用例结果？此操作不可撤销！", + "Deleted {runs} run(s) and {results} result(s).": "已删除 {runs} 条跑批记录、{results} 条用例结果。", + "Run {id} was kept: {reason}": "跑批 {id} 未删除：{reason}", + "Failed to delete the runs.": "删除跑批记录失败。", + "Evaluation scope": "评估范围", + "No test cases are registered yet.": "还没有登记任何测试用例。", + "batch {n}": "第 {n} 批", + "Write by Chat": "用对话写", + "Show": "展开", + "Hide": "收起", + "Undo": "撤销", + "Say what the case should cover, in your own words. Every reply edits the draft on this page -- nothing is stored until you press Save.": "用自己的话说说这个用例应该覆盖什么。每次回复都会直接改这页上的草稿——按下保存之前什么都不会存进去。", + "For example: the resident says the fridge is leaking and asks when someone will come; it should reach the work order agent and look the order up.": "例如：住户说冰箱在漏水，问什么时候有人来；应该转到工单 agent 并查一下工单。", + "Working on it...": "处理中……", + "Your instruction": "你的指令", + "Add a turn, change an assertion, ask what a field means...": "加一轮对话、改一条断言、或者问某个字段是什么意思……", + "Enter sends. Shift+Enter starts a new line.": "Enter 发送，Shift+Enter 换行。", + "Send": "发送", + "Changed in this reply": "本次回复改动了", + "The assistant could not produce a valid draft:": "助手没能给出一份能保存的草稿：", + "Worth checking:": "建议确认一下：", + "changed by chat": "对话改动过", + "New Case by Chat": "用对话新建", + "This suite has no judge model configured, so chat authoring has no default to fall back on -- pick one above.": "这个套件没有配置判官模型，对话出题也就没有默认可用的模型——请在上面选一个。" } diff --git a/src/lib/services/agent-test-service.js b/src/lib/services/agent-test-service.js index cc41d747..f35d0839 100644 --- a/src/lib/services/agent-test-service.js +++ b/src/lib/services/agent-test-service.js @@ -112,6 +112,58 @@ export async function updateCase(id, body) { return response.data; } +/** + * Work out which cases a change needs to run, before running anything. + * + * Read-only: it plans a run, it does not start one. Triggering stays per-suite, + * so a caller takes the included case ids from here and triggers each suite that + * appears among them. + * @param {{ targetAgentIds?: string[], fullPlatform?: boolean, batch?: number | null }} body + * @returns {Promise} + */ +export async function selectScope(body) { + const url = endpoints.agentTestScopeUrl; + const response = await axios.post(url, body); + return response.data; +} + +/** + * Clear run history: delete the named runs and the case results underneath them. + * + * Bulk-only on the server, and the single-row button calls it with one id -- the + * guard that refuses to delete a still-running run is worth having in exactly one + * place. + * + * A run that is still executing is skipped rather than failing the whole call, so + * the response reports what actually went and what did not. + * @param {string[]} runIds + * @returns {Promise} + */ +export async function deleteRuns(runIds) { + const url = endpoints.agentTestRunDeleteUrl; + const response = await axios.post(url, { runIds: runIds }); + return response.data; +} + +/** + * Duplicate an agent test case inside its own suite. + * + * Server-side rather than a get-then-create here, so the copy carries every + * field the case has. Rebuilding the payload from this client would drop + * anything it does not know about, and a copy missing its mocks looks identical + * in the list right up to the run where it blocks every tool. + * + * The copy lands disabled -- an exact duplicate joining the next run would + * measure the same thing twice. + * @param {string} id + * @returns {Promise} + */ +export async function copyCase(id) { + const url = endpoints.agentTestCaseCopyUrl.replace("{id}", id); + const response = await axios.post(url); + return response.data; +} + /** * Delete a test case * @param {string} id @@ -208,3 +260,20 @@ export async function getMockTargets(agentId) { }); return response.data; } + +/** + * One turn of authoring a case by conversation. + * + * Stateless on the server: the whole chat and the whole current draft go up every time, and what + * comes back replaces the draft. Nothing is saved -- the returned draft still has to go through + * createCase/updateCase, which is the only path that validates the entry agent and the only one the + * user presses deliberately. + * + * @param {import('$agentTestTypes').AgentTestAuthorRequest} body + * @returns {Promise} + */ +export async function authorCase(body) { + const url = endpoints.agentTestCaseAuthorUrl; + const response = await axios.post(url, body); + return response.data; +} diff --git a/src/lib/services/api-endpoints.js b/src/lib/services/api-endpoints.js index 2309b13e..71dac76e 100644 --- a/src/lib/services/api-endpoints.js +++ b/src/lib/services/api-endpoints.js @@ -140,10 +140,14 @@ export const endpoints = { agentTestSuiteRunUrl: `${host}/agent-test/suites/{id}/run`, agentTestCaseListUrl: `${host}/agent-test/cases`, agentTestCaseDetailUrl: `${host}/agent-test/cases/{id}`, + agentTestCaseCopyUrl: `${host}/agent-test/cases/{id}/copy`, agentTestRunListUrl: `${host}/agent-test/runs`, agentTestRunDetailUrl: `${host}/agent-test/runs/{id}`, agentTestRunCancelUrl: `${host}/agent-test/runs/{id}/cancel`, + agentTestRunDeleteUrl: `${host}/agent-test/runs/delete`, agentTestRecordUrl: `${host}/agent-test/record`, - agentTestMockTargetsUrl: `${host}/agent-test/mock-targets` + agentTestCaseAuthorUrl: `${host}/agent-test/author`, + agentTestMockTargetsUrl: `${host}/agent-test/mock-targets`, + agentTestScopeUrl: `${host}/agent-test/scope` } diff --git a/src/lib/styles/pages/_agent-test.scss b/src/lib/styles/pages/_agent-test.scss index 9a19511e..1f098c9d 100644 --- a/src/lib/styles/pages/_agent-test.scss +++ b/src/lib/styles/pages/_agent-test.scss @@ -878,3 +878,58 @@ font-size: 0.875rem; color: var(--color-muted); } + +/* ======================================================================== + * Authoring chat (case editor right-hand column) + * ======================================================================== + * A transcript, not a chat product: no avatars, no timestamps, no streaming. + * The two roles are told apart by ground and alignment only, because the + * thing worth looking at is the form next to it -- the chat is the input + * device, not the output. + * + * The log is height-capped and scrolls on its own so that a long + * conversation cannot push the composer off the bottom of a sticky column, + * which is exactly where it stops being usable. + */ +.ats-chat-log { + display: flex; + flex-direction: column; + gap: 0.5rem; + max-height: 22rem; + margin: 0.75rem 0; + overflow-y: auto; + padding-right: 0.25rem; +} + +.ats-chat-msg { + max-width: 92%; + padding: 0.5rem 0.75rem; + border-radius: 0.75rem; + font-size: 0.8125rem; + line-height: 1.5; + white-space: pre-wrap; + word-break: break-word; +} + +.ats-chat-msg-user { + align-self: flex-end; + background-color: color-mix(in srgb, var(--color-primary) 12%, transparent); + color: rgb(31 41 55); + + .dark & { + color: rgb(229 231 235); + } +} + +.ats-chat-msg-agent { + align-self: flex-start; + border: 1px solid rgb(229 231 235); + background-color: rgb(249 250 251); + color: rgb(31 41 55); + + .dark & { + border-color: rgb(55 65 81); + background-color: rgb(17 24 39 / 0.6); + color: rgb(229 231 235); + } +} diff --git a/src/routes/page/agent-test/+page.svelte b/src/routes/page/agent-test/+page.svelte index 64ba51e7..66803110 100644 --- a/src/routes/page/agent-test/+page.svelte +++ b/src/routes/page/agent-test/+page.svelte @@ -10,7 +10,7 @@ import Select from '$lib/common/dropdowns/Select.svelte'; import { getAgentOptions } from '$lib/services/agent-service.js'; import { getSuites, createSuite, deleteSuite } from '$lib/services/agent-test-service.js'; - import { t } from '$lib/helpers/utils/agent-test.js'; + import { errorMessage, t } from '$lib/helpers/utils/agent-test.js'; const duration = 3000; const nameMaxLength = 200; @@ -220,7 +220,6 @@ - - +
+ + + + {$_('Plan a scope')} + + +
diff --git a/src/routes/page/agent-test/[suiteId]/+page.svelte b/src/routes/page/agent-test/[suiteId]/+page.svelte index 54c600fc..b433a8d3 100644 --- a/src/routes/page/agent-test/[suiteId]/+page.svelte +++ b/src/routes/page/agent-test/[suiteId]/+page.svelte @@ -16,14 +16,26 @@ getSuite, updateSuite, getCases, + copyCase, deleteCase, updateCase, getRuns, triggerRun, cancelRun, + deleteRuns, recordCases } from '$lib/services/agent-test-service.js'; - import { statusColor, isTerminalStatus, errorMessage, formatDateTime, formatDuration, t } from '$lib/helpers/utils/agent-test.js'; + import { + statusColor, + isTerminalStatus, + priorityTone, + severityTone, + effectiveBatch, + errorMessage, + formatDateTime, + formatDuration, + t + } from '$lib/helpers/utils/agent-test.js'; const duration = 3000; const nameMaxLength = 200; @@ -67,6 +79,13 @@ /** @type {import('$agentTestTypes').AgentTestCase | null} */ let caseToDelete = $state(null); + /** @type {string[]} */ + let selectedRunIds = $state([]); + + /** Runs the confirm dialog is currently asking about; empty means it is closed. */ + /** @type {import('$agentTestTypes').AgentTestRun[]} */ + let runsToDelete = $state([]); + /** true = the run modal was opened by "Run Selected", false = "Run All Enabled". */ let runPartial = $state(false); @@ -171,6 +190,14 @@ let canSaveSettings = $derived(!!settingsDraft.name?.trim() && settingsDraft.caseTimeoutSeconds > 0); let canRecord = $derived(!!recordConversationId?.trim()); let allCasesSelected = $derived(cases.length > 0 && selectedCaseIds.length === cases.length); + let allRunsSelected = $derived(runs.length > 0 && selectedRunIds.length === runs.length); + + /** + * Selected runs that can actually go. A run still executing is refused by the server, so counting + * it here would promise a delete that will not happen. + */ + let deletableSelectedRuns = $derived( + runs.filter(r => selectedRunIds.includes(r.id) && isTerminalStatus(r.status))); onMount(async () => { isLoading = true; @@ -258,6 +285,10 @@ function loadRuns() { return getRuns(suiteId).then(res => { runs = res || []; + // Drop selections for runs that are gone, or a later delete would post ids the server + // can only answer "already deleted" for. + const ids = new Set(runs.map(r => r.id)); + selectedRunIds = selectedRunIds.filter(id => ids.has(id)); }).catch(() => { runs = []; }); @@ -317,6 +348,55 @@ selectedCaseIds = allCasesSelected ? [] : cases.map(x => x.id); } + /** @param {string} runId */ + function toggleRun(runId) { + selectedRunIds = selectedRunIds.includes(runId) + ? selectedRunIds.filter(id => id !== runId) + : [...selectedRunIds, runId]; + } + + function toggleAllRuns() { + selectedRunIds = allRunsSelected ? [] : runs.map(x => x.id); + } + + /** @param {import('$agentTestTypes').AgentTestRun[]} target */ + function openDeleteRunsModal(target) { + runsToDelete = target; + } + + function closeDeleteRunsModal() { + runsToDelete = []; + } + + function confirmDeleteRuns() { + const target = runsToDelete.map(r => r.id); + runsToDelete = []; + if (target.length === 0) return; + + isLoading = true; + deleteRuns(target).then(res => { + const deleted = res?.deletedRunIds?.length || 0; + if (deleted > 0) { + notifySuccess(t('Deleted {runs} run(s) and {results} result(s).', { + runs: deleted, + results: res?.deletedResultCount || 0 + })); + } + // Reported, not swallowed: a still-running run is left alone on purpose, and the reason + // says what to do about it. Silently showing "deleted 2" while a third row stays put is + // how a user concludes the button is broken. + (res?.skipped || []).forEach((/** @type {any} */ s) => { + notifyError(t('Run {id} was kept: {reason}', { id: s.runId.substring(0, 8), reason: s.reason })); + }); + selectedRunIds = []; + return loadRuns(); + }).catch(err => { + notifyError(errorMessage(err, t('Failed to delete the runs.'))); + }).finally(() => { + isLoading = false; + }); + } + function goBack() { goto('/page/agent-test'); } @@ -326,10 +406,33 @@ goto(`/page/agent-test/${suiteId}/case/${caseId}`); } + /** @param {import('$agentTestTypes').AgentTestCase} testCase */ + function copyTestCase(testCase) { + isLoading = true; + copyCase(testCase.id).then(copy => { + notifySuccess(t('Copied to "{name}". It is disabled until you enable it.', { name: copy.name })); + // Straight into the copy: it lands disabled and named "(copy)", and neither is + // something anyone leaves as it is -- a copy is made in order to be edited. + goToCase(copy.id); + }).catch(err => { + notifyError(errorMessage(err, t('Failed to copy the case.'))); + isLoading = false; + }); + } + function goToNewCase() { goto(`/page/agent-test/${suiteId}/case/new`); } + /** + * Same editor, with the authoring chat already open -- see the `chat` query parameter there. + * A separate entry rather than a mode of its own: whatever the chat produces still has to be + * reviewed and saved in the normal editor, so landing anywhere else would only add a hop. + */ + function goToNewCaseWithChat() { + goto(`/page/agent-test/${suiteId}/case/new?chat=1`); + } + /** @param {string} runId */ function goToRun(runId) { goto(`/page/agent-test/run/${runId}`); @@ -396,18 +499,15 @@ /** @param {import('$agentTestTypes').AgentTestCase} testCase */ function toggleCaseEnabled(testCase) { isLoading = true; - // Full replace again: everything the editor does not touch here still has - // to travel, or flipping the toggle wipes turns/mocks/assertions. + // PUT is a full replace, so everything has to travel or flipping this toggle wipes it. + // Spread rather than listed field by field: a hand-written payload silently drops each + // field added to a case afterwards, which is how caseType, entryAgentId and history all + // came to be erased by a single click on this toggle. Server-owned keys (id, createDate, + // updateDate) ride along harmlessly -- the upsert request has no such properties, so model + // binding ignores them. updateCase(testCase.id, { - suiteId: testCase.suiteId, - name: testCase.name, - enabled: !testCase.enabled, - turns: testCase.turns || [], - assertions: testCase.assertions || [], - initialStates: testCase.initialStates || [], - mocks: testCase.mocks || [], - unmockedToolPolicy: testCase.unmockedToolPolicy || 'Block', - sourceConversationId: testCase.sourceConversationId + ...testCase, + enabled: !testCase.enabled }).then(() => { notifySuccess(testCase.enabled ? t('Case disabled.') : t('Case enabled.')); return loadCases(); @@ -584,6 +684,21 @@ errorText={errorText} /> + 0} + icon="warning" + title={t('Are you sure?')} + text={runsToDelete.length === 1 + ? t('Delete this run and its case results? You won\'t be able to revert this!') + : t('Delete {n} runs and all their case results? You won\'t be able to revert this!', { n: runsToDelete.length })} + confirmBtnText={t('Yes, delete it!')} + cancelBtnText={t('Cancel')} + confirmBtnColor="danger" + confirm={confirmDeleteRuns} + cancel={closeDeleteRunsModal} + toggleModal={closeDeleteRunsModal} +/> + openRecord()}> {$_('Record from Conversation')} + @@ -721,7 +839,10 @@

{$_('No test cases in this suite yet.')}

- + + + + {#if testCase.caseType === 'Routing'} + {$_('Routing case')} + {:else} + {$_('Agent case')} + {/if} + + + + {testCase.priority || 'P1'} + + +
+ {$_('batch {n}', { values: { n: effectiveBatch(testCase) } })} +
+ + + + {testCase.severity || 'S1'} + + {testCase.turns?.length || 0} + {testCase.history?.length || 0} {testCase.mocks?.length || 0} {assertionCount} @@ -819,6 +974,16 @@ +
  • + +
    • - +
    + {#if selectedRunIds.length > 0} + + {#if deletableSelectedRuns.length < selectedRunIds.length} + + + {$_('{n} still running and cannot be deleted', { + values: { n: selectedRunIds.length - deletableSelectedRuns.length } + })} + + {/if} + {/if} + +
    @@ -873,6 +1061,15 @@ + @@ -884,6 +1081,15 @@ {#each runs as run (run.id)} +
    + toggleAllRuns()} + /> + {$_('Status')} {$_('Result')} {$_('Scope')}
    + toggleRun(run.id)} + /> + {$_(run.status)} @@ -934,7 +1140,21 @@ - {#if !isTerminalStatus(run.status)} + {#if isTerminalStatus(run.status)} +
  • + +
    • + {:else} +
  • + {/if} + + + + + + {#if isChatOpen} +
    +

    + {$_('Say what the case should cover, in your own words. Every reply edits the draft on this page -- nothing is stored until you press Save.')} +

    + +
    + {$_('Model')} + + +
    + {$_('Enter sends. Shift+Enter starts a new line.')} + +
    + + {#if authorErrorText} + + {/if} + + {#if authorChanges.length > 0} + +
    +

    {$_('Changed in this reply')}

    +
      + {#each authorChanges as change} +
    • {change.field} — {change.detail}
      • + {/each} +
    +
    + {/if} + + {#if authorErrors.length > 0} + + {/if} + + {#if authorWarnings.length > 0} + + {/if} +
    + {/if} +
    + + +
    @@ -593,6 +1103,40 @@
    +
    + + + +

    + {#if form.caseType === 'Routing'} + {$_('One turn, entered on the Copilot entry agent, asserting only which agent took the conversation. The only type counted towards a run\'s routing accuracy.')} + {:else} + {$_('One agent\'s own behaviour. Enter directly on that agent to keep the router out of what is measured. A journey across several agents is an Agent case too -- use an agentChain assertion for the hand-offs.')} + {/if} +

    +
    +
    + + + + {#each CASE_PRIORITIES as priority} + + {/each} + +

    + {$_('P0 runs in the stop-loss batch, P2 does not block a release.')} +

    +
    +
    + + +

    + {$_('S0 is a stop, not a statistic. S2 is phrasing and experience.')} +

    +
    +
    + + +

    + {$_('Runs in batch {n}.', { values: { n: effectiveBatch(form) } })} +

    +
    +
    +
    + + +
    +

    + {$_('Runs in every scope, whatever changed, and always in batch 1.')} +

    +
    +
    + + +
    +
    + +
    + + +
    +

    + {$_('Never set automatically -- editing a case is not reviewing it.')} +

    +
    +
    + + +

    + {$_('For whoever reviews the result. Never evaluated -- an expected outcome a machine can check is an assertion, and belongs below where it will be enforced.')} +

    +
    +
    +
    + + +
    +
    +
    +
    + + + +
    +
    + {$_('History')} + {#if changedFields.includes('history')} + {$_('changed by chat')} + {/if} +
    +

    {form.history.length}

    +
    +
    + +
    +
    +
    +

    + {$_('Written into the conversation before the first turn runs, so an existing question-and-answer exchange becomes the fixed starting context for this case.')} + {$_('These messages are not driven through the model, cost no tokens, and never appear in the agent chain.')} +

    + {#each form.history as message, i} +
    +
    +
    + + +
    +
    + + +
    +
    + + + +
    +
    +
    + {/each} + {#if form.history.length === 0} +
    +

    {$_('No history. The case starts from an empty conversation.')}

    +
    + {/if} +
    +
    +
    @@ -743,7 +1513,12 @@
    -
    {$_('Initial States')}
    +
    + {$_('Initial States')} + {#if changedFields.includes('initialStates')} + {$_('changed by chat')} + {/if} +

    {form.initialStates.length}

    @@ -777,7 +1552,12 @@
    -
    {$_('Tool Mocks')}
    +
    + {$_('Tool Mocks')} + {#if changedFields.includes('mocks')} + {$_('changed by chat')} + {/if} +

    {form.mocks.length}

    @@ -911,5 +1691,6 @@
    + {/if} diff --git a/src/routes/page/agent-test/run/[runId]/+page.svelte b/src/routes/page/agent-test/run/[runId]/+page.svelte index 4f06daab..41062336 100644 --- a/src/routes/page/agent-test/run/[runId]/+page.svelte +++ b/src/routes/page/agent-test/run/[runId]/+page.svelte @@ -7,7 +7,15 @@ import HeadTitle from '$lib/common/shared/HeadTitle.svelte'; import LoadingToComplete from '$lib/common/spinners/LoadingToComplete.svelte'; import { getRun, cancelRun, triggerRun } from '$lib/services/agent-test-service.js'; - import { statusColor, isTerminalStatus, errorMessage, formatDuration, formatDateTime, t } from '$lib/helpers/utils/agent-test.js'; + import { + statusColor, + isTerminalStatus, + errorMessage, + formatDuration, + formatDateTime, + formatAgentChain, + t + } from '$lib/helpers/utils/agent-test.js'; const duration = 3000; const pollIntervalMs = 2000; @@ -52,6 +60,36 @@ /** Column order for the comparison grid; follows the run's own model order. */ let modelColumns = $derived((run?.models || []).map(m => ({ ...m, key: `${m.provider}/${m.model}` }))); + + /** + * Per-model routing accuracy, straight off the run. Shown as "passed / total" next to the + * percentage on purpose: the gate is expressed in percentage points, but 3/4 says how much the + * figure is worth trusting and 75% does not -- and with a handful of routing cases that is the + * difference between a gate and a guess. + */ + let routingAccuracies = $derived((run?.routingAccuracies || []).map(a => ({ + ...a, + label: a.model || $_('agent default'), + percent: a.caseCount > 0 ? Math.round((a.passedCount / a.caseCount) * 1000) / 10 : null + }))); + + /** + * Latency, token and cost figures per model, as stored on the run. A table rather than tiles + * because there is one row per model and the whole point is comparing them side by side. + */ + let performance = $derived((run?.performanceSummaries || []).map(s => ({ + ...s, + label: s.model || $_('agent default') + }))); + + /** + * The unit costs the cost column was computed from. Shown next to the figures because a cost is + * not comparable with another run unless these match -- a provider price change would otherwise + * read as a cost regression with nothing to point at. + */ + let pricingByModel = $derived(Object.fromEntries( + (run?.modelPricing || []).map(p => [p.model || '', p]) + )); let isComparison = $derived(modelColumns.length > 1); /** @@ -309,6 +347,71 @@
    + {#if routingAccuracies.length > 0} + +
    +
    {$_('Routing accuracy')}
    +
    + {#each routingAccuracies as accuracy (accuracy.label)} +
    + {accuracy.percent}% + + {accuracy.label} · {accuracy.passedCount}/{accuracy.caseCount} + +
    + {/each} +
    +
    + {/if} + {#if performance.length > 0} +
    +
    {$_('Latency, tokens and cost')}
    +
    + + + + + + + + + + + + + + {#each performance as row (row.label)} + {@const pricing = pricingByModel[row.model || '']} + + + + + + + + + + {/each} + +
    {$_('Model')}{$_('Cases')}{$_('Latency P50')}{$_('Latency P95')}{$_('Tokens')}{$_('Cost')}{$_('Unit cost (in / out)')}
    {row.label}{row.caseCount}{formatDuration(row.latencyP50Ms)}{formatDuration(row.latencyP95Ms)}{row.totalTokens}{row.totalCost.toFixed(4)} + {#if pricing && pricing.textInputCost != null} + {pricing.textInputCost} / {pricing.textOutputCost} + {:else} + + {$_('unknown')} + {/if} +
    +
    +

    + {$_('Percentiles are nearest-rank over the agent-call time of the cases that reached the model, so every figure is a duration some case actually took. Cases that failed before their first turn are excluded from the percentiles but still counted in tokens and cost.')} +

    +
    + {/if}
    {run.totalCount} @@ -455,7 +558,32 @@ {result.model} {/if} - {formatDuration(result.durationMs)} + + {formatDuration(result.durationMs)} + + {#if result.modelDurationMs > 0} + + + ({formatDuration(result.modelDurationMs)}) + + {/if} + {#if result.totalTokens > 0} + + {result.totalTokens} · {result.cost.toFixed(4)} + + {/if} + {#if result.agentChain?.length > 0} + + + {formatAgentChain(result.agentChain)} + + {/if} {#if result.conversationId} {result.conversationId} {/if} @@ -490,6 +618,12 @@
    {$_('Agent output')}
    {turn.output || '--'}
    + {#if turn.agentChain?.length > 0} +
    +
    {$_('Answered by')}
    +
    {formatAgentChain(turn.agentChain)}
    +
    + {/if} {#if turn.assertions?.length > 0} {@render assertionTable(turn.assertions)} {:else} diff --git a/src/routes/page/agent-test/scope/+page.svelte b/src/routes/page/agent-test/scope/+page.svelte new file mode 100644 index 00000000..1e8a16d7 --- /dev/null +++ b/src/routes/page/agent-test/scope/+page.svelte @@ -0,0 +1,255 @@ + + + + + + + +
    +
    +
    +
    +
    +
    + + + +
    +
    {$_('Plan a scope')}
    +

    + {$_('Which cases a change needs to run, and which it does not. Read-only -- it plans a run, it does not start one.')} +

    +
    +
    + +
    +
    +
    +
    +
    + + + + + + + +
    +
    +
    + + +
    +

    + {$_('Narrowing is switched off: no agent is demonstrably untouched.')} +

    +
    +
    + +
    +
    + + {#if errorText} + + {/if} +
    +
    +
    +
    + +{#if scope} +
    +
    +
    +
    +
    + + + +
    +
    {$_('Evaluation scope')}
    +

    + {scope.included.length} + {$_('included')} · + {scope.excluded.length} + {$_('excluded')} · + {$_('of {n} registered', { values: { n: scope.totalCases } })} +

    +
    +
    +
    +
    + {#if scope.totalCases === 0} +
    +

    {$_('No test cases are registered yet.')}

    +
    + {:else} +
    + + + + + + + + + + + + + + {#each [...scope.included, ...scope.excluded] as scoped (scoped.caseId)} + {@const included = scope.included.some(c => c.caseId === scoped.caseId)} + + + + + + + + + {/each} + +
    {$_('In scope')}{$_('Case')}{$_('Suite')}{$_('Batch')}{$_('Severity')}{$_('Reason')}
    + {#if included} + {$_('Yes')} + {:else} + {$_('No')} + {/if} + {scoped.caseName}{scoped.suiteName}{scoped.batch} + + {scoped.severity} + + + + {scoped.reason} + + {#if scoped.reason === 'unknownAgents'} + + {$_('No involved agents known, so it was included to be safe.')} + + {/if} +
    +
    + {/if} +
    +
    +
    +
    +{/if} diff --git a/svelte.config.js b/svelte.config.js index 130d3715..b1979395 100644 --- a/svelte.config.js +++ b/svelte.config.js @@ -78,6 +78,7 @@ const config = { "/page/knowledge-base/dictionary", "/page/knowledge-base/[embed]/[embedType]", "/page/agent-test", + "/page/agent-test/scope", "/page/agent-test/[suiteId]", "/page/agent-test/[suiteId]/case/[caseId]", "/page/agent-test/run/[runId]"