diff --git a/.github/scripts/eval-report.mjs b/.github/scripts/eval-report.mjs index 6e69faf..92ec880 100644 --- a/.github/scripts/eval-report.mjs +++ b/.github/scripts/eval-report.mjs @@ -2,6 +2,7 @@ import { existsSync, readFileSync, readdirSync, realpathSync, statSync } from "n import { relative, resolve, sep } from "node:path"; import { fileURLToPath } from "node:url"; import { parseArgs, stripVTControlCharacters } from "node:util"; +import { loadProjectConfig } from "@microsoft/vally/config"; const hasGraderError = (grade) => grade?.status === "error" || (grade?.details ?? []).some(hasGraderError); const stripTerminalControls = (text) => stripVTControlCharacters(String(text ?? "")) @@ -434,18 +435,20 @@ export function renderReport(summary, path, { baselineSummary, baselinePath } = return lines.join("\n"); } -export function resolveResultsPath(input, root = process.cwd()) { +export async function resolveResultsPath(input, root = process.cwd()) { if (!input) { - const base = resolve(root, "vally-results"); + const config = await loadProjectConfig(root); + const base = resolve(config.dir || root, config.paths.results); + const display = relative(root, base).split(sep).join("/") || "."; if (!existsSync(base)) { - throw new Error("No saved eval runs under vally-results/. Run evals first, or pass --results ."); + throw new Error(`No saved eval runs under ${display}/. Run evals first, or pass --results .`); } const runs = readdirSync(base, { withFileTypes: true }) .filter((entry) => entry.isDirectory() && existsSync(resolve(base, entry.name, "results.jsonl"))) .map((entry) => entry.name) .sort(); if (!runs.length) { - throw new Error("No saved eval runs under vally-results/. Run evals first, or pass --results ."); + throw new Error(`No saved eval runs under ${display}/. Run evals first, or pass --results .`); } return resolve(base, runs.at(-1), "results.jsonl"); } @@ -463,8 +466,8 @@ export function resolveResultsPath(input, root = process.cwd()) { return resolved; } -export function loadResults(path, root = process.cwd()) { - const file = resolveResultsPath(path, root); +export async function loadResults(path, root = process.cwd()) { + const file = await resolveResultsPath(path, root); const lines = readFileSync(file, "utf8").split(/\r?\n/).filter((line) => line.trim()); const records = lines.map((line) => JSON.parse(line)); const displayPath = relative(root, file).split(sep).join("/") || file; @@ -481,7 +484,8 @@ table highlighting fixed failures, regressions, and token deltas. Options: --results, --candidate, --run-dir Candidate results.jsonl file or run directory. Defaults to - the newest run under vally-results/. + the newest run under .vally.yaml's paths.results (Vally's + default is vally-results/ when not configured). --baseline Baseline results.jsonl file or run directory to compare against the candidate run. -h, --help Show this help message. @@ -490,7 +494,7 @@ Examples: npm run eval:report npm run eval:compare -- --baseline evals/_shared/fixtures/pr-description/baseline-results.jsonl --candidate evals/_shared/fixtures/pr-description/candidate-results.jsonl`; -function main(argv = process.argv.slice(2)) { +async function main(argv = process.argv.slice(2)) { let parsed; try { parsed = parseArgs({ @@ -520,8 +524,8 @@ function main(argv = process.argv.slice(2)) { const baselineInput = values.baseline ?? (positionals.length === 2 ? positionals[0] : undefined); const candidateInput = values.candidate ?? values.results ?? values["run-dir"] ?? (positionals.length === 2 ? positionals[1] : positionals[0]); - const candidate = loadResults(candidateInput); - const baseline = baselineInput ? loadResults(baselineInput) : undefined; + const candidate = await loadResults(candidateInput); + const baseline = baselineInput ? await loadResults(baselineInput) : undefined; const markdown = renderReport(candidate.summary, candidate.displayPath, { baselineSummary: baseline?.summary, baselinePath: baseline?.displayPath, @@ -534,7 +538,7 @@ function main(argv = process.argv.slice(2)) { if (process.argv[1] && realpathSync(fileURLToPath(import.meta.url)) === realpathSync(resolve(process.argv[1]))) { try { - main(); + await main(); } catch (error) { console.error(error instanceof Error ? error.message : String(error)); process.exitCode = 1; diff --git a/.github/scripts/eval-report.test.mjs b/.github/scripts/eval-report.test.mjs index 6286a10..303c157 100644 --- a/.github/scripts/eval-report.test.mjs +++ b/.github/scripts/eval-report.test.mjs @@ -65,6 +65,33 @@ const makeFooter = (runId = "run-1", hadExecutionErrors = false) => ({ evals: [], }); +test("eval-report follows native project results configuration and retains explicit paths and defaults", async (t) => { + const directory = resolve(root, ".github/scripts", `.report-path-test-${randomUUID()}`); + mkdirSync(directory); + t.after(() => rmSync(directory, { recursive: true, force: true })); + const configured = resolve(directory, "evals/results/2026-01-01/results.jsonl"); + const legacy = resolve(directory, "vally-results/2026-02-01/results.jsonl"); + for (const [file, stimulus] of [[configured, "configured-results"], [legacy, "native-default"]]) { + mkdirSync(resolve(file, ".."), { recursive: true }); + writeFileSync(file, `${JSON.stringify(makeTrial({ stimulus }))}\n${JSON.stringify(makeFooter())}\n`); + } + const configuration = resolve(directory, ".vally.yaml"); + writeFileSync(configuration, "paths:\n results: evals/results\n"); + assert.equal((await loadResults(undefined, directory)).file, configured); + assert.equal((await loadResults("vally-results/2026-02-01", directory)).file, legacy); + const cli = spawnSync(process.execPath, [resolve(root, ".github/scripts/eval-report.mjs")], { + cwd: directory, encoding: "utf8", timeout: 10000, + }); + assert.ifError(cli.error); + assert.equal(cli.status, 0, cli.stderr); + assert.match(cli.stdout, /configured-results/); + assert.doesNotMatch(cli.stdout, /native-default/); + writeFileSync(configuration, "{}\n"); + assert.equal((await loadResults(undefined, directory)).file, legacy); + writeFileSync(configuration, "paths: [\n"); + await assert.rejects(loadResults(undefined, directory), /parse|YAML|flow sequence/i); +}); + test("extractTrialMetrics and summarizeRun extract per-trial and aggregate tokens and duration", () => { const trial1 = makeTrial({ stimulus: "case-a", durationMs: 1500, inputTokens: 800, outputTokens: 200, totalTokens: 1000 }); const trial2 = makeTrial({ stimulus: "case-b", durationMs: 2500, inputTokens: 1200, outputTokens: 300, totalTokens: 1500, passed: false, gradeScore: 0.5 }); @@ -172,7 +199,7 @@ test("compareSummaries and renderReport surface fixed failures, regressions, and assert.match(markdown, /Single-run score, token, and duration deltas are observed signals for review/); }); -prDescriptionTest("eval-report CLI compares the shared pr-description baseline and candidate fixtures without writing GITHUB_STEP_SUMMARY", (t) => { +prDescriptionTest("eval-report CLI compares the shared pr-description baseline and candidate fixtures without writing GITHUB_STEP_SUMMARY", async (t) => { const tempDir = resolve(root, ".github/scripts", `.report-test-${randomUUID()}`); mkdirSync(tempDir, { recursive: true }); t.after(() => rmSync(tempDir, { recursive: true, force: true })); @@ -181,8 +208,8 @@ prDescriptionTest("eval-report CLI compares the shared pr-description baseline a const baselinePath = "evals/_shared/fixtures/pr-description/baseline-results.jsonl"; const candidatePath = "evals/_shared/fixtures/pr-description/candidate-results.jsonl"; - const loadedBase = loadResults(baselinePath, root); - const loadedCand = loadResults(candidatePath, root); + const loadedBase = await loadResults(baselinePath, root); + const loadedCand = await loadResults(candidatePath, root); assert.equal(loadedBase.summary.passedCount, 1); assert.equal(loadedCand.summary.passedCount, 2); diff --git a/.gitignore b/.gitignore index 94210d3..27cbd82 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ node_modules/ vally-results/ +evals/results/ .vally/ # Isolated Copilot homes from local install checks. .copilot-home/ diff --git a/.vally.yaml b/.vally.yaml index 02efee6..28de733 100644 --- a/.vally.yaml +++ b/.vally.yaml @@ -7,7 +7,7 @@ paths: evalFilenames: - eval.yaml - "*.eval.yaml" - results: vally-results + results: evals/results environments: marketplace: # Vally resolves these paths relative to each eval, not this file. @@ -20,11 +20,17 @@ environments: - ../../../../plugins/pr-description/skills/pr-description # Vally runs setup in each trial workspace; Copilot discovers these skills natively. commands: - - git clone --filter=blob:none --no-checkout --depth 1 --branch copilot-evals-v0.24.0 --sparse https://github.com/github/evals.git .vally-eval-source + - git clone --filter=blob:none --no-checkout --depth 1 --branch + copilot-evals-v0.24.0 --sparse https://github.com/github/evals.git + .vally-eval-source - git -C .vally-eval-source sparse-checkout set packages/plugin/skills - - git -C .vally-eval-source checkout --detach 34ddb5803aeaee8330ab70c19ad1a3f37fb77ded + - git -C .vally-eval-source checkout --detach + 34ddb5803aeaee8330ab70c19ad1a3f37fb77ded - >- - node -e "const fs = require('node:fs'); fs.cpSync('.vally-eval-source/packages/plugin/skills', '.github/skills', { recursive: true }); fs.rmSync('.vally-eval-source', { recursive: true });" + node -e "const fs = require('node:fs'); + fs.cpSync('.vally-eval-source/packages/plugin/skills', '.github/skills', + { recursive: true }); fs.rmSync('.vally-eval-source', { recursive: true + });" suites: smoke: description: One representative case for each skill and agent diff --git a/docs/checks-and-evals.md b/docs/checks-and-evals.md index a4d6988..3604cec 100644 --- a/docs/checks-and-evals.md +++ b/docs/checks-and-evals.md @@ -111,7 +111,8 @@ npm run eval -- --suite smoke ``` For these direct Vally runs, results follow `paths.results` in -[.vally.yaml](../.vally.yaml), currently `vally-results/`. +[.vally.yaml](../.vally.yaml), currently `evals/results/`. The report command +uses the same configured directory unless you supply an explicit results path. Summarize the newest saved run (verdicts, scores, input/output/total tokens, and duration) or compare a baseline run against a candidate run without calling