Use evals/results and honor Vally results configuration

.github/scripts/eval-report.mjs

@@ -2,6 +2,7 @@ import { existsSync, readFileSync, readdirSync, realpathSync, statSync } from "n
 import { relative, resolve, sep } from "node:path";
 import { fileURLToPath } from "node:url";
 import { parseArgs, stripVTControlCharacters } from "node:util";
+import { loadProjectConfig } from "@microsoft/vally/config";
 
 const hasGraderError = (grade) => grade?.status === "error" || (grade?.details ?? []).some(hasGraderError);
 const stripTerminalControls = (text) => stripVTControlCharacters(String(text ?? ""))
@@ -434,18 +435,20 @@ export function renderReport(summary, path, { baselineSummary, baselinePath } =
   return lines.join("\n");
 }
 
-export function resolveResultsPath(input, root = process.cwd()) {
+export async function resolveResultsPath(input, root = process.cwd()) {
   if (!input) {
-    const base = resolve(root, "vally-results");
+    const config = await loadProjectConfig(root);
+    const base = resolve(config.dir || root, config.paths.results);
+    const display = relative(root, base).split(sep).join("/") || ".";
     if (!existsSync(base)) {
-      throw new Error("No saved eval runs under vally-results/. Run evals first, or pass --results <path>.");
+      throw new Error(`No saved eval runs under ${display}/. Run evals first, or pass --results <path>.`);
     }
     const runs = readdirSync(base, { withFileTypes: true })
       .filter((entry) => entry.isDirectory() && existsSync(resolve(base, entry.name, "results.jsonl")))
       .map((entry) => entry.name)
       .sort();
     if (!runs.length) {
-      throw new Error("No saved eval runs under vally-results/. Run evals first, or pass --results <path>.");
+      throw new Error(`No saved eval runs under ${display}/. Run evals first, or pass --results <path>.`);
     }
     return resolve(base, runs.at(-1), "results.jsonl");
   }
@@ -463,8 +466,8 @@ export function resolveResultsPath(input, root = process.cwd()) {
   return resolved;
 }
 
-export function loadResults(path, root = process.cwd()) {
-  const file = resolveResultsPath(path, root);
+export async function loadResults(path, root = process.cwd()) {
+  const file = await resolveResultsPath(path, root);
   const lines = readFileSync(file, "utf8").split(/\r?\n/).filter((line) => line.trim());
   const records = lines.map((line) => JSON.parse(line));
   const displayPath = relative(root, file).split(sep).join("/") || file;
@@ -481,7 +484,8 @@ table highlighting fixed failures, regressions, and token deltas.
 Options:
   --results, --candidate, --run-dir <path>
                     Candidate results.jsonl file or run directory. Defaults to
-                    the newest run under vally-results/.
+                    the newest run under .vally.yaml's paths.results (Vally's
+                    default is vally-results/ when not configured).
   --baseline <path> Baseline results.jsonl file or run directory to compare
                     against the candidate run.
   -h, --help        Show this help message.
@@ -490,7 +494,7 @@ Examples:
   npm run eval:report
   npm run eval:compare -- --baseline evals/_shared/fixtures/pr-description/baseline-results.jsonl --candidate evals/_shared/fixtures/pr-description/candidate-results.jsonl`;
 
-function main(argv = process.argv.slice(2)) {
+async function main(argv = process.argv.slice(2)) {
   let parsed;
   try {
     parsed = parseArgs({
@@ -520,8 +524,8 @@ function main(argv = process.argv.slice(2)) {
   const baselineInput = values.baseline ?? (positionals.length === 2 ? positionals[0] : undefined);
   const candidateInput = values.candidate ?? values.results ?? values["run-dir"]
     ?? (positionals.length === 2 ? positionals[1] : positionals[0]);
-  const candidate = loadResults(candidateInput);
-  const baseline = baselineInput ? loadResults(baselineInput) : undefined;
+  const candidate = await loadResults(candidateInput);
+  const baseline = baselineInput ? await loadResults(baselineInput) : undefined;
   const markdown = renderReport(candidate.summary, candidate.displayPath, {
     baselineSummary: baseline?.summary,
     baselinePath: baseline?.displayPath,
@@ -534,7 +538,7 @@ function main(argv = process.argv.slice(2)) {
 
 if (process.argv[1] && realpathSync(fileURLToPath(import.meta.url)) === realpathSync(resolve(process.argv[1]))) {
   try {
-    main();
+    await main();
   } catch (error) {
     console.error(error instanceof Error ? error.message : String(error));
     process.exitCode = 1;

.github/scripts/eval-report.test.mjs

@@ -65,6 +65,33 @@ const makeFooter = (runId = "run-1", hadExecutionErrors = false) => ({
   evals: [],
 });
 
+test("eval-report follows native project results configuration and retains explicit paths and defaults", async (t) => {
+  const directory = resolve(root, ".github/scripts", `.report-path-test-${randomUUID()}`);
+  mkdirSync(directory);
+  t.after(() => rmSync(directory, { recursive: true, force: true }));
+  const configured = resolve(directory, "evals/results/2026-01-01/results.jsonl");
+  const legacy = resolve(directory, "vally-results/2026-02-01/results.jsonl");
+  for (const [file, stimulus] of [[configured, "configured-results"], [legacy, "native-default"]]) {
+    mkdirSync(resolve(file, ".."), { recursive: true });
+    writeFileSync(file, `${JSON.stringify(makeTrial({ stimulus }))}\n${JSON.stringify(makeFooter())}\n`);
+  }
+  const configuration = resolve(directory, ".vally.yaml");
+  writeFileSync(configuration, "paths:\n  results: evals/results\n");
+  assert.equal((await loadResults(undefined, directory)).file, configured);
+  assert.equal((await loadResults("vally-results/2026-02-01", directory)).file, legacy);
+  const cli = spawnSync(process.execPath, [resolve(root, ".github/scripts/eval-report.mjs")], {
+    cwd: directory, encoding: "utf8", timeout: 10000,
+  });
+  assert.ifError(cli.error);
+  assert.equal(cli.status, 0, cli.stderr);
+  assert.match(cli.stdout, /configured-results/);
+  assert.doesNotMatch(cli.stdout, /native-default/);
+  writeFileSync(configuration, "{}\n");
+  assert.equal((await loadResults(undefined, directory)).file, legacy);
+  writeFileSync(configuration, "paths: [\n");
+  await assert.rejects(loadResults(undefined, directory), /parse|YAML|flow sequence/i);
+});
+
 test("extractTrialMetrics and summarizeRun extract per-trial and aggregate tokens and duration", () => {
   const trial1 = makeTrial({ stimulus: "case-a", durationMs: 1500, inputTokens: 800, outputTokens: 200, totalTokens: 1000 });
   const trial2 = makeTrial({ stimulus: "case-b", durationMs: 2500, inputTokens: 1200, outputTokens: 300, totalTokens: 1500, passed: false, gradeScore: 0.5 });
@@ -172,7 +199,7 @@ test("compareSummaries and renderReport surface fixed failures, regressions, and
   assert.match(markdown, /Single-run score, token, and duration deltas are observed signals for review/);
 });
 
-prDescriptionTest("eval-report CLI compares the shared pr-description baseline and candidate fixtures without writing GITHUB_STEP_SUMMARY", (t) => {
+prDescriptionTest("eval-report CLI compares the shared pr-description baseline and candidate fixtures without writing GITHUB_STEP_SUMMARY", async (t) => {
   const tempDir = resolve(root, ".github/scripts", `.report-test-${randomUUID()}`);
   mkdirSync(tempDir, { recursive: true });
   t.after(() => rmSync(tempDir, { recursive: true, force: true }));
@@ -181,8 +208,8 @@ prDescriptionTest("eval-report CLI compares the shared pr-description baseline a
 
   const baselinePath = "evals/_shared/fixtures/pr-description/baseline-results.jsonl";
   const candidatePath = "evals/_shared/fixtures/pr-description/candidate-results.jsonl";
-  const loadedBase = loadResults(baselinePath, root);
-  const loadedCand = loadResults(candidatePath, root);
+  const loadedBase = await loadResults(baselinePath, root);
+  const loadedCand = await loadResults(candidatePath, root);
   assert.equal(loadedBase.summary.passedCount, 1);
   assert.equal(loadedCand.summary.passedCount, 2);
 

.gitignore

@@ -1,5 +1,6 @@
 node_modules/
 vally-results/
+evals/results/
 .vally/
 # Isolated Copilot homes from local install checks.
 .copilot-home/

.vally.yaml

@@ -7,7 +7,7 @@ paths:
   evalFilenames:
     - eval.yaml
     - "*.eval.yaml"
-  results: vally-results
+  results: evals/results
 environments:
   marketplace:
     # Vally resolves these paths relative to each eval, not this file.
@@ -20,11 +20,17 @@ environments:
       - ../../../../plugins/pr-description/skills/pr-description
     # Vally runs setup in each trial workspace; Copilot discovers these skills natively.
     commands:
-      - git clone --filter=blob:none --no-checkout --depth 1 --branch copilot-evals-v0.24.0 --sparse https://github.com/github/evals.git .vally-eval-source
+      - git clone --filter=blob:none --no-checkout --depth 1 --branch
+        copilot-evals-v0.24.0 --sparse https://github.com/github/evals.git
+        .vally-eval-source
       - git -C .vally-eval-source sparse-checkout set packages/plugin/skills
-      - git -C .vally-eval-source checkout --detach 34ddb5803aeaee8330ab70c19ad1a3f37fb77ded
+      - git -C .vally-eval-source checkout --detach
+        34ddb5803aeaee8330ab70c19ad1a3f37fb77ded
       - >-
-        node -e "const fs = require('node:fs'); fs.cpSync('.vally-eval-source/packages/plugin/skills', '.github/skills', { recursive: true }); fs.rmSync('.vally-eval-source', { recursive: true });"
+        node -e "const fs = require('node:fs');
+        fs.cpSync('.vally-eval-source/packages/plugin/skills', '.github/skills',
+        { recursive: true }); fs.rmSync('.vally-eval-source', { recursive: true
+        });"
 suites:
   smoke:
     description: One representative case for each skill and agent

docs/checks-and-evals.md

@@ -111,7 +111,8 @@ npm run eval -- --suite smoke
 ```
 
 For these direct Vally runs, results follow `paths.results` in
-[.vally.yaml](../.vally.yaml), currently `vally-results/`.
+[.vally.yaml](../.vally.yaml), currently `evals/results/`. The report command
+uses the same configured directory unless you supply an explicit results path.
 
 Summarize the newest saved run (verdicts, scores, input/output/total tokens,
 and duration) or compare a baseline run against a candidate run without calling