Add a native standard-model experiment and setup guidance

.github/scripts/validation.test.mjs

@@ -386,6 +386,7 @@ test("validation calls CI scripts, and paid eval activation is manual-only", ()
   assert.deepEqual(JSON.parse(readFileSync(resolve(root, "package.json"), "utf8")).scripts, {
     lint: "node .github/scripts/lint.mjs",
     eval: "vally eval --workers 1 --max-retries 1",
+    "eval:models": "vally experiment run evals/standard-models.experiment.yaml --output-dir evals/results/standard-models --require-pass",
     "eval:report": "node .github/scripts/eval-report.mjs",
     "eval:compare": "node .github/scripts/eval-report.mjs",
   });

README.md

@@ -199,6 +199,14 @@ request.
 [Evals](.github/workflows/evals.yml) runs only when manually started; pull
 requests never trigger paid evals.
 
+**Standard models:** `npm run eval:models` uses the native
+[model experiment](evals/standard-models.experiment.yaml) to test every retained
+skill and agent eval with Claude Sonnet 5.5 and GPT-6.1 Terra. These are starter
+defaults: during setup, customize `matrix.model.values` and its `baseline.model`
+to the models you want to test and can access. Preview without model calls with
+`npm run eval:models -- --dry-run`. Individual evals and Actions keep their
+existing single-model defaults; the experiment does not override them globally.
+
 See [Checks and evals](docs/checks-and-evals.md) for CI details, local test
 commands, and Bash and PowerShell eval setup.
 

docs/checks-and-evals.md

@@ -79,6 +79,34 @@ side effects.
 The `plugin-reviewer` eval needs the full Copilot CLI installed by `npm ci`.
 Point `COPILOT_CLI_PATH` at that pinned CLI before running the smoke suite.
 
+### Standard model roster
+
+[The native experiment](../evals/standard-models.experiment.yaml) is the single
+source for the standard model roster: `claude-sonnet-5.5` and `gpt-6.1-terra`.
+When adopting the template, replace `matrix.model.values` with your own model
+IDs and set `baseline.model` to one of them. Vally requires at least two variants;
+for a single model, use `npm run eval -- --model <model-id>` instead.
+
+After setting `COPILOT_CLI_PATH` as shown below, preview the selected evals,
+variants and configuration hashes without running setup, Docker or a model:
+
+```sh
+npm run eval:models -- --dry-run
+```
+
+`npm run eval:models` executes the planned variants and spends AI credits for
+each case under each model. The two-model roster is not a guarantee of account
+access or a quality claim. The baseline identifies the experiment's control,
+not a preferred winner; the command does not invoke an extra comparison judge.
+Only the model varies, preserving the evals, graders, fixtures and shared
+marketplace environment. The existing single-model commands and Actions do
+not consume this experiment.
+
+Vally 0.19 experiments have a separate output flag rather than reading
+`paths.results`. The npm command explicitly writes timestamped per-variant
+results under `evals/results/standard-models/`. If you customize the output
+location, update the command's `--output-dir` too.
+
 The `try-plugin-locally` smoke case prepares a fresh-session `--plugin-dir`
 command for the selected plugin in a developer's existing checkout. Vally
 already stages the candidate skills. The case checks the selected absolute

evals/standard-models.experiment.yaml

@@ -0,0 +1,14 @@
+name: standard-models
+evals:
+  - "*/skills/*/eval.yaml"
+  - "*/agents/*/eval.yaml"
+matrix:
+  model:
+    path: /defaults/model
+    values:
+      - claude-sonnet-5.5
+      - gpt-6.1-terra
+baseline:
+  model: claude-sonnet-5.5
+execution:
+  workers: 1

package.json

@@ -7,6 +7,7 @@
   "scripts": {
     "lint": "node .github/scripts/lint.mjs",
     "eval": "vally eval --workers 1 --max-retries 1",
+    "eval:models": "vally experiment run evals/standard-models.experiment.yaml --output-dir evals/results/standard-models --require-pass",
     "eval:report": "node .github/scripts/eval-report.mjs",
     "eval:compare": "node .github/scripts/eval-report.mjs"
   },