Default repository evals to native Docker with Vally 0.21

.github/scripts/docker-defaults.test.mjs

@@ -0,0 +1,130 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { randomUUID } from "node:crypto";
+import { spawnSync } from "node:child_process";
+import { dirname, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { ProjectContext, discoverAndFilterEvals } from "@microsoft/vally";
+import { resolveBackendSettings } from "../../node_modules/@microsoft/vally-cli/dist/backend-selection.js";
+import { DockerBackend } from "../../node_modules/@microsoft/vally/dist/backend/docker/docker-backend.js";
+import { parse } from "yaml";
+
+const root = fileURLToPath(new URL("../../", import.meta.url));
+
+function assertReviewerCoverage(projectRoot, evals) {
+  if (existsSync(resolve(projectRoot, "plugins/marketplace-maintainer"))) {
+    assert.ok(evals.some(({ spec }) => spec.name === "plugin-reviewer-independent"),
+      "retained marketplace-maintainer requires its independent reviewer eval");
+  }
+}
+
+function assertBackendInheritance(command) {
+  assert.doesNotMatch(command, /--(?:backend(?:-args)?|in-place)\b/);
+}
+
+test("native full-suite and explicit-spec launches inherit Docker for every retained eval", async () => {
+  const project = await ProjectContext.load(root);
+  const suite = project.resolveSuite("full");
+  const discovery = await discoverAndFilterEvals(project, { suiteEvals: suite.evals });
+  assert.deepEqual(discovery.errors, []);
+  assertReviewerCoverage(root, discovery.matched);
+  for (const { filePath, spec } of discovery.matched) {
+    const settings = resolveBackendSettings({
+      evalBackend: spec.defaults?.backend,
+      projectBackend: project.config.defaults?.backend,
+      projectDir: project.dir,
+      evalDir: dirname(filePath),
+      cliCwd: root,
+    });
+    assert.equal(settings.name, "docker", filePath);
+    assert.equal(settings.args["grade-network"], "none", filePath);
+  }
+  const pkg = JSON.parse(readFileSync(resolve(root, "package.json"), "utf8"));
+  assertBackendInheritance(pkg.scripts.eval);
+  const workflow = parse(readFileSync(resolve(root, ".github/workflows/evals.yml"), "utf8"));
+  const launches = workflow.jobs.evals.steps.filter((step) => /npm run eval/.test(step.run ?? ""));
+  assert.equal(launches.length, 1);
+  assertBackendInheritance(launches[0].run);
+});
+
+test("reviewer coverage permits plugin removal but rejects a missing retained reviewer eval", (t) => {
+  const directory = resolve(root, ".vally", `reviewer-coverage-${randomUUID()}`);
+  mkdirSync(directory, { recursive: true });
+  t.after(() => rmSync(directory, { recursive: true, force: true }));
+  assertReviewerCoverage(directory, []);
+  mkdirSync(resolve(directory, "plugins/marketplace-maintainer"), { recursive: true });
+  assert.throws(() => assertReviewerCoverage(directory, []), /requires its independent reviewer eval/);
+  assertReviewerCoverage(directory, [{ spec: { name: "plugin-reviewer-independent" } }]);
+});
+
+test("shared launchers reject backend and in-place overrides in either argument form", () => {
+  for (const command of ["vally eval --workers 1", "npm run eval -- --suite smoke --require-pass"]) {
+    assertBackendInheritance(command);
+    for (const option of [
+      "--backend local", "--backend=local", "--backend docker", "--backend=docker",
+      "--backend-args grade-network=default", "--backend-args=grade-network=default",
+      "--in-place", "--in-place=true",
+    ]) {
+      assert.throws(() => assertBackendInheritance(`${command} ${option}`), assert.AssertionError);
+    }
+  }
+});
+
+test("a missing Docker daemon rejects native preparation rather than selecting host execution", async () => {
+  const backend = new DockerBackend({
+    cwd: root,
+    cli: {
+      contextInspect: async () => ({ name: "test-local", dockerEndpoint: "unix:///var/run/docker.sock" }),
+      info: async () => { throw new Error("test Docker daemon unavailable"); },
+    },
+  });
+  await assert.rejects(backend.prepare({}, { runId: "model-free-docker-unavailable" }),
+    /test Docker daemon unavailable/);
+  await backend.shutdown();
+});
+
+test("the native CLI fails closed without Docker even when a model-free local executor is available", (t) => {
+  const directory = resolve(root, ".vally", `docker-unavailable-${randomUUID()}`);
+  mkdirSync(directory, { recursive: true });
+  t.after(() => rmSync(directory, { recursive: true, force: true }));
+  const spec = resolve(directory, "eval.yaml");
+  writeFileSync(spec, [
+    "name: model-free-docker-unavailable",
+    "defaults:",
+    "  executor: mock",
+    "  timeout: 1s",
+    "stimuli:",
+    "  - name: synthetic",
+    "    prompt: model-free",
+    "    graders:",
+    "      - type: completed",
+    "",
+  ].join("\n"));
+  const child = spawnSync(process.execPath, [
+    resolve(root, "node_modules/@microsoft/vally-cli/dist/index.js"),
+    "eval", "--eval-spec", spec, "--work-dir", root,
+    "--workspace", resolve(directory, "workspace"),
+    "--output-dir", resolve(directory, "outputs"),
+    "--workers", "1", "--runs", "1", "--max-retries", "0",
+    "--shutdown-timeout", "1s", "--require-pass",
+  ], {
+    cwd: root,
+    encoding: "utf8",
+    timeout: 20_000,
+    env: {
+      PATH: directory,
+      ...(process.env.SystemRoot && { SystemRoot: process.env.SystemRoot }),
+      HOME: directory,
+      USERPROFILE: directory,
+      TMP: directory,
+      TEMP: directory,
+      DOCKER_CONFIG: directory,
+      DOCKER_CONTEXT: "model-free-nonexistent-context",
+      VALLY_TELEMETRY_OPTOUT: "1",
+    },
+  });
+  assert.ifError(child.error);
+  assert.notEqual(child.status, 0, child.stdout + child.stderr);
+  assert.match(child.stdout + child.stderr, /docker.*ENOENT|ENOENT.*docker|Docker backend preparation/i);
+});

.github/workflows/evals.yml

@@ -15,9 +15,7 @@ jobs:
       copilot-requests: write
     env:
       VALLY_TELEMETRY_OPTOUT: '1'
-      # The pinned Copilot CLI, which the plugin-reviewer eval needs to delegate to a custom agent.
-      COPILOT_CLI_PATH: ${{ github.workspace }}/node_modules/@github/copilot/npm-loader.js
-      COPILOT_AUTO_UPDATE: 'false'
+      # .vally.yaml selects Docker; the appliance supplies the executor runtime.
     steps:
       - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
         with:
@@ -27,6 +25,7 @@ jobs:
           node-version: 24
           cache: npm
       - run: npm ci --no-audit --no-fund
+      - run: test "$(docker info --format '{{.OSType}}')" = linux && docker compose version
       - run: npm run eval -- --suite smoke --require-pass
         env:
           GITHUB_TOKEN: ${{ github.token }}

.vally.yaml

@@ -1,4 +1,9 @@
 # Vally configuration for the included plugins' evals.
+defaults:
+  backend:
+    name: docker
+    args:
+      grade-network: none
 paths:
   # evals/ mirrors plugins/: one eval per skill or agent, such as
   # evals/<plugin>/skills/<skill>/eval.yaml.

CONTRIBUTING.md

@@ -17,7 +17,7 @@ project you agree to abide by its terms.
 ## Submitting a pull request
 
 1. [Fork][fork] and clone the repository.
-1. Install the dependencies with Node.js 22.14 or newer: `npm ci`
+1. Install the dependencies with Node.js 22.15.0 or newer (24+ recommended): `npm ci`
 1. Make sure the lint passes on your machine: `npm run lint`
 1. Create a new branch: `git checkout -b my-branch-name`
 1. Make your change. For a new plugin, follow [Add a plugin](README.md#add-a-plugin).

README.md

@@ -42,8 +42,9 @@ For marketplace development and evals, use this shared setup.
 - **Node.js 24+ and npm** for repository tooling and the bundled
   [local-install validator](plugins/plugin-authoring/skills/try-plugin-locally/references/local-install.md#test-installation-without-publishing).
 - **GitHub Copilot CLI 1.0.85+**.
-- **Vally ([`@microsoft/vally-cli`](https://www.npmjs.com/package/@microsoft/vally-cli)) 0.19.0+**.
+- **Vally ([`@microsoft/vally-cli`](https://www.npmjs.com/package/@microsoft/vally-cli)) 0.21.0**, pinned in `package.json`.
 - **Git 2.28.0+**.
+- For paid repository evals, **Docker with Compose and a running Linux daemon**.
 
 ## Make it yours
 
@@ -199,6 +200,20 @@ request.
 [Evals](.github/workflows/evals.yml) runs only when manually started; pull
 requests never trigger paid evals.
 
+Repository evals **default to Vally's native Docker backend** in
+[.vally.yaml](.vally.yaml), including `npm run eval` and the manually dispatched
+eval workflow. `npm run eval -- --suite full` selects every retained case,
+including the independent reviewer agent. These runs still spend AI credits
+and require Docker, Compose, network access, and container-compatible
+authentication. Missing Docker or image preparation fails the run; it does not
+silently fall back to host execution. Vally builds and caches its version-matched
+default image locally.
+See [Native Docker evals and isolation limits](docs/checks-and-evals.md#native-docker-evals)
+before running. The pinned Evals plugin's Run entrypoint cannot read the new
+project backend default and currently rejects this configuration; use the native
+Vally CLI rather than assuming plugin Run is container-isolated. Paid CI remains
+manual-only.
+
 See [Checks and evals](docs/checks-and-evals.md) for CI details, local test
 commands, and Bash and PowerShell eval setup.
 

docs/checks-and-evals.md

@@ -10,6 +10,22 @@ npm ci
 npm run lint
 ```
 
+The lockfile keeps exact versions and integrity hashes without registry-specific
+tarball URLs. Preserve existing SHA-512 integrity values for unchanged package
+versions: a registry mirror that omits `dist.integrity` can cause npm to replace
+them with weaker SHA-1 values from `dist.shasum` during an upgrade. Removing
+`resolved` URLs does not require changing integrity values. After changing
+dependencies, preserve that format with npm's
+[`omit-lockfile-registry-resolved`](https://docs.npmjs.com/cli/v11/using-npm/config/#omit-lockfile-registry-resolved)
+option:
+
+```sh
+npm install --package-lock-only --ignore-scripts --omit-lockfile-registry-resolved
+```
+
+`npm ci` resolves tarball endpoints through the configured registry. A successful
+install through a development mirror does not verify public-registry availability.
+
 `npm run lint` checks local catalog consistency, then checks every skill under
 `plugins/` and every eval under `evals/` with the pinned
 [Vally](https://microsoft.github.io/vally/) CLI. A remote-only marketplace can
@@ -69,15 +85,27 @@ call a model or prove how a plugin behaves in a model session.
 
 **These runs start real Copilot sessions and spend AI credits.**
 
+Docker execution readiness means every selected case finishes task execution
+and grading without execution or grading errors. A completed behavioral
+**FAIL** is still a valid graded result, not an infrastructure error; preserve
+it for follow-up rather than changing skills, graders, or thresholds to force
+a pass. Commands using `--require-pass`, including the manual workflow,
+also require passing behavioral verdicts and can therefore exit unsuccessfully
+even when Docker execution and grading completed correctly.
+
 Skill evals use Vally's built-in `skill-invocation` grader: `required` for every
 positive and boundary case, and `disallowed` for negative cases. Invocation
 credit requires a matching `skill_activation` event. Successful file reads,
 searches, path mentions, and completed answers do not count as invocation.
 Custom program graders independently check the resulting files, output, and
 side effects.
 
-The `plugin-reviewer` eval needs the full Copilot CLI installed by `npm ci`.
-Point `COPILOT_CLI_PATH` at that pinned CLI before running the smoke suite.
+Repository evals use the Docker backend selected in
+[.vally.yaml](../.vally.yaml). The appliance supplies its own executor runtime;
+do not forward a host `COPILOT_CLI_PATH` into a Linux container. The independent
+`plugin-reviewer` eval stages its actual read-only custom agent and requires
+delegation through the runtime's task tool. It is included in both normal
+discovery and the full suite, not replaced by a companion skill.
 
 The `try-plugin-locally` smoke case prepares a fresh-session `--plugin-dir`
 command for the selected plugin in a developer's existing checkout. Vally
@@ -96,20 +124,111 @@ install-check recipes remain covered by model-free tests. Running a real
 installation check still requires the user's exact cleanup confirmation and
 host permissions; a headless denial is a blocker, not permission to bypass them.
 
+### Native Docker evals
+
+**These are real Copilot runs and still spend AI credits.** Use Vally 0.21.0's
+native [`docker` backend](https://microsoft.github.io/vally/guides/running-evals-in-docker/)
+through the repository's `defaults.backend` configuration. It applies to all
+seven saved evals, including explicit `--eval-spec` runs, `--suite full`, and
+the manual Actions workflow. The current graders are deterministic, so the
+project also defaults grading to `grade-network: none`. Missing Docker, an
+inaccessible daemon, or failed appliance preparation is an error, not permission
+to use the local backend.
+
+The pinned Evals plugin's full-engine project reader rejects the
+`defaults.backend` field before Run can execute. Thus plugin Run is currently
+unsupported for this repository configuration, **not** an alternate Docker
+launcher or a local fallback. Its Setup writer only covers paths and suites;
+this advanced native setting is maintained in `.vally.yaml`. Upstream plugin
+support is needed before its Run entrypoint can honor this Docker default.
+
+After `npm ci`, make sure `docker` is on `PATH`, your account can access a
+running **Linux** Docker daemon, and `docker compose version` succeeds. The
+first uncached run needs network access and enough disk space to build Vally's
+default appliance. Vally builds it from the installed core package and caches
+it as `vally-task:<content-hash>`. Its image/core versions must match exactly;
+let Vally select/build the image after upgrades rather than reusing an older
+registry image. No repository Dockerfile or manual image rebuild is needed.
+Host npm connectivity does not prove registry connectivity inside Docker's
+build environment. Resolve build-network or certificate-trust failures before
+running; do not disable TLS verification.
+
+On restricted-registry networks, Vally 0.21.0's default builder does not inherit
+host `.npmrc` files or expose an npm-registry build override. A separately
+reviewed, version-matched custom image can be selected explicitly with
+`--backend docker --backend-args task-image=<immutable-image-id>` and the
+intended role-network arguments described below. A successful run with that
+image verifies native task/grade execution, **not** the unmodified default-image
+bootstrap. Never tag a modified recipe with Vally's canonical default cache tag.
+
+The container uses an isolated Copilot home and cannot reuse the host's
+interactive Copilot login. Vally automatically forwards the recognized model
+authentication variables (`COPILOT_GITHUB_TOKEN`, `GH_TOKEN`, `GITHUB_TOKEN`)
+to the task when using `copilot-sdk`. Use only the credential you intend to
+expose. For example, with an existing authenticated `gh` account, run this
+complete two-case eval (positive and negative), without printing or saving the
+token:
+
 Bash:
 
 ```sh
-export COPILOT_CLI_PATH="$PWD/node_modules/@github/copilot/npm-loader.js"
-npm run eval -- --suite smoke
+export GH_TOKEN="$(gh auth token)"
+npm run eval -- --eval-spec evals/pr-description/skills/pr-description/eval.yaml --max-retries 0
+unset GH_TOKEN
 ```
 
 PowerShell:
 
 ```powershell
-$env:COPILOT_CLI_PATH = "$PWD\node_modules\@github\copilot\npm-loader.js"
-npm run eval -- --suite smoke
+$env:GH_TOKEN = gh auth token
+npm run eval -- --eval-spec evals/pr-description/skills/pr-description/eval.yaml --max-retries 0
+Remove-Item Env:GH_TOKEN
 ```
 
+This uses the eval's declared model and one trial per case. The native image
+supplies its own executor runtime, distinct from the host's pinned Copilot CLI.
+
+With the same explicitly selected authentication, `npm run eval -- --suite
+smoke` selects the smoke cases and `npm run eval -- --suite full --max-retries
+0` selects every case. The full suite includes the independent reviewer's
+positive delegations and negative no-delegation case. Review its complete
+case/model plan and approve its additional AI-credit exposure before starting.
+
+Native precedence is CLI backend, then eval backend, then project backend.
+An explicit `--backend` replaces the project backend argument layer. When
+using an independently reviewed custom image, repeat the intended role policy,
+for example `--backend docker --backend-args task-image=<immutable-image-id>
+--backend-args grade-network=none`; do not assume project arguments survive the
+explicit override. No existing eval overrides the Docker project default.
+
+Know the boundary before approving a run:
+
+- The host still runs Vally orchestration and trusted build/setup tooling.
+  Task execution and grading use separate containers and workspaces, even
+  when they share the cached image. Hidden grading files are not staged into
+  the task; grading receives captured task evidence. Graders within one trial
+  share their grade environment and must treat agent-authored files as untrusted.
+- Task and grade roles run as non-root with capabilities dropped and
+  `no-new-privileges`. Vally also uses root-running initialization/evidence
+  helpers with limited capabilities; Docker is not protection against every
+  kernel escape or a substitute for reviewing trusted code.
+- Vally's built-in role-network defaults are Docker's normal network, **not offline**.
+  The task needs network for Copilot and the shared marketplace environment's
+  pinned remote-skill fetch. `task-network=none` is only suitable for a
+  self-contained model-free probe. The project sets `grade-network=none`
+  because the current repository graders are deterministic; LLM or network-dependent
+  graders need their own reviewed networking and credentials.
+- The task can read and use its forwarded model credential and reach allowed
+  network services. Custom `task-env` and `grade-env` forwards are role-specific;
+  never forward judge secrets to the task. Review eval setup commands, network
+  effects, and credentials separately from the container boundary.
+- Vally exports task workspaces and releases trial resources. Check warnings
+  for preserved host paths or `docker-volume://` recovery locations if export
+  or cleanup fails. An interrupted host or daemon can leave resources behind;
+  do not remove other runs' containers or volumes.
+
+### Saved results
+
 For these direct Vally runs, results follow `paths.results` in
 [.vally.yaml](../.vally.yaml), currently `evals/results/`. The report command
 uses the same configured directory unless you supply an explicit results path.