Run skill evals with a shared marketplace environment

.github/scripts/eval-report.test.mjs

@@ -393,7 +393,9 @@ prDescriptionTest("pr-description eval stages the multi-skill marketplace cohort
   const evalDir = resolve(root, "evals/pr-description/skills/pr-description");
   const spec = parse(readFileSync(resolve(evalDir, "eval.yaml"), "utf8"));
   assert.equal(spec.scoring.threshold, 1);
-  const skills = spec.agent_environment.skills;
+  const project = parse(readFileSync(resolve(root, ".vally.yaml"), "utf8"));
+  assert.equal(spec.agent_environment, "marketplace");
+  const skills = project.environments.marketplace.skills;
   for (const configuredSkill of skills) {
     assert.ok(existsSync(resolve(evalDir, configuredSkill)), `configured cohort skill path is missing: ${configuredSkill}`);
   }

.vally.yaml

@@ -8,6 +8,23 @@ paths:
     - eval.yaml
     - "*.eval.yaml"
   results: vally-results
+environments:
+  marketplace:
+    # Vally resolves these paths relative to each eval, not this file.
+    skills:
+      - ../../../../plugins/plugin-authoring/skills/add-plugin
+      - ../../../../plugins/plugin-authoring/skills/try-plugin-locally
+      - ../../../../plugins/plugin-authoring/skills/submit-plugin
+      - ../../../../plugins/marketplace-maintainer/skills/update-remote-pins
+      - ../../../../plugins/marketplace-maintainer/skills/release-plugins
+      - ../../../../plugins/pr-description/skills/pr-description
+    # Vally runs setup in each trial workspace; Copilot discovers these skills natively.
+    commands:
+      - git clone --filter=blob:none --no-checkout --depth 1 --branch copilot-evals-v0.24.0 --sparse https://github.com/github/evals.git .vally-eval-source
+      - git -C .vally-eval-source sparse-checkout set packages/plugin/skills
+      - git -C .vally-eval-source checkout --detach 34ddb5803aeaee8330ab70c19ad1a3f37fb77ded
+      - >-
+        node -e "const fs = require('node:fs'); fs.cpSync('.vally-eval-source/packages/plugin/skills', '.github/skills', { recursive: true }); fs.rmSync('.vally-eval-source', { recursive: true });"
 suites:
   smoke:
     description: One representative case for each skill and agent

evals/marketplace-maintainer/skills/release-plugins/eval.yaml

@@ -6,12 +6,7 @@ defaults:
   model: claude-haiku-5.5
   timeout: 5m
   runs: 1
-# Both of the plugin's skills load, as they do when it's installed, so these
-# cases also check that Copilot picks the right one.
-agent_environment:
-  skills:
-    - ../../../../plugins/marketplace-maintainer/skills/update-remote-pins
-    - ../../../../plugins/marketplace-maintainer/skills/release-plugins
+agent_environment: marketplace
 grading_environment:
   files:
     - src: grader.mjs

evals/marketplace-maintainer/skills/update-remote-pins/eval.yaml

@@ -6,12 +6,7 @@ defaults:
   model: claude-haiku-5.5
   timeout: 5m
   runs: 1
-# Both of the plugin's skills load, as they do when it's installed, so these
-# cases also check that Copilot picks the right one.
-agent_environment:
-  skills:
-    - ../../../../plugins/marketplace-maintainer/skills/update-remote-pins
-    - ../../../../plugins/marketplace-maintainer/skills/release-plugins
+agent_environment: marketplace
 grading_environment:
   files:
     - src: grader.mjs

evals/plugin-authoring/skills/add-plugin/eval.yaml

@@ -6,13 +6,7 @@ defaults:
   model: claude-haiku-5.5
   timeout: 5m
   runs: 1
-# All of the plugin's skills load, as they do when it's installed, so these
-# cases also check that Copilot picks the right one.
-agent_environment:
-  skills:
-    - ../../../../plugins/plugin-authoring/skills/add-plugin
-    - ../../../../plugins/plugin-authoring/skills/try-plugin-locally
-    - ../../../../plugins/plugin-authoring/skills/submit-plugin
+agent_environment: marketplace
 grading_environment:
   files:
     - src: grader.mjs

evals/plugin-authoring/skills/submit-plugin/eval.yaml

@@ -6,13 +6,7 @@ defaults:
   model: claude-haiku-5.5
   timeout: 5m
   runs: 1
-# All of the plugin's skills load, as they do when it's installed, so these
-# cases also check that Copilot picks the right one.
-agent_environment:
-  skills:
-    - ../../../../plugins/plugin-authoring/skills/add-plugin
-    - ../../../../plugins/plugin-authoring/skills/try-plugin-locally
-    - ../../../../plugins/plugin-authoring/skills/submit-plugin
+agent_environment: marketplace
 grading_environment:
   files:
     - src: grader.mjs

evals/plugin-authoring/skills/try-plugin-locally/eval.yaml

@@ -6,13 +6,7 @@ defaults:
   model: claude-haiku-5.5
   timeout: 5m
   runs: 1
-# All of the plugin's skills load, as they do when it's installed, so these
-# cases also check that Copilot picks the right one.
-agent_environment:
-  skills:
-    - ../../../../plugins/plugin-authoring/skills/add-plugin
-    - ../../../../plugins/plugin-authoring/skills/try-plugin-locally
-    - ../../../../plugins/plugin-authoring/skills/submit-plugin
+agent_environment: marketplace
 grading_environment:
   files:
     - src: grader.mjs

evals/pr-description/skills/pr-description/eval.yaml

@@ -6,16 +6,7 @@ defaults:
   model: claude-haiku-5.5
   timeout: 5m
   runs: 1
-# Stage pr-description alongside the other marketplace skills so cases verify
-# realistic multi-skill routing and non-collision.
-agent_environment:
-  skills:
-    - ../../../../plugins/pr-description/skills/pr-description
-    - ../../../../plugins/plugin-authoring/skills/add-plugin
-    - ../../../../plugins/plugin-authoring/skills/try-plugin-locally
-    - ../../../../plugins/plugin-authoring/skills/submit-plugin
-    - ../../../../plugins/marketplace-maintainer/skills/update-remote-pins
-    - ../../../../plugins/marketplace-maintainer/skills/release-plugins
+agent_environment: marketplace
 grading_environment:
   files:
     - src: grader.mjs