ronniegeraghty · Nov 11, 2026 · Nov 11, 2026 · Nov 11, 2026 · Nov 11, 2026 · Nov 11, 2026
@@ -5,7 +5,7 @@ import { randomUUID } from "node:crypto";
 import { copyFileSync, cpSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
 import { resolve } from "node:path";
 import { fileURLToPath, pathToFileURL } from "node:url";
-import { createDefaultGraderRegistry } from "@microsoft/vally";
+import { createDefaultGraderRegistry, gradeInDisposableWorkspace } from "@microsoft/vally";
 import { parse } from "yaml";
 
 const root = fileURLToPath(new URL("../../", import.meta.url));
@@ -34,11 +34,11 @@ async function fixture(t) {
     "```sh", `copilot --plugin-dir "${path}"`, "```",
     "Session: not started", "Discovery: unverified", "Behavior: unverified",
   ].join("\n");
-  const grade = (text = output(), commands = [], toolCalls = [], results = []) => score("local-guidance", evaluate({
+  const grade = (text = output(), commands = [], toolCalls = [], results = [], extraEvents = []) => score("local-guidance", evaluate({
     output: text,
     events: [...[...commands.map((command) => ({ toolName: "powershell", arguments: { command } })), ...toolCalls].map((call, index) => ({
       type: "tool_call", data: { toolCallId: String(index), ...call },
-    })), ...results],
+    })), ...results, ...extraEvents],
   }, directory));
   return { directory, selected, output, grade };
 }
@@ -55,6 +55,50 @@ guidanceTest("local guidance accepts the selected absolute checkout path with sp
   if (process.platform === "win32") assert.equal(grade(output(selected.toLowerCase())).passed, true);
 });
 
+guidanceTest("local guidance checks the original agent path when Vally grades a workspace copy", async (t) => {
+  const { directory, output, grade } = await fixture(t);
+  const original = resolve(directory, "original trial");
+  const selected = resolve(original, "plugins/sample-notes");
+  const activation = [{ type: "skill_activation", data: {
+    name: "try-plugin-locally", path: resolve(original, "try-plugin-locally/SKILL.md"),
+  } }];
+  assert.equal(grade(output(selected), [], [], [], activation).passed, true);
+  for (const wrong of [resolve(directory, "plugins/sample-notes"), resolve(original, "installed/sample-notes"), "./plugins/sample-notes"]) {
+    assert.equal(grade(output(wrong), [], [], [], activation).passed, false, wrong);
+  }
+  assert.equal(grade(output(selected)).passed, false, "an answer cannot declare its own original workspace");
+});
+
+guidanceTest("local guidance accepts explicit tool-use approval without dropping the permission requirement", async (t) => {
+  const { output, grade } = await fixture(t);
+  const approved = output().replace("host trust and tool permissions", "host trust and approve tool use");
+  assert.equal(grade(approved).passed, true);
+  assert.equal(grade(approved.replace(" and approve tool use", "")).passed, false);
+});
+
+guidanceTest("native disposable grading preserves the selected agent-workspace identity", async (t) => {
+  const { directory, output } = await fixture(t);
+  const grader = createDefaultGraderRegistry().get("program");
+  const result = await gradeInDisposableWorkspace({
+    trajectory: {
+      output: output(),
+      workDir: directory,
+      events: [{ type: "skill_activation", data: {
+        name: "try-plugin-locally", path: resolve(directory, "try-plugin-locally/SKILL.md"),
+      } }],
+    },
+    graderFiles: [
+      { src: resolve(directory, "grader.mjs"), dest: ".grading/grader.mjs" },
+      { src: resolve(directory, "common.mjs"), dest: ".grading/common.mjs" },
+    ],
+    grade: async (trajectory) => {
+      assert.notEqual(trajectory.workDir, directory);
+      return grader.grade({ trajectory, config: { program: process.execPath, args: [".grading/grader.mjs"] } });
+    },
+  });
+  assert.equal(result.passed, true, result.evidence);
+});
+
 guidanceTest("local guidance rejects relative, wrong, installed, and published plugin sources", async (t) => {
   const { directory, output, grade } = await fixture(t);
   for (const path of [

@@ -70,6 +70,11 @@ releaseTest("unknown-release-base requires an actionable evidence request instea
     "This catalog is not ready to release. Please share the released base.",
     "To get this catalog ready to release, I need one thing first. Please provide the released base.",
     "I can't tell whether sample-notes is ready for release yet. Which released tag should I compare against?",
+    "Before I edit, I need:\n\n1. **The released base**: a tag or release commit SHA.\n2. **Or confirmation** that this is an unreleased first version.\n",
+    "Please provide:\n- A released tag.\n- Or confirmation of the initial release.\n",
+    "**Please provide:**\n- A released base.\n",
+    "Please provide:\n- A plugin name.\n- A released base.\n",
+    "Current blockers:\n- The plugin name is missing.\n- Please provide a released base.\n",
   ]) {
     assert.equal(grade(output).passed, true, output);
   }
@@ -80,6 +85,14 @@ releaseTest("unknown-release-base requires an actionable evidence request instea
     "sample-notes 1.0.1 is ready to release against the base; no edits needed.",
     "Please provide the released base. This catalog is ready to release.",
     "Please provide the released base. I did not find evidence, but it is release-ready.",
+    "I need:\n- A plugin name.\n\nThe released base is missing.",
+    "The release record is unknown.\n\nPlease provide:\n- A plugin name.\n",
+    "The released base is missing. Please provide:\n- A plugin name.",
+    "The released base is missing; Please provide:\n- A plugin name.\n",
+    "Current blockers:\n- The released base is missing.\n- Please provide a plugin name.",
+    "Please provide:\n- A plugin name. The released base is missing.",
+    "**Please provide:**\n- A plugin name. The released base is missing.",
+    "Please provide:\n- A plugin name; the released base is missing.",
   ]) {
     const result = grade(output);
     assert.equal(result.passed, false, output);

@@ -83,10 +83,13 @@ export function evaluate(trajectory, workspace, mode) {
     const clauses = trajectory.output.match(/[^.!?\n;,]+/g) ?? [];
     const evidence = /\b(?:released?\s+(?:base|tag|commit|record|version)|release\s+(?:record|base|tag)|first[- ]release|(?:first|initial)\s+release)\b/i;
     const request = /\b(?:please\s+(?:provide|share|supply|confirm|specify)|(?:can|could|would)\s+you\s+(?:provide|share|supply|confirm|specify)|I\s+(?:need|require))\b|\b(?:what|which)\b[^?]*\?/i;
+    const asksInList = [...trajectory.output.matchAll(/([^.!?\n;]+:)[*_]*\s*\n((?:\s*(?:\d+[.)]|[-*])\s+[^\n]+(?:\n|$))+)/g)]
+      .some(([, header, items]) => request.test(header) && items.trim().split(/\r?\n/).some((item) =>
+        evidence.test(item.replace(/^\s*(?:\d+[.)]|[-*])\s+/, "").split(/[.!?;,]/, 1)[0])));
     const ready = /\b(?:ready\s+(?:to|for)\s+(?:the\s+)?release|release[- ]ready)\b/i;
     const unready = /\b(?:(?:not|isn't|aren't|can't|cannot|unable\s+to)\b|(?:whether|if|before|until|to\s+(?:get|make)\b))[\s\S]*\b(?:ready\s+(?:to|for)\s+(?:the\s+)?release|release[- ]ready)\b/i;
     return [
-      check("Asked for a released base or first-release confirmation", statements.some((statement) =>
+      check("Asked for a released base or first-release confirmation", asksInList || statements.some((statement) =>
         request.test(statement) && evidence.test(statement))),
       check("Did not claim release readiness without evidence", !clauses.some((clause) => ready.test(clause) && !unready.test(clause))),
     ];

@@ -1,10 +1,16 @@
-import { isAbsolute, relative, resolve } from "node:path";
+import { basename, dirname, isAbsolute, relative, resolve } from "node:path";
 import { calls, check, isMain, json, matchesTool, runGrade } from "./common.mjs";
 
 export function evaluate(trajectory, workspace) {
   const catalog = json(workspace, "marketplace.json");
   const plugin = json(workspace, "plugins/sample-notes/plugin.json");
-  const selected = resolve(workspace, "plugins/sample-notes");
+  const activation = trajectory.events?.find((event) => event.type === "skill_activation"
+    && event.data?.name === "try-plugin-locally")?.data;
+  // Vally grades in a copy; the runtime activation retains the agent's original skill path.
+  const source = typeof activation?.path === "string" && isAbsolute(activation.path)
+    && basename(activation.path) === "SKILL.md" && basename(dirname(activation.path)) === "try-plugin-locally"
+    ? dirname(dirname(activation.path)) : workspace;
+  const selected = resolve(source, "plugins/sample-notes");
   const output = trajectory.output;
   const blocks = [...output.matchAll(/```[^\n]*\n([\s\S]*?)```/g)].map((match) => match[1]);
   const commandLines = blocks.flatMap((block) => block.trim().split(/\r?\n/));
@@ -34,7 +40,8 @@ export function evaluate(trajectory, workspace) {
       && operands.length === 1 && selectedPath(operands[0])),
     check("Explained a fresh session and source trust", /\b(?:fresh|new)\s+(?:Copilot CLI\s+)?session\b/i.test(output)
       && /\b(?:review|inspect)\b[^.\n]*\b(?:source|code|plugin)\b/i.test(output)
-      && /\btrust\b/i.test(output) && /\bpermissions?\b/i.test(output)),
+      && /\btrust\b/i.test(output)
+      && /\b(?:permissions?|approve\s+tool\s+use|approval\s+(?:for|of)\s+tool\s+use)\b/i.test(output)),
     check("Left the session and host checks honestly unverified", /\bSession:\s*not started\b/i.test(output)
       && /\bDiscovery:\s*unverified\b/i.test(output) && /\bBehaviou?r:\s*unverified\b/i.test(output)
       && !/\b(?:Discovery|Behaviou?r):\s*(?:verified|passed|successful)\b/i.test(output)),

@@ -13,7 +13,7 @@
     {
       "name": "marketplace-maintainer",
       "source": "./plugins/marketplace-maintainer",
-      "version": "0.1.0",
+      "version": "0.2.0",
       "description": "Review contributed plugins, maintain remote pins, and prepare marketplace releases."
     },
     {

@@ -1,5 +1,9 @@
 # Changelog
 
+## 0.2.0
+
+- Route confirmed first-release note requests through the release skill even when no files need changing.
+
 ## 0.1.0
 
 - Resolve annotated release tags through the GitHub commit API.

@@ -7,7 +7,7 @@ checkout of this marketplace.
 |---|---|
 | [plugin-reviewer](com.github.copilot/agents/plugin-reviewer.agent.md) | Review a contribution and report risks and inconsistencies, without editing anything |
 | [update-remote-pins](skills/update-remote-pins/SKILL.md) | Review upstream changes and add or update a remote pin |
-| [release-plugins](skills/release-plugins/SKILL.md) | Check versions and changelogs, draft release notes, or retire a plugin |
+| [release-plugins](skills/release-plugins/SKILL.md) | Check versions and changelogs, draft release notes including first releases, or retire a plugin |
 
 ## Install
 

@@ -1,7 +1,7 @@
 {
   "$schema": "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json",
   "name": "marketplace-maintainer",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "Review contributed plugins, maintain remote pins, and prepare marketplace releases.",
   "license": "MIT",
   "repository": "https://github.com/github/plugin-marketplace-starter"

@@ -1,6 +1,6 @@
 ---
 name: release-plugins
-description: "Prepare a marketplace release, fix catalog inconsistencies, or retire a stale plugin. Use whenever asked to get a catalog ready to release, check changed plugin versions and changelogs, draft release notes, or retire or remove a plugin from the marketplace. Don't use for a single contribution's PR draft, remote pin updates alone, or publishing without approval."
+description: "Prepare a marketplace release, fix catalog inconsistencies, or retire a stale plugin. Use whenever asked to get a catalog ready to release, check changed plugin versions and changelogs, draft marketplace release notes including confirmed first releases with no file changes, or retire or remove a plugin from the marketplace. Don't use for a single contribution's PR draft, remote pin updates alone, or publishing without approval."
 ---
 
 # Prepare a marketplace release