From dd12eb758939b7d19805b06aea012d0d1e8b031c Mon Sep 17 00:00:00 2001 From: Robert Gruen Date: Sat, 31 Oct 2026 14:12:12 -0700 Subject: [PATCH] Block failed agent outputs before sealing discussion replies Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/discussion-draft.lock.yml | 7 ++++++- .github/workflows/discussion-draft.md | 5 +++++ AUTOMATION.md | 6 +++++- tools/tests/discussion-draft.test.mjs | 13 ++++++++++++- tools/tests/discussion-workflows.test.mjs | 12 ++++++++++++ 5 files changed, 40 insertions(+), 3 deletions(-) diff --git a/.github/workflows/discussion-draft.lock.yml b/.github/workflows/discussion-draft.lock.yml index 895f412e..c2c356ac 100644 --- a/.github/workflows/discussion-draft.lock.yml +++ b/.github/workflows/discussion-draft.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"bc07e8cc2cf7c2de901a9156476f3fbdfa1235a5b33c6843f9614624cc54b1cb","body_hash":"ba62d944e8841d183be9c6cdfa13970c18e99177930dcd6d178e60420e068f54","compiler_version":"v0.93.5","strict":true,"agent_id":"copilot","agent_model":"gpt-6.1-sol","engine_versions":{"copilot":"1.0.101"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"7c8380b1ffb86eb153eb973bf694b44f1f4e8549320f86794635467636bb5b5c","body_hash":"ba62d944e8841d183be9c6cdfa13970c18e99177930dcd6d178e60420e068f54","compiler_version":"v0.93.5","strict":true,"agent_id":"copilot","agent_model":"gpt-6.1-sol","engine_versions":{"copilot":"1.0.101"}} # gh-aw-manifest: {"version":1,"secrets":["GH_AW_DEFAULT_OTLP_ENDPOINT","GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/checkout","sha":"d23441a48e516b6c34aea4fa41551a30e30af803","version":"d23441a48e516b6c34aea4fa41551a30e30af803"},{"repo":"actions/download-artifact","sha":"9000827ccba6bdab643e8b6fd33ac0654aef8333","version":"9000827ccba6bdab643e8b6fd33ac0654aef8333"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"3a2844b7e9c422d3c10d287c895573f7108da1b3"},{"repo":"actions/setup-node","sha":"949feb2413d6458794dcd2491c4babbbce0c15c1","version":"v7.1.0"},{"repo":"actions/upload-artifact","sha":"cf430e030ddbb5b0abf93d22962f4752f3646cd9","version":"cf430e030ddbb5b0abf93d22962f4752f3646cd9"},{"repo":"github/gh-aw/actions/setup","sha":"32ecafbc061c9af57e9361942099e381102ae304","version":"32ecafbc061c9af57e9361942099e381102ae304"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.70","digest":"sha256:b15ab48cbbab02904eb9bbd359a398db59690eebbea1e6934d47ed2bb407dd34","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.70@sha256:b15ab48cbbab02904eb9bbd359a398db59690eebbea1e6934d47ed2bb407dd34"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.70","digest":"sha256:9ac8f9c7294e892c916de3da9d2588dfd8cd4931e5c39b00143266d7ed9fd52a","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.70@sha256:9ac8f9c7294e892c916de3da9d2588dfd8cd4931e5c39b00143266d7ed9fd52a"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.70","digest":"sha256:b5bc53f4becfd96e926e63c83831f1b3c9512f713c83161d2b8c9b7204b3e917","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.70@sha256:b5bc53f4becfd96e926e63c83831f1b3c9512f713c83161d2b8c9b7204b3e917"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.37","digest":"sha256:9c7f242f6dec2d7d7b55d4c9e07872d59b85377c6cb3de46c915a5d1ffb91442","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.37@sha256:9c7f242f6dec2d7d7b55d4c9e07872d59b85377c6cb3de46c915a5d1ffb91442"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:23746305eeded20dc2726ab18bb081962d66ee3b2e09a54291afd1869ab3ef6d","pinned_image":"ghcr.io/github/gh-aw-node@sha256:23746305eeded20dc2726ab18bb081962d66ee3b2e09a54291afd1869ab3ef6d"},{"image":"ghcr.io/github/github-mcp-server:v2.0.3","digest":"sha256:250bc6ce70788207608ef74cef4397230ff2e8481218da74d6093e4bd9101132","pinned_image":"ghcr.io/github/github-mcp-server:v2.0.3@sha256:250bc6ce70788207608ef74cef4397230ff2e8481218da74d6093e4bd9101132"}],"mcp_servers":[{"name":"github","tools":["get_discussion","get_discussion_comments","get_file_contents","issue_read","list_issues","list_pull_requests","pull_request_read","search_issues","search_pull_requests"]},{"name":"safeoutputs","tools":["noop","report_incomplete","report_missing_data"]}],"threat_detection":{"mode":"enabled"},"admission":false} # This file was automatically generated by gh-aw (v0.93.5). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -2485,10 +2485,15 @@ jobs: - name: Validate and seal the draft id: seal uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + TYPECHAT_AGENT_OUTPUT_TYPES: ${{ needs.agent.outputs.output_types }} with: github-token: ${{ github.token }} retries: 0 script: | + if (process.env.TYPECHAT_AGENT_OUTPUT_TYPES !== "noop") { + throw new Error("Only a successful native noop completion can be sealed; missing, incomplete, or failure outputs are rejected."); + } const path = require("node:path"); const { pathToFileURL } = require("node:url"); const helper = await import(pathToFileURL(path.join(process.env.GITHUB_WORKSPACE, "tools", "discussion-draft.mjs")).href); diff --git a/.github/workflows/discussion-draft.md b/.github/workflows/discussion-draft.md index 71abae09..69e19821 100644 --- a/.github/workflows/discussion-draft.md +++ b/.github/workflows/discussion-draft.md @@ -198,10 +198,15 @@ jobs: - name: Validate and seal the draft id: seal uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + TYPECHAT_AGENT_OUTPUT_TYPES: ${{ needs.agent.outputs.output_types }} with: github-token: ${{ github.token }} retries: 0 script: | + if (process.env.TYPECHAT_AGENT_OUTPUT_TYPES !== "noop") { + throw new Error("Only a successful native noop completion can be sealed; missing, incomplete, or failure outputs are rejected."); + } const path = require("node:path"); const { pathToFileURL } = require("node:url"); const helper = await import(pathToFileURL(path.join(process.env.GITHUB_WORKSPACE, "tools", "discussion-draft.mjs")).href); diff --git a/AUTOMATION.md b/AUTOMATION.md index 381819fe..203c5983 100644 --- a/AUTOMATION.md +++ b/AUTOMATION.md @@ -561,7 +561,11 @@ it as completed work and skips the model. The model writes the candidate first, then emits its native completion signal. If it emits no terminal safe output, the pinned collector records an explicit `report_incomplete` signal rather than treating silence as successful work. The detection guard consumes the collected -output types; a missing signal cannot authorize sealing. +output types. Before rendering or sealing an artifact, the trusted sealing step +requires `noop` to be the only collected output type. Missing or unknown outputs +and `report_missing_data` or `report_incomplete` reports fail sealing, including +reports combined with `noop`. Both reply writers require a successful seal, so +publication cannot race the later conclusion job's handling of failure reports. The first owner-commanded [draft run](https://github.com/microsoft/TypeChat/actions/runs/41136773812) on 2026-10-30 exposed this preflight behavior. The synthetic pre-agent no-op caused diff --git a/tools/tests/discussion-draft.test.mjs b/tools/tests/discussion-draft.test.mjs index da30c50c..1d0938f1 100644 --- a/tools/tests/discussion-draft.test.mjs +++ b/tools/tests/discussion-draft.test.mjs @@ -1028,7 +1028,18 @@ for (const [kind, mode] of [["issue", "draft"], ["discussion", "draft"], ["issue const scanDirectory = downloadDirectory(sealing.find(step => step.name === "Download candidate scan binding")); fs.mkdirSync(scanDirectory); fs.copyFileSync(scanned.path, path.join(scanDirectory, "candidate-scan.json")); - const sealed = await execute(sealing.find(step => step.id === "seal"), draftRun); + const seal = sealing.find(step => step.id === "seal"); + for (const outputTypes of [undefined, "", "report_missing_data", "noop,report_missing_data", + "report_incomplete", "noop,report_incomplete", "report_missing_tool", "unknown"]) { + delete process.env.TYPECHAT_AGENT_OUTPUT_TYPES; + await assert.rejects(execute(seal, draftRun, outputTypes === undefined ? {} : { + TYPECHAT_AGENT_OUTPUT_TYPES: outputTypes, + }), /Only a successful native noop completion can be sealed/); + assert.equal(fs.existsSync(path.join(temporary, "typechat-sealed-draft")), false, + `${outputTypes} must not produce a sealed artifact despite a valid candidate and successful scan`); + assert.equal(fixture.state.posts.length, 0); + } + const sealed = await execute(seal, draftRun, { TYPECHAT_AGENT_OUTPUT_TYPES: "noop" }); const bundle = readJson(path.dirname(sealed.path), "sealed-draft.json", MAX_BUNDLE_BYTES); assert.equal(fixture.state.posts.length, 0); diff --git a/tools/tests/discussion-workflows.test.mjs b/tools/tests/discussion-workflows.test.mjs index 98de1c3c..1472bf20 100644 --- a/tools/tests/discussion-workflows.test.mjs +++ b/tools/tests/discussion-workflows.test.mjs @@ -222,6 +222,18 @@ test("draft preparation must not pre-complete the model with a native noop", () assert.ok(text.includes("After writing `discussion-draft.json`, call `safeoutputs-noop`")); }); +test("trusted sealing checks native completion before exposing an artifact to either writer", () => { + const seal = sourceSeal.find(step => step.id === "seal"); + const generated = compiled.jobs.seal.steps.find(step => step.id === "seal"); + assert.deepEqual(seal.env, { TYPECHAT_AGENT_OUTPUT_TYPES: "${{ needs.agent.outputs.output_types }}" }); + assert.deepEqual(generated.env, seal.env); + assert.equal(generated.with.script, seal.with.script); + assert.equal(agent.outputs.output_types, "${{ steps.collect_output.outputs.output_types }}"); + for (const kind of ["issue", "discussion"]) { + assert.ok(compiled.jobs[`reply_${kind}`].if.includes("needs.seal.result == 'success'")); + } +}); + test("the compiled detection guard accepts collected outputs but never treats a missing signal as success", () => { const working = fs.mkdtempSync(path.join(os.tmpdir(), "typechat-detection-guard-")); try {