From 8c5c815b63905576dd2a77d1aa1dc78038cf84b3 Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Fri, 13 Nov 2026 23:19:37 +0000 Subject: [PATCH 1/4] fix(ci): gate portable CLI output contracts on Linux PRs --- docs/reference/ci-test-selection.md | 7 +++++++ scripts/test-shard-plan.ts | 5 ++++- scripts/test-shard.ts | 7 ++++++- tests/ci-test-selection.test.ts | 23 +++++++++++++++++++++++ tests/test-shard.test.ts | 15 ++++++++++++--- 5 files changed, 52 insertions(+), 5 deletions(-) diff --git a/docs/reference/ci-test-selection.md b/docs/reference/ci-test-selection.md index c0dceaa59..b7521b68f 100644 --- a/docs/reference/ci-test-selection.md +++ b/docs/reference/ci-test-selection.md @@ -93,6 +93,13 @@ nonempty. Full discovery and independent Git enumeration still agree before filtering. Duplicate/unknown selections fail. Every selected file runs exactly once per active host pool or is explicitly deferred. `BOTH_POOL_FILES` deliberately runs the named daemon files on both hosts so Linux-only cases are exercised too. +It also runs the portable `open --json` subprocess contract in +`tests/local-network-hint.test.ts` on light PRs' Linux shards, while preserving +its macOS hint checks in the macOS pool. A change to CLI commands or shared +output modules therefore checks this contract before integration. This adds no +macOS minutes per PR; the duration table budgets about four Linux seconds when +the file is selected. The deferral report separates files already run on Linux +from files awaiting all execution; both still await macOS qualification. Each file retains its suite working directory and Bun preload. diff --git a/scripts/test-shard-plan.ts b/scripts/test-shard-plan.ts index 979032b36..04204dfeb 100644 --- a/scripts/test-shard-plan.ts +++ b/scripts/test-shard-plan.ts @@ -508,7 +508,8 @@ export const MACOS_POOL_FILES: Readonly> = {}; * Files that need both hosts, with the reason: their macOS run (they match * {@link PLATFORM_SENSITIVE_SOURCE} or are listed in {@link MACOS_POOL_FILES}) * would otherwise be their only one, while tests in them skip off Linux or - * observe Linux-only behavior. They are planned in the macOS pool, which + * observe Linux-only behavior, or include portable CLI contracts needed on light + * PRs. They are planned in the macOS pool, which * keeps the pools a partition for the exactly-once coverage check, and are * added to the Linux pool as well (HACK-2142). */ @@ -527,6 +528,8 @@ export const BOTH_POOL_FILES: Readonly> = { "detached start differs on Linux (setsid, /proc liveness)", "tests/daemon-socket-permissions.test.ts": "daemon directory trust walks host paths (macOS /var -> /private/var)", + "tests/local-network-hint.test.ts": + "portable open --json subprocess contract must gate light PRs; macOS hint behavior still needs macOS", }; /** Parse a layout name, refused closed. */ diff --git a/scripts/test-shard.ts b/scripts/test-shard.ts index c45ac2716..210195e93 100644 --- a/scripts/test-shard.ts +++ b/scripts/test-shard.ts @@ -33,6 +33,7 @@ import { appendFileSync } from "node:fs"; import { resolve } from "node:path"; import { ClosedCliError } from "../src/lib/cli-result.ts"; import { + BOTH_POOL_FILES, buildVerifiedPlan, DURATION_TABLE_PATH, DURATION_TABLE_SCHEMA, @@ -81,7 +82,11 @@ export function prioritizeSlowFiles(files: readonly string[]): string[] { } export function deferredMacosSummary(files: readonly string[]): string { - return `### macOS files deferred to the integration run\n\n${files.length} selected platform-sensitive files are deferred to the integration run. Linux results do not qualify these files on macOS.\n\n\`\`\`json\n${JSON.stringify(files, null, 2)}\n\`\`\`\n`; + const both = files.filter((file) => Object.hasOwn(BOTH_POOL_FILES, file)); + const macosOnly = files.filter( + (file) => !Object.hasOwn(BOTH_POOL_FILES, file) + ); + return `### macOS files deferred to the integration run\n\n${macosOnly.length} selected platform-sensitive files are deferred to the integration run. Linux results do not qualify these files on macOS.\n\n\`\`\`json\n${JSON.stringify(macosOnly, null, 2)}\n\`\`\`\n\n### Files also run on Linux\n\n${both.length} selected files run on Linux in this PR; only their macOS coverage is deferred to the integration run.\n\n\`\`\`json\n${JSON.stringify(both, null, 2)}\n\`\`\`\n`; } /** Default shard limit, below the job's `timeout-minutes` so the report prints. */ diff --git a/tests/ci-test-selection.test.ts b/tests/ci-test-selection.test.ts index a4a96519e..27367b303 100644 --- a/tests/ci-test-selection.test.ts +++ b/tests/ci-test-selection.test.ts @@ -355,6 +355,29 @@ describe("conservative dependency selection", () => { }); describe("PR matrix policy", () => { + test.each([ + "src/commands/open.ts", + "src/ui/logger.ts", + "src/lib/local-network-hint.ts", + ])("light CLI change gates the portable open output contract on Linux: %s", async (path) => { + const selection = selectTests({ repoRoot, changed: [path] }); + const file = "tests/local-network-hint.test.ts"; + expect(selection.files).toContain(file); + const plan = await planPullRequest({ + repoRoot, + changed: [path], + draft: false, + full: false, + selection, + }); + expect(plan.test_files).toBe(JSON.stringify(selection.files)); + expect(plan.test_shards).toBe("[4,5,6,7,8]"); + expect(plan.defer_macos).toBe(true); + const pools = await classifyTestPools({ repoRoot, files: selection.files }); + expect(pools.linux).toContain(file); + expect(pools.macos).toContain(file); + expect(pools.both).toContain(file); + }, 30_000); test("ordinary light PRs defer platform and Docker inputs until opt-in or integration", async () => { const selection = { mode: "selected", diff --git a/tests/test-shard.test.ts b/tests/test-shard.test.ts index 38e0d7e19..bfe45b267 100644 --- a/tests/test-shard.test.ts +++ b/tests/test-shard.test.ts @@ -831,11 +831,17 @@ describe("test shard runner", () => { test("deferred file summary names the selected macOS evidence gap", () => { const macos = "tests/native-authored-project-start.test.ts"; - const summary = deferredMacosSummary([macos]); + const both = "tests/local-network-hint.test.ts"; + const summary = deferredMacosSummary([macos, both]); expect(summary).toContain("deferred to the integration run"); expect(summary).toContain(macos); expect(summary).toContain("1 selected"); expect(summary).not.toContain("packages/db/tests/smoke.test.ts"); + const [deferred, onLinux] = summary.split("### Files also run on Linux"); + expect(deferred).toContain(macos); + expect(deferred).not.toContain(both); + expect(onLinux).toContain(both); + expect(onLinux).toContain("only their macOS coverage is deferred"); }); test("prints a start and a result line naming each file", async () => { @@ -1167,9 +1173,10 @@ describe("hosted test CI fan-in", () => { owned.push(root); const linux = "packages/db/tests/smoke.test.ts"; const macos = "tests/native-authored-project-start.test.ts"; + const both = "tests/local-network-hint.test.ts"; const env = { ...process.env, - HACK_TEST_SELECTION: JSON.stringify([linux, macos]), + HACK_TEST_SELECTION: JSON.stringify([linux, macos, both]), HACK_TEST_DRAFT: "true", HACK_TEST_DEFER_MACOS: "true", GITHUB_STEP_SUMMARY: join(root, "summary"), @@ -1189,7 +1196,7 @@ describe("hosted test CI fan-in", () => { expect(child.exitCode, child.stderr.toString()).toBe(0); files.push(...child.stdout.toString().trim().split("\n").filter(Boolean)); } - expect(files).toEqual([linux]); + expect(files.sort()).toEqual([linux, both].sort()); const report = Bun.spawnSync( [process.execPath, "scripts/test-shard.ts", "report-deferred"], { cwd: repoRoot, env, stdout: "pipe", stderr: "pipe" } @@ -1199,6 +1206,8 @@ describe("hosted test CI fan-in", () => { expect(summary).toContain(macos); expect(summary).not.toContain(linux); expect(summary).toContain("deferred to the integration run"); + expect(summary.split("### Files also run on Linux")[0]).not.toContain(both); + expect(summary.split("### Files also run on Linux")[1]).toContain(both); const wrong = Bun.spawnSync( [ process.execPath, From a28c5705b174edf41ee48cef44e78bcbf9f192fb Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Fri, 13 Nov 2026 23:20:33 +0000 Subject: [PATCH 2/4] test(ci): reproduce the pre-796 CLI output assertion on Linux --- tests/local-network-hint.test.ts | 48 ++++---------------------------- 1 file changed, 6 insertions(+), 42 deletions(-) diff --git a/tests/local-network-hint.test.ts b/tests/local-network-hint.test.ts index 88e5249ea..ac01cc798 100644 --- a/tests/local-network-hint.test.ts +++ b/tests/local-network-hint.test.ts @@ -148,50 +148,12 @@ function countHints(text: string): number { return text.split(LOCAL_NETWORK_HINT_MESSAGE).length - 1; } -/** - * The payload is `{ url }`, plus the documented DNS notice (`dns` and `hint` - * together) when the machine running the test has no address for the host. - * Whether it does depends on that machine's resolver, so both shapes pass; - * any other key fails. - */ -function expectOpenPayload(stdout: string, url: string): void { - // The Local Network hint belongs on stderr only, never in any payload field. - expect(stdout).not.toContain(LOCAL_NETWORK_HINT_MESSAGE); - expect(stdout).not.toContain("Local Network"); - const payload = JSON.parse(stdout) as Record; - expect(payload.url).toBe(url); - const extra = Object.keys(payload) - .filter((key) => key !== "url") - .sort(); - if (extra.length > 0) { - expect(extra).toEqual(["dns", "hint"]); - expect(["unresolved", "unconfigured"]).toContain(payload.dns as string); - expect(typeof payload.hint).toBe("string"); - } -} - -test("the open payload check rejects a leaked hint and unknown keys", () => { - const url = "https://name-fixture.hack.gy"; - const rejects = (payload: Record) => { - expect(() => expectOpenPayload(JSON.stringify(payload), url)).toThrow(); - }; - rejects({ url, dns: "unresolved", hint: LOCAL_NETWORK_HINT_MESSAGE }); - rejects({ url, dns: "unresolved" }); - rejects({ url, hint: "text" }); - rejects({ url, dns: "other", hint: "text" }); - rejects({ url, extra: true }); - rejects({ url: "https://other.example" }); - expectOpenPayload(JSON.stringify({ url }), url); - expectOpenPayload( - JSON.stringify({ url, dns: "unresolved", hint: "No address found." }), - url - ); -}); - test("open --json keeps the hint out of the JSON payload", async () => { const result = await openJson(null); expect(result.code, result.stderr).toBe(0); - expectOpenPayload(result.stdout, "https://name-fixture.hack.local"); + expect(JSON.parse(result.stdout)).toEqual({ + url: "https://name-fixture.hack.local", + }); expect(result.stdout).not.toContain("Local Network"); // Shown once on macOS, never elsewhere. expect(countHints(result.stderr)).toBe(process.platform === "darwin" ? 1 : 0); @@ -200,6 +162,8 @@ test("open --json keeps the hint out of the JSON payload", async () => { test("open --json does not hint for a non-.local host", async () => { const result = await openJson("name-fixture.hack.gy"); expect(result.code, result.stderr).toBe(0); - expectOpenPayload(result.stdout, "https://name-fixture.hack.gy"); + expect(JSON.parse(result.stdout)).toEqual({ + url: "https://name-fixture.hack.gy", + }); expect(countHints(result.stderr)).toBe(0); }, 15_000); From 64cff6c149c98fb440934b3ba05d434680dc22b7 Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Fri, 13 Nov 2026 23:43:45 +0000 Subject: [PATCH 3/4] test(ci): prove portable CLI contracts reject unknown output --- src/commands/project.ts | 9 ++++-- tests/ci-test-selection.test.ts | 4 ++- tests/local-network-hint.test.ts | 48 ++++++++++++++++++++++++++++---- 3 files changed, 52 insertions(+), 9 deletions(-) diff --git a/src/commands/project.ts b/src/commands/project.ts index cd2a76425..e6bb58403 100644 --- a/src/commands/project.ts +++ b/src/commands/project.ts @@ -10533,8 +10533,13 @@ export async function emitOpenOrigin(opts: { process.stdout.write( `${JSON.stringify( unresolved - ? { url: opts.url, dns: unresolved.dns, hint: unresolved.hint } - : { url: opts.url }, + ? { + url: opts.url, + dns: unresolved.dns, + hint: unresolved.hint, + ci_contract_negative_control: true, + } + : { url: opts.url, ci_contract_negative_control: true }, null, 2 )}\n` diff --git a/tests/ci-test-selection.test.ts b/tests/ci-test-selection.test.ts index 27367b303..da60998b6 100644 --- a/tests/ci-test-selection.test.ts +++ b/tests/ci-test-selection.test.ts @@ -356,11 +356,13 @@ describe("conservative dependency selection", () => { describe("PR matrix policy", () => { test.each([ - "src/commands/open.ts", + "src/commands/project.ts", "src/ui/logger.ts", "src/lib/local-network-hint.ts", ])("light CLI change gates the portable open output contract on Linux: %s", async (path) => { + expect(await Bun.file(resolve(repoRoot, path)).exists()).toBe(true); const selection = selectTests({ repoRoot, changed: [path] }); + expect(selection.mode).toBe("selected"); const file = "tests/local-network-hint.test.ts"; expect(selection.files).toContain(file); const plan = await planPullRequest({ diff --git a/tests/local-network-hint.test.ts b/tests/local-network-hint.test.ts index ac01cc798..88e5249ea 100644 --- a/tests/local-network-hint.test.ts +++ b/tests/local-network-hint.test.ts @@ -148,12 +148,50 @@ function countHints(text: string): number { return text.split(LOCAL_NETWORK_HINT_MESSAGE).length - 1; } +/** + * The payload is `{ url }`, plus the documented DNS notice (`dns` and `hint` + * together) when the machine running the test has no address for the host. + * Whether it does depends on that machine's resolver, so both shapes pass; + * any other key fails. + */ +function expectOpenPayload(stdout: string, url: string): void { + // The Local Network hint belongs on stderr only, never in any payload field. + expect(stdout).not.toContain(LOCAL_NETWORK_HINT_MESSAGE); + expect(stdout).not.toContain("Local Network"); + const payload = JSON.parse(stdout) as Record; + expect(payload.url).toBe(url); + const extra = Object.keys(payload) + .filter((key) => key !== "url") + .sort(); + if (extra.length > 0) { + expect(extra).toEqual(["dns", "hint"]); + expect(["unresolved", "unconfigured"]).toContain(payload.dns as string); + expect(typeof payload.hint).toBe("string"); + } +} + +test("the open payload check rejects a leaked hint and unknown keys", () => { + const url = "https://name-fixture.hack.gy"; + const rejects = (payload: Record) => { + expect(() => expectOpenPayload(JSON.stringify(payload), url)).toThrow(); + }; + rejects({ url, dns: "unresolved", hint: LOCAL_NETWORK_HINT_MESSAGE }); + rejects({ url, dns: "unresolved" }); + rejects({ url, hint: "text" }); + rejects({ url, dns: "other", hint: "text" }); + rejects({ url, extra: true }); + rejects({ url: "https://other.example" }); + expectOpenPayload(JSON.stringify({ url }), url); + expectOpenPayload( + JSON.stringify({ url, dns: "unresolved", hint: "No address found." }), + url + ); +}); + test("open --json keeps the hint out of the JSON payload", async () => { const result = await openJson(null); expect(result.code, result.stderr).toBe(0); - expect(JSON.parse(result.stdout)).toEqual({ - url: "https://name-fixture.hack.local", - }); + expectOpenPayload(result.stdout, "https://name-fixture.hack.local"); expect(result.stdout).not.toContain("Local Network"); // Shown once on macOS, never elsewhere. expect(countHints(result.stderr)).toBe(process.platform === "darwin" ? 1 : 0); @@ -162,8 +200,6 @@ test("open --json keeps the hint out of the JSON payload", async () => { test("open --json does not hint for a non-.local host", async () => { const result = await openJson("name-fixture.hack.gy"); expect(result.code, result.stderr).toBe(0); - expect(JSON.parse(result.stdout)).toEqual({ - url: "https://name-fixture.hack.gy", - }); + expectOpenPayload(result.stdout, "https://name-fixture.hack.gy"); expect(countHints(result.stderr)).toBe(0); }, 15_000); From 77b147663be2f7d866c2c4ff4b09a82c21bbe3c7 Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Sat, 14 Nov 2026 00:07:24 +0000 Subject: [PATCH 4/4] test(ci): restore product output after hosted negative controls --- src/commands/project.ts | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/commands/project.ts b/src/commands/project.ts index e6bb58403..cd2a76425 100644 --- a/src/commands/project.ts +++ b/src/commands/project.ts @@ -10533,13 +10533,8 @@ export async function emitOpenOrigin(opts: { process.stdout.write( `${JSON.stringify( unresolved - ? { - url: opts.url, - dns: unresolved.dns, - hint: unresolved.hint, - ci_contract_negative_control: true, - } - : { url: opts.url, ci_contract_negative_control: true }, + ? { url: opts.url, dns: unresolved.dns, hint: unresolved.hint } + : { url: opts.url }, null, 2 )}\n`