fix(design-diff): await native Bun process exits in benchmark runner

Author: Bill Leoutsakos

scripts/design-diff/README.md

@@ -102,6 +102,7 @@ scripts/design-diff/
   infrastructure.ts                     Rendering lockfile dependency closure
   report.ts                             Value previews and bounded JSON serialization
   benchmark.ts                          Immutable-engine historical replay
+  process.ts                            Native Bun process status and bounded diagnostics
   benchmark/comparisons.json            Frozen original/holdout comparison manifest
   memory.ts                             Bun parser-batch garbage collection
   tailwind.ts                           Pinned compiler and trusted merge convention
@@ -339,6 +340,8 @@ bun --no-env-file scripts/design-diff/benchmark.ts \
 Fetch manifest commit objects beforehand; missing history fails explicitly. The runner verifies
 the frozen comparison commits and GitHub file sets. Cache identity includes engine SHA, trusted
 config, lockfile, runtime and comparison commits, with report-content verification before reuse.
-It records per-comparison elapsed time and peak RSS separately from deterministic reports.
+It awaits native Bun process exit status and records per-comparison elapsed time and peak RSS
+separately from deterministic reports. Failed runs retain bounded stderr diagnostics in a
+separate file, without printing source findings to logs.
 `/usr/bin/time` is required (macOS or Linux); source findings are not printed. Review original
 and holdout rates separately, and inspect every disagreement against the source label.

scripts/design-diff/benchmark.ts

@@ -1,8 +1,9 @@
-import { execFileSync, spawn } from 'node:child_process'
+import { execFileSync } from 'node:child_process'
 import { createHash } from 'node:crypto'
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'
 import path from 'node:path'
 import { parseArgs } from 'node:util'
+import { runProcess } from '#design-diff/process'
 import type { Report } from '#design-diff/types'
 
 interface Comparison {
@@ -150,10 +151,11 @@ export async function benchmark(args = process.argv.slice(2)): Promise<void> {
       const started = performance.now()
       const env = { ...process.env }
       for (const key of ['DESIGN_DIFF_PR', 'DESIGN_DIFF_ENGINE_SHA', 'HEAD_SHA']) delete env[key]
-      const timeArgs = process.platform === 'darwin' ? ['-l'] : ['-v']
-      const proc = spawn(
-        '/usr/bin/time',
+      const metricsFile = `${stem}.time.txt`
+      const timeArgs = process.platform === 'darwin' ? ['-l'] : ['-v', '-o', metricsFile]
+      const execution = await runProcess(
         [
+          '/usr/bin/time',
           ...timeArgs,
           process.execPath,
           '--no-env-file',
@@ -165,22 +167,20 @@ export async function benchmark(args = process.argv.slice(2)): Promise<void> {
           '--output',
           reportFile,
         ],
-        { cwd: engine, env, stdio: ['ignore', 'ignore', 'pipe'], detached: true }
+        engine,
+        env
       )
-      let metrics = ''
-      proc.stderr.on('data', (chunk: Buffer) => {
-        if (metrics.length < 65536) metrics += chunk.toString()
-      })
-      const timeout = setTimeout(
-        () => {
-          if (proc.pid) process.kill(-proc.pid, 'SIGKILL')
-        },
-        15 * 60 * 1000
-      )
-      result.exitCode = await new Promise<number | null>((resolve, reject) => {
-        proc.on('error', reject)
-        proc.on('close', resolve)
-      }).finally(() => clearTimeout(timeout))
+      result.exitCode = execution.exitCode
+      const metrics =
+        process.platform === 'linux' && existsSync(metricsFile)
+          ? readFileSync(metricsFile, 'utf8')
+          : execution.stderr
+      if (execution.exitCode !== 0 || execution.timedOut)
+        writeFileSync(
+          `${stem}.stderr.txt`,
+          execution.stderr + (execution.truncated ? '\n[stderr truncated at 65536 bytes]\n' : ''),
+          { mode: 0o600 }
+        )
       result.seconds = Math.round((performance.now() - started) / 10) / 100
       const rss =
         process.platform === 'darwin'
@@ -189,6 +189,7 @@ export async function benchmark(args = process.argv.slice(2)): Promise<void> {
       result.peakMemoryBytes = rss
         ? Number(rss[1]) * (process.platform === 'darwin' ? 1 : 1024)
         : null
+      if (execution.timedOut) throw new Error('Analysis deadline exceeded')
       if (!existsSync(reportFile)) throw new Error('Engine did not produce a report')
       const bytes = readFileSync(reportFile)
       const report = JSON.parse(bytes.toString()) as Report

scripts/design-diff/process.ts

@@ -0,0 +1,65 @@
+/** Native Bun subprocess contract used by the benchmark, independent of Node stream events. */
+interface Runtime {
+  spawn(
+    args: string[],
+    options: {
+      cwd: string
+      env: NodeJS.ProcessEnv
+      stdin: 'ignore'
+      stdout: 'ignore'
+      stderr: 'pipe'
+      detached: true
+    }
+  ): { pid: number; exited: Promise<number>; stderr: ReadableStream<Uint8Array> }
+}
+
+/** Await the process exit status itself and drain bounded diagnostics separately. */
+export async function runProcess(
+  args: string[],
+  cwd: string,
+  env: NodeJS.ProcessEnv,
+  milliseconds = 900000
+) {
+  const runtime = (globalThis as typeof globalThis & { Bun?: Runtime }).Bun
+  if (!runtime) throw new Error('Benchmark subprocesses require Bun')
+  const proc = runtime.spawn(args, {
+    cwd,
+    env,
+    stdin: 'ignore',
+    stdout: 'ignore',
+    stderr: 'pipe',
+    detached: true,
+  })
+  const reader = proc.stderr.getReader()
+  const chunks: Uint8Array[] = []
+  let bytes = 0
+  let truncated = false
+  const drain = (async () => {
+    for (;;) {
+      const { done, value } = await reader.read()
+      if (done) break
+      const remaining = Math.max(0, 65536 - bytes)
+      if (value.length > remaining) truncated = true
+      if (remaining) {
+        chunks.push(value.subarray(0, remaining))
+        bytes += Math.min(remaining, value.length)
+      }
+    }
+    return Buffer.concat(chunks).toString('utf8')
+  })()
+  let timedOut = false
+  const timeout = setTimeout(() => {
+    timedOut = true
+    try {
+      process.kill(-proc.pid, 'SIGKILL')
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== 'ESRCH') throw error
+    }
+  }, milliseconds)
+  try {
+    const [exitCode, stderr] = await Promise.all([proc.exited, drain])
+    return { exitCode, stderr, truncated, timedOut }
+  } finally {
+    clearTimeout(timeout)
+  }
+}

scripts/design-diff/tests/process.test.ts

@@ -0,0 +1,22 @@
+import { execFileSync } from 'node:child_process'
+import { fileURLToPath } from 'node:url'
+import { expect, it } from 'vitest'
+
+const runner = fileURLToPath(new URL('../process.ts', import.meta.url))
+
+it.each([
+  ['exit status', 'process.stderr.write("metric");process.exit(7)', 5000, 7, false, false, 6],
+  ['bounded diagnostics', 'process.stderr.write("x".repeat(100000))', 5000, 0, false, true, 65536],
+  ['deadline', 'await new Promise(resolve=>setTimeout(resolve,30000))', 50, null, true, false, 0],
+])(
+  'measures native Bun subprocess %s',
+  (_name, source, deadline, code, timedOut, truncated, bytes) => {
+    const script = `import {runProcess} from ${JSON.stringify(runner)};const result=await runProcess([process.execPath,'--no-env-file','-e',${JSON.stringify(source)}],process.cwd(),process.env,${deadline});process.stdout.write(JSON.stringify({...result,stderr:result.stderr.length}));`
+    const result = JSON.parse(
+      execFileSync('bun', ['--no-env-file', '-e', script], { encoding: 'utf8', timeout: 10000 })
+    )
+    expect(result).toMatchObject({ timedOut, truncated, stderr: bytes })
+    if (code === null) expect(result.exitCode).not.toBe(0)
+    else expect(result.exitCode).toBe(code)
+  }
+)