diff --git a/api/runtime/src/worker/server.js b/api/runtime/src/worker/server.js index 036d869038..4aab4486a6 100644 --- a/api/runtime/src/worker/server.js +++ b/api/runtime/src/worker/server.js @@ -26,7 +26,7 @@ // Each call is made once; one that throws answers { thrown }. import { execFile } from "node:child_process"; import { createHash, randomBytes, timingSafeEqual } from "node:crypto"; -import { createReadStream, mkdirSync, readFileSync, readdirSync, rmSync } from "node:fs"; +import { createReadStream, mkdirSync, readFileSync, rmSync } from "node:fs"; import { createRequire } from "node:module"; import { createServer } from "node:net"; import { tmpdir } from "node:os"; @@ -101,7 +101,8 @@ mkdirSync(config.home, { recursive: true, mode: 0o700 }); // The addon opens each Store with a folder under TMPDIR named by pid. A // restarted container keeps its /tmp and may reuse the pid, so a private // folder per start keeps an earlier run's folders from refusing the open. -for (const name of readdirSync(tmpdir())) if (name.startsWith("layerfs-worker-")) rmSync(join(tmpdir(), name), { recursive: true, force: true }); +// Other homes may have live workers in the same temporary parent. Never sweep +// their directories; shutdown removes only this instance's directory below. const temp = join(tmpdir(), `layerfs-worker-${randomBytes(12).toString("hex")}`); mkdirSync(temp, { mode: 0o700 }); process.env.TMPDIR = temp; diff --git a/api/runtime/test/verify/worker_temp_real.mjs b/api/runtime/test/verify/worker_temp_real.mjs new file mode 100644 index 0000000000..e1f1cb67dc --- /dev/null +++ b/api/runtime/test/verify/worker_temp_real.mjs @@ -0,0 +1,144 @@ +// Independent homes sharing TMPDIR: starting/stopping a worker must not remove +// another live worker's overlay/spool directories or retained failure evidence. +// Real addon, Disposable Store, Docker and daemon, under tools/worktree.py run: +// --timeout 90 -- node api/runtime/test/verify/worker_temp_real.mjs container +// --timeout 90 -- node api/runtime/test/verify/worker_temp_real.mjs stdin +// stdin exercises the Windows worker's startup frame and graceful EOF shutdown; +// container exercises the macOS worker's environment startup (on Linux here). +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { randomBytes } from "node:crypto"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { frame, readFrames } from "@layerfs/agent/internal"; + +const mode = process.argv[2]; +assert.ok(["container", "stdin"].includes(mode), "choose container or stdin"); +const bundle = dirname(process.env.LAYERFS_NATIVE); +const server = fileURLToPath(new URL("../../src/worker/server.js", import.meta.url)); +const work = mkdtempSync(join(tmpdir(), "layerfs-worker-temp-test-")); +// All workers, including a broken version's broad cleanup, are confined here. +const shared = join(work, "tmp"); +mkdirSync(shared, { mode: 0o700 }); +const retained = join(shared, "layerfs-worker-retained-evidence"); +mkdirSync(retained); +writeFileSync(join(retained, "evidence"), "retain original failure"); +const workers = []; +let client, sandbox, ws, passed = false; + +async function bounded(promise, label, ms = 10_000) { + let timer; + try { + return await Promise.race([promise, new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(`${label} exceeded ${ms} ms`)), ms); + })]); + } finally { + clearTimeout(timer); + } +} + +async function start(name) { + const home = join(work, name), secret = randomBytes(32).toString("hex"); + mkdirSync(home, { mode: 0o700 }); + const before = new Set(readdirSync(shared)); + const child = spawn(process.execPath, [server, ...(mode === "stdin" ? ["--stdin"] : [])], { + env: { ...process.env, TMPDIR: shared, LAYERFS_HOME: home, LAYERFS_BUNDLE: bundle, + LAYERFS_WORKER_SECRET: secret, LAYERFS_WORKER_LISTEN: "127.0.0.1:0" }, + stdio: ["pipe", "pipe", "pipe"], + }); + const worker = { child, ended: new Promise((resolve) => child.once("exit", (code, signal) => resolve({ code, signal }))) }; + workers.push(worker); + let stderr = ""; + const ready = new Promise((resolve, reject) => { + child.once("error", reject); + child.once("exit", (code, signal) => reject(new Error(`${name} exited before readiness: ${code ?? signal}\n${stderr}`))); + child.stderr.on("data", (chunk) => { + stderr += chunk; + if (mode === "container") { + const match = /listening on 127\.0\.0\.1:(\d+)/.exec(stderr); + if (match) resolve(Number(match[1])); + } + }); + if (mode === "stdin") readFrames(child.stdout, (r) => resolve(r.port), reject); + else child.stdout.resume(); + }); + if (mode === "stdin") child.stdin.write(frame({ secret, home, bundle })); + const port = await bounded(ready, `${name} readiness`); + const made = readdirSync(shared).filter((entry) => !before.has(entry)); + assert.equal(made.length, 1, "each start creates exactly one private directory"); + worker.temp = join(shared, made[0]); + assert.equal(statSync(worker.temp).mode & 0o777, 0o700); + worker.marker = join(worker.temp, "live-owner"); + writeFileSync(worker.marker, name); + worker.endpoint = join(work, `${name}.json`); + writeFileSync(worker.endpoint, JSON.stringify({ home, port, secret, build: "test" })); + worker.home = home; + return worker; +} + +async function stop(worker) { + if (mode === "stdin") worker.child.stdin.end(); + else worker.child.kill("SIGTERM"); + const exit = await bounded(worker.ended, "worker shutdown", 5_000); + assert.deepEqual(exit, mode === "stdin" ? { code: 0, signal: null } : { code: null, signal: "SIGTERM" }); + if (mode === "stdin") assert.equal(existsSync(worker.temp), false, "graceful shutdown removes its own directory"); +} + +try { + const a = await start("home-a"); + process.env.LAYERFS_HOME = a.home; + process.env.LAYERFS_WORKER_ENDPOINT = a.endpoint; + const { LayerFS } = await import("@layerfs/agent"); + const source = join(work, "source"); + mkdirSync(source); + writeFileSync(join(source, "README.md"), "worker isolation\n"); + client = await LayerFS.init("worker-temp", source); + await client.branch.fork("work"); + await client.fs.write("work", "/before", "before B"); + sandbox = (await client.sandbox.create()).sandbox; + ws = await client.workspace.mount(sandbox, "work"); + assert.equal((await client.workspace.exec(ws, "printf before")).stdout, "before"); + + const b = await start("home-b"); + assert.notEqual(a.temp, b.temp); + assert.equal(a.child.exitCode, null, "A remains alive"); + await client.fs.write("work", "/after", "after B"); + assert.equal((await client.fs.read("work", "/after")).text, "after B"); + // Beyond the 1 MiB inline limit: this needs A's original spool directory. + const size = 2 * 1024 * 1024 + 1; + const ran = await client.workspace.exec(ws, `head -c ${size} /dev/zero | tr '\\0' x`); + assert.deepEqual([ran.exitCode, ran.stdout, ran.stderr], [0, "x".repeat(size), ""]); + assert.equal(readFileSync(a.marker, "utf8"), "home-a"); + assert.equal(readFileSync(b.marker, "utf8"), "home-b"); + assert.equal(readFileSync(join(retained, "evidence"), "utf8"), "retain original failure"); + + await stop(b); + assert.equal(readFileSync(a.marker, "utf8"), "home-a", "B shutdown preserves A"); + await client.fs.write("work", "/after-stop", "B stopped"); + assert.equal((await client.fs.read("work", "/after-stop")).text, "B stopped"); + await client.workspace.unmount(ws, true); + ws = null; + await client.sandbox.close(sandbox); + sandbox = null; + await client.close(); + client = null; + await stop(a); + assert.equal(readFileSync(join(retained, "evidence"), "utf8"), "retain original failure"); + console.log(`worker_temp_real ${mode}: ok (independent writes, spooled output and cleanup)`); + passed = true; +} finally { + if (ws) await client.workspace.unmount(ws, true).catch((error) => { console.error(error); process.exitCode = 1; }); + if (sandbox) await client.sandbox.close(sandbox).catch((error) => { console.error(error); process.exitCode = 1; }); + if (client) await client.close().catch((error) => { console.error(error); process.exitCode = 1; }); + for (const worker of workers) { + if (worker.child.exitCode === null && worker.child.signalCode === null) { + worker.child.kill("SIGKILL"); + await bounded(worker.ended, "failed-test worker cleanup", 5_000); + } + } + // Only a passing run releases its fixture. The launcher's failed-run custody + // retains original files and stopped containers for inspection otherwise. + if (passed) rmSync(work, { recursive: true, force: true }); +}