diff --git a/.fallowrc.json b/.fallowrc.json index 4cc768da..9dbdbd76 100644 --- a/.fallowrc.json +++ b/.fallowrc.json @@ -70,6 +70,11 @@ "maxCrap": 100, "reason": "Glue scripts for the delivery pipeline: git and transcript walks, option parsing, a Markdown shape check. They have no coverage pipeline, so CRAP is complexity squared; each is run against real PRs and transcripts when it changes, and none holds product logic." }, + { + "files": ["src/agent-device/host/agent-device-supervisor.test.ts"], + "maxCrap": 80, + "reason": "The supervisor's test arranges a process table, a clock, a filesystem and a scripted runner in one helper so each of its cases reads as one claim; the helper is branching setup, not logic under test." + }, { "files": ["e2e/fake-driver/fake-driver.ts"], "maxCrap": 45, diff --git a/.oxlintrc.json b/.oxlintrc.json index f030bb9a..a556cab7 100644 --- a/.oxlintrc.json +++ b/.oxlintrc.json @@ -29,6 +29,10 @@ "regex": "(?:^|/)android/(?:$|[^i]|i(?:$|[^n]|n(?:$|[^d]|d(?:$|[^e]|e(?:$|[^x]|x(?:$|[^\\.]|\\.(?:$|[^j]|j(?:$|[^s]|s.))))))))", "message": "drivers/android is entered through its index.js" }, + { + "regex": "(?:^|/)agent-device/(?:$|[^i]|i(?:$|[^n]|n(?:$|[^d]|d(?:$|[^e]|e(?:$|[^x]|x(?:$|[^\\.]|\\.(?:$|[^j]|j(?:$|[^s]|s.))))))))", + "message": "agent-device is entered through its index.js" + }, { "regex": "(?:^|/)(?:testing|test-[^/]*)\\.js$", "message": "only a *.test.ts file or a test-*.ts helper imports a testing.js or test-*.js file" @@ -74,6 +78,10 @@ ], "message": "core and leasing reach the filesystem and processes only through injected ports" }, + { + "regex": "(?:^|/)agent-device/", + "message": "the core, leasing and the drivers import nothing from the agent-device edge" + }, { "regex": "(?:^|/)(?:testing|test-[^/]*)\\.js$", "message": "only a *.test.ts file or a test-*.ts helper imports a testing.js or test-*.js file" @@ -127,6 +135,10 @@ ], "message": "core and leasing reach the filesystem and processes only through injected ports" }, + { + "regex": "(?:^|/)agent-device/", + "message": "the core, leasing and the drivers import nothing from the agent-device edge" + }, { "regex": "(?:^|/)(?:testing|test-[^/]*)\\.js$", "message": "only a *.test.ts file or a test-*.ts helper imports a testing.js or test-*.js file" @@ -202,6 +214,10 @@ "regex": "(?:^|/)daemon/(?:$|[^d]|d(?:$|[^i]|i(?:$|[^s]|s(?:$|[^p]|p(?:$|[^a]|a(?:$|[^t]|t(?:$|[^c]|c(?:$|[^h]|h(?:$|[^\\.]|\\.(?:$|[^j]|j(?:$|[^s]|s.)))))))))))", "message": "the gateway imports only the daemon's dispatch contract" }, + { + "regex": "(?:^|/)agent-device/(?:$|[^i]|i(?:$|[^n]|n(?:$|[^d]|d(?:$|[^e]|e(?:$|[^x]|x(?:$|[^\\.]|\\.(?:$|[^j]|j(?:$|[^s]|s.))))))))", + "message": "agent-device is entered through its index.js" + }, { "regex": "(?:^|/)(?:testing|test-[^/]*)\\.js$", "message": "only a *.test.ts file or a test-*.ts helper imports a testing.js or test-*.js file" @@ -239,6 +255,10 @@ "regex": "(?:^|/)gateway/", "message": "only the composition root, main.ts, knows a gateway exists" }, + { + "regex": "(?:^|/)agent-device/(?:$|[^i]|i(?:$|[^n]|n(?:$|[^d]|d(?:$|[^e]|e(?:$|[^x]|x(?:$|[^\\.]|\\.(?:$|[^j]|j(?:$|[^s]|s.))))))))", + "message": "agent-device is entered through its index.js" + }, { "regex": "(?:^|/)(?:testing|test-[^/]*)\\.js$", "message": "only a *.test.ts file or a test-*.ts helper imports a testing.js or test-*.js file" @@ -268,6 +288,103 @@ ] } }, + { + "files": ["src/drivers/**"], + "rules": { + "no-restricted-imports": [ + "error", + { + "patterns": [ + { + "regex": "(?:^|/)core/(?:$|[^i]|i(?:$|[^n]|n(?:$|[^d]|d(?:$|[^e]|e(?:$|[^x]|x(?:$|[^\\.]|\\.(?:$|[^j]|j(?:$|[^s]|s.))))))))", + "message": "core is entered through its index.js" + }, + { + "regex": "(?:^|/)leasing/(?:$|[^i]|i(?:$|[^n]|n(?:$|[^d]|d(?:$|[^e]|e(?:$|[^x]|x(?:$|[^\\.]|\\.(?:$|[^j]|j(?:$|[^s]|s.))))))))", + "message": "leasing is entered through its index.js" + }, + { + "regex": "(?:^|/)gateway/(?:$|[^i]|i(?:$|[^n]|n(?:$|[^d]|d(?:$|[^e]|e(?:$|[^x]|x(?:$|[^\\.]|\\.(?:$|[^j]|j(?:$|[^s]|s.))))))))", + "message": "gateway is entered through its index.js" + }, + { + "regex": "(?:^|/)ios/(?:$|[^i]|i(?:$|[^n]|n(?:$|[^d]|d(?:$|[^e]|e(?:$|[^x]|x(?:$|[^\\.]|\\.(?:$|[^j]|j(?:$|[^s]|s.))))))))", + "message": "drivers/ios is entered through its index.js" + }, + { + "regex": "(?:^|/)android/(?:$|[^i]|i(?:$|[^n]|n(?:$|[^d]|d(?:$|[^e]|e(?:$|[^x]|x(?:$|[^\\.]|\\.(?:$|[^j]|j(?:$|[^s]|s.))))))))", + "message": "drivers/android is entered through its index.js" + }, + { + "regex": "(?:^|/)agent-device/(?:$|[^i]|i(?:$|[^n]|n(?:$|[^d]|d(?:$|[^e]|e(?:$|[^x]|x(?:$|[^\\.]|\\.(?:$|[^j]|j(?:$|[^s]|s.))))))))", + "message": "agent-device is entered through its index.js" + }, + { + "regex": "(?:^|/)(?:testing|test-[^/]*)\\.js$", + "message": "only a *.test.ts file or a test-*.ts helper imports a testing.js or test-*.js file" + }, + { + "regex": "(?:^|/)agent-device/", + "message": "the core, leasing and the drivers import nothing from the agent-device edge" + } + ] + } + ] + } + }, + { + "files": ["src/agent-device/**"], + "rules": { + "no-restricted-imports": [ + "error", + { + "patterns": [ + { + "regex": "(?:^|/)core/(?:$|[^i]|i(?:$|[^n]|n(?:$|[^d]|d(?:$|[^e]|e(?:$|[^x]|x(?:$|[^\\.]|\\.(?:$|[^j]|j(?:$|[^s]|s.))))))))", + "message": "core is entered through its index.js" + }, + { + "regex": "(?:^|/)leasing/(?:$|[^i]|i(?:$|[^n]|n(?:$|[^d]|d(?:$|[^e]|e(?:$|[^x]|x(?:$|[^\\.]|\\.(?:$|[^j]|j(?:$|[^s]|s.))))))))", + "message": "leasing is entered through its index.js" + }, + { + "regex": "(?:^|/)gateway/(?:$|[^i]|i(?:$|[^n]|n(?:$|[^d]|d(?:$|[^e]|e(?:$|[^x]|x(?:$|[^\\.]|\\.(?:$|[^j]|j(?:$|[^s]|s.))))))))", + "message": "gateway is entered through its index.js" + }, + { + "regex": "(?:^|/)ios/(?:$|[^i]|i(?:$|[^n]|n(?:$|[^d]|d(?:$|[^e]|e(?:$|[^x]|x(?:$|[^\\.]|\\.(?:$|[^j]|j(?:$|[^s]|s.))))))))", + "message": "drivers/ios is entered through its index.js" + }, + { + "regex": "(?:^|/)android/(?:$|[^i]|i(?:$|[^n]|n(?:$|[^d]|d(?:$|[^e]|e(?:$|[^x]|x(?:$|[^\\.]|\\.(?:$|[^j]|j(?:$|[^s]|s.))))))))", + "message": "drivers/android is entered through its index.js" + }, + { + "regex": "(?:^|/)agent-device/(?:$|[^i]|i(?:$|[^n]|n(?:$|[^d]|d(?:$|[^e]|e(?:$|[^x]|x(?:$|[^\\.]|\\.(?:$|[^j]|j(?:$|[^s]|s.))))))))", + "message": "agent-device is entered through its index.js" + }, + { + "regex": "(?:^|/)(?:testing|test-[^/]*)\\.js$", + "message": "only a *.test.ts file or a test-*.ts helper imports a testing.js or test-*.js file" + }, + { + "group": [ + "fs", + "fs/**", + "node:fs", + "node:fs/**", + "child_process", + "child_process/**", + "node:child_process", + "node:child_process/**" + ], + "message": "agent-device reaches the filesystem and processes only through injected ports" + } + ] + } + ] + } + }, { "files": ["**/*.test.ts", "**/test-*.ts"], "rules": { @@ -298,6 +415,10 @@ { "regex": "(?:^|/)android/(?:$|[^i]|i(?:$|[^n]|n(?:$|[^d]|d(?:$|[^e]|e(?:$|[^x]|x(?:$|[^\\.]|\\.(?:$|[^j]|j(?:$|[^s]|s.))))))))", "message": "drivers/android is entered through its index.js" + }, + { + "regex": "(?:^|/)agent-device/(?:$|[^it]|i(?:$|[^n]|n(?:$|[^d]|d(?:$|[^e]|e(?:$|[^x]|x(?:$|[^\\.]|\\.(?:$|[^j]|j(?:$|[^s]|s.)))))))|t(?:$|[^e]|e(?:$|[^s]|s(?:$|[^t]|t(?:$|[^i]|i(?:$|[^n]|n(?:$|[^g]|g(?:$|[^\\.]|\\.(?:$|[^j]|j(?:$|[^s]|s.))))))))))", + "message": "agent-device is entered through its index.js, or testing.js from a test file" } ] } diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 8fd0e9f4..b024c5e8 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -35,6 +35,8 @@ edit yourself keeps the mode you give it until the next `config set`; run | `gateway.url` | **Worker side.** Base URL of the gateway this worker joins, e.g. `wss://gw.example:4700`; the worker dials `/v1/uplink` and upgrades it to a WebSocket. Joining grants that gateway the `admin` role on this daemon, and `gateway.token` rides on the upgrade request as a bearer credential, so use `wss://` — or plain `ws://` only over loopback or inside your own tunnel. `http://` and `https://` are rejected at load. Unset means "do not join a fleet": the default, and the only thing that changes about a joined worker. | unset | | `gateway.token` | **Worker side.** The join token (`simlock token create --role worker`, minted on the gateway) this worker presents when it opens its uplink. Required whenever `gateway.url` is set. `simlock config` shows it; the daemon log and the `daemon.started` event show `[redacted]`. It is stored in plain text in `config.json`, which `simlock config set` writes owner-only. | unset | | `gateway.label` | **Worker side.** Display name for this worker in `simlock worker list`, `status`, the console, and on the lease's `worker` block. Display-only: nothing routes on it and it need not be unique. | the worker's own id | +| `agentDevice.enabled` | **Worker side.** Run one agent-device daemon on this worker, started and watched by Simlock. It sees Simlock's devices and no others, refuses `boot` and `shutdown`, and keeps its files in `~/.simlock/agent-device`. If agent-device is missing, older than 0.21.25, speaks another RPC protocol or needs a newer Node than Simlock runs on, it is not started and Simlock runs on without it. A daemon a killed Simlock left behind is stopped at the next start. | `false` | +| `agentDevice.command` | **Worker side.** The agent-device executable: a name looked up on `PATH`, or a path. Used only when `agentDevice.enabled` is `true`. | `agent-device` | | `exec.timeoutMs` | **Worker side.** How long one `device.exec` command (`simlock simctl` / `simlock adb` against a gateway or over HTTP) may run before the worker kills it and the operation fails with `EXEC_TIMEOUT`. Authoritative: it bounds the process that actually runs. | `10 minutes` | | `gateway.routing` | **Gateway side.** Which routing policy places a queued request on a worker. `warm-then-free` is the only policy in v1. Among the workers that can serve the request, in order: a worker whose health is not `running`, or that has requests of its own waiting, is left out; a worker holding a ready device that fits goes first (the device's mode must fit the request, a request with no mode fits a device that serves the worker's default mode, and its runtime must be the one requested or, with none requested, the one the worker would pick: its default runtime if the model pairs with it, otherwise the only runtime the model pairs with); otherwise a worker with no free running slot for the platform or overall is left out (a running device that is not leased counts as free: the worker shuts it down to make room), a worker under its RAM budget for the platform is preferred over one at it (a worker at its budget is still asked when it is the only one left), and the one with the most free running capacity for the platform wins. A worker that is left out is busy, not unable: the request waits for it. Ties go to the lowest worker id. | `warm-then-free` | | `gateway.disconnectedRetentionMs` | **Gateway side.** How long a disconnected worker is kept (greyed, never dispatched to) before the gateway forgets it. The clock is held while the gateway still knows of gateway-issued leases on that worker, and that hold ends when the last of those leases passes its deadline. | `24 hours` | diff --git a/docs/internal/COMPONENTS.md b/docs/internal/COMPONENTS.md index 6a239f38..03980e90 100644 --- a/docs/internal/COMPONENTS.md +++ b/docs/internal/COMPONENTS.md @@ -105,6 +105,16 @@ modules with. | `LeaseHealthMonitor` | `src/leasing/lease-health-monitor.ts` | Observing leased devices on a tick, rebooting a crashed one under its lease, giving the lease up as `device-lost` when recovery is exhausted. | Touch an unleased device. | | `PhysicalDeviceWatcher` | `src/leasing/physical-device-watcher.ts` | The presence tick (every `health.probeIntervalMs`, whatever `health.enabled` says), the grace-period count of not-present reads, `absent` and its reason, the device-lost ending of a lease on a device that is gone, and the return reclaim of a device that is back (ADR 0022 §7). | Retry quarantine, grant, or run a driver verb but the presence listing on a leased device. | +## Agent-device edge + +`src/agent-device/` holds the agent-device daemon a worker runs beside its drivers (ADR 0023). It is entered only through `src/agent-device/index.ts`; the core, leasing and the drivers import nothing from it, and its non-test files import no `fs` or `child_process`. + +| Component | Where | Owns | Does not | +|---|---|---|---| +| `AgentDeviceSupervisor` | `src/agent-device/host/agent-device-supervisor.ts` | Starting, watching and stopping the one agent-device daemon: reaping one a killed simlock left (by a pid record and an identity check), the private state dir and policy file, the daemon's environment, readiness through `daemon.json`, the port and `/health`, restart on death, and the availability state (`off`, `starting`, `running`, `unavailable` with a reason and remedy). | Serve agent-device traffic; report to `simlock doctor`; decide which devices exist (the drivers' environments do). | +| Install lookup, compatibility, daemon environment | `src/agent-device/host/install.ts`, `src/agent-device/shared/` | Finding the agent-device install and its daemon entry; the minimum versions and RPC protocol; the environment and policy data. | Run a process. | +| `AgentDeviceHealthProbe` | `src/agent-device/host/health.ts` | The `GET /health` read, with a time limit. | Judge the answer. | + ## Drivers | Component | Where | Owns | Does not | diff --git a/e2e/fake-driver/fake-driver.ts b/e2e/fake-driver/fake-driver.ts index 6419afb0..4c31b4ed 100644 --- a/e2e/fake-driver/fake-driver.ts +++ b/e2e/fake-driver/fake-driver.ts @@ -601,6 +601,10 @@ export class OutOfProcessFakeDriver implements Driver { return this.#lastKnownEstimateMs?.[estimate.operation] ?? 0; } + virtualLeaseEnvironment(): Readonly> { + return {}; + } + leaseEnvironment( device: VirtualDriverDevice | PhysicalDriverDevice, ): Readonly> { diff --git a/src/agent-device/host/agent-device-supervisor.test.ts b/src/agent-device/host/agent-device-supervisor.test.ts new file mode 100644 index 00000000..95fec996 --- /dev/null +++ b/src/agent-device/host/agent-device-supervisor.test.ts @@ -0,0 +1,1485 @@ +import { describe, expect, it, vi } from "vitest"; + +import { + FakeClock, + FakeProcessSupervisor, + FakeTcpProbe, + JsonLinesLogger, + MemoryFilesystem, + MemoryLogSink, + ScriptedProcessRunner, + type ProcessResult, + type ScriptedProcessExpectation, +} from "../../ports/index.js"; +import { FakeAgentDeviceHealth } from "../testing.js"; +import { + AgentDeviceSupervisor, + type AgentDeviceAvailability, + type AgentDevicePlatform, +} from "./agent-device-supervisor.js"; + +const home = "/home/simlock/.simlock"; +const stateDir = `${home}/agent-device`; +const recordPath = `${home}/agent-device-daemon.json`; +const policyPath = `${stateDir}/policy.json`; +const daemonJsonPath = `${stateDir}/daemon.json`; +const nodePath = "/opt/node/bin/node"; +const packageRoot = "/opt/ad/node_modules/agent-device"; +const realCli = `${packageRoot}/bin/agent-device.mjs`; +const internalEntry = `${packageRoot}/dist/src/internal/daemon.js`; +const fallbackEntry = `${packageRoot}/dist/src/daemon.js`; +const httpPort = 4310; +const token = "daemon-token"; +const healthyBody = { rpcProtocolVersion: 2, service: "agent-device-daemon" }; +const restartCommand = ["simlock daemon stop", "simlock daemon start"]; + +type Behaviour = "dies" | "ignores-sigterm" | "immortal"; +type DaemonJson = "ready" | "none" | "other-pid"; + +class ProcessTable extends FakeProcessSupervisor { + readonly behaviours = new Map(); + readonly onDeath = new Map void>(); + + override signal(pid: number, signal: NodeJS.Signals): void { + super.signal(pid, signal); + this.apply(pid, signal); + } + + apply(pid: number, signal: NodeJS.Signals): void { + const behaviour = this.behaviours.get(pid) ?? "dies"; + if (behaviour === "immortal") return; + if (behaviour === "ignores-sigterm" && signal !== "SIGKILL") return; + this.markDead(pid); + this.onDeath.get(pid)?.(); + } +} + +interface Child { + readonly pid: number; + readonly args: readonly string[]; + exit(code: number | null): void; +} + +interface Options { + readonly baseEnvironment?: NodeJS.ProcessEnv; + readonly command?: string; + readonly daemonJson?: (startIndex: number) => DaemonJson; + readonly driverEnvironment?: Readonly>; + readonly enabled?: boolean; + readonly expectations?: readonly ScriptedProcessExpectation[]; + readonly filesystem?: MemoryFilesystem; + readonly home?: string; + readonly layout?: (filesystem: MemoryFilesystem) => Promise; + readonly leaseMaxTtlMs?: number; + readonly listening?: boolean; + readonly livePids?: readonly number[]; + readonly nodeVersion?: string; + readonly platforms?: readonly AgentDevicePlatform[]; + readonly record?: { readonly pid: number; readonly entry: string; readonly startedAt: number }; + readonly spawnThrows?: Error; + readonly stateDirContents?: (filesystem: MemoryFilesystem) => Promise; + readonly behaviours?: ReadonlyMap; +} + +async function arrange(options: Options = {}) { + const clock = new FakeClock(1_000_000); + const filesystem = options.filesystem ?? new MemoryFilesystem(); + const simlockHome = options.home ?? home; + const sink = new MemoryLogSink(); + const logger = new JsonLinesLogger({ clock, level: "debug", sink }); + const probe = new FakeTcpProbe(); + const health = new FakeAgentDeviceHealth(); + const processes = new ProcessTable(options.livePids ?? []); + for (const [pid, behaviour] of options.behaviours ?? []) processes.behaviours.set(pid, behaviour); + const runner = new ScriptedProcessRunner(options.expectations ?? [versionRun(), daemonSpawn()]); + const children: Child[] = []; + const events: string[] = []; + let starts = 0; + + health.respondWith(healthyBody); + await installAgentDevice(filesystem, simlockHome, options); + if (options.record !== undefined) { + await filesystem.writeFileAtomic( + `${simlockHome}/agent-device-daemon.json`, + JSON.stringify(options.record), + ); + } + await options.stateDirContents?.(filesystem); + + const spawn = runner.spawn.bind(runner); + vi.spyOn(runner, "spawn").mockImplementation((command, args, spawnOptions) => { + const handle = spawn(command, args, spawnOptions); + if (command !== nodePath || !isDaemonEntry(args[0])) return handle; + if (options.spawnThrows !== undefined) throw options.spawnThrows; + const pid = handle.pid; + let exit!: (result: ProcessResult) => void; + const exited = new Promise((resolve) => { + exit = resolve; + }); + const child: Child = { + args: [...args], + exit(code) { + events.push(`exit ${pid}`); + processes.markDead(pid); + exit({ code, stderr: "", stdout: "" }); + }, + pid, + }; + children.push(child); + processes.markAlive(pid); + processes.onDeath.set(pid, () => child.exit(null)); + events.push(`spawn ${pid}`); + const startIndex = starts; + starts += 1; + const mode = options.daemonJson?.(startIndex) ?? "ready"; + if (mode !== "none") { + void filesystem.writeFileAtomic( + `${simlockHome}/agent-device/daemon.json`, + JSON.stringify({ + httpPort, + pid: mode === "ready" ? pid : pid + 1_000, + token, + version: "0.21.25-dev", + }), + ); + } + if (mode === "ready" && options.listening !== false) probe.startListening(httpPort); + return { + kill(signal = "SIGTERM") { + processes.signals.push({ pid, signal }); + processes.apply(pid, signal); + handle.kill(signal); + }, + pid, + stderr: handle.stderr, + stdout: handle.stdout, + unref: () => handle.unref(), + wait: () => Promise.race([handle.wait(), exited]), + }; + }); + + const agentDevice = new AgentDeviceSupervisor({ + baseEnvironment: options.baseEnvironment ?? { + HOME: "/home/operator", + PATH: "/usr/local/bin:/usr/bin", + }, + clock, + config: { command: options.command ?? "agent-device", enabled: options.enabled ?? true }, + driverEnvironment: options.driverEnvironment ?? {}, + filesystem, + healthProbe: health, + home: simlockHome, + leaseMaxTtlMs: options.leaseMaxTtlMs ?? 4 * 60 * 60_000, + logger, + nodePath, + nodeVersion: options.nodeVersion ?? "22.14.0", + platforms: options.platforms ?? ["ios", "android"], + processRunner: runner, + processSupervisor: processes, + tcpProbe: probe, + }); + const harness = { + agentDevice, + children, + clock, + events, + filesystem, + health, + probe, + processes, + runner, + sink, + started: undefined as Promise | undefined, + begin(): Promise { + const started = agentDevice.start(); + started.catch(() => undefined); + harness.started = started; + return started; + }, + availability: () => agentDevice.availability(), + }; + return harness; +} + +type Harness = Awaited>; + +async function run(options: Options = {}): Promise { + const harness = await arrange(options); + await advancing(harness, harness.begin()); + return harness; +} + +function isDaemonEntry(arg: string | undefined): boolean { + return arg === internalEntry || arg === fallbackEntry; +} + +async function installAgentDevice( + filesystem: MemoryFilesystem, + simlockHome: string, + options: Options, +): Promise { + if (!(await filesystem.exists(simlockHome))) await filesystem.mkdirp(simlockHome); + if (options.command?.includes("/") !== true) { + await filesystem.mkdirp("/usr/local/bin"); + } + await filesystem.mkdirp(`${packageRoot}/bin`); + await filesystem.mkdirp(`${packageRoot}/dist/src/internal`); + await filesystem.writeFileAtomic(realCli, "#!/usr/bin/env node\n"); + filesystem.defineAttributes(realCli, { mode: 0o755 }); + await filesystem.writeFileAtomic(`${packageRoot}/package.json`, '{"name":"agent-device"}'); + await filesystem.writeFileAtomic(internalEntry, ""); + filesystem.defineSymlink("/usr/local/bin/agent-device", realCli); + await options.layout?.(filesystem); +} + +function versionRun(output = "0.21.25-dev\n", extra: Partial = {}) { + return { + match: { args: [realCli, "--version"], command: nodePath }, + result: { code: 0, stderr: "", stdout: output }, + ...extra, + } satisfies ScriptedProcessExpectation; +} + +function daemonSpawn( + entry = internalEntry, + extra: Partial = {}, + simlockHome = home, +): ScriptedProcessExpectation { + return { + hangs: true, + match: { args: [entry, `--simlock-home=${simlockHome}`], command: nodePath }, + ...extra, + }; +} + +function psRun(pid: number, line: string, code = 0): ScriptedProcessExpectation { + return { + match: { args: ["-ww", "-o", "args=", "-p", String(pid)], command: "ps" }, + result: { code, stderr: "", stdout: line === "" ? "" : `${line}\n` }, + }; +} + +function ownLine(entry = internalEntry, simlockHome = home): string { + return `${nodePath} ${entry} --simlock-home=${simlockHome}`; +} + +function record(pid: number, entry = internalEntry) { + return { entry, pid, startedAt: 1 }; +} + +async function flush(): Promise { + await new Promise((resolve) => setImmediate(resolve)); +} + +async function advanceBy(harness: Harness, ms: number): Promise { + for (let elapsed = 0; elapsed < ms; elapsed += 100) { + await flush(); + harness.clock.advance(100); + } + await flush(); +} + +async function advancing(harness: Harness, work: Promise): Promise { + let settled = false; + const tracked = work.finally(() => { + settled = true; + }); + tracked.catch(() => undefined); + for (let tick = 0; tick < 4_000 && !settled; tick += 1) { + await flush(); + harness.clock.advance(100); + } + return tracked; +} + +async function advanceUntil( + harness: Harness, + done: () => boolean, + limitMs = 120_000, +): Promise { + for (let elapsed = 0; elapsed < limitMs && !done(); elapsed += 100) { + await flush(); + harness.clock.advance(100); + } + await flush(); +} + +function stateOf(harness: Harness): AgentDeviceAvailability["state"] { + return harness.availability().state; +} + +function unavailable(harness: Harness) { + const availability = harness.availability(); + if (availability.state !== "unavailable") { + throw new Error(`expected unavailable, got ${availability.state}`); + } + return availability; +} + +function running(harness: Harness) { + const availability = harness.availability(); + if (availability.state !== "running") { + throw new Error(`expected running, got ${availability.state}`); + } + return availability; +} + +function commandsRun(harness: Harness): string[][] { + return harness.runner.calls.map((call) => [call.command, ...call.args]); +} + +function daemonSpawns(harness: Harness): string[][] { + return commandsRun(harness).filter((call) => call[0] === nodePath && isDaemonEntry(call[1])); +} + +function warnings(harness: Harness): string[] { + return harness.sink.records + .filter((entry) => entry.level === "warn") + .map((entry) => JSON.stringify(entry)); +} + +async function readJson(harness: Harness, path: string): Promise> { + return JSON.parse(await harness.filesystem.readFile(path)) as Record; +} + +describe("AgentDeviceSupervisor off and starting", () => { + it("with agentDevice.enabled false, start() reaps a recorded daemon, starts no process, runs no --version, and availability is off before and after", async () => { + const harness = await arrange({ + enabled: false, + expectations: [psRun(900, ownLine())], + livePids: [900], + record: record(900), + }); + expect(stateOf(harness)).toBe("off"); + + await advancing(harness, harness.begin()); + + expect(harness.processes.signals).toEqual([{ pid: 900, signal: "SIGTERM" }]); + expect(commandsRun(harness)).toEqual([["ps", "-ww", "-o", "args=", "-p", "900"]]); + await expect(harness.filesystem.exists(recordPath)).resolves.toBe(false); + expect(stateOf(harness)).toBe("off"); + }); + + it("enabled, availability is starting before start() settles", async () => { + const harness = await arrange({ daemonJson: () => "none" }); + expect(stateOf(harness)).toBe("starting"); + + harness.begin(); + await advanceBy(harness, 1_000); + + expect(stateOf(harness)).toBe("starting"); + expect(harness.children).toHaveLength(1); + }); +}); + +describe("AgentDeviceSupervisor finding agent-device", () => { + const pathLayout = async (filesystem: MemoryFilesystem) => { + await filesystem.mkdirp("/dir-bin/agent-device"); + await filesystem.mkdirp("/noexec"); + await filesystem.writeFileAtomic("/noexec/agent-device", "not executable"); + filesystem.defineAttributes("/noexec/agent-device", { mode: 0o644 }); + await filesystem.mkdirp("/exec"); + filesystem.defineSymlink("/exec/agent-device", realCli); + await filesystem.mkdirp("/later"); + await filesystem.writeFileAtomic("/later/agent-device", "later"); + filesystem.defineAttributes("/later/agent-device", { mode: 0o755 }); + }; + + it("finds a name in the first PATH entry holding an executable file of that name, skipping an earlier directory, non-executable file and missing entry", async () => { + const harness = await run({ + baseEnvironment: { PATH: "/missing:/dir-bin:/noexec:/exec:/later" }, + layout: pathLayout, + }); + + expect(commandsRun(harness)[0]).toEqual([nodePath, realCli, "--version"]); + expect(stateOf(harness)).toBe("running"); + }); + + it("uses a command with a slash as given and resolves its real path, with the package root two levels above", async () => { + const harness = await run({ + command: "/opt/custom/agent-device", + layout: async (filesystem) => { + await filesystem.mkdirp("/opt/custom"); + filesystem.defineSymlink("/opt/custom/agent-device", realCli); + }, + }); + + expect(commandsRun(harness)[0]).toEqual([nodePath, realCli, "--version"]); + expect(daemonSpawns(harness)[0]?.[1]).toBe(internalEntry); + }); + + it("reports a command found nowhere as not-found with the PATH message, and runs nothing", async () => { + const harness = await run({ + baseEnvironment: { PATH: "/nowhere" }, + expectations: [], + }); + + expect(unavailable(harness)).toMatchObject({ + message: 'agent-device not found: "agent-device" is not on PATH', + reason: "not-found", + }); + expect(unavailable(harness).remedy).toContain("agentDevice.command"); + expect(unavailable(harness).remedy).toContain("0.21.25 or later"); + for (const phrase of restartCommand) expect(unavailable(harness).remedy).toContain(phrase); + expect(harness.runner.calls).toEqual([]); + }); + + it.each([ + ["a missing path", async () => undefined], + [ + "a directory", + async (filesystem: MemoryFilesystem) => { + await filesystem.mkdirp("/opt/x/agent-device"); + }, + ], + [ + "a file with no execute bit", + async (filesystem: MemoryFilesystem) => { + await filesystem.mkdirp("/opt/x"); + await filesystem.writeFileAtomic("/opt/x/agent-device", "plain"); + filesystem.defineAttributes("/opt/x/agent-device", { mode: 0o644 }); + }, + ], + ])( + "reports a command with a slash that is %s as not-found and runs nothing", + async (_case, layout) => { + const harness = await run({ command: "/opt/x/agent-device", expectations: [], layout }); + + expect(unavailable(harness)).toMatchObject({ + message: "agent-device not found: /opt/x/agent-device is not an executable file", + reason: "not-found", + }); + expect(harness.runner.calls).toEqual([]); + }, + ); +}); + +describe("AgentDeviceSupervisor checking the install", () => { + it.each([ + [ + "a missing package.json", + async (filesystem: MemoryFilesystem) => filesystem.rm(`${packageRoot}/package.json`), + "package.json", + ], + [ + "a package.json naming another package", + async (filesystem: MemoryFilesystem) => + filesystem.writeFileAtomic(`${packageRoot}/package.json`, '{"name":"other-tool"}'), + "package.json", + ], + [ + "neither daemon entry", + async (filesystem: MemoryFilesystem) => filesystem.rm(internalEntry), + "daemon", + ], + ])("treats %s as not-agent-device and spawns nothing", async (_case, layout, word) => { + const harness = await run({ expectations: [versionRun()], layout }); + + expect(unavailable(harness).reason).toBe("not-agent-device"); + expect(unavailable(harness).message).toContain(word); + expect(unavailable(harness).remedy).toContain("agentDevice.command"); + for (const phrase of restartCommand) expect(unavailable(harness).remedy).toContain(phrase); + expect(daemonSpawns(harness)).toEqual([]); + }); + + it("treats a --version that exits non-zero, prints nothing, or runs past 10 s as three distinct not-agent-device answers, and spawns nothing", async () => { + const messages: string[] = []; + for (const expectation of [ + versionRun("", { result: { code: 3, stderr: "boom", stdout: "" } }), + versionRun(""), + versionRun("", { hangs: true }), + ]) { + const harness = await arrange({ expectations: [expectation] }); + const started = Date.now(); + await advancing(harness, harness.begin()); + + expect(unavailable(harness).reason).toBe("not-agent-device"); + expect(daemonSpawns(harness)).toEqual([]); + messages.push(unavailable(harness).message); + if (expectation.hangs === true) { + expect(unavailable(harness).message).toContain("timed out"); + expect(harness.clock.now()).toBeGreaterThanOrEqual(1_000_000 + 10_000); + } + expect(Date.now() - started).toBeLessThan(60_000); + } + expect(messages[0]).toContain("3"); + expect(new Set(messages).size).toBe(3); + }); + + it("reports 0.20.3 as wrong-version with the found-and-need message", async () => { + const harness = await run({ expectations: [versionRun("0.20.3\n")] }); + + expect(unavailable(harness)).toMatchObject({ + message: "found 0.20.3, need 0.21.25 or later", + reason: "wrong-version", + }); + expect(unavailable(harness).remedy).toContain("0.21.25 or later"); + for (const phrase of restartCommand) expect(unavailable(harness).remedy).toContain(phrase); + }); + + it.each(["abc", "0.21", "0.9.100"])("reports %s as wrong-version", async (printed) => { + const harness = await run({ expectations: [versionRun(`${printed}\n`)] }); + + expect(unavailable(harness).reason).toBe("wrong-version"); + expect(daemonSpawns(harness)).toEqual([]); + }); + + it.each(["0.21.25-dev", "0.21.25", "0.22.0", "0.100.0", "1.0.0+build.7"])( + "starts a daemon for the version %s", + async (printed) => { + const harness = await run({ expectations: [versionRun(`${printed}\n`), daemonSpawn()] }); + + expect(daemonSpawns(harness)).toHaveLength(1); + expect(stateOf(harness)).toBe("running"); + }, + ); + + it("reports Node 22.3.0 as wrong-version naming both Node versions, and starts on Node 22.12.0", async () => { + const old = await run({ expectations: [versionRun()], nodeVersion: "22.3.0" }); + + expect(unavailable(old)).toMatchObject({ + message: "agent-device needs Node 22.12.0 or later, simlock runs on Node 22.3.0", + reason: "wrong-version", + }); + expect(unavailable(old).remedy).toContain("Node 22.12.0 or later"); + for (const phrase of restartCommand) expect(unavailable(old).remedy).toContain(phrase); + expect(daemonSpawns(old)).toEqual([]); + + const exact = await run({ nodeVersion: "22.12.0" }); + expect(stateOf(exact)).toBe("running"); + }); +}); + +describe("AgentDeviceSupervisor spawning", () => { + it("runs --version then spawns the internal daemon entry with the home marker, stdio ignored and the handle unreffed", async () => { + const harness = await run(); + + expect(commandsRun(harness)).toEqual([ + [nodePath, realCli, "--version"], + [nodePath, internalEntry, `--simlock-home=${home}`], + ]); + expect(harness.runner.calls[1]?.options.stdio).toBe("ignore"); + expect(harness.runner.handles[1]?.unreffed).toBe(true); + }); + + it("spawns dist/src/daemon.js when the internal entry does not exist", async () => { + const harness = await run({ + expectations: [versionRun(), daemonSpawn(fallbackEntry)], + layout: async (filesystem) => { + await filesystem.rm(internalEntry); + await filesystem.writeFileAtomic(fallbackEntry, ""); + }, + }); + + expect(daemonSpawns(harness)).toEqual([[nodePath, fallbackEntry, `--simlock-home=${home}`]]); + }); + + it("gives the child no AGENT_DEVICE_ key from the base environment, keeps PATH and HOME, and sets the state, claims, mode, idle, policy and lease keys", async () => { + const harness = await run({ + baseEnvironment: { + AGENT_DEVICE_CLAIMS_DIR: "/operator/claims", + AGENT_DEVICE_SESSION: "operator-session", + AGENT_DEVICE_STATE_DIR: "/operator/state", + HOME: "/home/operator", + PATH: "/usr/local/bin:/usr/bin", + }, + }); + + expect(harness.runner.calls[1]?.options.env).toEqual({ + AGENT_DEVICE_CLAIMS_DIR: `${stateDir}/device-claims`, + AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: "0", + AGENT_DEVICE_DAEMON_POLICY: policyPath, + AGENT_DEVICE_DAEMON_SERVER_MODE: "http", + AGENT_DEVICE_LEASE_MAX_TTL_MS: String(24 * 60 * 60_000), + AGENT_DEVICE_STATE_DIR: stateDir, + HOME: "/home/operator", + PATH: "/usr/local/bin:/usr/bin", + }); + }); + + it.each([ + [4 * 60 * 60_000, 24 * 60 * 60_000], + [12 * 60 * 60_000, 24 * 60 * 60_000], + [20 * 60 * 60_000, 40 * 60 * 60_000], + ])( + "sets the lease ceiling for lease.maxTtlMs %i to %i and reports the same in running", + async (maxTtlMs, ceiling) => { + const harness = await run({ leaseMaxTtlMs: maxTtlMs }); + + expect(harness.runner.calls[1]?.options.env?.["AGENT_DEVICE_LEASE_MAX_TTL_MS"]).toBe( + String(ceiling), + ); + expect(running(harness).leaseMaxTtlMs).toBe(ceiling); + }, + ); + + it("passes every key and value of the drivers' environment unchanged, and copies the iOS device set to the agent-device key", async () => { + const harness = await run({ + driverEnvironment: { + ANDROID_ADB_SERVER_PORT: "5199", + CUSTOM_DRIVER_KEY: "custom value", + SIMLOCK_IOS_DEVICE_SET: "/devices/ios", + }, + }); + const env = harness.runner.calls[1]?.options.env ?? {}; + + expect(env).toMatchObject({ + AGENT_DEVICE_IOS_SIMULATOR_DEVICE_SET: "/devices/ios", + ANDROID_ADB_SERVER_PORT: "5199", + CUSTOM_DRIVER_KEY: "custom value", + SIMLOCK_IOS_DEVICE_SET: "/devices/ios", + }); + expect(env["AGENT_DEVICE_ANDROID_DEVICE_ALLOWLIST"]).toBeUndefined(); + expect(running(harness).platforms).toEqual(["ios", "android"]); + }); + + it("points the daemon at an empty private directory when the iOS driver is off", async () => { + const harness = await run({ + driverEnvironment: { ANDROID_ADB_SERVER_PORT: "5199" }, + platforms: ["android"], + }); + + expect(harness.runner.calls[1]?.options.env?.["AGENT_DEVICE_IOS_SIMULATOR_DEVICE_SET"]).toBe( + `${stateDir}/no-ios-devices`, + ); + await expect(harness.filesystem.readdir(`${stateDir}/no-ios-devices`)).resolves.toEqual([]); + await expect(harness.filesystem.lstat(`${stateDir}/no-ios-devices`)).resolves.toMatchObject({ + kind: "directory", + mode: 0o700, + }); + expect(running(harness).platforms).toEqual(["android"]); + }); + + it("allows only a serial no device has when the Android driver is off", async () => { + const harness = await run({ + driverEnvironment: { SIMLOCK_IOS_DEVICE_SET: "/devices/ios" }, + platforms: ["ios"], + }); + + expect(harness.runner.calls[1]?.options.env?.["AGENT_DEVICE_ANDROID_DEVICE_ALLOWLIST"]).toBe( + "simlock-no-android-device", + ); + expect(running(harness).platforms).toEqual(["ios"]); + }); + + it("writes exactly the policy file before the spawn", async () => { + const harness = await arrange(); + const atSpawn: string[] = []; + const arranged = vi.mocked(harness.runner.spawn).getMockImplementation(); + if (arranged === undefined) throw new Error("arrange installs the spawn spy"); + vi.mocked(harness.runner.spawn).mockImplementation((command, args, options) => { + if (isDaemonEntry(args[0])) { + void harness.filesystem.readFile(policyPath).then((contents) => atSpawn.push(contents)); + } + return arranged(command, args, options); + }); + + await advancing(harness, harness.begin()); + + const expected = + '{"version":1,"commands":{"deny":["boot","shutdown"]},"capabilities":{"deny":["device-shutdown"]}}'; + expect(atSpawn).toEqual([expected]); + await expect(harness.filesystem.readFile(policyPath)).resolves.toBe(expected); + }); +}); + +describe("AgentDeviceSupervisor state directory", () => { + it("creates a missing state directory with mode 0700", async () => { + const harness = await run(); + + await expect(harness.filesystem.lstat(stateDir)).resolves.toMatchObject({ + kind: "directory", + mode: 0o700, + }); + }); + + it("sets an existing state directory to mode 0700", async () => { + const harness = await run({ + stateDirContents: async (filesystem) => { + await filesystem.mkdir(stateDir, { mode: 0o755 }); + }, + }); + + await expect(harness.filesystem.lstat(stateDir)).resolves.toMatchObject({ mode: 0o700 }); + expect(stateOf(harness)).toBe("running"); + }); + + it.each([ + [ + "a symlink", + async (filesystem: MemoryFilesystem) => { + await filesystem.mkdirp("/elsewhere"); + filesystem.defineSymlink(stateDir, "/elsewhere"); + }, + ], + [ + "a file", + async (filesystem: MemoryFilesystem) => + filesystem.writeFileAtomic(stateDir, "not a directory"), + ], + [ + "a directory owned by another user", + async (filesystem: MemoryFilesystem) => { + await filesystem.mkdir(stateDir, { mode: 0o700 }); + filesystem.defineAttributes(stateDir, { uid: (process.getuid?.() ?? 0) + 1 }); + }, + ], + ])( + "treats %s as the state directory as state-dir, naming the path, and spawns nothing", + async (_case, stateDirContents) => { + const harness = await run({ expectations: [versionRun()], stateDirContents }); + + expect(unavailable(harness).reason).toBe("state-dir"); + expect(unavailable(harness).message).toContain(stateDir); + expect(unavailable(harness).remedy).toContain(stateDir); + for (const phrase of restartCommand) expect(unavailable(harness).remedy).toContain(phrase); + expect(daemonSpawns(harness)).toEqual([]); + }, + ); + + it("treats a chmod that throws on an existing state directory as state-dir naming the path and the cause", async () => { + const filesystem = new MemoryFilesystem(); + vi.spyOn(filesystem, "chmod").mockRejectedValue(new Error("EPERM: operation not permitted")); + const harness = await run({ + expectations: [versionRun()], + filesystem, + stateDirContents: async (memory) => { + await memory.mkdir(stateDir, { mode: 0o755 }); + }, + }); + + expect(unavailable(harness).reason).toBe("state-dir"); + expect(unavailable(harness).message).toContain(stateDir); + expect(unavailable(harness).message).toContain("EPERM"); + expect(daemonSpawns(harness)).toEqual([]); + }); +}); + +describe("AgentDeviceSupervisor pid record", () => { + it("records the child's pid and entry while daemon.json is not yet written", async () => { + const harness = await arrange({ daemonJson: () => "none" }); + harness.begin(); + await advanceBy(harness, 1_000); + + expect(await readJson(harness, recordPath)).toMatchObject({ + entry: internalEntry, + pid: harness.children[0]?.pid, + }); + expect(typeof (await readJson(harness, recordPath))["startedAt"]).toBe("number"); + }); + + it("kills the child and reports start-failed naming the error when the record write throws", async () => { + const harness = await arrange(); + const write = harness.filesystem.writeFileAtomic.bind(harness.filesystem); + vi.spyOn(harness.filesystem, "writeFileAtomic").mockImplementation(async (path, ...rest) => { + if (path === recordPath) throw new Error("ENOSPC: no space left"); + return write(path, ...rest); + }); + + await advancing(harness, harness.begin()); + + expect(unavailable(harness)).toMatchObject({ reason: "start-failed" }); + expect(unavailable(harness).message).toContain("ENOSPC"); + expect(harness.processes.signals).toEqual([ + { pid: harness.children[0]?.pid, signal: "SIGTERM" }, + ]); + }); +}); + +describe("AgentDeviceSupervisor readiness", () => { + it("is running with the version, base URL, token and generation 1 once daemon.json names the child, the port listens and /health answers protocol 2", async () => { + const harness = await run(); + + expect(harness.availability()).toEqual({ + baseUrl: `http://127.0.0.1:${httpPort}`, + generation: 1, + leaseMaxTtlMs: 24 * 60 * 60_000, + platforms: ["ios", "android"], + state: "running", + token, + version: "0.21.25-dev", + }); + expect(harness.health.reads).toContain(`http://127.0.0.1:${httpPort}`); + }); + + it("is not ready while daemon.json names another pid, and ready once it names the child", async () => { + const harness = await arrange({ daemonJson: () => "other-pid" }); + harness.begin(); + await advanceBy(harness, 5_000); + expect(stateOf(harness)).toBe("starting"); + + const pid = harness.children[0]?.pid; + await harness.filesystem.writeFileAtomic( + daemonJsonPath, + JSON.stringify({ httpPort, pid, token, version: "0.21.25-dev" }), + ); + harness.probe.startListening(httpPort); + await advanceUntil(harness, () => stateOf(harness) === "running", 10_000); + + expect(stateOf(harness)).toBe("running"); + }); + + it("is not ready while the port is not listening, and ready once it is", async () => { + const harness = await arrange({ listening: false }); + harness.begin(); + await advanceBy(harness, 5_000); + expect(stateOf(harness)).toBe("starting"); + + harness.probe.startListening(httpPort); + await advanceUntil(harness, () => stateOf(harness) === "running", 10_000); + + expect(stateOf(harness)).toBe("running"); + }); + + it("is not ready while a health read rejects, and ready once it answers", async () => { + const harness = await arrange(); + harness.health.rejectWith(new Error("ECONNREFUSED")); + harness.begin(); + await advanceBy(harness, 5_000); + expect(stateOf(harness)).toBe("starting"); + + harness.health.respondWith(healthyBody); + await advanceUntil(harness, () => stateOf(harness) === "running", 10_000); + + expect(stateOf(harness)).toBe("running"); + }); + + it("stops the child and reports wrong-version naming both protocols when /health answers protocol 3", async () => { + const harness = await arrange(); + harness.health.respondWith({ rpcProtocolVersion: 3, service: "agent-device-daemon" }); + + await advancing(harness, harness.begin()); + + expect(unavailable(harness)).toMatchObject({ + message: "agent-device 0.21.25-dev speaks RPC protocol 3, need 2", + reason: "wrong-version", + }); + expect(unavailable(harness).remedy).toContain("RPC protocol 2"); + for (const phrase of restartCommand) expect(unavailable(harness).remedy).toContain(phrase); + expect(harness.processes.signals).toEqual([ + { pid: harness.children[0]?.pid, signal: "SIGTERM" }, + ]); + }); + + it.each([ + ["another service", { rpcProtocolVersion: 2, service: "something-else" }], + ["no numeric protocol", { rpcProtocolVersion: "2", service: "agent-device-daemon" }], + ])( + "stops the child and reports not-agent-device without retrying when /health answers with %s", + async (_case, body) => { + const harness = await arrange(); + harness.health.respondWith(body); + + await advancing(harness, harness.begin()); + + expect(unavailable(harness)).toMatchObject({ + message: "/health did not answer as agent-device", + reason: "not-agent-device", + }); + expect(harness.processes.signals).toEqual([ + { pid: harness.children[0]?.pid, signal: "SIGTERM" }, + ]); + await advanceBy(harness, 120_000); + expect(daemonSpawns(harness)).toHaveLength(1); + expect(stateOf(harness)).toBe("unavailable"); + }, + ); +}); + +describe("AgentDeviceSupervisor failed starts", () => { + it("kills the child after 15 s without readiness, removes the record only after it exits, and reports start-failed naming the timeout", async () => { + const harness = await arrange({ daemonJson: () => "none" }); + const remove = harness.filesystem.rm.bind(harness.filesystem); + vi.spyOn(harness.filesystem, "rm").mockImplementation(async (path) => { + if (path === recordPath) harness.events.push("record removed"); + return remove(path); + }); + const spawnedAt = harness.clock.now(); + + await advancing(harness, harness.begin()); + + expect(unavailable(harness)).toMatchObject({ reason: "start-failed" }); + expect(unavailable(harness).message).toContain("timed out"); + expect(unavailable(harness).remedy).toContain("30 s"); + expect(unavailable(harness).remedy).toContain("simlock daemon logs"); + expect(harness.clock.now() - spawnedAt).toBeGreaterThanOrEqual(15_000); + expect(harness.clock.now() - spawnedAt).toBeLessThan(16_000); + const pid = harness.children[0]?.pid; + expect(harness.processes.signals).toEqual([{ pid, signal: "SIGTERM" }]); + expect(harness.events.slice(-2)).toEqual([`exit ${pid}`, "record removed"]); + await expect(harness.filesystem.exists(recordPath)).resolves.toBe(false); + }); + + it("reports start-failed naming the exit code when the child exits before readiness", async () => { + const harness = await arrange({ daemonJson: () => "none" }); + harness.begin(); + await advanceBy(harness, 1_000); + + harness.children[0]?.exit(7); + await advancing(harness, harness.started as Promise); + + expect(unavailable(harness)).toMatchObject({ reason: "start-failed" }); + expect(unavailable(harness).message).toContain("7"); + }); + + it("reports start-failed and writes no record when the spawn throws", async () => { + const harness = await run({ spawnThrows: new Error("EACCES: permission denied") }); + + expect(warnings(harness).some((line) => line.includes("EACCES"))).toBe(true); + expect(unavailable(harness)).toMatchObject({ reason: "start-failed" }); + expect(unavailable(harness).message).toContain("EACCES"); + await expect(harness.filesystem.exists(recordPath)).resolves.toBe(false); + }); + + it("keeps the record and names the pid when the child is still alive 2 s after SIGKILL", async () => { + const harness = await arrange({ daemonJson: () => "none" }); + harness.begin(); + await advanceBy(harness, 500); + const pid = harness.children[0]?.pid as number; + harness.processes.behaviours.set(pid, "immortal"); + + await advancing(harness, harness.started as Promise); + + expect(harness.processes.signals.map((entry) => entry.signal)).toEqual(["SIGTERM", "SIGKILL"]); + expect(unavailable(harness)).toMatchObject({ reason: "start-failed" }); + expect(unavailable(harness).message).toContain(String(pid)); + expect(await readJson(harness, recordPath)).toMatchObject({ pid }); + }); + + it("resolves start() with start-failed carrying the message when the filesystem throws unexpectedly while writing the policy file", async () => { + const harness = await arrange(); + const write = harness.filesystem.writeFileAtomic.bind(harness.filesystem); + vi.spyOn(harness.filesystem, "writeFileAtomic").mockImplementation(async (path, ...rest) => { + if (path === policyPath) throw new Error("disk on fire"); + return write(path, ...rest); + }); + + await expect(advancing(harness, harness.begin())).resolves.toBeUndefined(); + + expect(unavailable(harness)).toMatchObject({ reason: "start-failed" }); + expect(unavailable(harness).message).toContain("disk on fire"); + expect(daemonSpawns(harness)).toEqual([]); + }); +}); + +describe("AgentDeviceSupervisor reaping a daemon a killed simlock left", () => { + const recorded = { livePids: [900], record: record(900) }; + + it("sends SIGTERM to a live recorded pid whose line ends with the marker and holds the entry, removes the record once it is gone, and starts a new daemon", async () => { + const harness = await run({ + ...recorded, + expectations: [psRun(900, ownLine()), versionRun(), daemonSpawn()], + }); + + expect(harness.processes.signals).toEqual([{ pid: 900, signal: "SIGTERM" }]); + expect(harness.processes.isAlive(900)).toBe(false); + expect(await readJson(harness, recordPath)).toMatchObject({ pid: harness.children[0]?.pid }); + expect(stateOf(harness)).toBe("running"); + }); + + it("escalates to SIGKILL after 10 s when the recorded pid ignores SIGTERM and is still identified", async () => { + const harness = await arrange({ + ...recorded, + behaviours: new Map([[900, "ignores-sigterm"]]), + expectations: [psRun(900, ownLine()), psRun(900, ownLine()), versionRun(), daemonSpawn()], + }); + const startedAt = harness.clock.now(); + + await advancing(harness, harness.begin()); + + expect(harness.processes.signals).toEqual([ + { pid: 900, signal: "SIGTERM" }, + { pid: 900, signal: "SIGKILL" }, + ]); + expect(harness.clock.now() - startedAt).toBeGreaterThanOrEqual(10_000); + expect(stateOf(harness)).toBe("running"); + }); + + it("sends no SIGKILL to a pid whose second ps line no longer passes the identity check", async () => { + const harness = await run({ + ...recorded, + behaviours: new Map([[900, "ignores-sigterm"]]), + expectations: [ + psRun(900, ownLine()), + psRun(900, "/usr/bin/vim notes.txt"), + versionRun(), + daemonSpawn(), + ], + }); + + expect(harness.processes.signals).toEqual([{ pid: 900, signal: "SIGTERM" }]); + }); + + it("keeps the record and reports start-failed naming the pid when it is still alive 2 s after SIGKILL", async () => { + const harness = await run({ + ...recorded, + behaviours: new Map([[900, "immortal"]]), + expectations: [psRun(900, ownLine()), psRun(900, ownLine())], + }); + + expect(harness.processes.signals.map((entry) => entry.signal)).toEqual(["SIGTERM", "SIGKILL"]); + expect(unavailable(harness)).toMatchObject({ reason: "start-failed" }); + expect(unavailable(harness).message).toContain("900"); + expect(await readJson(harness, recordPath)).toMatchObject({ pid: 900 }); + expect(daemonSpawns(harness)).toEqual([]); + }); + + it("spawns with the resolved home as the marker when SIMLOCK_HOME is reached through a symlink, and a later start reaps that child", async () => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/private/tmp/x"); + await filesystem.mkdirp("/tmp"); + filesystem.defineSymlink("/tmp/x", "/private/tmp/x"); + const first = await run({ + expectations: [versionRun(), daemonSpawn(internalEntry, {}, "/private/tmp/x")], + filesystem, + home: "/tmp/x", + }); + const childPid = first.children[0]?.pid as number; + expect(first.children[0]?.args).toEqual([internalEntry, "--simlock-home=/private/tmp/x"]); + + const second = await run({ + expectations: [ + psRun(childPid, ownLine(internalEntry, "/private/tmp/x")), + versionRun(), + daemonSpawn(internalEntry, {}, "/private/tmp/x"), + ], + filesystem, + home: "/tmp/x", + livePids: [childPid], + }); + + expect(second.processes.signals).toEqual([{ pid: childPid, signal: "SIGTERM" }]); + }); + + it("reaps the live pid daemon.json names when there is no record and the line ends with the marker", async () => { + const harness = await run({ + expectations: [psRun(901, ownLine()), versionRun(), daemonSpawn()], + livePids: [901], + stateDirContents: async (filesystem) => { + await filesystem.mkdir(stateDir, { mode: 0o700 }); + await filesystem.writeFileAtomic( + daemonJsonPath, + JSON.stringify({ httpPort, pid: 901, token }), + ); + }, + }); + + expect(harness.processes.signals).toEqual([{ pid: 901, signal: "SIGTERM" }]); + }); + + it("does not read daemon.json and signals nothing when the state directory is a symlink", async () => { + const harness = await run({ + expectations: [versionRun()], + livePids: [901], + stateDirContents: async (filesystem) => { + await filesystem.mkdirp("/elsewhere"); + await filesystem.writeFileAtomic("/elsewhere/daemon.json", JSON.stringify({ pid: 901 })); + filesystem.defineSymlink(stateDir, "/elsewhere"); + }, + }); + + expect(harness.processes.signals).toEqual([]); + expect(commandsRun(harness).filter((call) => call[0] === "ps")).toEqual([]); + expect(unavailable(harness).reason).toBe("state-dir"); + }); + + it.each([ + ["a dead recorded pid", [], []], + ["a ps line of another command", [900], [psRun(900, "/usr/bin/vim notes.txt")]], + ["a line holding the entry but no marker", [900], [psRun(900, `${nodePath} ${internalEntry}`)]], + [ + "a line with another home's marker", + [900], + [psRun(900, ownLine(internalEntry, "/other/home"))], + ], + [ + "a line with this home's marker but another entry than the recorded one", + [900], + [psRun(900, ownLine("/elsewhere/daemon.js"))], + ], + ] as const)("removes the record and signals nothing for %s", async (_case, livePids, ps) => { + // why: the spawn throws, so no new record can stand in for the old one: only a removal empties it. + const harness = await run({ + expectations: [...ps, versionRun(), daemonSpawn()], + livePids, + record: record(900), + spawnThrows: new Error("no spawn"), + }); + + expect(harness.processes.signals).toEqual([]); + expect(await harness.filesystem.exists(recordPath)).toBe(false); + }); + + it("signals nothing when daemon.json names another pid than the recorded one", async () => { + const harness = await run({ + expectations: [psRun(900, ownLine()), versionRun(), daemonSpawn()], + livePids: [900], + record: record(900), + stateDirContents: async (filesystem) => { + await filesystem.mkdir(stateDir, { mode: 0o700 }); + await filesystem.writeFileAtomic(daemonJsonPath, JSON.stringify({ pid: 777 })); + }, + }); + + expect(harness.processes.signals).toEqual([]); + expect(await readJson(harness, recordPath)).toMatchObject({ pid: harness.children[0]?.pid }); + expect(stateOf(harness)).toBe("running"); + }); + + it("removes the record when daemon.json names another pid than the recorded one", async () => { + const harness = await run({ + expectations: [psRun(900, ownLine()), versionRun(), daemonSpawn()], + livePids: [900], + record: record(900), + spawnThrows: new Error("no spawn"), + stateDirContents: async (filesystem) => { + await filesystem.mkdir(stateDir, { mode: 0o700 }); + await filesystem.writeFileAtomic(daemonJsonPath, JSON.stringify({ pid: 777 })); + }, + }); + + expect(await harness.filesystem.exists(recordPath)).toBe(false); + }); + + it.each([ + ["fails", [psRun(900, "", 1)]], + ["prints nothing", [psRun(900, "")]], + ["runs past 5 s", [{ ...psRun(900, ""), hangs: true }]], + ] as const)( + "signals nothing, keeps the record, starts nothing and reports start-failed naming the pid when ps %s", + async (_case, expectations) => { + const harness = await run({ expectations, livePids: [900], record: record(900) }); + + expect(harness.processes.signals).toEqual([]); + expect(unavailable(harness)).toMatchObject({ reason: "start-failed" }); + expect(unavailable(harness).message).toContain("900"); + expect(await readJson(harness, recordPath)).toMatchObject({ pid: 900 }); + expect(commandsRun(harness).filter((call) => call[0] === nodePath)).toEqual([]); + }, + ); + + it("logs a warning naming the pid, stays off and runs nothing 30 s later when ps fails with the edge off", async () => { + const harness = await run({ + enabled: false, + expectations: [psRun(900, "", 1)], + livePids: [900], + record: record(900), + }); + const callsAfterStart = harness.runner.calls.length; + + await advanceBy(harness, 60_000); + + expect(warnings(harness).some((line) => line.includes("900"))).toBe(true); + expect(stateOf(harness)).toBe("off"); + expect(harness.runner.calls).toHaveLength(callsAfterStart); + }); +}); + +describe("AgentDeviceSupervisor after the daemon is running", () => { + async function runningFor( + options: Options, + ms: number, + ): Promise<{ harness: Harness; readyAt: number }> { + const harness = await run(options); + expect(stateOf(harness)).toBe("running"); + const readyAt = harness.clock.now(); + await advanceBy(harness, ms); + return { harness, readyAt }; + } + + it("starts a child that exited 60 s after ready again at once with the same environment, as generation 2, and warns with the exit code", async () => { + const { harness } = await runningFor( + { expectations: [versionRun(), daemonSpawn(), versionRun(), daemonSpawn()] }, + 60_000, + ); + const firstEnv = harness.runner.calls[1]?.options.env; + + harness.children[0]?.exit(42); + await flush(); + expect(stateOf(harness)).toBe("starting"); + await advanceUntil(harness, () => stateOf(harness) === "running", 5_000); + + expect(daemonSpawns(harness)).toHaveLength(2); + expect(harness.runner.calls[3]?.options.env).toEqual(firstEnv); + expect(running(harness).generation).toBe(2); + expect(warnings(harness).some((line) => line.includes("42"))).toBe(true); + }); + + it("reports start-failed for a child that exits 5 s after ready, and starts it again only 30 s later", async () => { + const { harness } = await runningFor( + { expectations: [versionRun(), daemonSpawn(), versionRun(), daemonSpawn()] }, + 5_000, + ); + + harness.children[0]?.exit(9); + await flush(); + + expect(unavailable(harness)).toMatchObject({ reason: "start-failed" }); + expect(unavailable(harness).message).toMatch( + /agent-device exited .*9.* 5 s after it was ready/, + ); + await advanceBy(harness, 29_000); + expect(daemonSpawns(harness)).toHaveLength(1); + await advanceUntil(harness, () => stateOf(harness) === "running", 5_000); + expect(daemonSpawns(harness)).toHaveLength(2); + }); + + it("times the stable uptime from ready, not from the spawn: a child ready 10 s after the spawn that exits 62 s after it is start-failed", async () => { + const harness = await arrange({ + daemonJson: () => "none", + expectations: [versionRun(), daemonSpawn(), versionRun(), daemonSpawn()], + }); + void harness.begin(); + await advanceUntil(harness, () => harness.children.length === 1, 5_000); + const spawnedAt = harness.clock.now(); + await advanceBy(harness, 10_000); + await harness.filesystem.writeFileAtomic( + daemonJsonPath, + JSON.stringify({ httpPort, pid: harness.children[0]?.pid, token }), + ); + harness.probe.startListening(httpPort); + await advanceUntil(harness, () => stateOf(harness) === "running", 5_000); + await advanceBy(harness, spawnedAt + 62_000 - harness.clock.now()); + + harness.children[0]?.exit(9); + await flush(); + + expect(unavailable(harness)).toMatchObject({ reason: "start-failed" }); + expect(daemonSpawns(harness)).toHaveLength(1); + }); + + it("reports start-failed for a child that exits 59 s after ready instead of starting it at once", async () => { + const { harness } = await runningFor( + { expectations: [versionRun(), daemonSpawn(), versionRun(), daemonSpawn()] }, + 59_000, + ); + + harness.children[0]?.exit(9); + await flush(); + + expect(unavailable(harness)).toMatchObject({ reason: "start-failed" }); + expect(daemonSpawns(harness)).toHaveLength(1); + }); + + it("retries a failed first start 30 s later and is running when that succeeds", async () => { + const harness = await arrange({ + daemonJson: (index) => (index === 0 ? "none" : "ready"), + expectations: [versionRun(), daemonSpawn(), versionRun(), daemonSpawn()], + }); + await advancing(harness, harness.begin()); + expect(unavailable(harness).reason).toBe("start-failed"); + const failedAt = harness.clock.now(); + + await advanceBy(harness, 29_000); + expect(daemonSpawns(harness)).toHaveLength(1); + await advanceUntil(harness, () => stateOf(harness) === "running", 5_000); + + expect(daemonSpawns(harness)).toHaveLength(2); + expect(harness.clock.now() - failedAt).toBeGreaterThanOrEqual(30_000); + expect(harness.clock.now() - failedAt).toBeLessThanOrEqual(30_500); + expect(running(harness).generation).toBe(1); + }); + + it("retries a start that failed after a restart 30 s later", async () => { + const harness = await arrange({ + daemonJson: (index) => (index === 1 ? "none" : "ready"), + expectations: [ + versionRun(), + daemonSpawn(), + versionRun(), + daemonSpawn(), + versionRun(), + daemonSpawn(), + ], + }); + await advancing(harness, harness.begin()); + await advanceBy(harness, 60_000); + harness.children[0]?.exit(1); + await advanceUntil(harness, () => unavailableState(harness), 30_000); + expect(unavailable(harness).reason).toBe("start-failed"); + + await advanceUntil(harness, () => stateOf(harness) === "running", 60_000); + + expect(daemonSpawns(harness)).toHaveLength(3); + expect(stateOf(harness)).toBe("running"); + }); + + function unavailableState(harness: Harness): boolean { + return stateOf(harness) === "unavailable"; + } + + it.each([ + ["not-found", { baseEnvironment: { PATH: "/nowhere" }, expectations: [] }], + [ + "not-agent-device", + { + expectations: [versionRun()], + layout: async (filesystem: MemoryFilesystem) => + filesystem.rm(`${packageRoot}/package.json`), + }, + ], + ["wrong-version", { expectations: [versionRun("0.20.3\n")] }], + [ + "state-dir", + { + expectations: [versionRun()], + stateDirContents: async (filesystem: MemoryFilesystem) => { + await filesystem.mkdirp("/elsewhere"); + filesystem.defineSymlink(stateDir, "/elsewhere"); + }, + }, + ], + ] satisfies [string, Options][])("does not retry a %s", async (reason, options) => { + const harness = await run(options); + expect(unavailable(harness).reason).toBe(reason); + const calls = harness.runner.calls.length; + + await advanceBy(harness, 120_000); + + expect(harness.runner.calls).toHaveLength(calls); + expect(unavailable(harness).reason).toBe(reason); + }); +}); + +describe("AgentDeviceSupervisor.stop", () => { + it("sends SIGTERM to the child, removes the record once it is gone, starts nothing more and leaves availability off", async () => { + const harness = await run(); + const pid = harness.children[0]?.pid; + const calls = harness.runner.calls.length; + + await advancing(harness, harness.agentDevice.stop()); + + expect(harness.processes.signals).toEqual([{ pid, signal: "SIGTERM" }]); + await expect(harness.filesystem.exists(recordPath)).resolves.toBe(false); + expect(harness.events.slice(-1)).toEqual([`exit ${pid}`]); + await advanceBy(harness, 120_000); + expect(harness.runner.calls).toHaveLength(calls); + expect(stateOf(harness)).toBe("off"); + }); + + it("sends SIGKILL after 10 s when the child ignores SIGTERM", async () => { + const harness = await run(); + const pid = harness.children[0]?.pid as number; + harness.processes.behaviours.set(pid, "ignores-sigterm"); + const stoppedAt = harness.clock.now(); + + await advancing(harness, harness.agentDevice.stop()); + + expect(harness.processes.signals).toEqual([ + { pid, signal: "SIGTERM" }, + { pid, signal: "SIGKILL" }, + ]); + expect(harness.clock.now() - stoppedAt).toBeGreaterThanOrEqual(10_000); + await expect(harness.filesystem.exists(recordPath)).resolves.toBe(false); + }); + + it("keeps the record and warns naming the pid when the child is alive 2 s after SIGKILL, and availability still ends off", async () => { + const harness = await run(); + const pid = harness.children[0]?.pid as number; + harness.processes.behaviours.set(pid, "immortal"); + + await advancing(harness, harness.agentDevice.stop()); + + expect(await readJson(harness, recordPath)).toMatchObject({ pid }); + expect(warnings(harness).some((line) => line.includes(String(pid)))).toBe(true); + expect(stateOf(harness)).toBe("off"); + }); + + it("warns naming the pid and keeps the record when stop() ends a start whose child survives SIGKILL", async () => { + const harness = await arrange({ daemonJson: () => "none" }); + harness.begin(); + await advanceBy(harness, 1_000); + const pid = harness.children[0]?.pid as number; + harness.processes.behaviours.set(pid, "immortal"); + + await advancing(harness, harness.agentDevice.stop()); + await advancing(harness, harness.started as Promise); + + expect(await readJson(harness, recordPath)).toMatchObject({ pid }); + expect(warnings(harness).some((line) => line.includes(String(pid)))).toBe(true); + expect(stateOf(harness)).toBe("off"); + }); + + it("spawns nothing when stop() is asked while the state directory is being prepared", async () => { + const harness = await arrange(); + const writeFileAtomic = harness.filesystem.writeFileAtomic.bind(harness.filesystem); + let stopping: Promise | undefined; + vi.spyOn(harness.filesystem, "writeFileAtomic").mockImplementation(async (path, data) => { + if (path === `${stateDir}/policy.json`) stopping = harness.agentDevice.stop(); + await writeFileAtomic(path, data); + }); + + await advancing(harness, harness.begin()); + await advancing(harness, stopping as Promise); + + expect(daemonSpawns(harness)).toHaveLength(0); + expect(stateOf(harness)).toBe("off"); + }); + + it("cancels a pending retry", async () => { + const harness = await run({ daemonJson: () => "none" }); + expect(unavailable(harness).reason).toBe("start-failed"); + const calls = harness.runner.calls.length; + + await advancing(harness, harness.agentDevice.stop()); + await advanceBy(harness, 120_000); + + expect(harness.runner.calls).toHaveLength(calls); + expect(stateOf(harness)).toBe("off"); + }); + + it("ends a start in progress with the child killed and availability off", async () => { + const harness = await arrange({ daemonJson: () => "none" }); + harness.begin(); + await advanceBy(harness, 1_000); + const pid = harness.children[0]?.pid; + + await advancing(harness, harness.agentDevice.stop()); + await advancing(harness, harness.started as Promise); + + expect(harness.processes.signals).toEqual([{ pid, signal: "SIGTERM" }]); + expect(stateOf(harness)).toBe("off"); + await advanceBy(harness, 120_000); + expect(daemonSpawns(harness)).toHaveLength(1); + expect(stateOf(harness)).toBe("off"); + }); + + it("logs no start-failed warning when stop() cancels a start", async () => { + const harness = await arrange({ daemonJson: () => "none" }); + harness.begin(); + await advanceBy(harness, 1_000); + + await advancing(harness, harness.agentDevice.stop()); + await advancing(harness, harness.started as Promise); + + expect(warnings(harness).filter((line) => line.includes("start failed"))).toEqual([]); + expect(stateOf(harness)).toBe("off"); + }); + + it("leaves no unhandled rejection when a health read rejects after stop() was asked during the port check", async () => { + const harness = await arrange(); + harness.health.rejectWith(new Error("ECONNREFUSED")); + const isListening = harness.probe.isListening.bind(harness.probe); + let stopping: Promise | undefined; + vi.spyOn(harness.probe, "isListening").mockImplementation(async (port) => { + stopping = harness.agentDevice.stop(); + return isListening(port); + }); + const unhandled: unknown[] = []; + const record = (reason: unknown): void => { + unhandled.push(reason); + }; + process.on("unhandledRejection", record); + try { + harness.begin(); + await advanceBy(harness, 5_000); + await advancing(harness, stopping as Promise); + await new Promise((resolve) => setImmediate(resolve)); + } finally { + process.off("unhandledRejection", record); + } + + expect(unhandled).toEqual([]); + expect(stateOf(harness)).toBe("off"); + }); +}); diff --git a/src/agent-device/host/agent-device-supervisor.ts b/src/agent-device/host/agent-device-supervisor.ts new file mode 100644 index 00000000..7c1bc231 --- /dev/null +++ b/src/agent-device/host/agent-device-supervisor.ts @@ -0,0 +1,773 @@ +import { + type Clock, + type Filesystem, + isMissingPathError, + type Logger, + type PathDetails, + type ProcessHandle, + type ProcessResult, + type ProcessRunner, + type ProcessSupervisor, + type TcpProbe, + type TimerHandle, +} from "../../ports/index.js"; +import { + AGENT_DEVICE_RPC_PROTOCOL_VERSION, + isVersionAtLeast, + MIN_AGENT_DEVICE_VERSION, + MIN_NODE_VERSION, +} from "../shared/compatibility.js"; +import { + AGENT_DEVICE_POLICY, + agentDeviceDaemonEnvironment, + agentDeviceLeaseMaxTtlMs, + NO_ANDROID_DEVICE_SERIAL, +} from "../shared/daemon-environment.js"; +import type { AgentDeviceHealthProbe } from "./health.js"; +import { findAgentDevice } from "./install.js"; + +export type AgentDevicePlatform = "ios" | "android"; + +type UnavailableReason = + | "not-found" + | "not-agent-device" + | "wrong-version" + | "state-dir" + | "start-failed"; + +export type AgentDeviceAvailability = + | { readonly state: "off" } + | { readonly state: "starting" } + | { + readonly state: "running"; + readonly version: string; + readonly baseUrl: string; + readonly token: string; + readonly generation: number; + readonly leaseMaxTtlMs: number; + readonly platforms: readonly AgentDevicePlatform[]; + } + | { + readonly state: "unavailable"; + readonly reason: UnavailableReason; + readonly message: string; + readonly remedy: string; + }; + +export interface AgentDeviceSupervisorOptions { + readonly baseEnvironment: NodeJS.ProcessEnv; + readonly clock: Clock; + readonly config: { readonly enabled: boolean; readonly command: string }; + readonly driverEnvironment: Readonly>; + readonly filesystem: Filesystem; + readonly healthProbe: AgentDeviceHealthProbe; + readonly home: string; + readonly leaseMaxTtlMs: number; + readonly logger: Logger; + readonly nodePath: string; + readonly nodeVersion: string; + readonly platforms: readonly AgentDevicePlatform[]; + readonly processRunner: ProcessRunner; + readonly processSupervisor: ProcessSupervisor; + readonly tcpProbe: TcpProbe; +} + +const POLL_MS = 100; +const PS_TIMEOUT_MS = 5_000; +const VERSION_TIMEOUT_MS = 10_000; +const READY_TIMEOUT_MS = 15_000; +const TERM_GRACE_MS = 10_000; +const KILL_GRACE_MS = 2_000; +const RETRY_MS = 30_000; +const STABLE_AFTER_READY_MS = 60_000; +const MARKER = "--simlock-home="; +const RESTART = "then restart the daemon (`simlock daemon stop`, then `simlock daemon start`)"; + +type RemedyKind = UnavailableReason | "node" | "protocol"; + +interface Child { + readonly pid: number; + exit: ProcessResult | undefined; + readyAt: number; +} + +interface PidRecord { + readonly pid: number; + readonly entry: string; + readonly startedAt: number; +} + +type Raced = { readonly timedOut: false; readonly value: T } | { readonly timedOut: true }; + +type ReadyOutcome = + | { readonly kind: "ready"; readonly baseUrl: string; readonly token: string } + | { readonly kind: "exited" } + | { readonly kind: "cancelled" } + | { readonly kind: "refused"; readonly reason: UnavailableReason; readonly message: string }; + +type ReapOutcome = { readonly ok: true } | { readonly ok: false; readonly message: string }; + +function messageOf(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function describeExit(result: ProcessResult): string { + return result.code === null ? "after a signal" : `with code ${result.code}`; +} + +function endpointOf( + daemon: Record | undefined, + pid: number, +): { readonly httpPort: number; readonly token: string } | undefined { + if (daemon === undefined || daemon["pid"] !== pid) return undefined; + const { httpPort, token } = daemon; + return typeof httpPort === "number" && typeof token === "string" + ? { httpPort, token } + : undefined; +} + +function judgeHealth(body: unknown, version: string): ReadyOutcome | undefined { + const health = (typeof body === "object" && body !== null ? body : {}) as Record; + const protocol = health["rpcProtocolVersion"]; + if (health["service"] !== "agent-device-daemon" || typeof protocol !== "number") { + return { + kind: "refused", + message: "/health did not answer as agent-device", + reason: "not-agent-device", + }; + } + if (protocol === AGENT_DEVICE_RPC_PROTOCOL_VERSION) return undefined; + return { + kind: "refused", + message: `agent-device ${version} speaks RPC protocol ${protocol}, need ${AGENT_DEVICE_RPC_PROTOCOL_VERSION}`, + reason: "wrong-version", + }; +} + +export class AgentDeviceSupervisor { + readonly #o: AgentDeviceSupervisorOptions; + #stateDir: string; + #recordPath: string; + #availability: AgentDeviceAvailability; + #generation = 0; + #stopped = false; + #child: Child | undefined; + #current: Promise = Promise.resolve(); + #retry: TimerHandle | undefined; + readonly #stopWaiters = new Set<() => void>(); + + constructor(options: AgentDeviceSupervisorOptions) { + this.#o = options; + this.#stateDir = `${options.home}/agent-device`; + this.#recordPath = `${options.home}/agent-device-daemon.json`; + this.#availability = options.config.enabled ? { state: "starting" } : { state: "off" }; + } + + availability(): AgentDeviceAvailability { + return this.#availability; + } + + async start(): Promise { + await this.#begin(); + } + + async stop(): Promise { + this.#stopped = true; + for (const wake of this.#stopWaiters) wake(); + this.#cancelRetry(); + await this.#current; + const child = this.#child; + this.#child = undefined; + if (child !== undefined) { + const gone = child.exit !== undefined || (await this.#terminate(child.pid)); + if (gone) { + await this.#removeRecord(); + } else { + this.#o.logger.warn( + `agent-device daemon pid ${child.pid} is still alive after stop; its record is kept for the next start`, + ); + } + } + this.#availability = { state: "off" }; + } + + #begin(): Promise { + this.#current = this.#run(); + return this.#current; + } + + async #run(): Promise { + if (this.#stopped) return; + try { + await this.#startOnce(); + } catch (error: unknown) { + this.#fail("start-failed", messageOf(error)); + } + if (!this.#stopped && this.#availability.state === "unavailable") { + if (this.#availability.reason === "start-failed") this.#scheduleRetry(); + } + } + + #scheduleRetry(): void { + this.#cancelRetry(); + this.#retry = this.#o.clock.setTimer(RETRY_MS, () => { + this.#retry = undefined; + void this.#begin(); + }); + } + + #cancelRetry(): void { + if (this.#retry === undefined) return; + this.#o.clock.cancel(this.#retry); + this.#retry = undefined; + } + + #remedy(kind: RemedyKind): string { + switch (kind) { + case "not-found": + return `install agent-device ${MIN_AGENT_DEVICE_VERSION} or later, or set agentDevice.command to its path, ${RESTART}`; + case "not-agent-device": + return `point agentDevice.command at an agent-device install, ${RESTART}`; + case "wrong-version": + return `install agent-device ${MIN_AGENT_DEVICE_VERSION} or later, ${RESTART}`; + case "node": + return `run simlock on Node ${MIN_NODE_VERSION} or later, ${RESTART}`; + case "protocol": + return `install an agent-device that speaks RPC protocol ${AGENT_DEVICE_RPC_PROTOCOL_VERSION}, ${RESTART}`; + case "state-dir": + return `make ${this.#stateDir} a directory owned by the user simlock runs as, ${RESTART}`; + case "start-failed": + return "simlock tries again every 30 s; see the daemon log (`simlock daemon logs`)"; + } + } + + #fail(reason: UnavailableReason, message: string, remedyKind?: RemedyKind): void { + if (reason === "start-failed" && !this.#stopped) + this.#o.logger.warn(`agent-device start failed: ${message}`); + this.#availability = { + state: "unavailable", + reason, + message, + remedy: this.#remedy(remedyKind ?? reason), + }; + } + + async #startOnce(): Promise { + const o = this.#o; + const realHome = await o.filesystem.realpath(o.home); + this.#stateDir = `${realHome}/agent-device`; + this.#recordPath = `${realHome}/agent-device-daemon.json`; + if (o.config.enabled) this.#availability = { state: "starting" }; + if (!(await this.#reapStep(realHome))) return; + const found = await this.#checkInstall(); + if (found === undefined || this.#stopped) return; + const env = await this.#prepareLaunch(); + if (env === undefined || this.#stopped) return; + const child = await this.#launch(found.entry, realHome, env); + if (child === undefined) return; + await this.#awaitReady(child, found.version); + } + + async #reapStep(realHome: string): Promise { + const o = this.#o; + const reaped = await this.#reap(realHome); + if (this.#stopped) return false; + if (!reaped.ok) { + if (o.config.enabled) { + this.#fail("start-failed", reaped.message); + } else { + o.logger.warn(reaped.message); + this.#availability = { state: "off" }; + } + return false; + } + if (!o.config.enabled) this.#availability = { state: "off" }; + return o.config.enabled; + } + + async #checkInstall(): Promise<{ readonly entry: string; readonly version: string } | undefined> { + const o = this.#o; + const install = await findAgentDevice( + o.filesystem, + o.config.command, + o.baseEnvironment["PATH"], + ); + if (!install.ok) { + this.#fail(install.reason, install.message); + return undefined; + } + if (this.#stopped) return undefined; + const version = await this.#readVersion(install.realPath); + if (typeof version !== "string") { + this.#fail("not-agent-device", version.message); + return undefined; + } + return this.#versionsAcceptable(version) ? { entry: install.entry, version } : undefined; + } + + #versionsAcceptable(version: string): boolean { + if (!isVersionAtLeast(version, MIN_AGENT_DEVICE_VERSION)) { + this.#fail("wrong-version", `found ${version}, need ${MIN_AGENT_DEVICE_VERSION} or later`); + return false; + } + if (!isVersionAtLeast(this.#o.nodeVersion, MIN_NODE_VERSION)) { + this.#fail( + "wrong-version", + `agent-device needs Node ${MIN_NODE_VERSION} or later, simlock runs on Node ${this.#o.nodeVersion}`, + "node", + ); + return false; + } + return true; + } + + async #prepareLaunch(): Promise { + const problem = await this.#prepareStateDir(); + if (problem !== undefined) { + this.#fail("state-dir", problem); + return undefined; + } + const policyPath = `${this.#stateDir}/policy.json`; + await this.#o.filesystem.writeFileAtomic(policyPath, JSON.stringify(AGENT_DEVICE_POLICY)); + return this.#environment(policyPath); + } + + async #launch( + entry: string, + realHome: string, + env: NodeJS.ProcessEnv, + ): Promise { + const o = this.#o; + let handle: ProcessHandle; + try { + handle = o.processRunner.spawn(o.nodePath, [entry, `${MARKER}${realHome}`], { + env, + stdio: "ignore", + }); + } catch (error: unknown) { + this.#fail("start-failed", `could not start agent-device: ${messageOf(error)}`); + return undefined; + } + handle.unref(); + const child: Child = { exit: undefined, pid: handle.pid, readyAt: 0 }; + const exited = (result: ProcessResult) => { + child.exit = result; + this.#onChildExit(child); + }; + handle.wait().then(exited, () => exited({ code: null, stderr: "", stdout: "" })); + + try { + const record: PidRecord = { entry, pid: child.pid, startedAt: o.clock.now() }; + await o.filesystem.writeFileAtomic(this.#recordPath, JSON.stringify(record)); + } catch (error: unknown) { + await this.#stopFailedChild( + child, + "start-failed", + `could not write the pid record: ${messageOf(error)}`, + ); + return undefined; + } + return child; + } + + async #awaitReady(child: Child, version: string): Promise { + const signal = { aborted: false }; + const raced = await this.#race(this.#waitReady(child, version, signal), READY_TIMEOUT_MS); + signal.aborted = true; + if (raced.timedOut) { + await this.#stopFailedChild( + child, + "start-failed", + `agent-device was not ready within ${READY_TIMEOUT_MS / 1000} s (timed out)`, + ); + return; + } + await this.#settle(child, version, raced.value); + } + + async #settle(child: Child, version: string, outcome: ReadyOutcome): Promise { + switch (outcome.kind) { + case "cancelled": + await this.#stopFailedChild(child, "start-failed", "stopped"); + this.#availability = { state: "off" }; + return; + case "exited": + await this.#removeRecord(); + this.#fail( + "start-failed", + `agent-device exited ${describeExit(child.exit as ProcessResult)} before it was ready`, + ); + return; + case "refused": + await this.#stopFailedChild( + child, + outcome.reason, + outcome.message, + outcome.reason === "wrong-version" ? "protocol" : undefined, + ); + return; + case "ready": + this.#markRunning(child, version, outcome); + } + } + + #markRunning( + child: Child, + version: string, + ready: { readonly baseUrl: string; readonly token: string }, + ): void { + const o = this.#o; + this.#generation += 1; + child.readyAt = o.clock.now(); + this.#child = child; + this.#availability = { + baseUrl: ready.baseUrl, + generation: this.#generation, + leaseMaxTtlMs: agentDeviceLeaseMaxTtlMs(o.leaseMaxTtlMs), + platforms: o.platforms, + state: "running", + token: ready.token, + version, + }; + if (child.exit !== undefined) this.#onChildExit(child); + } + + #onChildExit(child: Child): void { + if (this.#stopped || this.#child !== child) return; + this.#child = undefined; + const result = child.exit as ProcessResult; + const elapsedMs = this.#o.clock.now() - child.readyAt; + this.#o.logger.warn(`agent-device daemon pid ${child.pid} exited ${describeExit(result)}`); + if (elapsedMs >= STABLE_AFTER_READY_MS) { + void this.#begin(); + return; + } + this.#fail( + "start-failed", + `agent-device exited ${describeExit(result)}, ${Math.floor(elapsedMs / 1000)} s after it was ready`, + ); + this.#scheduleRetry(); + } + + async #readVersion(realPath: string): Promise { + const o = this.#o; + let raced: Raced; + try { + raced = await this.#race( + o.processRunner.run(o.nodePath, [realPath, "--version"], { timeoutMs: VERSION_TIMEOUT_MS }), + VERSION_TIMEOUT_MS, + ); + } catch (error: unknown) { + return { message: `agent-device --version failed: ${messageOf(error)}` }; + } + if (raced.timedOut) { + return { message: `agent-device --version timed out after ${VERSION_TIMEOUT_MS / 1000} s` }; + } + if (raced.value.code !== 0) { + return { message: `agent-device --version exited with code ${raced.value.code}` }; + } + const version = raced.value.stdout.split("\n")[0]?.trim() ?? ""; + return version === "" ? { message: "agent-device --version printed nothing" } : version; + } + + async #prepareStateDir(): Promise { + const path = this.#stateDir; + let details; + try { + details = await this.#o.filesystem.lstat(path); + } catch (error: unknown) { + if (!isMissingPathError(error)) return `${path} cannot be inspected: ${messageOf(error)}`; + return this.#createStateDir(); + } + return this.#tightenStateDir(details); + } + + async #createStateDir(): Promise { + try { + await this.#o.filesystem.mkdir(this.#stateDir, { mode: 0o700 }); + } catch (error: unknown) { + return `${this.#stateDir} cannot be created: ${messageOf(error)}`; + } + return undefined; + } + + async #tightenStateDir(details: PathDetails): Promise { + const path = this.#stateDir; + if (details.kind !== "directory") return `${path} is a ${details.kind}, not a directory`; + const uid = process.getuid?.(); + if (uid !== undefined && details.uid !== uid) return `${path} is owned by another user`; + if (details.mode === 0o700) return undefined; + try { + await this.#o.filesystem.chmod(path, 0o700); + } catch (error: unknown) { + return `${path} cannot be set to mode 0700: ${messageOf(error)}`; + } + return undefined; + } + + async #environment(policyPath: string): Promise { + const o = this.#o; + const env: NodeJS.ProcessEnv = {}; + for (const [key, value] of Object.entries(o.baseEnvironment)) { + if (!key.startsWith("AGENT_DEVICE_")) env[key] = value; + } + Object.assign( + env, + o.driverEnvironment, + agentDeviceDaemonEnvironment({ + leaseMaxTtlMs: o.leaseMaxTtlMs, + policyPath, + stateDir: this.#stateDir, + }), + ); + if (o.platforms.includes("ios")) { + const deviceSet = o.driverEnvironment["SIMLOCK_IOS_DEVICE_SET"]; + if (deviceSet !== undefined) env["AGENT_DEVICE_IOS_SIMULATOR_DEVICE_SET"] = deviceSet; + } else { + const empty = `${this.#stateDir}/no-ios-devices`; + try { + await o.filesystem.mkdir(empty, { mode: 0o700 }); + } catch (error: unknown) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; + await o.filesystem.chmod(empty, 0o700); + } + env["AGENT_DEVICE_IOS_SIMULATOR_DEVICE_SET"] = empty; + } + if (!o.platforms.includes("android")) { + env["AGENT_DEVICE_ANDROID_DEVICE_ALLOWLIST"] = NO_ANDROID_DEVICE_SERIAL; + } + return env; + } + + async #waitReady( + child: Child, + version: string, + signal: { aborted: boolean }, + ): Promise { + for (;;) { + await this.#untilStop(this.#sleep(POLL_MS)); + if (signal.aborted || this.#stopped) return { kind: "cancelled" }; + if (child.exit !== undefined) return { kind: "exited" }; + const outcome = await this.#probe(child, version); + if (outcome !== undefined) return outcome; + } + } + + async #probe(child: Child, version: string): Promise { + const o = this.#o; + const endpoint = endpointOf(await this.#readDaemonJson(), child.pid); + if (endpoint === undefined) return undefined; + if (!(await o.tcpProbe.isListening(endpoint.httpPort))) return undefined; + const baseUrl = `http://127.0.0.1:${endpoint.httpPort}`; + let body: unknown; + try { + const read = await this.#untilStop(o.healthProbe.read(baseUrl)); + if (read === "stopped") return { kind: "cancelled" }; + body = read; + } catch { + return undefined; + } + return judgeHealth(body, version) ?? { baseUrl, kind: "ready", token: endpoint.token }; + } + + async #readDaemonJson(): Promise | undefined> { + try { + const parsed = JSON.parse( + await this.#o.filesystem.readFile(`${this.#stateDir}/daemon.json`), + ) as unknown; + return typeof parsed === "object" && parsed !== null + ? (parsed as Record) + : undefined; + } catch { + return undefined; + } + } + + async #stopFailedChild( + child: Child, + reason: UnavailableReason, + message: string, + remedyKind?: RemedyKind, + ): Promise { + const gone = child.exit !== undefined || (await this.#terminate(child.pid)); + if (gone) { + await this.#removeRecord(); + this.#fail(reason, message, remedyKind); + return; + } + this.#o.logger.warn( + `agent-device daemon pid ${child.pid} is still alive after SIGKILL; its record is kept for the next start`, + ); + this.#fail( + reason, + `${message}; pid ${child.pid} is still alive and its record is kept`, + remedyKind, + ); + } + + async #terminate(pid: number): Promise { + const supervisor = this.#o.processSupervisor; + supervisor.signal(pid, "SIGTERM"); + if (await this.#waitGone(pid, TERM_GRACE_MS)) return true; + supervisor.signal(pid, "SIGKILL"); + return this.#waitGone(pid, KILL_GRACE_MS); + } + + async #waitGone(pid: number, ms: number): Promise { + const deadline = this.#o.clock.now() + ms; + while (this.#o.processSupervisor.isAlive(pid)) { + if (this.#o.clock.now() >= deadline) return false; + await this.#sleep(POLL_MS); + } + return true; + } + + async #reap(realHome: string): Promise { + const record = await this.#readRecord(); + if (record !== undefined) { + if (!this.#o.processSupervisor.isAlive(record.pid)) { + await this.#removeRecord(); + return { ok: true }; + } + return this.#reapPid(record.pid, realHome, record.entry); + } + if (!(await this.#stateDirIsOurs())) return { ok: true }; + const pid = (await this.#readDaemonJson())?.["pid"]; + if (typeof pid !== "number" || !this.#o.processSupervisor.isAlive(pid)) return { ok: true }; + return this.#reapPid(pid, realHome, undefined); + } + + async #reapPid(pid: number, realHome: string, entry: string | undefined): Promise { + const supervisor = this.#o.processSupervisor; + const inconclusive = (why: string): ReapOutcome => ({ + message: `the agent-device daemon a previous simlock left (pid ${pid}) ${why}`, + ok: false, + }); + + const first = await this.#identify(pid, realHome, entry); + if (first === "other") { + await this.#removeRecord(); + return { ok: true }; + } + if (first === "unknown") + return inconclusive("could not be identified, so it was not signalled"); + + supervisor.signal(pid, "SIGTERM"); + if (await this.#waitGone(pid, TERM_GRACE_MS)) { + await this.#removeRecord(); + return { ok: true }; + } + const second = await this.#identify(pid, realHome, entry); + if (second === "other") { + await this.#removeRecord(); + return { ok: true }; + } + if (second === "unknown") + return inconclusive("could not be identified again, so it was not killed"); + + supervisor.signal(pid, "SIGKILL"); + if (await this.#waitGone(pid, KILL_GRACE_MS)) { + await this.#removeRecord(); + return { ok: true }; + } + return inconclusive("is still alive after SIGKILL"); + } + + async #identify( + pid: number, + realHome: string, + entry: string | undefined, + ): Promise<"own" | "other" | "unknown"> { + const line = await this.#psLine(pid); + if (line === undefined) return "unknown"; + if (!line.endsWith(` ${MARKER}${realHome}`)) return "other"; + if (entry !== undefined && !line.includes(entry)) return "other"; + if (await this.#stateDirIsOurs()) { + const named = (await this.#readDaemonJson())?.["pid"]; + if (named !== undefined && named !== pid) return "other"; + } + return "own"; + } + + async #psLine(pid: number): Promise { + try { + const raced = await this.#race( + this.#o.processRunner.run("ps", ["-ww", "-o", "args=", "-p", String(pid)], { + timeoutMs: PS_TIMEOUT_MS, + }), + PS_TIMEOUT_MS, + ); + if (raced.timedOut || raced.value.code !== 0) return undefined; + const line = raced.value.stdout.split("\n")[0]?.trim() ?? ""; + return line === "" ? undefined : line; + } catch { + return undefined; + } + } + + async #stateDirIsOurs(): Promise { + try { + const details = await this.#o.filesystem.lstat(this.#stateDir); + const uid = process.getuid?.(); + return details.kind === "directory" && (uid === undefined || details.uid === uid); + } catch { + return false; + } + } + + async #readRecord(): Promise { + try { + const parsed = JSON.parse( + await this.#o.filesystem.readFile(this.#recordPath), + ) as Partial; + return typeof parsed.pid === "number" && typeof parsed.entry === "string" + ? { entry: parsed.entry, pid: parsed.pid, startedAt: parsed.startedAt ?? 0 } + : undefined; + } catch { + return undefined; + } + } + + async #removeRecord(): Promise { + await this.#o.filesystem.rm(this.#recordPath); + } + + // why: a waiter per poll on one never-settling promise would pile up reactions across retries. + async #untilStop(work: Promise): Promise { + if (this.#stopped) { + work.catch(() => undefined); + return "stopped"; + } + let wake!: () => void; + const stopped = new Promise<"stopped">((resolve) => { + wake = () => resolve("stopped"); + }); + this.#stopWaiters.add(wake); + try { + return await Promise.race([work, stopped]); + } finally { + this.#stopWaiters.delete(wake); + } + } + + #sleep(ms: number): Promise { + return new Promise((resolve) => { + this.#o.clock.setTimer(ms, resolve); + }); + } + + async #race(work: Promise, ms: number): Promise> { + let timer: TimerHandle | undefined; + const limit = new Promise>((resolve) => { + timer = this.#o.clock.setTimer(ms, () => resolve({ timedOut: true })); + }); + try { + return await Promise.race([ + work.then((value): Raced => ({ timedOut: false, value })), + limit, + ]); + } finally { + if (timer !== undefined) this.#o.clock.cancel(timer); + } + } +} diff --git a/src/agent-device/host/health.test.ts b/src/agent-device/host/health.test.ts new file mode 100644 index 00000000..ec8ca89b --- /dev/null +++ b/src/agent-device/host/health.test.ts @@ -0,0 +1,64 @@ +import { createServer, type Server } from "node:http"; +import type { AddressInfo } from "node:net"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { NodeAgentDeviceHealthProbe } from "./health.js"; + +const servers: Server[] = []; + +afterEach(async () => { + await Promise.all( + servers.splice(0).map( + (server) => + new Promise((resolve) => { + server.closeAllConnections(); + server.close(() => resolve()); + }), + ), + ); +}); + +async function serve( + handler: (url: string | undefined) => { readonly body: string } | "never", +): Promise<{ readonly baseUrl: string; readonly urls: string[] }> { + const urls: string[] = []; + const server = createServer((request, response) => { + urls.push(request.url ?? ""); + const answer = handler(request.url); + if (answer === "never") return; + response.setHeader("content-type", "application/json"); + response.end(answer.body); + }); + servers.push(server); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + return { baseUrl: `http://127.0.0.1:${(server.address() as AddressInfo).port}`, urls }; +} + +describe("NodeAgentDeviceHealthProbe", () => { + it("returns the JSON body of GET /health", async () => { + const { baseUrl, urls } = await serve(() => ({ + body: JSON.stringify({ rpcProtocolVersion: 2, service: "agent-device-daemon" }), + })); + + await expect(new NodeAgentDeviceHealthProbe().read(baseUrl)).resolves.toEqual({ + rpcProtocolVersion: 2, + service: "agent-device-daemon", + }); + expect(urls).toEqual(["/health"]); + }); + + it("rejects on a body that is not JSON", async () => { + const { baseUrl } = await serve(() => ({ body: "not json" })); + + await expect(new NodeAgentDeviceHealthProbe().read(baseUrl)).rejects.toThrow(); + }); + + it("rejects a server that never answers once its time limit passes", async () => { + const { baseUrl } = await serve(() => "never"); + const started = Date.now(); + + await expect(new NodeAgentDeviceHealthProbe(50).read(baseUrl)).rejects.toThrow(); + expect(Date.now() - started).toBeLessThan(2_000); + }); +}); diff --git a/src/agent-device/host/health.ts b/src/agent-device/host/health.ts new file mode 100644 index 00000000..4752006c --- /dev/null +++ b/src/agent-device/host/health.ts @@ -0,0 +1,16 @@ +export interface AgentDeviceHealthProbe { + read(baseUrl: string): Promise; +} + +const DEFAULT_HEALTH_TIMEOUT_MS = 5_000; + +export class NodeAgentDeviceHealthProbe implements AgentDeviceHealthProbe { + constructor(readonly timeoutMs: number = DEFAULT_HEALTH_TIMEOUT_MS) {} + + async read(baseUrl: string): Promise { + const response = await fetch(`${baseUrl}/health`, { + signal: AbortSignal.timeout(this.timeoutMs), + }); + return (await response.json()) as unknown; + } +} diff --git a/src/agent-device/host/install.ts b/src/agent-device/host/install.ts new file mode 100644 index 00000000..f6f9af62 --- /dev/null +++ b/src/agent-device/host/install.ts @@ -0,0 +1,82 @@ +import { dirname } from "node:path"; + +import { type Filesystem } from "../../ports/index.js"; + +export type AgentDeviceInstall = + | { readonly ok: true; readonly realPath: string; readonly entry: string } + | { + readonly ok: false; + readonly reason: "not-found" | "not-agent-device"; + readonly message: string; + }; + +async function isExecutableFile(filesystem: Filesystem, path: string): Promise { + try { + const details = await filesystem.stat(path); + return details.kind === "file" && ((details.mode ?? 0) & 0o111) !== 0; + } catch { + return false; + } +} + +async function locate( + filesystem: Filesystem, + command: string, + pathVariable: string | undefined, +): Promise { + if (command.includes("/")) { + return (await isExecutableFile(filesystem, command)) ? command : undefined; + } + for (const directory of (pathVariable ?? "").split(":")) { + if (directory === "") continue; + const candidate = `${directory}/${command}`; + if (await isExecutableFile(filesystem, candidate)) return candidate; + } + return undefined; +} + +export async function findAgentDevice( + filesystem: Filesystem, + command: string, + pathVariable: string | undefined, +): Promise { + const found = await locate(filesystem, command, pathVariable); + const notFound = { + ok: false, + reason: "not-found", + message: command.includes("/") + ? `agent-device not found: ${command} is not an executable file` + : `agent-device not found: "${command}" is not on PATH`, + } as const; + if (found === undefined) return notFound; + + let realPath: string; + try { + realPath = await filesystem.realpath(found); + } catch { + return notFound; + } + + const packageRoot = dirname(dirname(realPath)); + const notAgentDevice = (message: string) => + ({ ok: false, reason: "not-agent-device", message }) as const; + let packageName: unknown; + try { + packageName = ( + JSON.parse(await filesystem.readFile(`${packageRoot}/package.json`)) as { name?: unknown } + ).name; + } catch { + return notAgentDevice(`${packageRoot}/package.json is missing or unreadable`); + } + if (packageName !== "agent-device") { + return notAgentDevice(`${packageRoot}/package.json does not name agent-device`); + } + + for (const entry of [ + `${packageRoot}/dist/src/internal/daemon.js`, + `${packageRoot}/dist/src/daemon.js`, + ]) { + if (await filesystem.exists(entry)) return { ok: true, realPath, entry }; + } + return notAgentDevice(`${packageRoot} has no agent-device daemon entry under dist/src`); +} diff --git a/src/agent-device/index.ts b/src/agent-device/index.ts new file mode 100644 index 00000000..15db789b --- /dev/null +++ b/src/agent-device/index.ts @@ -0,0 +1,4 @@ +export { AgentDeviceSupervisor } from "./host/agent-device-supervisor.js"; +// fallow-ignore-next-line unused-type -- the availability contract the front (#472) and doctor (#471) read. +export type { AgentDeviceAvailability } from "./host/agent-device-supervisor.js"; +export { type AgentDeviceHealthProbe, NodeAgentDeviceHealthProbe } from "./host/health.js"; diff --git a/src/agent-device/shared/compatibility.ts b/src/agent-device/shared/compatibility.ts new file mode 100644 index 00000000..9bc589a8 --- /dev/null +++ b/src/agent-device/shared/compatibility.ts @@ -0,0 +1,20 @@ +export const MIN_AGENT_DEVICE_VERSION = "0.21.25"; +export const AGENT_DEVICE_RPC_PROTOCOL_VERSION = 2; +export const MIN_NODE_VERSION = "22.12.0"; + +function numbersOf(version: string): readonly [number, number, number] | undefined { + const match = /^(\d+)\.(\d+)\.(\d+)/.exec(version.trim()); + if (match === null) return undefined; + return [Number(match[1]), Number(match[2]), Number(match[3])]; +} + +export function isVersionAtLeast(version: string, minimum: string): boolean { + const found = numbersOf(version); + const needed = numbersOf(minimum); + if (found === undefined || needed === undefined) return false; + for (let index = 0; index < 3; index += 1) { + const difference = (found[index] ?? 0) - (needed[index] ?? 0); + if (difference !== 0) return difference > 0; + } + return true; +} diff --git a/src/agent-device/shared/daemon-environment.ts b/src/agent-device/shared/daemon-environment.ts new file mode 100644 index 00000000..7a062751 --- /dev/null +++ b/src/agent-device/shared/daemon-environment.ts @@ -0,0 +1,28 @@ +const DAY_MS = 24 * 60 * 60_000; + +export function agentDeviceLeaseMaxTtlMs(leaseMaxTtlMs: number): number { + return Math.max(DAY_MS, 2 * leaseMaxTtlMs); +} + +export const AGENT_DEVICE_POLICY = { + version: 1, + commands: { deny: ["boot", "shutdown"] }, + capabilities: { deny: ["device-shutdown"] }, +}; + +export const NO_ANDROID_DEVICE_SERIAL = "simlock-no-android-device"; + +export function agentDeviceDaemonEnvironment(paths: { + readonly stateDir: string; + readonly policyPath: string; + readonly leaseMaxTtlMs: number; +}): Readonly> { + return { + AGENT_DEVICE_STATE_DIR: paths.stateDir, + AGENT_DEVICE_CLAIMS_DIR: `${paths.stateDir}/device-claims`, + AGENT_DEVICE_DAEMON_SERVER_MODE: "http", + AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: "0", + AGENT_DEVICE_DAEMON_POLICY: paths.policyPath, + AGENT_DEVICE_LEASE_MAX_TTL_MS: String(agentDeviceLeaseMaxTtlMs(paths.leaseMaxTtlMs)), + }; +} diff --git a/src/agent-device/testing.ts b/src/agent-device/testing.ts new file mode 100644 index 00000000..a80d577d --- /dev/null +++ b/src/agent-device/testing.ts @@ -0,0 +1,24 @@ +import type { AgentDeviceHealthProbe } from "./host/health.js"; + +type Answer = { readonly body: unknown } | { readonly error: Error } | "never"; + +export class FakeAgentDeviceHealth implements AgentDeviceHealthProbe { + readonly reads: string[] = []; + #answer: Answer = "never"; + + respondWith(body: unknown): void { + this.#answer = { body }; + } + + rejectWith(error: Error): void { + this.#answer = { error }; + } + + // fallow-ignore-next-line unused-class-member -- reached through the AgentDeviceHealthProbe port by the supervisor. + async read(baseUrl: string): Promise { + this.reads.push(baseUrl); + if (this.#answer === "never") return new Promise(() => undefined); + if ("error" in this.#answer) throw this.#answer.error; + return this.#answer.body; + } +} diff --git a/src/cli/index.test.ts b/src/cli/index.test.ts index 847bcb8c..625e1955 100644 --- a/src/cli/index.test.ts +++ b/src/cli/index.test.ts @@ -5812,6 +5812,7 @@ async function startInMemoryDaemon(options: { function testConfig(): Config { return { mode: "worker", + agentDevice: { command: "agent-device", enabled: false }, exec: { timeoutMs: 600_000 }, diskPressure: { freeBytesThreshold: 10 * gibibyte }, gateway: { diff --git a/src/contract/operations.test.ts b/src/contract/operations.test.ts index 87442773..e4434710 100644 --- a/src/contract/operations.test.ts +++ b/src/contract/operations.test.ts @@ -1091,6 +1091,7 @@ describe("operation input/output round trips", () => { defaultModels: {}, emulator: { headless: true, gpu: "host", audio: false, bootAnimation: false }, }, + agentDevice: { command: "agent-device", enabled: false }, stalledTransition: { thresholdMultiplier: 1, minimumThresholdMs: 1 }, }; expect(OPERATIONS["config.get"].output.parse(config)).toBeDefined(); @@ -1147,6 +1148,7 @@ describe("operation input/output round trips", () => { defaultModels: {}, emulator: { headless: true, gpu: "host", audio: false, bootAnimation: false }, }, + agentDevice: { command: "agent-device", enabled: false }, stalledTransition: { thresholdMultiplier: 1, minimumThresholdMs: 1 }, }; const config = (pool: object) => ({ diff --git a/src/contract/protocol.ts b/src/contract/protocol.ts index dd7ef7f8..c5c03604 100644 --- a/src/contract/protocol.ts +++ b/src/contract/protocol.ts @@ -10,7 +10,7 @@ export interface ProtocolRange { const protocolRangeSchema = z.object({ min: z.number().int(), max: z.number().int() }); /** Widen only once a second version is kept alive side by side with the first; a bump that leaves no shim behind moves both ends. */ -export const PROTOCOL_VERSION_RANGE: ProtocolRange = { min: 29, max: 29 }; +export const PROTOCOL_VERSION_RANGE: ProtocolRange = { min: 30, max: 30 }; const LEGACY_DAEMON_PROTOCOL_VERSION = 2; diff --git a/src/contract/schemas.ts b/src/contract/schemas.ts index 374160ab..0cb10788 100644 --- a/src/contract/schemas.ts +++ b/src/contract/schemas.ts @@ -840,6 +840,7 @@ export const configSchema = z.object({ host: z.string(), port: z.number(), }), + agentDevice: z.object({ enabled: z.boolean(), command: z.string() }), health: z.object({ enabled: z.boolean(), probeIntervalMs: z.number(), diff --git a/src/core/cleanup/idle-destroy.test.ts b/src/core/cleanup/idle-destroy.test.ts index 2bbe9778..260b3071 100644 --- a/src/core/cleanup/idle-destroy.test.ts +++ b/src/core/cleanup/idle-destroy.test.ts @@ -7,6 +7,7 @@ const gibibyte = 1024 ** 3; const config: Config = { mode: "worker", + agentDevice: { command: "agent-device", enabled: false }, exec: { timeoutMs: 600_000 }, diskPressure: { freeBytesThreshold: 0 }, gateway: { diff --git a/src/core/cleanup/idle-shutdown.test.ts b/src/core/cleanup/idle-shutdown.test.ts index 5d6fd88e..911c9138 100644 --- a/src/core/cleanup/idle-shutdown.test.ts +++ b/src/core/cleanup/idle-shutdown.test.ts @@ -5,6 +5,7 @@ import { idleShutdownRule } from "./idle-shutdown.js"; const config: Config = { mode: "worker", + agentDevice: { command: "agent-device", enabled: false }, exec: { timeoutMs: 600_000 }, diskPressure: { freeBytesThreshold: 0 }, gateway: { diff --git a/src/core/config.test.ts b/src/core/config.test.ts index 4deccb7d..1ad35306 100644 --- a/src/core/config.test.ts +++ b/src/core/config.test.ts @@ -1939,3 +1939,67 @@ describe("configSchema agrees with the config the daemon actually loads", () => ); }); }); + +describe("agentDevice config", () => { + async function loadWith(contents: unknown, warn?: (message: string) => void) { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic(configPath, JSON.stringify(contents)); + return loadConfig({ + configPath, + filesystem, + systemStats: createStats(), + ...(warn === undefined ? {} : { warn }), + }); + } + + it("defaults agentDevice.enabled to false and agentDevice.command to agent-device", async () => { + const config = await loadWith({}); + + expect(dottedValue(config, "agentDevice.enabled")).toBe(false); + expect(dottedValue(config, "agentDevice.command")).toBe("agent-device"); + }); + + it("keeps an explicit agentDevice block", async () => { + const config = await loadWith({ + agentDevice: { command: "/opt/ad/bin/agent-device", enabled: true }, + }); + + expect(dottedValue(config, "agentDevice.enabled")).toBe(true); + expect(dottedValue(config, "agentDevice.command")).toBe("/opt/ad/bin/agent-device"); + }); + + it("refuses a non-boolean agentDevice.enabled, naming the key", async () => { + await expect(loadWith({ agentDevice: { enabled: true } })).resolves.toBeDefined(); + + await expect(loadWith({ agentDevice: { enabled: "yes" } })).rejects.toThrow( + "agentDevice.enabled", + ); + }); + + it.each([ + ["an empty string", ""], + ["a number", 7], + ])("refuses agentDevice.command as %s, naming the key", async (_case, command) => { + await expect(loadWith({ agentDevice: { command: "agent-device" } })).resolves.toBeDefined(); + + await expect(loadWith({ agentDevice: { command } })).rejects.toThrow("agentDevice.command"); + }); + + it("a loaded config parsed through configSchema keeps both keys", async () => { + const config = await loadWith({ agentDevice: { command: "ad", enabled: true } }); + + const parsed = configSchema.parse(config) as unknown as Config; + expect(dottedValue(parsed, "agentDevice.enabled")).toBe(true); + expect(dottedValue(parsed, "agentDevice.command")).toBe("ad"); + }); + + it("loads a gateway config with agentDevice and warns that the key is worker-only", async () => { + const warn = vi.fn(); + + const config = await loadWith({ agentDevice: { enabled: true }, mode: "gateway" }, warn); + + expect(warn).toHaveBeenCalledWith(expect.stringContaining('Ignoring "agentDevice"')); + expect(dottedValue(config, "agentDevice.enabled")).toBe(true); + }); +}); diff --git a/src/core/config.ts b/src/core/config.ts index 00bc45f3..604044d2 100644 --- a/src/core/config.ts +++ b/src/core/config.ts @@ -250,6 +250,10 @@ export interface Config { readonly bootAnimation: boolean; }; }; + readonly agentDevice: { + readonly enabled: boolean; + readonly command: string; + }; /** * ADR 0005 §3/§6. Both sides of the fleet live under one key, and which half is read depends * on `mode`: @@ -718,6 +722,7 @@ function defaultConfig( bootAnimation: true, }, }, + agentDevice: { enabled: false, command: "agent-device" }, gateway: { disconnectedRetentionMs: DEFAULT_DISCONNECTED_RETENTION_MS, execTimeoutMs: DEFAULT_GATEWAY_EXEC_TIMEOUT_MS, @@ -870,6 +875,7 @@ function configValidators(strategy: CapacityStrategyName): Record = {}): Config { return { mode: "worker", + agentDevice: { command: "agent-device", enabled: false }, exec: { timeoutMs: 600_000 }, diskPressure: { freeBytesThreshold: 1 }, gateway: { diff --git a/src/core/driver.ts b/src/core/driver.ts index f579b0d0..035887da 100644 --- a/src/core/driver.ts +++ b/src/core/driver.ts @@ -349,6 +349,7 @@ export interface Driver { leaseEnvironment( device: VirtualDriverDevice | PhysicalDriverDevice, ): Readonly>; + virtualLeaseEnvironment(): Readonly>; /** Android must reap the adb server it supervises by pid; nothing else can. */ dispose?(): Promise; /** Spelled `| undefined` so a driver that decides at construction can still say it has none. */ diff --git a/src/core/fake-driver.test.ts b/src/core/fake-driver.test.ts index a31a824c..86c51721 100644 --- a/src/core/fake-driver.test.ts +++ b/src/core/fake-driver.test.ts @@ -254,6 +254,19 @@ describe("FakeDriver", () => { }); }); + it("answers virtualLeaseEnvironment with its configured lease environment, and leaseEnvironment for a virtual device equals it", () => { + const driver = new FakeDriver({ + clock: new FakeClock(), + leaseEnvironment: { SIMLOCK_IOS_DEVICE_SET: "/tmp/devices/ios" }, + platform: "ios", + }); + + expect(driver.virtualLeaseEnvironment()).toEqual({ + SIMLOCK_IOS_DEVICE_SET: "/tmp/devices/ios", + }); + expect(driver.leaseEnvironment(VIRTUAL_DEVICE)).toEqual(driver.virtualLeaseEnvironment()); + }); + it("reports a staged reality entry whatever it is named, the way a root does", () => { const driver = new FakeDriver({ clock: new FakeClock(), platform: "ios" }); driver.setManagedReality({ diff --git a/src/core/fake-driver.ts b/src/core/fake-driver.ts index 662ea106..dcb9d55d 100644 --- a/src/core/fake-driver.ts +++ b/src/core/fake-driver.ts @@ -635,6 +635,10 @@ export class FakeDriver implements Driver { return this.#estimateMs?.reclaim ?? 0; } + virtualLeaseEnvironment(): Readonly> { + return this.#leaseEnvironment; + } + leaseEnvironment(_device: DriverDevice): Readonly> { return this.#leaseEnvironment; } diff --git a/src/core/nuke.test.ts b/src/core/nuke.test.ts index 3153ca02..b13333d4 100644 --- a/src/core/nuke.test.ts +++ b/src/core/nuke.test.ts @@ -157,6 +157,7 @@ function sequence() { function config(): Config { return { mode: "worker", + agentDevice: { command: "agent-device", enabled: false }, exec: { timeoutMs: 600_000 }, diskPressure: { freeBytesThreshold: 1 }, gateway: { diff --git a/src/core/reaper.test.ts b/src/core/reaper.test.ts index 600fe63b..665184ac 100644 --- a/src/core/reaper.test.ts +++ b/src/core/reaper.test.ts @@ -53,6 +53,7 @@ class MutableFreeDiskFilesystem extends MemoryFilesystem { function config(): Config { return { mode: "worker", + agentDevice: { command: "agent-device", enabled: false }, exec: { timeoutMs: 600_000 }, diskPressure: { freeBytesThreshold: 10 * gibibyte }, gateway: { diff --git a/src/daemon-protocol/index.test.ts b/src/daemon-protocol/index.test.ts index 6b370620..78adef7f 100644 --- a/src/daemon-protocol/index.test.ts +++ b/src/daemon-protocol/index.test.ts @@ -9,7 +9,7 @@ import { describe("daemon protocol", () => { it("keeps the current protocol version and newline framing", () => { - expect(DAEMON_PROTOCOL_VERSION).toBe(29); + expect(DAEMON_PROTOCOL_VERSION).toBe(30); expect(serializeFrame({ id: 1, type: "hello" })).toBe('{"id":1,"type":"hello"}\n'); }); diff --git a/src/daemon/dispatcher.test.ts b/src/daemon/dispatcher.test.ts index ee2bf2f6..0dd4feb5 100644 --- a/src/daemon/dispatcher.test.ts +++ b/src/daemon/dispatcher.test.ts @@ -3547,6 +3547,7 @@ function testConfig( ): Config { return { mode: "worker", + agentDevice: { command: "agent-device", enabled: false }, gateway: { disconnectedRetentionMs: 24 * 60 * 60_000, execTimeoutMs: 11 * 60_000, diff --git a/src/daemon/main.test.ts b/src/daemon/main.test.ts index f386133a..f073a25a 100644 --- a/src/daemon/main.test.ts +++ b/src/daemon/main.test.ts @@ -14,6 +14,7 @@ import { REDACTED_VALUE, Registry, } from "../core/index.js"; +import { FakeAgentDeviceHealth } from "../agent-device/testing.js"; import { FakeDriver } from "../core/testing.js"; import { IosSimctlDriver } from "../drivers/ios/index.js"; import { EventBus } from "../bus/index.js"; @@ -94,7 +95,7 @@ describe("the event file sink", () => { logger: new JsonLinesLogger({ clock, level: "debug", sink: new MemoryLogSink() }), statePath: join(directory, "state.json"), version: "1.2.3", - } as StartDaemonOptions); + }); runningDaemons.push(daemon); expect(await readFile(join(directory, "events.jsonl.1"), "utf8")).toBe(`{"timestamp":500}\n`); @@ -670,7 +671,7 @@ describe("startDaemon wires core and leasing together", () => { logger: new JsonLinesLogger({ clock, level: "debug", sink: new MemoryLogSink() }), statePath, version: "1.2.3", - } as StartDaemonOptions); + }); runningDaemons.push(daemon); const names = (await readFile(join(directory, "events.jsonl"), "utf8")) @@ -991,7 +992,7 @@ describe("startDaemon stopped while starting", () => { socketPath, statePath: join(directory, "state.json"), version: "1.2.3", - } as StartDaemonOptions); + }); const client = await connectRetrying(socketPath); try { @@ -1107,7 +1108,7 @@ describe("startDaemon stop with a warm target", () => { logger: new JsonLinesLogger({ clock, level: "debug", sink: new MemoryLogSink() }), statePath: join(directory, "state.json"), version: "1.2.3", - } as StartDaemonOptions); + }); runningDaemons.push(daemon); await expect .poll(() => driver.calls.filter((call) => call.operation === "makeReady").length) @@ -2308,3 +2309,235 @@ async function pollUntil(condition: () => boolean, timeoutMs = 2_000): Promise setTimeout(resolve, 5)); } } + +describe("startDaemon with the agent-device edge", () => { + const packageRoot = "/opt/ad/node_modules/agent-device"; + const realCli = `${packageRoot}/bin/agent-device.mjs`; + const entry = `${packageRoot}/dist/src/internal/daemon.js`; + const command = "/opt/ad/bin/agent-device"; + const lease = { + model: "iPhone 16", + osVersion: "26.5", + platform: "ios", + requesterId: "agent-1", + } as const; + const agent = { + manageEventSubscription: () => undefined, + principal: "agent", + role: "agent", + } as const; + + class KillingSupervisor extends FakeProcessSupervisor { + readonly kills = new Map void>(); + + override signal(pid: number, signal: NodeJS.Signals): void { + super.signal(pid, signal); + this.markDead(pid); + this.kills.get(pid)?.(); + } + } + + async function setUp( + config: unknown, + options: { readonly directory?: string; readonly record?: { readonly pid: number } } = {}, + ) { + const directory = options.directory ?? (await mkdtemp(join(tmpdir(), "simlock-main-ad-"))); + temporaryDirectories.push(directory); + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp(directory); + await filesystem.mkdirp("/opt/ad/bin"); + await filesystem.mkdirp(`${packageRoot}/bin`); + await filesystem.mkdirp(`${packageRoot}/dist/src/internal`); + await filesystem.writeFileAtomic(realCli, "#!/usr/bin/env node\n"); + filesystem.defineAttributes(realCli, { mode: 0o755 }); + await filesystem.writeFileAtomic(`${packageRoot}/package.json`, '{"name":"agent-device"}'); + await filesystem.writeFileAtomic(entry, ""); + filesystem.defineSymlink(command, realCli); + await filesystem.writeFileAtomic(join(directory, "config.json"), JSON.stringify(config)); + if (options.record !== undefined) { + await filesystem.writeFileAtomic( + join(directory, "agent-device-daemon.json"), + JSON.stringify({ entry, pid: options.record.pid, startedAt: 1 }), + ); + } + return { directory, filesystem }; + } + + function versionAndDaemon(directory: string): ScriptedProcessRunner { + return new ScriptedProcessRunner([ + { + match: { args: [realCli, "--version"], command: process.execPath }, + result: { code: 0, stderr: "", stdout: "0.21.25-dev\n" }, + }, + { + hangs: true, + match: { args: [entry, `--simlock-home=${directory}`], command: process.execPath }, + }, + ]); + } + + async function startEdge( + setup: Awaited>, + ports: { + readonly drivers?: StartDaemonOptions["drivers"]; + readonly processRunner: ScriptedProcessRunner; + readonly processSupervisor: FakeProcessSupervisor; + }, + ) { + const clock = new FakeClock(1_000); + const sink = new MemoryLogSink(); + const daemon = await startDaemon({ + agentDeviceHealth: new FakeAgentDeviceHealth(), + clock, + dataDirectory: setup.directory, + drivers: ports.drivers ?? [ + new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }), + ], + filesystem: setup.filesystem, + hostInfo: new FakeHostInfo({ arch: "arm64", os: "macOS", osVersion: "15.5" }), + logger: new JsonLinesLogger({ clock, level: "debug", sink }), + processRunner: ports.processRunner, + processSupervisor: ports.processSupervisor, + statePath: join(setup.directory, "state.json"), + tcpProbe: new FakeTcpProbe(), + version: "1.2.3", + }); + runningDaemons.push(daemon); + return { daemon, sink }; + } + + it("resolves and grants a lease to an ordinary client while agent-device's /health never answers, and stops the child before the drivers are disposed", async () => { + const setup = await setUp({ agentDevice: { command, enabled: true } }); + const events: string[] = []; + const processSupervisor = new KillingSupervisor(); + const runner = versionAndDaemon(setup.directory); + const spawn = runner.spawn.bind(runner); + vi.spyOn(runner, "spawn").mockImplementation((spawned, args, options) => { + const handle = spawn(spawned, args, options); + if (args[0] === entry) { + processSupervisor.markAlive(handle.pid); + processSupervisor.kills.set(handle.pid, () => handle.kill()); + void handle.wait().then(() => events.push("agent-device exited")); + } + return handle; + }); + const clock = new FakeClock(1_000); + class DisposingDriver extends FakeDriver { + async dispose(): Promise { + events.push("driver disposed"); + } + } + + const { daemon } = await startEdge(setup, { + drivers: [new DisposingDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" })], + processRunner: runner, + processSupervisor, + }); + await pollUntil(() => runner.calls.length === 2); + + expect(runner.calls.map((call) => call.args[0])).toEqual([realCli, entry]); + await expect(daemon.dispatch("lease.request", lease, agent)).resolves.toMatchObject({ + lease: { id: expect.any(String) }, + }); + + await daemon.stop("test"); + + expect(events).toEqual(["agent-device exited", "driver disposed"]); + }); + + async function spawnedEnvironment(drivers: (clock: FakeClock) => StartDaemonOptions["drivers"]) { + const setup = await setUp({ agentDevice: { command, enabled: true } }); + const runner = versionAndDaemon(setup.directory); + await startEdge(setup, { + drivers: drivers(new FakeClock(1_000)), + processRunner: runner, + processSupervisor: new FakeProcessSupervisor(), + }); + await pollUntil(() => runner.calls.length === 2); + return runner.calls[1]?.options.env ?? {}; + } + + it("hands the drivers' lease environment to the agent-device daemon", async () => { + const env = await spawnedEnvironment((clock) => [ + new FakeDriver({ + availableOsVersions: ["26.5"], + clock, + leaseEnvironment: { SIMLOCK_IOS_DEVICE_SET: "/sets/ios", SIMLOCK_TEST_MARKER: "kept" }, + platform: "ios", + }), + ]); + + expect(env["SIMLOCK_TEST_MARKER"]).toBe("kept"); + expect(env["AGENT_DEVICE_IOS_SIMULATOR_DEVICE_SET"]).toBe("/sets/ios"); + }); + + it("hides a platform whose driver is off from the agent-device daemon, and only that one", async () => { + const iosOnly = await spawnedEnvironment((clock) => [ + new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }), + ]); + const both = await spawnedEnvironment((clock) => [ + new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }), + new FakeDriver({ availableOsVersions: ["14"], clock, platform: "android" }), + ]); + + expect(iosOnly["AGENT_DEVICE_ANDROID_DEVICE_ALLOWLIST"]).toBe("simlock-no-android-device"); + expect(both["AGENT_DEVICE_ANDROID_DEVICE_ALLOWLIST"]).toBeUndefined(); + }); + + it("still starts and grants a lease when agent-device is not found", async () => { + const setup = await setUp({ + agentDevice: { command: "/opt/missing/agent-device", enabled: true }, + }); + const runner = new ScriptedProcessRunner([]); + + const { daemon } = await startEdge(setup, { + processRunner: runner, + processSupervisor: new FakeProcessSupervisor(), + }); + await pollUntil(() => false, 100); + + await expect(daemon.dispatch("lease.request", lease, agent)).resolves.toMatchObject({ + lease: { id: expect.any(String) }, + }); + expect(runner.calls).toEqual([]); + }); + + it("builds no supervisor in gateway mode: no ps and no agent-device process run, and no signal is sent", async () => { + const setup = await setUp( + { + agentDevice: { command, enabled: true }, + http: { enabled: true, host: "127.0.0.1", port: await freeLoopbackPort() }, + mode: "gateway", + }, + { record: { pid: 900 } }, + ); + const runner = new ScriptedProcessRunner([]); + const processSupervisor = new FakeProcessSupervisor([900]); + + await startEdge(setup, { drivers: [], processRunner: runner, processSupervisor }); + await pollUntil(() => false, 100); + + expect(runner.calls).toEqual([]); + expect(processSupervisor.signals).toEqual([]); + }); + + it("runs no ps and sends no signal in a second daemon on the same home that loses the socket claim", async () => { + const first = await setUp({ agentDevice: { enabled: false } }); + await startEdge(first, { + processRunner: new ScriptedProcessRunner([]), + processSupervisor: new FakeProcessSupervisor(), + }); + const second = await setUp( + { agentDevice: { command, enabled: true } }, + { directory: first.directory, record: { pid: 900 } }, + ); + const runner = new ScriptedProcessRunner([]); + const processSupervisor = new FakeProcessSupervisor([900]); + + await expect(startEdge(second, { processRunner: runner, processSupervisor })).rejects.toThrow(); + await pollUntil(() => false, 100); + + expect(runner.calls).toEqual([]); + expect(processSupervisor.signals).toEqual([]); + }); +}); diff --git a/src/daemon/main.ts b/src/daemon/main.ts index 5906fd39..9db627d2 100644 --- a/src/daemon/main.ts +++ b/src/daemon/main.ts @@ -2,6 +2,11 @@ import { homedir } from "node:os"; import { join, resolve } from "node:path"; import { pathToFileURL } from "node:url"; +import { + AgentDeviceSupervisor, + type AgentDeviceHealthProbe, + NodeAgentDeviceHealthProbe, +} from "../agent-device/index.js"; import { EVENT_FILE_NAME, EventBus, EventHistory, type EventBusLogger } from "../bus/index.js"; import { fitHostFacts } from "../contract/index.js"; import { @@ -101,6 +106,7 @@ import { GatewayUplink } from "./gateway-uplink.js"; import { createCredentialRoleResolver } from "./session.js"; export interface StartDaemonOptions { + readonly agentDeviceHealth?: AgentDeviceHealthProbe; readonly clock?: Clock; readonly configOverrides?: ConfigOverrides; readonly configPath?: string; @@ -398,6 +404,26 @@ export async function startDaemon(options: StartDaemonOptions = {}): Promise driver.virtualLeaseEnvironment()), + ), + filesystem, + healthProbe: options.agentDeviceHealth ?? new NodeAgentDeviceHealthProbe(), + home: dataDirectory, + leaseMaxTtlMs: config.lease.maxTtlMs, + logger: logger.child("agent-device"), + nodePath: process.execPath, + nodeVersion: process.versions.node, + platforms: drivers.map((driver) => driver.platform), + processRunner, + processSupervisor, + tcpProbe, + }); const daemon = new DaemonServer({ // Closes the warm pool the moment a stop is asked for, ahead of every await in the stop. beginStop: () => core.closeWarmPool(), @@ -454,6 +480,9 @@ export async function startDaemon(options: StartDaemonOptions = {}): Promise driver.dispose?.())); for (const [index, disposal] of disposals.entries()) { if (disposal.status === "rejected") { @@ -492,6 +521,9 @@ export async function startDaemon(options: StartDaemonOptions = {}): Promise { socketClaimed = true; + // why: not awaited, since simlock's start never waits for agent-device, and a daemon that lost + // the socket race never gets here, so it reaps nothing. + void agentDevice.start(); // Dialled here, once the socket is claimed and `#readyPromise` is set. The gateway's first // `status.get` never parks (it still answers `starting` during convergence), but its // `events.subscribe` does park on startup readiness, so the refresh that follows the diff --git a/src/daemon/server.test.ts b/src/daemon/server.test.ts index 7687ec21..a68eee7a 100644 --- a/src/daemon/server.test.ts +++ b/src/daemon/server.test.ts @@ -191,8 +191,9 @@ describe("DaemonServer", () => { [26, "cannot read a physical device's presence"], [27, "cannot read config.get's lease.maxPerRequester"], [28, "cannot read config.get's lease.endWaitMs"], + [29, "cannot read config.get's agentDevice"], ])( - "answers PROTOCOL_VERSION_UNSUPPORTED, naming protocol 29, to a client on protocol %i, which %s", + "answers PROTOCOL_VERSION_UNSUPPORTED, naming protocol 30, to a client on protocol %i, which %s", async (version) => { const harness = await createHarness(); const previous = await createClient(harness.socketPath); @@ -202,7 +203,7 @@ describe("DaemonServer", () => { ).resolves.toMatchObject({ error: { code: "PROTOCOL_VERSION_UNSUPPORTED", - details: { client: { min: version, max: version }, daemon: { min: 29, max: 29 } }, + details: { client: { min: version, max: version }, daemon: { min: 30, max: 30 } }, }, ok: false, }); @@ -3400,6 +3401,7 @@ function testConfig( ): Config { return { mode: "worker", + agentDevice: { command: "agent-device", enabled: false }, exec: { timeoutMs: 600_000 }, diskPressure: { freeBytesThreshold: 10 * gibibyte }, gateway: { diff --git a/src/drivers/android/index.test.ts b/src/drivers/android/index.test.ts index ca5b8855..f3197596 100644 --- a/src/drivers/android/index.test.ts +++ b/src/drivers/android/index.test.ts @@ -1571,6 +1571,15 @@ describe("AndroidDriver", () => { expect(reality.processes).toEqual([expect.objectContaining({ deviceId: "simlock_running" })]); }); + it("answers virtualLeaseEnvironment with simlock's adb port, and leaseEnvironment for a virtual device equals it", async () => { + const driver = await createDriver(await androidFilesystem(), new ScriptedProcessRunner([])); + + expect(driver.virtualLeaseEnvironment()).toEqual({ + ANDROID_ADB_SERVER_PORT: String(adbServerPort), + }); + expect(driver.leaseEnvironment(VIRTUAL_DEVICE)).toEqual(driver.virtualLeaseEnvironment()); + }); + it("hands a lease holder of a physical device its serial and the default adb server", async () => { const driver = await createDriver(await androidFilesystem(), new ScriptedProcessRunner([])); diff --git a/src/drivers/android/index.ts b/src/drivers/android/index.ts index ff803833..661fb246 100644 --- a/src/drivers/android/index.ts +++ b/src/drivers/android/index.ts @@ -646,13 +646,17 @@ export class AndroidDriver implements Driver { }); } + virtualLeaseEnvironment(): Readonly> { + // why: without this their `adb` talks to the shared server, which cannot see the emulator. + return { ANDROID_ADB_SERVER_PORT: String(this.#adbServerPort) }; + } + leaseEnvironment(device: DriverDevice): Readonly> { // A physical device is on the default adb server, not Simlock's emulator server. if (isPhysicalDriverDevice(device)) { return { ANDROID_ADB_SERVER_PORT: "5037", ANDROID_SERIAL: device.deviceId }; } - // Without this their `adb` talks to the shared server, which cannot see the emulator. - return { ANDROID_ADB_SERVER_PORT: String(this.#adbServerPort) }; + return this.virtualLeaseEnvironment(); } readonly passthroughTools = ANDROID_PASSTHROUGH_TOOLS; diff --git a/src/drivers/ios/index.test.ts b/src/drivers/ios/index.test.ts index f914fcf8..213cdc79 100644 --- a/src/drivers/ios/index.test.ts +++ b/src/drivers/ios/index.test.ts @@ -1587,6 +1587,13 @@ describe("IosSimctlDriver", () => { expect(driver.leaseEnvironment(VIRTUAL_DEVICE)).toEqual({ SIMLOCK_IOS_DEVICE_SET: deviceRoot }); }); + it("answers virtualLeaseEnvironment with the device set, and leaseEnvironment for a virtual device equals it", async () => { + const driver = await createDriver(new ScriptedProcessRunner([])); + + expect(driver.virtualLeaseEnvironment()).toEqual({ SIMLOCK_IOS_DEVICE_SET: deviceRoot }); + expect(driver.leaseEnvironment(VIRTUAL_DEVICE)).toEqual(driver.virtualLeaseEnvironment()); + }); + it("hands a lease holder of a physical device its UDID and no simulator device set", async () => { const driver = await createDriver(new ScriptedProcessRunner([])); diff --git a/src/drivers/ios/index.ts b/src/drivers/ios/index.ts index 4b5715ae..176bc99e 100644 --- a/src/drivers/ios/index.ts +++ b/src/drivers/ios/index.ts @@ -1532,9 +1532,13 @@ export class IosSimctlDriver implements Driver { ]; } + virtualLeaseEnvironment(): Readonly> { + return { SIMLOCK_IOS_DEVICE_SET: this.#deviceRoot }; + } + leaseEnvironment(device: DriverDevice): Readonly> { if (isPhysicalDriverDevice(device)) return { SIMLOCK_DEVICE_UDID: device.deviceId }; - return { SIMLOCK_IOS_DEVICE_SET: this.#deviceRoot }; + return this.virtualLeaseEnvironment(); } readonly passthroughTools = IOS_PASSTHROUGH_TOOLS; diff --git a/src/gateway/dispatcher.test.ts b/src/gateway/dispatcher.test.ts index 8bd33e50..30156309 100644 --- a/src/gateway/dispatcher.test.ts +++ b/src/gateway/dispatcher.test.ts @@ -29,6 +29,7 @@ import { import { WorkerRegistry } from "./worker-registry.js"; const gatewayConfig = { + agentDevice: { command: "agent-device", enabled: false }, capacity: { strategy: "fixed" as const, config: { maxRunning: 4 } }, diskPressure: { freeBytesThreshold: 1 }, downloads: { acceptAndroidLicenses: false, policy: "on-request" as const, timeoutMs: 1 }, diff --git a/src/gateway/service.test.ts b/src/gateway/service.test.ts index 2cd948d1..9bdd4e99 100644 --- a/src/gateway/service.test.ts +++ b/src/gateway/service.test.ts @@ -1207,7 +1207,7 @@ describe("GatewayService", () => { min: PROTOCOL_VERSION_RANGE.max - 1, max: PROTOCOL_VERSION_RANGE.max - 1, }; - expect(PROTOCOL_VERSION_RANGE.max).toBe(29); + expect(PROTOCOL_VERSION_RANGE.max).toBe(30); expect(negotiateProtocolVersion(PROTOCOL_VERSION_RANGE, previous)).toBeUndefined(); const harness = fleet(); await harness.service.start(); diff --git a/src/http/test-fakes.ts b/src/http/test-fakes.ts index 522f22c6..c7e59543 100644 --- a/src/http/test-fakes.ts +++ b/src/http/test-fakes.ts @@ -32,6 +32,7 @@ export function testConfig( ): Config { return { mode, + agentDevice: { command: "agent-device", enabled: false }, gateway: { disconnectedRetentionMs: 24 * 60 * 60_000, execTimeoutMs: 11 * 60_000, diff --git a/src/leasing/acquisition-planner.test.ts b/src/leasing/acquisition-planner.test.ts index 849f2b79..01239b85 100644 --- a/src/leasing/acquisition-planner.test.ts +++ b/src/leasing/acquisition-planner.test.ts @@ -16,6 +16,7 @@ const gibibyte = 1024 ** 3; const spec = { physical: false, model: "iPhone 16", osVersion: "26.5", platform: "ios" } as const; const config: Config = { mode: "worker", + agentDevice: { command: "agent-device", enabled: false }, exec: { timeoutMs: 600_000 }, diskPressure: { freeBytesThreshold: 10 * gibibyte }, gateway: { diff --git a/src/leasing/create-leasing.test.ts b/src/leasing/create-leasing.test.ts index 9dca1a29..952fdcdb 100644 --- a/src/leasing/create-leasing.test.ts +++ b/src/leasing/create-leasing.test.ts @@ -45,6 +45,7 @@ function config( ): Config { return { mode: "worker", + agentDevice: { command: "agent-device", enabled: false }, exec: { timeoutMs: 600_000 }, diskPressure: { freeBytesThreshold: 10 * gibibyte }, gateway: { diff --git a/src/leasing/lease-acquisition-coordinator.test.ts b/src/leasing/lease-acquisition-coordinator.test.ts index a04d42b7..4bf73ec9 100644 --- a/src/leasing/lease-acquisition-coordinator.test.ts +++ b/src/leasing/lease-acquisition-coordinator.test.ts @@ -63,6 +63,7 @@ const request = { function config(maxDevices = 1, maxRunning = 1): Config { return { mode: "worker", + agentDevice: { command: "agent-device", enabled: false }, exec: { timeoutMs: 600_000 }, diskPressure: { freeBytesThreshold: 10 * gibibyte }, gateway: { diff --git a/src/leasing/lease-end-participants.test.ts b/src/leasing/lease-end-participants.test.ts index faca01d9..0d1b6930 100644 --- a/src/leasing/lease-end-participants.test.ts +++ b/src/leasing/lease-end-participants.test.ts @@ -112,6 +112,7 @@ function config(options: EngineOptions): Config { const ios = options.ios ?? { maxDevices: 1, maxRunning: 1 }; return { mode: "worker", + agentDevice: { command: "agent-device", enabled: false }, exec: { timeoutMs: 600_000 }, diskPressure: { freeBytesThreshold: 10 * gibibyte }, gateway: { diff --git a/src/leasing/lease-health-monitor.test.ts b/src/leasing/lease-health-monitor.test.ts index 34140b81..3ea64933 100644 --- a/src/leasing/lease-health-monitor.test.ts +++ b/src/leasing/lease-health-monitor.test.ts @@ -24,6 +24,7 @@ const spec = { physical: false, model: "iPhone 16", osVersion: "26.5", platform: function config(overrides: Partial = {}): Config { return { mode: "worker", + agentDevice: { command: "agent-device", enabled: false }, exec: { timeoutMs: 600_000 }, diskPressure: { freeBytesThreshold: 10 * gibibyte }, gateway: { diff --git a/src/leasing/physical-device-watcher.test.ts b/src/leasing/physical-device-watcher.test.ts index d38c426b..f02e17a1 100644 --- a/src/leasing/physical-device-watcher.test.ts +++ b/src/leasing/physical-device-watcher.test.ts @@ -46,6 +46,7 @@ function config(options: HarnessOptions): Config { const quarantineBackoffMs = options.quarantineBackoffMs ?? 30_000; return { mode: "worker", + agentDevice: { command: "agent-device", enabled: false }, exec: { timeoutMs: 600_000 }, diskPressure: { freeBytesThreshold: 10 * gibibyte }, gateway: { diff --git a/src/leasing/physical-leasing.test.ts b/src/leasing/physical-leasing.test.ts index f6525238..8f9b619c 100644 --- a/src/leasing/physical-leasing.test.ts +++ b/src/leasing/physical-leasing.test.ts @@ -41,6 +41,7 @@ const virtualRequest = { function config(leaseOverrides: object = {}): Config { return { mode: "worker", + agentDevice: { command: "agent-device", enabled: false }, exec: { timeoutMs: 600_000 }, diskPressure: { freeBytesThreshold: 10 * gibibyte }, gateway: { diff --git a/src/lint-rules.test.ts b/src/lint-rules.test.ts index 38a6c2ec..3fc30492 100644 --- a/src/lint-rules.test.ts +++ b/src/lint-rules.test.ts @@ -278,4 +278,51 @@ describe("import rules enforced by pnpm lint", () => { expectLintPass("src/daemon/main.ts", `import { x } from "../leasing/index.js";\n`); expectLintError("src/daemon/main.ts", `import { x } from "../leasing/wait-queue.js";\n`); }); + + it("fails a file under core, leasing, drivers/ios or drivers/android importing agent-device's index", () => { + expectLintError("src/core/x.ts", `import { x } from "../agent-device/index.js";\n`); + expectLintError("src/leasing/x.ts", `import { x } from "../agent-device/index.js";\n`); + expectLintError("src/drivers/ios/x.ts", `import { x } from "../../agent-device/index.js";\n`); + expectLintError( + "src/drivers/android/x.ts", + `import { x } from "../../agent-device/index.js";\n`, + ); + }); + + it("fails a daemon file importing agent-device's supervisor file, and passes its index", () => { + expectLintError( + "src/daemon/x.ts", + `import { x } from "../agent-device/host/agent-device-supervisor.js";\n`, + ); + expectLintPass("src/daemon/x.ts", `import { x } from "../agent-device/index.js";\n`); + }); + + it("passes agent-device's testing file in a daemon test file and in agent-device's own test, and fails it in a daemon file", () => { + expectLintPass("src/daemon/x.test.ts", `import { x } from "../agent-device/testing.js";\n`); + expectLintPass("src/agent-device/host/x.test.ts", `import { x } from "../testing.js";\n`); + expectLintError("src/daemon/x.ts", `import { x } from "../agent-device/testing.js";\n`); + }); + + it.each([ + "src/agent-device/host/x.ts", + "src/agent-device/shared/x.ts", + "src/agent-device/index.ts", + ])( + "fails %s importing fs, node:fs, node:fs/promises, child_process or node:child_process", + (path) => { + for (const module of [ + "fs", + "node:fs", + "node:fs/promises", + "child_process", + "node:child_process", + ]) { + expectLintError(path, `import { x } from "${module}";\n`); + } + }, + ); + + it("passes an agent-device test file importing node:fs", () => { + expectLintPass("src/agent-device/host/x.test.ts", `import { x } from "node:fs";\n`); + }); }); diff --git a/src/ports/filesystem.ts b/src/ports/filesystem.ts index 8bfae698..ef9a9859 100644 --- a/src/ports/filesystem.ts +++ b/src/ports/filesystem.ts @@ -298,6 +298,7 @@ export class MemoryFilesystem implements Filesystem { options?: WriteFileAtomicOptions, ): Promise { this.#failIfDefined(path); + path = this.#underResolvedParent(path); this.#requireDirectory(parentPath(path)); this.#entries.set(path, { ...this.#newEntry({ kind: "file", contents }), @@ -307,6 +308,7 @@ export class MemoryFilesystem implements Filesystem { async writeFileExclusive(path: string, contents: string): Promise { this.#failIfDefined(path); + path = this.#underResolvedParent(path); if (this.#entries.has(path)) { throw errnoError("EEXIST", `File already exists: ${path}`); @@ -411,6 +413,7 @@ export class MemoryFilesystem implements Filesystem { async mkdir(path: string, options: { readonly mode?: number } = {}): Promise { this.#failIfDefined(path); + path = this.#underResolvedParent(path); if (this.#entries.has(path)) { throw errnoError("EEXIST", `File already exists: ${path}`); @@ -510,7 +513,6 @@ export class MemoryFilesystem implements Filesystem { } /** Test-only: rewrites an existing entry's ownership or permission bits. */ - // fallow-ignore-next-line unused-class-member -- test-only state the port itself cannot create. defineAttributes( path: string, attributes: { readonly uid?: number; readonly mode?: number }, @@ -589,6 +591,13 @@ export class MemoryFilesystem implements Filesystem { return entry?.kind === "symlink" ? this.#resolveLinks(entry.target, hops + 1) : resolved; } + #underResolvedParent(path: string): string { + return joinMemoryPath( + this.#resolveLinks(parentPath(path), 0), + path.slice(path.lastIndexOf("/") + 1), + ); + } + #requireDirectory(path: string): void { const entry = this.#entryAt(path); if (entry.kind !== "directory") {