Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 11 additions & 5 deletions packages/core/src/runner/runner.ts
Original file line number Diff line number Diff line change
Expand Up @@ -413,7 +413,10 @@ interface Job {
timedOut: boolean;
/** The watchdog stopped it: its report (the run's error). */
stalled?: string;
/** Last line Claude Code wrote, and when the process started (ms) — what the watchdog looks at. */
/**
* Last line Claude Code wrote, and when the process started — what the watchdog looks at. On the awake clock
* (`performance.now()`, which stops while the computer sleeps): time asleep is no stall.
*/
lastOutputAt?: number;
spawnedAt?: number;
lastLabel: string;
Expand Down Expand Up @@ -478,7 +481,10 @@ export function activeRunForConversation(conversationId: string): string | null
return queued;
}

/** A running run as the watchdog sees it. `waiting`: it stands still on purpose (a pause, the usage limit, compacting). */
/**
* A running run as the watchdog sees it, its times on the awake clock (`performance.now()`). `waiting`: it stands still
* on purpose (a pause, the usage limit, compacting).
*/
export interface WatchedRun {
runId: string;
agentId: string;
Expand Down Expand Up @@ -1743,8 +1749,8 @@ async function spawnClaude(
throw err;
}
job.proc = proc;
job.spawnedAt ??= Date.now();
job.lastOutputAt = Date.now();
job.spawnedAt ??= performance.now();
job.lastOutputAt = performance.now();
if (halted(job)) killTree(proc);
try {
proc.stdin.write(prompt);
Expand All @@ -1755,7 +1761,7 @@ async function spawnClaude(
const stderrP = readTail(proc.stderr, STDERR_TAIL_BYTES);
const tail = tailLines(outPath, proc.exited, (raw) => {
const line = raw.trim();
job.lastOutputAt = Date.now();
job.lastOutputAt = performance.now();
if (!line) return;
let event: unknown;
try {
Expand Down
4 changes: 2 additions & 2 deletions packages/core/src/runner/watchdog.ts
Original file line number Diff line number Diff line change
Expand Up @@ -199,8 +199,8 @@ function intervene(run: WatchedRun, verdict: Verdict): void {
}
}

/** One look at every running run. */
export function checkRuns(at = Date.now()): number {
/** One look at every running run. `at` is on the awake clock: the time the computer slept doesn't count as a stall. */
export function checkRuns(at = performance.now()): number {
const limits = watchdogLimits();
if (!limits) return 0;
let stopped = 0;
Expand Down
3 changes: 2 additions & 1 deletion packages/core/src/services/power.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ const SHORT_LIMIT_MS = 60 * 60_000;
/** On quit: how long a switch under way may finish before sleep is given back regardless. */
const STOP_WAIT_MS = 1_000;
const RELEASE_FAILED = "Godmode couldn't let the Mac sleep again. Click \"Let the Mac sleep\" in Settings → General and enter your Mac password.";
const HOLD_FAILED = "Godmode is no longer allowed to keep the Mac awake. Turn lid-closed mode off and on again.";
const HOLD_FAILED = "Godmode is no longer allowed to keep the Mac awake, so it sleeps when the lid closes. Turn lid-closed mode off and on again in Settings → General and enter your Mac password once.";

export interface Exec {
code: number;
Expand Down Expand Up @@ -233,6 +233,7 @@ function failed(message: string) {
if (lastError === message) return;
lastError = message;
if (message === RELEASE_FAILED) notify("error", "Your Mac can't sleep with the lid closed", message, "/settings/general");
else notify("warning", "Your Mac sleeps with the lid closed", message, "/settings/general");
}

/** Bring `disablesleep` in line with the setting, the runs and the battery. */
Expand Down
7 changes: 4 additions & 3 deletions packages/core/src/services/workspaceSources.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import { all, get, insert, run, tx } from "../db";
import { bus } from "../events/bus";
import { logger } from "../log";
import { moveToTrash } from "../agents/repo";
import { badRequest, childEnv, hostnameOf, newId, notFound, now, which } from "../util";
import { awakeTimeout, badRequest, childEnv, hostnameOf, newId, notFound, now, which } from "../util";
import { normalizeWorkingDirectory, workingDirectoryProblem } from "./folders";

const log = logger("sources");
Expand Down Expand Up @@ -443,7 +443,8 @@ async function exec(
const stdin = opts.input === undefined ? "ignore" : new Blob([opts.input]);
const proc = Bun.spawn([bin, ...safe, ...args], { cwd: opts.cwd, env: opts.env, stdin, stdout: "pipe", stderr: "pipe" });
let timedOut = false;
const timer = setTimeout(() => {
// Awake time only: a fetch the computer slept through isn't timed out the moment it wakes.
const cancelTimeout = awakeTimeout(() => {
timedOut = true;
proc.kill();
}, opts.timeoutMs);
Expand All @@ -457,7 +458,7 @@ async function exec(
const [stdout, stderr] = stopped ? await Promise.race([output, Bun.sleep(500).then(() => ["", ""] as const)]) : await output;
return { ok: code === 0 && !stopped, stdout, stderr, timedOut, ms: Math.round(performance.now() - started) };
} finally {
clearTimeout(timer);
cancelTimeout();
opts.signal?.removeEventListener("abort", stop);
}
}
Expand Down
16 changes: 14 additions & 2 deletions packages/core/test/heartbeats.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import { RUN_WATCHDOG, heartbeatAllowedAt, nextAllowedAt, nextHeartbeatAt, norma
import { argValue, invocations, makeAgent, setupEnv, until, type TestEnv } from "./fixtures/runner-harness";
import { get, run as sql } from "../src/db";
import { getAgent, updateAgent } from "../src/agents/service";
import { getRun, listRuns, waitForRun } from "../src/runner/runner";
import { cancelRun, getRun, listRuns, waitForRun, watchedRuns } from "../src/runner/runner";
import { __setWatchdogForTests, assess, checkRuns, listWatchdogEvents } from "../src/runner/watchdog";
import { __setHeartbeatsForTests, beat, heartbeatState, listBeats, sweepHeartbeats } from "../src/services/heartbeats";
import { startChat } from "../src/services/conversations";
Expand Down Expand Up @@ -318,6 +318,18 @@ describe("watchdog", () => {
}
});

test("time the computer slept is no stall: runs are timed on the awake clock", async () => {
const a = await makeAgent({ name: "Sleeper" });
const { run } = await startChat({ agentId: a.id, content: "SLEEP until done" });
await until(() => watchedRuns().some((r) => r.runId === run.id), 10_000, "run to start");
const watched = watchedRuns().find((r) => r.runId === run.id)!;
expect(watched.lastOutputAt).toBeLessThanOrEqual(performance.now());
expect(watched.lastOutputAt).toBeGreaterThan(performance.now() - 60_000);
expect(checkRuns()).toBe(0);
await cancelRun(run.id);
await waitForRun(run.id, 10_000);
});

test("a stalled ticket tries again once, then waits blocked", async () => {
const a = await makeAgent({ name: "Stuck Ticket" });
__setWatchdogForTests({ stallMs: 300 });
Expand All @@ -340,7 +352,7 @@ describe("watchdog", () => {
test("turned off, it leaves runs alone", () => {
updateSettings({ runner: { watchdog: false } });
try {
expect(checkRuns(Date.now() + 10 * 3_600_000)).toBe(0);
expect(checkRuns(performance.now() + 10 * 3_600_000)).toBe(0);
} finally {
updateSettings({ runner: { watchdog: true } });
}
Expand Down
1 change: 1 addition & 0 deletions packages/core/test/power.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -253,6 +253,7 @@ describe("lid-closed mode", () => {
const status = await powerStatus();
expect(status).toMatchObject({ enabled: true, active: false, state: "error", permission: false });
expect(status.error).toContain("no longer allowed");
expect(all<{ title: string }>("SELECT title FROM notifications WHERE title = 'Your Mac sleeps with the lid closed'")).toHaveLength(1);
});

test("a paused run keeps the Mac awake: the answer it waits for must reach it", async () => {
Expand Down
Loading