Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions packages/core/src/db/migrations.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1699,6 +1699,19 @@ CREATE TABLE artifact_versions (
sql: /* sql */ `
-- Device groups: the computer an automation's schedule runs on ("primary" or a device id; NULL = any).
ALTER TABLE routines ADD COLUMN run_on TEXT;
`,
},
{
id: 104,
name: "self_repair_tickets",
sql: /* sql */ `
-- The log a fix ticket came from: emptied once its pull request is merged, so the same problems are never fixed twice.
CREATE TABLE IF NOT EXISTS self_repair_tickets (
task_id TEXT PRIMARY KEY REFERENCES tasks(id) ON DELETE CASCADE,
source TEXT NOT NULL,
filed_at TEXT NOT NULL,
cleared_at TEXT
);
`,
},
];
47 changes: 44 additions & 3 deletions packages/core/src/diagnostics/selfRepair.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,21 +4,22 @@
*
* Asked for by the human (Settings → Logs, a runner's "Fix in code"), it always files one. Checks that run by
* themselves (the self_repair tool) only file one for problems that are new: seen lately, not already in an open
* ticket, and not filed for the build that still runs.
* ticket, and not filed for the build that still runs. Once a fix is merged, the log it came from is emptied.
*/
import type { LogEntry, LogIssue, SelfRepairInput, SelfRepairResult, Task, TaskActor } from "@godmode/shared";
import { MAX_TASK_DESCRIPTION_LENGTH, taskAttachmentMarkdown } from "@godmode/shared";
import { BUILD, VERSION, config } from "../config";
import { bus } from "../events/bus";
import { getAgent, getDefaultAgentId } from "../agents/service";
import { all, get, run as sql } from "../db";
import { logger, scrub } from "../log";
import { listRunners, runnerDiagnostics } from "../remote/runners";
import { clearRunnerLog, listRunners, runnerDiagnostics } from "../remote/runners";
import { audit } from "../services/audit";
import { getSettings } from "../services/settings";
import { saveTaskAttachment } from "../tasks/attachments";
import { createTask } from "../tasks/service";
import { badRequest, now } from "../util";
import { buildLogReport, fingerprint, issuesOf, listLogEntries } from "./logs";
import { buildLogReport, clearLogs, fingerprint, issuesOf, listLogEntries } from "./logs";

const log = logger("self-repair");

Expand Down Expand Up @@ -131,6 +132,28 @@ function agentTicketsToday(): number {
return get<{ n: number }>("SELECT COUNT(*) AS n FROM tasks WHERE created_by LIKE 'agent:%' AND labels LIKE '%\"autofix\"%' AND created_at > ?", since)?.n ?? 0;
}

/** Empty the logs whose fix tickets were merged (all of them, or one source's): what they show is fixed. */
export async function clearFixedLogs(onlySource?: string): Promise<void> {
const merged = all<{ task_id: string; source: string }>(
`SELECT s.task_id, s.source FROM self_repair_tickets s JOIN tasks t ON t.id = s.task_id
WHERE s.cleared_at IS NULL AND t.pr_state = 'merged'${onlySource ? " AND s.source = ?" : ""}`,
...(onlySource ? [onlySource] : []),
);
for (const source of new Set(merged.map((m) => m.source))) {
try {
if (source === "local") clearLogs();
else await clearRunnerLog(source);
} catch (err) {
// An offline runner's log is emptied before its next check.
log.warn(`could not clear the log of ${source === "local" ? "this computer" : `runner ${source}`} after its fix was merged`, err);
continue;
}
const ts = now();
for (const m of merged.filter((x) => x.source === source)) sql("UPDATE self_repair_tickets SET cleared_at = ? WHERE task_id = ?", ts, m.task_id);
log.info(`cleared the log of ${source === "local" ? "this computer" : `runner ${source}`}: its fix was merged`);
}
}

/** File a fix ticket for what a log (this computer's or a runner's) shows. */
export async function fixFromLog(source: RepairSource, opts: RepairOptions = {}): Promise<SelfRepairResult> {
if (config().role === "runner") throw badRequest("A runner's problems are fixed from the computer it works for.");
Expand All @@ -142,6 +165,7 @@ export async function fixFromLog(source: RepairSource, opts: RepairOptions = {})
}
const id = sourceId(source);
const onlyNew = !!opts.onlyNew;
await clearFixedLogs(id);

let where: string;
let label: string;
Expand Down Expand Up @@ -194,6 +218,7 @@ export async function fixFromLog(source: RepairSource, opts: RepairOptions = {})
opts.actor ?? "user",
);
const ts = now();
sql("INSERT OR REPLACE INTO self_repair_tickets (task_id, source, filed_at) VALUES (?, ?, ?)", task.id, id, ts);
for (const i of issues) {
sql(
"INSERT INTO self_repair_issues (source, key, task_id, filed_at, build) VALUES (?, ?, ?, ?, ?) ON CONFLICT(source, key) DO UPDATE SET task_id = excluded.task_id, filed_at = excluded.filed_at, build = excluded.build",
Expand Down Expand Up @@ -235,3 +260,19 @@ export async function selfRepairCheck(
}
return out;
}

let unsubscribe: (() => void) | null = null;

/** A merged fix empties its log right away (and what was merged while Godmode was off, now). */
export function startSelfRepair(): void {
if (config().role === "runner") return;
unsubscribe ??= bus.on((e) => {
if (e.type === "task.event" && e.event.kind === "pr_merged") void clearFixedLogs().catch((err) => log.warn("could not clear the logs of merged fixes", err));
});
void clearFixedLogs().catch((err) => log.warn("could not clear the logs of merged fixes", err));
}

export function stopSelfRepair(): void {
unsubscribe?.();
unsubscribe = null;
}
3 changes: 3 additions & 0 deletions packages/core/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ import { cleanUp, fixAll, installUpdates, startMaintenance, stopMaintenance } fr
import { RECOMMENDED, scanCleanup } from "./services/cleanup";
import { checkUpdates } from "./services/updates";
import { resourceSnapshot, startDiagnostics, stopDiagnostics } from "./diagnostics/monitor";
import { startSelfRepair, stopSelfRepair } from "./diagnostics/selfRepair";
import { getModelCatalog } from "./runner/models";
import { refreshMobileAccess, startMobileAccess, stopMobileAccess } from "./mobile/access";
import { answerHealth, HEALTH_PATH, runnerFile, runningRunner, runRunnerCli, servingRunner, USAGE as RUNNER_USAGE, type RunnerProcess } from "./remote/cli";
Expand Down Expand Up @@ -217,6 +218,7 @@ async function serve(values: Record<string, unknown>, role?: CoreConfig["role"])
startAutomationEvents();
startAppTriggers();
startMessaging();
startSelfRepair();
startTasks();
startHeartbeats();
startCalls();
Expand All @@ -231,6 +233,7 @@ async function serve(values: Record<string, unknown>, role?: CoreConfig["role"])
stopAutomationEvents();
await stopMessaging();
stopTasks();
stopSelfRepair();
stopHeartbeats();
stopCalls();
};
Expand Down
7 changes: 7 additions & 0 deletions packages/core/src/remote/runners.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1211,6 +1211,13 @@ export async function runnerDiagnostics(runnerId: string): Promise<{ runner: Rem
return { runner: getRunner(runnerId), entries, report: Buffer.from(res.body).toString("utf8"), health: health ? describeHealth(health) : null };
}

/** Empty a runner's diagnostic log (its fix was merged). */
export async function clearRunnerLog(runnerId: string): Promise<void> {
const l = link(runnerId);
l.assertOnline();
await l.json("DELETE", "/api/logs", undefined, { timeoutMs: 30_000 });
}

/** For the autofix chat's tools: a command on the runner. */
export async function runnerExec(runnerId: string, command: string, timeoutSec?: number): Promise<{ code: number | null; stdout: string; stderr: string; timedOut: boolean }> {
return link(runnerId).json("POST", "/api/link/exec", { command, ...(timeoutSec ? { timeoutMs: Math.min(timeoutSec, 900) * 1000 } : {}) }, { timeoutMs: 16 * 60_000 });
Expand Down
23 changes: 20 additions & 3 deletions packages/core/test/self-repair.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@ import type { Agent } from "@godmode/shared";
import { makeAgent, setupEnv, until, type TestEnv } from "./fixtures/runner-harness";
import { startSmartGitServer, type SmartGitServer } from "./fixtures/smart-git-server";
import { BUILD, config } from "../src/config";
import { run as sql } from "../src/db";
import { clearLogs } from "../src/diagnostics/logs";
import { fixFromLog, selfRepairCheck } from "../src/diagnostics/selfRepair";
import { get, run as sql } from "../src/db";
import { clearLogs, listLogEntries } from "../src/diagnostics/logs";
import { clearFixedLogs, fixFromLog, selfRepairCheck } from "../src/diagnostics/selfRepair";
import { logger, setFileLogLevel, setLogDir } from "../src/log";
import { getSettings, updateSettings } from "../src/services/settings";
import { readTaskAttachment } from "../src/tasks/attachments";
Expand Down Expand Up @@ -135,4 +135,21 @@ describe("fixing Godmode's code from its logs", () => {
expect(task?.autoMerge).toBe(false);
expect(task?.description).toContain("waits for the human's review");
});

test("a merged fix empties the log it came from, so its problems are not fixed twice", async () => {
log.error("could not write the vault index");
const { task } = await fixFromLog({ kind: "local" }, { autoMerge: false, actor: "user" });
await clearFixedLogs();
expect(listLogEntries({ search: "could not write the vault index" })).toHaveLength(1);

sql("UPDATE tasks SET pr_state = 'merged' WHERE id = ?", task!.id);
await clearFixedLogs();
expect(listLogEntries({ level: "warn" })).toHaveLength(0);
expect(get<{ cleared_at: string | null }>("SELECT cleared_at FROM self_repair_tickets WHERE task_id = ?", task!.id)?.cleared_at).toBeTruthy();

// Cleared once: problems logged after that stay for the next fix.
log.error("could not write the vault index");
await clearFixedLogs();
expect(listLogEntries({ search: "could not write the vault index" })).toHaveLength(1);
});
});
Loading