Track settled worker models and failure telemetry

Author: TheGreatAxios

src/subagent/run-settlement.test.ts

@@ -0,0 +1,117 @@
+import { expect, test } from "bun:test";
+import { mkdtemp } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+import type { ReactorEmittedEvent } from "@intx/inference";
+
+import { withMockedModuleDuring } from "../../tests/helpers/mock-module.js";
+import { createPermissionGate } from "../permission/gate.js";
+import type { SubAgentRunSettlement } from "./types.js";
+
+const permissionGate = createPermissionGate({
+  approvals: [],
+  interactive: false,
+  skipPermissions: true,
+});
+
+test("rejected workers settle prior rollups with the latest observed model", async () => {
+  const cwd = await mkdtemp(join(tmpdir(), "corbits-run-settlement-"));
+  const originalError = new Error("worker failed after prior activity");
+  let settlement: Readonly<SubAgentRunSettlement> | undefined;
+
+  const caught = await withMockedModuleDuring(
+    import.meta.resolve("../agent/live-tool-dispatch.js"),
+    (real: typeof import("../agent/live-tool-dispatch.js")) => ({
+      ...real,
+      createAgentWithLiveToolDispatch: async () => ({
+        send: async () => {
+          await new Promise((resolve) => setTimeout(resolve, 20));
+          throw originalError;
+        },
+        stream: () =>
+          (async function* (): AsyncGenerator<ReactorEmittedEvent> {
+            yield {
+              type: "tool.start",
+              seq: 1,
+              data: { call: { id: "call-1", name: "read_file", arguments: {} } },
+            } as ReactorEmittedEvent;
+            yield {
+              type: "tool.done",
+              seq: 2,
+              data: {
+                call: { id: "call-1", name: "read_file", arguments: {} },
+                result: { callId: "call-1", content: "failed", isError: true },
+              },
+            } as ReactorEmittedEvent;
+            yield {
+              type: "inference.done",
+              seq: 3,
+              data: {
+                turn: { role: "assistant", content: [], model: "backup-model", timestamp: 0 },
+                usage: {
+                  input: 11,
+                  output: 7,
+                  cacheRead: 3,
+                  cacheWrite: 2,
+                  thinking: 5,
+                },
+                source: {
+                  sourceId: "backup-source",
+                  provider: "backup",
+                  model: "backup-model",
+                },
+              },
+            } as ReactorEmittedEvent;
+          })(),
+        deliver: () => {},
+        close: async () => {},
+        setSource: () => {},
+        setSources: () => {},
+        history: async () => [],
+        checkpoints: async () => [],
+        readAt: async () => [],
+        blobReader: {},
+      }),
+    }),
+    async () => {
+      const { runSubAgent } = await import("./run.js");
+      try {
+        await runSubAgent({
+          cwd,
+          workdirBase: join(cwd, ".ctx"),
+          permissionGate,
+          provider: {
+            providerName: "initial",
+            baseURL: "http://localhost",
+            model: "initial-model",
+          },
+          description: "settlement probe",
+          prompt: "do work then fail",
+          onRunSettled: (summary) => {
+            settlement = summary;
+          },
+        });
+      } catch (error) {
+        return error;
+      }
+      throw new Error("expected runSubAgent to reject");
+    },
+  );
+
+  expect(caught).toBe(originalError);
+  expect(settlement).toMatchObject({
+    turn_count: 1,
+    input_tokens: 11,
+    output_tokens: 7,
+    cache_read_tokens: 3,
+    cache_write_tokens: 2,
+    reasoning_tokens: 5,
+    tool_call_count: 1,
+    tool_error_count: 1,
+    error_count: 1,
+    model: "backup-model",
+    terminal_reason: "error",
+  });
+  expect(Object.isFrozen(settlement)).toBe(true);
+});

src/subagent/run.ts

@@ -375,9 +375,10 @@ export async function runSubAgent(params: RunSubAgentParams): Promise<RunSubAgen
   };
   let terminalReason: SubAgentTerminalReason = "error";
   let errorCount = 0;
+  const settlementState = { latestModel: params.provider.model };
 
   try {
-    const result = await runSubAgentInner(params, telemetryRollup);
+    const result = await runSubAgentInner(params, telemetryRollup, settlementState);
     terminalReason = result.stopReason ?? "complete";
     return result;
   } catch (error) {
@@ -390,7 +391,7 @@ export async function runSubAgent(params: RunSubAgentParams): Promise<RunSubAgen
           ...telemetryRollup,
           error_count: errorCount,
           duration_ms: Date.now() - startedAt,
-          model: params.provider.model,
+          model: settlementState.latestModel,
           terminal_reason: terminalReason,
         }),
       );
@@ -403,6 +404,7 @@ export async function runSubAgent(params: RunSubAgentParams): Promise<RunSubAgen
 async function runSubAgentInner(
   params: RunSubAgentParams,
   telemetryRollup: SubAgentTelemetryRollup,
+  settlementState: { latestModel: string },
 ): Promise<RunSubAgentResult> {
   await seedPricingMetadataFromCache({
     cachePath: defaultPricingCachePath(),
@@ -916,6 +918,9 @@ async function runSubAgentInner(
         const result = (event as { data?: { result?: { isError?: unknown } } }).data?.result;
         if (result?.isError === true) telemetryRollup.tool_error_count += 1;
       }
+      if (event.type === "inference.done") {
+        settlementState.latestModel = event.data.source.model;
+      }
       if (onTurnBoundary(event)) {
         telemetryRollup.turn_count += 1;
         const usage = (

src/subagent/spawn-agent-worktree.test.ts

@@ -9,6 +9,7 @@ import { createFleetRecords, createSpawnAgentTool } from "./agent-fleet.js";
 import { createSubAgentSessionStore } from "./session-store.js";
 import { createPermissionGate } from "../permission/gate.js";
 import type { RunSubAgentParams, RunSubAgentResult } from "./types.js";
+import type { Telemetry } from "../telemetry/index.js";
 
 const run = promisify(execFile);
 
@@ -24,6 +25,19 @@ const provider = {
   model: "test-model",
 };
 
+function telemetryCapture() {
+  const events: { event: string; properties: Record<string, unknown> }[] = [];
+  const telemetry: Telemetry = {
+    enabled: true,
+    installationId: "test",
+    capture: (event, properties = {}) => events.push({ event, properties }),
+    captureIntentional: () => false,
+    flush: async () => {},
+    discard: () => {},
+  };
+  return { telemetry, events };
+}
+
 const tempDirs: string[] = [];
 
 afterEach(async () => {
@@ -108,17 +122,20 @@ describe("spawn_agent worktree isolation", () => {
     tempDirs.push(workdirBase);
 
     let ran = false;
+    const { telemetry, events } = telemetryCapture();
+    const sessions = createSubAgentSessionStore();
     const tool = createSpawnAgentTool({
       permissionGate: testPermissionGate,
       cwd: notARepo,
       getWorkdirBase: () => workdirBase,
       provider,
       useWorktree: true,
+      telemetry,
       run: async () => {
         ran = true;
         return { report: "no" };
       },
-      sessions: createSubAgentSessionStore(),
+      sessions,
       fleetRecords: createFleetRecords(),
     });
     if (tool.kind !== "full") throw new Error("expected full tool");
@@ -132,6 +149,25 @@ describe("spawn_agent worktree isolation", () => {
     );
     expect(result.isError).toBe(true);
     expect(ran).toBe(false);
+    expect(sessions.list()).toHaveLength(1);
+    expect(sessions.list()[0]?.status).toBe("failed");
+    expect(events.filter((event) => event.event === "subagent_start")).toHaveLength(1);
+    const ends = events.filter((event) => event.event === "subagent_end");
+    expect(ends).toHaveLength(1);
+    expect(ends[0]?.properties).toMatchObject({
+      status: "failed",
+      stop_reason: "setup_error",
+      model: "test-model",
+      turn_count: 0,
+      input_tokens: 0,
+      output_tokens: 0,
+      cache_read_tokens: 0,
+      cache_write_tokens: 0,
+      reasoning_tokens: 0,
+      tool_call_count: 0,
+      tool_error_count: 0,
+    });
+    expect(typeof ends[0]?.properties.duration_ms).toBe("number");
   });
 
   test("defers worktree cleanup while the session is retained for followup", async () => {

src/subagent/task-tool-worktree.test.ts

@@ -8,6 +8,7 @@ import { promisify } from "node:util";
 import { createTaskTool } from "./task-tool.js";
 import type { RunSubAgentParams } from "./types.js";
 import { createPermissionGate } from "../permission/gate.js";
+import type { Telemetry } from "../telemetry/index.js";
 
 const run = promisify(execFile);
 
@@ -23,6 +24,19 @@ const provider = {
   model: "test-model",
 };
 
+function telemetryCapture() {
+  const events: { event: string; properties: Record<string, unknown> }[] = [];
+  const telemetry: Telemetry = {
+    enabled: true,
+    installationId: "test",
+    capture: (event, properties = {}) => events.push({ event, properties }),
+    captureIntentional: () => false,
+    flush: async () => {},
+    discard: () => {},
+  };
+  return { telemetry, events };
+}
+
 async function callTask(
   tool: ReturnType<typeof createTaskTool>,
   args: Record<string, unknown>,
@@ -122,12 +136,14 @@ describe("createTaskTool worktree isolation", () => {
     tempDirs.push(workdirBase);
 
     let ran = false;
+    const { telemetry, events } = telemetryCapture();
     const tool = createTaskTool({
       permissionGate: testPermissionGate,
       cwd: notARepo,
       getWorkdirBase: () => workdirBase,
       provider,
       useWorktree: true,
+      telemetry,
       run: async () => {
         ran = true;
         return { report: "done" };
@@ -143,6 +159,23 @@ describe("createTaskTool worktree isolation", () => {
     expect(result).toContain("Error:");
     expect(result).toContain("not inside a git repository");
     expect(ran).toBe(false);
+    expect(events.filter((event) => event.event === "subagent_start")).toHaveLength(1);
+    const ends = events.filter((event) => event.event === "subagent_end");
+    expect(ends).toHaveLength(1);
+    expect(ends[0]?.properties).toMatchObject({
+      status: "failed",
+      stop_reason: "setup_error",
+      model: "test-model",
+      turn_count: 0,
+      input_tokens: 0,
+      output_tokens: 0,
+      cache_read_tokens: 0,
+      cache_write_tokens: 0,
+      reasoning_tokens: 0,
+      tool_call_count: 0,
+      tool_error_count: 0,
+    });
+    expect(typeof ends[0]?.properties.duration_ms).toBe("number");
   });
 
   test("preserves a worktree the sub-agent left dirty, with a notice in the report", async () => {