Abort a model that loops on repeated output

The stall watchdog measured silence, so a model streaming the same
line on repeat never tripped it: tokens kept arriving, lastActivityAt
kept refreshing, and the run burned tokens until a human noticed and
interrupted it. The watchdog now also folds streamed text into a
bounded buffer and checks it for a line repeated past a threshold,
aborting immediately on detection regardless of how fast the loop is
producing output. The recovery message names it as the model
repeating itself and reports the tokens spent on the looped span, so
a retry reads as reasonable rather than papering over a hang.

Author: TheGreatAxios

src/tui-opentui/runtime-bridge.ts

@@ -40,10 +40,12 @@ import {
 import { quotaWaitSeconds, shouldAutoRetryQuota } from "./quota-retry.js"
 import {
   applyStallRecovery,
+  repetitionRecoveryMessage,
   shouldAbortForStall,
   shouldNoticeStall,
   STALL_NOTICE_MESSAGE,
   STALL_NOTICE_MS,
+  STALL_RECOVERY_MESSAGE,
   STALL_TIMEOUT_MS,
 } from "./stall-watchdog.js"
 import {
@@ -850,6 +852,19 @@ export function attachSessionBridge(
       return
     }
 
+    // Content-based, not time-based: a repeating line means the model is
+    // stuck regardless of how fast it is producing it, so this is checked
+    // before the silence clock rather than folded into it.
+    if (bag.turn.status === "running" && bag.turn.repeating) {
+      const repeatedTokens =
+        bag.turn.streamTokenCount - (bag.turn.repeatingSinceTokenCount ?? 0)
+      applyStallRecovery(
+        { abort: doInterrupt, notify: (message) => setStatusFlash(shell, message) },
+        repetitionRecoveryMessage(repeatedTokens),
+      )
+      return
+    }
+
     const stallArgs = {
       status: bag.turn.status,
       awaitingResponse: bag.turn.awaitingResponse,
@@ -861,10 +876,10 @@ export function attachSessionBridge(
     }
 
     if (shouldAbortForStall(stallArgs)) {
-      applyStallRecovery({
-        abort: doInterrupt,
-        notify: (message) => setStatusFlash(shell, message),
-      })
+      applyStallRecovery(
+        { abort: doInterrupt, notify: (message) => setStatusFlash(shell, message) },
+        STALL_RECOVERY_MESSAGE,
+      )
       return
     }
 

src/tui-opentui/stall-watchdog.test.ts

@@ -2,6 +2,8 @@ import { describe, expect, test } from "bun:test"
 
 import {
   applyStallRecovery,
+  detectRepetition,
+  repetitionRecoveryMessage,
   shouldAbortForStall,
   shouldNoticeStall,
   STALL_NOTICE_MS,
@@ -81,14 +83,65 @@ describe("shouldAbortForStall", () => {
 })
 
 describe("applyStallRecovery", () => {
-  test("aborts then notifies", () => {
+  test("aborts then notifies with the default message", () => {
     const calls: string[] = []
     applyStallRecovery({
       abort: () => calls.push("abort"),
       notify: (m) => calls.push(m),
     })
     expect(calls).toEqual(["abort", STALL_RECOVERY_MESSAGE])
   })
+
+  test("aborts then notifies with a supplied message", () => {
+    const calls: string[] = []
+    applyStallRecovery(
+      { abort: () => calls.push("abort"), notify: (m) => calls.push(m) },
+      "custom message",
+    )
+    expect(calls).toEqual(["abort", "custom message"])
+  })
+})
+
+describe("detectRepetition", () => {
+  test("finds nothing in fresh, varied output", () => {
+    const text = [
+      "I'll check the callId emission path first.",
+      "Running the search now.",
+      "Found three matches across the module.",
+    ].join("\n")
+    expect(detectRepetition(text).repeating).toBe(false)
+  })
+
+  test("flags a line repeated past the occurrence threshold", () => {
+    const line1 =
+      "I'll verify callId emission and remaining edges, then write the ranked findings."
+    const line2 = "Confirming callId emission, then writing the ranked findings."
+    const text = Array(4).fill([line1, line2]).flat().join("\n")
+    const check = detectRepetition(text)
+    expect(check.repeating).toBe(true)
+    expect(check.repeatedLine).toBe(line1)
+    expect(check.occurrences).toBeGreaterThanOrEqual(3)
+  })
+
+  test("ignores short recurring lines", () => {
+    const text = Array(6).fill("Checking...").join("\n")
+    expect(detectRepetition(text).repeating).toBe(false)
+  })
+
+  test("does not flag two occurrences", () => {
+    const line =
+      "I'll verify callId emission and remaining edges, then write the ranked findings."
+    const text = [line, "some other progress here.", line].join("\n")
+    expect(detectRepetition(text).repeating).toBe(false)
+  })
+})
+
+describe("repetitionRecoveryMessage", () => {
+  test("names degeneration and attributes the looped tokens", () => {
+    const message = repetitionRecoveryMessage(42)
+    expect(message).toContain("repeating itself")
+    expect(message).toContain("42")
+  })
 })
 
 describe("shouldNoticeStall", () => {

src/tui-opentui/stall-watchdog.ts

@@ -21,6 +21,43 @@ export type ShouldAbortForStallArgs = {
   readonly streamingType: "text" | "thinking" | "tool" | null
 }
 
+// A repeated line has to be long enough that short, legitimately-recurring
+// fragments ("Let me check.") do not trip the guard.
+const REPETITION_MIN_LINE_LENGTH = 20
+// How far back into the streamed text to look for repeats. Bounded so the
+// check stays cheap however long the turn runs.
+const REPETITION_LOOKBACK_LINES = 12
+// Three verbatim repeats of the same substantial line is not a coincidence
+// of phrasing — it is the model looping.
+const REPETITION_MIN_OCCURRENCES = 3
+
+export type RepetitionCheck = {
+  readonly repeating: boolean
+  readonly repeatedLine: string | null
+  readonly occurrences: number
+}
+
+/**
+ * Whether the tail of the streamed text is dominated by a line repeated
+ * verbatim. Pure text-in, decision-out: the caller owns accumulating the
+ * buffer across deltas and cycles within a turn.
+ */
+export function detectRepetition(text: string): RepetitionCheck {
+  const lines = text
+    .split("\n")
+    .map((line) => line.trim())
+    .filter((line) => line.length >= REPETITION_MIN_LINE_LENGTH)
+  const tail = lines.slice(-REPETITION_LOOKBACK_LINES)
+  const counts = new Map<string, number>()
+  for (const line of tail) counts.set(line, (counts.get(line) ?? 0) + 1)
+  for (const [line, occurrences] of counts) {
+    if (occurrences >= REPETITION_MIN_OCCURRENCES) {
+      return { repeating: true, repeatedLine: line, occurrences }
+    }
+  }
+  return { repeating: false, repeatedLine: null, occurrences: 0 }
+}
+
 /**
  * Whether silence of `thresholdMs` counts as stuck at all. Shared by the notice
  * and the abort so they never disagree about which runs are stalled — only
@@ -62,19 +99,35 @@ export function shouldNoticeStall(args: ShouldNoticeStallArgs): boolean {
   return silentPastThreshold(args, args.stallNoticeMs)
 }
 
-/** Shown while the run is silent; names the state and the way out. */
+/**
+ * Shown while nothing is arriving at all. Never fires while tokens are
+ * flowing — a model looping on repeated content is still producing output,
+ * so it is reported by `repetitionRecoveryMessage` instead, not this one.
+ */
 export const STALL_NOTICE_MESSAGE = "no response for a while — ctrl+c to interrupt"
 
 export const STALL_RECOVERY_MESSAGE =
   "stopped after no response — send again to retry"
 
+/**
+ * Shown once a repeated line aborts the turn. Named as degeneration, not a
+ * generic failure, so a retry reads as the reasonable next step rather than
+ * papering over a suspected hang or network fault.
+ */
+export function repetitionRecoveryMessage(repeatedTokens: number): string {
+  return `stopped after repeating itself — ~${repeatedTokens} tokens looped — send again to retry`
+}
+
 export type ApplyStallRecoveryDeps = {
   /** Abort the in-flight run through the session port. */
   readonly abort: () => void
   readonly notify: (message: string) => void
 }
 
-export function applyStallRecovery(deps: ApplyStallRecoveryDeps): void {
+export function applyStallRecovery(
+  deps: ApplyStallRecoveryDeps,
+  message: string = STALL_RECOVERY_MESSAGE,
+): void {
   deps.abort()
-  deps.notify(STALL_RECOVERY_MESSAGE)
+  deps.notify(message)
 }

src/tui-opentui/turn-monitor.test.ts

@@ -369,6 +369,61 @@ describe("stall watchdog", () => {
   })
 })
 
+describe("repetition guard", () => {
+  test("aborts a looping model without waiting on the stall clock", async () => {
+    await withTestRenderer(async (h) => {
+      const t: Harness = await setup(h)
+      try {
+        t.bridge.submit("build it", "immediate")
+        t.port.clear()
+
+        const line1 =
+          "I'll verify callId emission and remaining edges, then write the ranked findings.\n"
+        const line2 = "Confirming callId emission, then writing the ranked findings.\n"
+
+        // Tokens keep landing every tick — a real stall would never fire here.
+        for (let i = 0; i < 6; i++) {
+          t.bridge.handle({
+            type: "inference.text.delta",
+            data: { token: i % 2 === 0 ? line1 : line2 },
+          })
+          t.advance(10)
+          t.tick()
+        }
+
+        expect(t.port.calls).toEqual([{ op: "interrupt" }])
+        expect(t.shell.statusFlash).toContain("repeating itself")
+        expect(t.shell.statusFlash).not.toBe(STALL_NOTICE_MESSAGE)
+      } finally {
+        t.bridge.dispose()
+      }
+    })
+  })
+
+  test("a slow but progressing turn is never killed", async () => {
+    await withTestRenderer(async (h) => {
+      const t: Harness = await setup(h)
+      try {
+        t.bridge.submit("build it", "immediate")
+        t.port.clear()
+
+        for (let i = 0; i < 5; i++) {
+          t.bridge.handle({
+            type: "inference.text.delta",
+            data: { token: `distinct progress update number ${i}\n` },
+          })
+          t.advance(500)
+          t.tick()
+        }
+
+        expect(t.port.calls).toEqual([])
+      } finally {
+        t.bridge.dispose()
+      }
+    })
+  })
+})
+
 describe("reasoning settles to a summary", () => {
   test("a closed thinking row carries its elapsed time", async () => {
     await withTestRenderer(async (h) => {

src/tui-opentui/turn-state.test.ts

@@ -9,7 +9,12 @@ import {
 } from "./turn-state.js"
 
 const fold = (
-  events: readonly { type: string; data?: unknown; state?: string }[],
+  events: readonly {
+    type: string
+    data?: unknown
+    state?: string
+    text?: string
+  }[],
   startMs = 0,
 ) =>
   events.reduce(
@@ -160,3 +165,69 @@ describe("turn transitions", () => {
     expect(s.isProcessing).toBe(true)
   })
 })
+
+describe("repetition tracking", () => {
+  const line1 =
+    "I'll verify callId emission and remaining edges, then write the ranked findings."
+  const line2 =
+    "Confirming callId emission, then writing the ranked findings."
+
+  const textDelta = (text: string) => ({
+    type: "inference.text.delta",
+    data: { token: text },
+  })
+
+  test("varied streamed text is never flagged", () => {
+    const s = fold([
+      { type: "inference.start" },
+      textDelta("I'll check the callId path.\n"),
+      textDelta("Running the search now.\n"),
+      textDelta("Found the match.\n"),
+    ])
+    expect(s.repeating).toBe(false)
+    expect(s.repeatingSinceTokenCount).toBeNull()
+  })
+
+  test("a line looping past the threshold flips repeating and latches the token count", () => {
+    const deltas = Array(4)
+      .fill([line1, line2])
+      .flat()
+      .map((line) => textDelta(`${line}\n`))
+    const s = fold([{ type: "inference.start" }, ...deltas])
+    expect(s.repeating).toBe(true)
+    // Three text deltas land before the third `line1` repeat crosses the
+    // occurrence threshold and latches the count.
+    expect(s.repeatingSinceTokenCount).toBe(5)
+  })
+
+  test("repetition tracked across a tool cycle survives connector.reply with tools outstanding", () => {
+    const deltas = Array(4)
+      .fill([line1, line2])
+      .flat()
+      .map((line) => textDelta(`${line}\n`))
+    const withTool = turnStateFromEvent(
+      fold([{ type: "inference.start" }, ...deltas]),
+      { type: "tool.start", data: { call: { id: "c1", name: "grep" } } },
+      100,
+    )
+    const afterReply = turnStateFromEvent(
+      withTool,
+      { type: "connector.reply" },
+      101,
+    )
+    expect(afterReply.repeating).toBe(true)
+    expect(afterReply.streamText.length).toBeGreaterThan(0)
+  })
+
+  test("a fresh submit clears the repetition state", () => {
+    const deltas = Array(4)
+      .fill([line1, line2])
+      .flat()
+      .map((line) => textDelta(`${line}\n`))
+    const looping = fold([{ type: "inference.start" }, ...deltas])
+    const restarted = turnStateOnSubmit(looping, 200)
+    expect(restarted.repeating).toBe(false)
+    expect(restarted.repeatingSinceTokenCount).toBeNull()
+    expect(restarted.streamText).toBe("")
+  })
+})