Detect disconnected MCP servers and automatically reconnect them

docs/MCP.md

@@ -124,6 +124,33 @@ file for that HTTP endpoint. Built-in Exa is disable-only — Alt+R notifies and
 does not open the confirm. While a local `.corbits/settings.json` `mcpServers`
 list is in effect, these actions edit that local file; add remains blocked.
 
+## Dropped Transports Reconnect in the Background
+
+When a connected server's transport dies unexpectedly, the row goes
+**reconnecting** instead of failed: the redial attempt and the retained tool
+count stay on the row (for example, `acme — reconnecting · attempt 3 · 2
+tools`), and backoff redials behind the scenes with a jittered delay that
+grows per attempt and caps at 30 seconds. While reconnecting, the server's
+tools stay mounted but fail fast — calling one returns "MCP server is
+reconnecting; retry the call once it reports connected." rather than hanging
+or dispatching against a dead transport.
+
+Pressing Enter on a failed or reconnecting row retries now with a single dial
+instead of waiting for backoff, reusing the saved settings row — it never
+adds a second server. A failed retry leaves the row failed with no backoff
+until the next manual retry. Redials that hit an authorization wall or a
+terminal failure stop on their own and drop the tools; only a manual retry
+brings those rows back.
+
+This contrasts with deliberate teardown: **Alt+D** (disable) and **Alt+R**
+(remove) drop the server's tools at once and stop any redial loop, while a
+transport death keeps the tools mounted as fail-fast stubs and keeps
+redialing. A `tool_search` miss waits up to 1 second for in-flight handshakes
+and looks again; when a reconnecting server holds tools that could match the
+query, the search takes one extra 500 ms extension for the redial to remount
+them. Servers still waiting on authorization never earn that extension — they
+settle only when authorization completes out of band.
+
 ## Server Kinds
 
 A server is reached one of two ways:

src/agent/tool-search.test.ts

@@ -1,4 +1,4 @@
-import { describe, test, expect } from "bun:test";
+import { describe, test, expect, jest } from "bun:test";
 import type { ToolDefinition } from "@intx/types/runtime";
 import {
   createToolIndex,
@@ -9,6 +9,8 @@ import {
   coreToolNamesForSessionMode,
   CORE_TOOL_NAMES,
   CATALOG_TOOL_NAMES,
+  TOOL_SEARCH_PENDING_WAIT_MS,
+  TOOL_SEARCH_RECONNECT_WAIT_MS,
   type ToolAvailability,
 } from "./tool-search.js";
 
@@ -243,6 +245,15 @@ function call(
   return tool.handler(args, new AbortController().signal);
 }
 
+async function flushMicrotasks(rounds = 100): Promise<void> {
+  for (let i = 0; i < rounds; i++) await Promise.resolve();
+}
+
+async function advanceAndFlush(ms: number): Promise<void> {
+  jest.advanceTimersByTime(ms);
+  await flushMicrotasks();
+}
+
 describe("createToolSearchTool", () => {
   test("promotes matches and lists them back to the model", async () => {
     const promoted: string[] = [];
@@ -368,6 +379,111 @@ describe("createToolSearchTool", () => {
       /still connecting|still starting up|retry shortly/i,
     );
   });
+
+  test("a genuine miss with nothing pending never consults the reconnect predicate", async () => {
+    let matchChecks = 0;
+    const tool = createToolSearchTool({
+      search: () => [],
+      lookup: () => undefined,
+      promote: () => undefined,
+      awaitPendingConnections: async () => 0,
+      hasReconnectingMatch: () => {
+        matchChecks += 1;
+        return true;
+      },
+    });
+    const out = await call(tool, { query: "nonsense" });
+    expect(out).toContain("different keywords");
+    expect(matchChecks).toBe(0);
+  });
+
+  test("a reconnecting match earns one short extension and finds the remounted tool", async () => {
+    jest.useFakeTimers();
+    try {
+      const live: ToolDefinition[] = [];
+      const timeouts: (number | undefined)[] = [];
+      const matchQueries: string[] = [];
+      const promoted: string[] = [];
+      let searches = 0;
+      const tool = createToolSearchTool({
+        search: (query) => {
+          searches += 1;
+          return createToolIndex(() => live).search(query);
+        },
+        lookup: (name) => live.find((def) => def.name === name),
+        promote: (names) => promoted.push(...names),
+        awaitPendingConnections: async (timeoutMs?: number) => {
+          timeouts.push(timeoutMs);
+          await new Promise((resolve) => setTimeout(resolve, timeoutMs ?? 0));
+          return live.length === 0 ? 1 : 0;
+        },
+        hasReconnectingMatch: (query) => {
+          matchQueries.push(query);
+          return true;
+        },
+      });
+      const pending = call(tool, { query: "linear tracker" });
+      await advanceAndFlush(TOOL_SEARCH_PENDING_WAIT_MS);
+      // The redial remounts mid-extension: dropped stubs at tier 1, live set
+      // back before the extension elapses.
+      live.push({
+        name: "mcp__linear__create_issue",
+        description: "Create an issue in the tracker",
+        inputSchema: { type: "object", properties: {}, required: [] },
+      });
+      await advanceAndFlush(TOOL_SEARCH_RECONNECT_WAIT_MS);
+      const out = await pending;
+      expect(out).toContain("mcp__linear__create_issue");
+      expect(promoted).toContain("mcp__linear__create_issue");
+      expect(timeouts).toEqual([
+        TOOL_SEARCH_PENDING_WAIT_MS,
+        TOOL_SEARCH_RECONNECT_WAIT_MS,
+      ]);
+      expect(searches).toBe(3);
+      expect(matchQueries).toEqual(["linear tracker"]);
+    } finally {
+      jest.useRealTimers();
+    }
+  });
+
+  test("a needs-auth miss never earns the extension and stays within the tier-1 bound", async () => {
+    jest.useFakeTimers();
+    try {
+      const timeouts: (number | undefined)[] = [];
+      let searches = 0;
+      let matchChecks = 0;
+      const tool = createToolSearchTool({
+        search: () => {
+          searches += 1;
+          return [];
+        },
+        lookup: () => undefined,
+        promote: () => undefined,
+        awaitPendingConnections: async (timeoutMs?: number) => {
+          timeouts.push(timeoutMs);
+          await new Promise((resolve) => setTimeout(resolve, timeoutMs ?? 0));
+          return 1;
+        },
+        // A needs-auth server never populates the reconnect map, so the
+        // toolset predicate stays false for it.
+        hasReconnectingMatch: () => {
+          matchChecks += 1;
+          return false;
+        },
+      });
+      const pending = call(tool, { query: "notion" });
+      await advanceAndFlush(TOOL_SEARCH_PENDING_WAIT_MS);
+      // Far past any extension: no second wait may be outstanding.
+      await advanceAndFlush(10_000);
+      const out = await pending;
+      expect(out).toMatch(/retry.*shortly/i);
+      expect(timeouts).toEqual([TOOL_SEARCH_PENDING_WAIT_MS]);
+      expect(searches).toBe(2);
+      expect(matchChecks).toBe(1);
+    } finally {
+      jest.useRealTimers();
+    }
+  });
 });
 
 describe("advertisedTools", () => {

src/agent/tool-search.ts

@@ -285,13 +285,26 @@ export interface ToolSearchDeps {
   // below so even a stuck dependency can never hang the call. Omitted callers
   // (tests, ad-hoc indexes) have no pending handshakes to wait for.
   awaitPendingConnections?: (timeoutMs?: number) => Promise<number>;
+  // True when a reconnecting MCP server holds retained tools that could match
+  // `query`, justifying one short extra wait for the redial to remount them.
+  // Only transport-death reconnects qualify — a needs-auth server settles
+  // solely via out-of-band authorization, so extending the wait for one can
+  // never help and this must stay false for them.
+  hasReconnectingMatch?: (query: string) => boolean;
 }
 
 // Brief bound a tool_search miss waits for in-flight MCP handshakes before
 // answering. A hung authorization must never hang the call, so both the
 // toolset wait and the handler race below are capped by this.
 export const TOOL_SEARCH_PENDING_WAIT_MS = 1_000;
 
+// Short extension past the tier-1 wait, taken at most once and only when a
+// reconnecting server holds tools that could match the query. Covers the
+// redial window where the stubs are dropped and the live set is not yet
+// remounted. Never taken for needs-auth: those servers settle solely via
+// out-of-band authorization.
+export const TOOL_SEARCH_RECONNECT_WAIT_MS = 500;
+
 const ToolSearchArgs = type({ query: "string" });
 
 // Render one discovered tool as name, description, and pretty-printed input
@@ -312,21 +325,19 @@ function indent(text: string, pad: string): string {
     .join("\n");
 }
 
-// Race the dependency's pending-count wait against the same bound, so a
+// Race the dependency's pending-count wait against a bound, so a
 // stuck dependency (hung OAuth that never settles) cannot hang the call.
 // Resolves undefined when this race itself times out.
 async function racePendingCount(
   awaitPending: (timeoutMs?: number) => Promise<number>,
+  timeoutMs: number,
 ): Promise<number | undefined> {
   let timer: ReturnType<typeof setTimeout> | undefined;
   try {
     return await Promise.race([
-      awaitPending(TOOL_SEARCH_PENDING_WAIT_MS),
+      awaitPending(timeoutMs),
       new Promise<undefined>((resolve) => {
-        timer = setTimeout(
-          () => resolve(undefined),
-          TOOL_SEARCH_PENDING_WAIT_MS,
-        );
+        timer = setTimeout(() => resolve(undefined), timeoutMs);
       }),
     ]);
   } finally {
@@ -347,13 +358,29 @@ export function createToolSearchTool(deps: ToolSearchDeps): AgentTool {
         return "Error: tool_search requires a non-empty query.";
       let names = deps.search(query);
       if (names.length === 0 && deps.awaitPendingConnections !== undefined) {
-        // Miss while connectors start up: wait briefly, then re-search so
-        // late-mounting tools land. The race bounds even a stuck dependency
-        // (hung OAuth) — undefined means the wait itself timed out.
-        const stillPending = await racePendingCount(
+        // Tier 1 — miss while connectors start up: wait briefly, then
+        // re-search so late-mounting tools land. The race bounds even a stuck
+        // dependency (hung OAuth) — undefined means the wait itself timed out.
+        let stillPending = await racePendingCount(
           deps.awaitPendingConnections,
+          TOOL_SEARCH_PENDING_WAIT_MS,
         );
         names = deps.search(query);
+        if (
+          names.length === 0 &&
+          (stillPending ?? 1) > 0 &&
+          deps.hasReconnectingMatch?.(query) === true
+        ) {
+          // Tier 2 — a reconnecting server holds tools that could match: one
+          // short extension for the redial to remount them, then a final
+          // re-search. Needs-auth never qualifies (see the dep contract), so
+          // a hung authorization still answers within the tier-1 bound.
+          stillPending = await racePendingCount(
+            deps.awaitPendingConnections,
+            TOOL_SEARCH_RECONNECT_WAIT_MS,
+          );
+          names = deps.search(query);
+        }
         if (names.length === 0 && (stillPending ?? 1) > 0) {
           const detail =
             stillPending === undefined