Share capped response-body reader between web_fetch and go model catalog

Author: Gustav-Simonsson

src/provider/opencode-go-models.ts

@@ -5,6 +5,7 @@ import {
   OPENCODE_GO_MODEL_IDS,
 } from "../../packages/opencode-go/src/index.js";
 import { requestModelsEndpoint } from "./models-endpoint.js";
+import { readCappedBody } from "../util/capped-body.js";
 
 const GoModelsResponse = type({
   data: type({ id: "string" }).array(),
@@ -49,48 +50,19 @@ async function readCatalogJson(
     return { ok: false, message: oversizeMessage("bytes") };
   }
 
-  const body = response.body;
-  if (body === null) {
-    try {
-      const text = await response.text();
-      if (new TextEncoder().encode(text).byteLength > MAX_GO_CATALOG_BYTES) {
-        return { ok: false, message: oversizeMessage("bytes") };
-      }
-      const value: unknown = JSON.parse(text);
-      return { ok: true, value };
-    } catch (error) {
-      return { ok: false, message: error instanceof Error ? error.message : String(error) };
-    }
-  }
-
-  const reader = body.getReader();
-  const chunks: Uint8Array[] = [];
-  let total = 0;
+  let text: string;
+  let truncated: boolean;
   try {
-    for (;;) {
-      const { done, value } = await reader.read();
-      if (done) break;
-      if (value === undefined) continue;
-      total += value.byteLength;
-      if (total > MAX_GO_CATALOG_BYTES) {
-        await reader.cancel().catch(() => undefined);
-        return { ok: false, message: oversizeMessage("bytes") };
-      }
-      chunks.push(value);
-    }
+    ({ text, truncated } = await readCappedBody(response, MAX_GO_CATALOG_BYTES));
   } catch (error) {
     return { ok: false, message: error instanceof Error ? error.message : String(error) };
   }
-
-  const buffer = new Uint8Array(total);
-  let offset = 0;
-  for (const chunk of chunks) {
-    buffer.set(chunk, offset);
-    offset += chunk.byteLength;
+  if (truncated) {
+    return { ok: false, message: oversizeMessage("bytes") };
   }
 
   try {
-    const value: unknown = JSON.parse(new TextDecoder().decode(buffer));
+    const value: unknown = JSON.parse(text);
     return { ok: true, value };
   } catch (error) {
     return { ok: false, message: error instanceof Error ? error.message : String(error) };

src/tools/web-fetch.ts

@@ -5,6 +5,7 @@ import type { ToolCall, ToolDefinition, ToolResult } from "@intx/types/runtime";
 
 import { checkUrlForSsrf } from "./ssrf-guard.js";
 import { htmlToMarkdown, htmlToText } from "./html-convert.js";
+import { readCappedBody } from "../util/capped-body.js";
 import { COMMAND_NAME } from "../branding.js";
 import type { MCPClient } from "../mcp/client.js";
 import pkg from "../../package.json" with { type: "json" };
@@ -63,44 +64,6 @@ function looksLikeBotBlock(status: number): boolean {
   return status === 403 || status === 429 || status === 999;
 }
 
-async function readCapped(
-  response: Response,
-  capBytes: number,
-): Promise<{ text: string; truncated: boolean }> {
-  const body = response.body;
-  if (body === null) return { text: await response.text(), truncated: false };
-  const reader = body.getReader();
-  const chunks: Uint8Array[] = [];
-  let total = 0;
-  let truncated = false;
-  for (;;) {
-    const { done, value } = await reader.read();
-    if (done) break;
-    if (value === undefined) continue;
-    const remaining = capBytes - total;
-    if (remaining <= 0) {
-      truncated = true;
-      await reader.cancel().catch(() => undefined);
-      break;
-    }
-    const slice = value.byteLength > remaining ? value.slice(0, remaining) : value;
-    chunks.push(slice);
-    total += slice.byteLength;
-    if (slice.byteLength < value.byteLength) {
-      truncated = true;
-      await reader.cancel().catch(() => undefined);
-      break;
-    }
-  }
-  const buffer = new Uint8Array(total);
-  let offset = 0;
-  for (const chunk of chunks) {
-    buffer.set(chunk, offset);
-    offset += chunk.byteLength;
-  }
-  return { text: new TextDecoder().decode(buffer), truncated };
-}
-
 async function fetchOnce(
   url: string,
   userAgent: string,
@@ -176,14 +139,14 @@ export async function runWebFetch(
     }
 
     if (!response.ok) {
-      const { text } = await readCapped(response, 8192);
+      const { text } = await readCappedBody(response, 8192);
       return {
         ok: false,
         error: `Fetch of ${currentUrl} failed with status ${response.status}: ${text.slice(0, 500)}`,
       };
     }
 
-    const { text: body, truncated } = await readCapped(response, MAX_FETCH_BYTES);
+    const { text: body, truncated } = await readCappedBody(response, MAX_FETCH_BYTES);
     const contentType = response.headers.get("content-type") ?? "";
     const isHtml = contentType.includes("html") || /^\s*<(!doctype|html)/i.test(body);
 

src/util/capped-body.ts

@@ -0,0 +1,67 @@
+// Shared byte-capped response-body reader.
+//
+// Two callers used to carry their own copy of the same subtle byte-accounting
+// loop (accumulate stream chunks up to a cap, cancel the reader the moment the
+// cap is exceeded so the upstream socket is not drained, concatenate, decode):
+//   - web_fetch (src/tools/web-fetch.ts) caps page bodies at 5MB and error
+//     snippets at 8KB, keeping the first `capBytes` bytes and flagging the cut.
+//   - the OpenCode Go model catalog (src/provider/opencode-go-models.ts) caps
+//     the live /models response so an oversized or hostile catalog cannot blow
+//     process memory, rejecting (rather than keeping a prefix) when over.
+// Keeping the reader here means a cap-accounting fix (off-by-one, cancel
+// discipline, chunk slicing) lands once instead of drifting across copies.
+
+export interface CappedBody {
+  /** The body's content, decoded as UTF-8 and sliced to at most `capBytes` bytes. */
+  text: string;
+  /** True when the body was longer than `capBytes`; reading stopped at the cap. */
+  truncated: boolean;
+}
+
+/**
+ * Read up to `capBytes` bytes of a Response body. Returns the decoded prefix
+ * plus whether the body was cut off; the reader is cancelled as soon as the cap
+ * is exceeded so oversized bodies are not drained. A Response whose body is
+ * null (synthetic responses, no-content statuses) is read via `text()` and
+ * truncation is judged from the decoded byte length.
+ */
+export async function readCappedBody(response: Response, capBytes: number): Promise<CappedBody> {
+  const body = response.body;
+  if (body === null) {
+    const text = await response.text();
+    return {
+      text,
+      truncated: new TextEncoder().encode(text).byteLength > capBytes,
+    };
+  }
+  const reader = body.getReader();
+  const chunks: Uint8Array[] = [];
+  let total = 0;
+  let truncated = false;
+  for (;;) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    if (value === undefined) continue;
+    const remaining = capBytes - total;
+    if (remaining <= 0) {
+      truncated = true;
+      await reader.cancel().catch(() => undefined);
+      break;
+    }
+    const slice = value.byteLength > remaining ? value.slice(0, remaining) : value;
+    chunks.push(slice);
+    total += slice.byteLength;
+    if (slice.byteLength < value.byteLength) {
+      truncated = true;
+      await reader.cancel().catch(() => undefined);
+      break;
+    }
+  }
+  const buffer = new Uint8Array(total);
+  let offset = 0;
+  for (const chunk of chunks) {
+    buffer.set(chunk, offset);
+    offset += chunk.byteLength;
+  }
+  return { text: new TextDecoder().decode(buffer), truncated };
+}