crawler: 10 s fetch, 30 s render, slow or broken targets parked 30 days (0.49.0)

package.json

@@ -1,6 +1,6 @@
 {
   "name": "pitworld",
-  "version": "0.48.0",
+  "version": "0.49.0",
   "mcpName": "world.pit/pit",
   "description": "pit.world MCP server: private web search and Moshpit name lookups for AI agents (stdio, no API key). Also a web app, API, CLI and TUI.",
   "license": "MIT",

services/crawler/src/config.ts

@@ -12,6 +12,22 @@ export const VERSION = '0.1.0'
 export const BOT = 'pitbot'
 export const USER_AGENT = `Mozilla/5.0 (compatible; ${BOT}/${VERSION}; +https://pit.world/bot)`
 
+/**
+ * The fleet crawler rule (Anthony, 2026-11-08): "crawlers should timeout after 10 seconds for
+ * fetch, 30 for browser ... We don't want broken slow shit. Flag to recheck later in a month."
+ *
+ * One plain GET of a third-party address (page, robots.txt, sitemap, card read): 10 s for all
+ * of it, DNS, every redirect hop and the body. Calls to our own services (nichedb, Obscura's
+ * MCP transport, rssamplifier OPML) keep their own budgets.
+ */
+export const CRAWL_FETCH_TIMEOUT_MS = 10_000
+/** One browser render through Obscura: navigation to `load`, a short settle, the read back. */
+export const CRAWL_RENDER_TIMEOUT_MS = 30_000
+/** The bounded settle after `load`, for scripts that paint just after it. Never networkidle. */
+export const CRAWL_RENDER_SETTLE_MS = 1_500
+/** A host or page that timed out, would not connect or answered 5xx is left alone this long, then read once more. */
+export const CRAWL_RECHECK_MS = 30 * 86_400_000
+
 export type Role = 'site' | 'page' | 'render' | 'publish'
 
 export type Config = {

services/crawler/src/crawler.ts

@@ -4,7 +4,8 @@
  * The BullMQ wiring (queues.ts) calls these; the tests call them directly
  * with an in-memory frontier and a fake web.
  */
-import type { Config } from './config.ts'
+import { CRAWL_RECHECK_MS, type Config } from './config.ts'
+import { failureKind, HOST_KINDS, HOST_PAGE_FAILURE_WINDOW_MS, HOST_PAGE_FAILURES, parkFor, statusKind, type FailureKind } from './failures.ts'
 import { extract, needsRender, type Extracted } from './extract.ts'
 import { politeGet, readSitemapXml } from './fetcher.ts'
 import { endpointsIn, handshake, mcpRecord, type McpJob, WELL_KNOWN } from './mcp.ts'
@@ -135,17 +136,64 @@ export function createCrawler(deps: Deps) {
     }
   }
 
-  async function robotsFor(site: string): Promise<Robots | 'unavailable'> {
-    const kept = await frontier.getRobots(site)
+  const errorText = (error: unknown) => (error instanceof Error ? error.message : String(error)).slice(0, 200)
+
+  /**
+   * Park a host: nothing reads from it until `ms` from now (Redis, which every worker asks
+   * before a fetch), and the site row carries the flag, so the recrawl skips it until then
+   * and gives it exactly one recheck after (Postgres).
+   */
+  async function flagHost(site: string, kind: FailureKind, detail: string, ms = CRAWL_RECHECK_MS): Promise<void> {
+    await frontier.markDown(site, ms)
+    await siteStore.flag(site, `${kind}: ${detail}`, Date.now() + ms).catch((error) => log(`flag ${site}: ${errorText(error)}`))
+    await frontier.bump('sites_flagged')
+    await frontier.log({ kind: 'fail', site, note: `parked ${ms >= 86_400_000 ? `${Math.round(ms / 86_400_000)} d` : `${Math.round(ms / 60_000)} min`}: ${kind}, ${detail}`.slice(0, 200) })
+  }
+
+  /**
+   * Park one page for 30 days (a timeout or a 5xx may be that page alone). Enough of them
+   * from one site inside the hour and the host is parked too: a dead host is not read page
+   * by page for a thousand pages.
+   */
+  async function flagPage(job: PageJob, kind: FailureKind, detail: string, retryAfter?: string): Promise<string> {
+    if (HOST_KINDS.has(kind)) {
+      await flagHost(job.site, kind, `${detail} (${job.url})`, parkFor(kind, retryAfter))
+      return `host parked: ${kind}`
+    }
+    await frontier.markPageDown(job.url, CRAWL_RECHECK_MS)
+    await frontier.bump('pages_flagged')
+    await frontier.log({ kind: 'fail', site: job.site, url: job.url, note: `parked 30 d: ${kind}, ${detail}`.slice(0, 200) })
+    const n = await frontier.noteFailure(job.site, HOST_PAGE_FAILURE_WINDOW_MS)
+    if (n === HOST_PAGE_FAILURES) await flagHost(job.site, kind, `${n} pages failed within the hour, last ${detail}`)
+    return `page parked: ${kind}`
+  }
+
+  /** `state`: a siteState() the caller already has, saving the round trip. */
+  async function robotsFor(site: string, given?: { robots: Robots | null; downMs: number }): Promise<Robots | 'unavailable'> {
+    const state = given ?? (await frontier.siteState(site))
+    if (state.downMs > 0) return 'unavailable'
+    const kept = state.robots
     // Cached before AI-crawler rules were kept: read again, so a training opt-out is never missed.
     if (kept && kept.ai) return kept
     let robots: Robots = EMPTY
     try {
       const r = await get(`https://${site}/robots.txt`, site, 'text/plain,*/*;q=0.5')
-      // 5xx: the site cannot say, so it is read another day (Google's reading). 4xx: no rules.
-      if (r.status >= 500) return 'unavailable'
+      // 5xx: the site cannot say (Google's reading), so it is parked and asked again in 30 days.
+      // 429: parked for its Retry-After (capped). Any other 4xx: no rules.
+      const failed = statusKind(r.status)
+      if (failed) {
+        await flagHost(site, failed, `robots.txt answered HTTP ${r.status}`, parkFor(failed, r.retryAfter))
+        return 'unavailable'
+      }
       if (r.status < 400 && !/html/.test(r.contentType)) robots = parseRobots(r.body)
-    } catch {
+    } catch (error) {
+      // A host that times out or will not connect on robots.txt would do the same for every
+      // page: it was read as "no rules" and each page then timed out three times.
+      const kind = failureKind(error)
+      if (kind) {
+        await flagHost(site, kind, `robots.txt: ${errorText(error)}`)
+        return 'unavailable'
+      }
       robots = { ...EMPTY, read: 'error' }
     }
     // TDMRep's site-wide file: rarely there, read with robots.txt once a day.
@@ -334,16 +382,19 @@ export function createCrawler(deps: Deps) {
     // Once per cycle per site, however many jobs for it are queued.
     if (!(await frontier.claimStart(site, config.recrawlDays * 86_400_000))) return { admitted: 0, outcome: 'already started this cycle' }
     if (!(await resolver(site))) {
+      // No public address (often no address at all): parked and asked again in 30 days.
       await frontier.bump('sites_refused')
-      await frontier.log({ kind: 'skip', site, note: 'not on the public internet' })
+      await flagHost(site, 'dns', 'no public address')
       return { admitted: 0, outcome: 'not public' }
     }
     const robots = await robotsFor(site)
     if (robots === 'unavailable') {
       await frontier.bump('sites_unavailable')
-      await frontier.log({ kind: 'skip', site, note: 'robots.txt answered 5xx; trying another day' })
       return { admitted: 0, outcome: 'robots.txt unavailable' }
     }
+    // Read fine: a flag from an earlier cycle (this was its recheck) is cleared.
+    await frontier.clearDown(site)
+    await siteStore.clearFlag(site).catch((error) => log(`unflag ${site}: ${errorText(error)}`))
     if (!allows(robots, '/')) {
       await frontier.bump('sites_disallowed')
       await frontier.log({ kind: 'skip', site, note: 'robots.txt disallows pitbot' })
@@ -415,7 +466,10 @@ export function createCrawler(deps: Deps) {
     if ((await frontier.outboxLength()) > config.outboxHigh) {
       return { delayMs: 20_000 + Math.floor(Math.random() * 20_000), outcome: 'waiting for publishing to catch up' }
     }
-    const robots = await robotsFor(job.site)
+    const state = await frontier.siteState(job.site, job.url)
+    const parked = parkedOutcome(state)
+    if (parked) return parked
+    const robots = await robotsFor(job.site, state)
     if (robots === 'unavailable') return { outcome: 'robots.txt unavailable' }
     const gap = gapFor(robots)
     const slot = await frontier.lease(job.site, gap)
@@ -424,11 +478,25 @@ export function createCrawler(deps: Deps) {
       await frontier.bump('pages_disallowed')
       return { outcome: 'disallowed' }
     }
-    const r = await get(job.url, job.site)
+    let r: Awaited<ReturnType<typeof get>>
+    try {
+      r = await get(job.url, job.site)
+    } catch (error) {
+      // Not thrown on to BullMQ: a retry in 30 s of a page that just timed out is the slow
+      // broken read the rule is about. Anything else (a bug, Redis) still throws and retries.
+      const kind = failureKind(error)
+      if (!kind) throw error
+      return { outcome: await flagPage(job, kind, errorText(error)) }
+    }
     if (r.offsite) {
       if (config.discover) await scheduleSites([r.offsite], 'redirect')
       return { outcome: `redirects to ${r.offsite}` }
     }
+    const failed = statusKind(r.status)
+    if (failed) {
+      await frontier.bump('pages_failed')
+      return { outcome: await flagPage(job, failed, `HTTP ${r.status}`, r.retryAfter) }
+    }
     if (r.status >= 400) {
       await frontier.bump('pages_failed')
       await frontier.log({ kind: 'fail', site: job.site, url: job.url, note: `HTTP ${r.status}` })
@@ -446,14 +514,38 @@ export function createCrawler(deps: Deps) {
     return { outcome: await accept(doc, optOut ? { ...job, optOut } : job, robots) }
   }
 
+  /**
+   * A parked host or page is not read. A short park (a 429's Retry-After) is waited out;
+   * a long one drops the job, and the site's recheck queues the page again in 30 days.
+   */
+  function parkedOutcome(state: { downMs: number; pageDown: boolean }): PageOutcome | null {
+    if (state.pageDown) return { outcome: 'page parked' }
+    if (state.downMs <= 0) return null
+    if (state.downMs <= MAX_SPREAD_MS) return { delayMs: state.downMs, outcome: 'waiting out a parked host' }
+    return { outcome: 'host parked' }
+  }
+
   async function renderPage(job: PageJob): Promise<PageOutcome> {
     if (!deps.renderer?.size) return { outcome: 'no renderer' }
-    const robots = await robotsFor(job.site)
+    const state = await frontier.siteState(job.site, job.url)
+    const parked = parkedOutcome(state)
+    if (parked) return parked
+    const robots = await robotsFor(job.site, state)
     if (robots === 'unavailable') return { outcome: 'robots.txt unavailable' }
     const gap = gapFor(robots)
     const slot = await frontier.lease(job.site, gap)
     if (slot.waitMs > 0) return { delayMs: spreadDelay(slot.waitMs, slot.pending, gap), outcome: 'waiting for the site' }
-    const rendered = await deps.renderer.render(job.url)
+    let rendered: Rendered
+    try {
+      rendered = await deps.renderer.render(job.url)
+    } catch (error) {
+      // Out of the 30 s budget, or the browser could not reach the page: flagged, not retried.
+      // Obscura's own trouble (HTTP errors from our container) still throws and retries.
+      const kind = failureKind(error)
+      if (!kind) throw error
+      await frontier.bump('renders_failed')
+      return { outcome: await flagPage(job, kind, `render: ${errorText(error)}`) }
+    }
     const landed = normalizeUrl(rendered.url) ?? job.url
     if (!sameSite(landed, job.site)) return { outcome: `rendered onto ${siteOf(landed)}` }
     const doc = extract(rendered.html, landed, { depth: job.depth, rendered: true, crawler: config.userAgent })

services/crawler/src/failures.ts

@@ -0,0 +1,79 @@
+/**
+ * Slow or broken targets: what counts as one, and how long it is left alone.
+ *
+ * A timeout, a name that does not resolve, a bad certificate, a refused or reset
+ * connection, or a 5xx is not retried within minutes (BullMQ's backoff used to read a
+ * dead host's every page three times). The host or page is flagged instead and gets
+ * exactly one recheck after CRAWL_RECHECK_MS: success clears the flag, failure sets it
+ * another 30 days. A 429 parks the host for its Retry-After, capped at the same 30 days.
+ * Every other 4xx keeps its old meaning (the page is dropped, robots.txt reads as no rules).
+ */
+import { CRAWL_RECHECK_MS } from './config.ts'
+
+export type FailureKind = 'timeout' | 'dns' | 'tls' | 'connection' | 'http5xx' | 'http429'
+
+/** Kinds that are the host's, not one page's: every page of it would fail the same way. */
+export const HOST_KINDS: ReadonlySet<FailureKind> = new Set(['dns', 'tls', 'connection', 'http429'])
+
+/** Page timeouts or 5xx from one site inside an hour before the whole site is parked. */
+export const HOST_PAGE_FAILURES = 5
+export const HOST_PAGE_FAILURE_WINDOW_MS = 3_600_000
+
+/** A 429 without a usable Retry-After waits a day; with one, never less than a minute. */
+const RETRY_AFTER_DEFAULT_MS = 86_400_000
+const RETRY_AFTER_MIN_MS = 60_000
+
+/** A thrown fetch (or render) error as a failure kind; null when it is not the target's fault. */
+export function failureKind(error: unknown): FailureKind | null {
+  if (!error || typeof error !== 'object') return null
+  const e = error as { name?: unknown; code?: unknown; message?: unknown; ours?: unknown }
+  // Our own service failed (src/obscura.ts ObscuraError): not the target's doing.
+  if (e.ours === true) return null
+  const name = String(e.name ?? '')
+  const code = String(e.code ?? '')
+  const message = String(e.message ?? '')
+  if (name === 'TimeoutError' || name === 'AbortError' || /\btimed? ?out\b|timeout/i.test(message)) return 'timeout'
+  if (/^(ENOTFOUND|EAI_AGAIN|EAI_NONAME|EAI_NODATA)$/.test(code) || /getaddrinfo|ENOTFOUND|ERR_NAME_NOT_RESOLVED|not on the public internet/i.test(message)) return 'dns'
+  if (/CERT|SSL|TLS/i.test(code) || /certificate|\bssl\b|\btls\b|ERR_CERT_|ERR_SSL_/i.test(message)) return 'tls'
+  if (/^(ECONNREFUSED|ECONNRESET|EHOSTUNREACH|ENETUNREACH|EPIPE|ECONNABORTED|ConnectionClosed|ConnectionRefused)$/.test(code) || /ECONNREFUSED|ECONNRESET|EHOSTUNREACH|ENETUNREACH|ERR_CONNECTION_|socket (connection was )?closed|connection (was )?(closed|reset|refused)/i.test(message))
+    return 'connection'
+  return null
+}
+
+/** A status as a failure kind: 5xx and 429 only. */
+export function statusKind(status: number): FailureKind | null {
+  if (status === 429) return 'http429'
+  if (status >= 500 && status <= 599) return 'http5xx'
+  return null
+}
+
+/** Retry-After (seconds or an HTTP date) in ms from now, clamped to [1 min, 30 days]; a day when absent or unreadable. */
+export function retryAfterMs(header: string | null | undefined, now = Date.now(), cap = CRAWL_RECHECK_MS): number {
+  const raw = (header ?? '').trim()
+  let ms = RETRY_AFTER_DEFAULT_MS
+  if (/^\d+$/.test(raw)) ms = Number(raw) * 1000
+  else if (raw) {
+    const at = Date.parse(raw)
+    if (Number.isFinite(at)) ms = at - now
+  }
+  return Math.min(cap, Math.max(RETRY_AFTER_MIN_MS, ms))
+}
+
+/** How long a failure of this kind keeps its target away. */
+export function parkFor(kind: FailureKind, retryAfter?: string | null, now = Date.now()): number {
+  return kind === 'http429' ? retryAfterMs(retryAfter, now) : CRAWL_RECHECK_MS
+}
+
+/** The flag a site row carries while it is parked. */
+export type SiteFlag = { status: 'down'; lastError: string; failedAt: number; retryAfter: number }
+
+export function flagFor(error: string, untilMs: number, now = Date.now()): SiteFlag {
+  return { status: 'down', lastError: error.slice(0, 300), failedAt: now, retryAfter: untilMs }
+}
+
+/** A scheduled site is due again once its cycle has run out AND any flag on it has. */
+export function isDue(row: { scheduledAt: number | null; retryAfter?: number | null }, cycleMs: number, now = Date.now()): boolean {
+  if (row.scheduledAt === null) return false
+  if (row.scheduledAt > now - cycleMs) return false
+  return !row.retryAfter || row.retryAfter <= now
+}

services/crawler/src/fetcher.ts

@@ -1,7 +1,12 @@
 /**
  * One polite GET: a named user agent, a timeout, a size cap, and redirects
  * followed by hand so every hop is checked against the private-network guard.
+ *
+ * The timeout is one budget for the whole GET (CRAWL_FETCH_TIMEOUT_MS, 10 s): the name
+ * lookup, every redirect hop and the body. It used to be 20 s per hop, so five redirects
+ * could hold a worker for over a minute and a slow body was never cut off.
  */
+import { CRAWL_FETCH_TIMEOUT_MS } from './config.ts'
 import { normalizeUrl, type Resolver } from './url.ts'
 
 export type Fetched = {
@@ -15,20 +20,45 @@ export type Fetched = {
   offsite?: string
   /** The page's own opt-out headers (X-Robots-Tag, TDMRep's tdm-reservation), lowercased. */
   optOut?: { robotsTag: string; tdmReservation: string }
+  /** Retry-After, as sent (a 429 or 503 may carry one). */
+  retryAfter?: string
 }
 
 type Fetch = typeof fetch
 
 export const MAX_BYTES = 2 * 1024 * 1024
 const MAX_REDIRECTS = 5
 
-export async function readCapped(response: Response, max: number): Promise<string> {
+/** Reject when the signal fires, whatever `work` is waiting on (a name lookup has no signal of its own). */
+export function withSignal<T>(work: Promise<T>, signal: AbortSignal): Promise<T> {
+  if (signal.aborted) return Promise.reject(signal.reason)
+  return new Promise<T>((resolve, reject) => {
+    const abort = () => reject(signal.reason)
+    signal.addEventListener('abort', abort, { once: true })
+    work.then(
+      (v) => {
+        signal.removeEventListener('abort', abort)
+        resolve(v)
+      },
+      (e) => {
+        signal.removeEventListener('abort', abort)
+        reject(e)
+      },
+    )
+  })
+}
+
+export async function readCapped(response: Response, max: number, signal?: AbortSignal): Promise<string> {
   if (!response.body) return ''
   const reader = response.body.getReader()
   const chunks: Uint8Array[] = []
   let size = 0
   while (true) {
-    const { done, value } = await reader.read()
+    // A body that trickles in past the budget is a failure, not a short page.
+    const { done, value } = signal ? await withSignal(reader.read(), signal).catch(async (error) => {
+      await reader.cancel().catch(() => {})
+      throw error
+    }) : await reader.read()
     if (done) break
     size += value.byteLength
     if (size > max) {
@@ -46,18 +76,19 @@ export async function politeGet(
   options: { userAgent: string; resolver: Resolver; fetch?: Fetch; timeoutMs?: number; accept?: string; site?: string; maxBytes?: number },
 ): Promise<Fetched> {
   const fetchImpl = options.fetch ?? fetch
+  const signal = AbortSignal.timeout(options.timeoutMs ?? CRAWL_FETCH_TIMEOUT_MS)
   let current = url
   for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {
     const host = new URL(current).hostname
-    if (!(await options.resolver(host))) throw new Error(`refused: ${host} is not on the public internet`)
+    if (!(await withSignal(options.resolver(host), signal))) throw new Error(`refused: ${host} is not on the public internet`)
     const response = await fetchImpl(current, {
       redirect: 'manual',
       headers: {
         'user-agent': options.userAgent,
         accept: options.accept ?? 'text/html,application/xhtml+xml;q=0.9,*/*;q=0.1',
         'accept-language': 'en;q=1, *;q=0.5',
       },
-      signal: AbortSignal.timeout(options.timeoutMs ?? 20_000),
+      signal,
     })
     if (response.status >= 300 && response.status < 400 && response.headers.get('location')) {
       await response.body?.cancel().catch(() => {})
@@ -72,10 +103,11 @@ export async function politeGet(
     }
     const contentType = (response.headers.get('content-type') ?? '').toLowerCase()
     const textual = /text\/|html|xml|json/.test(contentType) || contentType === ''
-    const body = textual ? await readCapped(response, options.maxBytes ?? MAX_BYTES) : ''
+    const body = textual ? await readCapped(response, options.maxBytes ?? MAX_BYTES, signal) : ''
     if (!textual) await response.body?.cancel().catch(() => {})
     const optOut = { robotsTag: (response.headers.get('x-robots-tag') ?? '').toLowerCase(), tdmReservation: (response.headers.get('tdm-reservation') ?? '').trim() }
-    return { url, finalUrl: current, status: response.status, contentType, body, optOut }
+    const retryAfter = response.headers.get('retry-after')
+    return { url, finalUrl: current, status: response.status, contentType, body, optOut, ...(retryAfter ? { retryAfter } : {}) }
   }
   throw new Error('too many redirects')
 }