Crawlers: 10 s fetch, 30 s render, 30-day recheck for slow or broken targets

app/actions/socialPosting.ts

@@ -549,6 +549,9 @@ export async function saveFeedAutopostSettings(
         ignore_paths: ignorePaths,
         status: "idle",
         last_error: null,
+        // A saved config is a fresh start: drop any 30-day crawl flag.
+        failed_at: null,
+        next_check_at: null,
       },
       { onConflict: "project_id" },
     );

lib/audit/fetch.ts

@@ -1,14 +1,16 @@
 import type { FetchedPage } from "./types";
 import { smartFetch } from "../onion";
+import { fetchBudgetMs } from "../crawl/policy";
 
 const UA = "CrawlProofBot/1.0 (+https://crawlproof.com/bot)";
-const TIMEOUT_MS = 15_000;
 const MAX_BYTES = 4 * 1024 * 1024;
 
 export async function fetchPage(url: string): Promise<FetchedPage> {
   const started = Date.now();
   const controller = new AbortController();
-  const timer = setTimeout(() => controller.abort(), TIMEOUT_MS);
+  // 10 s clearnet, the separate Tor budget for .onion (lib/crawl/policy.ts);
+  // the timer covers the body read below too.
+  const timer = setTimeout(() => controller.abort(), fetchBudgetMs(url));
   try {
     // smartFetch routes .onion targets through Tor (TOR_SOCKS_URL); everything
     // else is a normal fetch.

lib/audit/render.ts

@@ -1,4 +1,5 @@
 import type { FetchedPage } from "./types";
+import { CRAWL_BROWSER_SETTLE_MS, CRAWL_BROWSER_TIMEOUT_MS } from "../crawl/policy";
 
 // Lazily-imported to avoid loading Playwright when not needed.
 let _chromium: typeof import("playwright").chromium | null = null;
@@ -25,7 +26,15 @@ export async function renderPage(url: string): Promise<{
     browser = await chromium.launch({ args: ["--no-sandbox"] });
     const ctx = await browser.newContext({ userAgent: UA });
     const page = await ctx.newPage();
-    await page.goto(url, { waitUntil: "networkidle", timeout: 20_000 });
+    // Fleet crawler policy: 30 s for the whole render. Wait for `load`, then a
+    // bounded settle for late XHR — never an open-ended networkidle, which a
+    // page that polls never reaches. A navigation timeout throws and is
+    // returned as `error`, not read as an empty page.
+    const deadline = Date.now() + CRAWL_BROWSER_TIMEOUT_MS;
+    page.setDefaultTimeout(CRAWL_BROWSER_TIMEOUT_MS);
+    await page.goto(url, { waitUntil: "load", timeout: CRAWL_BROWSER_TIMEOUT_MS });
+    const settle = Math.min(CRAWL_BROWSER_SETTLE_MS, Math.max(0, deadline - Date.now()));
+    if (settle > 0) await page.waitForLoadState("networkidle", { timeout: settle }).catch(() => {});
     const html = await page.content();
     const text = await page.evaluate(() => document.body?.innerText ?? "");
     return {
@@ -49,6 +58,9 @@ export async function renderPage(url: string): Promise<{
 
 export async function attachRendered(page: FetchedPage): Promise<FetchedPage> {
   const r = await renderPage(page.url);
+  // A failed render is recorded as a failure, not as a page with no text:
+  // checks that read renderedText fall back to the raw HTML instead.
+  if (r.error) return { ...page, renderError: r.error };
   return {
     ...page,
     renderedHtml: r.html,

lib/audit/types.ts

@@ -23,6 +23,8 @@ export type FetchedPage = {
   renderedText?: string;
   renderedHtml?: string;
   renderedBytes?: number;
+  /** Why the headless render failed (timeout, navigation error); unset when it worked. */
+  renderError?: string;
   error?: string;
 };
 

lib/crawl/policy.ts

@@ -0,0 +1,178 @@
+// Fleet crawler policy (Anthony, 2026-11-08): "crawlers should timeout after
+// 10 seconds for fetch, 30 for browser (js, images, css). We don't want broken
+// slow shit. Flag to recheck later in a month."
+//
+// One place for the numbers, so a crawler never grows its own private budget
+// again, plus the pure functions that decide what a failure means for the
+// next check.
+//
+// Scope: fetches of THIRD-PARTY urls (pages, feeds, sitemaps, PDFs, og reads)
+// and headless renders of third-party pages. NOT in scope, and deliberately
+// left on their own fast cadence: uptime monitors (lib/uptime.ts), ad
+// link-health (lib/ads/linkHealth.ts), and calls to our own services or
+// vendors (Supabase, CoinPay, AI providers, the c0mpute gateway).
+
+/** Total budget for one plain HTTP fetch of a third-party URL: connect + headers + body. */
+export const CRAWL_FETCH_TIMEOUT_MS = 10_000;
+
+/**
+ * Total budget for a fetch routed through Tor (.onion targets). A Tor circuit
+ * alone routinely takes several seconds to build, so the clearnet 10 s would
+ * fail healthy onion sites; this is the separate, documented ceiling and it
+ * must never exceed the browser budget.
+ */
+export const TOR_FETCH_TIMEOUT_MS = 30_000;
+
+/** Total navigation budget for a headless-browser render of a third-party page. */
+export const CRAWL_BROWSER_TIMEOUT_MS = 30_000;
+
+/**
+ * After `load`/`domcontentloaded`, how long a render may wait for late XHR
+ * before it reads the page anyway. Bounded: `networkidle` never fires on a
+ * page that polls or holds a socket open.
+ */
+export const CRAWL_BROWSER_SETTLE_MS = 2_000;
+
+/** How long a slow or broken target is left alone before its one recheck. */
+export const CRAWL_RECHECK_MS = 30 * 24 * 60 * 60 * 1000;
+
+/** Ceiling on an honoured Retry-After, so a 429 can never park a target longer than a broken one. */
+export const RETRY_AFTER_CAP_MS = CRAWL_RECHECK_MS;
+
+/** Budget for one fetch of `url`: Tor gets its own, everything else the crawl budget. */
+export function fetchBudgetMs(url: string): number {
+  try {
+    return /\.onion$/i.test(new URL(url).hostname) ? TOR_FETCH_TIMEOUT_MS : CRAWL_FETCH_TIMEOUT_MS;
+  } catch {
+    return CRAWL_FETCH_TIMEOUT_MS;
+  }
+}
+
+/** AbortSignal carrying the crawl budget. Pass it to fetch AND keep it alive while reading the body. */
+export function crawlSignal(ms: number = CRAWL_FETCH_TIMEOUT_MS): AbortSignal {
+  return AbortSignal.timeout(ms);
+}
+
+/** Thrown by crawlers for a non-2xx response, so the status reaches the classifier. */
+export class CrawlHttpError extends Error {
+  readonly status: number;
+  readonly retryAfter: string | null;
+  constructor(status: number, retryAfter: string | null = null, message?: string) {
+    super(message ?? `HTTP ${status}`);
+    this.name = "CrawlHttpError";
+    this.status = status;
+    this.retryAfter = retryAfter;
+  }
+}
+
+export type CrawlFailureKind =
+  /** timeout, DNS/TLS/connection error, HTTP 5xx: park for CRAWL_RECHECK_MS. */
+  | "slow_or_broken"
+  /** HTTP 429: honour Retry-After, capped. */
+  | "rate_limited"
+  /** anything else (4xx, unparseable body, empty feed): the caller's existing semantics. */
+  | "other";
+
+const NETWORK_CODES = new Set([
+  "ENOTFOUND",
+  "EAI_AGAIN",
+  "ECONNREFUSED",
+  "ECONNRESET",
+  "ECONNABORTED",
+  "ETIMEDOUT",
+  "EHOSTUNREACH",
+  "ENETUNREACH",
+  "EPIPE",
+  "UND_ERR_CONNECT_TIMEOUT",
+  "UND_ERR_HEADERS_TIMEOUT",
+  "UND_ERR_BODY_TIMEOUT",
+  "UND_ERR_SOCKET",
+  "UND_ERR_CLOSED",
+]);
+
+function errorCode(err: unknown): string | null {
+  if (!err || typeof err !== "object") return null;
+  const e = err as { code?: unknown; cause?: unknown };
+  if (typeof e.code === "string") return e.code;
+  if (e.cause && typeof e.cause === "object") {
+    const c = (e.cause as { code?: unknown }).code;
+    if (typeof c === "string") return c;
+  }
+  return null;
+}
+
+/** Status of a failure: a number for an HTTP response, null for a thrown error. */
+export function classifyHttpStatus(status: number): CrawlFailureKind {
+  if (status === 429) return "rate_limited";
+  if (status >= 500 && status <= 599) return "slow_or_broken";
+  return "other";
+}
+
+/** What a thrown fetch error (or CrawlHttpError) means for the target. */
+export function classifyCrawlError(err: unknown): CrawlFailureKind {
+  if (err instanceof CrawlHttpError) return classifyHttpStatus(err.status);
+  if (!err || typeof err !== "object") return "other";
+  const e = err as { name?: unknown; message?: unknown };
+  const name = typeof e.name === "string" ? e.name : "";
+  const message = typeof e.message === "string" ? e.message : "";
+  // AbortSignal.timeout rejects with TimeoutError; a manual controller.abort() with AbortError.
+  if (name === "TimeoutError" || name === "AbortError") return "slow_or_broken";
+  const code = errorCode(err);
+  if (code && (NETWORK_CODES.has(code) || code.startsWith("ERR_TLS") || code.includes("CERT"))) {
+    return "slow_or_broken";
+  }
+  if ([...NETWORK_CODES].some((c) => message.includes(c))) return "slow_or_broken";
+  // undici's catch-all for DNS/TCP/TLS failure, and Node's own abort wording.
+  if (/fetch failed|aborted|timed? ?out|socket hang up|getaddrinfo|certificate|ssl|tls/i.test(message)) {
+    return "slow_or_broken";
+  }
+  return "other";
+}
+
+/** Parse a Retry-After header (seconds or HTTP date) to milliseconds from `now`, capped. */
+export function retryAfterMs(header: string | null | undefined, now: Date): number | null {
+  if (!header) return null;
+  const trimmed = header.trim();
+  let ms: number | null = null;
+  if (/^\d+$/.test(trimmed)) {
+    ms = Number(trimmed) * 1000;
+  } else {
+    const at = Date.parse(trimmed);
+    if (!Number.isNaN(at)) ms = at - now.getTime();
+  }
+  if (ms === null || !Number.isFinite(ms)) return null;
+  return Math.min(Math.max(0, ms), RETRY_AFTER_CAP_MS);
+}
+
+/** The next time a slow or broken target may be touched: exactly CRAWL_RECHECK_MS out. */
+export function crawlRecheckAt(now: Date): Date {
+  return new Date(now.getTime() + CRAWL_RECHECK_MS);
+}
+
+/**
+ * When to look at a failed target next, or null to leave the caller's own
+ * schedule in charge (the "other" kind: keep existing 4xx semantics).
+ *
+ * slow_or_broken -> now + 30 days. rate_limited -> Retry-After (capped at 30
+ * days), or null without a usable header so the caller's backoff applies.
+ */
+export function nextCheckAfterFailure(
+  kind: CrawlFailureKind,
+  now: Date,
+  retryAfter?: string | null,
+): Date | null {
+  if (kind === "slow_or_broken") return crawlRecheckAt(now);
+  if (kind === "rate_limited") {
+    const ms = retryAfterMs(retryAfter ?? null, now);
+    return ms === null ? null : new Date(now.getTime() + ms);
+  }
+  return null;
+}
+
+/** True once a parked target's 30 days are up and it is owed its one recheck. */
+export function recheckDue(lastAttemptAt: string | Date | null | undefined, now: Date): boolean {
+  if (!lastAttemptAt) return true;
+  const at = typeof lastAttemptAt === "string" ? Date.parse(lastAttemptAt) : lastAttemptAt.getTime();
+  if (Number.isNaN(at)) return true;
+  return now.getTime() - at >= CRAWL_RECHECK_MS;
+}

lib/lx/feedCrawl.ts

@@ -19,11 +19,12 @@
 import type { SupabaseClient } from "@supabase/supabase-js";
 import { itemEntries } from "./feedTopics";
 import { resolveMasters, tokens } from "./topicPlan";
+import { CRAWL_FETCH_TIMEOUT_MS, CRAWL_RECHECK_MS } from "@/lib/crawl/policy";
 
 const RSSAMPLIFIER = "https://rssamplifier.com";
 
-/** Per-feed budget. Generous next to delivery's, because nothing waits on it. */
-const FETCH_TIMEOUT_MS = 12_000;
+/** Per-feed budget: the fleet crawl budget (lib/crawl/policy.ts). */
+const FETCH_TIMEOUT_MS = CRAWL_FETCH_TIMEOUT_MS;
 
 /** Feeds read per sweep. Bounded so one tick cannot run for an hour. */
 const PER_SWEEP = 25;
@@ -41,6 +42,17 @@ export const REFRESH_AFTER_MS = 6 * 60 * 60 * 1000; // 6h
  */
 const GIVE_UP_AFTER = 8;
 
+/**
+ * Parked is not dead for ever: a given_up source gets one recheck every
+ * CRAWL_RECHECK_MS (30 days) after its last attempt. Success puts it back to
+ * active with a clean failure count; failure leaves it parked and stamps
+ * last_fetch_at, which starts the next 30 days.
+ */
+export const GIVEN_UP_RECHECK_MS = CRAWL_RECHECK_MS;
+
+/** Parked sources rechecked per sweep, so a mass give-up cannot crowd out live feeds. */
+const RECHECKS_PER_SWEEP = 5;
+
 /** Items kept per source. Older ones are pruned so the table stays bounded. */
 const KEEP_PER_SOURCE = 40;
 
@@ -132,6 +144,15 @@ export async function crawlFeeds(
     .order("last_fetch_at", { ascending: true, nullsFirst: true })
     .limit(PER_SWEEP);
 
+  const recheckBefore = new Date(Date.now() - GIVEN_UP_RECHECK_MS).toISOString();
+  const { data: parked } = await supabase
+    .from("lx_feed_source")
+    .select("id, topic, url, consecutive_failures")
+    .eq("status", "given_up")
+    .lt("last_fetch_at", recheckBefore)
+    .order("last_fetch_at", { ascending: true })
+    .limit(RECHECKS_PER_SWEEP);
+
   const result: FeedSweepResult = {
     sources,
     fetched: 0,
@@ -140,7 +161,7 @@ export async function crawlFeeds(
     gaveUp: 0,
   };
 
-  for (const source of (batch ?? []) as Array<{
+  for (const source of [...(batch ?? []), ...(parked ?? [])] as Array<{
     id: string;
     topic: string;
     url: string;