diff --git a/app/actions/socialPosting.ts b/app/actions/socialPosting.ts index 0d6e1241..bf730b8f 100644 --- a/app/actions/socialPosting.ts +++ b/app/actions/socialPosting.ts @@ -549,6 +549,9 @@ export async function saveFeedAutopostSettings( ignore_paths: ignorePaths, status: "idle", last_error: null, + // A saved config is a fresh start: drop any 30-day crawl flag. + failed_at: null, + next_check_at: null, }, { onConflict: "project_id" }, ); diff --git a/lib/audit/fetch.ts b/lib/audit/fetch.ts index 9cce6521..3ce28bea 100644 --- a/lib/audit/fetch.ts +++ b/lib/audit/fetch.ts @@ -1,14 +1,16 @@ import type { FetchedPage } from "./types"; import { smartFetch } from "../onion"; +import { fetchBudgetMs } from "../crawl/policy"; const UA = "CrawlProofBot/1.0 (+https://crawlproof.com/bot)"; -const TIMEOUT_MS = 15_000; const MAX_BYTES = 4 * 1024 * 1024; export async function fetchPage(url: string): Promise { const started = Date.now(); const controller = new AbortController(); - const timer = setTimeout(() => controller.abort(), TIMEOUT_MS); + // 10 s clearnet, the separate Tor budget for .onion (lib/crawl/policy.ts); + // the timer covers the body read below too. + const timer = setTimeout(() => controller.abort(), fetchBudgetMs(url)); try { // smartFetch routes .onion targets through Tor (TOR_SOCKS_URL); everything // else is a normal fetch. diff --git a/lib/audit/render.ts b/lib/audit/render.ts index bebf9728..a5c024a1 100644 --- a/lib/audit/render.ts +++ b/lib/audit/render.ts @@ -1,4 +1,5 @@ import type { FetchedPage } from "./types"; +import { CRAWL_BROWSER_SETTLE_MS, CRAWL_BROWSER_TIMEOUT_MS } from "../crawl/policy"; // Lazily-imported to avoid loading Playwright when not needed. let _chromium: typeof import("playwright").chromium | null = null; @@ -25,7 +26,15 @@ export async function renderPage(url: string): Promise<{ browser = await chromium.launch({ args: ["--no-sandbox"] }); const ctx = await browser.newContext({ userAgent: UA }); const page = await ctx.newPage(); - await page.goto(url, { waitUntil: "networkidle", timeout: 20_000 }); + // Fleet crawler policy: 30 s for the whole render. Wait for `load`, then a + // bounded settle for late XHR — never an open-ended networkidle, which a + // page that polls never reaches. A navigation timeout throws and is + // returned as `error`, not read as an empty page. + const deadline = Date.now() + CRAWL_BROWSER_TIMEOUT_MS; + page.setDefaultTimeout(CRAWL_BROWSER_TIMEOUT_MS); + await page.goto(url, { waitUntil: "load", timeout: CRAWL_BROWSER_TIMEOUT_MS }); + const settle = Math.min(CRAWL_BROWSER_SETTLE_MS, Math.max(0, deadline - Date.now())); + if (settle > 0) await page.waitForLoadState("networkidle", { timeout: settle }).catch(() => {}); const html = await page.content(); const text = await page.evaluate(() => document.body?.innerText ?? ""); return { @@ -49,6 +58,9 @@ export async function renderPage(url: string): Promise<{ export async function attachRendered(page: FetchedPage): Promise { const r = await renderPage(page.url); + // A failed render is recorded as a failure, not as a page with no text: + // checks that read renderedText fall back to the raw HTML instead. + if (r.error) return { ...page, renderError: r.error }; return { ...page, renderedHtml: r.html, diff --git a/lib/audit/types.ts b/lib/audit/types.ts index 5980cb46..7d2d8fc9 100644 --- a/lib/audit/types.ts +++ b/lib/audit/types.ts @@ -23,6 +23,8 @@ export type FetchedPage = { renderedText?: string; renderedHtml?: string; renderedBytes?: number; + /** Why the headless render failed (timeout, navigation error); unset when it worked. */ + renderError?: string; error?: string; }; diff --git a/lib/crawl/policy.ts b/lib/crawl/policy.ts new file mode 100644 index 00000000..65ab4d2d --- /dev/null +++ b/lib/crawl/policy.ts @@ -0,0 +1,178 @@ +// Fleet crawler policy (Anthony, 2026-11-08): "crawlers should timeout after +// 10 seconds for fetch, 30 for browser (js, images, css). We don't want broken +// slow shit. Flag to recheck later in a month." +// +// One place for the numbers, so a crawler never grows its own private budget +// again, plus the pure functions that decide what a failure means for the +// next check. +// +// Scope: fetches of THIRD-PARTY urls (pages, feeds, sitemaps, PDFs, og reads) +// and headless renders of third-party pages. NOT in scope, and deliberately +// left on their own fast cadence: uptime monitors (lib/uptime.ts), ad +// link-health (lib/ads/linkHealth.ts), and calls to our own services or +// vendors (Supabase, CoinPay, AI providers, the c0mpute gateway). + +/** Total budget for one plain HTTP fetch of a third-party URL: connect + headers + body. */ +export const CRAWL_FETCH_TIMEOUT_MS = 10_000; + +/** + * Total budget for a fetch routed through Tor (.onion targets). A Tor circuit + * alone routinely takes several seconds to build, so the clearnet 10 s would + * fail healthy onion sites; this is the separate, documented ceiling and it + * must never exceed the browser budget. + */ +export const TOR_FETCH_TIMEOUT_MS = 30_000; + +/** Total navigation budget for a headless-browser render of a third-party page. */ +export const CRAWL_BROWSER_TIMEOUT_MS = 30_000; + +/** + * After `load`/`domcontentloaded`, how long a render may wait for late XHR + * before it reads the page anyway. Bounded: `networkidle` never fires on a + * page that polls or holds a socket open. + */ +export const CRAWL_BROWSER_SETTLE_MS = 2_000; + +/** How long a slow or broken target is left alone before its one recheck. */ +export const CRAWL_RECHECK_MS = 30 * 24 * 60 * 60 * 1000; + +/** Ceiling on an honoured Retry-After, so a 429 can never park a target longer than a broken one. */ +export const RETRY_AFTER_CAP_MS = CRAWL_RECHECK_MS; + +/** Budget for one fetch of `url`: Tor gets its own, everything else the crawl budget. */ +export function fetchBudgetMs(url: string): number { + try { + return /\.onion$/i.test(new URL(url).hostname) ? TOR_FETCH_TIMEOUT_MS : CRAWL_FETCH_TIMEOUT_MS; + } catch { + return CRAWL_FETCH_TIMEOUT_MS; + } +} + +/** AbortSignal carrying the crawl budget. Pass it to fetch AND keep it alive while reading the body. */ +export function crawlSignal(ms: number = CRAWL_FETCH_TIMEOUT_MS): AbortSignal { + return AbortSignal.timeout(ms); +} + +/** Thrown by crawlers for a non-2xx response, so the status reaches the classifier. */ +export class CrawlHttpError extends Error { + readonly status: number; + readonly retryAfter: string | null; + constructor(status: number, retryAfter: string | null = null, message?: string) { + super(message ?? `HTTP ${status}`); + this.name = "CrawlHttpError"; + this.status = status; + this.retryAfter = retryAfter; + } +} + +export type CrawlFailureKind = + /** timeout, DNS/TLS/connection error, HTTP 5xx: park for CRAWL_RECHECK_MS. */ + | "slow_or_broken" + /** HTTP 429: honour Retry-After, capped. */ + | "rate_limited" + /** anything else (4xx, unparseable body, empty feed): the caller's existing semantics. */ + | "other"; + +const NETWORK_CODES = new Set([ + "ENOTFOUND", + "EAI_AGAIN", + "ECONNREFUSED", + "ECONNRESET", + "ECONNABORTED", + "ETIMEDOUT", + "EHOSTUNREACH", + "ENETUNREACH", + "EPIPE", + "UND_ERR_CONNECT_TIMEOUT", + "UND_ERR_HEADERS_TIMEOUT", + "UND_ERR_BODY_TIMEOUT", + "UND_ERR_SOCKET", + "UND_ERR_CLOSED", +]); + +function errorCode(err: unknown): string | null { + if (!err || typeof err !== "object") return null; + const e = err as { code?: unknown; cause?: unknown }; + if (typeof e.code === "string") return e.code; + if (e.cause && typeof e.cause === "object") { + const c = (e.cause as { code?: unknown }).code; + if (typeof c === "string") return c; + } + return null; +} + +/** Status of a failure: a number for an HTTP response, null for a thrown error. */ +export function classifyHttpStatus(status: number): CrawlFailureKind { + if (status === 429) return "rate_limited"; + if (status >= 500 && status <= 599) return "slow_or_broken"; + return "other"; +} + +/** What a thrown fetch error (or CrawlHttpError) means for the target. */ +export function classifyCrawlError(err: unknown): CrawlFailureKind { + if (err instanceof CrawlHttpError) return classifyHttpStatus(err.status); + if (!err || typeof err !== "object") return "other"; + const e = err as { name?: unknown; message?: unknown }; + const name = typeof e.name === "string" ? e.name : ""; + const message = typeof e.message === "string" ? e.message : ""; + // AbortSignal.timeout rejects with TimeoutError; a manual controller.abort() with AbortError. + if (name === "TimeoutError" || name === "AbortError") return "slow_or_broken"; + const code = errorCode(err); + if (code && (NETWORK_CODES.has(code) || code.startsWith("ERR_TLS") || code.includes("CERT"))) { + return "slow_or_broken"; + } + if ([...NETWORK_CODES].some((c) => message.includes(c))) return "slow_or_broken"; + // undici's catch-all for DNS/TCP/TLS failure, and Node's own abort wording. + if (/fetch failed|aborted|timed? ?out|socket hang up|getaddrinfo|certificate|ssl|tls/i.test(message)) { + return "slow_or_broken"; + } + return "other"; +} + +/** Parse a Retry-After header (seconds or HTTP date) to milliseconds from `now`, capped. */ +export function retryAfterMs(header: string | null | undefined, now: Date): number | null { + if (!header) return null; + const trimmed = header.trim(); + let ms: number | null = null; + if (/^\d+$/.test(trimmed)) { + ms = Number(trimmed) * 1000; + } else { + const at = Date.parse(trimmed); + if (!Number.isNaN(at)) ms = at - now.getTime(); + } + if (ms === null || !Number.isFinite(ms)) return null; + return Math.min(Math.max(0, ms), RETRY_AFTER_CAP_MS); +} + +/** The next time a slow or broken target may be touched: exactly CRAWL_RECHECK_MS out. */ +export function crawlRecheckAt(now: Date): Date { + return new Date(now.getTime() + CRAWL_RECHECK_MS); +} + +/** + * When to look at a failed target next, or null to leave the caller's own + * schedule in charge (the "other" kind: keep existing 4xx semantics). + * + * slow_or_broken -> now + 30 days. rate_limited -> Retry-After (capped at 30 + * days), or null without a usable header so the caller's backoff applies. + */ +export function nextCheckAfterFailure( + kind: CrawlFailureKind, + now: Date, + retryAfter?: string | null, +): Date | null { + if (kind === "slow_or_broken") return crawlRecheckAt(now); + if (kind === "rate_limited") { + const ms = retryAfterMs(retryAfter ?? null, now); + return ms === null ? null : new Date(now.getTime() + ms); + } + return null; +} + +/** True once a parked target's 30 days are up and it is owed its one recheck. */ +export function recheckDue(lastAttemptAt: string | Date | null | undefined, now: Date): boolean { + if (!lastAttemptAt) return true; + const at = typeof lastAttemptAt === "string" ? Date.parse(lastAttemptAt) : lastAttemptAt.getTime(); + if (Number.isNaN(at)) return true; + return now.getTime() - at >= CRAWL_RECHECK_MS; +} diff --git a/lib/lx/feedCrawl.ts b/lib/lx/feedCrawl.ts index f20c4ae5..559790e4 100644 --- a/lib/lx/feedCrawl.ts +++ b/lib/lx/feedCrawl.ts @@ -19,11 +19,12 @@ import type { SupabaseClient } from "@supabase/supabase-js"; import { itemEntries } from "./feedTopics"; import { resolveMasters, tokens } from "./topicPlan"; +import { CRAWL_FETCH_TIMEOUT_MS, CRAWL_RECHECK_MS } from "@/lib/crawl/policy"; const RSSAMPLIFIER = "https://rssamplifier.com"; -/** Per-feed budget. Generous next to delivery's, because nothing waits on it. */ -const FETCH_TIMEOUT_MS = 12_000; +/** Per-feed budget: the fleet crawl budget (lib/crawl/policy.ts). */ +const FETCH_TIMEOUT_MS = CRAWL_FETCH_TIMEOUT_MS; /** Feeds read per sweep. Bounded so one tick cannot run for an hour. */ const PER_SWEEP = 25; @@ -41,6 +42,17 @@ export const REFRESH_AFTER_MS = 6 * 60 * 60 * 1000; // 6h */ const GIVE_UP_AFTER = 8; +/** + * Parked is not dead for ever: a given_up source gets one recheck every + * CRAWL_RECHECK_MS (30 days) after its last attempt. Success puts it back to + * active with a clean failure count; failure leaves it parked and stamps + * last_fetch_at, which starts the next 30 days. + */ +export const GIVEN_UP_RECHECK_MS = CRAWL_RECHECK_MS; + +/** Parked sources rechecked per sweep, so a mass give-up cannot crowd out live feeds. */ +const RECHECKS_PER_SWEEP = 5; + /** Items kept per source. Older ones are pruned so the table stays bounded. */ const KEEP_PER_SOURCE = 40; @@ -132,6 +144,15 @@ export async function crawlFeeds( .order("last_fetch_at", { ascending: true, nullsFirst: true }) .limit(PER_SWEEP); + const recheckBefore = new Date(Date.now() - GIVEN_UP_RECHECK_MS).toISOString(); + const { data: parked } = await supabase + .from("lx_feed_source") + .select("id, topic, url, consecutive_failures") + .eq("status", "given_up") + .lt("last_fetch_at", recheckBefore) + .order("last_fetch_at", { ascending: true }) + .limit(RECHECKS_PER_SWEEP); + const result: FeedSweepResult = { sources, fetched: 0, @@ -140,7 +161,7 @@ export async function crawlFeeds( gaveUp: 0, }; - for (const source of (batch ?? []) as Array<{ + for (const source of [...(batch ?? []), ...(parked ?? [])] as Array<{ id: string; topic: string; url: string; diff --git a/lib/lx/sitemapCrawl.ts b/lib/lx/sitemapCrawl.ts index de160814..219cd4d8 100644 --- a/lib/lx/sitemapCrawl.ts +++ b/lib/lx/sitemapCrawl.ts @@ -18,8 +18,9 @@ import type { SupabaseClient } from "@supabase/supabase-js"; import OpenAI from "openai"; +import { CRAWL_FETCH_TIMEOUT_MS, CrawlHttpError, crawlRecheckAt } from "@/lib/crawl/policy"; -const FETCH_TIMEOUT_MS = 8000; +const FETCH_TIMEOUT_MS = CRAWL_FETCH_TIMEOUT_MS; const UA = "Crawlproof-LinkExchange/1.0 (+https://crawlproof.com)"; const MAX_PAGES = 200; const HTML_CONCURRENCY = 5; @@ -52,6 +53,88 @@ async function fetchText(url: string): Promise { } } +// The root sitemap, or a thrown error saying why not. Unlike fetchText this +// does not swallow the failure: a sitemap that times out or 5xxs used to be +// recorded as "empty", which reads as "this site has no pages" and is never +// retried. +async function fetchRoot(url: string): Promise { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS); + try { + const res = await fetch(url, { + method: "GET", + redirect: "follow", + signal: controller.signal, + headers: { "user-agent": UA }, + }); + if (!res.ok) throw new CrawlHttpError(res.status, res.headers.get("retry-after")); + return await res.text(); + } catch (err) { + if (controller.signal.aborted) throw new Error(`timed out after ${FETCH_TIMEOUT_MS / 1000}s`); + throw err; + } finally { + clearTimeout(timer); + } +} + +/** + * Columns for a failed crawl: flag it and schedule the one recheck 30 days out + * (fleet crawler policy, lib/crawl/policy.ts). Covers our own failures + * (embedding quota, DB) too: before this an errored sitemap was never crawled + * again unless somebody pressed refresh. + */ +export function sitemapFailureFields(status: string, now: Date) { + return { + sitemap_status: status, + last_sitemap_fetch_at: now.toISOString(), + sitemap_failed_at: now.toISOString(), + sitemap_next_check_at: crawlRecheckAt(now).toISOString(), + }; +} + +/** Columns for a crawl that finished: clears any flag. */ +export function sitemapSuccessFields(status: string, now: Date) { + return { + sitemap_status: status, + last_sitemap_fetch_at: now.toISOString(), + sitemap_failed_at: null, + sitemap_next_check_at: null, + }; +} + +/** A recheck claim holds the row this long, so a crash mid-crawl is retried rather than lost. */ +const RECHECK_CLAIM_MS = 24 * 60 * 60 * 1000; + +/** + * Sites whose failed sitemap is owed its 30-day recheck, claimed (next check + * pushed a day out) so an overlapping sweep cannot crawl them twice. The crawl + * itself then writes the real outcome. + */ +export async function claimDueSitemapRechecks( + supabase: SupabaseClient, + now: Date = new Date(), + limit = 5, +): Promise { + const { data } = await supabase + .from("lx_site") + .select("id") + .lte("sitemap_next_check_at", now.toISOString()) + .order("sitemap_next_check_at", { ascending: true }) + .limit(limit); + const ids: string[] = []; + for (const row of (data ?? []) as Array<{ id: string }>) { + const { data: claimed } = await supabase + .from("lx_site") + .update({ sitemap_next_check_at: new Date(now.getTime() + RECHECK_CLAIM_MS).toISOString() }) + .eq("id", row.id) + .lte("sitemap_next_check_at", now.toISOString()) + .select("id") + .maybeSingle(); + if (claimed) ids.push(row.id); + } + return ids; +} + // Extract ... values. Works for both urlset and sitemapindex. export function extractLocs(xml: string): string[] { const out: string[] = []; @@ -135,8 +218,7 @@ async function collectUrls(xml: string, depth: number, acc: string[]): Promise { - const root = await fetchText(sitemapUrl); - if (!root) return []; + const root = await fetchRoot(sitemapUrl); const acc: string[] = []; await collectUrls(root, 0, acc); return acc; @@ -263,10 +345,9 @@ export async function crawlSitemap( } catch (err) { await supabase .from("lx_site") - .update({ - sitemap_status: `error: ${err instanceof Error ? err.message : "fetch failed"}`, - last_sitemap_fetch_at: new Date().toISOString(), - }) + .update( + sitemapFailureFields(`error: ${err instanceof Error ? err.message : "fetch failed"}`, new Date()), + ) .eq("id", site.id); return { ok: false, @@ -280,10 +361,7 @@ export async function crawlSitemap( if (urls.length === 0) { await supabase .from("lx_site") - .update({ - sitemap_status: "empty", - last_sitemap_fetch_at: new Date().toISOString(), - }) + .update(sitemapSuccessFields("empty", new Date())) .eq("id", site.id); return { ok: true, discovered: 0, fetched: 0, embedded: 0 }; } @@ -308,10 +386,12 @@ export async function crawlSitemap( } catch (err) { await supabase .from("lx_site") - .update({ - sitemap_status: `error: embedding ${err instanceof Error ? err.message : "failed"}`, - last_sitemap_fetch_at: new Date().toISOString(), - }) + .update( + sitemapFailureFields( + `error: embedding ${err instanceof Error ? err.message : "failed"}`, + new Date(), + ), + ) .eq("id", site.id); return { ok: false, @@ -345,10 +425,7 @@ export async function crawlSitemap( if (error) { await supabase .from("lx_site") - .update({ - sitemap_status: `error: ${error.message}`, - last_sitemap_fetch_at: new Date().toISOString(), - }) + .update(sitemapFailureFields(`error: ${error.message}`, new Date())) .eq("id", site.id); return { ok: false, @@ -362,10 +439,7 @@ export async function crawlSitemap( await supabase .from("lx_site") - .update({ - sitemap_status: `ok (${rows.length} pages)`, - last_sitemap_fetch_at: now, - }) + .update(sitemapSuccessFields(`ok (${rows.length} pages)`, new Date(now))) .eq("id", site.id); return { diff --git a/lib/onion.ts b/lib/onion.ts index 1eaeff73..f24031a6 100644 --- a/lib/onion.ts +++ b/lib/onion.ts @@ -38,6 +38,8 @@ function torDispatcher(): Dispatcher { command: "connect", // host is the .onion — SOCKS5 sends it to Tor to resolve (socks5h). destination: { host: opts.hostname, port: destPort }, + // Circuit build only; the whole request is bounded by TOR_FETCH_TIMEOUT_MS + // (lib/crawl/policy.ts) in the caller. timeout: 20_000, }) .then(({ socket }) => { diff --git a/lib/outreach/discover.ts b/lib/outreach/discover.ts index f2c103e2..494c6955 100644 --- a/lib/outreach/discover.ts +++ b/lib/outreach/discover.ts @@ -27,6 +27,7 @@ import { looksLikeLoginWall } from "./loginWall"; import { extractPerson, type ExtractedPerson } from "./person"; import { findNextPageUrl } from "./pagination"; import type { SeedCredentials } from "./seedLogin"; +import { crawlSignal } from "@/lib/crawl/policy"; /** A person found on a page, with where they were found. */ export type DiscoveredPerson = ExtractedPerson & { sourceUrl: string }; @@ -288,7 +289,7 @@ async function fetchHtml( try { const res = await fetch(url, { headers: { "user-agent": SEED_UA }, - signal: AbortSignal.timeout(15_000), + signal: crawlSignal(), redirect: "follow", }); if (!res.ok) return { ok: false, status: res.status, error: `HTTP ${res.status}` }; diff --git a/lib/outreach/documents.ts b/lib/outreach/documents.ts index bc176e1f..23df13f3 100644 --- a/lib/outreach/documents.ts +++ b/lib/outreach/documents.ts @@ -11,10 +11,11 @@ import { extractText, getDocumentProxy } from "unpdf"; import { discoverContactEmails, normalizeHost, type ContactCandidate } from "./cold"; +import { CRAWL_FETCH_TIMEOUT_MS } from "@/lib/crawl/policy"; const MAX_PDF_BYTES = 12 * 1024 * 1024; const MAX_PDF_PAGES = 25; -const FETCH_TIMEOUT_MS = 20_000; +const FETCH_TIMEOUT_MS = CRAWL_FETCH_TIMEOUT_MS; /** Filenames that suggest a document naming humans, best first. */ const PROMISING_NAME = diff --git a/lib/outreach/render.ts b/lib/outreach/render.ts index 98089100..97c08175 100644 --- a/lib/outreach/render.ts +++ b/lib/outreach/render.ts @@ -30,9 +30,12 @@ import type { SeedCredentials } from "./seedLogin"; import type { CodeWaiter } from "@/lib/sp/verificationChallenge"; import dns from "node:dns/promises"; import net from "node:net"; +import { CRAWL_BROWSER_SETTLE_MS, CRAWL_BROWSER_TIMEOUT_MS } from "@/lib/crawl/policy"; -const NAV_TIMEOUT_MS = 25_000; -const SETTLE_MS = 2_500; +// Fleet crawler policy (lib/crawl/policy.ts): one 30 s budget per navigation +// covering load, settle and scrolling, and a settle of at most 2 s. +const NAV_TIMEOUT_MS = CRAWL_BROWSER_TIMEOUT_MS; +const SETTLE_MS = CRAWL_BROWSER_SETTLE_MS; const IDLE_SHUTDOWN_MS = 60_000; const MAX_HTML_BYTES = 3 * 1024 * 1024; /** Bounded so an infinite feed cannot hold a render open forever. */ @@ -113,6 +116,12 @@ export type RenderResult = loginRequired?: boolean; }; +/** Wait for network quiet, but never longer than `ms`; a page that never goes quiet just moves on. */ +async function settle(page: import("playwright").Page, ms: number): Promise { + if (ms <= 0) return; + await page.waitForLoadState("networkidle", { timeout: ms }).catch(() => {}); +} + /** Page titles a bot-protection interstitial serves instead of the real page. */ const CHALLENGE_RE = /just a moment|attention required|verifying you are human|checking your browser/i; @@ -165,6 +174,9 @@ export async function renderPage( const page = await context.newPage(); page.setDefaultTimeout(NAV_TIMEOUT_MS); + let deadline = Date.now() + NAV_TIMEOUT_MS; + const left = () => Math.max(0, deadline - Date.now()); + const response = await page.goto(url, { waitUntil: "domcontentloaded", timeout: NAV_TIMEOUT_MS, @@ -174,9 +186,7 @@ export async function renderPage( // Let XHR-driven listings arrive. networkidle is the obvious choice but // never fires on pages that poll or hold a socket open, so this waits for // quiet with a hard ceiling instead. - await page - .waitForLoadState("networkidle", { timeout: SETTLE_MS * 2 }) - .catch(() => page.waitForTimeout(SETTLE_MS)); + await settle(page, Math.min(SETTLE_MS, left())); // Directories commonly load a first page of entries and fetch the rest on // scroll. Rendering without scrolling therefore sees a third of a list and @@ -185,13 +195,13 @@ export async function renderPage( // static page costs one measurement rather than the full budget. let previousHeight = 0; for (let i = 0; i < MAX_SCROLLS; i++) { + // The scroll pass spends whatever the navigation left of the budget. + if (left() < SCROLL_SETTLE_MS) break; const height = await page.evaluate(() => document.body.scrollHeight).catch(() => 0); if (!height || height === previousHeight) break; previousHeight = height; await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight)).catch(() => {}); - await page - .waitForLoadState("networkidle", { timeout: SCROLL_SETTLE_MS }) - .catch(() => page.waitForTimeout(SCROLL_SETTLE_MS)); + await settle(page, Math.min(SCROLL_SETTLE_MS, left())); } const title = await page.title().catch(() => ""); @@ -226,10 +236,10 @@ export async function renderPage( }; } + // A fresh navigation after sign-in gets a fresh budget. + deadline = Date.now() + NAV_TIMEOUT_MS; await page.goto(url, { waitUntil: "domcontentloaded", timeout: NAV_TIMEOUT_MS }); - await page - .waitForLoadState("networkidle", { timeout: SETTLE_MS * 2 }) - .catch(() => page.waitForTimeout(SETTLE_MS)); + await settle(page, Math.min(SETTLE_MS, left())); html = await page.content(); finalUrl = page.url(); diff --git a/lib/promote/ingest.ts b/lib/promote/ingest.ts index 253d8275..513afb4c 100644 --- a/lib/promote/ingest.ts +++ b/lib/promote/ingest.ts @@ -12,16 +12,27 @@ import type { SupabaseClient } from "@supabase/supabase-js"; import { parseFeed, type ParsedFeedItem } from "@/lib/promote/feedParse"; import { canonicalizeUrl, normalizeUrlForIdentity, urlHash } from "@/lib/promote/normalizeUrl"; +import { + CRAWL_FETCH_TIMEOUT_MS, + CRAWL_RECHECK_MS, + CrawlHttpError, + classifyCrawlError, + nextCheckAfterFailure, +} from "@/lib/crawl/policy"; const USER_AGENT = "CrawlProofPromote/1.0 (+https://crawlproof.com)"; -const FETCH_TIMEOUT_MS = 15_000; const MAX_ITEMS_PER_FEED = 50; const MAX_BODY_BYTES = 5_000_000; // A failing feed backs off geometrically instead of being polled every 15 // minutes forever. Capped so a feed that comes back is noticed within a day. +// That daily cap only applies to "soft" failures (a 4xx, a page that is not a +// feed); a slow or broken feed (timeout, DNS/TLS/connection error, 5xx) is +// parked for 30 days by the fleet crawler policy (lib/crawl/policy.ts). const MAX_BACKOFF_MULTIPLIER = 32; const MAX_BACKOFF_SECONDS = 86_400; +/** Soft failures in a row before the feed stops being polled daily and is parked too. */ +const PARK_SOFT_FAILURES_AFTER = 8; export type FetchLike = ( url: string, @@ -183,14 +194,26 @@ async function ingestOneFeed( if (feed.etag) headers["if-none-match"] = feed.etag; if (feed.last_modified) headers["if-modified-since"] = feed.last_modified; + // One budget for connect + headers + body: the timer stays armed until the + // body has been read, so a server that trickles bytes cannot hold the pass. const controller = new AbortController(); - const timer = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS); - let response: Awaited>; + const timer = setTimeout(() => controller.abort(), CRAWL_FETCH_TIMEOUT_MS); try { - response = await doFetch(feed.feed_url, { headers, signal: controller.signal }); + return await ingestFetched(supabase, feed, now, doFetch, headers, controller.signal); } finally { clearTimeout(timer); } +} + +async function ingestFetched( + supabase: SupabaseClient, + feed: FeedRow, + now: Date, + doFetch: FetchLike, + headers: Record, + signal: AbortSignal, +): Promise<{ unchanged: boolean; failed: boolean; itemsStored: number; linksCreated: number }> { + const response = await doFetch(feed.feed_url, { headers, signal }); // Unchanged since last time: the common case, and the cheap one. if (response.status === 304) { @@ -208,7 +231,16 @@ async function ingestOneFeed( } if (!response.ok) { - await recordFailure(supabase, feed, now, new Error(`Feed returned HTTP ${response.status}`)); + await recordFailure( + supabase, + feed, + now, + new CrawlHttpError( + response.status, + response.headers.get("retry-after"), + `Feed returned HTTP ${response.status}`, + ), + ); return { unchanged: false, failed: true, itemsStored: 0, linksCreated: 0 }; } @@ -375,11 +407,36 @@ async function recordFailure( last_fetched_at: now.toISOString(), consecutive_failures: failures, last_error: err instanceof Error ? err.message : String(err), - next_fetch_at: nextFetchAt({ ...feed, consecutive_failures: failures }, now, true), + next_fetch_at: failureNextFetchAt(feed, now, err), }) .eq("id", feed.id); } +/** + * When a feed that just failed is fetched next. + * + * Slow or broken (timeout, DNS/TLS/connection error, 5xx): 30 days out, then + * exactly one recheck; success clears it, failure parks it another 30 days. + * 429: its Retry-After, capped at 30 days. Anything else keeps the geometric + * backoff, until PARK_SOFT_FAILURES_AFTER in a row, so a feed that 404s is + * not polled daily for ever. + */ +export function failureNextFetchAt( + feed: Pick, + now: Date, + err: unknown, +): string { + const failures = (feed.consecutive_failures ?? 0) + 1; + const kind = classifyCrawlError(err); + const retryAfter = err instanceof CrawlHttpError ? err.retryAfter : null; + const flagged = nextCheckAfterFailure(kind, now, retryAfter); + if (flagged) return flagged.toISOString(); + if (failures >= PARK_SOFT_FAILURES_AFTER) { + return new Date(now.getTime() + CRAWL_RECHECK_MS).toISOString(); + } + return nextFetchAt({ ...feed, consecutive_failures: failures }, now, true); +} + /** Geometric backoff while a feed is failing, plain interval while it is healthy. */ export function nextFetchAt( feed: Pick, diff --git a/lib/promote/sources.ts b/lib/promote/sources.ts index 1b492e58..2fd23f11 100644 --- a/lib/promote/sources.ts +++ b/lib/promote/sources.ts @@ -9,6 +9,7 @@ import type { SupabaseClient } from "@supabase/supabase-js"; import { parseFeed } from "@/lib/promote/feedParse"; import { topicFeedUrl, type KeywordInput } from "@/lib/promote/keywords"; import type { FetchLike } from "@/lib/promote/ingest"; +import { CRAWL_FETCH_TIMEOUT_MS } from "@/lib/crawl/policy"; // Same guard the audit engine applies to user-supplied targets: a feed URL is // fetched by our server, so it must not be able to point at our own network. @@ -16,7 +17,7 @@ import type { FetchLike } from "@/lib/promote/ingest"; const PRIVATE_HOSTS = /^(localhost|127\.|10\.|192\.168\.|169\.254\.|172\.(1[6-9]|2\d|3[01])\.|::1|fc00:|fd00:|.*\.local)$/i; -const VALIDATE_TIMEOUT_MS = 15_000; +const VALIDATE_TIMEOUT_MS = CRAWL_FETCH_TIMEOUT_MS; export type FeedKind = "rssamplifier_topic" | "custom_feed" | "project_feed"; diff --git a/lib/sp/feedAutopost.ts b/lib/sp/feedAutopost.ts index 3ff63e42..c5a96abc 100644 --- a/lib/sp/feedAutopost.ts +++ b/lib/sp/feedAutopost.ts @@ -18,6 +18,12 @@ import { shouldAttachImage, type ImageStylePref, } from "@/lib/sp/imageGen"; +import { + CRAWL_FETCH_TIMEOUT_MS, + CrawlHttpError, + classifyCrawlError, + nextCheckAfterFailure, +} from "@/lib/crawl/policy"; export type FeedAutopostClients = { anthropic?: Anthropic | null; @@ -25,10 +31,29 @@ export type FeedAutopostClients = { }; const USER_AGENT = "CrawlProofSocialFeed/1.0"; -const FETCH_TIMEOUT_MS = 12_000; +const FETCH_TIMEOUT_MS = CRAWL_FETCH_TIMEOUT_MS; const MAX_RSS_ITEMS = 50; const MAX_SITEMAP_URLS = 500; const FEED_POLL_EVERY_MS = 15 * 60 * 1000; +/** A feed URL answering 4xx is a config problem, not an outage: look again daily, not every 15 min. */ +const FEED_4XX_RECHECK_MS = 24 * 60 * 60 * 1000; + +/** + * The next_check_at a failed feed check earns, or null to keep the plain + * 15-minute cadence (our own failures: locked keys, DB, rendering, posting). + * + * Fleet crawler policy (lib/crawl/policy.ts): timeout, DNS/TLS/connection + * error or 5xx parks the feed for 30 days, then one recheck; 429 honours + * Retry-After (capped); a 4xx is retried daily instead of every 15 minutes. + */ +export function feedFailureNextCheck(err: unknown, now: Date): Date | null { + const kind = classifyCrawlError(err); + const retryAfter = err instanceof CrawlHttpError ? err.retryAfter : null; + const flagged = nextCheckAfterFailure(kind, now, retryAfter); + if (flagged) return flagged; + if (err instanceof CrawlHttpError) return new Date(now.getTime() + FEED_4XX_RECHECK_MS); + return null; +} // Per-(user, platform) cap on autoposted feed items, across every project // that user owns. Manual posts are not counted. When the gate fires the @@ -133,11 +158,15 @@ export async function processDueSocialFeeds( ): Promise { const now = opts.now ?? new Date(); const cutoff = new Date(now.getTime() - FEED_POLL_EVERY_MS).toISOString(); + const nowIso = now.toISOString(); const { data, error } = await supabase .from("sp_feed_config") .select("id, user_id, project_id, feed_type, feed_url, ignore_paths, last_checked_at") .eq("enabled", true) - .or(`last_checked_at.is.null,last_checked_at.lt.${cutoff}`) + // Due by cadence AND not parked by a failure (next_check_at in the future). + .or( + `and(or(last_checked_at.is.null,last_checked_at.lt.${cutoff}),or(next_check_at.is.null,next_check_at.lte.${nowIso}))`, + ) .order("last_checked_at", { ascending: true, nullsFirst: true }) .limit(opts.limit ?? 10); @@ -283,6 +312,8 @@ async function processSocialFeedConfig( last_success_at: nowIso, last_item_at: newestTimestamp(items) ?? null, last_error: null, + failed_at: null, + next_check_at: null, }) .eq("id", config.id); @@ -298,9 +329,15 @@ async function processSocialFeedConfig( } catch (err) { const message = err instanceof Error ? err.message : String(err); console.error(`[sp] feed check ${config.id} (${config.feed_url ?? "auto"}) failed: ${message}`); + const nextCheck = feedFailureNextCheck(err, opts.now); await supabase .from("sp_feed_config") - .update({ status: "error", last_error: message }) + .update({ + status: "error", + last_error: nextCheck ? `${message} (next check ${nextCheck.toISOString().slice(0, 10)})` : message, + failed_at: nowIso, + next_check_at: nextCheck ? nextCheck.toISOString() : null, + }) .eq("id", config.id); return { ok: false, configId: config.id, error: message }; } @@ -711,8 +748,17 @@ async function fetchText(url: string): Promise { }, signal: ac.signal, }); - if (!res.ok) throw new Error(`Feed returned HTTP ${res.status}`); + if (!res.ok) { + throw new CrawlHttpError(res.status, res.headers.get("retry-after"), `Feed returned HTTP ${res.status}`); + } return await res.text(); + } catch (err) { + if (ac.signal.aborted) { + const timeout = new Error(`Feed timed out after ${FETCH_TIMEOUT_MS / 1000}s`); + timeout.name = "TimeoutError"; + throw timeout; + } + throw err; } finally { clearTimeout(timer); } diff --git a/supabase/migrations/20261108170000_crawl_recheck_30d.sql b/supabase/migrations/20261108170000_crawl_recheck_30d.sql new file mode 100644 index 00000000..b4bdf9fe --- /dev/null +++ b/supabase/migrations/20261108170000_crawl_recheck_30d.sql @@ -0,0 +1,33 @@ +-- Fleet crawler policy (lib/crawl/policy.ts): a slow or broken target +-- (timeout, DNS/TLS/connection error, HTTP 5xx) is flagged and left alone for +-- 30 days, then gets exactly one recheck. Success clears the flag; failure +-- flags it another 30 days. +-- +-- promo_feed already has next_fetch_at / last_error; lx_feed_source parks with +-- status='given_up' + last_fetch_at. These two had nowhere to keep the flag. +-- Apply by hand, one file, in dev2 supabase-db (no `supabase db push`). + +-- Social feed autopost: before this a dead feed was refetched every 15 minutes. +alter table public.sp_feed_config add column if not exists failed_at timestamptz; +alter table public.sp_feed_config add column if not exists next_check_at timestamptz; +comment on column public.sp_feed_config.next_check_at is + 'Set 30 days out when the feed timed out / was unreachable / 5xx; the poller skips it until then. Null = normal 15-minute cadence.'; + +-- Link-exchange sitemaps: an error was only ever text in sitemap_status and +-- nothing retried it. +alter table public.lx_site add column if not exists sitemap_failed_at timestamptz; +alter table public.lx_site add column if not exists sitemap_next_check_at timestamptz; +comment on column public.lx_site.sitemap_next_check_at is + 'When a failed sitemap crawl gets its one recheck (30 days after the failure). Null = nothing scheduled.'; + +create index if not exists lx_site_sitemap_recheck_idx + on public.lx_site (sitemap_next_check_at) + where sitemap_next_check_at is not null; + +-- Sitemaps already sitting in an error state get their recheck 30 days after +-- the failure that put them there (immediately, if that is already past). +update public.lx_site + set sitemap_failed_at = coalesce(last_sitemap_fetch_at, now()), + sitemap_next_check_at = coalesce(last_sitemap_fetch_at, now()) + interval '30 days' + where sitemap_status like 'error%' + and sitemap_next_check_at is null; diff --git a/tests/crawl-policy.test.ts b/tests/crawl-policy.test.ts new file mode 100644 index 00000000..8b189608 --- /dev/null +++ b/tests/crawl-policy.test.ts @@ -0,0 +1,219 @@ +// Fleet crawler policy: 10 s per fetch, 30 s per browser render, and a slow or +// broken target is flagged and left alone for 30 days, then rechecked once. +import { describe, expect, it } from "vitest"; +import { + CRAWL_BROWSER_SETTLE_MS, + CRAWL_BROWSER_TIMEOUT_MS, + CRAWL_FETCH_TIMEOUT_MS, + CRAWL_RECHECK_MS, + CrawlHttpError, + RETRY_AFTER_CAP_MS, + TOR_FETCH_TIMEOUT_MS, + classifyCrawlError, + classifyHttpStatus, + crawlRecheckAt, + fetchBudgetMs, + nextCheckAfterFailure, + recheckDue, + retryAfterMs, +} from "@/lib/crawl/policy"; +import { failureNextFetchAt } from "@/lib/promote/ingest"; +import { feedFailureNextCheck } from "@/lib/sp/feedAutopost"; +import { claimDueSitemapRechecks, sitemapFailureFields, sitemapSuccessFields } from "@/lib/lx/sitemapCrawl"; +import { crawlFeeds, GIVEN_UP_RECHECK_MS } from "@/lib/lx/feedCrawl"; +import { makeFakeSupabase } from "./promote/fake-supabase"; + +const NOW = new Date("2026-11-08T12:00:00.000Z"); +const DAY = 24 * 60 * 60 * 1000; + +describe("budgets", () => { + it("is 10 s per fetch, 30 s per render, settle <= 2 s, 30 days to recheck", () => { + expect(CRAWL_FETCH_TIMEOUT_MS).toBe(10_000); + expect(CRAWL_BROWSER_TIMEOUT_MS).toBe(30_000); + expect(CRAWL_BROWSER_SETTLE_MS).toBeLessThanOrEqual(2_000); + expect(CRAWL_RECHECK_MS).toBe(30 * DAY); + }); + + it("gives Tor its own budget, never above 30 s", () => { + expect(TOR_FETCH_TIMEOUT_MS).toBeGreaterThan(CRAWL_FETCH_TIMEOUT_MS); + expect(TOR_FETCH_TIMEOUT_MS).toBeLessThanOrEqual(30_000); + expect(fetchBudgetMs("http://expyuzz4wqqyqhjn.onion/")).toBe(TOR_FETCH_TIMEOUT_MS); + expect(fetchBudgetMs("https://example.com/")).toBe(CRAWL_FETCH_TIMEOUT_MS); + expect(fetchBudgetMs("not a url")).toBe(CRAWL_FETCH_TIMEOUT_MS); + }); +}); + +describe("classify", () => { + it("treats 5xx as broken, 429 as rate limited, other 4xx as the caller's business", () => { + expect(classifyHttpStatus(500)).toBe("slow_or_broken"); + expect(classifyHttpStatus(503)).toBe("slow_or_broken"); + expect(classifyHttpStatus(429)).toBe("rate_limited"); + expect(classifyHttpStatus(404)).toBe("other"); + expect(classifyHttpStatus(403)).toBe("other"); + }); + + it("treats timeouts and network failures as slow or broken", () => { + const timeout = new Error("The operation was aborted due to timeout"); + timeout.name = "TimeoutError"; + expect(classifyCrawlError(timeout)).toBe("slow_or_broken"); + const abort = new Error("This operation was aborted"); + abort.name = "AbortError"; + expect(classifyCrawlError(abort)).toBe("slow_or_broken"); + expect(classifyCrawlError(Object.assign(new TypeError("fetch failed"), { cause: { code: "ENOTFOUND" } }))).toBe( + "slow_or_broken", + ); + expect(classifyCrawlError(new Error("ECONNRESET"))).toBe("slow_or_broken"); + expect(classifyCrawlError({ code: "CERT_HAS_EXPIRED", message: "certificate has expired" })).toBe( + "slow_or_broken", + ); + expect(classifyCrawlError(new CrawlHttpError(502))).toBe("slow_or_broken"); + }); + + it("leaves content problems alone", () => { + expect(classifyCrawlError(new Error("Feed contained no usable entries"))).toBe("other"); + expect(classifyCrawlError(new CrawlHttpError(404))).toBe("other"); + expect(classifyCrawlError(null)).toBe("other"); + }); +}); + +describe("next check", () => { + it("parks a slow or broken target for exactly 30 days", () => { + expect(crawlRecheckAt(NOW).getTime() - NOW.getTime()).toBe(30 * DAY); + expect(nextCheckAfterFailure("slow_or_broken", NOW)?.toISOString()).toBe("2026-12-08T12:00:00.000Z"); + }); + + it("honours Retry-After on a 429 but caps it at 30 days", () => { + expect(retryAfterMs("120", NOW)).toBe(120_000); + expect(retryAfterMs("Sun, 08 Nov 2026 13:00:00 GMT", NOW)).toBe(3_600_000); + expect(retryAfterMs(String(365 * 86_400), NOW)).toBe(RETRY_AFTER_CAP_MS); + expect(retryAfterMs("garbage", NOW)).toBeNull(); + expect(nextCheckAfterFailure("rate_limited", NOW, "60")?.getTime()).toBe(NOW.getTime() + 60_000); + expect(nextCheckAfterFailure("rate_limited", NOW, null)).toBeNull(); + }); + + it("leaves other failures on the caller's schedule", () => { + expect(nextCheckAfterFailure("other", NOW)).toBeNull(); + }); + + it("owes a recheck once 30 days have passed, not before", () => { + expect(recheckDue(new Date(NOW.getTime() - 29 * DAY), NOW)).toBe(false); + expect(recheckDue(new Date(NOW.getTime() - 30 * DAY), NOW)).toBe(true); + expect(recheckDue(null, NOW)).toBe(true); + }); +}); + +describe("promote feeds", () => { + const feed = { fetch_interval_seconds: 900, consecutive_failures: 0 }; + + it("parks a timed-out or 5xx feed for 30 days instead of a geometric retry", () => { + const timeout = Object.assign(new Error("aborted"), { name: "AbortError" }); + expect(failureNextFetchAt(feed, NOW, timeout)).toBe("2026-12-08T12:00:00.000Z"); + expect(failureNextFetchAt(feed, NOW, new CrawlHttpError(503))).toBe("2026-12-08T12:00:00.000Z"); + }); + + it("keeps the backoff for a 404, until it has failed long enough to park", () => { + expect(failureNextFetchAt(feed, NOW, new CrawlHttpError(404))).toBe("2026-11-08T12:15:00.000Z"); + expect(failureNextFetchAt({ ...feed, consecutive_failures: 7 }, NOW, new CrawlHttpError(404))).toBe( + "2026-12-08T12:00:00.000Z", + ); + }); + + it("follows Retry-After on a 429", () => { + expect(failureNextFetchAt(feed, NOW, new CrawlHttpError(429, "3600"))).toBe("2026-11-08T13:00:00.000Z"); + }); +}); + +describe("social feed autopost", () => { + it("parks an unreachable feed for 30 days, a 4xx for a day, our own errors not at all", () => { + const timeout = Object.assign(new Error("Feed timed out after 10s"), { name: "TimeoutError" }); + expect(feedFailureNextCheck(timeout, NOW)?.toISOString()).toBe("2026-12-08T12:00:00.000Z"); + expect(feedFailureNextCheck(new CrawlHttpError(500), NOW)?.toISOString()).toBe("2026-12-08T12:00:00.000Z"); + expect(feedFailureNextCheck(new CrawlHttpError(404), NOW)?.toISOString()).toBe("2026-11-09T12:00:00.000Z"); + expect(feedFailureNextCheck(new Error("Project not found."), NOW)).toBeNull(); + }); +}); + +describe("sitemaps", () => { + it("flags a failed crawl with a recheck 30 days out and clears it on success", () => { + const failed = sitemapFailureFields("error: timed out after 10s", NOW); + expect(failed.sitemap_next_check_at).toBe("2026-12-08T12:00:00.000Z"); + expect(failed.sitemap_failed_at).toBe(NOW.toISOString()); + const ok = sitemapSuccessFields("ok (3 pages)", NOW); + expect(ok.sitemap_next_check_at).toBeNull(); + expect(ok.sitemap_failed_at).toBeNull(); + }); + + it("claims only sites whose recheck is due, and pushes the claim forward", async () => { + const { client, db } = makeFakeSupabase({ + lx_site: [ + { id: "due", sitemap_next_check_at: "2026-11-01T00:00:00.000Z" }, + { id: "later", sitemap_next_check_at: "2026-12-01T00:00:00.000Z" }, + { id: "healthy", sitemap_next_check_at: null }, + ], + }); + const ids = await claimDueSitemapRechecks(client, NOW); + expect(ids).toEqual(["due"]); + const due = db.lx_site.find((r) => r.id === "due")!; + expect(Date.parse(due.sitemap_next_check_at)).toBeGreaterThan(NOW.getTime()); + }); +}); + +describe("lx feed sources", () => { + it("gives a parked source its one recheck after 30 days and reactivates it on success", async () => { + expect(GIVEN_UP_RECHECK_MS).toBe(30 * DAY); + const updates: Array> = []; + const parked = { id: "src-1", topic: "seo", url: "https://rssamplifier.com/topics/seo.rss", consecutive_failures: 8 }; + const filtersSeen: string[] = []; + const supabase = { + from(table: string) { + const state: { op: string; status?: string; payload?: Record } = { op: "select" }; + const b: any = { + select: () => b, + eq: (col: string, v: string) => { + if (col === "status") state.status = v; + return b; + }, + or: () => b, + lt: (col: string, v: string) => { + filtersSeen.push(`${col}<${v}`); + return b; + }, + order: () => b, + limit: () => b, + not: () => b, + delete: () => { + state.op = "delete"; + return b; + }, + upsert: () => b, + update: (p: Record) => { + state.op = "update"; + state.payload = p; + updates.push(p); + return b; + }, + then: (res: (v: unknown) => unknown) => { + if (table === "lx_feed_source" && state.op === "select" && state.status === "given_up") { + return Promise.resolve({ data: [parked], error: null }).then(res); + } + return Promise.resolve({ data: [], error: null }).then(res); + }, + }; + return b; + }, + }; + const feed = `How small blogs win search traffic in 2026https://blog.example/a`; + const fetchImpl = (async () => new Response(feed, { status: 200 })) as typeof fetch; + + const result = await crawlFeeds(supabase as any, fetchImpl); + + expect(result.fetched).toBe(1); + expect(result.succeeded).toBe(1); + const cutoff = filtersSeen.find((f) => f.startsWith("last_fetch_at<")); + expect(cutoff).toBeDefined(); + expect(Date.now() - Date.parse(cutoff!.split("<")[1])).toBeGreaterThanOrEqual(30 * DAY - 60_000); + const last = updates.at(-1)!; + expect(last.status).toBe("active"); + expect(last.consecutive_failures).toBe(0); + }); +}); diff --git a/tests/promote/ingest.test.ts b/tests/promote/ingest.test.ts index c16d39b0..480c769e 100644 --- a/tests/promote/ingest.test.ts +++ b/tests/promote/ingest.test.ts @@ -249,6 +249,47 @@ describe("ingestDueFeeds", () => { expect(result.feedsFailed).toBe(1); expect(db.promo_feed[0].consecutive_failures).toBe(1); expect(db.promo_feed[0].last_error).toContain("500"); + // A 5xx is a broken target: parked 30 days (lib/crawl/policy.ts). + expect(db.promo_feed[0].next_fetch_at).toBe("2026-09-17T12:00:00.000Z"); + }); + + it("parks a feed that times out for 30 days", async () => { + const { client, db } = makeFakeSupabase(seed(), CONSTRAINTS); + const fetchImpl: FetchLike = async () => { + throw Object.assign(new Error("The operation was aborted due to timeout"), { name: "TimeoutError" }); + }; + + await ingestDueFeeds(client, { fetchImpl, now: NOW }); + + expect(db.promo_feed[0].next_fetch_at).toBe("2026-09-17T12:00:00.000Z"); + }); + + it("clears the flag when the recheck succeeds", async () => { + const { client, db } = makeFakeSupabase( + seed({ + promo_feed: [ + { + id: "feed-bitcoin", + feed_url: "https://rssamplifier.com/topics/bitcoin.rss", + kind: "rssamplifier_topic", + etag: null, + last_modified: null, + fetch_interval_seconds: 900, + consecutive_failures: 3, + last_error: "The operation was aborted due to timeout", + next_fetch_at: "2026-08-18T11:00:00.000Z", + }, + ], + }), + CONSTRAINTS, + ); + const fetchImpl: FetchLike = async () => response(bitcoinFeed); + + await ingestDueFeeds(client, { fetchImpl, now: NOW }); + + expect(db.promo_feed[0].consecutive_failures).toBe(0); + expect(db.promo_feed[0].last_error).toBeNull(); + expect(db.promo_feed[0].next_fetch_at).toBe("2026-08-18T12:15:00.000Z"); }); it("treats a page that is not a feed as a failure", async () => { diff --git a/worker/index.ts b/worker/index.ts index a009a051..67230f65 100644 --- a/worker/index.ts +++ b/worker/index.ts @@ -27,7 +27,7 @@ import { quoteFromFindings, type Quote } from "../lib/audit/quote"; import type { Finding } from "../lib/audit/types"; import { auditStuckAfterMinutes, auditStuckAfterMs } from "../lib/audit/timeouts"; import OpenAI from "openai"; -import { crawlSitemap } from "../lib/lx/sitemapCrawl"; +import { claimDueSitemapRechecks, crawlSitemap } from "../lib/lx/sitemapCrawl"; import { DataForSeoClient } from "../lib/lx/dataforseo"; import { researchKeywords } from "../lib/lx/keywordsResearch"; import Anthropic from "@anthropic-ai/sdk"; @@ -1593,6 +1593,31 @@ setInterval( FEED_CRAWL_TICK_MS, ); +// Sitemap rechecks (fleet crawler policy, lib/crawl/policy.ts). +// +// A failed link-exchange sitemap crawl is flagged with sitemap_next_check_at +// 30 days out. This hands each one its single recheck when the date comes up; +// the crawl itself clears the flag on success or sets another 30 days. +const SITEMAP_RECHECK_TICK_MS = 60 * 60 * 1000; // 1h +let sitemapRecheckRunning = false; +async function sitemapRecheckSweep() { + if (sitemapRecheckRunning) return; + sitemapRecheckRunning = true; + try { + const ids = await claimDueSitemapRechecks(supabase); + for (const id of ids) { + console.log(`[worker] lx sitemap 30-day recheck ${id}`); + await processLxSitemap(id); + } + } finally { + sitemapRecheckRunning = false; + } +} +setInterval( + () => sitemapRecheckSweep().catch((e) => console.error("[worker] sitemap recheck sweep", e)), + SITEMAP_RECHECK_TICK_MS, +); + // Ad autobid. // // Every live campaign with autobid on (the default) gets its bid recomputed by @@ -1677,6 +1702,7 @@ server.listen(port, bindHost, () => { promoteIngestSweep().catch((e) => console.error("[worker] promote ingest", e)); sessionRefreshSweep().catch((e) => console.error("[worker] session refresh sweep", e)); feedCrawlSweep().catch((e) => console.error("[worker] feed crawl sweep", e)); + sitemapRecheckSweep().catch((e) => console.error("[worker] sitemap recheck sweep", e)); autobidSweep().catch((e) => console.error("[worker] autobid sweep", e)); adLinkSweep().catch((e) => console.error("[worker] ad link sweep", e)); // Long-lived BullMQ consumer rather than an interval sweep: a render takes