From 8452ccf03c3addc70327b6ad9cdffb53ec1bedde Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sun, 8 Nov 2026 17:03:36 +0000 Subject: [PATCH 1/2] Crawler rule: 10 s fetch, 30 s render, slow/broken sources flagged 30 days - core/crawl.js: CRAWL_FETCH_TIMEOUT_MS 10 s, CRAWL_BROWSER_TIMEOUT_MS 30 s, BULK_DOWNLOAD_TIMEOUT_MS 1 h, FLAG_DAYS 30, failureClass() - http.request default 30 s -> 10 s; download keeps an hour; thrown errors marked upstream, HTTP errors carry status - opensite TIMEOUT_MS 15 s -> 10 s; storefront/buyvps/statuspage/openvuln page and API reads drop their 15-30 s literals for the default - renderWithObscura: one 30 s budget, waitUntil load + bounded settle instead of networkidle 60 s, rejects on timeout, always closes the tab - ingest: a timeout, DNS/TLS/connect error or 5xx sets next_run_at 30 days out with sources.flag/flagged_at (migration 0044); boot's rescheduleFailedSources skips flagged rows Co-Authored-By: Claude Opus 5.5 --- packages/adapters/src/buyvps.js | 2 +- packages/adapters/src/openvuln.js | 4 +- packages/adapters/src/statuspage.js | 2 +- packages/adapters/src/storefront.js | 57 +++-- packages/core/package.json | 1 + packages/core/src/crawl.js | 104 +++++++++ packages/core/src/http.js | 41 +++- packages/core/src/index.js | 7 + packages/core/src/ingest.js | 23 +- packages/core/src/opensite.js | 6 +- packages/db/migrations/0044_source_flags.sql | 15 ++ packages/db/src/queries.js | 7 + test/crawl-rule.test.js | 225 +++++++++++++++++++ 13 files changed, 464 insertions(+), 30 deletions(-) create mode 100644 packages/core/src/crawl.js create mode 100644 packages/db/migrations/0044_source_flags.sql create mode 100644 test/crawl-rule.test.js diff --git a/packages/adapters/src/buyvps.js b/packages/adapters/src/buyvps.js index 1f6013e6..526d2108 100644 --- a/packages/adapters/src/buyvps.js +++ b/packages/adapters/src/buyvps.js @@ -188,7 +188,7 @@ export const buyvps = defineAdapter({ }); if (html) via = 'obscura'; } - if (!html) html = await http.text(url, { headers: { accept: 'text/html' }, timeoutMs: 30_000 }); + if (!html) html = await http.text(url, { headers: { accept: 'text/html' } }); const fetchedAt = new Date(); const plans = parsePlans(html); const items = plans.map((p) => toItem(p, fetchedAt)).filter(Boolean); diff --git a/packages/adapters/src/openvuln.js b/packages/adapters/src/openvuln.js index de1a430a..8b6e4daa 100644 --- a/packages/adapters/src/openvuln.js +++ b/packages/adapters/src/openvuln.js @@ -86,7 +86,7 @@ export async function readIndex(doc, fetchedFrom, http, { limit = 500 } = {}) { if (!entry?.summary && entry?.url) { if (!sameHost(entry.url, fetchedFrom)) continue; try { - record = await http.json(entry.url, { timeoutMs: 20_000 }); + record = await http.json(entry.url); } catch { continue; } @@ -142,7 +142,7 @@ export const openvuln = defineAdapter({ for (const url of urls) { if (Date.now() > deadline) break; try { - const doc = await http.json(url, { timeoutMs: 20_000 }); + const doc = await http.json(url); const { items: got, rejected } = await readIndex(doc, url, http); if (rejected) notes.push(`${hostOf(url)} rejected (${rejected})`); else { diff --git a/packages/adapters/src/statuspage.js b/packages/adapters/src/statuspage.js index 10b2b30f..b884cd4e 100644 --- a/packages/adapters/src/statuspage.js +++ b/packages/adapters/src/statuspage.js @@ -96,7 +96,7 @@ export const statuspage = defineAdapter({ for (const host of hosts) { if (Date.now() > deadline) break; try { - const res = await http.json(`https://${host}/api/v2/incidents.json`, { timeoutMs: 15_000 }); + const res = await http.json(`https://${host}/api/v2/incidents.json`); for (const inc of (res.incidents ?? []).slice(0, 50)) items.push(toItem(host, res.page, inc)); } catch (err) { diff --git a/packages/adapters/src/storefront.js b/packages/adapters/src/storefront.js index 49e22de7..c7b832ac 100644 --- a/packages/adapters/src/storefront.js +++ b/packages/adapters/src/storefront.js @@ -1,5 +1,6 @@ import { readFileSync } from 'node:fs'; import { defineAdapter } from '@nichedb/core/adapter'; +import { BROWSER_SETTLE_MS, CRAWL_BROWSER_TIMEOUT_MS } from '@nichedb/core/crawl'; import { offer, offerItem } from './hosting.js'; import { kindOf, @@ -173,10 +174,27 @@ export function productItem(product, { platform, domain, provider, providerName, return item; } -/** Ask an Obscura MCP server for a page's rendered HTML. */ -export async function renderWithObscura(mcpUrl, url, { timeoutMs = 60_000 } = {}) { - const call = async (id, name, args) => { - const res = await fetch(mcpUrl, { +/** + * Ask an Obscura MCP server for a page's rendered HTML. + * + * The fleet crawler rule: 30 s for the whole render (navigate, settle, read), + * waiting for `load` plus a bounded settle, never `networkidle` (a page with a + * chat widget or an analytics beacon never goes idle). Running out of time + * rejects, so the caller records a failure instead of reading half a page. + */ +export async function renderWithObscura( + mcpUrl, + url, + { + timeoutMs = CRAWL_BROWSER_TIMEOUT_MS, + settleMs = BROWSER_SETTLE_MS, + sleep = Bun.sleep, + fetchImpl = fetch, + } = {}, +) { + const deadline = AbortSignal.timeout(timeoutMs); + const call = async (id, name, args, signal = deadline) => { + const res = await fetchImpl(mcpUrl, { method: 'POST', headers: { 'content-type': 'application/json', @@ -188,7 +206,7 @@ export async function renderWithObscura(mcpUrl, url, { timeoutMs = 60_000 } = {} method: 'tools/call', params: { name, arguments: args }, }), - signal: AbortSignal.timeout(timeoutMs), + signal, }); const text = await res.text(); const body = text.includes('data:') ? text.split('data:').pop() : text; @@ -196,12 +214,22 @@ export async function renderWithObscura(mcpUrl, url, { timeoutMs = 60_000 } = {} if (json.error) throw new Error(json.error.message ?? 'obscura error'); return json.result?.content?.find((c) => c.type === 'text')?.text ?? ''; }; - await call(1, 'browser_navigate', { url, waitUntil: 'networkidle' }); - const html = await call(2, 'browser_evaluate', { - expression: 'document.documentElement.outerHTML', - }); - await call(3, 'browser_close', {}).catch(() => {}); - return html; + try { + await call(1, 'browser_navigate', { url, waitUntil: 'load', timeout: timeoutMs }); + // Late scripts get a moment, bounded and inside the same budget. + await Promise.race([ + sleep(Math.min(settleMs, 2_000)), + new Promise((_, reject) => + deadline.addEventListener('abort', () => reject(deadline.reason), { once: true }), + ), + ]); + return await call(2, 'browser_evaluate', { + expression: 'document.documentElement.outerHTML', + }); + } finally { + // Close the tab whatever happened, on its own short budget. + await call(3, 'browser_close', {}, AbortSignal.timeout(5_000)).catch(() => {}); + } } /** @@ -220,7 +248,7 @@ export async function readHost( const u = new URL(url); if (!robots.has(u.origin)) { requests++; - const txt = await http.request(`${u.origin}/robots.txt`, { timeoutMs: 15_000 }).then( + const txt = await http.request(`${u.origin}/robots.txt`).then( (r) => (r.ok ? r.text() : ''), () => '', ); @@ -231,10 +259,7 @@ export async function readHost( if (requests >= maxRequests || Date.now() > deadline) return null; if (requests > 1) await sleep(1000); requests++; - const res = await http.request(url, { - headers: { accept: 'text/html,*/*' }, - timeoutMs: 15_000, - }); + const res = await http.request(url, { headers: { accept: 'text/html,*/*' } }); const html = (await res.text()).slice(0, 600_000); return { status: res.status, diff --git a/packages/core/package.json b/packages/core/package.json index 64f44743..e7a66ed1 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -6,6 +6,7 @@ "exports": { ".": "./src/index.js", "./adapter": "./src/adapter.js", + "./crawl": "./src/crawl.js", "./http": "./src/http.js", "./names": "./src/names.js", "./opensite": "./src/opensite.js", diff --git a/packages/core/src/crawl.js b/packages/core/src/crawl.js new file mode 100644 index 00000000..f70c6ac3 --- /dev/null +++ b/packages/core/src/crawl.js @@ -0,0 +1,104 @@ +/** + * The fleet crawler rule (2026-11-08): "crawlers should timeout after 10 + * seconds for fetch, 30 for browser (js, images, css). We don't want broken + * slow shit. Flag to recheck later in a month." + * + * One place for the numbers, and the pure function that decides whether a + * failed run was the upstream being slow or broken. On its own, with no + * imports, like retry.js: http.js, ingest.js and the adapters all need it. + */ + +/** One plain HTTP request to a third-party URL: connect, headers and body. */ +export const CRAWL_FETCH_TIMEOUT_MS = 10_000; + +/** One headless-browser render (JS, images, CSS): navigation plus settle. */ +export const CRAWL_BROWSER_TIMEOUT_MS = 30_000; + +/** The bounded settle after `load` in a browser render; never networkidle. */ +export const BROWSER_SETTLE_MS = 1_500; + +/** + * A bulk dataset download (a dump of gigabytes, `http.download`). Not a page + * or an API call, so not the 10 s rule: an hour for the whole transfer. + */ +export const BULK_DOWNLOAD_TIMEOUT_MS = 60 * 60_000; + +/** How long a source whose upstream was slow or broken is left alone. */ +export const FLAG_DAYS = 30; +export const FLAG_MS = FLAG_DAYS * 86_400_000; + +// Timeouts anywhere between us and the upstream (Bun: TimeoutError; undici: UND_ERR_*). +const SLOW_CODES = new Set([ + 'TimeoutError', + 'ETIMEDOUT', + 'ESOCKETTIMEDOUT', + 'UND_ERR_CONNECT_TIMEOUT', + 'UND_ERR_HEADERS_TIMEOUT', + 'UND_ERR_BODY_TIMEOUT', +]); +// The upstream cannot be reached or spoken to: no such host, refused, reset, TLS. +const BROKEN_CODES = new Set([ + 'ENOTFOUND', + 'ECONNREFUSED', + 'ECONNRESET', + 'EHOSTUNREACH', + 'ENETUNREACH', + 'EHOSTDOWN', + 'EPIPE', + 'EPROTO', + 'ConnectionRefused', + 'ConnectionClosed', + 'FailedToOpenSocket', + 'UND_ERR_SOCKET', +]); +const TLS_CODE = + /^(ERR_TLS_|ERR_SSL_|CERT_|UNABLE_TO_|DEPTH_ZERO_SELF_SIGNED_CERT|SELF_SIGNED_CERT_IN_CHAIN|HOSTNAME_MISMATCH)/; + +function codesOf(err) { + const out = []; + for (let e = err, depth = 0; e && depth < 4; e = e.cause, depth++) { + if (typeof e !== 'object' && typeof e !== 'function') break; + for (const c of [e.code, e.name]) if (typeof c === 'string') out.push(c); + } + return out; +} + +/** + * 'slow' (a timeout), 'broken' (DNS, TLS, refused/reset connection, HTTP + * 5xx) or null for anything else: a 4xx, a parse error, a missing key, a bug + * of ours, a database error. Those keep the short retry in retry.js. + * + * EAI_AGAIN is null on purpose: it is our own resolver failing for a moment + * (dev2's Docker DNS has failed every host at once), not the upstream. An + * error can force the answer with `flag: 'slow' | 'broken' | null`. + */ +export function failureClass(err) { + if (!err || (typeof err !== 'object' && typeof err !== 'function')) return null; + if ('flag' in err && (err.flag === 'slow' || err.flag === 'broken' || err.flag === null)) + return err.flag; + const codes = codesOf(err); + if (codes.includes('EAI_AGAIN')) return null; + // Our own database failing is never the upstream's fault: Bun SQL errors + // carry ERR_POSTGRES_* codes, and a refused Postgres must not park sources. + if (codes.some((c) => c.startsWith('ERR_POSTGRES') || c === 'PostgresError' || c === 'SQLError')) + return null; + // A system code only says "upstream" on a fetch error: Bun's fetch throws a + // TypeError carrying it, and makeHttp marks what it throws `upstream`. + const fromFetch = + err.upstream === true || err.name === 'TypeError' || err.name === 'TimeoutError'; + if (codes.includes('TimeoutError')) return 'slow'; + if (fromFetch && codes.some((c) => SLOW_CODES.has(c))) return 'slow'; + const status = Number(err.status ?? err.statusCode); + if (Number.isFinite(status) && status >= 500 && status <= 599) return 'broken'; + if (fromFetch && codes.some((c) => BROKEN_CODES.has(c) || TLS_CODE.test(c))) return 'broken'; + const message = String(err.message ?? ''); + // httpError's message, for an error rethrown without its status. + if (/^5\d\d from https?:\/\//.test(message)) return 'broken'; + if (/^The operation timed out\.?$/.test(message)) return 'slow'; + return null; +} + +/** When a source flagged now is looked at again. */ +export function flaggedUntil(now = Date.now()) { + return new Date(now + FLAG_MS); +} diff --git a/packages/core/src/http.js b/packages/core/src/http.js index 52b10757..c9d1be04 100644 --- a/packages/core/src/http.js +++ b/packages/core/src/http.js @@ -5,14 +5,32 @@ * police the User-Agent (SEC refuses a blank one outright) and the rate. So this * always identifies the deployment, always has a timeout, and turns a 429 into * one polite wait rather than a retry storm. + * + * The timeout is the fleet crawler rule's 10 s for a page or API call + * (CRAWL_FETCH_TIMEOUT_MS); an adapter reading a big document passes its own, + * and `download` keeps an hour for a dump. What it throws is marked + * `upstream` (and an HTTP error carries `status`), so ingest can tell a slow + * or broken upstream, flagged for 30 days, from anything else. */ import { mkdir, open, stat, unlink } from 'node:fs/promises'; import { dirname } from 'node:path'; +import { BULK_DOWNLOAD_TIMEOUT_MS, CRAWL_FETCH_TIMEOUT_MS } from './crawl.js'; import { requestMusicBrainz, retryDelay } from './musicbrainz-http.js'; export function makeHttp({ userAgent, log = () => {} }) { - async function request(url, { headers = {}, timeoutMs = 30_000, method = 'GET', body } = {}) { + async function request(url, opts = {}) { + try { + return await send(url, opts); + } catch (err) { + throw markUpstream(err); + } + } + + async function send( + url, + { headers = {}, timeoutMs = CRAWL_FETCH_TIMEOUT_MS, method = 'GET', body } = {}, + ) { const doFetch = () => fetch(url, { method, @@ -39,7 +57,7 @@ export function makeHttp({ userAgent, log = () => {} }) { * * The dumps this serves are 1.7 to 11 GB, so the body is streamed chunk by * chunk onto an open file handle and never held in memory; the timeout covers - * the whole transfer and defaults to an hour, because `request`'s 30 s is a + * the whole transfer and defaults to an hour, because `request`'s 10 s is a * ceiling on an API call, not on a download. The user agent goes with it: * Podcast Index answers 403 without one. * @@ -59,7 +77,7 @@ export function makeHttp({ userAgent, log = () => {} }) { async function download( url, filePath, - { timeoutMs = 60 * 60_000, headers = {}, onProgress } = {}, + { timeoutMs = BULK_DOWNLOAD_TIMEOUT_MS, headers = {}, onProgress } = {}, ) { await mkdir(dirname(filePath), { recursive: true }); const have = (await stat(filePath).catch(() => null))?.size ?? 0; @@ -165,7 +183,22 @@ async function httpError(res, url) { } catch { // An unreadable body is no worse than the message without it. } - return new Error(`${res.status} from ${String(url).slice(0, 400)}${detail}`); + return Object.assign(new Error(`${res.status} from ${String(url).slice(0, 400)}${detail}`), { + status: res.status, + upstream: true, + }); +} + +/** Mark a fetch failure as the upstream's, keeping the error itself (its name, code and cause). */ +function markUpstream(err) { + if (err && typeof err === 'object') { + try { + err.upstream = true; + } catch { + // A frozen error stays as it is. + } + } + return err; } /** The total out of `Content-Range: bytes 0-99/1000` (or the `bytes` star form of a 416), or null. */ diff --git a/packages/core/src/index.js b/packages/core/src/index.js index 1c778be1..6aebba84 100644 --- a/packages/core/src/index.js +++ b/packages/core/src/index.js @@ -4,5 +4,12 @@ export * from './dump.js'; export { describeEnrichers, enrichPending } from './enrich.js'; export { scanFeeds } from './feedscan.js'; export { makeHttp } from './http.js'; +export { + CRAWL_BROWSER_TIMEOUT_MS, + CRAWL_FETCH_TIMEOUT_MS, + FLAG_DAYS, + failureClass, + flaggedUntil, +} from './crawl.js'; export { ADAPTERS, adapterByName, describeAdapters, envFor, runSource } from './ingest.js'; export { COLLECTIONS, DEFAULT_FEEDS, ensureDefaults } from './seed.js'; diff --git a/packages/core/src/ingest.js b/packages/core/src/ingest.js index ca4aa954..f4e9d960 100644 --- a/packages/core/src/ingest.js +++ b/packages/core/src/ingest.js @@ -6,6 +6,7 @@ import { retryTransient } from '@nichedb/db/resilience'; import { normaliseItem } from './adapter.js'; import { makeHttp } from './http.js'; import { profileItem } from './profiles.js'; +import { failureClass, flaggedUntil } from './crawl.js'; import { retryMinutes } from './retry.js'; const UA = () => @@ -158,7 +159,11 @@ export async function runSource( return { ...totals }; } catch (err) { const message = String(err?.message ?? err).slice(0, 1000); - const nextRunAt = await retryAt(source).catch(() => null); + // The fleet crawler rule: an upstream that timed out, failed DNS/TLS or a + // connection, or answered 5xx is flagged and left alone for 30 days, then + // tried once; anything else takes the short doubling retry. + const flag = failureClass(err); + const nextRunAt = flag ? flaggedUntil() : await retryAt(source).catch(() => null); /* * A run that died because Postgres went away must still be closed once it * is back. Left 'running', the row parks the source: `dueSources` treats it @@ -169,11 +174,21 @@ export async function runSource( * most, before giving up and leaving the row to the reaper as before. */ await retryTransient( - () => q.finishRun({ runId, sourceId: source.id, status: 'error', error: message, nextRunAt }), + () => + q.finishRun({ + runId, + sourceId: source.id, + status: 'error', + error: message, + nextRunAt, + flag, + }), { label: `${tag} closing the run`, log: l, budgetMs: 3 * 60_000, ...closeRetry }, ); - l(`failed: ${message}${nextRunAt ? `; retrying at ${nextRunAt.toISOString()}` : ''}`); - return { error: message }; + l( + `failed${flag ? ` (${flag} upstream, flagged)` : ''}: ${message}${nextRunAt ? `; retrying at ${nextRunAt.toISOString()}` : ''}`, + ); + return flag ? { error: message, flag } : { error: message }; } } diff --git a/packages/core/src/opensite.js b/packages/core/src/opensite.js index 81e8b486..dc8ae3b3 100644 --- a/packages/core/src/opensite.js +++ b/packages/core/src/opensite.js @@ -15,6 +15,8 @@ * wants to see what each one saw. */ +import { CRAWL_FETCH_TIMEOUT_MS } from './crawl.js'; + export const VERSION = '0.1'; export const SPEC = 'https://logicsrc.com/opensite'; export const AGENT = `OpenSite/${VERSION} (+${SPEC})`; @@ -36,9 +38,9 @@ export const KINDS = [ 'other', ]; -/** The limits the spec sets on a read. */ +/** The limits on a read: the spec's size cap, and the fleet crawler rule's 10 s (was 15). */ export const MAX_BYTES = 2 * 1024 * 1024; -export const TIMEOUT_MS = 15_000; +export const TIMEOUT_MS = CRAWL_FETCH_TIMEOUT_MS; export const MAX_REDIRECTS = 5; /** A record is never bigger than this; JSON-LD is the first thing cut. */ export const MAX_RECORD_BYTES = 256 * 1024; diff --git a/packages/db/migrations/0044_source_flags.sql b/packages/db/migrations/0044_source_flags.sql new file mode 100644 index 00000000..93314f37 --- /dev/null +++ b/packages/db/migrations/0044_source_flags.sql @@ -0,0 +1,15 @@ +-- The fleet crawler rule (2026-11-08): a source whose upstream was slow or +-- broken (timeout, DNS/TLS/connection error, HTTP 5xx) is flagged and left +-- alone for 30 days (next_run_at), then tried once. `flag` is 'slow' or +-- 'broken' while that holds, `flagged_at` when it was set (the failure time); +-- both clear on the next run that ends any other way. Boot's +-- rescheduleFailedSources skips flagged rows, or every deploy would undo it. +-- +-- sources is small (hundreds of rows) and both columns are nullable with no +-- default, so this is a catalogue change, not a rewrite. lock_timeout keeps a +-- busy worker from holding the deploy: the migration fails and retries. +set local lock_timeout = '10s'; + +alter table sources + add column if not exists flag text check (flag in ('slow', 'broken')), + add column if not exists flagged_at timestamptz; diff --git a/packages/db/src/queries.js b/packages/db/src/queries.js index 0ba938c8..2aad8c32 100644 --- a/packages/db/src/queries.js +++ b/packages/db/src/queries.js @@ -666,7 +666,11 @@ export async function finishRun({ note = null, cursor, nextRunAt, + flag = null, }) { + // flag: 'slow' | 'broken' when the upstream failed the crawler rule (the + // caller sets nextRunAt 30 days out); cleared by any other outcome. + const flagged = status !== 'ok' && (flag === 'slow' || flag === 'broken') ? flag : null; await sql` update runs set finished_at = now(), status = ${status}, seen = ${seen}, added = ${added}, updated = ${updated}, error = ${error}, note = ${note} @@ -676,6 +680,8 @@ export async function finishRun({ update sources set last_ok_at = case when ${status === 'ok'} then now() else last_ok_at end, last_error = ${status === 'ok' ? null : error}, + flag = ${flagged}, + flagged_at = case when ${flagged !== null} then now() else null end, cursor = coalesce(${cursor === undefined ? null : JSON.stringify(cursor)}::jsonb, cursor), next_run_at = coalesce(${nextRunAt ?? null}, next_run_at), item_count = case @@ -715,6 +721,7 @@ export async function rescheduleFailedSources({ retryMinutes = 15 } = {}) { updated_at = now() where enabled and last_error is not null + and flag is null and next_run_at > now() + make_interval(mins => ${retryMinutes}) returning slug `; diff --git a/test/crawl-rule.test.js b/test/crawl-rule.test.js new file mode 100644 index 00000000..1779c55c --- /dev/null +++ b/test/crawl-rule.test.js @@ -0,0 +1,225 @@ +import { describe, expect, mock, test } from 'bun:test'; + +/** + * The fleet crawler rule (2026-11-08): 10 s per fetch, 30 s per browser + * render, and a source whose upstream was slow or broken is flagged and left + * alone for 30 days instead of retried in minutes. runSource runs for real + * against a recording stand-in for the queries module, as dump-ingest does. + */ +process.env.DATABASE_URL ??= 'postgres://test:test@localhost:5432/test'; +process.env.SITE_URL ??= 'https://nichedb.test'; + +const finishes = []; +const state = { source: null, errorsBefore: 0 }; +const realQueries = await import('../packages/db/src/queries.js'); +mock.module('../packages/db/src/queries.js', () => ({ + ...realQueries, + getSourceById: async () => state.source, + consecutiveErrors: async () => state.errorsBefore, + startRun: async () => 9, + finishRun: async (args) => { + finishes.push(args); + }, +})); + +const { runSource } = await import('../packages/core/src/ingest.js'); +const { + BULK_DOWNLOAD_TIMEOUT_MS, + CRAWL_BROWSER_TIMEOUT_MS, + CRAWL_FETCH_TIMEOUT_MS, + FLAG_DAYS, + failureClass, +} = await import('../packages/core/src/crawl.js'); +const { TIMEOUT_MS } = await import('../packages/core/src/opensite.js'); +const { makeHttp } = await import('../packages/core/src/http.js'); +const { renderWithObscura } = await import('../packages/adapters/src/storefront.js'); + +const DAY = 86_400_000; +const sysErr = (code, message = code) => Object.assign(new TypeError(message), { code }); + +describe('crawler rule: the numbers', () => { + test('10 s fetch, 30 s browser, an hour for a bulk download, 30 days flagged', () => { + expect(CRAWL_FETCH_TIMEOUT_MS).toBe(10_000); + expect(CRAWL_BROWSER_TIMEOUT_MS).toBe(30_000); + expect(BULK_DOWNLOAD_TIMEOUT_MS).toBe(60 * 60_000); + expect(FLAG_DAYS).toBe(30); + expect(TIMEOUT_MS).toBe(10_000); + }); +}); + +describe('failureClass', () => { + test('timeouts are slow', () => { + expect( + failureClass(Object.assign(new Error('The operation timed out.'), { name: 'TimeoutError' })), + ).toBe('slow'); + expect(failureClass(new Error('The operation timed out.'))).toBe('slow'); + expect(failureClass(Object.assign(sysErr('UND_ERR_CONNECT_TIMEOUT'), { upstream: true }))).toBe( + 'slow', + ); + }); + test('DNS, refused, TLS and 5xx are broken', () => { + expect(failureClass(sysErr('ENOTFOUND', 'getaddrinfo ENOTFOUND gone.example'))).toBe('broken'); + expect(failureClass(sysErr('ECONNREFUSED'))).toBe('broken'); + expect(failureClass(sysErr('CERT_HAS_EXPIRED', 'certificate has expired'))).toBe('broken'); + expect( + failureClass(Object.assign(new Error('502 from https://a.example/x'), { status: 502 })), + ).toBe('broken'); + expect(failureClass(new Error('503 from https://a.example/x: Service Unavailable'))).toBe( + 'broken', + ); + }); + test('4xx, parse errors, missing keys, our resolver and our database keep the short retry', () => { + expect( + failureClass(Object.assign(new Error('404 from https://a.example/x'), { status: 404 })), + ).toBe(null); + expect(failureClass(new SyntaxError('JSON Parse error: Unexpected identifier'))).toBe(null); + expect(failureClass(new Error('IGDB_CLIENT_ID is not set'))).toBe(null); + expect(failureClass(sysErr('EAI_AGAIN', 'getaddrinfo EAI_AGAIN api.example'))).toBe(null); + expect( + failureClass( + Object.assign(new Error('Connection closed'), { code: 'ERR_POSTGRES_CONNECTION_CLOSED' }), + ), + ).toBe(null); + // A refused Postgres is not a fetch error: no TypeError, no upstream mark. + expect( + failureClass( + Object.assign(new Error('connect ECONNREFUSED 10.0.0.5:5432'), { code: 'ECONNREFUSED' }), + ), + ).toBe(null); + expect(failureClass(new Error('canceling statement due to statement timeout'))).toBe(null); + }); +}); + +describe('runSource: a slow or broken upstream is flagged for 30 days', () => { + const fail = (err) => ({ name: 'fake', pull: async () => Promise.reject(err) }); + const run = (adapter) => { + state.source = { + id: 3, + slug: 'fake-src', + adapter: 'fake', + enabled: true, + cadence_minutes: 60, + config: {}, + cursor: {}, + }; + return runSource(3, { + log: () => {}, + resolveAdapter: () => adapter, + closeRetry: { budgetMs: 0 }, + }); + }; + + test('a timeout: flag slow, next run 30 days out', async () => { + finishes.length = 0; + const before = Date.now(); + const out = await run( + fail(Object.assign(new Error('The operation timed out.'), { name: 'TimeoutError' })), + ); + expect(out.flag).toBe('slow'); + const f = finishes.at(-1); + expect(f.status).toBe('error'); + expect(f.flag).toBe('slow'); + expect(f.nextRunAt.getTime() - before).toBeGreaterThanOrEqual(30 * DAY - 1000); + expect(f.nextRunAt.getTime() - before).toBeLessThan(30 * DAY + 5000); + }); + + test('a 5xx: flag broken, 30 days', async () => { + finishes.length = 0; + await run( + fail( + Object.assign(new Error('502 from https://api.example/x'), { status: 502, upstream: true }), + ), + ); + const f = finishes.at(-1); + expect(f.flag).toBe('broken'); + expect(f.nextRunAt.getTime() - Date.now()).toBeGreaterThan(29 * DAY); + }); + + test('a 4xx keeps the 15-minute doubling retry and no flag', async () => { + finishes.length = 0; + state.errorsBefore = 0; + await run(fail(Object.assign(new Error('404 from https://api.example/x'), { status: 404 }))); + const f = finishes.at(-1); + expect(f.flag).toBe(null); + expect(Math.abs(f.nextRunAt.getTime() - (Date.now() + 15 * 60_000))).toBeLessThan(5000); + }); +}); + +describe('http: what it throws says it was the upstream', () => { + test('a 5xx carries status and upstream; ingest reads it as broken', async () => { + const server = Bun.serve({ port: 0, fetch: () => new Response('down', { status: 502 }) }); + try { + const http = makeHttp({ userAgent: 'test' }); + const err = await http.json(`http://localhost:${server.port}/x`).catch((e) => e); + expect(err.status).toBe(502); + expect(err.upstream).toBe(true); + expect(failureClass(err)).toBe('broken'); + } finally { + server.stop(true); + } + }); + test('a slow answer past timeoutMs rejects as a timeout (slow)', async () => { + const server = Bun.serve({ + port: 0, + fetch: async () => { + await Bun.sleep(500); + return new Response('late'); + }, + }); + try { + const http = makeHttp({ userAgent: 'test' }); + const err = await http + .text(`http://localhost:${server.port}/x`, { timeoutMs: 50 }) + .catch((e) => e); + expect(err.upstream).toBe(true); + expect(failureClass(err)).toBe('slow'); + } finally { + server.stop(true); + } + }); +}); + +describe('renderWithObscura: load + bounded settle, never networkidle, and a timeout rejects', () => { + const reply = (text) => + new Response(JSON.stringify({ jsonrpc: '2.0', result: { content: [{ type: 'text', text }] } })); + test('navigates with waitUntil load and a 30 s budget, settles at most 2 s, closes the tab', async () => { + const calls = []; + const slept = []; + const html = await renderWithObscura('http://obscura.test/mcp', 'https://shop.example/', { + sleep: async (ms) => void slept.push(ms), + fetchImpl: async (_url, init) => { + const body = JSON.parse(init.body); + calls.push(body.params); + return reply(body.params.name === 'browser_evaluate' ? 'ok' : 'done'); + }, + }); + expect(html).toBe('ok'); + expect(calls[0]).toEqual({ + name: 'browser_navigate', + arguments: { url: 'https://shop.example/', waitUntil: 'load', timeout: 30_000 }, + }); + expect(calls.map((c) => c.name)).toEqual([ + 'browser_navigate', + 'browser_evaluate', + 'browser_close', + ]); + expect(slept.every((ms) => ms <= 2_000)).toBe(true); + }); + test('a render past its budget rejects (and still closes)', async () => { + const names = []; + const err = await renderWithObscura('http://obscura.test/mcp', 'https://slow.example/', { + timeoutMs: 50, + fetchImpl: (_url, init) => { + const body = JSON.parse(init.body); + names.push(body.params.name); + if (body.params.name === 'browser_close') return Promise.resolve(reply('closed')); + return new Promise((_, reject) => + init.signal.addEventListener('abort', () => reject(init.signal.reason), { once: true }), + ); + }, + }).catch((e) => e); + expect(err?.name).toBe('TimeoutError'); + expect(failureClass(err)).toBe('slow'); + expect(names).toContain('browser_close'); + }); +}); From e4783437efce5aae2517c2a9da96a8a2302b42d7 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sun, 8 Nov 2026 17:11:55 +0000 Subject: [PATCH 2/2] Lint: sort crawl.js imports/exports Co-Authored-By: Claude Opus 5.5 --- packages/core/src/index.js | 8 ++++---- packages/core/src/ingest.js | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/core/src/index.js b/packages/core/src/index.js index 6aebba84..dc34aa8b 100644 --- a/packages/core/src/index.js +++ b/packages/core/src/index.js @@ -1,9 +1,5 @@ export * from './adapter.js'; export { computeCollectionStats, refreshCollectionStats } from './collection-stats.js'; -export * from './dump.js'; -export { describeEnrichers, enrichPending } from './enrich.js'; -export { scanFeeds } from './feedscan.js'; -export { makeHttp } from './http.js'; export { CRAWL_BROWSER_TIMEOUT_MS, CRAWL_FETCH_TIMEOUT_MS, @@ -11,5 +7,9 @@ export { failureClass, flaggedUntil, } from './crawl.js'; +export * from './dump.js'; +export { describeEnrichers, enrichPending } from './enrich.js'; +export { scanFeeds } from './feedscan.js'; +export { makeHttp } from './http.js'; export { ADAPTERS, adapterByName, describeAdapters, envFor, runSource } from './ingest.js'; export { COLLECTIONS, DEFAULT_FEEDS, ensureDefaults } from './seed.js'; diff --git a/packages/core/src/ingest.js b/packages/core/src/ingest.js index f4e9d960..bda4ba05 100644 --- a/packages/core/src/ingest.js +++ b/packages/core/src/ingest.js @@ -4,9 +4,9 @@ import * as profiles from '@nichedb/db/profiles'; import * as q from '@nichedb/db/queries'; import { retryTransient } from '@nichedb/db/resilience'; import { normaliseItem } from './adapter.js'; +import { failureClass, flaggedUntil } from './crawl.js'; import { makeHttp } from './http.js'; import { profileItem } from './profiles.js'; -import { failureClass, flaggedUntil } from './crawl.js'; import { retryMinutes } from './retry.js'; const UA = () =>