crawler: 10 s fetch budget, 30-day flag for slow/broken feeds, dead rechecked monthly

apps/web/src/app/[slug]/page.jsx

@@ -424,8 +424,8 @@ export default async function FeedPage({ params, base }) {
 
       {feed.status === 'dead' && (
         <p className="notice">
-          This feed stopped responding, so we no longer crawl it. The archive below is what we
-          collected while it was live.
+          This feed stopped responding, so we only check it once a month now and it is back as
+          soon as it answers. The archive below is what we collected while it was live.
         </p>
       )}
 

apps/web/src/app/api/feeds/[slug]/refresh/route.js

@@ -43,7 +43,7 @@ export async function POST(_req, { params }) {
   const client = db();
   const feed = await q.feedBySlug(client, slug);
   if (!feed) return json({ error: 'not-found' }, 404);
-  if (feed.status === 'dead') return json({ ok: false, state: 'dead', error: 'This feed has been retired after repeated failures.' }, 409);
+  if (feed.status === 'dead') return json({ ok: false, state: 'dead', error: 'This feed has been retired after repeated failures; it is rechecked once a month.' }, 409);
 
   const last = Date.parse(String(feed.last_fetched_at ?? ''));
   if (Number.isFinite(last) && Date.now() - last < RECENT_MS) {

apps/web/src/lib/reasons.js

@@ -20,6 +20,10 @@ export const REASONS = {
   'no-feed-found': 'no feed on the site',
   timeout: 'site did not respond',
   'fetch-failed': 'site could not be reached',
+  'connection-failed': 'site refused or dropped the connection',
+  'tls-failed': 'site has a broken HTTPS certificate',
+  'dns-failed': 'site name does not resolve',
+  'dns-retry': 'site name could not be looked up just now',
   'blocked-host': 'not a public address',
   // Emitted by the checker but never given words until the log started showing
   // rejections one by one, where a bare code is the most common thing on screen.

packages/db/migrations-pg/20261108165240_dead_feeds_recheck.sql

@@ -0,0 +1,27 @@
+-- 'dead' is no longer for ever (fleet crawler rule, 2026-11-08: flag a slow or
+-- broken target and recheck it in a month). A retired feed comes back to the
+-- crawler once its next_fetch_at passes, read by recheckFeeds() in queries.js
+-- off this partial index: every other due index is `where status <> 'dead'`.
+--
+-- Concurrently, as feeds_due_tier_idx was: migrate() runs each statement on
+-- its own with no transaction around the file.
+create index concurrently if not exists feeds_recheck_idx
+  on feeds (next_fetch_at)
+  where status = 'dead';
+
+-- Feeds retired before this change carry a next_fetch_at from the old ladder
+-- (a day at most after their last attempt), which would make all of them due
+-- at once. Re-stamp each to thirty days after its last attempt, which is when
+-- the new rule would have scheduled its recheck. Idempotent: rerunning it
+-- computes the same value from the same column. recheckFeeds only offers a
+-- small share of each tick, so the ones already past thirty days drain over a
+-- day or two rather than in one burst.
+update feeds
+   set next_fetch_at = to_char(
+         (last_fetched_at::timestamptz + interval '30 days') at time zone 'UTC',
+         'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')
+ where status = 'dead'
+   and last_fetched_at is not null
+   and next_fetch_at < to_char(
+         (last_fetched_at::timestamptz + interval '30 days') at time zone 'UTC',
+         'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"');

packages/db/src/queries.js

@@ -2894,6 +2894,39 @@ export async function expressFeeds(db, limit = 25) {
   return rows;
 }
 
+/**
+ * The share of a tick offered to retired feeds that are due their monthly
+ * recheck. Small and bounded, so the 31k feeds that were already dead when the
+ * recheck arrived drain over a day or two instead of landing in one tick; slots
+ * the live tiers cannot fill are handed to it as well.
+ */
+const RECHECK_SHARE = 0.05;
+
+/**
+ * Retired ('dead') feeds whose recheck is due, oldest first.
+ *
+ * 'dead' used to be for ever: every due query filters `status <> 'dead'`, so a
+ * blog that was down for a fortnight in August was never looked at again. It
+ * now comes back once its next_fetch_at passes -- thirty days after the attempt
+ * that retired it -- for exactly one recheck. Served by `feeds_recheck_idx`, the
+ * partial index on `status = 'dead'`.
+ *
+ * @param {Client} db
+ * @param {number} [limit]
+ * @returns {Promise<object[]>}
+ */
+export async function recheckFeeds(db, limit = 25) {
+  if (limit <= 0) return [];
+  const { rows } = await db.execute({
+    sql: `select ${DUE_COLUMNS}
+          from feeds
+          where status = 'dead' and next_fetch_at <= ?
+          order by next_fetch_at asc limit ?`,
+    args: [nowIso(), limit],
+  });
+  return rows;
+}
+
 /**
  * A feed's crawl tier, from its interval alone: 0 hot (read at least hourly),
  * 1 active (daily), 2 slow, 3 dormant (gone quiet, read weekly), 4 archived
@@ -2928,13 +2961,18 @@ export async function dueFeeds(db, limit = 25) {
   // Bounded rather than unbounded even though the express table is tiny: the
   // point of a reserved share is that it is reserved in both directions.
   const express = await expressFeeds(db, Math.floor(limit * EXPRESS_SHARE));
-  const room = limit - express.length;
-  if (room <= 0) return express;
+  if (limit - express.length <= 0) return express;
+
+  // Retired feeds due their monthly recheck: a small reserved share first, and
+  // whatever the live tiers leave over at the end.
+  const rechecks = await recheckFeeds(db, Math.max(1, Math.floor((limit - express.length) * RECHECK_SHARE)));
+  const room = limit - express.length - rechecks.length;
 
   const now = nowIso();
-  const out = [...express];
+  const out = [...express, ...rechecks];
   const exhausted = new Set();
-  const seen = new Set(express.map((r) => String(r.id)));
+  const seen = new Set(out.map((r) => String(r.id)));
+  if (room <= 0) return out;
 
   const take = async (tier, want) => {
     // The express rows are excluded rather than deduplicated afterwards: they
@@ -2969,6 +3007,15 @@ export async function dueFeeds(db, limit = 25) {
     if (carry <= 0) break;
     if (!exhausted.has(tier)) carry -= await take(tier, carry);
   }
+  if (carry > 0) {
+    for (const row of await recheckFeeds(db, carry + rechecks.length)) {
+      if (carry <= 0) break;
+      if (seen.has(String(row.id))) continue;
+      seen.add(String(row.id));
+      out.push(row);
+      carry -= 1;
+    }
+  }
   return out;
 }
 
@@ -3505,26 +3552,33 @@ export async function markUnchanged(db, id, minutes, signals = {}) {
 /**
  * Record a failed crawl and push the next attempt out.
  *
+ * Ten consecutive failures retire a feed (status 'dead': hidden from listings,
+ * page kept), but not for ever: `recheckFeeds` hands a dead feed back to the
+ * crawler once its `next_fetch_at` passes, which the crawler sets thirty days
+ * out (see failureMinutes in @rssamplifier/ingest).
+ *
  * @param {Client} db
  * @param {string} id
  * @param {string} error
  * @param {number} errorCount consecutive failures including this one
- * @param {number} minutes backoff
+ * @param {number} minutes until the next attempt
+ * @param {{ keepInterval?: boolean }} [opts] keepInterval: leave
+ *   fetch_interval_minutes alone (a month-long flag is not the feed's cadence)
  */
-export async function markCrawlFailure(db, id, error, errorCount, minutes) {
+export async function markCrawlFailure(db, id, error, errorCount, minutes, opts = {}) {
   const now = nowIso();
   await db.execute({
     sql: `update feeds set
             status = ?, last_fetched_at = ?, last_error = ?, error_count = ?,
-            fetch_interval_minutes = ?, next_fetch_at = ?, updated_at = ?
+            fetch_interval_minutes = coalesce(?, fetch_interval_minutes), next_fetch_at = ?, updated_at = ?
           where id = ?`,
     args: [
       // Ten consecutive failures is a dead feed: stop crawling, keep the page.
       errorCount >= 10 ? 'dead' : 'error',
       now,
       error,
       errorCount,
-      minutes,
+      opts.keepInterval ? null : minutes,
       nowIso(minutes * 60_000),
       now,
       id,

packages/db/test/db.test.js

@@ -557,17 +557,51 @@ test('crawl failure backs off and eventually marks the feed dead', async () => {
   row = await q.feedBySlug(db, 'test-blog');
   assert.equal(row.status, 'dead', 'ten consecutive failures retires the feed');
 
-  // A dead feed must drop out of the crawl queue and the public listing.
-  await db.execute({
-    sql: 'update feeds set next_fetch_at = ? where id = ?',
-    args: [nowIso(-60_000), id],
-  });
-  assert.equal((await q.dueFeeds(db, 10)).length, 0, 'dead feeds are not re-crawled');
+  // A dead feed drops out of the public listing at once, and out of the crawl
+  // queue until its recheck comes due.
   assert.equal(
     (await q.listFeeds(db)).some((f) => f.slug === 'test-blog'),
     false,
     'dead feeds are hidden from the index',
   );
+  await db.execute({
+    sql: 'update feeds set next_fetch_at = ? where id = ?',
+    args: [nowIso(29 * 86_400_000), id],
+  });
+  assert.equal((await q.dueFeeds(db, 10)).length, 0, 'not before its recheck');
+  assert.equal((await q.recheckFeeds(db, 10)).length, 0);
+
+  // But not dead for ever: once next_fetch_at passes it is offered for one recheck.
+  await db.execute({
+    sql: 'update feeds set next_fetch_at = ? where id = ?',
+    args: [nowIso(-60_000), id],
+  });
+  assert.deepEqual(
+    (await q.recheckFeeds(db, 10)).map((f) => String(f.id)),
+    [id],
+    'a dead feed comes back for its monthly recheck',
+  );
+  assert.deepEqual(
+    (await q.dueFeeds(db, 10)).map((f) => String(f.id)),
+    [id],
+    'and the crawler is handed it',
+  );
+
+  // A failed recheck parks it another month, still dead, interval untouched.
+  await q.markCrawlFailure(db, id, 'timeout', 11, 43_200, { keepInterval: true });
+  row = await q.feedBySlug(db, 'test-blog');
+  assert.equal(row.status, 'dead');
+  assert.equal(Number(row.fetch_interval_minutes), 1440, 'a flag is not a cadence');
+  const days = (Date.parse(row.next_fetch_at) - Date.now()) / 86_400_000;
+  assert.ok(days > 29.9 && days <= 30, `next check in 30 days -> ${days.toFixed(2)}`);
+  assert.equal((await q.dueFeeds(db, 10)).length, 0);
+
+  // A successful recheck restores it.
+  await q.markUnchanged(db, id, 60);
+  row = await q.feedBySlug(db, 'test-blog');
+  assert.equal(row.status, 'active', 'success clears the flag');
+  assert.equal(Number(row.error_count), 0);
+  assert.equal((await q.listFeeds(db)).some((f) => f.slug === 'test-blog'), true);
 });
 
 test('submission audit and rate-limit window', async () => {

packages/feed/index.js

@@ -29,7 +29,17 @@ export {
 export { extractKeywords, feedTopics, tokenize, topicSlug, singularize } from './src/keywords.js';
 export { sanitizeHtml, textLength } from './src/sanitize.js';
 export { normalizeUrl, findFeedLinks, guessFeedUrls, looksLikeFeed } from './src/discover.js';
-export { safeFetch, safeFetchBytes, resolveFeed, isBlockedAddress, isPublicHost, fetchPage } from './src/fetch.js';
+export {
+  safeFetch,
+  safeFetchBytes,
+  resolveFeed,
+  isBlockedAddress,
+  isPublicHost,
+  hostVerdict,
+  fetchErrorCode,
+  fetchPage,
+  CRAWL_FETCH_TIMEOUT_MS,
+} from './src/fetch.js';
 export { imageSize } from './src/imagesize.js';
 export {
   findFeedCard,