crawler: a dead feed that fails its monthly recheck stays dead

packages/db/src/queries.js

@@ -2845,7 +2845,7 @@ export async function recentlyCrawled(db, limit = 20) {
 }
 
 /** The columns a crawl needs off a feed row. Shared by both due queries. */
-const DUE_COLUMNS = `id, slug, title, feed_url, error_count, fetch_interval_minutes, source_kind,
+const DUE_COLUMNS = `id, slug, title, feed_url, status, error_count, fetch_interval_minutes, source_kind,
                  item_count, last_published_at,
                  http_etag, http_last_modified, content_hash, change_log,
                  social_network, social_ref, social_config`;
@@ -3569,11 +3569,14 @@ export async function markCrawlFailure(db, id, error, errorCount, minutes, opts
   const now = nowIso();
   await db.execute({
     sql: `update feeds set
-            status = ?, last_fetched_at = ?, last_error = ?, error_count = ?,
+            status = case when status = 'dead' then 'dead' else ? end,
+            last_fetched_at = ?, last_error = ?, error_count = ?,
             fetch_interval_minutes = coalesce(?, fetch_interval_minutes), next_fetch_at = ?, updated_at = ?
           where id = ?`,
     args: [
       // Ten consecutive failures is a dead feed: stop crawling, keep the page.
+      // A dead feed that fails its monthly recheck stays dead whatever its
+      // count says (some were retired by older rules with fewer than ten).
       errorCount >= 10 ? 'dead' : 'error',
       now,
       error,

packages/db/test/db.test.js

@@ -596,6 +596,10 @@ test('crawl failure backs off and eventually marks the feed dead', async () => {
   assert.ok(days > 29.9 && days <= 30, `next check in 30 days -> ${days.toFixed(2)}`);
   assert.equal((await q.dueFeeds(db, 10)).length, 0);
 
+  // Retired with a low count by an older rule: a failed recheck keeps it dead.
+  await q.markCrawlFailure(db, id, 'invalid-url', 3, 43_200, { keepInterval: true });
+  assert.equal((await q.feedBySlug(db, 'test-blog')).status, 'dead', 'dead stays dead');
+
   // A successful recheck restores it.
   await q.markUnchanged(db, id, 60);
   row = await q.feedBySlug(db, 'test-blog');

packages/ingest/src/crawl.js

@@ -357,7 +357,14 @@ export async function crawlFeed(db, feed, opts = {}) {
     // A social source's failures come from a vendor (Arctic Shift, RSSHub) and
     // only "no such source" reaches here, so the month-long flag is for feeds
     // and scraped pages we fetched ourselves.
-    const minutes = social ? backoffMinutes(errorCount) : failureMinutes(resolved.error, errorCount);
+    // A retired feed failing its monthly recheck waits another month, whatever
+    // its count (some were retired with fewer than DEAD_AFTER failures).
+    const minutes =
+      feed.status === 'dead'
+        ? RECHECK_MINUTES
+        : social
+          ? backoffMinutes(errorCount)
+          : failureMinutes(resolved.error, errorCount);
     // A month-long wait is a flag, not a cadence: the feed's own interval is
     // kept so that its tier and its rhythm survive to the recheck, and a
     // recovered feed is not mistaken for one that publishes monthly.

packages/ingest/test/recheck.test.js

@@ -100,3 +100,15 @@ test('a 429 is still a throttle, capped at a day, never a flag', async () => {
   const hours = (Date.parse(next) - Date.now()) / 3_600_000;
   assert.ok(hours > 23.9 && hours <= 24, `capped at a day -> ${hours}`);
 });
+
+test('a dead feed that fails its recheck waits another month, even with a low count', async () => {
+  const db = recorder();
+  await crawlFeed(db, { ...feed, status: 'dead', error_count: 3 }, {
+    resolve: async () => ({ ok: false, error: 'invalid-url' }),
+  });
+  const call = db.calls.find((c) => /last_error/.test(c.sql));
+  assert.match(call.sql, /when status = 'dead' then 'dead'/, 'never downgraded to error');
+  const next = call.args[5];
+  const days = (Date.parse(next) - Date.now()) / 86_400_000;
+  assert.ok(days > 29.99 && days <= 30, `30 days -> ${days}`);
+});