From 92a044a06ce698de42ac1866d6c977866dfd29c9 Mon Sep 17 00:00:00 2001
From: Anthony Ettinger <anthony@profullstack.com>
Date: Sun, 8 Nov 2026 17:52:29 +0000
Subject: [PATCH] crawler: a dead feed that fails its monthly recheck stays
 dead

Follow-up to #275. 721 dead feeds were retired by older rules with fewer
than ten failures (garbage URLs, not-a-hostname); a failed recheck
recomputed status from error_count and downgraded them to 'error', putting
them back in listings and on the 1h..1d ladder. markCrawlFailure now keeps
'dead', and crawlFeed waits 30 days for any dead feed (status is in
DUE_COLUMNS).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 packages/db/src/queries.js           |  7 +++++--
 packages/db/test/db.test.js          |  4 ++++
 packages/ingest/src/crawl.js         |  9 ++++++++-
 packages/ingest/test/recheck.test.js | 12 ++++++++++++
 4 files changed, 29 insertions(+), 3 deletions(-)

diff --git a/packages/db/src/queries.js b/packages/db/src/queries.js
index 48a3f83..38ebcb4 100644
--- a/packages/db/src/queries.js
+++ b/packages/db/src/queries.js
@@ -2845,7 +2845,7 @@ export async function recentlyCrawled(db, limit = 20) {
 }
 
 /** The columns a crawl needs off a feed row. Shared by both due queries. */
-const DUE_COLUMNS = `id, slug, title, feed_url, error_count, fetch_interval_minutes, source_kind,
+const DUE_COLUMNS = `id, slug, title, feed_url, status, error_count, fetch_interval_minutes, source_kind,
                  item_count, last_published_at,
                  http_etag, http_last_modified, content_hash, change_log,
                  social_network, social_ref, social_config`;
@@ -3569,11 +3569,14 @@ export async function markCrawlFailure(db, id, error, errorCount, minutes, opts
   const now = nowIso();
   await db.execute({
     sql: `update feeds set
-            status = ?, last_fetched_at = ?, last_error = ?, error_count = ?,
+            status = case when status = 'dead' then 'dead' else ? end,
+            last_fetched_at = ?, last_error = ?, error_count = ?,
             fetch_interval_minutes = coalesce(?, fetch_interval_minutes), next_fetch_at = ?, updated_at = ?
           where id = ?`,
     args: [
       // Ten consecutive failures is a dead feed: stop crawling, keep the page.
+      // A dead feed that fails its monthly recheck stays dead whatever its
+      // count says (some were retired by older rules with fewer than ten).
       errorCount >= 10 ? 'dead' : 'error',
       now,
       error,
diff --git a/packages/db/test/db.test.js b/packages/db/test/db.test.js
index a0977f1..a283f50 100644
--- a/packages/db/test/db.test.js
+++ b/packages/db/test/db.test.js
@@ -596,6 +596,10 @@ test('crawl failure backs off and eventually marks the feed dead', async () => {
   assert.ok(days > 29.9 && days <= 30, `next check in 30 days -> ${days.toFixed(2)}`);
   assert.equal((await q.dueFeeds(db, 10)).length, 0);
 
+  // Retired with a low count by an older rule: a failed recheck keeps it dead.
+  await q.markCrawlFailure(db, id, 'invalid-url', 3, 43_200, { keepInterval: true });
+  assert.equal((await q.feedBySlug(db, 'test-blog')).status, 'dead', 'dead stays dead');
+
   // A successful recheck restores it.
   await q.markUnchanged(db, id, 60);
   row = await q.feedBySlug(db, 'test-blog');
diff --git a/packages/ingest/src/crawl.js b/packages/ingest/src/crawl.js
index 11591e4..adfa82d 100644
--- a/packages/ingest/src/crawl.js
+++ b/packages/ingest/src/crawl.js
@@ -357,7 +357,14 @@ export async function crawlFeed(db, feed, opts = {}) {
     // A social source's failures come from a vendor (Arctic Shift, RSSHub) and
     // only "no such source" reaches here, so the month-long flag is for feeds
     // and scraped pages we fetched ourselves.
-    const minutes = social ? backoffMinutes(errorCount) : failureMinutes(resolved.error, errorCount);
+    // A retired feed failing its monthly recheck waits another month, whatever
+    // its count (some were retired with fewer than DEAD_AFTER failures).
+    const minutes =
+      feed.status === 'dead'
+        ? RECHECK_MINUTES
+        : social
+          ? backoffMinutes(errorCount)
+          : failureMinutes(resolved.error, errorCount);
     // A month-long wait is a flag, not a cadence: the feed's own interval is
     // kept so that its tier and its rhythm survive to the recheck, and a
     // recovered feed is not mistaken for one that publishes monthly.
diff --git a/packages/ingest/test/recheck.test.js b/packages/ingest/test/recheck.test.js
index b449caa..a7d953c 100644
--- a/packages/ingest/test/recheck.test.js
+++ b/packages/ingest/test/recheck.test.js
@@ -100,3 +100,15 @@ test('a 429 is still a throttle, capped at a day, never a flag', async () => {
   const hours = (Date.parse(next) - Date.now()) / 3_600_000;
   assert.ok(hours > 23.9 && hours <= 24, `capped at a day -> ${hours}`);
 });
+
+test('a dead feed that fails its recheck waits another month, even with a low count', async () => {
+  const db = recorder();
+  await crawlFeed(db, { ...feed, status: 'dead', error_count: 3 }, {
+    resolve: async () => ({ ok: false, error: 'invalid-url' }),
+  });
+  const call = db.calls.find((c) => /last_error/.test(c.sql));
+  assert.match(call.sql, /when status = 'dead' then 'dead'/, 'never downgraded to error');
+  const next = call.args[5];
+  const days = (Date.parse(next) - Date.now()) / 86_400_000;
+  assert.ok(days > 29.99 && days <= 30, `30 days -> ${days}`);
+});