diff --git a/packages/db/src/queries.js b/packages/db/src/queries.js index 48a3f83..38ebcb4 100644 --- a/packages/db/src/queries.js +++ b/packages/db/src/queries.js @@ -2845,7 +2845,7 @@ export async function recentlyCrawled(db, limit = 20) { } /** The columns a crawl needs off a feed row. Shared by both due queries. */ -const DUE_COLUMNS = `id, slug, title, feed_url, error_count, fetch_interval_minutes, source_kind, +const DUE_COLUMNS = `id, slug, title, feed_url, status, error_count, fetch_interval_minutes, source_kind, item_count, last_published_at, http_etag, http_last_modified, content_hash, change_log, social_network, social_ref, social_config`; @@ -3569,11 +3569,14 @@ export async function markCrawlFailure(db, id, error, errorCount, minutes, opts const now = nowIso(); await db.execute({ sql: `update feeds set - status = ?, last_fetched_at = ?, last_error = ?, error_count = ?, + status = case when status = 'dead' then 'dead' else ? end, + last_fetched_at = ?, last_error = ?, error_count = ?, fetch_interval_minutes = coalesce(?, fetch_interval_minutes), next_fetch_at = ?, updated_at = ? where id = ?`, args: [ // Ten consecutive failures is a dead feed: stop crawling, keep the page. + // A dead feed that fails its monthly recheck stays dead whatever its + // count says (some were retired by older rules with fewer than ten). errorCount >= 10 ? 'dead' : 'error', now, error, diff --git a/packages/db/test/db.test.js b/packages/db/test/db.test.js index a0977f1..a283f50 100644 --- a/packages/db/test/db.test.js +++ b/packages/db/test/db.test.js @@ -596,6 +596,10 @@ test('crawl failure backs off and eventually marks the feed dead', async () => { assert.ok(days > 29.9 && days <= 30, `next check in 30 days -> ${days.toFixed(2)}`); assert.equal((await q.dueFeeds(db, 10)).length, 0); + // Retired with a low count by an older rule: a failed recheck keeps it dead. + await q.markCrawlFailure(db, id, 'invalid-url', 3, 43_200, { keepInterval: true }); + assert.equal((await q.feedBySlug(db, 'test-blog')).status, 'dead', 'dead stays dead'); + // A successful recheck restores it. await q.markUnchanged(db, id, 60); row = await q.feedBySlug(db, 'test-blog'); diff --git a/packages/ingest/src/crawl.js b/packages/ingest/src/crawl.js index 11591e4..adfa82d 100644 --- a/packages/ingest/src/crawl.js +++ b/packages/ingest/src/crawl.js @@ -357,7 +357,14 @@ export async function crawlFeed(db, feed, opts = {}) { // A social source's failures come from a vendor (Arctic Shift, RSSHub) and // only "no such source" reaches here, so the month-long flag is for feeds // and scraped pages we fetched ourselves. - const minutes = social ? backoffMinutes(errorCount) : failureMinutes(resolved.error, errorCount); + // A retired feed failing its monthly recheck waits another month, whatever + // its count (some were retired with fewer than DEAD_AFTER failures). + const minutes = + feed.status === 'dead' + ? RECHECK_MINUTES + : social + ? backoffMinutes(errorCount) + : failureMinutes(resolved.error, errorCount); // A month-long wait is a flag, not a cadence: the feed's own interval is // kept so that its tier and its rhythm survive to the recheck, and a // recovered feed is not mistaken for one that publishes monthly. diff --git a/packages/ingest/test/recheck.test.js b/packages/ingest/test/recheck.test.js index b449caa..a7d953c 100644 --- a/packages/ingest/test/recheck.test.js +++ b/packages/ingest/test/recheck.test.js @@ -100,3 +100,15 @@ test('a 429 is still a throttle, capped at a day, never a flag', async () => { const hours = (Date.parse(next) - Date.now()) / 3_600_000; assert.ok(hours > 23.9 && hours <= 24, `capped at a day -> ${hours}`); }); + +test('a dead feed that fails its recheck waits another month, even with a low count', async () => { + const db = recorder(); + await crawlFeed(db, { ...feed, status: 'dead', error_count: 3 }, { + resolve: async () => ({ ok: false, error: 'invalid-url' }), + }); + const call = db.calls.find((c) => /last_error/.test(c.sql)); + assert.match(call.sql, /when status = 'dead' then 'dead'/, 'never downgraded to error'); + const next = call.args[5]; + const days = (Date.parse(next) - Date.now()) / 86_400_000; + assert.ok(days > 29.99 && days <= 30, `30 days -> ${days}`); +});