Crawler rule: own services keep the short retry

packages/core/src/crawl.js

@@ -54,6 +54,24 @@ const BROKEN_CODES = new Set([
 const TLS_CODE =
   /^(ERR_TLS_|ERR_SSL_|CERT_|UNABLE_TO_|DEPTH_ZERO_SELF_SIGNED_CERT|SELF_SIGNED_CERT_IN_CHAIN|HOSTNAME_MISMATCH)/;
 
+/**
+ * Our own services and the vendors behind them (rule 4 of the crawler rule):
+ * a house site, a pit.world host, our Supabase projects. A failure talking to
+ * one of these is ours to fix, not a slow third party, so it keeps the short
+ * retry instead of 30 days. Matched on the host the request went to.
+ */
+export const OWN_HOSTS =
+  /(^|\.)(pit\.world|profullstack\.com|nichedb\.dev|bittorrented\.com|ch4nnels\.com|c0upons\.com|outreachgraph\.com|smshub\.dev|rssamplifier\.com|p0dcasters\.com|logicsrc\.com|c0mpute\.com|crawlproof\.com|r4ck\.dev|weedforcrypto\.com|supabase\.co|coinpayportal\.com)$/i;
+
+/** The host a failed request went to: makeHttp records it; otherwise the first one the message names. */
+export function failedHost(err) {
+  const raw = err?.upstreamUrl ?? String(err?.message ?? '');
+  const m =
+    String(raw).match(/https?:\/\/([^/\s:?#]+)/i) ??
+    String(raw).match(/\b(?:ENOTFOUND|EAI_AGAIN)\s+([a-z0-9.-]+\.[a-z]{2,})/i);
+  return m ? m[1].toLowerCase() : null;
+}
+
 function codesOf(err) {
   const out = [];
   for (let e = err, depth = 0; e && depth < 4; e = e.cause, depth++) {
@@ -76,6 +94,8 @@ export function failureClass(err) {
   if (!err || (typeof err !== 'object' && typeof err !== 'function')) return null;
   if ('flag' in err && (err.flag === 'slow' || err.flag === 'broken' || err.flag === null))
     return err.flag;
+  const host = failedHost(err);
+  if (host && OWN_HOSTS.test(host)) return null;
   const codes = codesOf(err);
   if (codes.includes('EAI_AGAIN')) return null;
   // Our own database failing is never the upstream's fault: Bun SQL errors

packages/core/src/http.js

@@ -23,7 +23,7 @@ export function makeHttp({ userAgent, log = () => {} }) {
     try {
       return await send(url, opts);
     } catch (err) {
-      throw markUpstream(err);
+      throw markUpstream(err, url);
     }
   }
 
@@ -186,14 +186,16 @@ async function httpError(res, url) {
   return Object.assign(new Error(`${res.status} from ${String(url).slice(0, 400)}${detail}`), {
     status: res.status,
     upstream: true,
+    upstreamUrl: String(url),
   });
 }
 
 /** Mark a fetch failure as the upstream's, keeping the error itself (its name, code and cause). */
-function markUpstream(err) {
+function markUpstream(err, url) {
   if (err && typeof err === 'object') {
     try {
       err.upstream = true;
+      err.upstreamUrl ??= String(url);
     } catch {
       // A frozen error stays as it is.
     }

test/crawl-rule.test.js

@@ -90,6 +90,33 @@ describe('failureClass', () => {
   });
 });
 
+describe('our own services keep the short retry (rule 4)', () => {
+  test('a house site, a pit.world host or our Supabase is never flagged', () => {
+    expect(failureClass(sysErr('ENOTFOUND', 'getaddrinfo ENOTFOUND abcxyz.supabase.co'))).toBe(
+      null,
+    );
+    expect(
+      failureClass(
+        Object.assign(new Error('502 from https://ch4nnels.com/api/v1/seedboxes'), { status: 502 }),
+      ),
+    ).toBe(null);
+    expect(
+      failureClass(
+        Object.assign(new Error('The operation timed out.'), {
+          name: 'TimeoutError',
+          upstreamUrl: 'https://built.indie.pit.world/api/v1/products',
+        }),
+      ),
+    ).toBe(null);
+    // A third party on a lookalike name is still a third party.
+    expect(
+      failureClass(
+        Object.assign(new Error('502 from https://notpit.world.example/x'), { status: 502 }),
+      ),
+    ).toBe('broken');
+  });
+});
+
 describe('runSource: a slow or broken upstream is flagged for 30 days', () => {
   const fail = (err) => ({ name: 'fake', pull: async () => Promise.reject(err) });
   const run = (adapter) => {