From 20db2f8b20b042713b9be3dcf7b676fdc4948a34 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Fri, 25 Sep 2026 15:30:46 +0000 Subject: [PATCH] robots.txt: refuse GoogleOther Google's non-Search crawler walked every filter combination of /api/v1/search and search.csv, 134 unpaid 402s in an hour on dev2. It indexes nothing for Search, so refusing it by name costs no ranking; an IP ban on 66.249.x would take Googlebot down with it. Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/web/src/routes/static.js | 6 ++++++ test/api.test.js | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/apps/web/src/routes/static.js b/apps/web/src/routes/static.js index ce60a9e..fb9fcde 100644 --- a/apps/web/src/routes/static.js +++ b/apps/web/src/routes/static.js @@ -88,6 +88,12 @@ export function registerStatic(app) { c.text( gateway.robotsTxt({ disallow: ['/login', '/signup', '/settings', '/auth/', '/api/auth/'], + // Google's non-Search crawler walked every filter combination of + // /api/v1/search and search.csv: 134 unpaid 402s in an hour on dev2, + // 2026-09-25. It indexes nothing for Search, so refusing it costs us + // no ranking. Refuse it by name: an IP ban on 66.249.x would take + // Googlebot down with it. + refused: ['GoogleOther'], sitemap: `${config.siteUrl}/sitemap.xml`, }), ), diff --git a/test/api.test.js b/test/api.test.js index eeeb0a7..f65ee35 100644 --- a/test/api.test.js +++ b/test/api.test.js @@ -303,6 +303,11 @@ if (!url) { 404, ); }); + test('robots.txt refuses GoogleOther and still welcomes Googlebot', async () => { + const txt = await (await app.request('/robots.txt')).text(); + expect(txt).toContain('User-agent: GoogleOther\nDisallow: /\n'); + expect(txt).toContain('User-agent: *\nAllow: /'); + }); test('training crawlers get 402, readers do not', async () => { expect( (