fix(route-rules): allow rules beat a catch-all noindex header

docs/content/2.guides/4.route-rules.md

@@ -23,6 +23,14 @@ The rules are applied using the following logic:
 - `<meta name="robots">`{lang="html"}
 - `/robots.txt` disallow entry - When [disallowNonIndexableRoutes](/docs/robots/api/config#disallownonindexableroutes) is enabled
 
+A more specific rule wins. With a default-deny setup, the allowed routes get an `X-Robots-Tag` header with the [robotsEnabledValue](/docs/robots/api/config#robotsenabledvalue) config.
+Prerendered pages and sitemaps resolve route rules per URL, so the explicit allow beats the catch-all.
+
+Static hosts send every matching `_headers` rule.
+A noindex rule that also covers an allowed route would send noindex to that route.
+The module drops such a rule from the output.
+The denied pages keep the `<meta name="robots">` tag, and `robots.txt` crawling rules still apply.
+
 ## Inline Route Rules
 
 Requires enabling the experimental `inlineRouteRules`, see the [defineRouteRules](https://nuxt.com/docs/api/utils/define-route-rules) documentation

src/header-rules.ts

@@ -8,6 +8,7 @@ export interface RobotsHeaderRulesInput {
   routeRules: Record<string, RobotsRouteRuleConfig | undefined>
   /** Whether the site is indexable at build time. */
   indexable: boolean
+  robotsEnabledValue: string
   robotsDisabledValue: string
   /** The build assets directory, such as `/_nuxt/`. */
   buildAssetsDir: string
@@ -29,10 +30,25 @@ export function resolveRobotsHeaderRules(input: RobotsHeaderRulesInput): Record<
     [`${input.buildAssetsDir}**`, `${input.buildAssetsDir}file.js`],
   ])
   const headerRules: Record<string, string> = {}
+  const allowRoutes: string[] = []
+  const noIndexRoutes: string[] = []
   for (const [route, rules] of Object.entries(input.routeRules)) {
     const robotRule = normaliseRobotsRouteRule(rules)
-    if (robotRule && (!robotRule.allow || robotRule.rule?.split(',').some(part => ['noai', 'noimageai'].includes(part.trim()))))
-      headerRules[route] = robotRule.rule || input.robotsDisabledValue
+    if (!robotRule)
+      continue
+    headerRules[route] = robotRule.rule || (robotRule.allow ? input.robotsEnabledValue : input.robotsDisabledValue)
+    ;(robotRule.allow ? allowRoutes : noIndexRoutes).push(route)
+  }
+
+  // A noindex rule covering an allowed route would send noindex to it: Nitro merges the more
+  // specific allow over the catch-all, but static hosts apply every matching rule. Drop the
+  // overlapping rule; the meta tag and the robots.txt crawling rules cover the denied paths.
+  for (const route of noIndexRoutes) {
+    if (!allowRoutes.length || !isNoIndexRule(headerRules[route]!))
+      continue
+    const matchNoIndexRule = createNitroRouteRuleMatcher<{ rule?: string }>({ nitro: { routeRules: { [route]: { rule: headerRules[route] } } } })
+    if (allowRoutes.some(allowRoute => matchNoIndexRule(allowRoute.replaceAll('**', 'x').replaceAll('*', 'x')).rule))
+      delete headerRules[route]
   }
 
   // If a user rule already sends noindex on build assets, drop the weaker built-in rule.

src/module.ts

@@ -543,6 +543,7 @@ export default defineNuxtModule<ModuleOptions>({
           routeRules: nuxt.options.routeRules || {},
           indexable: (typeof useSiteConfig().indexable !== 'undefined' ? String(useSiteConfig().indexable) === 'true' : useSiteConfig().env === 'production')
             && !config.groups.some(group => asArray(group.userAgent).includes('*') && asArray(group.disallow).includes('/')),
+          robotsEnabledValue: config.robotsEnabledValue,
           robotsDisabledValue: config.robotsDisabledValue,
           buildAssetsDir: nuxt.options.app.buildAssetsDir,
         })

src/runtime/server/routes/robots-txt.ts

@@ -66,8 +66,11 @@ export default defineEventHandler(async (e) => {
   }
   if (credits) {
     // A catch-all noindex route rule disables indexing, and robots.txt still allows crawling.
-    const catchAllRule = normaliseRobotsRouteRule(useRuntimeConfig().nitro?.routeRules?.['/**'] as RobotsRouteRuleConfig | undefined)
-    const siteIndexable = indexable && !robotsTxtCtx.groups.some(group => group.userAgent.includes('*') && group.disallow.includes('/')) && !(catchAllRule?.allow === false && isNoIndexRule(catchAllRule.rule || robotsDisabledValue))
+    // More specific allow rules win the route rule merge, so the site stays indexable.
+    const routeRules = (useRuntimeConfig().nitro?.routeRules || {}) as Record<string, RobotsRouteRuleConfig | undefined>
+    const catchAllRule = normaliseRobotsRouteRule(routeRules['/**'])
+    const hasAllowRule = Object.values(routeRules).some(rules => normaliseRobotsRouteRule(rules)?.allow === true)
+    const siteIndexable = indexable && !robotsTxtCtx.groups.some(group => group.userAgent.includes('*') && group.disallow.includes('/')) && !(catchAllRule?.allow === false && isNoIndexRule(catchAllRule.rule || robotsDisabledValue) && !hasAllowRule)
     robotsTxt = [
       `# START nuxt-robots (${siteIndexable ? 'indexable' : 'indexing disabled'})`,
       robotsTxt,

test/e2e/headersRouteRuleAllow.test.ts

@@ -0,0 +1,41 @@
+import { createResolver } from '@nuxt/kit'
+import { $fetch, setup } from '@nuxt/test-utils'
+import { describe, expect, it } from 'vitest'
+import { fetchRobotsHeaders, findBuildAsset } from './utils/headers'
+
+const { resolve } = createResolver(import.meta.url)
+
+process.env.NODE_ENV = 'production'
+await setup({
+  rootDir: resolve('../fixtures/static-headers'),
+  build: true,
+  nuxtConfig: {
+    routeRules: {
+      '/**': { robots: false },
+      '/': { robots: true },
+    },
+    nitro: {
+      prerender: {
+        routes: ['/'],
+      },
+    },
+  },
+})
+
+// A default-deny config with explicit allows. The catch-all noindex header must not
+// reach the allowed route, which Nitro serves as a prerendered static file.
+describe('a catch-all `robots: false` route rule with an allowed route', () => {
+  it('sends the enabled value on the prerendered allowed route', async () => {
+    expect(await fetchRobotsHeaders('/')).toEqual(['index, follow, max-image-preview:large, max-snippet:-1, max-video-preview:-1'])
+  })
+  it('sends one noindex header on build assets', async () => {
+    expect(await fetchRobotsHeaders(findBuildAsset())).toEqual(['noindex'])
+  })
+  it('drops the overlapping catch-all header from denied static files', async () => {
+    expect(await fetchRobotsHeaders('/api/data.json')).toEqual([])
+  })
+  it('labels robots.txt as indexable', async () => {
+    const robotsTxt = await $fetch<string>('/robots.txt')
+    expect(robotsTxt).toContain('# START nuxt-robots (indexable)')
+  })
+})

test/e2e/headersRouteRuleAllowStatic.test.ts

@@ -0,0 +1,34 @@
+import { createResolver } from '@nuxt/kit'
+import { setup } from '@nuxt/test-utils'
+import { describe, expect, it } from 'vitest'
+import { findBuildAsset, readHeadersFile, staticRobotsHeaders } from './utils/headers'
+
+const { resolve } = createResolver(import.meta.url)
+
+process.env.NODE_ENV = 'production'
+await setup({
+  rootDir: resolve('../fixtures/static-headers'),
+  build: true,
+  server: false,
+  nuxtConfig: {
+    nitro: { preset: 'cloudflare-module' },
+    routeRules: {
+      '/**': { robots: false },
+      '/': { robots: true },
+    },
+  },
+})
+
+// Static hosts apply every matching `_headers` rule, so an overlapping noindex rule
+// would send noindex to the allowed route. It must drop out of `_headers`.
+describe('_headers for a catch-all `robots: false` route rule with an allowed route', () => {
+  it('sends only the enabled value on the allowed route', () => {
+    expect(staticRobotsHeaders(readHeadersFile(), '/')).toEqual(['index, follow, max-image-preview:large, max-snippet:-1, max-video-preview:-1'])
+  })
+  it('sends no header on denied static files, robots.txt crawling rules cover them', () => {
+    expect(staticRobotsHeaders(readHeadersFile(), '/api/data.json')).toEqual([])
+  })
+  it('sends one noindex header on build assets', () => {
+    expect(staticRobotsHeaders(readHeadersFile(), findBuildAsset())).toEqual(['noindex'])
+  })
+})

test/fixtures/issue338/server/plugins/hosts.ts

@@ -10,4 +10,3 @@ export default defineNitroPlugin((nitroApp) => {
       ctx.groups[0]!.disallow = ['/']
   })
 })
-

test/unit/headerRules.test.ts

@@ -7,10 +7,13 @@ const buildAssetRules = {
   '/_nuxt/**': { robots: 'noindex' },
 }
 
+const enabledValue = 'index, follow, max-image-preview:large, max-snippet:-1, max-video-preview:-1'
+
 function resolve(routeRules: Record<string, any>, indexable = true) {
   return resolveRobotsHeaderRules({
     routeRules: { ...routeRules, ...buildAssetRules },
     indexable,
+    robotsEnabledValue: enabledValue,
     robotsDisabledValue: 'noindex, nofollow',
     buildAssetsDir: '/_nuxt/',
   })
@@ -30,9 +33,31 @@ describe('resolveRobotsHeaderRules', () => {
     expect(resolve({ '/secret/**': { robots: false } })).toMatchObject({ '/secret/**': 'noindex, nofollow' })
   })
 
-  it('sends no header for an indexable rule', () => {
+  it('sends the rule or enabled value for indexable rules', () => {
     expect(resolve({ '/open/**': { robots: true }, '/custom': { robots: 'index, follow' } }))
-      .toEqual({ '/_nuxt': 'noindex', '/_nuxt/**': 'noindex' })
+      .toEqual({
+        '/open/**': enabledValue,
+        '/custom': 'index, follow',
+        '/_nuxt': 'noindex',
+        '/_nuxt/**': 'noindex',
+      })
+  })
+
+  it('stops a catch-all noindex rule from covering an allowed route', () => {
+    // Nitro merges the explicit allow over the catch-all, but static hosts apply every
+    // matching rule. The catch-all must drop out so the allowed route is indexable.
+    expect(resolve({ '/**': { robots: false }, '/': { robots: true } }))
+      .toEqual({ '/': enabledValue, '/_nuxt': 'noindex', '/_nuxt/**': 'noindex' })
+  })
+
+  it('keeps a noindex rule that does not cover an allowed route', () => {
+    expect(resolve({ '/admin/**': { robots: false }, '/': { robots: true } }))
+      .toEqual({
+        '/': enabledValue,
+        '/admin/**': 'noindex, nofollow',
+        '/_nuxt': 'noindex',
+        '/_nuxt/**': 'noindex',
+      })
   })
 
   it('drops the build asset rules when a catch-all rule already sends noindex', () => {