diff --git a/docs/content/2.guides/4.route-rules.md b/docs/content/2.guides/4.route-rules.md
index 7730ff7f..4a067cb1 100644
--- a/docs/content/2.guides/4.route-rules.md
+++ b/docs/content/2.guides/4.route-rules.md
@@ -23,6 +23,14 @@ The rules are applied using the following logic:
- ``{lang="html"}
- `/robots.txt` disallow entry - When [disallowNonIndexableRoutes](/docs/robots/api/config#disallownonindexableroutes) is enabled
+A more specific rule wins. With a default-deny setup, the allowed routes get an `X-Robots-Tag` header with the [robotsEnabledValue](/docs/robots/api/config#robotsenabledvalue) config.
+Prerendered pages and sitemaps resolve route rules per URL, so the explicit allow beats the catch-all.
+
+Static hosts send every matching `_headers` rule.
+A noindex rule that also covers an allowed route would send noindex to that route.
+The module drops such a rule from the output.
+The denied pages keep the `` tag, and `robots.txt` crawling rules still apply.
+
## Inline Route Rules
Requires enabling the experimental `inlineRouteRules`, see the [defineRouteRules](https://nuxt.com/docs/api/utils/define-route-rules) documentation
diff --git a/src/header-rules.ts b/src/header-rules.ts
index 101b6c0a..a5900b84 100644
--- a/src/header-rules.ts
+++ b/src/header-rules.ts
@@ -8,6 +8,7 @@ export interface RobotsHeaderRulesInput {
routeRules: Record
/** Whether the site is indexable at build time. */
indexable: boolean
+ robotsEnabledValue: string
robotsDisabledValue: string
/** The build assets directory, such as `/_nuxt/`. */
buildAssetsDir: string
@@ -29,10 +30,25 @@ export function resolveRobotsHeaderRules(input: RobotsHeaderRulesInput): Record<
[`${input.buildAssetsDir}**`, `${input.buildAssetsDir}file.js`],
])
const headerRules: Record = {}
+ const allowRoutes: string[] = []
+ const noIndexRoutes: string[] = []
for (const [route, rules] of Object.entries(input.routeRules)) {
const robotRule = normaliseRobotsRouteRule(rules)
- if (robotRule && (!robotRule.allow || robotRule.rule?.split(',').some(part => ['noai', 'noimageai'].includes(part.trim()))))
- headerRules[route] = robotRule.rule || input.robotsDisabledValue
+ if (!robotRule)
+ continue
+ headerRules[route] = robotRule.rule || (robotRule.allow ? input.robotsEnabledValue : input.robotsDisabledValue)
+ ;(robotRule.allow ? allowRoutes : noIndexRoutes).push(route)
+ }
+
+ // A noindex rule covering an allowed route would send noindex to it: Nitro merges the more
+ // specific allow over the catch-all, but static hosts apply every matching rule. Drop the
+ // overlapping rule; the meta tag and the robots.txt crawling rules cover the denied paths.
+ for (const route of noIndexRoutes) {
+ if (!allowRoutes.length || !isNoIndexRule(headerRules[route]!))
+ continue
+ const matchNoIndexRule = createNitroRouteRuleMatcher<{ rule?: string }>({ nitro: { routeRules: { [route]: { rule: headerRules[route] } } } })
+ if (allowRoutes.some(allowRoute => matchNoIndexRule(allowRoute.replaceAll('**', 'x').replaceAll('*', 'x')).rule))
+ delete headerRules[route]
}
// If a user rule already sends noindex on build assets, drop the weaker built-in rule.
diff --git a/src/module.ts b/src/module.ts
index 61a79c18..42cef946 100644
--- a/src/module.ts
+++ b/src/module.ts
@@ -543,6 +543,7 @@ export default defineNuxtModule({
routeRules: nuxt.options.routeRules || {},
indexable: (typeof useSiteConfig().indexable !== 'undefined' ? String(useSiteConfig().indexable) === 'true' : useSiteConfig().env === 'production')
&& !config.groups.some(group => asArray(group.userAgent).includes('*') && asArray(group.disallow).includes('/')),
+ robotsEnabledValue: config.robotsEnabledValue,
robotsDisabledValue: config.robotsDisabledValue,
buildAssetsDir: nuxt.options.app.buildAssetsDir,
})
diff --git a/src/runtime/server/routes/robots-txt.ts b/src/runtime/server/routes/robots-txt.ts
index 15792d4a..4dc0d047 100644
--- a/src/runtime/server/routes/robots-txt.ts
+++ b/src/runtime/server/routes/robots-txt.ts
@@ -66,8 +66,11 @@ export default defineEventHandler(async (e) => {
}
if (credits) {
// A catch-all noindex route rule disables indexing, and robots.txt still allows crawling.
- const catchAllRule = normaliseRobotsRouteRule(useRuntimeConfig().nitro?.routeRules?.['/**'] as RobotsRouteRuleConfig | undefined)
- const siteIndexable = indexable && !robotsTxtCtx.groups.some(group => group.userAgent.includes('*') && group.disallow.includes('/')) && !(catchAllRule?.allow === false && isNoIndexRule(catchAllRule.rule || robotsDisabledValue))
+ // More specific allow rules win the route rule merge, so the site stays indexable.
+ const routeRules = (useRuntimeConfig().nitro?.routeRules || {}) as Record
+ const catchAllRule = normaliseRobotsRouteRule(routeRules['/**'])
+ const hasAllowRule = Object.values(routeRules).some(rules => normaliseRobotsRouteRule(rules)?.allow === true)
+ const siteIndexable = indexable && !robotsTxtCtx.groups.some(group => group.userAgent.includes('*') && group.disallow.includes('/')) && !(catchAllRule?.allow === false && isNoIndexRule(catchAllRule.rule || robotsDisabledValue) && !hasAllowRule)
robotsTxt = [
`# START nuxt-robots (${siteIndexable ? 'indexable' : 'indexing disabled'})`,
robotsTxt,
diff --git a/test/e2e/headersRouteRuleAllow.test.ts b/test/e2e/headersRouteRuleAllow.test.ts
new file mode 100644
index 00000000..86bd3cf2
--- /dev/null
+++ b/test/e2e/headersRouteRuleAllow.test.ts
@@ -0,0 +1,41 @@
+import { createResolver } from '@nuxt/kit'
+import { $fetch, setup } from '@nuxt/test-utils'
+import { describe, expect, it } from 'vitest'
+import { fetchRobotsHeaders, findBuildAsset } from './utils/headers'
+
+const { resolve } = createResolver(import.meta.url)
+
+process.env.NODE_ENV = 'production'
+await setup({
+ rootDir: resolve('../fixtures/static-headers'),
+ build: true,
+ nuxtConfig: {
+ routeRules: {
+ '/**': { robots: false },
+ '/': { robots: true },
+ },
+ nitro: {
+ prerender: {
+ routes: ['/'],
+ },
+ },
+ },
+})
+
+// A default-deny config with explicit allows. The catch-all noindex header must not
+// reach the allowed route, which Nitro serves as a prerendered static file.
+describe('a catch-all `robots: false` route rule with an allowed route', () => {
+ it('sends the enabled value on the prerendered allowed route', async () => {
+ expect(await fetchRobotsHeaders('/')).toEqual(['index, follow, max-image-preview:large, max-snippet:-1, max-video-preview:-1'])
+ })
+ it('sends one noindex header on build assets', async () => {
+ expect(await fetchRobotsHeaders(findBuildAsset())).toEqual(['noindex'])
+ })
+ it('drops the overlapping catch-all header from denied static files', async () => {
+ expect(await fetchRobotsHeaders('/api/data.json')).toEqual([])
+ })
+ it('labels robots.txt as indexable', async () => {
+ const robotsTxt = await $fetch('/robots.txt')
+ expect(robotsTxt).toContain('# START nuxt-robots (indexable)')
+ })
+})
diff --git a/test/e2e/headersRouteRuleAllowStatic.test.ts b/test/e2e/headersRouteRuleAllowStatic.test.ts
new file mode 100644
index 00000000..f809c937
--- /dev/null
+++ b/test/e2e/headersRouteRuleAllowStatic.test.ts
@@ -0,0 +1,34 @@
+import { createResolver } from '@nuxt/kit'
+import { setup } from '@nuxt/test-utils'
+import { describe, expect, it } from 'vitest'
+import { findBuildAsset, readHeadersFile, staticRobotsHeaders } from './utils/headers'
+
+const { resolve } = createResolver(import.meta.url)
+
+process.env.NODE_ENV = 'production'
+await setup({
+ rootDir: resolve('../fixtures/static-headers'),
+ build: true,
+ server: false,
+ nuxtConfig: {
+ nitro: { preset: 'cloudflare-module' },
+ routeRules: {
+ '/**': { robots: false },
+ '/': { robots: true },
+ },
+ },
+})
+
+// Static hosts apply every matching `_headers` rule, so an overlapping noindex rule
+// would send noindex to the allowed route. It must drop out of `_headers`.
+describe('_headers for a catch-all `robots: false` route rule with an allowed route', () => {
+ it('sends only the enabled value on the allowed route', () => {
+ expect(staticRobotsHeaders(readHeadersFile(), '/')).toEqual(['index, follow, max-image-preview:large, max-snippet:-1, max-video-preview:-1'])
+ })
+ it('sends no header on denied static files, robots.txt crawling rules cover them', () => {
+ expect(staticRobotsHeaders(readHeadersFile(), '/api/data.json')).toEqual([])
+ })
+ it('sends one noindex header on build assets', () => {
+ expect(staticRobotsHeaders(readHeadersFile(), findBuildAsset())).toEqual(['noindex'])
+ })
+})
diff --git a/test/fixtures/issue338/server/plugins/hosts.ts b/test/fixtures/issue338/server/plugins/hosts.ts
index 91763bed..9b005312 100644
--- a/test/fixtures/issue338/server/plugins/hosts.ts
+++ b/test/fixtures/issue338/server/plugins/hosts.ts
@@ -10,4 +10,3 @@ export default defineNitroPlugin((nitroApp) => {
ctx.groups[0]!.disallow = ['/']
})
})
-
diff --git a/test/unit/headerRules.test.ts b/test/unit/headerRules.test.ts
index 758705b4..c2c1ea0b 100644
--- a/test/unit/headerRules.test.ts
+++ b/test/unit/headerRules.test.ts
@@ -7,10 +7,13 @@ const buildAssetRules = {
'/_nuxt/**': { robots: 'noindex' },
}
+const enabledValue = 'index, follow, max-image-preview:large, max-snippet:-1, max-video-preview:-1'
+
function resolve(routeRules: Record, indexable = true) {
return resolveRobotsHeaderRules({
routeRules: { ...routeRules, ...buildAssetRules },
indexable,
+ robotsEnabledValue: enabledValue,
robotsDisabledValue: 'noindex, nofollow',
buildAssetsDir: '/_nuxt/',
})
@@ -30,9 +33,31 @@ describe('resolveRobotsHeaderRules', () => {
expect(resolve({ '/secret/**': { robots: false } })).toMatchObject({ '/secret/**': 'noindex, nofollow' })
})
- it('sends no header for an indexable rule', () => {
+ it('sends the rule or enabled value for indexable rules', () => {
expect(resolve({ '/open/**': { robots: true }, '/custom': { robots: 'index, follow' } }))
- .toEqual({ '/_nuxt': 'noindex', '/_nuxt/**': 'noindex' })
+ .toEqual({
+ '/open/**': enabledValue,
+ '/custom': 'index, follow',
+ '/_nuxt': 'noindex',
+ '/_nuxt/**': 'noindex',
+ })
+ })
+
+ it('stops a catch-all noindex rule from covering an allowed route', () => {
+ // Nitro merges the explicit allow over the catch-all, but static hosts apply every
+ // matching rule. The catch-all must drop out so the allowed route is indexable.
+ expect(resolve({ '/**': { robots: false }, '/': { robots: true } }))
+ .toEqual({ '/': enabledValue, '/_nuxt': 'noindex', '/_nuxt/**': 'noindex' })
+ })
+
+ it('keeps a noindex rule that does not cover an allowed route', () => {
+ expect(resolve({ '/admin/**': { robots: false }, '/': { robots: true } }))
+ .toEqual({
+ '/': enabledValue,
+ '/admin/**': 'noindex, nofollow',
+ '/_nuxt': 'noindex',
+ '/_nuxt/**': 'noindex',
+ })
})
it('drops the build asset rules when a catch-all rule already sends noindex', () => {