diff --git a/docs/content/2.guides/4.route-rules.md b/docs/content/2.guides/4.route-rules.md index 7730ff7f..4a067cb1 100644 --- a/docs/content/2.guides/4.route-rules.md +++ b/docs/content/2.guides/4.route-rules.md @@ -23,6 +23,14 @@ The rules are applied using the following logic: - ``{lang="html"} - `/robots.txt` disallow entry - When [disallowNonIndexableRoutes](/docs/robots/api/config#disallownonindexableroutes) is enabled +A more specific rule wins. With a default-deny setup, the allowed routes get an `X-Robots-Tag` header with the [robotsEnabledValue](/docs/robots/api/config#robotsenabledvalue) config. +Prerendered pages and sitemaps resolve route rules per URL, so the explicit allow beats the catch-all. + +Static hosts send every matching `_headers` rule. +A noindex rule that also covers an allowed route would send noindex to that route. +The module drops such a rule from the output. +The denied pages keep the `` tag, and `robots.txt` crawling rules still apply. + ## Inline Route Rules Requires enabling the experimental `inlineRouteRules`, see the [defineRouteRules](https://nuxt.com/docs/api/utils/define-route-rules) documentation diff --git a/src/header-rules.ts b/src/header-rules.ts index 101b6c0a..a5900b84 100644 --- a/src/header-rules.ts +++ b/src/header-rules.ts @@ -8,6 +8,7 @@ export interface RobotsHeaderRulesInput { routeRules: Record /** Whether the site is indexable at build time. */ indexable: boolean + robotsEnabledValue: string robotsDisabledValue: string /** The build assets directory, such as `/_nuxt/`. */ buildAssetsDir: string @@ -29,10 +30,25 @@ export function resolveRobotsHeaderRules(input: RobotsHeaderRulesInput): Record< [`${input.buildAssetsDir}**`, `${input.buildAssetsDir}file.js`], ]) const headerRules: Record = {} + const allowRoutes: string[] = [] + const noIndexRoutes: string[] = [] for (const [route, rules] of Object.entries(input.routeRules)) { const robotRule = normaliseRobotsRouteRule(rules) - if (robotRule && (!robotRule.allow || robotRule.rule?.split(',').some(part => ['noai', 'noimageai'].includes(part.trim())))) - headerRules[route] = robotRule.rule || input.robotsDisabledValue + if (!robotRule) + continue + headerRules[route] = robotRule.rule || (robotRule.allow ? input.robotsEnabledValue : input.robotsDisabledValue) + ;(robotRule.allow ? allowRoutes : noIndexRoutes).push(route) + } + + // A noindex rule covering an allowed route would send noindex to it: Nitro merges the more + // specific allow over the catch-all, but static hosts apply every matching rule. Drop the + // overlapping rule; the meta tag and the robots.txt crawling rules cover the denied paths. + for (const route of noIndexRoutes) { + if (!allowRoutes.length || !isNoIndexRule(headerRules[route]!)) + continue + const matchNoIndexRule = createNitroRouteRuleMatcher<{ rule?: string }>({ nitro: { routeRules: { [route]: { rule: headerRules[route] } } } }) + if (allowRoutes.some(allowRoute => matchNoIndexRule(allowRoute.replaceAll('**', 'x').replaceAll('*', 'x')).rule)) + delete headerRules[route] } // If a user rule already sends noindex on build assets, drop the weaker built-in rule. diff --git a/src/module.ts b/src/module.ts index 61a79c18..42cef946 100644 --- a/src/module.ts +++ b/src/module.ts @@ -543,6 +543,7 @@ export default defineNuxtModule({ routeRules: nuxt.options.routeRules || {}, indexable: (typeof useSiteConfig().indexable !== 'undefined' ? String(useSiteConfig().indexable) === 'true' : useSiteConfig().env === 'production') && !config.groups.some(group => asArray(group.userAgent).includes('*') && asArray(group.disallow).includes('/')), + robotsEnabledValue: config.robotsEnabledValue, robotsDisabledValue: config.robotsDisabledValue, buildAssetsDir: nuxt.options.app.buildAssetsDir, }) diff --git a/src/runtime/server/routes/robots-txt.ts b/src/runtime/server/routes/robots-txt.ts index 15792d4a..4dc0d047 100644 --- a/src/runtime/server/routes/robots-txt.ts +++ b/src/runtime/server/routes/robots-txt.ts @@ -66,8 +66,11 @@ export default defineEventHandler(async (e) => { } if (credits) { // A catch-all noindex route rule disables indexing, and robots.txt still allows crawling. - const catchAllRule = normaliseRobotsRouteRule(useRuntimeConfig().nitro?.routeRules?.['/**'] as RobotsRouteRuleConfig | undefined) - const siteIndexable = indexable && !robotsTxtCtx.groups.some(group => group.userAgent.includes('*') && group.disallow.includes('/')) && !(catchAllRule?.allow === false && isNoIndexRule(catchAllRule.rule || robotsDisabledValue)) + // More specific allow rules win the route rule merge, so the site stays indexable. + const routeRules = (useRuntimeConfig().nitro?.routeRules || {}) as Record + const catchAllRule = normaliseRobotsRouteRule(routeRules['/**']) + const hasAllowRule = Object.values(routeRules).some(rules => normaliseRobotsRouteRule(rules)?.allow === true) + const siteIndexable = indexable && !robotsTxtCtx.groups.some(group => group.userAgent.includes('*') && group.disallow.includes('/')) && !(catchAllRule?.allow === false && isNoIndexRule(catchAllRule.rule || robotsDisabledValue) && !hasAllowRule) robotsTxt = [ `# START nuxt-robots (${siteIndexable ? 'indexable' : 'indexing disabled'})`, robotsTxt, diff --git a/test/e2e/headersRouteRuleAllow.test.ts b/test/e2e/headersRouteRuleAllow.test.ts new file mode 100644 index 00000000..86bd3cf2 --- /dev/null +++ b/test/e2e/headersRouteRuleAllow.test.ts @@ -0,0 +1,41 @@ +import { createResolver } from '@nuxt/kit' +import { $fetch, setup } from '@nuxt/test-utils' +import { describe, expect, it } from 'vitest' +import { fetchRobotsHeaders, findBuildAsset } from './utils/headers' + +const { resolve } = createResolver(import.meta.url) + +process.env.NODE_ENV = 'production' +await setup({ + rootDir: resolve('../fixtures/static-headers'), + build: true, + nuxtConfig: { + routeRules: { + '/**': { robots: false }, + '/': { robots: true }, + }, + nitro: { + prerender: { + routes: ['/'], + }, + }, + }, +}) + +// A default-deny config with explicit allows. The catch-all noindex header must not +// reach the allowed route, which Nitro serves as a prerendered static file. +describe('a catch-all `robots: false` route rule with an allowed route', () => { + it('sends the enabled value on the prerendered allowed route', async () => { + expect(await fetchRobotsHeaders('/')).toEqual(['index, follow, max-image-preview:large, max-snippet:-1, max-video-preview:-1']) + }) + it('sends one noindex header on build assets', async () => { + expect(await fetchRobotsHeaders(findBuildAsset())).toEqual(['noindex']) + }) + it('drops the overlapping catch-all header from denied static files', async () => { + expect(await fetchRobotsHeaders('/api/data.json')).toEqual([]) + }) + it('labels robots.txt as indexable', async () => { + const robotsTxt = await $fetch('/robots.txt') + expect(robotsTxt).toContain('# START nuxt-robots (indexable)') + }) +}) diff --git a/test/e2e/headersRouteRuleAllowStatic.test.ts b/test/e2e/headersRouteRuleAllowStatic.test.ts new file mode 100644 index 00000000..f809c937 --- /dev/null +++ b/test/e2e/headersRouteRuleAllowStatic.test.ts @@ -0,0 +1,34 @@ +import { createResolver } from '@nuxt/kit' +import { setup } from '@nuxt/test-utils' +import { describe, expect, it } from 'vitest' +import { findBuildAsset, readHeadersFile, staticRobotsHeaders } from './utils/headers' + +const { resolve } = createResolver(import.meta.url) + +process.env.NODE_ENV = 'production' +await setup({ + rootDir: resolve('../fixtures/static-headers'), + build: true, + server: false, + nuxtConfig: { + nitro: { preset: 'cloudflare-module' }, + routeRules: { + '/**': { robots: false }, + '/': { robots: true }, + }, + }, +}) + +// Static hosts apply every matching `_headers` rule, so an overlapping noindex rule +// would send noindex to the allowed route. It must drop out of `_headers`. +describe('_headers for a catch-all `robots: false` route rule with an allowed route', () => { + it('sends only the enabled value on the allowed route', () => { + expect(staticRobotsHeaders(readHeadersFile(), '/')).toEqual(['index, follow, max-image-preview:large, max-snippet:-1, max-video-preview:-1']) + }) + it('sends no header on denied static files, robots.txt crawling rules cover them', () => { + expect(staticRobotsHeaders(readHeadersFile(), '/api/data.json')).toEqual([]) + }) + it('sends one noindex header on build assets', () => { + expect(staticRobotsHeaders(readHeadersFile(), findBuildAsset())).toEqual(['noindex']) + }) +}) diff --git a/test/fixtures/issue338/server/plugins/hosts.ts b/test/fixtures/issue338/server/plugins/hosts.ts index 91763bed..9b005312 100644 --- a/test/fixtures/issue338/server/plugins/hosts.ts +++ b/test/fixtures/issue338/server/plugins/hosts.ts @@ -10,4 +10,3 @@ export default defineNitroPlugin((nitroApp) => { ctx.groups[0]!.disallow = ['/'] }) }) - diff --git a/test/unit/headerRules.test.ts b/test/unit/headerRules.test.ts index 758705b4..c2c1ea0b 100644 --- a/test/unit/headerRules.test.ts +++ b/test/unit/headerRules.test.ts @@ -7,10 +7,13 @@ const buildAssetRules = { '/_nuxt/**': { robots: 'noindex' }, } +const enabledValue = 'index, follow, max-image-preview:large, max-snippet:-1, max-video-preview:-1' + function resolve(routeRules: Record, indexable = true) { return resolveRobotsHeaderRules({ routeRules: { ...routeRules, ...buildAssetRules }, indexable, + robotsEnabledValue: enabledValue, robotsDisabledValue: 'noindex, nofollow', buildAssetsDir: '/_nuxt/', }) @@ -30,9 +33,31 @@ describe('resolveRobotsHeaderRules', () => { expect(resolve({ '/secret/**': { robots: false } })).toMatchObject({ '/secret/**': 'noindex, nofollow' }) }) - it('sends no header for an indexable rule', () => { + it('sends the rule or enabled value for indexable rules', () => { expect(resolve({ '/open/**': { robots: true }, '/custom': { robots: 'index, follow' } })) - .toEqual({ '/_nuxt': 'noindex', '/_nuxt/**': 'noindex' }) + .toEqual({ + '/open/**': enabledValue, + '/custom': 'index, follow', + '/_nuxt': 'noindex', + '/_nuxt/**': 'noindex', + }) + }) + + it('stops a catch-all noindex rule from covering an allowed route', () => { + // Nitro merges the explicit allow over the catch-all, but static hosts apply every + // matching rule. The catch-all must drop out so the allowed route is indexable. + expect(resolve({ '/**': { robots: false }, '/': { robots: true } })) + .toEqual({ '/': enabledValue, '/_nuxt': 'noindex', '/_nuxt/**': 'noindex' }) + }) + + it('keeps a noindex rule that does not cover an allowed route', () => { + expect(resolve({ '/admin/**': { robots: false }, '/': { robots: true } })) + .toEqual({ + '/': enabledValue, + '/admin/**': 'noindex, nofollow', + '/_nuxt': 'noindex', + '/_nuxt/**': 'noindex', + }) }) it('drops the build asset rules when a catch-all rule already sends noindex', () => {