From b551f4bda5e9ef30533e94ecf642e252d9f2de17 Mon Sep 17 00:00:00 2001 From: Bruno Perez Date: Tue, 28 Jul 2026 10:13:31 +0200 Subject: [PATCH] fix: keep API JSON out of Google results Google had started indexing /api/v1/*.json endpoints (18 URLs in Search Console). Those are machine payloads, not pages, and they dilute the crawl budget that should go to the model, provider and parameter pages. Serve /api/v1 with X-Robots-Tag: noindex (vercel.json for prod, an Express middleware for dev/test parity) and drop rel="nofollow" on every internal link to a .json endpoint. The endpoints stay public and crawlable on purpose: a robots.txt Disallow would hide the noindex and strand the already-discovered URLs in the index. Also adds /api, the HTML docs page, to the sitemap. It was canonical and indexable but missing from it. --- src/build/build.ts | 7 +++-- src/build/render-api.ts | 4 +-- src/data/robots.ts | 26 +++++++++++++++++ src/data/urls.ts | 3 ++ src/server/app.ts | 9 ++++++ src/views/model.ejs | 2 +- src/views/partials/footer.ejs | 4 +-- src/views/partials/how_to_use.ejs | 12 ++++---- tests/robots.test.ts | 47 +++++++++++++++++++++++++++++++ tests/server.test.ts | 34 ++++++++++++++++++++++ vercel.json | 4 +++ 11 files changed, 139 insertions(+), 13 deletions(-) create mode 100644 src/data/robots.ts create mode 100644 tests/robots.test.ts diff --git a/src/build/build.ts b/src/build/build.ts index 42dda34..2c2be7d 100644 --- a/src/build/build.ts +++ b/src/build/build.ts @@ -18,7 +18,9 @@ import { REPO_ROOT, } from "../data/paths.js"; import { SITE_URL } from "../data/site.js"; +import { buildRobotsTxt } from "../data/robots.js"; import { + API_PATH, GLOSSARY_PATH, modelPagePath, parameterPagePath, @@ -57,11 +59,11 @@ async function writeLlmsFiles(models: Model[]): Promise { } async function writeRobotsAndSitemap(models: Model[]): Promise { - const robots = `# AI agents welcome. Machine-readable overview: ${SITE_URL}/llms.txt\nUser-agent: *\nAllow: /\nSitemap: ${SITE_URL}/sitemap.xml\n`; - await fs.writeFile(path.join(DIST_DIR, "robots.txt"), robots, "utf8"); + await fs.writeFile(path.join(DIST_DIR, "robots.txt"), buildRobotsTxt(SITE_URL), "utf8"); // Sitemaps list canonical, indexable HTML pages only — the JSON API and the // .txt agent files are intentionally excluded (they're not search results). + // /api is the HTML documentation page, so it belongs here. const today = new Date().toISOString().slice(0, 10); const dates = gitLastmodMap(REPO_ROOT); // Aggregate pages (home, glossary, providers, parameters) track the build date; @@ -69,6 +71,7 @@ async function writeRobotsAndSitemap(models: Model[]): Promise { const entries: { path: string; priority: string; lastmod: string }[] = [ { path: "/", priority: "1.0", lastmod: today }, { path: GLOSSARY_PATH, priority: "0.7", lastmod: today }, + { path: API_PATH, priority: "0.5", lastmod: today }, ...uniqueProviders(models).map((provider) => ({ path: providerPagePath(provider), priority: "0.8", diff --git a/src/build/render-api.ts b/src/build/render-api.ts index 4a5797c..50e88f0 100644 --- a/src/build/render-api.ts +++ b/src/build/render-api.ts @@ -2,7 +2,7 @@ import path from "node:path"; import ejs from "ejs"; import { VIEWS_DIR } from "../data/paths.js"; import { SITE_NAME, SITE_URL } from "../data/site.js"; -import { absolute } from "../data/urls.js"; +import { API_PATH, absolute } from "../data/urls.js"; import { type Model } from "../schema/model.js"; import { hubLinks, renderShell, viewHelpers } from "./render.js"; @@ -19,7 +19,7 @@ export async function renderApiPage(allModels: Model[]): Promise { { title: API_TITLE, description: API_DESCRIPTION, - canonicalUrl: absolute(SITE_URL, "/api"), + canonicalUrl: absolute(SITE_URL, API_PATH), structuredData: "{}", providerHubs: hubLinks(allModels), }, diff --git a/src/data/robots.ts b/src/data/robots.ts new file mode 100644 index 0000000..ef9c4ea --- /dev/null +++ b/src/data/robots.ts @@ -0,0 +1,26 @@ +// robots.txt builder. Kept as a pure function of the site origin so the crawl +// policy is unit-testable instead of buried in the build script. + +/** + * Crawl policy for the site. + * + * The JSON API under /api/v1 is deliberately left crawlable: it is served with + * `X-Robots-Tag: noindex` (see vercel.json and src/server/app.ts), and a + * `Disallow` here would hide that header from crawlers — Google would keep the + * already-discovered endpoints in the index as bare URLs instead of dropping + * them. Search engines belong on the HTML pages; the JSON is for programmatic + * callers, which ignore robots directives anyway. + */ +export function buildRobotsTxt(siteUrl: string): string { + return [ + `# AI agents welcome. Machine-readable overview: ${siteUrl}/llms.txt`, + "User-agent: *", + "Allow: /", + "", + "# /api/v1/*.json is crawlable on purpose so crawlers can read its noindex", + "# header. Do not Disallow it — that would strand those URLs in the index.", + "", + `Sitemap: ${siteUrl}/sitemap.xml`, + "", + ].join("\n"); +} diff --git a/src/data/urls.ts b/src/data/urls.ts index 55e9463..d8d21ee 100644 --- a/src/data/urls.ts +++ b/src/data/urls.ts @@ -19,6 +19,9 @@ export function providerPagePath(provider: string): string { /** Parameter glossary page — the hub that links out to each parameter page. */ export const GLOSSARY_PATH = "/glossary"; +/** API documentation page. The HTML docs, not the JSON endpoints under /api/v1. */ +export const API_PATH = "/api"; + /** * URL-safe slug for a parameter path: lowercased, with nested-path dots turned into * hyphens, e.g. `thinking.type` → `thinking-type`. Underscores are kept so that diff --git a/src/server/app.ts b/src/server/app.ts index 5ed6505..32b8ac9 100644 --- a/src/server/app.ts +++ b/src/server/app.ts @@ -28,6 +28,15 @@ export function makeApp(loadModels: LoadModels): express.Express { app.use("/assets", express.static(DIST_ASSETS_DIR, { maxAge: 0 })); + // The JSON API is for programmatic callers, not search results: `noindex` keeps + // crawlers on the HTML pages while every endpoint stays fetchable. Scoped to + // /api/v1 so the HTML docs page at /api keeps its place in the index. Production + // is static, so vercel.json carries the same header — this is dev/test parity. + app.use("/api/v1", (_req, res, next) => { + res.setHeader("X-Robots-Tag", "noindex"); + next(); + }); + app.get("/", async (_req, res, next) => { try { const models = await loadModels(); diff --git a/src/views/model.ejs b/src/views/model.ejs index fe51ef3..032d66d 100644 --- a/src/views/model.ejs +++ b/src/views/model.ejs @@ -110,7 +110,7 @@ The full model definition as served by the API. Copy it or open the endpoint directly.

- <%= jsonPath %> + <%= jsonPath %>