fix: keep API JSON out of Google results

src/build/build.ts

@@ -18,7 +18,9 @@ import {
   REPO_ROOT,
 } from "../data/paths.js";
 import { SITE_URL } from "../data/site.js";
+import { buildRobotsTxt } from "../data/robots.js";
 import {
+  API_PATH,
   GLOSSARY_PATH,
   modelPagePath,
   parameterPagePath,
@@ -57,18 +59,19 @@ async function writeLlmsFiles(models: Model[]): Promise<void> {
 }
 
 async function writeRobotsAndSitemap(models: Model[]): Promise<void> {
-  const robots = `# AI agents welcome. Machine-readable overview: ${SITE_URL}/llms.txt\nUser-agent: *\nAllow: /\nSitemap: ${SITE_URL}/sitemap.xml\n`;
-  await fs.writeFile(path.join(DIST_DIR, "robots.txt"), robots, "utf8");
+  await fs.writeFile(path.join(DIST_DIR, "robots.txt"), buildRobotsTxt(SITE_URL), "utf8");
 
   // Sitemaps list canonical, indexable HTML pages only — the JSON API and the
   // .txt agent files are intentionally excluded (they're not search results).
+  // /api is the HTML documentation page, so it belongs here.
   const today = new Date().toISOString().slice(0, 10);
   const dates = gitLastmodMap(REPO_ROOT);
   // Aggregate pages (home, glossary, providers, parameters) track the build date;
   // each model URL carries the commit date of its own YAML so lastmod stays honest.
   const entries: { path: string; priority: string; lastmod: string }[] = [
     { path: "/", priority: "1.0", lastmod: today },
     { path: GLOSSARY_PATH, priority: "0.7", lastmod: today },
+    { path: API_PATH, priority: "0.5", lastmod: today },
     ...uniqueProviders(models).map((provider) => ({
       path: providerPagePath(provider),
       priority: "0.8",

src/build/render-api.ts

@@ -2,7 +2,7 @@ import path from "node:path";
 import ejs from "ejs";
 import { VIEWS_DIR } from "../data/paths.js";
 import { SITE_NAME, SITE_URL } from "../data/site.js";
-import { absolute } from "../data/urls.js";
+import { API_PATH, absolute } from "../data/urls.js";
 import { type Model } from "../schema/model.js";
 import { hubLinks, renderShell, viewHelpers } from "./render.js";
 
@@ -19,7 +19,7 @@ export async function renderApiPage(allModels: Model[]): Promise<string> {
     {
       title: API_TITLE,
       description: API_DESCRIPTION,
-      canonicalUrl: absolute(SITE_URL, "/api"),
+      canonicalUrl: absolute(SITE_URL, API_PATH),
       structuredData: "{}",
       providerHubs: hubLinks(allModels),
     },

src/data/robots.ts

@@ -0,0 +1,26 @@
+// robots.txt builder. Kept as a pure function of the site origin so the crawl
+// policy is unit-testable instead of buried in the build script.
+
+/**
+ * Crawl policy for the site.
+ *
+ * The JSON API under /api/v1 is deliberately left crawlable: it is served with
+ * `X-Robots-Tag: noindex` (see vercel.json and src/server/app.ts), and a
+ * `Disallow` here would hide that header from crawlers — Google would keep the
+ * already-discovered endpoints in the index as bare URLs instead of dropping
+ * them. Search engines belong on the HTML pages; the JSON is for programmatic
+ * callers, which ignore robots directives anyway.
+ */
+export function buildRobotsTxt(siteUrl: string): string {
+  return [
+    `# AI agents welcome. Machine-readable overview: ${siteUrl}/llms.txt`,
+    "User-agent: *",
+    "Allow: /",
+    "",
+    "# /api/v1/*.json is crawlable on purpose so crawlers can read its noindex",
+    "# header. Do not Disallow it — that would strand those URLs in the index.",
+    "",
+    `Sitemap: ${siteUrl}/sitemap.xml`,
+    "",
+  ].join("\n");
+}

src/data/urls.ts

@@ -19,6 +19,9 @@ export function providerPagePath(provider: string): string {
 /** Parameter glossary page — the hub that links out to each parameter page. */
 export const GLOSSARY_PATH = "/glossary";
 
+/** API documentation page. The HTML docs, not the JSON endpoints under /api/v1. */
+export const API_PATH = "/api";
+
 /**
  * URL-safe slug for a parameter path: lowercased, with nested-path dots turned into
  * hyphens, e.g. `thinking.type` → `thinking-type`. Underscores are kept so that

src/server/app.ts

@@ -28,6 +28,15 @@ export function makeApp(loadModels: LoadModels): express.Express {
 
   app.use("/assets", express.static(DIST_ASSETS_DIR, { maxAge: 0 }));
 
+  // The JSON API is for programmatic callers, not search results: `noindex` keeps
+  // crawlers on the HTML pages while every endpoint stays fetchable. Scoped to
+  // /api/v1 so the HTML docs page at /api keeps its place in the index. Production
+  // is static, so vercel.json carries the same header — this is dev/test parity.
+  app.use("/api/v1", (_req, res, next) => {
+    res.setHeader("X-Robots-Tag", "noindex");
+    next();
+  });
+
   app.get("/", async (_req, res, next) => {
     try {
       const models = await loadModels();

src/views/model.ejs

@@ -110,7 +110,7 @@
       The full model definition as served by the API. Copy it or open the endpoint directly.
     </p>
     <div class="mt-3 flex items-center gap-3">
-      <a href="<%= jsonPath %>" target="_blank" rel="noopener noreferrer" class="text-sm text-accent hover:text-accent-hover"><%= jsonPath %></a>
+      <a href="<%= jsonPath %>" target="_blank" rel="noopener noreferrer nofollow" class="text-sm text-accent hover:text-accent-hover"><%= jsonPath %></a>
     </div>
     <div class="relative mt-4">
       <button type="button" data-copy-json class="absolute right-2 top-2 inline-flex items-center gap-1.5 border border-slate-200 bg-white px-2.5 py-1 text-xs font-medium text-slate-600 hover:bg-warm-hover hover:text-slate-900 dark:border-slate-700 dark:bg-slate-900 dark:text-slate-300 dark:hover:bg-slate-800 dark:hover:text-white">

src/views/partials/footer.ejs

@@ -21,8 +21,8 @@
       <div class="flex flex-col gap-2 text-sm">
         <a href="/api" class="text-slate-500 hover:text-slate-900 dark:text-slate-400 dark:hover:text-white">API</a>
         <a href="/glossary" class="text-slate-500 hover:text-slate-900 dark:text-slate-400 dark:hover:text-white">Glossary</a>
-        <a href="/api/v1/models.json" target="_blank" rel="noopener noreferrer" class="inline-flex items-center gap-1 text-slate-500 hover:text-slate-900 dark:text-slate-400 dark:hover:text-white">JSON API <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="currentColor" class="h-3 w-3 shrink-0" aria-hidden="true"><path d="M17 16V7H8v2h5.59l-6.3 6.29 1.42 1.42 6.29-6.3V16z"></path></svg></a>
-        <a href="/api/v1/schema.json" target="_blank" rel="noopener noreferrer" class="inline-flex items-center gap-1 text-slate-500 hover:text-slate-900 dark:text-slate-400 dark:hover:text-white">JSON Schema <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="currentColor" class="h-3 w-3 shrink-0" aria-hidden="true"><path d="M17 16V7H8v2h5.59l-6.3 6.29 1.42 1.42 6.29-6.3V16z"></path></svg></a>
+        <a href="/api/v1/models.json" target="_blank" rel="noopener noreferrer nofollow" class="inline-flex items-center gap-1 text-slate-500 hover:text-slate-900 dark:text-slate-400 dark:hover:text-white">JSON API <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="currentColor" class="h-3 w-3 shrink-0" aria-hidden="true"><path d="M17 16V7H8v2h5.59l-6.3 6.29 1.42 1.42 6.29-6.3V16z"></path></svg></a>
+        <a href="/api/v1/schema.json" target="_blank" rel="noopener noreferrer nofollow" class="inline-flex items-center gap-1 text-slate-500 hover:text-slate-900 dark:text-slate-400 dark:hover:text-white">JSON Schema <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="currentColor" class="h-3 w-3 shrink-0" aria-hidden="true"><path d="M17 16V7H8v2h5.59l-6.3 6.29 1.42 1.42 6.29-6.3V16z"></path></svg></a>
         <a href="/llms.txt" target="_blank" rel="noopener noreferrer" class="inline-flex items-center gap-1 text-slate-500 hover:text-slate-900 dark:text-slate-400 dark:hover:text-white">llms.txt <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="currentColor" class="h-3 w-3 shrink-0" aria-hidden="true"><path d="M17 16V7H8v2h5.59l-6.3 6.29 1.42 1.42 6.29-6.3V16z"></path></svg></a>
       </div>
 

src/views/partials/how_to_use.ejs

@@ -63,29 +63,29 @@
     <section class="space-y-3">
       <h3 class="text-sm font-semibold uppercase tracking-wider text-slate-500 dark:text-slate-400">Catalog API</h3>
       <p>The full catalog is static JSON, CORS-enabled, served from the edge.</p>
-      <pre class="overflow-x-auto rounded-lg bg-slate-100 px-3 py-2 font-mono text-xs text-slate-800 dark:bg-slate-800 dark:text-slate-200"><code>curl <a class="underline decoration-dotted underline-offset-2 hover:text-accent" href="/api/v1/models.json">https://modelparams.dev/api/v1/models.json</a></code></pre>
+      <pre class="overflow-x-auto rounded-lg bg-slate-100 px-3 py-2 font-mono text-xs text-slate-800 dark:bg-slate-800 dark:text-slate-200"><code>curl <a class="underline decoration-dotted underline-offset-2 hover:text-accent" href="/api/v1/models.json" rel="nofollow">https://modelparams.dev/api/v1/models.json</a></code></pre>
       <p>
         Each entry is keyed by <code class="rounded bg-slate-100 px-1.5 py-0.5 font-mono text-xs dark:bg-slate-800">provider/model</code> for API-key variants; subscription variants append <code class="rounded bg-slate-100 px-1.5 py-0.5 font-mono text-xs dark:bg-slate-800">-subscription</code>.
       </p>
       <p>
         If you only need the params for one model contract, use the providerless endpoint. Subscription contracts are model slugs with <code class="rounded bg-slate-100 px-1.5 py-0.5 font-mono text-xs dark:bg-slate-800">-subscription</code>.
       </p>
-      <pre class="overflow-x-auto rounded-lg bg-slate-100 px-3 py-2 font-mono text-xs text-slate-800 dark:bg-slate-800 dark:text-slate-200"><code>curl <a class="underline decoration-dotted underline-offset-2 hover:text-accent" href="/api/v1/params/gpt-5.5.json">https://modelparams.dev/api/v1/params/gpt-5.5.json</a>
-curl <a class="underline decoration-dotted underline-offset-2 hover:text-accent" href="/api/v1/params/gpt-5.5-subscription.json">https://modelparams.dev/api/v1/params/gpt-5.5-subscription.json</a></code></pre>
+      <pre class="overflow-x-auto rounded-lg bg-slate-100 px-3 py-2 font-mono text-xs text-slate-800 dark:bg-slate-800 dark:text-slate-200"><code>curl <a class="underline decoration-dotted underline-offset-2 hover:text-accent" href="/api/v1/params/gpt-5.5.json" rel="nofollow">https://modelparams.dev/api/v1/params/gpt-5.5.json</a>
+curl <a class="underline decoration-dotted underline-offset-2 hover:text-accent" href="/api/v1/params/gpt-5.5-subscription.json" rel="nofollow">https://modelparams.dev/api/v1/params/gpt-5.5-subscription.json</a></code></pre>
     </section>
 
     <section class="space-y-3">
       <h3 class="text-sm font-semibold uppercase tracking-wider text-slate-500 dark:text-slate-400">Single model</h3>
-      <pre class="overflow-x-auto rounded-lg bg-slate-100 px-3 py-2 font-mono text-xs text-slate-800 dark:bg-slate-800 dark:text-slate-200"><code>curl <a class="underline decoration-dotted underline-offset-2 hover:text-accent" href="/api/v1/models/anthropic/claude-opus-4-7.json">https://modelparams.dev/api/v1/models/anthropic/claude-opus-4-7.json</a>
-curl <a class="underline decoration-dotted underline-offset-2 hover:text-accent" href="/api/v1/models/anthropic/claude-opus-4-7-subscription.json">https://modelparams.dev/api/v1/models/anthropic/claude-opus-4-7-subscription.json</a></code></pre>
+      <pre class="overflow-x-auto rounded-lg bg-slate-100 px-3 py-2 font-mono text-xs text-slate-800 dark:bg-slate-800 dark:text-slate-200"><code>curl <a class="underline decoration-dotted underline-offset-2 hover:text-accent" href="/api/v1/models/anthropic/claude-opus-4-7.json" rel="nofollow">https://modelparams.dev/api/v1/models/anthropic/claude-opus-4-7.json</a>
+curl <a class="underline decoration-dotted underline-offset-2 hover:text-accent" href="/api/v1/models/anthropic/claude-opus-4-7-subscription.json" rel="nofollow">https://modelparams.dev/api/v1/models/anthropic/claude-opus-4-7-subscription.json</a></code></pre>
     </section>
 
     <section class="space-y-3">
       <h3 class="text-sm font-semibold uppercase tracking-wider text-slate-500 dark:text-slate-400">JSON Schema</h3>
       <p>
         Every entry validates against a JSON Schema you can use in your editor or pipeline.
       </p>
-      <pre class="overflow-x-auto rounded-lg bg-slate-100 px-3 py-2 font-mono text-xs text-slate-800 dark:bg-slate-800 dark:text-slate-200"><code>curl <a class="underline decoration-dotted underline-offset-2 hover:text-accent" href="/api/v1/schema.json">https://modelparams.dev/api/v1/schema.json</a></code></pre>
+      <pre class="overflow-x-auto rounded-lg bg-slate-100 px-3 py-2 font-mono text-xs text-slate-800 dark:bg-slate-800 dark:text-slate-200"><code>curl <a class="underline decoration-dotted underline-offset-2 hover:text-accent" href="/api/v1/schema.json" rel="nofollow">https://modelparams.dev/api/v1/schema.json</a></code></pre>
       <p>
         Add this header to any YAML you author for autocomplete in VS Code:
       </p>

tests/robots.test.ts

@@ -0,0 +1,47 @@
+import fs from "node:fs";
+import path from "node:path";
+import { describe, expect, it } from "vitest";
+import { REPO_ROOT } from "../src/data/paths.js";
+import { buildRobotsTxt } from "../src/data/robots.js";
+
+const SITE = "https://modelparams.dev";
+
+describe("buildRobotsTxt", () => {
+  const robots = buildRobotsTxt(SITE);
+
+  it("allows crawling and points at the sitemap and llms.txt", () => {
+    expect(robots).toContain("User-agent: *");
+    expect(robots).toContain("Allow: /");
+    expect(robots).toContain(`Sitemap: ${SITE}/sitemap.xml`);
+    expect(robots).toContain(`${SITE}/llms.txt`);
+  });
+
+  it("never disallows the JSON API, which would hide its noindex header", () => {
+    expect(robots).not.toMatch(/^Disallow:\s*\/api/m);
+  });
+
+  it("ends with a trailing newline", () => {
+    expect(robots.endsWith("\n")).toBe(true);
+  });
+});
+
+describe("vercel.json crawl headers", () => {
+  const config = JSON.parse(fs.readFileSync(path.join(REPO_ROOT, "vercel.json"), "utf8")) as {
+    headers: { source: string; headers: { key: string; value: string }[] }[];
+  };
+
+  // Production serves dist/ statically, so vercel.json — not the Express app — is
+  // what actually keeps the JSON API out of the search index.
+  it("serves /api/v1 with X-Robots-Tag: noindex", () => {
+    const rule = config.headers.find((entry) => entry.source === "/api/v1/(.*)");
+    expect(rule).toBeTruthy();
+    expect(rule!.headers).toContainEqual({ key: "X-Robots-Tag", value: "noindex" });
+  });
+
+  it("does not noindex anything outside /api/v1", () => {
+    const noindexed = config.headers
+      .filter((entry) => entry.headers.some((header) => header.key === "X-Robots-Tag"))
+      .map((entry) => entry.source);
+    expect(noindexed).toEqual(["/api/v1/(.*)"]);
+  });
+});

tests/server.test.ts

@@ -233,6 +233,40 @@ describe("GET /models/:provider/:slug", () => {
   });
 });
 
+describe("crawl policy", () => {
+  it.each([
+    "/api/v1/models.json",
+    "/api/v1/schema.json",
+    "/api/v1/params/claude-opus-4-7.json",
+    "/api/v1/models/anthropic/claude-opus-4-7.json",
+  ])("noindexes %s so it stays out of search results", async (path) => {
+    const res = await get(path);
+    expect(res.headers.get("x-robots-tag")).toBe("noindex");
+  });
+
+  it("noindexes JSON 404s too, so bad endpoints cannot get indexed either", async () => {
+    const res = await get("/api/v1/models/anthropic/does-not-exist.json");
+    expect(res.status).toBe(404);
+    expect(res.headers.get("x-robots-tag")).toBe("noindex");
+  });
+
+  it.each(["/", "/api", "/glossary", "/parameters/temperature", "/providers/anthropic"])(
+    "leaves %s indexable",
+    async (path) => {
+      const res = await get(path);
+      expect(res.status).toBe(200);
+      expect(res.headers.get("x-robots-tag")).toBeNull();
+    },
+  );
+
+  it("does not hand crawlers a followable link to a model's JSON", async () => {
+    const body = await get("/models/anthropic/claude-opus-4-7").then((r) => r.text());
+    expect(body).toContain(
+      '<a href="/api/v1/models/anthropic/claude-opus-4-7.json" target="_blank" rel="noopener noreferrer nofollow"',
+    );
+  });
+});
+
 describe("llms.txt feeds", () => {
   it("serves llms.txt as plain text", async () => {
     const res = await get("/llms.txt");

vercel.json

@@ -14,6 +14,10 @@
         { "key": "Cache-Control", "value": "public, max-age=300, s-maxage=3600" }
       ]
     },
+    {
+      "source": "/api/v1/(.*)",
+      "headers": [{ "key": "X-Robots-Tag", "value": "noindex" }]
+    },
     {
       "source": "/(llms.txt|llms-full.txt)",
       "headers": [