Docs search: index page content, not just titles

apps/website/e2e/docs.spec.ts

@@ -159,6 +159,71 @@ test.describe('Docs slug page', () => {
     await page.goto('/docs/langgraph/getting-started/introduction');
     await expect(page.locator('nav[aria-label="Breadcrumb"]')).toHaveCount(1);
   });
+
+  test('finds a term that appears only in body prose and lands on its section', async ({ page }) => {
+    // Desktop breakpoint: the "Search docs" control-plane button is directly
+    // visible without opening the mobile navigation dialog first (see
+    // workspace-shell.spec.ts's desktop-rail tests).
+    await page.setViewportSize({ width: 1440, height: 900 });
+    await page.goto(route);
+
+    // Open by clicking the trigger, matching workspace-shell.spec.ts. A
+    // keyboard shortcut would depend on how the runner maps Meta, and the
+    // button is the affordance a real user has anyway. The control may also
+    // exist in the mobile drawer's DOM, so scope to the first match.
+    await page.getByRole('button', { name: 'Search docs' }).first().click();
+
+    const dialog = page.getByRole('dialog', { name: 'Search documentation' });
+    await expect(dialog).toBeVisible();
+
+    // "checkpointer" is prose inside the persistence guide (7 occurrences
+    // outside fenced code) and is in no page title anywhere in
+    // docs-config.ts, so a title-only search returns nothing for it. This
+    // only passes if server-side content search is genuinely wired up.
+    await dialog
+      .getByRole('combobox', { name: 'Search documentation...' })
+      .fill('checkpointer');
+
+    const hit = dialog.getByRole('option').filter({ hasText: /checkpointer/i }).first();
+    await expect(hit).toBeVisible({ timeout: 10000 });
+
+    // Prove the match came from server-side content search, not the instant
+    // client-side title matcher: it must sit under the "In page content"
+    // group, and its own row must show a snippet containing the term (a
+    // future regression that merely surfaced a title match must not keep
+    // this green).
+    await expect(dialog.getByText('In page content')).toBeVisible();
+    await expect(hit.locator('.docs-search-result-snippet')).toContainText(/checkpointer/i);
+
+    await hit.click();
+
+    // The deep link must land on a real section anchor, not just a
+    // well-formed but dangling fragment that scrolls nowhere.
+    await expect(page).toHaveURL(/\/docs\/langgraph\/.*#.+/);
+    const url = new URL(page.url());
+    expect(url.hash.length).toBeGreaterThan(1);
+    await expect(page.locator(url.hash)).toBeVisible();
+  });
+
+  test('search shows the empty state for a term that appears nowhere in the docs', async ({ page }) => {
+    // Negative control: without this, a matcher that returns everything for
+    // everything would make the content-hit test above pass by accident.
+    await page.setViewportSize({ width: 1440, height: 900 });
+    await page.goto(route);
+
+    await page.getByRole('button', { name: 'Search docs' }).first().click();
+
+    const dialog = page.getByRole('dialog', { name: 'Search documentation' });
+    await expect(dialog).toBeVisible();
+
+    // Confirmed absent from the whole docs content/config tree via grep.
+    await dialog
+      .getByRole('combobox', { name: 'Search documentation...' })
+      .fill('zqxvantibrackle');
+
+    await expect(dialog.getByText('No results found')).toBeVisible({ timeout: 10000 });
+    await expect(dialog.getByRole('option')).toHaveCount(0);
+  });
 });
 
 test.describe('a2ui docs', () => {

apps/website/next.config.ts

@@ -17,6 +17,11 @@ export const nextConfig: WithNxOptions = {
       '../../cockpit/**/*.ts',
       '../../deployments/ag-ui-mastra/*.mjs',
       '../../nx.json',
+      // The docs search route reads these at request time. Unlike
+      // api/markdown it cannot be statically generated, so without this the
+      // route deploys with no corpus and returns empty for every query —
+      // silently, and only in production.
+      'content/docs/**/*.mdx',
     ],
   },
   skipTrailingSlashRedirect: true,

apps/website/src/app/api/docs-search/route.spec.ts

@@ -0,0 +1,84 @@
+// @vitest-environment node
+import { describe, expect, it } from 'vitest';
+import { buildIndex, GET } from './route';
+
+const call = (q: string) =>
+  GET(new Request(`http://localhost/api/docs-search?q=${encodeURIComponent(q)}`));
+
+describe('GET /api/docs-search', () => {
+  it('finds a page by a term that appears only in its body prose', async () => {
+    // "checkpointer" is prose in the LangGraph persistence guide, and is in no
+    // page title — exactly the query the old title-only search could not serve.
+    const res = await call('checkpointer');
+    expect(res.status).toBe(200);
+    const { results } = await res.json();
+    expect(results.length).toBeGreaterThan(0);
+    expect(results.some((r: { href: string }) => r.href.includes('/docs/langgraph/'))).toBe(true);
+  });
+
+  it('returns hits that carry a snippet and marks', async () => {
+    const { results } = await (await call('checkpointer')).json();
+    expect(typeof results[0].snippet).toBe('string');
+    expect(Array.isArray(results[0].marks)).toBe(true);
+  });
+
+  it('returns a hit shaped exactly as the wire contract, no extra fields', async () => {
+    const { results } = await (await call('checkpointer')).json();
+    expect(Object.keys(results[0]).sort()).toEqual(
+      ['href', 'title', 'heading', 'libraryTitle', 'snippet', 'marks'].sort()
+    );
+  });
+
+  it('returns empty without scanning for a query under two characters', async () => {
+    const { results } = await (await call('a')).json();
+    expect(results).toEqual([]);
+  });
+
+  it('searches at exactly the two-character boundary rather than short-circuiting', async () => {
+    // "ag" is a real substring in this corpus (ag-ui) — proves the >= 2 path
+    // actually reaches searchIndexedDocs instead of also being swallowed by
+    // the short-query guard.
+    const { results } = await (await call('ag')).json();
+    expect(results.length).toBeGreaterThan(0);
+  });
+
+  it('returns empty for a missing query parameter', async () => {
+    const res = await GET(new Request('http://localhost/api/docs-search'));
+    const { results } = await res.json();
+    expect(results).toEqual([]);
+  });
+
+  it('treats a whitespace-only query the same as an empty one', async () => {
+    const { results } = await (await call('   ')).json();
+    expect(results).toEqual([]);
+  });
+
+  it('is cacheable, because the corpus only changes on deploy', async () => {
+    const res = await call('streaming');
+    expect(res.headers.get('cache-control')).toContain('max-age=');
+  });
+
+  it('sets Cache-Control on the empty-result response too, not only populated ones', async () => {
+    const res = await call('a');
+    expect(res.headers.get('cache-control')).toContain('max-age=');
+  });
+});
+
+describe('buildIndex', () => {
+  it('skips a single document that throws without breaking the rest of the corpus', () => {
+    const entries = [
+      { library: 'lib', section: 'guides', slug: 'good' },
+      { library: 'lib', section: 'guides', slug: 'bad' },
+    ];
+
+    const result = buildIndex(entries, (entry) => {
+      if (entry.slug === 'bad') {
+        throw new Error('simulated malformed doc');
+      }
+      return { title: 'Good Doc', body: '# Good Doc\n\nSome findable prose here.' };
+    });
+
+    expect(result).toHaveLength(1);
+    expect(result[0].slug).toBe('good');
+  });
+});

apps/website/src/app/api/docs-search/route.ts

@@ -0,0 +1,105 @@
+import { NextResponse } from 'next/server';
+import { getAllDocSlugs, getDocBySlug } from '../../../lib/docs';
+import { getLibraryConfig } from '../../../lib/docs-config';
+import { indexDocSections } from '../../../lib/docs-search-index';
+import { searchIndexedDocs, type IndexedDoc } from '../../../lib/docs-search-query';
+
+const MIN_QUERY_LENGTH = 2;
+
+export interface DocSlugEntry {
+  library: string;
+  section: string;
+  slug: string;
+}
+
+/** The slice of `ResolvedDoc` the indexer actually needs. */
+interface ResolvableDoc {
+  title: string;
+  body: string;
+}
+
+function resolveDoc(entry: DocSlugEntry): ResolvableDoc | null {
+  return getDocBySlug(entry.library, entry.section, entry.slug);
+}
+
+/**
+ * Build the search index from a list of doc slugs.
+ *
+ * `resolveDoc` is a parameter — not just a closure over `lib/docs` — so a
+ * test can inject a resolver that throws for one entry without touching the
+ * filesystem, proving the per-document guard below actually guards.
+ *
+ * One malformed doc must not take down search for the whole instance: the
+ * index is built once at module scope (see `getIndex`), so an uncaught throw
+ * here would fail every request that instance ever serves, not just the one
+ * request that happened to trigger the (re)build.
+ */
+export function buildIndex(
+  entries: DocSlugEntry[],
+  resolve: (entry: DocSlugEntry) => ResolvableDoc | null = resolveDoc
+): IndexedDoc[] {
+  return entries.flatMap((entry) => {
+    try {
+      const doc = resolve(entry);
+      if (!doc) return [];
+      return [
+        {
+          library: entry.library,
+          libraryTitle: getLibraryConfig(entry.library)?.title ?? entry.library,
+          section: entry.section,
+          slug: entry.slug,
+          title: doc.title,
+          sections: indexDocSections(doc.body),
+        },
+      ];
+    } catch {
+      return [];
+    }
+  });
+}
+
+/**
+ * Built once per instance, not per request.
+ *
+ * This reads MDX from disk at request time, which is why
+ * `content/docs/**` has to be traced into the deployed function — see
+ * `outputFileTracingIncludes` in next.config.ts. The route cannot be
+ * statically generated the way `api/markdown` is, because the query space is
+ * unbounded.
+ */
+let index: IndexedDoc[] | null = null;
+
+function getIndex(): IndexedDoc[] {
+  if (!index) {
+    index = buildIndex(getAllDocSlugs());
+  }
+  return index;
+}
+
+export async function GET(request: Request): Promise<Response> {
+  const query = new URL(request.url).searchParams.get('q')?.trim() ?? '';
+
+  if (query.length < MIN_QUERY_LENGTH) {
+    return NextResponse.json(
+      { results: [] },
+      {
+        headers: {
+          // Same cacheability as the populated response — a short/empty
+          // query is just as deterministic a function of the corpus.
+          'Cache-Control': 'public, max-age=300',
+        },
+      }
+    );
+  }
+
+  return NextResponse.json(
+    { results: searchIndexedDocs(getIndex(), query) },
+    {
+      headers: {
+        // The corpus only changes on deploy, so repeated queries are served
+        // by the CDN rather than waking this function.
+        'Cache-Control': 'public, max-age=300',
+      },
+    }
+  );
+}