From 9d61da58ea548fe2fd367ce10736520f7a9628f0 Mon Sep 17 00:00:00 2001 From: Kieron Lanning Date: Mon, 28 Sep 2026 11:26:44 +0100 Subject: [PATCH] feat: added llms per-project --- README.md | 13 +++- package.json | 2 +- src/astro.config.ts | 16 ++++- src/scripts/check-generated.ts | 30 ++++++++++ src/src/components/SiteFooter.astro | 17 +++++- src/src/components/SiteNav.astro | 58 +++++++++++++++--- src/src/components/starlight/PageTitle.astro | 26 ++++++++ src/src/lib/llms.ts | 13 ++++ src/src/pages/docs/index.astro | 31 ++++++++++ src/src/pages/projects/[project].astro | 16 +++++ src/tests/dist/built-output.test.ts | 62 ++++++++++++++++++++ 11 files changed, 268 insertions(+), 16 deletions(-) create mode 100644 src/src/lib/llms.ts diff --git a/README.md b/README.md index 99e933f..eee4ea8 100644 --- a/README.md +++ b/README.md @@ -282,8 +282,17 @@ changesets automation publishes `vX.Y.Z-prerelease.N` tags with the GitHub The build generates `/llms.txt`, `/llms-small.txt`, and `/llms-full.txt` (Starlight's `starlight-llms-txt` plugin) using canonical production URLs and -the aggregated documentation. The built outputs are asserted by `tests/dist/` -and `just check-generated`. +the aggregated documentation. + +Each documented project is also emitted as its own scoped bundle at +`/_llms-txt/.txt` via the plugin's `customSets` option. The project +page, the project's documentation overview page, and the documentation portal +all link to that path, and `/llms.txt` lists every bundle under +`Documentation Sets`. Because `rawContent` is enabled, each bundle is the raw +aggregated Markdown for that project (the same content that feeds +`llms-full.txt`, scoped down). + +The built outputs are asserted by `tests/dist/` and `just check-generated`. ## Site versioning and releases diff --git a/package.json b/package.json index 83ebe09..aaf4ded 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "purview-dev", - "version": "0.3.3", + "version": "0.3.4", "private": true, "description": "Purview-Dev public website and unified documentation portal (workspace root).", "license": "MIT", diff --git a/src/astro.config.ts b/src/astro.config.ts index 6cb020f..89c418c 100644 --- a/src/astro.config.ts +++ b/src/astro.config.ts @@ -19,6 +19,19 @@ const projects = loadProjects(); const docsManifest = readDocsManifest(); const docsProjects = projects.filter((project) => project.docs && project.status !== 'archived'); +/** + * Per-project LLM bundles. `starlight-llms-txt` emits one file per `customSets` + * entry at `/_llms-txt/.txt`, where the slug is `github-slugger`'s slug of + * the set label. Every documented project's display name slugifies to exactly + * its project id, so `docs//**` maps to `/_llms-txt/.txt` + * and the UI links straight to that path. The build's link crawl (and the dist + * tests) fail if a future project name breaks that correspondence. + */ +const docsSets = docsProjects.map((project) => ({ + label: project.name, + description: project.shortDescription, + paths: [`docs/${project.id}/**`], +})); const previewBadge = { text: 'Preview', variant: 'caution' } as const; const sidebarTopics = [ { label: 'Documentation home', link: '/docs/' }, @@ -180,9 +193,10 @@ export default defineConfig({ description: 'GitHub and NuGet release information.', }, ], - promote: ['index*'], + promote: ['index*', 'docs/*/index'], demote: [], exclude: ['dotnet-logging-source-generators/**'], + customSets: docsSets, minify: { note: true, tip: true, diff --git a/src/scripts/check-generated.ts b/src/scripts/check-generated.ts index ae6e1e8..70bd008 100644 --- a/src/scripts/check-generated.ts +++ b/src/scripts/check-generated.ts @@ -8,6 +8,7 @@ import { DOCS_OUTPUT_DIR, readDocsManifest, } from '../src/lib/docs/aggregate'; +import { loadProjects } from '../src/lib/manifest/load'; import { isReleaseCache, readReleaseCache } from '../src/lib/releases/cache'; const DIST = resolve('dist'); @@ -124,6 +125,33 @@ function requireDistFile(file: string): void { } } +/** + * Per-project `llms.txt` bundles are emitted by `starlight-llms-txt`'s + * `customSets` option, one file per documented project at + * `/_llms-txt/.txt`. The project pages, the project documentation + * overview, and the documentation portal all link to these paths, so a missing + * bundle is a broken link — and the link crawl only sees the pages that link it. + */ +function validateProjectLlmsBundles(): void { + const projects = loadProjects().filter( + (project) => project.docs && project.status !== 'archived', + ); + for (const project of projects) { + requireDistFile(`_llms-txt/${project.id}.txt`); + } + + const entrypointPath = resolve(DIST, 'llms.txt'); + if (!existsSync(entrypointPath)) { + return; + } + const entrypoint = readFileSync(entrypointPath, 'utf8'); + for (const project of projects) { + if (!entrypoint.includes(`/_llms-txt/${project.id}.txt`)) { + fail(`llms.txt does not link the per-project bundle for "${project.id}".`); + } + } +} + async function scanForSecrets(): Promise { const files = await glob('**/*', { cwd: DIST, onlyFiles: true }); for (const file of files) { @@ -166,6 +194,8 @@ async function run(): Promise { requireDistFile('sitemap-index.xml'); requireDistFile('robots.txt'); + validateProjectLlmsBundles(); + const llmsFull = readFileSync(resolve(DIST, 'llms-full.txt'), 'utf8'); const llms = readFileSync(resolve(DIST, 'llms.txt'), 'utf8'); if ( diff --git a/src/src/components/SiteFooter.astro b/src/src/components/SiteFooter.astro index e8f89e6..e93809f 100644 --- a/src/src/components/SiteFooter.astro +++ b/src/src/components/SiteFooter.astro @@ -1,6 +1,7 @@ --- import SiteBrand from '~/components/SiteBrand.astro'; import { NAV_ITEMS } from '~/components/SiteNav.astro'; +import { LLMS_LINK_ATTRS } from '~/lib/llms'; import { loadProjects } from '~/lib/manifest/load'; import { getReleaseInfo } from '~/lib/site-version'; import { OWNER, SITE } from '~/lib/site'; @@ -76,9 +77,19 @@ const releasesUrl = `${SITE.githubUrl}/${OWNER}.github.io/releases`; diff --git a/src/src/components/SiteNav.astro b/src/src/components/SiteNav.astro index c3297d4..b49d89d 100644 --- a/src/src/components/SiteNav.astro +++ b/src/src/components/SiteNav.astro @@ -1,12 +1,24 @@ --- import { withBase } from '~/lib/urls'; -export const NAV_ITEMS = [ +type NavItem = { + label: string; + href: string; + /** + * Optional icon for the item. In the horizontal nav an icon item renders as a + * compact icon-only button; the vertical (mobile) menu keeps the label so the + * list does not leave a lone glyph dangling next to the text items. + */ + icon?: 'home'; +}; + +export const NAV_ITEMS: readonly NavItem[] = [ + { label: 'Home', href: '/', icon: 'home' }, { label: 'Projects', href: '/projects/' }, { label: 'Documentation', href: '/docs/' }, { label: 'Releases', href: '/releases/' }, { label: 'About', href: '/about/' }, -] as const; +]; interface Props { currentPath: string; @@ -25,30 +37,58 @@ function isActive(href: string): boolean { ---
    { NAV_ITEMS.map((item) => { const active = isActive(withBase(item.href)); + // Icon items collapse to an icon-only button in the horizontal nav, using + // tighter horizontal padding than the text items. The visible label moves + // into `aria-label`/`title` so the control stays accessible. + const iconOnly = item.icon !== undefined && !vertical; return (
  • - {item.label} + {item.icon === 'home' && ( + + )} + {!iconOnly && {item.label}}
    • ); }) } -
\ No newline at end of file + diff --git a/src/src/components/starlight/PageTitle.astro b/src/src/components/starlight/PageTitle.astro index a6d04fd..16b74f4 100644 --- a/src/src/components/starlight/PageTitle.astro +++ b/src/src/components/starlight/PageTitle.astro @@ -1,6 +1,7 @@ --- import StatusBadge from '~/components/StatusBadge.astro'; import { isStale, STALE_AFTER_DAYS } from '~/lib/docs/staleness'; +import { LLMS_LINK_ATTRS } from '~/lib/llms'; import { withBase } from '~/lib/urls'; const { entry } = Astro.locals.starlightRoute; @@ -8,6 +9,17 @@ const data = entry?.data; const lastReviewed = typeof data?.lastReviewed === 'string' ? data.lastReviewed : undefined; const stale = lastReviewed ? isStale(lastReviewed) : false; const tags = Array.isArray(data?.tags) ? data.tags : []; +// Project overview pages surface the per-project `llms.txt` bundle emitted by +// the `customSets` option in `astro.config.ts`. Starlight resolves an index +// page to its directory id (`docs/`), while the content collection — +// and therefore the plugin's bundle paths — use `docs//index`, so both +// forms are accepted here. Other aggregated pages keep the heading lean and only +// link the global files from the site footer. +const entryId = typeof entry?.id === 'string' ? entry.id : undefined; +const isProjectOverview = entryId !== undefined && /^docs\/[^/]+(\/index)?$/.test(entryId); +const sourceProject = typeof data?.sourceProject === 'string' ? data.sourceProject : undefined; +const llmsHref = + isProjectOverview && sourceProject ? withBase(`/_llms-txt/${sourceProject}.txt`) : null; ---

{data?.title}

@@ -66,6 +78,20 @@ const tags = Array.isArray(data?.tags) ? data.tags : []; ) } + { + llmsHref && ( + + + llms.txt + + + ) + } ) } diff --git a/src/src/lib/llms.ts b/src/src/lib/llms.ts new file mode 100644 index 0000000..1883ddb --- /dev/null +++ b/src/src/lib/llms.ts @@ -0,0 +1,13 @@ +/** + * Link attributes for the generated LLM text bundles — `llms.txt`, + * `llms-small.txt`, `llms-full.txt`, and the per-project + * `/_llms-txt/.txt` sets. + * + * These are plain-text files rather than site pages, so every link to one is + * treated like an external link and opens in a new tab. Spreading the object + * keeps the call sites short enough to stay on one line. + */ +export const LLMS_LINK_ATTRS = { + rel: 'noopener noreferrer', + target: '_blank', +} as const; diff --git a/src/src/pages/docs/index.astro b/src/src/pages/docs/index.astro index 6160a0d..b475111 100644 --- a/src/src/pages/docs/index.astro +++ b/src/src/pages/docs/index.astro @@ -1,5 +1,6 @@ --- import SiteLayout from '~/layouts/SiteLayout.astro'; +import { LLMS_LINK_ATTRS } from '~/lib/llms'; import { loadProjects } from '~/lib/manifest/load'; import { readDocsManifest } from '~/lib/docs/aggregate'; import { projectRepoEnrichment } from '~/lib/releases/repo'; @@ -76,5 +77,35 @@ const pageCount = docsManifest?.projects.reduce( + +
+

Machine-readable documentation

+

+ Each project's documentation is also published as a single llms.txt bundle for + language models and other tooling. The whole site is available as + llms.txt, + + llms-small.txt + , and + + llms-full.txt + . +

+ +
diff --git a/src/src/pages/projects/[project].astro b/src/src/pages/projects/[project].astro index d054ef7..66385d4 100644 --- a/src/src/pages/projects/[project].astro +++ b/src/src/pages/projects/[project].astro @@ -5,6 +5,7 @@ import PackageVersions, { type PackageRow } from '~/components/islands/PackageVe import StatusBadge from '~/components/StatusBadge.astro'; import { buildBadge, nugetDownloadsBadge, nugetVersionBadge, actionsUrl } from '~/lib/badges'; import { installAnchorId, installKindFor, installTabs } from '~/lib/install'; +import { LLMS_LINK_ATTRS } from '~/lib/llms'; import { loadProjects } from '~/lib/manifest/load'; import { projectRepoEnrichment } from '~/lib/releases/repo'; import { projectReleaseSummary } from '~/lib/releases/transform'; @@ -44,6 +45,9 @@ const supersededBy = : undefined; const primaryPackage = project.packages.find((pkg) => pkg.primary) ?? project.packages[0]; const docsPath = project.docs ? withBase(`/docs/${project.id}/`) : null; +// Per-project machine-readable bundle emitted by `starlight-llms-txt`'s +// `customSets` option (one file per documented project at /_llms-txt/.txt). +const llmsPath = project.docs ? withBase(`/_llms-txt/${project.id}.txt`) : null; const targetFrameworks = project.targetFrameworks ?? []; const installSections = project.status !== 'archived' ? @@ -137,6 +141,18 @@ const packageRows: PackageRow[] = releases.packages.map((pkg) => ({ ) } + { + llmsPath && ( + + llms.txt + + ) + } { }); }); +describe('built per-project llms outputs', () => { + const docsProjects = loadProjects().filter( + (project) => project.docs && project.status !== 'archived', + ); + + // The plugin derives each bundle's slug from its `customSets` label + // (`github-slugger` of the project name). Asserting the file exists under the + // project id fails the build if a future display name stops slugifying to its + // id, which is the invariant the UI links rely on. + test('a scoped bundle is built for every documented project', () => { + expect(docsProjects.length).toBeGreaterThan(0); + for (const project of docsProjects) { + const content = requireBuilt(`_llms-txt/${project.id}.txt`); + expect(content.trim().length, `${project.id} bundle is empty`).toBeGreaterThan(0); + expect(content, `${project.id} bundle lacks aggregated content`).toContain('Purview'); + } + }); + + test('the llms.txt entrypoint links every scoped bundle', () => { + const content = requireBuilt('llms.txt'); + for (const project of docsProjects) { + expect(content, `llms.txt does not link ${project.id}`).toContain( + `https://purview.dev/_llms-txt/${project.id}.txt`, + ); + } + }); + + test('scoped bundles exclude secrets and local build paths', () => { + for (const project of docsProjects) { + const content = requireBuilt(`_llms-txt/${project.id}.txt`); + expect(content).not.toMatch(/\bghp_[A-Za-z0-9]{36,}\b/); + expect(content).not.toMatch(/[A-Za-z]:\\/); + expect(content).not.toMatch(/node_modules[/\\]/); + } + }); +}); +describe('built llms links', () => { + // The LLM text bundles are plain files rather than site pages, so every link + // to one must opt into the external-link treatment. + test('every llms text link opens in a new tab like an external link', () => { + const { glob } = require('fast-glob'); + const files = glob.sync('**/*.html', { cwd: DIST }); + const anchor = + /]*href="[^"]*(?:llms(?:-small|-full)?\.txt|_llms-txt\/[^"]+\.txt)"[^>]*>/g; + + let checked = 0; + for (const file of files) { + const content = readFileSync(resolve(DIST, file), 'utf8'); + for (const tag of content.match(anchor) ?? []) { + checked += 1; + expect(tag, `${file}: ${tag}`).toContain('target="_blank"'); + expect(tag, `${file}: ${tag}`).toContain('rel="noopener noreferrer"'); + } + } + + // Guard against the assertions silently passing if the regex stops matching. + expect(checked).toBeGreaterThan(0); + }); +}); + describe('built SEO outputs', () => { test('sitemap index exists and references the canonical site', () => { const content = requireBuilt('sitemap-index.xml');