From c69c07ac35d2bcfb38eaffc6a1ba8c2acedaa44e Mon Sep 17 00:00:00 2001 From: Oto Macenauer Date: Fri, 14 Aug 2026 17:01:09 +0200 Subject: [PATCH] perf: self-host Inter and stop copying what can be linked MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Inter came from fonts.googleapis.com on every page view, sending each reader's IP and User-Agent to a third party and leaving the deployment with one external dependency it otherwise does not have — mermaid is already vendored for exactly this reason. It is now served from this origin via @fontsource-variable/inter, declared in marketplace.css. Only the two latin subsets, upright, are declared, which is what the Google request asked for; importing the package's own CSS would emit the cyrillic, greek and vietnamese faces into dist/ as well. That lets the CSP drop both third-party hosts: style-src and font-src are now 'self' (plus data: and the existing 'unsafe-inline'). nginx.headers.conf and the Express mirror move together, as they must. Alongside it, the build-time costs from #51: - Sub-app assets are hardlinked from apps/ into public/ instead of copied, falling back to a copy when the filesystem will not link. CSS is deliberately still copied: the url() rewrite edits the destination in place and a link would write that back into the artifact. - getStaticPaths no longer attaches the whole resolved registry to every route. The masthead names one app and the catchall titles the page with it, so each page carries only that app's card. - collectHtmlFiles and copyAssets read directory entries with withFileTypes instead of a statSync per file, which also means a symlink is seen as one rather than followed. - loadRegistry memoises on the mtimes of apps.json and .single-page.json, so a repeated getStaticPaths costs two statSync calls rather than two reads and parses, and astro dev still notices an edit. Two items in that issue were already fixed: copyDir moved to scripts/artifacts.js with withFileTypes when it was hardened, and fetch-apps.js no longer dynamically imports copyFileSync. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QqFK6yffibtCBTF8xZ4hXW --- CLAUDE.md | 2 +- README.md | 4 +++ nginx.headers.conf | 8 +++-- package-lock.json | 10 ++++++ package.json | 1 + scripts/build-vite.js | 67 +++++++++++++++++++++++++---------- src/components/Masthead.astro | 9 ++--- src/layouts/Base.astro | 7 ++-- src/pages/[...path].astro | 5 ++- src/pages/index.astro | 2 +- src/styles/marketplace.css | 34 +++++++++++++++++- src/utils/apps.js | 58 ++++++++++++++++++++++++------ tests/build-integrity.spec.js | 45 +++++++++++++++++------ tests/fragment-server.mjs | 4 +-- tests/nginx-config.spec.js | 11 ++++++ 15 files changed, 210 insertions(+), 57 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index a0abb52..fc8b5de 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -72,7 +72,7 @@ Orchestrator: `scripts/build-vite.js`. Flags: `--local`, `--headless`. - `src/pages/index.astro` — Landing catalog page - `src/utils/apps.js` — `getAppPages()` enumerates sub-app HTML (manifest-driven or filesystem crawl) - `src/utils/transform.js` — `transformSubAppHtml()`: URL rewriting, document splitting (head/body/title/body-class), headless transforms -- `src/layouts/Base.astro` — The one document shell: head, fonts, marketplace CSS, ``, shadow-DOM compat styles +- `src/layouts/Base.astro` — The one document shell: head, marketplace CSS (which carries the self-hosted Inter faces), ``, shadow-DOM compat styles - `src/components/Masthead.astro` — Persistent Knowledge base header + Library/current-app sub-nav (all pages, both modes) - `src/components/AppCard.astro`, `src/components/AppIcon.astro` — Catalog card and its icon - `src/templates/shadow-compat.js` — Shadow-DOM design-token styles, injected into the body by the layout diff --git a/README.md b/README.md index 269b6c0..7270247 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,10 @@ build time it: Each app keeps its own sidebar, routing, and internal navigation. +The output makes **no third-party requests at runtime**: the Inter typeface is +self-hosted and mermaid is vendored into each artifact, so the deployment works +unchanged behind restricted egress and sends no reader's IP to a CDN. + --- ## Architecture diff --git a/nginx.headers.conf b/nginx.headers.conf index 7c783e2..7d91f08 100644 --- a/nginx.headers.conf +++ b/nginx.headers.conf @@ -56,7 +56,11 @@ add_header Referrer-Policy "strict-origin" always; # is a far weaker injection primitive than script. Tightening it means hashing # or hoisting styles the same way — worth doing, not worth blocking this on. # +# style-src and font-src are otherwise 'self': Inter is served from this origin +# (#54), so no third-party font host needs allowlisting. font-src keeps data: +# for sub-app CSS that inlines a face. +# # frame-src allows https: because `type: "iframe"` registry entries embed # arbitrary external documentation sites; blob: is for the web-fragments -# reframing iframe. font-src covers Google Fonts until they are self-hosted (#54). -add_header Content-Security-Policy "default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; img-src 'self' data: https:; font-src 'self' data: https://fonts.gstatic.com; connect-src 'self'; frame-src 'self' https: blob:; worker-src 'self' blob:; frame-ancestors 'self'; base-uri 'none'; form-action 'self'; object-src 'none'" always; +# reframing iframe. +add_header Content-Security-Policy "default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data: https:; font-src 'self' data:; connect-src 'self'; frame-src 'self' https: blob:; worker-src 'self' blob:; frame-ancestors 'self'; base-uri 'none'; form-action 'self'; object-src 'none'" always; diff --git a/package-lock.json b/package-lock.json index 7f8626c..5fe7dbc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,6 +8,7 @@ "name": "knowledge-base", "version": "1.0.0", "dependencies": { + "@fontsource-variable/inter": "^5.3.0", "ajv": "^8.17.1", "parse5": "^7.3.0" }, @@ -1178,6 +1179,15 @@ "node": ">=18" } }, + "node_modules/@fontsource-variable/inter": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/@fontsource-variable/inter/-/inter-5.3.0.tgz", + "integrity": "sha512-OupL48va4JNofb97w6NYeF9S7W/kHNKM0Er8Dem5nqi4jeOLrVJDoE8tZEpnMJmtkvNbB1EIPPwHcdkF6b1oUA==", + "license": "OFL-1.1", + "funding": { + "url": "https://github.com/sponsors/ayuhito" + } + }, "node_modules/@img/colour": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.1.0.tgz", diff --git a/package.json b/package.json index cd06e7d..ef52bda 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "test:container": "npx playwright test --config=playwright.config.docker.js" }, "dependencies": { + "@fontsource-variable/inter": "^5.3.0", "ajv": "^8.17.1", "parse5": "^7.3.0" }, diff --git a/scripts/build-vite.js b/scripts/build-vite.js index ad7181b..3104207 100644 --- a/scripts/build-vite.js +++ b/scripts/build-vite.js @@ -16,7 +16,7 @@ * Sub-app pages are rendered by src/pages/[...path].astro via getStaticPaths. */ -import { existsSync, readFileSync, writeFileSync, mkdirSync, rmSync, copyFileSync, readdirSync, statSync } from 'node:fs'; +import { existsSync, readFileSync, writeFileSync, mkdirSync, rmSync, copyFileSync, linkSync, readdirSync } from 'node:fs'; import { join, dirname, resolve, isAbsolute } from 'node:path'; import { fileURLToPath } from 'node:url'; import { homedir } from 'node:os'; @@ -125,6 +125,24 @@ function installBundle(app, stageDir, sourceLabel) { function fail(msg) { throw new Error(msg); } +/** + * Hardlinks a file, copying only if the filesystem will not link it. + * + * Every sub-app asset is already written twice — once by the tarball extraction + * into apps/{slug}/, once by Astro's static copy of public/ into dist/. The hop + * in between does not need a third set of bytes, and for an artifact carrying + * images or fonts that is the largest of the three. A link is not possible + * across devices (EXDEV) or on a filesystem without hardlink support, hence the + * fallback rather than a bare linkSync. + */ +function linkOrCopy(src, dest) { + try { + linkSync(src, dest); + } catch { + copyFileSync(src, dest); + } +} + async function build() { const startMs = Date.now(); const modeLabel = [LOCAL_MODE && 'local', HEADLESS && 'headless'].filter(Boolean).join(', '); @@ -280,23 +298,34 @@ async function build() { */ function copyAssets(src, dest, slug) { mkdirSync(dest, { recursive: true }); - for (const entry of readdirSync(src).sort()) { - const s = join(src, entry); - const d = join(dest, entry); - if (statSync(s).isDirectory()) copyAssets(s, d, slug); - else if (!entry.endsWith('.html')) { + // withFileTypes: the entry type comes out of the directory read that already + // happened, instead of a statSync syscall per file — and a symlink reports as + // one rather than as whatever it points at. + const entries = readdirSync(src, { withFileTypes: true }).sort((a, b) => (a.name < b.name ? -1 : 1)); + + for (const entry of entries) { + if (entry.isSymbolicLink()) continue; + const s = join(src, entry.name); + const d = join(dest, entry.name); + + if (entry.isDirectory()) { copyAssets(s, d, slug); continue; } + if (!entry.isFile() || entry.name.endsWith('.html')) continue; + + if (entry.name.endsWith('.css')) { + // A real copy, not a link: the rewrite below edits the destination in + // place, and a hardlink would write that edit back into apps/{slug}/. copyFileSync(s, d); - if (entry.endsWith('.css')) { - const css = readFileSync(d, 'utf8'); - // url(/path) | url('/path') | url("/path") → url(/{prefix}/{slug}/path). - // The (?!\/) guard skips protocol-relative //host/…; data: and #ref - // never match, since neither starts with a slash. - const rewritten = css.replace( - /url\(\s*(['"]?)\/(?!\/)/g, - 'url($1/' + PATH_PREFIX + '/' + slug + '/', - ); - if (rewritten !== css) writeFileSync(d, rewritten); - } + const css = readFileSync(d, 'utf8'); + // url(/path) | url('/path') | url("/path") → url(/{prefix}/{slug}/path). + // The (?!\/) guard skips protocol-relative //host/…; data: and #ref + // never match, since neither starts with a slash. + const rewritten = css.replace( + /url\(\s*(['"]?)\/(?!\/)/g, + 'url($1/' + PATH_PREFIX + '/' + slug + '/', + ); + if (rewritten !== css) writeFileSync(d, rewritten); + } else { + linkOrCopy(s, d); } } } @@ -353,7 +382,9 @@ async function build() { // 4. Summary step('4/4 Build complete'); const elapsed = ((Date.now() - startMs) / 1000).toFixed(1); - const slugs = readdirSync(APPS_DIR).filter(d => statSync(join(APPS_DIR, d)).isDirectory()); + const slugs = readdirSync(APPS_DIR, { withFileTypes: true }) + .filter(e => e.isDirectory()) + .map(e => e.name); const appList = slugs.map(s => ' \u2022 \x1b[36m' + s + '\x1b[0m → dist/' + s + '/').join('\n'); console.log('\n\x1b[32m✓\x1b[0m \x1b[1m' + (slugs.length + iframeApps.length) + ' app(s) integrated\x1b[0m in ' + elapsed + 's\n\n Apps:\n' + appList + '\n Prefix: \x1b[33m' + PATH_PREFIX + '\x1b[0m\n'); } diff --git a/src/components/Masthead.astro b/src/components/Masthead.astro index 82f7500..7ae73af 100644 --- a/src/components/Masthead.astro +++ b/src/components/Masthead.astro @@ -20,16 +20,17 @@ interface App { } interface Props { - apps: App[]; - activeSlug?: string; + /** The app being viewed, or undefined on the catalog. Not the whole registry: + the masthead names one app, and a per-page copy of every app's metadata is + what Astro would otherwise serialise into every route (#51). */ + activeApp?: App; base?: string; /** True when the current page is the active app's own index (its crumb is then inert). */ appRoot?: boolean; } -const { apps, activeSlug = '', base = '/knowledge-base', appRoot = false } = Astro.props; +const { activeApp, base = '/knowledge-base', appRoot = false } = Astro.props; -const activeApp = apps.find(a => a.slug === activeSlug); const onLibrary = !activeApp; --- diff --git a/src/layouts/Base.astro b/src/layouts/Base.astro index 1cb12e6..d79cf84 100644 --- a/src/layouts/Base.astro +++ b/src/layouts/Base.astro @@ -71,11 +71,8 @@ const bodyClasses = [ */} - - {/* data-astro-transition-persist keeps the font stylesheet stable across - ClientRouter navigation so it isn't churned on every page swap — avoids - web-fragments #297 accumulation. */} - + {/* Inter is self-hosted: its @font-face lives in marketplace.css, which the + layout already imports, so there is nothing to link here (#54). */} diff --git a/src/pages/[...path].astro b/src/pages/[...path].astro index 408eebb..e6602b4 100644 --- a/src/pages/[...path].astro +++ b/src/pages/[...path].astro @@ -26,13 +26,12 @@ export function getStaticPaths() { } const props = Astro.props; -const { slug, appHeadless, apps, title } = props; +const { slug, appHeadless, app, title } = props; const parts = props.iframe ? null : transformSubAppHtml(readFileSync(props.file, 'utf-8'), slug, props.fileRelDir, PATH_PREFIX); -const app = apps.find((a: any) => a.slug === slug); const pageTitle = title ?? parts?.title ?? app?.name ?? 'Knowledge base'; // The app's own crumb in the masthead is inert on the app index and a link deeper in. @@ -47,7 +46,7 @@ const atAppRoot = !props.fileRelDir; > {parts && } - + {props.iframe ? (
diff --git a/src/pages/index.astro b/src/pages/index.astro index 781eaa6..34f3a1e 100644 --- a/src/pages/index.astro +++ b/src/pages/index.astro @@ -14,7 +14,7 @@ const buildDate = new Date().toISOString().slice(0, 10); --- - +
diff --git a/src/styles/marketplace.css b/src/styles/marketplace.css index 7195d05..5811a49 100644 --- a/src/styles/marketplace.css +++ b/src/styles/marketplace.css @@ -4,6 +4,38 @@ @source "../src/templates/**/*.js"; @source "../scripts/**/*.js"; +/* ── Typeface ─────────────────────────────────────────────────────────────── + Inter, self-hosted from @fontsource-variable/inter. + + It used to come from fonts.googleapis.com, which sent every reader's IP and + User-Agent to a third party on every page view and made the font the one + external dependency in an otherwise self-contained deployment — mermaid is + already vendored for the same reason. Self-hosting also keeps the CSP at + 'self' for both style-src and font-src (#54). + + Only the two latin subsets are declared, and only the upright faces: the + Google request asked for exactly those. Importing the package's own CSS would + emit the cyrillic, greek and vietnamese subsets into dist/ as well, which no + reader would ever fetch. The `unicode-range` values are the package's, so a + reader who does need another script still degrades to the fallback stack + rather than getting mojibake. */ +@font-face { + font-family: 'Inter Variable'; + font-style: normal; + font-display: swap; + font-weight: 100 900; + src: url('@fontsource-variable/inter/files/inter-latin-opsz-normal.woff2') format('woff2-variations'); + unicode-range: U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD; +} +@font-face { + font-family: 'Inter Variable'; + font-style: normal; + font-display: swap; + font-weight: 100 900; + src: url('@fontsource-variable/inter/files/inter-latin-ext-opsz-normal.woff2') format('woff2-variations'); + unicode-range: U+0100-02BA,U+02BD-02C5,U+02C7-02CC,U+02CE-02D7,U+02DD-02FF,U+0304,U+0308,U+0329,U+1D00-1DBF,U+1E00-1E9F,U+1EF2-1EFF,U+2020,U+20A0-20AB,U+20AD-20C0,U+2113,U+2C60-2C7F,U+A720-A7FF; +} + /* ── Design tokens ──────────────────────────────────────────────────────── */ @theme { /* Brand palette */ @@ -16,7 +48,7 @@ --color-kb-950: #1b0e12; /* Typography */ - --font-sans: Inter, 'Noto Sans', ui-sans-serif, system-ui, sans-serif; + --font-sans: 'Inter Variable', Inter, 'Noto Sans', ui-sans-serif, system-ui, sans-serif; --font-mono: 'SF Mono', 'Fira Code', 'Fira Mono', ui-monospace, monospace; } diff --git a/src/utils/apps.js b/src/utils/apps.js index 42a1bcf..a98d487 100644 --- a/src/utils/apps.js +++ b/src/utils/apps.js @@ -6,6 +6,23 @@ import { existsSync, readFileSync, readdirSync, statSync } from 'node:fs'; import { join, relative, dirname } from 'node:path'; import { isSinglePage, readExpansionMap, resolveRegistry } from './single-page.js'; +/** + * Cached result of loadRegistry, keyed by cwd and invalidated by mtime. + * + * getAppPages calls loadRegistry, and Astro may call getStaticPaths more than + * once per build, so the registry was being re-read, re-parsed and re-resolved + * for no new information. Keying on the mtimes rather than on cwd alone keeps + * `astro dev` honest: editing apps.json is still picked up on the next request, + * at the cost of two statSync calls instead of two full reads and parses. + */ +let registryCache = null; + +/** Modification stamp of the two files the registry is built from. */ +function registryStamp(cwd) { + const mtime = (p) => (existsSync(p) ? statSync(p).mtimeMs : 0); + return `${mtime(join(cwd, 'apps.json'))}:${mtime(join(cwd, 'apps', '.single-page.json'))}`; +} + /** * Reads the effective app registry. * @@ -16,24 +33,36 @@ import { isSinglePage, readExpansionMap, resolveRegistry } from './single-page.j * @param {string} cwd - project root (process.cwd()) */ export function loadRegistry(cwd) { + const stamp = registryStamp(cwd); + if (registryCache && registryCache.cwd === cwd && registryCache.stamp === stamp) { + return registryCache.apps; + } + const appsJson = join(cwd, 'apps.json'); const registry = existsSync(appsJson) ? JSON.parse(readFileSync(appsJson, 'utf-8')) : []; - return resolveRegistry(registry, readExpansionMap(cwd), (msg) => { + const apps = resolveRegistry(registry, readExpansionMap(cwd), (msg) => { process.stderr.write(` \x1b[33m⚠\x1b[0m ${msg}\n`); }); + + registryCache = { cwd, stamp, apps }; + return apps; } -/** Recursively collect every .html file under a directory. */ +/** Recursively collect every .html file under a directory. Symlinks are skipped. */ export function collectHtmlFiles(dir) { if (!existsSync(dir)) return []; const results = []; - for (const entry of readdirSync(dir)) { - const full = join(dir, entry); - if (statSync(full).isDirectory()) results.push(...collectHtmlFiles(full)); - else if (entry.endsWith('.html')) results.push(full); + // withFileTypes: the type comes from the directory read that already happened, + // rather than a statSync per entry, and a symlink is reported as one instead + // of as its target. + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if (entry.isSymbolicLink()) continue; + const full = join(dir, entry.name); + if (entry.isDirectory()) results.push(...collectHtmlFiles(full)); + else if (entry.isFile() && entry.name.endsWith('.html')) results.push(full); } return results; } @@ -43,6 +72,13 @@ export function collectHtmlFiles(dir) { * Each descriptor contains the route path, the absolute file path, * the app slug, and the effective headless flag. * + * Note what is *not* in a descriptor: the registry. getStaticPaths returns one + * entry per HTML file and every one of them used to carry the whole resolved + * registry, so the route table grew with apps × pages × apps. (It cost build + * memory, not output bytes — these props are never serialised into the static + * HTML.) Only the current app's card is needed: the masthead names it and the + * catchall titles the page with it. + * * @param {string} cwd - project root (process.cwd()) * @param {boolean} headless - global headless default */ @@ -55,6 +91,8 @@ export function getAppPages(cwd, headless) { for (const app of apps) { const appDir = join(appsDir, app.slug); const appHeadless = app.headless ?? headless; + /** The only registry data a sub-app page needs. */ + const card = { slug: app.slug, name: app.name ?? app.slug, icon: app.icon }; if (app.type === 'iframe') { // iFrame onboarding mode: no artifact on disk — emit a single route that @@ -65,7 +103,7 @@ export function getAppPages(cwd, headless) { slug: app.slug, fileRelDir: '', appHeadless, - apps, + app: card, title: app.name ?? app.slug, section: null, iframe: true, @@ -84,7 +122,7 @@ export function getAppPages(cwd, headless) { slug: app.slug, fileRelDir: '', appHeadless, - apps, + app: card, title: app.name ?? app.slug, section: null, singlePage: true, @@ -106,7 +144,7 @@ export function getAppPages(cwd, headless) { slug: app.slug, fileRelDir, appHeadless, - apps, + app: card, title: page.title, section: page.section ?? null, }); @@ -124,7 +162,7 @@ export function getAppPages(cwd, headless) { slug: app.slug, fileRelDir, appHeadless, - apps, + app: card, title: null, section: null, }); diff --git a/tests/build-integrity.spec.js b/tests/build-integrity.spec.js index d9aae88..1813ae2 100644 --- a/tests/build-integrity.spec.js +++ b/tests/build-integrity.spec.js @@ -20,6 +20,16 @@ const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..'); const DIST = join(ROOT, 'dist'); const read = (rel) => readFileSync(join(DIST, rel), 'utf8'); +/** Every .html file under dist/, recursively. */ +function htmlFiles(dir, acc = []) { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const full = join(dir, entry.name); + if (entry.isDirectory()) htmlFiles(full, acc); + else if (entry.name.endsWith('.html')) acc.push(full); + } + return acc; +} + /** * The marketplace stylesheet a page loads. Astro injects this from * Base.astro's CSS import, so the name is content-hashed and changes whenever @@ -289,16 +299,6 @@ test.describe('Masthead', () => { // ───────────────────────────────────────────────────────────────────────────── test.describe('no inline scripts in the build output', () => { - /** Every .html file under dist/, recursively. */ - function htmlFiles(dir, acc = []) { - for (const entry of readdirSync(dir, { withFileTypes: true })) { - const full = join(dir, entry.name); - if (entry.isDirectory()) htmlFiles(full, acc); - else if (entry.name.endsWith('.html')) acc.push(full); - } - return acc; - } - test('every