diff --git a/docs/packages/cli.mdx b/docs/packages/cli.mdx index 5c5eb3e7bf..4b41364315 100644 --- a/docs/packages/cli.mdx +++ b/docs/packages/cli.mdx @@ -186,10 +186,41 @@ npx hyperframes capture https://example.com --json Screenshots: 12 Assets: 45 + Dropped: 9 (6 size-floor, 3 cap-reached) Sections: 15 Fonts: sohne-var ``` +`Dropped` is how many assets the page referenced that are **not** in the +folder, and why. It is printed only when it is non-zero, and `--json` always +carries it as a `dropped` object. Without it, a capture of a spare page and a +capture that a limit truncated are the same three-line summary, and the only +way to tell them apart is to open the page yourself. + +| Reason | What it means | +| ------------------ | ------------------------------------------------------------------------------------------------------ | +| `size-floor` | Fetched, then judged too small to be a real asset rather than a spacer or tracking pixel. | +| `budget-exhausted` | `--capture-budget` ran out before this one was reached. Raise it, or pass `--skip-vision` to buy time. | +| `cap-reached` | A per-run or per-family limit was already met: 30 inline SVGs, 30 fonts, 6 faces per family. | +| `unavailable` | The request or the write failed: network error, timeout, refused address, bad status, disk. | + +Three of those are decisions the capture made and one is a failure it hit, so a +run that is thin with an all-zero `dropped` is a thin page, and a run that is +thin with counts on it was cut short. + +A page that declares no `` still gets a favicon: capture falls +back to `/favicon.ico` and then `/apple-touch-icon.png` at the site root, the +same paths a browser requests on its own, and keeps the first that answers with +an image. A declared icon link always wins, and a fallback that answers with +nothing usable is counted under `unavailable`. + +```json +{ + "assets": 45, + "dropped": { "size-floor": 6, "budget-exhausted": 0, "cap-reached": 3, "unavailable": 0 } +} +``` + | Flag | Description | | ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------- | | `--output, -o` | Output directory. Default `./capture`, then `./capture-2/`, `./capture-3/`, … if that name is taken. | @@ -207,7 +238,8 @@ metadata, and contact sheets — plus whatever Lottie, video, and WebGL context the page exposed. It is raw material for an agent, not a finished composition; the `/product-launch-video` workflow uses it when a real product has to appear on screen. Dynamic sites, protected pages, and unusual media loaders produce -partial results, so read the warnings and contact sheets before you build. +partial results, so read `dropped`, the warnings, and the contact sheets before +you build. For AI image descriptions, set `GEMINI_API_KEY` in a `.env` file (~$0.001/image), or `OPENROUTER_API_KEY` to route any vision model through diff --git a/packages/cli/src/capture/assetDownloader.test.ts b/packages/cli/src/capture/assetDownloader.test.ts index 4028da90c8..65bddd8e84 100644 --- a/packages/cli/src/capture/assetDownloader.test.ts +++ b/packages/cli/src/capture/assetDownloader.test.ts @@ -4,10 +4,12 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { downloadAndRewriteFonts, + downloadAssets, isPrivateUrl, safeFetch, toStandaloneSvg, } from "./assetDownloader.js"; +import type { DesignTokens } from "./types.js"; describe("isPrivateUrl — SSRF denylist (security: F-003)", () => { it("blocks loopback, private, and metadata IPv4", () => { @@ -184,3 +186,224 @@ describe("downloadAndRewriteFonts — attempt caps", () => { } }); }); + +describe("drop counts — why a referenced asset is not in the capture", () => { + afterEach(() => vi.unstubAllGlobals()); + + /** `n` @font-face rules, each naming a DIFFERENT family, so only the global cap can bite. */ + function fontCss(n: number): string { + return Array.from( + { length: n }, + (_, i) => + `@font-face { font-family: Family${i}; src: url(https://fonts${i}.example/font-${i}.woff2); }`, + ).join("\n"); + } + + function withTempDir(run: (dir: string) => Promise): Promise { + const dir = mkdtempSync(join(tmpdir(), "hf-drops-")); + return run(dir).finally(() => rmSync(dir, { recursive: true, force: true })); + } + + it("counts every face the budget never let it reach, not just the one it stopped on", async () => { + // Four declared, zero budget: the honest number is four, and a warning string could only + // ever have said "some". + await withTempDir(async (dir) => { + const fetchMock = vi.fn(); + vi.stubGlobal("fetch", fetchMock); + const { drops } = await downloadAndRewriteFonts(fontCss(4), dir, { remainingMs: () => 0 }); + expect(fetchMock).not.toHaveBeenCalled(); + expect(drops["budget-exhausted"]).toBe(4); + expect(drops["cap-reached"]).toBe(0); + expect(drops.unavailable).toBe(0); + }); + }); + + it("separates the faces the global cap refused from the ones that failed", async () => { + // 35 declared, cap 30: 30 are attempted and every attempt 503s, 5 are never reached. + await withTempDir(async (dir) => { + vi.stubGlobal( + "fetch", + vi.fn(async () => new Response("no", { status: 503 })), + ); + const { drops } = await downloadAndRewriteFonts(fontCss(35), dir); + expect(drops["cap-reached"]).toBe(5); + expect(drops.unavailable).toBe(30); + expect(drops["budget-exhausted"]).toBe(0); + }); + }); + + it("counts the faces the per-family cap refused", async () => { + // 10 rules, all one family, per-family cap 6: 6 attempted, 4 refused. + await withTempDir(async (dir) => { + vi.stubGlobal( + "fetch", + vi.fn(async () => new Response("no", { status: 503 })), + ); + const css = Array.from( + { length: 10 }, + (_, i) => + `@font-face { font-family: Shared; src: url(https://fonts.example/f-${i}.woff2); }`, + ).join("\n"); + const { drops } = await downloadAndRewriteFonts(css, dir); + expect(drops["cap-reached"]).toBe(4); + expect(drops.unavailable).toBe(6); + }); + }); + + it("reports all zeroes when nothing was refused, which is what makes thin readable", async () => { + // The whole point of the tally: this page declared one face and we have it. A reader can now + // tell this apart from a page that declared thirty and got truncated to one. + await withTempDir(async (dir) => { + vi.stubGlobal( + "fetch", + vi.fn(async () => new Response(new Uint8Array(2048), { status: 200 })), + ); + const { css, drops } = await downloadAndRewriteFonts(fontCss(1), dir); + expect(css).toContain("assets/fonts/font-0.woff2"); + expect(drops).toEqual({ + "size-floor": 0, + "budget-exhausted": 0, + "cap-reached": 0, + unavailable: 0, + }); + }); + }); + + /** The two fields `downloadAssets` reads off the token bundle. */ + function tokensWithNoSvgs(): DesignTokens { + return { svgs: [], sections: [], ogImage: "" } as unknown as DesignTokens; + } + + it("counts an image dropped for being under the raster floor", async () => { + // 9 KB is under the 10 KB floor. Nothing lands, and the reason is now on the record. + await withTempDir(async (dir) => { + vi.stubGlobal( + "fetch", + vi.fn(async () => new Response(new Uint8Array(9000), { status: 200 })), + ); + const { assets, drops } = await downloadAssets(tokensWithNoSvgs(), dir, [ + { type: "Image", url: "https://cdn.example/hero.png", contexts: ["img[src]"] }, + ] as never); + expect(assets).toEqual([]); + expect(drops["size-floor"]).toBe(1); + expect(drops.unavailable).toBe(0); + }); + }); + + it("counts every catalogued image the budget never let it reach", async () => { + await withTempDir(async (dir) => { + const fetchMock = vi.fn(); + vi.stubGlobal("fetch", fetchMock); + const catalog = Array.from({ length: 7 }, (_, i) => ({ + type: "Image", + url: `https://cdn.example/img-${i}.png`, + contexts: ["img[src]"], + })); + const { assets, drops } = await downloadAssets( + tokensWithNoSvgs(), + dir, + catalog as never, + [], + { remainingMs: () => 0 }, + ); + expect(fetchMock).not.toHaveBeenCalled(); + expect(assets).toEqual([]); + expect(drops["budget-exhausted"]).toBe(7); + }); + }); + + it("falls back to the site root's well-known icon paths when the page declares none", async () => { + // The case that used to produce no favicon AND no drop reason: a page whose head declares + // no icon link, on a site that serves `/favicon.ico` perfectly well. + await withTempDir(async (dir) => { + const fetchMock = vi.fn( + async (url: string) => + new Response(new Uint8Array([0, 0, 1, 0]), { + status: url.endsWith("/favicon.ico") ? 200 : 404, + headers: { "content-type": "image/x-icon" }, + }), + ); + vi.stubGlobal("fetch", fetchMock); + const { assets, drops } = await downloadAssets(tokensWithNoSvgs(), dir, [], [], { + pageUrl: "https://brand.example/pricing?ref=x", + }); + expect(assets).toEqual([ + { + url: "https://brand.example/favicon.ico", + localPath: "assets/favicon.ico", + type: "favicon", + }, + ]); + expect(drops.unavailable).toBe(0); + }); + }); + + it("counts a well-known icon path that answers with a page, never drops it silently", async () => { + // A site with no icon answers `/favicon.ico` with its 200 HTML shell. Nothing is kept and + // both guesses are on the record. + await withTempDir(async (dir) => { + vi.stubGlobal( + "fetch", + vi.fn( + async () => + new Response("", { + status: 200, + headers: { "content-type": "text/html" }, + }), + ), + ); + const { assets, drops } = await downloadAssets(tokensWithNoSvgs(), dir, [], [], { + pageUrl: "https://brand.example/", + }); + expect(assets).toEqual([]); + expect(drops.unavailable).toBe(2); + }); + }); + + it("keeps a declared icon link ahead of the well-known paths", async () => { + // The control: when the page vouches for an icon, the guess must not be attempted at all. + await withTempDir(async (dir) => { + const fetchMock = vi.fn( + async (_url: string) => + new Response(new Uint8Array([137, 80, 78, 71]), { + status: 200, + headers: { "content-type": "image/png" }, + }), + ); + vi.stubGlobal("fetch", fetchMock); + const { assets } = await downloadAssets( + tokensWithNoSvgs(), + dir, + [], + [{ rel: "icon", href: "https://cdn.example/brand/icon-512.png" }], + { pageUrl: "https://brand.example/" }, + ); + expect(assets).toEqual([ + { + url: "https://cdn.example/brand/icon-512.png", + localPath: "assets/favicon.png", + type: "favicon", + }, + ]); + expect(fetchMock.mock.calls.map((c) => c[0])).toEqual([ + "https://cdn.example/brand/icon-512.png", + ]); + }); + }); + + it("counts the inline SVGs the 30-per-run cap refused", async () => { + // 34 inline SVGs on the page, 30 kept: the four the cap dropped are now countable. + await withTempDir(async (dir) => { + const svgs = Array.from({ length: 34 }, (_, i) => ({ + outerHTML: ``, + isLogo: false, + })); + const { assets, drops } = await downloadAssets( + { svgs, sections: [], ogImage: "" } as unknown as DesignTokens, + dir, + ); + expect(assets).toHaveLength(30); + expect(drops["cap-reached"]).toBe(4); + }); + }); +}); diff --git a/packages/cli/src/capture/assetDownloader.ts b/packages/cli/src/capture/assetDownloader.ts index 21a881f3b9..c36bc6c61e 100644 --- a/packages/cli/src/capture/assetDownloader.ts +++ b/packages/cli/src/capture/assetDownloader.ts @@ -13,6 +13,84 @@ import type { CatalogedAsset } from "./assetCataloger.js"; interface DownloadBudgetOptions { remainingMs?: () => number; + /** + * The page's final URL, used to resolve the well-known icon paths when the page declares + * no icon link at all. Omit it and a page with no declared icon simply gets no favicon. + */ + pageUrl?: string; +} + +/** + * The paths a browser requests on its own when a document declares no ``. + * Order is the browser's preference order, and the first one that answers wins. + * + * This exists because "the page declared no icon" and "the site has no icon" are different + * facts, and only the first is visible in the DOM. A large site can serve a perfectly good + * `/favicon.ico` while declaring none of the ~150 `` tags in its head as an icon; + * scraping only declared links returns nothing for it and, worse, attempts nothing, so not + * even a drop reason gets recorded. + */ +const WELL_KNOWN_ICON_PATHS = ["/favicon.ico", "/apple-touch-icon.png"]; + +/** + * Declared icon links if the page has any, otherwise the site root's well-known paths. + * Declared links always win: the page vouched for them, a well-known path is a guess. + */ +function iconCandidates( + declared: Array<{ rel: string; href: string }>, + pageUrl: string | undefined, +): Array<{ rel: string; href: string }> { + if (declared.length > 0) return declared; + if (!pageUrl) return []; + try { + return WELL_KNOWN_ICON_PATHS.map((path) => ({ + rel: "icon", + href: new URL(path, pageUrl).href, + })); + } catch { + return []; + } +} + +/** + * Why an asset the page referenced is not in the capture. + * + * Three of these are DECISIONS this downloader made and one is a FAILURE it hit, which is the + * split a reader actually needs: a capture that is thin because the page is thin looks exactly + * like a capture that is thin because a limit truncated it, and neither used to say so. + * + * Every member is counted at the single line that performs the drop, so a count can never + * disagree with the branch it describes. + */ +export type AssetDropReason = + /** Fetched, then judged too small to be a real asset rather than a spacer or tracking pixel. */ + | "size-floor" + /** The post-navigation clock ran out before this one was reached. */ + | "budget-exhausted" + /** A per-run or per-family limit was already met. */ + | "cap-reached" + /** The request or the write failed: network error, timeout, refused address, bad status, disk. */ + | "unavailable"; + +export type AssetDropCounts = Record; + +/** A tally with every reason at zero — the shape a caller merges into. */ +export function noDrops(): AssetDropCounts { + return { "size-floor": 0, "budget-exhausted": 0, "cap-reached": 0, unavailable: 0 }; +} + +/** Sum two tallies. Used to fold the font pass and the asset pass into one capture-wide count. */ +export function mergeDrops(a: AssetDropCounts, b: AssetDropCounts): AssetDropCounts { + const total = noDrops(); + for (const reason of Object.keys(total) as AssetDropReason[]) { + total[reason] = a[reason] + b[reason]; + } + return total; +} + +/** How many assets were dropped in total, for a caller deciding whether to say anything at all. */ +export function totalDrops(drops: AssetDropCounts): number { + return Object.values(drops).reduce((sum, n) => sum + n, 0); } // SVGs: hash-of-bytes filename so it can't drift from content; label-derived names mis-assigned brands. @@ -54,18 +132,24 @@ export async function downloadAssets( catalogedAssets?: CatalogedAsset[], faviconLinks?: Array<{ rel: string; href: string }>, options: DownloadBudgetOptions = {}, -): Promise { +): Promise<{ assets: DownloadedAsset[]; drops: AssetDropCounts }> { const assetsDir = join(outputDir, "assets"); mkdirSync(assetsDir, { recursive: true }); const assets: DownloadedAsset[] = []; + const drops = noDrops(); const downloadedUrls = new Set(); mkdirSync(join(outputDir, "assets", "svgs"), { recursive: true }); const usedSvgNames = new Set(); - for (let i = 0; i < tokens.svgs.length && i < 30; i++) { + const MAX_INLINE_SVGS = 30; + drops["cap-reached"] += Math.max(0, tokens.svgs.length - MAX_INLINE_SVGS); + for (let i = 0; i < tokens.svgs.length && i < MAX_INLINE_SVGS; i++) { const svg = tokens.svgs[i]!; - if (!svg.outerHTML || svg.outerHTML.length < 50) continue; + if (!svg.outerHTML || svg.outerHTML.length < 50) { + drops["size-floor"]++; + continue; + } // Hash the bytes that actually land on disk, so the filename still can't drift from content. const svgFile = toStandaloneSvg(svg.outerHTML); const slug = svgContentHashSlug(svgFile, !!svg.isLogo); @@ -82,14 +166,21 @@ export async function downloadAssets( writeFileSync(join(outputDir, localPath), svgFile, "utf-8"); assets.push({ url: "", localPath, type: "svg" }); } catch { - /* skip */ + drops.unavailable++; } } // 2. Favicon - for (const icon of faviconLinks || []) { + // `fetchBuffer` already refuses an HTML or XML body served under a 200, which is exactly + // how a site with no icon answers a well-known path, so a guess that misses lands in + // `unavailable` rather than writing a login page to `assets/favicon.ico`. + const icons = iconCandidates(faviconLinks || [], options.pageUrl); + for (const [index, icon] of icons.entries()) { const remainingMs = options.remainingMs?.() ?? 10_000; - if (remainingMs <= 0) break; + if (remainingMs <= 0) { + drops["budget-exhausted"] += icons.length - index; + break; + } if (!icon.href) continue; try { const ext = extname(new URL(icon.href).pathname) || ".ico"; @@ -101,8 +192,9 @@ export async function downloadAssets( assets.push({ url: icon.href, localPath, type: "favicon" }); break; } + drops.unavailable++; } catch { - /* skip */ + drops.unavailable++; } } @@ -158,7 +250,10 @@ export async function downloadAssets( const usedNames = new Set(); for (let i = 0; i < toDownload.length; i += BATCH_SIZE) { const remainingMs = options.remainingMs?.() ?? 10_000; - if (remainingMs <= 0) break; + if (remainingMs <= 0) { + drops["budget-exhausted"] += toDownload.length - i; + break; + } const batch = toDownload.slice(i, i + BATCH_SIZE); const results = await Promise.allSettled( batch.map(async ({ url, isPoster, catalog }) => { @@ -166,15 +261,27 @@ export async function downloadAssets( const pathExt = extname(parsedUrl.pathname); const ext = pathExt && pathExt.length <= 5 ? pathExt : ".jpg"; const buffer = await fetchBuffer(url, Math.min(10_000, remainingMs)); - if (!buffer) return null; + if (!buffer) { + drops.unavailable++; + return null; + } const isSvg = ext === ".svg" || url.includes(".svg"); const minSize = isSvg ? 200 : 10000; - if (buffer.length < minSize) return null; + if (buffer.length < minSize) { + drops["size-floor"]++; + return null; + } return { url, isPoster, parsedUrl, ext, buffer, catalog }; }), ); for (const result of results) { - if (result.status !== "fulfilled" || !result.value) continue; + // A rejection never reached a drop site of its own, so it is counted here. A fulfilled + // `null` already counted itself above; counting it again here would double it. + if (result.status === "rejected") { + drops.unavailable++; + continue; + } + if (!result.value) continue; const { url, isPoster, parsedUrl, ext, buffer, catalog } = result.value; try { let slug: string; @@ -201,7 +308,7 @@ export async function downloadAssets( assets.push({ url, localPath, type: "image" }); imgIdx++; } catch { - /* skip */ + drops.unavailable++; } } } @@ -212,18 +319,25 @@ export async function downloadAssets( try { const ext = extname(new URL(tokens.ogImage).pathname) || ".jpg"; const localPath = `assets/og-image${ext}`; - const buffer = - remainingMs > 0 ? await fetchBuffer(tokens.ogImage, Math.min(10_000, remainingMs)) : null; - if (buffer && buffer.length > 5000) { - writeFileSync(join(outputDir, localPath), buffer); - assets.push({ url: tokens.ogImage, localPath, type: "image" }); + if (remainingMs <= 0) { + drops["budget-exhausted"]++; + } else { + const buffer = await fetchBuffer(tokens.ogImage, Math.min(10_000, remainingMs)); + if (!buffer) { + drops.unavailable++; + } else if (buffer.length <= 5000) { + drops["size-floor"]++; + } else { + writeFileSync(join(outputDir, localPath), buffer); + assets.push({ url: tokens.ogImage, localPath, type: "image" }); + } } } catch { - /* skip */ + drops.unavailable++; } } - return assets; + return { assets, drops }; } /** Normalize URL for deduplication — unwrap Next.js image proxy, strip w/q params */ @@ -251,9 +365,10 @@ export async function downloadAndRewriteFonts( css: string, outputDir: string, options: DownloadBudgetOptions = {}, -): Promise { +): Promise<{ css: string; drops: AssetDropCounts }> { const assetsDir = join(outputDir, "assets", "fonts"); mkdirSync(assetsDir, { recursive: true }); + const drops = noDrops(); const fontUrlRegex = /url\(['"]?(https?:\/\/[^'")\s]+\.(?:woff2?|ttf|otf)[^'")\s]*?)['"]?\)/g; const fontUrls = new Set(); @@ -262,7 +377,7 @@ export async function downloadAndRewriteFonts( if (match[1]) fontUrls.add(match[1]); } - if (fontUrls.size === 0) return css; + if (fontUrls.size === 0) return { css, drops }; // Limit font download attempts to bound worst-case egress and latency. Google Fonts serves // 20+ unicode-range subsets per weight, so successes alone cannot be the bound: six transient @@ -293,13 +408,22 @@ export async function downloadAndRewriteFonts( let rewritten = css; let count = 0; - for (const fontUrl of sortedUrls) { + for (const [index, fontUrl] of sortedUrls.entries()) { const remainingMs = options.remainingMs?.() ?? 10_000; - if (remainingMs <= 0) break; - if (count >= MAX_TOTAL_FONTS) break; + if (remainingMs <= 0) { + drops["budget-exhausted"] += sortedUrls.length - index; + break; + } + if (count >= MAX_TOTAL_FONTS) { + drops["cap-reached"] += sortedUrls.length - index; + break; + } const family = getFamilyForUrl(fontUrl); const familyCount = familyCounts.get(family) || 0; - if (familyCount >= MAX_FONTS_PER_FAMILY) continue; + if (familyCount >= MAX_FONTS_PER_FAMILY) { + drops["cap-reached"]++; + continue; + } familyCounts.set(family, familyCount + 1); count++; @@ -313,13 +437,15 @@ export async function downloadAndRewriteFonts( if (buffer) { writeFileSync(localPath, buffer); rewritten = rewritten.split(fontUrl).join(relativePath); + } else { + drops.unavailable++; } } catch { - /* skip */ + drops.unavailable++; } } - return rewritten; + return { css: rewritten, drops }; } // Reserved/loopback/private IPv4 blocks as [firstOctet, secondOctetLo, secondOctetHi]. diff --git a/packages/cli/src/capture/index.ts b/packages/cli/src/capture/index.ts index 9f65ea054c..7cd6d48f7f 100644 --- a/packages/cli/src/capture/index.ts +++ b/packages/cli/src/capture/index.ts @@ -16,7 +16,13 @@ import { extractHtml } from "./htmlExtractor.js"; // captureScreenshots removed — full-page screenshot replaces per-section shots import { extractTokens } from "./tokenExtractor.js"; import { extractDesignStyles } from "./designStyleExtractor.js"; -import { downloadAssets, downloadAndRewriteFonts } from "./assetDownloader.js"; +import { + downloadAssets, + downloadAndRewriteFonts, + mergeDrops, + noDrops, + totalDrops, +} from "./assetDownloader.js"; import { extractFontMetadata } from "./fontMetadataExtractor.js"; import { normalizeErrorMessage } from "../utils/errorMessage.js"; import { diag } from "../ui/diagnostics.js"; @@ -574,27 +580,30 @@ export async function captureWebsite( return iconEls.map(function(l) { return { rel: l.rel, href: l.href }; }); })()`)) as Array<{ rel: string; href: string }>; + // Read the FINAL url (after any redirect) before the page goes away: it is what the + // well-known icon paths must resolve against when the page declared no icon link. + const finalPageUrl = page1.url(); + await page1.close(); phase("core-extraction", "completed"); - // Download fonts and rewrite URLs to local paths - if (remainingMs() > 0) { - phase("fonts", "started"); - extracted.headHtml = await downloadAndRewriteFonts(extracted.headHtml, outputDir, { - remainingMs, - }); - phase( - "fonts", - remainingMs() > 0 ? "completed" : "degraded", - remainingMs() > 0 ? undefined : "budget-exhausted", - ); - } else { - warnings.push( - "Capture budget exhausted before font downloads; extracted font tokens were preserved.", - ); - phase("fonts", "degraded", "budget-exhausted"); - } + // Download fonts and rewrite URLs to local paths. + // + // Called even with the budget already gone, which is the point: its own loop is the only + // thing that knows how many faces the page declared, so letting it run and record + // `budget-exhausted` for every one of them replaces a warning string that could only ever + // say "some". A zero budget means it breaks on the first url, so this costs no network. + phase("fonts", "started"); + const fontPass = await downloadAndRewriteFonts(extracted.headHtml, outputDir, { + remainingMs, + }); + extracted.headHtml = fontPass.css; + phase( + "fonts", + remainingMs() > 0 ? "completed" : "degraded", + remainingMs() > 0 ? undefined : "budget-exhausted", + ); // Identify each downloaded font by reading its OpenType name table. // Modern frameworks hash font filenames; this manifest tells the @@ -651,25 +660,41 @@ export async function captureWebsite( // Download assets — single pass using the catalog for best image quality let assets: CaptureResult["assets"] = []; + let assetDrops = noDrops(); if (!skipAssets) { - if (remainingMs() > 0) { - phase("assets", "started"); - progress("assets", "Downloading assets..."); - assets = await downloadAssets(tokens, outputDir, catalogedAssets, faviconLinks, { - remainingMs, - }); - phase( - "assets", - remainingMs() > 0 ? "completed" : "degraded", - remainingMs() > 0 ? undefined : "budget-exhausted", - ); - } else { - warnings.push("Capture budget exhausted before asset downloads; extraction continued."); - phase("assets", "degraded", "budget-exhausted"); - } + // Called even with the budget already gone, for the reason the font pass is: the loop that + // skips an asset is the only thing that can say how many it skipped. + phase("assets", "started"); + progress("assets", "Downloading assets..."); + const assetPass = await downloadAssets(tokens, outputDir, catalogedAssets, faviconLinks, { + remainingMs, + pageUrl: finalPageUrl, + }); + assets = assetPass.assets; + assetDrops = assetPass.drops; + phase( + "assets", + remainingMs() > 0 ? "completed" : "degraded", + remainingMs() > 0 ? undefined : "budget-exhausted", + ); } else { phase("assets", "degraded", "disabled"); } + // One capture-wide tally, summed from the two passes that own the drops. The warning is + // DERIVED from it rather than written alongside it, so the prose and the number cannot + // disagree the way two separately-authored budget strings could. + const dropped = mergeDrops(fontPass.drops, assetDrops); + const droppedTotal = totalDrops(dropped); + if (droppedTotal > 0) { + const breakdown = Object.entries(dropped) + .filter(([, n]) => n > 0) + .map(([reason, n]) => `${n} ${reason}`) + .join(", "); + warnings.push( + `${droppedTotal} referenced asset(s) are not in this capture (${breakdown}). ` + + "A thin capture with no drops is a thin page; this one was truncated.", + ); + } // Join in-section media URLs → downloaded local paths, then re-write // tokens.json. Downstream page recreation MUST reference local files: @@ -885,6 +910,7 @@ export async function captureWebsite( screenshots, tokens, assets, + dropped, animationCatalog, warnings, lastPhase, diff --git a/packages/cli/src/capture/types.ts b/packages/cli/src/capture/types.ts index f64b5f318d..2e965ad13e 100644 --- a/packages/cli/src/capture/types.ts +++ b/packages/cli/src/capture/types.ts @@ -79,6 +79,13 @@ export interface CaptureResult { tokens: DesignTokens; /** Downloaded asset paths (relative to projectDir) */ assets: DownloadedAsset[]; + /** + * How many referenced assets are NOT here, by reason. + * + * Without this, a capture of a page with three images and a capture truncated to three images + * are the same object. All zeroes means the capture kept everything it was offered. + */ + dropped: import("./assetDownloader.js").AssetDropCounts; /** Animation catalog (captured during full-JS page load) */ animationCatalog?: import("./animationCataloger.js").AnimationCatalog; /** Errors/warnings encountered during capture */ diff --git a/packages/cli/src/commands/capture.ts b/packages/cli/src/commands/capture.ts index 7d4d3a2ed4..e0bde6b29e 100644 --- a/packages/cli/src/commands/capture.ts +++ b/packages/cli/src/commands/capture.ts @@ -207,6 +207,7 @@ export default defineCommand({ title: result.title, screenshots: result.screenshots.length, assets: result.assets.length, + dropped: result.dropped, detectedSections: result.tokens.sections.length, fonts: result.tokens.fonts.map((f) => f.family), fontsDetailed: result.tokens.fonts, @@ -225,6 +226,18 @@ export default defineCommand({ console.log(); console.log(` ${c.dim("Screenshots:")} ${result.screenshots.length}`); console.log(` ${c.dim("Assets:")} ${result.assets.length}`); + const droppedTotal = Object.values(result.dropped).reduce((sum, n) => sum + n, 0); + if (droppedTotal > 0) { + const breakdown = Object.entries(result.dropped) + .filter(function (entry) { + return entry[1] > 0; + }) + .map(function (entry) { + return entry[1] + " " + entry[0]; + }) + .join(", "); + console.log(` ${c.dim("Dropped:")} ${droppedTotal} (${breakdown})`); + } console.log(` ${c.dim("Sections:")} ${result.tokens.sections.length}`); console.log( ` ${c.dim("Fonts:")} ${result.tokens.fonts