diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index 4c6ac21f39..830523ee0f 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -333,8 +333,8 @@ jobs: # apps/data emits index.json, manifest.json and the manifest/ shards # into suite/results/, the single results tree. It never re-emits the # published date directories (decision D2) and never writes an - # index.html -- the published index.html is the live site at - # benchmarks.vlt.sh and survives via keep_files. + # index.html -- the published index.html is a live page at + # benchmarks-data.vlt.io (the gh-pages CNAME) and survives via keep_files. run: | vlt install vlt run --workspace=apps/data build diff --git a/apps/data/src/client.ts b/apps/data/src/client.ts index 3aabc9a515..da194c82bf 100644 --- a/apps/data/src/client.ts +++ b/apps/data/src/client.ts @@ -37,7 +37,7 @@ export type FetchLike = (url: string) => Promise<{ }>; export interface DataClientOptions { - /** Base URL of the served tree, e.g. `https://benchmarks.vlt.sh`. */ + /** Base URL of the served tree, e.g. `https://benchmarks-data.vlt.io`. */ baseUrl: string; /** Injectable for tests; defaults to global fetch. */ fetch?: FetchLike; diff --git a/apps/data/src/index.ts b/apps/data/src/index.ts index 0f8813f979..fe84553d83 100644 --- a/apps/data/src/index.ts +++ b/apps/data/src/index.ts @@ -28,4 +28,13 @@ export { type ParsedDataFile, } from "./paths.ts"; export { scanRoots } from "./scan.ts"; -export { buildSummary, type DateSummary, type SummaryResult } from "./summary.ts"; +export { + buildSummary, + SUMMARY_PACKAGE_COUNTS, + SUMMARY_VERSIONS, + summaryPackageCounts, + summaryVersions, + type DateSummary, + type SummaryCounts, + type SummaryResult, +} from "./summary.ts"; diff --git a/apps/data/src/serve.ts b/apps/data/src/serve.ts index ccc0b69889..4218b4612b 100644 --- a/apps/data/src/serve.ts +++ b/apps/data/src/serve.ts @@ -2,9 +2,9 @@ * Minimal static server for the assembled tree. * * Deliberately dependency-free (`node:http` only) and deliberately does not - * serve a directory index. The published root's `index.html` IS the live site - * at benchmarks.vlt.sh, and this app must never be the thing that puts an - * index page there. + * serve a directory index. The published root's `index.html` is a live page + * at benchmarks-data.vlt.io (the gh-pages CNAME), and this app must never be + * the thing that puts an index page there. */ import { createServer, type Server } from "node:http"; import { createReadStream, statSync } from "node:fs"; diff --git a/apps/data/src/summary.ts b/apps/data/src/summary.ts index aca48b66ba..119a029699 100644 --- a/apps/data/src/summary.ts +++ b/apps/data/src/summary.ts @@ -9,10 +9,16 @@ * Not `chart-data.json`. That file gives a failed run a time, and nothing in * it says the run failed. Every result here keeps its `exit_codes`, so a * consumer applies the same rule to it as to the file it came from. + * + * The date's package counts and `versions.json` ride along, + * verbatim, under reserved `#` keys beside the variations -- so a summary + * written before they existed is still a valid one, and a reader that only + * looks up variations never sees them. Each is optional: absent means "not in + * this summary", and the reader falls back to the file; it never means zero. */ -import type { BenchmarkResult } from "@benchmarks/types"; +import type { BenchmarkResult, PackageCount, Versions } from "@benchmarks/types"; import type { DirectoryListing } from "./manifest.ts"; -import { parseDataFile } from "./paths.ts"; +import { parseDataFile, VERSIONS_FILE } from "./paths.ts"; /** What a trend needs of a result, exactly as the benchmark file has it. */ export type SummaryResult = Pick; @@ -22,9 +28,42 @@ export type DateSummary = { [variation: string]: { [fixture: string]: SummaryResult[] | undefined } | undefined; }; +/** Variation, then fixture, to that count file as published. */ +export type SummaryCounts = { + [variation: string]: { [fixture: string]: T | undefined } | undefined; +}; + +export const SUMMARY_PACKAGE_COUNTS = "#packageCounts"; +export const SUMMARY_VERSIONS = "#versions"; + +/* The reserved keys, read without letting them widen every variation's type. */ +const extra = (summary: DateSummary, key: string): unknown => + (summary as { readonly [key: string]: unknown })[key]; + +/** This summary's package counts, or `undefined` when it does not carry them. */ +export const summaryPackageCounts = ( + summary: DateSummary, +): SummaryCounts | undefined => + extra(summary, SUMMARY_PACKAGE_COUNTS) as SummaryCounts | undefined; + +/** The date's `versions.json`, or `undefined` when the summary does not carry it. */ +export const summaryVersions = (summary: DateSummary): Versions | undefined => + extra(summary, SUMMARY_VERSIONS) as Versions | undefined; + +const isObject = (value: unknown): value is object => + typeof value === "object" && value !== null && !Array.isArray(value); + +const parse = (text: string): unknown => { + try { + return JSON.parse(text); + } catch { + return undefined; + } +}; + /** * Summarise one date directory. `read` returns a file's text; a file that - * does not parse as a benchmark file is reported in `skipped`, not guessed at. + * does not parse as what its name says is reported in `skipped`, not guessed at. */ export const buildSummary = ( listing: DirectoryListing, @@ -32,16 +71,33 @@ export const buildSummary = ( ): { summary: DateSummary; skipped: string[] } => { // A variation is whatever the file name says, `__proto__` included. const summary: DateSummary = Object.create(null); + const packageCounts: SummaryCounts = Object.create(null); const skipped: string[] = []; for (const file of listing.files) { + if (file === VERSIONS_FILE) { + const versions = parse(read(file)); + if (isObject(versions)) { + (summary as { [key: string]: unknown })[SUMMARY_VERSIONS] = versions; + } else skipped.push(file); + continue; + } const parsed = parseDataFile(file); - if (parsed?.kind !== "benchmark") continue; - let results: unknown; - try { - results = (JSON.parse(read(file)) as { results?: unknown }).results; - } catch { - results = undefined; + if (parsed === undefined) continue; + // A variation named like a reserved key would be read as one. + if (parsed.variation.startsWith("#")) { + skipped.push(file); + continue; + } + // Process counts stay out until something reads them from here. + if (parsed.kind === "process-count") continue; + const content = parse(read(file)); + if (parsed.kind === "package-count") { + if (isObject(content)) { + (packageCounts[parsed.variation] ??= {})[parsed.fixture] = content as PackageCount; + } else skipped.push(file); + continue; } + const results = isObject(content) ? (content as { results?: unknown }).results : undefined; if (!Array.isArray(results) || !results.every((r) => typeof r === "object" && r !== null)) { skipped.push(file); continue; @@ -50,5 +106,8 @@ export const buildSummary = ( ({ command, mean, stddev, exit_codes }) => ({ command, mean, stddev, exit_codes }), ); } + if (Object.keys(packageCounts).length > 0) { + (summary as { [key: string]: unknown })[SUMMARY_PACKAGE_COUNTS] = packageCounts; + } return { summary, skipped }; }; diff --git a/apps/data/test/served-tree.test.ts b/apps/data/test/served-tree.test.ts index df633fe73c..8082585abc 100644 --- a/apps/data/test/served-tree.test.ts +++ b/apps/data/test/served-tree.test.ts @@ -72,7 +72,7 @@ describe.skipIf(skip)("served tree, built from real gh-pages data", () => { mkdirSync(join(root, "results-astro-clean"), { recursive: true }); writeFileSync(join(root, "results-astro-clean", "benchmarks.json"), "{}\n"); writeFileSync(join(root, "index.html"), "live site\n"); - writeFileSync(join(root, "CNAME"), "benchmarks.vlt.sh\n"); + writeFileSync(join(root, "CNAME"), "benchmarks-data.vlt.io\n"); // Run the REAL CLI rather than reimplementing what it does. An earlier // version of this fixture built the index and shards inline, which meant a diff --git a/apps/data/test/summary.test.ts b/apps/data/test/summary.test.ts index 2d2504469d..76643d51d3 100644 --- a/apps/data/test/summary.test.ts +++ b/apps/data/test/summary.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { buildSummary } from "../src/summary.ts"; +import { buildSummary, summaryPackageCounts, summaryVersions } from "../src/summary.ts"; describe("buildSummary", () => { it("lifts each benchmark file's results verbatim, exit codes and all", () => { @@ -30,8 +30,69 @@ describe("buildSummary", () => { }, "registry-clean": { next: [] }, ["__proto__"]: { next: [] }, + "#packageCounts": { clean: { next: {} } }, }); expect(Object.prototype).not.toHaveProperty("next"); expect(skipped).toEqual(["astro-clean.json", "vue-clean.json"]); }); + + it("carries the date's counts and versions verbatim, beside the variations", () => { + const counts = { npm: { count: 110, minCount: 108, maxCount: 112 }, bun: { count: 100 } }; + const processes = { npm: { count: 4 } }; + const versions = { npm: "11.0.0", vlt: "1.1.0" }; + const files: { [name: string]: string } = { + "next-clean.json": JSON.stringify({ + results: [{ command: "npm", mean: 2, stddev: 0, exit_codes: [0] }], + }), + "next-clean-package-count.json": JSON.stringify(counts), + "next-clean-process-count.json": JSON.stringify(processes), + "next-registry-clean-package-count.json": JSON.stringify({ npm: { count: 90 } }), + "astro-clean-package-count.json": "not json", + "versions.json": JSON.stringify(versions), + }; + const { summary, skipped } = buildSummary( + { name: "2026-10-19", files: Object.keys(files) }, + (file) => files[file] ?? "", + ); + expect(summaryPackageCounts(summary)).toEqual({ + clean: { next: counts }, + "registry-clean": { next: { npm: { count: 90 } } }, + }); + // Nothing reads process counts from a summary, so it does not carry them. + expect( + Object.keys(summary) + .filter((k) => k.startsWith("#")) + .sort(), + ).toEqual(["#packageCounts", "#versions"]); + expect(summaryVersions(summary)).toEqual(versions); + // A reader that looks up variations sees exactly what it did before. + expect(summary["clean"]).toEqual({ + next: [{ command: "npm", mean: 2, stddev: 0, exit_codes: [0] }], + }); + expect(skipped).toEqual(["astro-clean-package-count.json"]); + }); + + it("leaves out what the date did not publish, rather than writing it empty", () => { + const { summary } = buildSummary({ name: "2026-10-19", files: ["next-run.json"] }, () => + JSON.stringify({ results: [] }), + ); + expect(summary).toEqual({ run: { next: [] } }); + expect(summaryPackageCounts(summary)).toBeUndefined(); + expect(summaryVersions(summary)).toBeUndefined(); + }); + + it("an old summary, written before counts and versions, reads as carrying none", () => { + const old = JSON.parse(JSON.stringify({ clean: { next: [] } })); + expect(summaryPackageCounts(old)).toBeUndefined(); + expect(summaryVersions(old)).toBeUndefined(); + }); + + it("will not let a variation pose as a reserved key", () => { + const { summary, skipped } = buildSummary( + { name: "2026-10-19", files: ["next-#versions.json"] }, + () => JSON.stringify({ results: [] }), + ); + expect(summaryVersions(summary)).toBeUndefined(); + expect(skipped).toEqual(["next-#versions.json"]); + }); }); diff --git a/apps/www/.env.example b/apps/www/.env.example index 6f3143c2ff..0f054a7c68 100644 --- a/apps/www/.env.example +++ b/apps/www/.env.example @@ -7,8 +7,9 @@ # Local development, after `vlt run --workspace=apps/data build`: # vlt run --workspace=apps/data start # serves results/ on :8811 # -# Production is the gh-pages origin, which serves the same tree: -# NEXT_PUBLIC_DATA_URL=https://benchmarks.vlt.sh +# Production is the gh-pages origin (its CNAME), which serves the same tree. +# Not benchmarks.vlt.sh: that is the site, and it answers HTML for any path. +# NEXT_PUBLIC_DATA_URL=https://benchmarks-data.vlt.io NEXT_PUBLIC_DATA_URL=http://localhost:8811 # Optional server-side runtime override, checked first when set. diff --git a/apps/www/.env.local b/apps/www/.env.local index 6f3143c2ff..0f054a7c68 100644 --- a/apps/www/.env.local +++ b/apps/www/.env.local @@ -7,8 +7,9 @@ # Local development, after `vlt run --workspace=apps/data build`: # vlt run --workspace=apps/data start # serves results/ on :8811 # -# Production is the gh-pages origin, which serves the same tree: -# NEXT_PUBLIC_DATA_URL=https://benchmarks.vlt.sh +# Production is the gh-pages origin (its CNAME), which serves the same tree. +# Not benchmarks.vlt.sh: that is the site, and it answers HTML for any path. +# NEXT_PUBLIC_DATA_URL=https://benchmarks-data.vlt.io NEXT_PUBLIC_DATA_URL=http://localhost:8811 # Optional server-side runtime override, checked first when set. diff --git a/apps/www/src/app/[section]/[[...params]]/page.tsx b/apps/www/src/app/[section]/[[...params]]/page.tsx index 6d0b8c227e..46100c0233 100644 --- a/apps/www/src/app/[section]/[[...params]]/page.tsx +++ b/apps/www/src/app/[section]/[[...params]]/page.tsx @@ -37,7 +37,8 @@ import { formatDuration, longDate } from "@/lib/findings"; import { fixtureBars } from "@/lib/figures"; import { pageMetadata } from "@/lib/metadata"; import { ogFigure } from "@/lib/og-figure"; -import { loadReportParams, reportHref, reportParams } from "@/lib/params"; +import { toTrendColumns } from "@/lib/chart-model"; +import { loadReportParams, rangeLabel, reportHref, reportParams } from "@/lib/params"; import { loadCatalog } from "@/lib/catalog"; import { dataClient } from "@/lib/data"; import { loadReport } from "@/lib/report"; @@ -196,14 +197,17 @@ const Report = async ({ const timeBars = report.timeBars; const perPackageBars = report.perPackageBars; - /* What the chart needs of each run besides its plotted means. */ - const trendDetail = trend.points.map( - ({ - byTool: _byTool, - perPackage: _perPackage, - perPackageByFixture: _perPackageByFixture, - ...rest - }) => rest, + /* What the chart needs of each run besides its plotted means, by column: one + object, which both trend islands get and the page carries once. */ + const trendDetail = toTrendColumns( + trend.points.map( + ({ + byTool: _byTool, + perPackage: _perPackage, + perPackageByFixture: _perPackageByFixture, + ...rest + }) => rest, + ), ); const fixtureCount = report.fixtures.length; const across = @@ -322,7 +326,7 @@ const Report = async ({ per mark, of which All time draws over a + * thousand. + */ +.mark-dnf, +.mark-version, +.mark-unusual { + display: inline-block; + flex: none; + width: 0.625rem; + height: 0.625rem; + background-color: currentColor; + mask: var(--mark) center / contain no-repeat; +} + +.mark-dnf { + --mark: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 24 24' fill='none' stroke='black' stroke-width='2.5' stroke-linecap='round' stroke-linejoin='round'%3E%3Ccircle cx='12' cy='12' r='10'/%3E%3Cpath d='m15 9-6 6'/%3E%3Cpath d='m9 9 6 6'/%3E%3C/svg%3E"); +} + +.mark-version { + --mark: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 24 24' fill='none' stroke='black' stroke-width='2.5' stroke-linecap='round' stroke-linejoin='round'%3E%3Cpath d='M12.586 2.586A2 2 0 0 0 11.172 2H4a2 2 0 0 0-2 2v7.172a2 2 0 0 0 .586 1.414l8.704 8.704a2.426 2.426 0 0 0 3.42 0l6.58-6.58a2.426 2.426 0 0 0 0-3.42z'/%3E%3Ccircle cx='7.5' cy='7.5' r='.5' fill='black'/%3E%3C/svg%3E"); +} + +.mark-unusual { + --mark: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 24 24' fill='none' stroke='black' stroke-width='2.5' stroke-linecap='round' stroke-linejoin='round'%3E%3Cpath d='m21.73 18-8-14a2 2 0 0 0-3.48 0l-8 14A2 2 0 0 0 4 21h16a2 2 0 0 0 1.73-3'/%3E%3Cpath d='M12 9v4'/%3E%3Cpath d='M12 17h.01'/%3E%3C/svg%3E"); +} + +/* Forced colours replace the background a mask paints with the page's own + colour, which hid the marks. There they take the system text colour, as the + inline SVG strokes they replaced did. */ +@media (forced-colors: active) { + .mark-dnf, + .mark-version, + .mark-unusual { + forced-color-adjust: none; + background-color: CanvasText; + } +} diff --git a/apps/www/src/app/llms.txt/route.ts b/apps/www/src/app/llms.txt/route.ts index 0f8d3b2cc5..5629f25738 100644 --- a/apps/www/src/app/llms.txt/route.ts +++ b/apps/www/src/app/llms.txt/route.ts @@ -17,7 +17,7 @@ Every page is available as Markdown: append \`.md\` to its URL, or request the p ## Reading the results ${READING.map((line) => `- ${line}`).join("\n")} -- Page parameters: \`date\`, \`fixture\`, \`range\` (30, 60, 90 or 120 days of trend), \`from\` and \`to\` (an exact trend window), \`hide\` (comma-separated tools), \`metric\` (total, change, baseline or per-package), \`exclude\` (\`YYYY-MM-DD\` or \`tool@YYYY-MM-DD\`, comma-separated). +- Page parameters: \`date\`, \`fixture\`, \`range\` (today, 1w, 1m, 1y or all: rolling days of trend ending at the run; default 1m), \`from\` and \`to\` (an exact trend window), \`hide\` (comma-separated tools), \`metric\` (total, change, baseline or per-package), \`exclude\` (\`YYYY-MM-DD\` or \`tool@YYYY-MM-DD\`, comma-separated). ## API and MCP diff --git a/apps/www/src/app/openapi.json/route.ts b/apps/www/src/app/openapi.json/route.ts index 494f148092..f65ae141bf 100644 --- a/apps/www/src/app/openapi.json/route.ts +++ b/apps/www/src/app/openapi.json/route.ts @@ -1,4 +1,4 @@ -import { RANGES } from "@/lib/params"; +import { LEGACY_RANGES, PRESETS } from "@/lib/params"; import { METRICS } from "@/lib/figures"; import { SITE_DESCRIPTION, siteOrigin } from "@/lib/metadata"; import { SECTIONS } from "@/lib/sections"; @@ -42,13 +42,13 @@ const viewParams = [ ), query( "range", - { type: "integer", enum: RANGES, default: 30 }, - "Trend window: this many calendar days ending at the run.", + { type: "string", enum: PRESETS.map((p) => p.value), default: "1m" }, + `Trend window in rolling calendar days ending at the run: today (the run alone), 1w (7), 1m (30), 1y (365) or all (every published run up to it). Legacy day counts ${LEGACY_RANGES.join(", ")} are still accepted.`, ), query( "from", date, - "Exact trend window start; with `to`. Both must be published dates, less than 120 days apart.", + "Exact trend window start; with `to`. Both must be published dates, in order; any span.", ), query("to", date, "Exact trend window end; no later than the run."), query( diff --git a/apps/www/src/components/charts-echarts.tsx b/apps/www/src/components/charts-echarts.tsx index 15def1fe8e..bd69e210fa 100644 --- a/apps/www/src/components/charts-echarts.tsx +++ b/apps/www/src/components/charts-echarts.tsx @@ -23,7 +23,7 @@ import { type ReactNode, } from "react"; import { ChartLegend } from "@/components/chart-legend"; -import { PinnedRun, type PinnedNote, type TrendDetail } from "@/components/pinned-run"; +import { onFixture, PinnedRun, type PinnedNote, type TrendDetail } from "@/components/pinned-run"; import { RangePicker } from "@/components/range-picker"; import { usePending } from "@/components/report-pending"; import { Callout, ChartFrame, Figure } from "@/components/report"; @@ -45,7 +45,7 @@ import { tooltipRow, tooltipShell, } from "@/components/ui/echarts-tooltip"; -import { toolColor } from "@/lib/chart-model"; +import { rowsOf, toolColor, trendOf, type DenseSeries, type TrendColumns } from "@/lib/chart-model"; import { EVIL_AXIS_LABEL, EVIL_SPLIT_LINE, EVIL_TICK_DOT, escapeHtml } from "@/lib/chart-style"; import { bridges, @@ -187,7 +187,7 @@ export interface TimeSeriesTool { export const TimeSeriesChart = ({ id, title, - rows, + series, tools, section, baseline, @@ -195,7 +195,8 @@ export const TimeSeriesChart = ({ yLabel, markers = [], versions = [], - trend = [], + trend: columns, + fixture, perPackage = true, focus = null, fixedMetric, @@ -216,8 +217,12 @@ export const TimeSeriesChart = ({ /** A tab row in place of the title tab (`FigureTabStrip`). */ tabs?: ReactNode; firstTabActive?: boolean; - /** Dense rows from `toDenseChartRows`: holes as `null`, never as zero. */ - rows: readonly { date: string; [tool: string]: string | number | null }[]; + /** + * The plotted values by column (`toDenseSeries`): holes as `null`, never as + * zero. Sent this way rather than as a row per calendar day with every + * tool's key repeated -- the lighter page -- and unpacked here, once. + */ + series: DenseSeries; /** Every tool these files contain, in ranking order -- hidden ones included. */ tools: readonly TimeSeriesTool[]; section: SectionId; @@ -233,9 +238,12 @@ export const TimeSeriesChart = ({ /** * Each run's coverage, failures, spread and per-fixture seconds (from * `loadScenario`), for the DNF marks, the band, the tooltip and the pinned - * run's panel. + * run's panel. By column (`toTrendColumns`), the lighter page, and unpacked + * here once. */ - trend?: readonly TrendDetail[]; + trend?: TrendColumns; + /** Each run as this one fixture saw it (the per-fixture chart's tab). */ + fixture?: string; /** * Whether "Per package" can be offered: there are package counts for what * is plotted. Not for task runners: nothing is installed. @@ -252,6 +260,12 @@ export const TimeSeriesChart = ({ readonly custom: boolean; }; }) => { + /* Once per series: a fixture tab remounts the chart with its own. */ + const rows = useMemo(() => rowsOf(series), [series]); + const trend: readonly TrendDetail[] = useMemo(() => { + const runs = columns === undefined ? [] : trendOf(columns); + return fixture === undefined ? runs : onFixture(runs, fixture); + }, [columns, fixture]); const { resolvedTheme } = useTheme(); /* Server writes run in the report's transitions; a tools change leaves the lines live. */ const { data: reloading, startTransition, startToolsTransition } = usePending(); @@ -1376,7 +1390,7 @@ export const TimeSeriesChart = ({ color: failed.length === 1 ? paint(failed[0] ?? "") : "var(--foreground)", }} > - + {failed.length === 1 ? null : failed.length} ))} @@ -1395,7 +1409,6 @@ export const TimeSeriesChart = ({ /> ); /* A version change (a tag) or an unusual run (a warning): an icon on the line, in its colour. */ - const Icon = kind === "ring" ? Tag : TriangleAlert; return ( - + ); })} diff --git a/apps/www/src/components/fixture-trend.tsx b/apps/www/src/components/fixture-trend.tsx index f9c2b9bd44..d15d6dd43d 100644 --- a/apps/www/src/components/fixture-trend.tsx +++ b/apps/www/src/components/fixture-trend.tsx @@ -1,36 +1,12 @@ "use client"; import { useState, type ComponentProps } from "react"; +import type { DenseSeries } from "@/lib/chart-model"; import { TimeSeriesChart } from "@/components/charts-echarts"; -import type { TrendDetail } from "@/components/pinned-run"; import { FigureTabStrip } from "@/components/tabbed-figure"; import { FixtureName } from "@/components/tool-name"; import { fixtureLabel } from "@/lib/tools"; -type Rows = ComponentProps["rows"]; - -/** - * Each run as one fixture saw it: the failures there and no others, nothing - * partial (one fixture is all or nothing), and that fixture's own spread for - * the band. - */ -const onFixture = (trend: readonly TrendDetail[], fixture: string): TrendDetail[] => - trend.map((d) => ({ - ...d, - fixtures: d.fixtures.filter((f) => f === fixture), - partial: {}, - failed: Object.fromEntries( - Object.entries(d.failed) - .map(([tool, list]) => [tool, list.filter((e) => e.fixture === fixture)] as const) - .filter(([, list]) => list.length > 0), - ), - rsd: Object.fromEntries( - Object.entries(d.rsdByFixture).flatMap(([tool, by]) => - by[fixture] === undefined ? [] : [[tool, by[fixture]]], - ), - ), - })); - /** * Per-package time on each fixture, run by run: the report's own * "Performance over time" chart, whole -- range, display, pin, brush, value @@ -47,9 +23,9 @@ export const FixtureTimeSeries = ({ ...chart }: Omit< ComponentProps, - "rows" | "yLabel" | "tabs" | "firstTabActive" | "fixedMetric" | "scope" + "series" | "yLabel" | "tabs" | "firstTabActive" | "fixedMetric" | "scope" > & { - fixtures: readonly { readonly fixture: string; readonly rows: Rows }[]; + fixtures: readonly { readonly fixture: string; readonly series: DenseSeries }[]; }) => { const [active, setActive] = useState(fixtures[0]?.fixture ?? ""); const shown = fixtures.find((f) => f.fixture === active) ?? fixtures[0]; @@ -60,8 +36,9 @@ export const FixtureTimeSeries = ({ {...chart} id={id} title={title} - rows={shown.rows} - trend={onFixture(trend ?? [], shown.fixture)} + series={shown.series} + trend={trend} + fixture={shown.fixture} yLabel={`Time per installed package on ${fixtureLabel(shown.fixture)}`} fixedMetric="per-package" scope="pp" diff --git a/apps/www/src/components/pinned-run.tsx b/apps/www/src/components/pinned-run.tsx index ebd5a84e5d..a6a062aac8 100644 --- a/apps/www/src/components/pinned-run.tsx +++ b/apps/www/src/components/pinned-run.tsx @@ -14,6 +14,28 @@ import type { SectionId } from "@/lib/sections"; export type TrendDetail = Omit; +/** + * Each run as one fixture saw it: the failures there and no others, nothing + * partial (one fixture is all or nothing), and that fixture's own spread for + * the band. + */ +export const onFixture = (trend: readonly TrendDetail[], fixture: string): TrendDetail[] => + trend.map((d) => ({ + ...d, + fixtures: d.fixtures.filter((f) => f === fixture), + partial: {}, + failed: Object.fromEntries( + Object.entries(d.failed) + .map(([tool, list]) => [tool, list.filter((e) => e.fixture === fixture)] as const) + .filter(([, list]) => list.length > 0), + ), + rsd: Object.fromEntries( + Object.entries(d.rsdByFixture).flatMap(([tool, by]) => + by[fixture] === undefined ? [] : [[tool, by[fixture]]], + ), + ), + })); + /** One tool on the pinned run, everything the panel says about it. */ export interface PinnedTool { readonly tool: string; diff --git a/apps/www/src/components/range-picker.tsx b/apps/www/src/components/range-picker.tsx index 856e3d87f2..39e10982fa 100644 --- a/apps/www/src/components/range-picker.tsx +++ b/apps/www/src/components/range-picker.tsx @@ -1,14 +1,15 @@ "use client"; -import { CalendarDays, ChevronDown } from "lucide-react"; +import { ArrowRight, CalendarDays, ChevronDown } from "lucide-react"; import { useQueryStates } from "nuqs"; -import { useState } from "react"; +import { useState, useSyncExternalStore, useTransition } from "react"; import type { DateRange } from "react-day-picker"; import { usePending } from "@/components/report-pending"; import { Button } from "@/components/ui/button"; import { Calendar } from "@/components/ui/calendar"; import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover"; -import { MAX_WINDOW_DAYS, RANGES, reportParams } from "@/lib/params"; +import { PRESETS, rangeLabel, reportParams } from "@/lib/params"; +import { cn } from "@/lib/utils"; /* * Dates here are calendar days with no time and no zone. They are built and @@ -22,21 +23,57 @@ const toDate = (iso: string): Date => { const toIso = (date: Date): string => `${date.getFullYear()}-${String(date.getMonth() + 1).padStart(2, "0")}-${String(date.getDate()).padStart(2, "0")}`; +const short = (d: Date, year: boolean) => + d.toLocaleDateString("en-US", { + month: "short", + day: "numeric", + ...(year ? { year: "numeric" } : {}), + }); + const words = (from: string, to: string): string => { const a = toDate(from); const b = toDate(to); - const short = (d: Date, year: boolean) => - d.toLocaleDateString("en-US", { - month: "short", - day: "numeric", - ...(year ? { year: "numeric" } : {}), - }); + if (from === to) return short(b, true); return `${short(a, a.getFullYear() !== b.getFullYear())} – ${short(b, true)}`; }; +/* Two months side by side from `sm`, one below it. Only read once the + popover is open, which is always on the client, so the server's answer + never reaches the page. */ +const SM = "(min-width: 40rem)"; +const subscribe = (onChange: () => void) => { + const mql = window.matchMedia(SM); + mql.addEventListener("change", onChange); + return () => mql.removeEventListener("change", onChange); +}; +const useWide = () => + useSyncExternalStore( + subscribe, + () => window.matchMedia(SM).matches, + () => true, + ); + +/** The month a calendar of `months` opens on, so that `to`'s month is the last one shown. */ +const openingMonth = (to: string, first: string, months: number): Date => { + const end = toDate(to); + const open = new Date(end.getFullYear(), end.getMonth() - (months - 1), 1); + const start = toDate(first); + return open < start ? new Date(start.getFullYear(), start.getMonth(), 1) : open; +}; + /** * The trend's date range: presets, or an exact start and end from a calendar. * + * PRESETS apply at once. They are rolling days ending at the run shown + * (`lib/params.ts`), and "1 month" is the default, so choosing it clears the + * parameter. An old `range=90` link is labelled "Last 90 days" and highlights + * no preset. + * + * THE CALENDAR is a draft: picking days changes nothing until Apply, and + * Escape or a click outside throws the draft away. It reaches across the + * whole archive -- there is no longest window -- so its caption is a month + * and a year dropdown rather than a long walk of "previous". + * * ONLY PUBLISHED DATES CAN BE CHOSEN. Every other day is disabled -- days * after the run being shown, days before the archive begins, and the days in * between on which nothing ran -- so a range cannot start or end on a hole, @@ -45,10 +82,12 @@ const words = (from: string, to: string): string => { * one. * * This is a server parameter: the window decides which files are fetched. So - * the write is not shallow, and the page re-renders with the new data. + * the write is not shallow, and it runs in the report's pending transition, + * which keeps the chart on screen, marked stale, until the new data arrives. * * The calendar is react-day-picker's, with its keyboard and screen-reader - * behaviour as shipped. + * behaviour as shipped. The popover is Radix's: it traps focus, closes on + * Escape and returns focus to the trigger. */ export const RangePicker = ({ dates, @@ -67,26 +106,30 @@ export const RangePicker = ({ custom: boolean; }) => { const { startTransition } = usePending(); + /* The range's own write, inside the report's: the report goes stale for + any server parameter, but only this one makes the URL's window news. */ + const [ranging, startRanging] = useTransition(); const [params, setParams] = useQueryStates( { range: reportParams.range, from: reportParams.from, to: reportParams.to }, - { shallow: false, startTransition }, + { shallow: false, startTransition: (write) => startTransition(() => startRanging(write)) }, ); const [open, setOpen] = useState(false); - const [draft, setDraft] = useState({ - from: toDate(from), - to: toDate(to), - }); + const [draft, setDraft] = useState(); - const published = new Set(dates.filter((d) => d <= end)); - const first = dates[0] ?? end; - const ready = draft?.from !== undefined && draft.to !== undefined; + /* While the server re-renders a new window, the URL's choice is shown at + once; otherwise the server's, which is the truth when it refused a + hand-edited range. */ + const asked = ranging && params.from !== null && params.to !== null; + const exact = ranging ? asked : custom; + const [start, stop] = asked ? [params.from ?? from, params.to ?? to] : [from, to]; + const label = exact ? words(start, stop) : rangeLabel(params.range); return ( { setOpen(next); - if (next) setDraft({ from: toDate(from), to: toDate(to) }); + if (next) setDraft({ from: toDate(start), to: toDate(stop) }); }} > @@ -96,65 +139,150 @@ export const RangePicker = ({ - -
- {RANGES.map((days) => { - const selected = !custom && params.range === days; + { + const target = event.currentTarget as HTMLElement; + const first = + target.querySelector("[aria-pressed=true]") ?? + target.querySelector("[data-slot=calendar] td button[tabindex='0']"); + if (first === null) return; + event.preventDefault(); + first.focus(); + }} + /* On a phone it can be taller than the room on either side of the trigger; it scrolls rather than leave the screen. */ + className="flex max-h-(--radix-popover-content-available-height) w-auto max-w-[calc(100vw-2rem)] flex-col gap-0 overflow-y-auto p-0 sm:flex-row" + > +
+ {PRESETS.map((preset) => { + const selected = !exact && params.range === preset.value; return ( ); })}
-
- !published.has(toIso(date))} - /> -
-

- Days with no published run cannot be chosen. Up to {MAX_WINDOW_DAYS} days. -

- -
-
+ { + void setParams(range); + setOpen(false); + }} + />
); }; + +/** The calendar half: a draft range, its readout, and Apply. */ +const Custom = ({ + dates, + end, + to, + draft, + onDraft, + onApply, +}: { + dates: readonly string[]; + end: string; + to: string; + draft: DateRange | undefined; + onDraft: (range: DateRange | undefined) => void; + onApply: (range: { from: string; to: string }) => void; +}) => { + const months = useWide() ? 2 : 1; + const shown = dates.filter((d) => d <= end); + const published = new Set(shown); + const first = shown[0] ?? end; + + const a = draft?.from === undefined ? undefined : toIso(draft.from); + // A range is complete with both ends; one picked day is a range of one. + const b = draft?.to === undefined ? undefined : toIso(draft.to); + const runs = a === undefined ? 0 : shown.filter((d) => d >= a && d <= (b ?? a)).length; + + return ( +
+ onDraft(range ?? { from: day, to: undefined })} + resetOnSelect + numberOfMonths={months} + captionLayout="dropdown" + defaultMonth={openingMonth(to, first, months)} + startMonth={toDate(first)} + endMonth={toDate(end)} + disabled={(date) => !published.has(toIso(date))} + className="mx-auto" + /> +
+ {/* What Apply will show, read out as it changes. */} +

+ {a === undefined ? ( + Pick a start and an end + ) : ( + <> + Selected: + {short(toDate(a), b === undefined || a.slice(0, 4) !== b.slice(0, 4))} + + to + + {b === undefined ? "pick an end" : short(toDate(b), true)} + + + · {runs} {runs === 1 ? "run" : "runs"} + + + )} +

+ +
+
+ ); +}; diff --git a/apps/www/src/lib/api.ts b/apps/www/src/lib/api.ts index 7851a37ceb..40d7cc8abd 100644 --- a/apps/www/src/lib/api.ts +++ b/apps/www/src/lib/api.ts @@ -4,9 +4,9 @@ * it, and say what would have worked. */ import type { DataClient } from "@benchmarks/data"; +import { rowsOf } from "./chart-model"; import { dataClient } from "./data"; import { metricRows } from "./figures"; -import { MAX_WINDOW_DAYS } from "./params"; import { ignoredSelection, loadReport, type ReportModel, type ReportQuery } from "./report"; import { SECTIONS } from "./sections"; @@ -91,7 +91,7 @@ export const strictReport = async ({ status: 400, code: "invalid_window", message: `The window ${query.from ?? "?"} to ${query.to ?? "?"} cannot be used.`, - hint: `from and to must both be published dates, from no later than to, to no later than the run shown, and less than ${MAX_WINDOW_DAYS} days apart. /api/v1/runs lists the published dates.`, + hint: `from and to must both be published dates, from no later than to, and to no later than the run shown. /api/v1/runs lists the published dates.`, }, }; } @@ -127,8 +127,8 @@ export const historyOf = (report: ReportModel, tools?: readonly string[]) => { window: report.filters.window, excluded: report.filters.exclude, tools: chosen, - /* One row per calendar day; `null` is a day with no value, never a zero. */ - rows: metricRows(report.trend.rows, all, plotted, report.baseline).map((row) => ({ + /* One row per calendar day (per run past MAX_DENSE_DAYS); `null` is no value, never a zero. */ + rows: metricRows(rowsOf(report.trend.series), all, plotted, report.baseline).map((row) => ({ date: row.date, ...pick(row), })), diff --git a/apps/www/src/lib/chart-model.ts b/apps/www/src/lib/chart-model.ts index 1e5d1146c9..f603464637 100644 --- a/apps/www/src/lib/chart-model.ts +++ b/apps/www/src/lib/chart-model.ts @@ -162,8 +162,12 @@ export const spreadOf = ( const DAY = 86_400_000; -/** Cap on densification, so a two-year window cannot produce 700 rows. */ -export const MAX_DENSE_DAYS = 120; +/** + * Cap on densification: about three years, so the whole archive (440 days and + * growing) is spaced by calendar time on every window, All time included. A + * row per day is cheap at this size; past it the fallback below takes over. + */ +export const MAX_DENSE_DAYS = 1100; /** * Rows covering every calendar day between the first and last published run. @@ -216,6 +220,155 @@ export const toDenseChartRows = ( return { rows, dense: true, spanDays }; }; +/** One row of a trend chart: a date and each tool's value, `null` for a hole. */ +export type ChartRow = { date: string; [tool: string]: string | number | null }; + +/** + * Dense rows, by column: what the server sends a chart island instead of one + * object per calendar day with every tool's key repeated. Lossless -- the + * values are the rows' own, unrounded -- and `rowsOf` turns it back into + * exactly those rows, so the charts' internals never see the difference. + */ +export interface DenseSeries { + /** First calendar day, YYYY-MM-DD. Index i is start + i days. */ + readonly start: string; + /** Calendar days covered; every array below has this length. */ + readonly length: number; + /** False past MAX_DENSE_DAYS: then `dates` lists each index's day (the sparse fallback). */ + readonly dense: boolean; + readonly dates?: readonly string[]; + /** One column per tool, null for a hole (never 0). */ + readonly values: { readonly [tool: string]: readonly (number | null)[] }; +} + +/** `toDenseChartRows`, by column. */ +export const toDenseSeries = ( + points: readonly Plotted[], + tools: readonly string[], +): DenseSeries => { + const { rows, dense } = toDenseChartRows(points, tools); + /* Dense rows are consecutive days, so their dates are implied. */ + const implied = dense && rows.length > 0; + return { + start: rows[0]?.date ?? "", + length: rows.length, + dense: implied, + ...(implied ? {} : { dates: rows.map((r) => r.date) }), + values: Object.fromEntries( + tools.map((tool) => [tool, rows.map((r) => (typeof r[tool] === "number" ? r[tool] : null))]), + ), + }; +}; + +/** The rows a `DenseSeries` encodes: `toDenseChartRows(points, tools).rows`, exactly. */ +export const rowsOf = (series: DenseSeries): ChartRow[] => { + const first = Date.parse(`${series.start}T00:00:00Z`); + const columns = Object.entries(series.values); + return Array.from({ length: series.length }, (_, i) => { + const date = series.dense + ? new Date(first + i * DAY).toISOString().slice(0, 10) + : (series.dates?.[i] ?? ""); + const row: ChartRow = { date }; + for (const [tool, column] of columns) row[tool] = column[i] ?? null; + return row; + }); +}; + +/** What a chart island needs of each run besides its plotted values. */ +export type TrendRun = Pick< + TrendPoint, + "date" | "fixtures" | "partial" | "failed" | "byFixture" | "rsd" | "rsdByFixture" +>; + +type Column = readonly (number | null)[]; + +/** + * Each run's detail, by column: what the server sends instead of one object + * per run with every tool and fixture key repeated. The per-fixture seconds + * and spreads are most of a long trend's bytes, so they become one column per + * (tool, fixture) over the runs, `null` where that run has no value. Partial + * and failed are sparse and stay per run. Lossless -- nothing is rounded -- + * and `trendOf` gives back the runs. + */ +export interface TrendColumns { + readonly dates: readonly string[]; + /** Every fixture any run published, in first-seen order. */ + readonly fixtures: readonly string[]; + /** Which of `fixtures` each run published, by index, in the run's own order. */ + readonly published: readonly (readonly number[])[]; + readonly partial: readonly TrendRun["partial"][]; + readonly failed: readonly TrendRun["failed"][]; + readonly rsd: { readonly [tool: string]: Column }; + readonly byFixture: { readonly [tool: string]: { readonly [fixture: string]: Column } }; + readonly rsdByFixture: { readonly [tool: string]: { readonly [fixture: string]: Column } }; +} + +/** The runs' detail by column. */ +export const toTrendColumns = (runs: readonly TrendRun[]): TrendColumns => { + const fixtures = [...new Set(runs.flatMap((r) => r.fixtures))]; + const nested = ( + pick: (r: TrendRun) => { readonly [tool: string]: { readonly [fixture: string]: number } }, + ) => { + const out: { [tool: string]: { [fixture: string]: (number | null)[] } } = {}; + runs.forEach((run, i) => { + for (const [tool, byFixture] of Object.entries(pick(run))) { + for (const [fixture, value] of Object.entries(byFixture)) { + ((out[tool] ??= {})[fixture] ??= Array.from({ length: runs.length }, () => null))[i] = + value; + } + } + }); + return out; + }; + const rsd: { [tool: string]: (number | null)[] } = {}; + runs.forEach((run, i) => { + for (const [tool, value] of Object.entries(run.rsd)) { + (rsd[tool] ??= Array.from({ length: runs.length }, () => null))[i] = value; + } + }); + return { + dates: runs.map((r) => r.date), + fixtures, + published: runs.map((r) => r.fixtures.map((f) => fixtures.indexOf(f))), + partial: runs.map((r) => r.partial), + failed: runs.map((r) => r.failed), + rsd, + byFixture: nested((r) => r.byFixture), + rsdByFixture: nested((r) => r.rsdByFixture), + }; +}; + +/** The runs a `TrendColumns` encodes: a tool or fixture with no value on a run is absent, as it was. */ +export const trendOf = (columns: TrendColumns): TrendRun[] => { + const unnest = ( + by: TrendColumns["byFixture"], + i: number, + ): { [tool: string]: { [fixture: string]: number } } => { + const out: { [tool: string]: { [fixture: string]: number } } = {}; + for (const [tool, byFixture] of Object.entries(by)) { + for (const [fixture, column] of Object.entries(byFixture)) { + const value = column[i]; + if (typeof value === "number") (out[tool] ??= {})[fixture] = value; + } + } + return out; + }; + return columns.dates.map((date, i) => ({ + date, + fixtures: (columns.published[i] ?? []).map((f) => columns.fixtures[f] ?? ""), + partial: columns.partial[i] ?? {}, + failed: columns.failed[i] ?? {}, + byFixture: unnest(columns.byFixture, i), + rsd: Object.fromEntries( + Object.entries(columns.rsd).flatMap(([tool, column]) => { + const value = column[i]; + return typeof value === "number" ? [[tool, value]] : []; + }), + ), + rsdByFixture: unnest(columns.rsdByFixture, i), + })); +}; + /* ------------------------------------------------------------------ */ /* Presentation tokens shared by every chart */ /* ------------------------------------------------------------------ */ diff --git a/apps/www/src/lib/load.ts b/apps/www/src/lib/load.ts index 37ef5a4d4e..8b868f572e 100644 --- a/apps/www/src/lib/load.ts +++ b/apps/www/src/lib/load.ts @@ -8,7 +8,12 @@ * and it gives a failed run a time. The files are the measurement. */ import type { BenchmarkFile, Fixture, PackageCount, ProcessCount } from "@benchmarks/types"; -import { DataClient, type DateSummary } from "@benchmarks/data"; +import { + DataClient, + summaryPackageCounts, + type DateSummary, + type SummaryResult, +} from "@benchmarks/data"; import { probe } from "./data"; import { applyExclusions } from "./figures"; import { averageFiles, usable } from "./measurements"; @@ -54,8 +59,16 @@ export interface TrendPoint { export interface ScenarioData { readonly bundles: readonly FixtureBundle[]; readonly trend: readonly TrendPoint[]; - /** What the trend actually cost, so the number is reported rather than guessed. */ - readonly trendCost: { readonly requests: number; readonly dates: number }; + /** + * What the trend actually cost, so the number is reported rather than + * guessed: its requests, the runs it drew, and what a run without a summary + * costs, which is what cut the window when `dates` is short of it. + */ + readonly trendCost: { + readonly requests: number; + readonly dates: number; + readonly perRun: number; + }; } const asFixture = (name: string): Fixture => name as Fixture; @@ -172,12 +185,31 @@ export const trendPoint = ( }; }; +/* What a summary leaves out of a result; nothing on a trend reads it. */ +const unsummarised = { + median: Number.NaN, + min: Number.NaN, + max: Number.NaN, + user: Number.NaN, + system: Number.NaN, + times: [], +}; + +/** A summarised result as a benchmark file's: the fields a trend reads, verbatim. */ +const fromSummary = (results: readonly SummaryResult[]): BenchmarkFile => ({ + results: results.map((r) => ({ ...r, ...unsummarised })), +}); + /** * The trend series behind the sparklines and the time chart. * - * Cost is `dates x fixtures` requests, doubled when package counts are asked - * for; `trendCost` reports the real number. Dates the scenario did not run on - * contribute no point, which is why the series can be shorter than the window. + * A date with a summary (from `loadSummaries`) reads its files out of it: the + * same mean, stddev and exit codes, so the same DNF rule, for one fetch already + * made. A date without costs one request per fixture. Package counts, when + * asked for, come from the summary too where it carries that fixture's, and + * otherwise cost one request per fixture that ran; `requests` counts what this function + * fetched. Dates the scenario did not run on contribute no + * point, which is why the series can be shorter than the window. */ const loadTrend = async ( client: DataClient, @@ -185,91 +217,132 @@ const loadTrend = async ( fixtures: readonly string[], variation: string, withCounts: boolean, + summaries: ReadonlyMap = new Map(), ): Promise<{ trend: TrendPoint[]; requests: number }> => { + let requests = 0; const perDate = await Promise.all( dates.map(async (date) => { - const loaded = await Promise.all( + const summary = summaries.get(date); + const files = await Promise.all( fixtures.map(async (fixture) => { - const [file, counts] = await Promise.all([ - probe(() => client.getBenchmark(date, asFixture(fixture), variation)), - withCounts - ? probe(() => - client.getPackageCount(date, asFixture(fixture), variation), - ) - : null, - ]); - return { - fixture, - file: file.state === "present" ? file.value : null, - counts: counts?.state === "present" ? counts.value : null, - }; + if (summary !== undefined) { + const results = summary[variation]?.[fixture]; + return { fixture, file: results === undefined ? null : fromSummary(results) }; + } + requests++; + const file = await probe(() => + client.getBenchmark(date, asFixture(fixture), variation), + ); + return { fixture, file: file.state === "present" ? file.value : null }; }), ); - return trendPoint( - date, - loaded.flatMap((l) => (l.file === null ? [] : [{ fixture: l.fixture, file: l.file }])), - loaded.flatMap((l) => - l.counts === null ? [] : [{ fixture: l.fixture, counts: l.counts }], - ), + const ran = files.flatMap((f) => + f.file === null ? [] : [{ fixture: f.fixture, file: f.file }], ); + /* + * A fixture the summary has no count for falls back to its file: the + * build skips a count file it cannot parse, and absent is never zero. + */ + const summarised = summary === undefined ? undefined : summaryPackageCounts(summary); + const counts = withCounts + ? await Promise.all( + ran.map(async ({ fixture }) => { + const carried = summarised?.[variation]?.[fixture]; + if (carried !== undefined) return [{ fixture, counts: carried }]; + requests++; + const c = await probe(() => + client.getPackageCount(date, asFixture(fixture), variation), + ); + return c.state === "present" ? [{ fixture, counts: c.value }] : []; + }), + ) + : []; + return trendPoint(date, ran, counts.flat()); }), ); - return { - trend: perDate.filter((p): p is TrendPoint => p !== null), - requests: dates.length * fixtures.length * (withCounts ? 2 : 1), - }; + return { trend: perDate.filter((p): p is TrendPoint => p !== null), requests }; }; +/** + * The shown run and its trend. `summaries` may still be loading: the run's + * own files do not wait for it, and only the trend does, since which runs it + * can afford depends on which have one. + */ export const loadScenario = async ( client: DataClient, date: string, variation: string, fixtures: readonly string[], - trendDates: readonly string[], - options: { readonly exclude?: readonly string[]; readonly perPackage?: boolean } = {}, + windowDates: readonly string[], + options: { + readonly exclude?: readonly string[]; + readonly perPackage?: boolean; + /** From `loadSummaries`: a summarised run costs one fetch, not one per fixture. */ + readonly summaries?: Summaries | Promise; + } = {}, ): Promise => { + const withCounts = options.perPackage === true; + /* A run without a summary: a file per fixture, and its count where that is wanted. */ + const perRun = fixtures.length * (withCounts ? 2 : 1); const [bundles, trend] = await Promise.all([ Promise.all(fixtures.map((f) => loadFixture(client, date, f, variation))), - loadTrend(client, trendDates, fixtures, variation, options.perPackage === true), + (async () => { + const summaries = await options.summaries; + const dates = trendDates(windowDates, perRun, new Set(summaries?.keys())); + return { + dates, + ...(await loadTrend(client, dates, fixtures, variation, withCounts, summaries)), + }; + })(), ]); return { bundles: bundles.filter((b): b is FixtureBundle => b !== null), trend: applyExclusions(trend.trend, options.exclude ?? []), - trendCost: { requests: trend.requests, dates: trendDates.length }, + trendCost: { + requests: trend.requests + (options.summaries === undefined ? 0 : windowDates.length), + dates: trend.dates.length, + perRun, + }, }; }; +/** Each date's summary, from `loadSummaries`. */ +export type Summaries = ReadonlyMap; + /** - * What the derived "average" view's trend may fetch per page. One run costs a - * file per (fixture, scenario) -- 54 for package managers -- so a 30-day - * window would be about 1,600 requests; the window is cut to its most recent - * runs instead. + * What a trend may fetch per page beyond its summaries. A run without one + * costs a request per file it reads -- for the derived "average" a file per + * (fixture, scenario), 54 for package managers, so a 30-day window would be + * about 1,600 requests -- and the window is cut to its most recent such runs + * instead. A missing or failed summary never fans out past it. */ -export const AVERAGE_TREND_BUDGET = 400; +export const TREND_BUDGET = 400; /** - * The window's dates the average's trend draws: every date with a summary, - * which costs one fetch, and of the rest the most recent whose files fit the - * budget, never fewer than two. + * The window's dates a trend draws: every date with a summary, which costs + * one fetch, and of the rest the most recent whose files fit the budget, + * never fewer than two. */ -export const averageTrendDates = ( +export const trendDates = ( dates: readonly string[], - cellsPerDate: number, + requestsPerDate: number, summarised: ReadonlySet = new Set(), ): string[] => { - const raw = dates - .filter((d) => !summarised.has(d)) - .slice(-Math.max(2, Math.floor(AVERAGE_TREND_BUDGET / Math.max(cellsPerDate, 1)))); - return dates.filter((d) => summarised.has(d) || raw.includes(d)); + const raw = new Set( + dates + .filter((d) => !summarised.has(d)) + .slice(-Math.max(2, Math.floor(TREND_BUDGET / Math.max(requestsPerDate, 1)))), + ); + return dates.filter((d) => summarised.has(d) || raw.has(d)); }; /** Each date's summary, where a deploy has written one; a date without is left out. */ export const loadSummaries = async ( client: DataClient, dates: readonly string[], -): Promise> => { +): Promise => { const found = await Promise.all(dates.map((d) => probe(() => client.getSummary(d)))); return new Map( dates.flatMap((d, i) => { @@ -286,16 +359,6 @@ export interface RunCell { readonly file: BenchmarkFile; } -/* What a summary leaves out of a result; nothing on a trend reads it. */ -const unsummarised = { - median: Number.NaN, - min: Number.NaN, - max: Number.NaN, - user: Number.NaN, - system: Number.NaN, - times: [], -}; - const loadCells = async ( client: DataClient, date: string, @@ -306,15 +369,7 @@ const loadCells = async ( return Object.entries(plan).flatMap(([fixture, variations]) => variations.flatMap((variation) => { const results = summary[variation]?.[fixture]; - return results === undefined - ? [] - : [ - { - fixture, - variation, - file: { results: results.map((r) => ({ ...r, ...unsummarised })) }, - }, - ]; + return results === undefined ? [] : [{ fixture, variation, file: fromSummary(results) }]; }), ); } @@ -341,21 +396,26 @@ export const loadAverage = async ( client: DataClient, date: string, plan: { readonly [fixture: string]: readonly string[] }, - trendDates: readonly string[], + windowDates: readonly string[], options: { readonly exclude?: readonly string[]; - /** From `loadSummaries`: a summarised run costs one fetch, not one per file. */ - readonly summaries?: ReadonlyMap; + /** From `loadSummaries`, possibly still loading, as for `loadScenario`. */ + readonly summaries?: Summaries | Promise; } = {}, ): Promise => { // The run shown is usually the trend's last date; fetched once either way, - // and from its files, which the report reads in full. - const dates = trendDates.includes(date) ? trendDates : [...trendDates, date]; - const summaryOf = (d: string) => (d === date ? undefined : options.summaries?.get(d)); - const loaded = await Promise.all(dates.map((d) => loadCells(client, d, plan, summaryOf(d)))); - const trend = trendDates.flatMap( - (d, i) => trendPoint(d, averageFiles(loaded[i] ?? [], plan)) ?? [], + // and from its files, which the report reads in full, without waiting for + // the summaries. + const shown = loadCells(client, date, plan); + const summaries = await options.summaries; + const perRun = Object.values(plan).flat().length; + const drawn = trendDates(windowDates, perRun, new Set(summaries?.keys())); + const dates = drawn.includes(date) ? drawn : [...drawn, date]; + const summaryOf = (d: string) => (d === date ? undefined : summaries?.get(d)); + const loaded = await Promise.all( + dates.map((d) => (d === date ? shown : loadCells(client, d, plan, summaryOf(d)))), ); + const trend = drawn.flatMap((d, i) => trendPoint(d, averageFiles(loaded[i] ?? [], plan)) ?? []); const cells = loaded[dates.indexOf(date)] ?? []; return { @@ -367,14 +427,11 @@ export const loadAverage = async ( })), trend: applyExclusions(trend, options.exclude ?? []), trendCost: { - requests: dates.reduce( - (sum, d) => - sum + - (options.summaries === undefined ? 0 : 1) + - (summaryOf(d) === undefined ? Object.values(plan).flat().length : 0), - 0, - ), - dates: trendDates.length, + requests: + (summaries === undefined ? 0 : windowDates.length) + + dates.filter((d) => summaryOf(d) === undefined).length * perRun, + dates: drawn.length, + perRun, }, }; }; diff --git a/apps/www/src/lib/markdown.ts b/apps/www/src/lib/markdown.ts index d25fbb19eb..27c7bbbd8b 100644 --- a/apps/www/src/lib/markdown.ts +++ b/apps/www/src/lib/markdown.ts @@ -16,7 +16,8 @@ import { import { formatDuration, formatVersus } from "./findings"; import { formatMs, type CountRow, type FixtureRow } from "./measurements"; import { METHOD, RESOURCES, RUNNER, RUNNER_NOTE } from "./metadata"; -import { reportHref } from "./params"; +import { rowsOf } from "./chart-model"; +import { rangeLabel, reportHref } from "./params"; import type { TrendPoint } from "./load"; import type { ReportModel } from "./report"; import type { VersionMarker } from "./version-markers"; @@ -107,12 +108,31 @@ const counts = (report: ReportModel, rows: readonly CountRow[]): string => ]), ); +/* + * How a trend's CSV is laid out: a row a calendar day, or -- past + * MAX_DENSE_DAYS, where the chart stops spacing runs by date -- a row a run. + */ +const layout = (perDay: boolean) => + perDay + ? { rows: "one row per calendar day", row: "day" } + : { rows: "one row per run (the window is too long for a row a day)", row: "run" }; + +const sentence = (text: string) => text.charAt(0).toUpperCase() + text.slice(1); + +const DAY_MS = 86_400_000; + +/* Whether a history's rows are consecutive days, as a dense series' are. */ +const perDay = (rows: readonly { readonly date: string }[]): boolean => + rows.every( + (r, i) => i === 0 || Date.parse(r.date) - Date.parse(rows[i - 1]?.date ?? "") === DAY_MS, + ); + const label = (report: ReportModel, tool: string): string => report.leaderboard.find((r) => r.tool === tool)?.label ?? tool; /** * The trend as agents read it best: a CSV block, one row per calendar day - * with empty cells for gaps, then the DNFs, coverage, version changes and + * (or per run, past MAX_DENSE_DAYS) with empty cells for gaps, then the DNFs, coverage, version changes and * outliers that explain them. */ const trendLines = ({ @@ -220,7 +240,7 @@ export const reportMarkdown = (report: ReportModel, origin: string): string => { `- Measured: mean ${report.measure} time ${filters.fixture === null ? `across ${fixtureCount} ${fixtureCount === 1 ? "fixture" : "fixtures"}` : `on the ${fixtureLabel(filters.fixture)} fixture`}${scenario.averageOf === null ? "" : `, averaged over ${scenario.averageOf.scenarios.length} scenarios (derived, not measured)`}, on GitHub-hosted ubuntu-24.04-arm runners.${section.id === "registries" ? " Every registry is installed from with the same npm client." : ""}`, `- Filters in effect:`, ` - fixture: ${filters.fixture === null ? `all (${report.fixtures.join(", ")})` : filters.fixture}`, - ` - trend window: ${filters.window.custom ? "custom" : `last ${filters.range} days`}, ${filters.window.from} to ${filters.window.to}`, + ` - trend window: ${filters.window.custom ? "custom" : rangeLabel(filters.range).toLowerCase()}, ${filters.window.from} to ${filters.window.to}`, ` - trend metric: ${METRIC_TEXT[metric].name}`, ` - hidden tools: ${filters.hide.length === 0 ? "none" : filters.hide.join(", ")}`, ` - excluded runs: ${filters.exclude.length === 0 ? "none" : filters.exclude.join(", ")}`, @@ -353,13 +373,19 @@ export const reportMarkdown = (report: ReportModel, origin: string): string => { "", ); } + const grid = layout(trend.series.dense); add( "### Performance over time", "", - `${metric === "total" ? `Mean ${report.measure} time in seconds` : METRIC_TEXT[metric].unit}, one row per calendar day from ${filters.window.from} to ${filters.window.to}.${trend.cut === null ? "" : ` ${trend.cut}`} An empty cell is a day with no published value for that tool: no run, a DNF, a partial run or an exclusion. The chart bridges those with a dotted line by default; nothing is interpolated here. For other windows use the history endpoint: ${abs(`/api/v1/${section.id}/${encodeURIComponent(scenario.variation)}/history`)}`, + `${metric === "total" ? `Mean ${report.measure} time in seconds` : METRIC_TEXT[metric].unit}, ${grid.rows} from ${filters.window.from} to ${filters.window.to}.${trend.cut === null ? "" : ` ${trend.cut}`} An empty cell is a ${grid.row} with no published value for that tool: no run, a DNF, a partial run or an exclusion. The chart bridges those with a dotted line by default; nothing is interpolated here. For other windows use the history endpoint: ${abs(`/api/v1/${section.id}/${encodeURIComponent(scenario.variation)}/history`)}`, "", ...trendLines({ - rows: metricRows(trend.rows, [...report.ranked, ...report.unranked], metric, report.baseline), + rows: metricRows( + rowsOf(trend.series), + [...report.ranked, ...report.unranked], + metric, + report.baseline, + ), runs: trend.points, markers: trend.markers, outliers: trend.outliers, @@ -419,10 +445,11 @@ export const reportMarkdown = (report: ReportModel, origin: string): string => { add( "### Per-package time by fixture (ms per installed package, derived)", "", - "Each fixture's per-package time, run by run, one row per calendar day; an empty cell is no value for that tool on that day.", + `Each fixture's per-package time, run by run, ${grid.rows}; an empty cell is no value for that tool on that ${grid.row}.`, "", ); - for (const { fixture, rows } of trend.perPackageByFixture) { + for (const { fixture, series } of trend.perPackageByFixture) { + const rows = rowsOf(series); add( `#### ${fixtureLabel(fixture)}`, "", @@ -523,6 +550,7 @@ export const reportMarkdown = (report: ReportModel, origin: string): string => { /** A scenario's history (`historyOf`) as Markdown: what was asked, then the trend. */ export const historyMarkdown = (history: History, origin: string): string => { const section = history.section as SectionId; + const grid = layout(perDay(history.rows)); const label = (tool: string) => toolLabel(tool, section); return [ `# ${SECTION_DEFS[section].label} — ${history.scenario}: history`, @@ -534,7 +562,7 @@ export const historyMarkdown = (history: History, origin: string): string => { `- Tools: ${history.tools.map((t) => (label(t) === t ? t : `${label(t)} (${t})`)).join(", ")}`, `- Excluded runs: ${history.excluded.length === 0 ? "none" : history.excluded.join(", ")}`, "", - "One row per calendar day. An empty cell is a day with no comparable value for that tool: no run, a DNF, a partial run or an exclusion. Nothing is interpolated.", + `${sentence(grid.rows)}. An empty cell is a ${grid.row} with no comparable value for that tool: no run, a DNF, a partial run or an exclusion. Nothing is interpolated.`, "", ...trendLines({ rows: history.rows, diff --git a/apps/www/src/lib/mcp.ts b/apps/www/src/lib/mcp.ts index 8dc59e123a..c3a45a68af 100644 --- a/apps/www/src/lib/mcp.ts +++ b/apps/www/src/lib/mcp.ts @@ -43,7 +43,7 @@ export const inputs = { section, scenario, tools: tools.optional(), - from: date.optional().describe("Window start, a published date; with to. At most 120 days."), + from: date.optional().describe("Window start, a published date; with to. Any span."), to: date .optional() .describe("Window end, a published date. Omitted: the 30 days to the latest run."), diff --git a/apps/www/src/lib/metadata.ts b/apps/www/src/lib/metadata.ts index f1f50c27c9..afec204fa7 100644 --- a/apps/www/src/lib/metadata.ts +++ b/apps/www/src/lib/metadata.ts @@ -26,7 +26,8 @@ const https = (host?: string) => (host ? `https://${host}` : undefined); * out (vltpkg/www/docs/src/lib/site-url.ts, after Next's own social-image * fallback): localhost in development, so a card previews on the machine * serving it; the preview URL on a Vercel preview; the project's production - * domain in production, and benchmarks.vlt.sh where nothing says otherwise. + * domain in production, and benchmarks.vlt.io where nothing says otherwise + * (benchmarks.vlt.sh only redirects there). * SITE_URL overrides all of it. */ export const siteUrl = new URL( @@ -36,7 +37,7 @@ export const siteUrl = new URL( : process.env.VERCEL_ENV === "preview" ? https(process.env.VERCEL_BRANCH_URL ?? process.env.VERCEL_URL) : https(process.env.VERCEL_PROJECT_PRODUCTION_URL)) ?? - "https://benchmarks.vlt.sh", + "https://benchmarks.vlt.io", ); /** diff --git a/apps/www/src/lib/opengraph/templates.tsx b/apps/www/src/lib/opengraph/templates.tsx index 34ba86d105..5437098f98 100644 --- a/apps/www/src/lib/opengraph/templates.tsx +++ b/apps/www/src/lib/opengraph/templates.tsx @@ -10,6 +10,7 @@ import { join } from "node:path"; import { createElement, type ComponentType, type ReactElement, type ReactNode } from "react"; import { Vlt } from "@/components/icons/vlt"; import { TOOL_ICONS } from "@/components/tool-name"; +import { rowsOf } from "@/lib/chart-model"; import { shownBars } from "@/lib/figures"; import { formatDuration, longDate } from "@/lib/findings"; import { formatMs } from "@/lib/measurements"; @@ -443,7 +444,7 @@ export const FigureCard = ({ report.trend.perPackageByFixture[0]) : null; if (lane != null) extra = [`on ${fixtureLabel(lane.fixture)}`]; - const rows = lane?.rows ?? report.trend.rows; + const rows = rowsOf(lane?.series ?? report.trend.series); const unit = per || report.trend.plotted === "per-package" ? "ms" : "s"; const tools = order.filter((t) => rows.some((r) => typeof r[t] === "number")); const { svg, max } = lineSvg(rows, tools, colors, report.baseline, PLOT); diff --git a/apps/www/src/lib/params.ts b/apps/www/src/lib/params.ts index 00c5a40147..a11241d0b8 100644 --- a/apps/www/src/lib/params.ts +++ b/apps/www/src/lib/params.ts @@ -23,16 +23,51 @@ */ import { createLoader, + createParser, createSerializer, parseAsArrayOf, - parseAsNumberLiteral, parseAsString, parseAsStringLiteral, } from "nuqs/server"; import { METRICS } from "./figures"; -/** Trend windows, in calendar days ending at the run being shown. */ -export const RANGES = [30, 60, 90, 120] as const; +/** + * The trend windows offered, in rolling calendar days ending at the run being + * shown -- not calendar months. `today` is that run alone; `all` has no lower + * bound: every published date up to it. + */ +export const PRESETS = [ + { value: "today", label: "Today", days: 1 }, + { value: "1w", label: "1 week", days: 7 }, + { value: "1m", label: "1 month", days: 30 }, + { value: "1y", label: "1 year", days: 365 }, + { value: "all", label: "All time", days: null }, +] as const; +export type Preset = (typeof PRESETS)[number]["value"]; +/** Old links: `range=30|60|90|120` keep meaning N rolling days. Parsed, never written. */ +export const LEGACY_RANGES = [30, 60, 90, 120] as const; +export type RangeValue = Preset | (typeof LEGACY_RANGES)[number]; + +const presetOf = (value: string): (typeof PRESETS)[number] | undefined => + PRESETS.find((p) => p.value === value); + +/** How many days a range reaches back, or `null` for no lower bound. */ +export const rangeDays = (range: RangeValue): number | null => + typeof range === "number" ? range : (presetOf(range)?.days ?? null); + +/** A range in words: the preset's label, or "Last N days" for an old link. */ +export const rangeLabel = (range: RangeValue): string => + typeof range === "number" ? `Last ${range} days` : (presetOf(range)?.label ?? range); + +/** A preset name or a legacy day count; anything else is not a range. */ +const parseAsRange = createParser({ + parse: (value) => { + if (presetOf(value) !== undefined) return value as Preset; + const legacy = LEGACY_RANGES.find((n) => String(n) === value); + return legacy ?? null; + }, + serialize: String, +}); export const SCALES = ["linear", "log"] as const; export const SMOOTHING = ["off", "median7"] as const; export const GAPS = ["connect", "hole"] as const; @@ -42,8 +77,8 @@ export const reportParams = { date: parseAsString, /** SERVER. One fixture; absent means all of them, averaged. */ fixture: parseAsString, - /** SERVER. A preset trend window: this many days, ending at the run shown. */ - range: parseAsNumberLiteral(RANGES).withDefault(30), + /** SERVER. A preset trend window, ending at the run shown; see `PRESETS`. */ + range: parseAsRange.withDefault("1m"), /** SERVER. An exact trend window. Both must be published dates; see `trendWindow`. */ from: parseAsString, to: parseAsString, @@ -148,9 +183,6 @@ export const withTableSort = ( /* The trend window */ /* ------------------------------------------------------------------ */ -/** The longest window offered: the span that can be spaced by calendar date. */ -export const MAX_WINDOW_DAYS = 120; - const DAY = 86_400_000; const day = (date: string): number => Date.parse(`${date}T00:00:00Z`); @@ -161,6 +193,8 @@ export interface TrendWindow { readonly custom: boolean; readonly from: string; readonly to: string; + /** The preset asked for; the one applied whenever `custom` is false. */ + readonly range: RangeValue; } /** @@ -168,10 +202,14 @@ export interface TrendWindow { * * An exact window is honoured only if it can be: both ends are dates the * archive actually published (a range cannot start or end on a hole), `from` - * is not after `to`, `to` is not after the run being shown, and the span is - * within {@link MAX_WINDOW_DAYS}. Anything else -- a hand-edited URL, a link - * that outlived its dates -- falls back to the preset rather than to an empty - * chart or an error. + * is not after `to`, and `to` is not after the run being shown. There is no + * cap on the span: the axis is spaced by calendar time for any window the + * archive can hold (see MAX_DENSE_DAYS). Anything else -- a hand-edited URL, + * a link that outlived its dates -- falls back to the preset rather than to + * an empty chart or an error. + * + * The same rule serves the site, `/api/v1` and MCP; agents get no separate + * cap. */ export const trendWindow = ({ dates, @@ -184,7 +222,7 @@ export const trendWindow = ({ readonly dates: readonly string[]; /** The run the report is showing. */ readonly end: string; - readonly range: number; + readonly range: RangeValue; readonly from: string | null; readonly to: string | null; }): TrendWindow => { @@ -194,15 +232,16 @@ export const trendWindow = ({ dates.includes(from) && dates.includes(to) && from <= to && - to <= end && - (day(to) - day(from)) / DAY < MAX_WINDOW_DAYS + to <= end ) { - return { dates: dates.filter((d) => d >= from && d <= to), custom: true, from, to }; + return { dates: dates.filter((d) => d >= from && d <= to), custom: true, from, to, range }; } + const days = rangeDays(range); const inRange = dates.filter((d) => { if (d > end) return false; + if (days === null) return true; const age = (day(end) - day(d)) / DAY; - return Number.isFinite(age) && age < Math.min(range, MAX_WINDOW_DAYS); + return Number.isFinite(age) && age < days; }); - return { dates: inRange, custom: false, from: inRange[0] ?? end, to: end }; + return { dates: inRange, custom: false, from: inRange[0] ?? end, to: end, range }; }; diff --git a/apps/www/src/lib/report.ts b/apps/www/src/lib/report.ts index 2e70b2eb3a..a8792c84e5 100644 --- a/apps/www/src/lib/report.ts +++ b/apps/www/src/lib/report.ts @@ -8,23 +8,17 @@ * and the same file is the source of the section lists this report refuses * to trust. */ -import { benchmarkFile, type DataClient } from "@benchmarks/data"; +import { benchmarkFile, summaryVersions, type DataClient } from "@benchmarks/data"; import type { Fixture, Versions } from "@benchmarks/types"; import type { inferParserType } from "nuqs/server"; import { publishedVariations } from "./catalog"; -import { toDenseChartRows } from "./chart-model"; +import { rowsOf, toDenseSeries, type DenseSeries } from "./chart-model"; import { citeFor, type Citation } from "./cite"; import { dataClient, probe } from "./data"; import { flagOutliers, meanBars, type Bar } from "./figures"; import { formatDuration, formatRatio, type Finding } from "./findings"; import { scenarioFacts, scopedFinding } from "./headline"; -import { - averageTrendDates, - loadAverage, - loadScenario, - loadSummaries, - type TrendPoint, -} from "./load"; +import { loadAverage, loadScenario, loadSummaries, type TrendPoint } from "./load"; import { countRows, perPackageByFixture, @@ -155,16 +149,19 @@ export interface ReportModel { readonly outline: readonly OutlineSection[]; readonly trend: { readonly points: readonly TrendPoint[]; - /** One row per calendar day, holes as `null`, in the plotted metric. */ - readonly rows: readonly { date: string; [tool: string]: string | number | null }[]; - /** What `rows` holds: total seconds, or milliseconds per installed package. */ + /** + * One row per calendar day, holes as `null`, in the plotted metric -- by + * column, as the chart island receives it; `rowsOf` gives the rows. + */ + readonly series: DenseSeries; + /** What `series` holds: total seconds, or milliseconds per installed package. */ readonly plotted: "total" | "per-package"; /** Whether per-package can be plotted here at all. */ readonly perPackage: boolean; /** Each fixture's milliseconds per installed package, one row per calendar day, holes as `null`. */ readonly perPackageByFixture: readonly { readonly fixture: string; - readonly rows: readonly { date: string; [tool: string]: string | number | null }[]; + readonly series: DenseSeries; }[]; /** Whether the window ends at the run shown, so "since last run" is about it. */ readonly endsHere: boolean; @@ -346,14 +343,16 @@ export const loadReport = async ({ }); /* - * Trend window: a preset number of calendar days ending at this run, or an - * exact from/to. Resolved and validated in `trendWindow`. + * Trend window: a preset of rolling calendar days ending at this run (up to + * the whole archive), or an exact from/to. Resolved and validated in + * `trendWindow`; there is no cap. * * By TIME, not by count. The archive is missing 53 calendar days, so "the * last 16 runs" can reach back the better part of a year, and a line joining * runs ten months apart is not a trend -- it is two unrelated states drawn - * as a slope. Every window is inside MAX_DENSE_DAYS (120), so the axis is - * always spaced by real calendar time and a gap is always a gap. + * as a slope. MAX_DENSE_DAYS (1,100) holds the whole archive, so the axis is + * spaced by real calendar time and a gap is a gap; a window past it falls + * back to an evenly spaced axis that says so. */ const window = trendWindow({ dates: index.dates, @@ -368,24 +367,25 @@ export const loadReport = async ({ * them -- the report's numbers are about it. */ const exclude = query.exclude.filter((e) => e.split("@").at(-1) !== date); - /* The derived view's trend reads each run's summary; a run without one is cut to what the request budget allows. */ + /* + * Both views' trends read each run's summary: one fetch a run, whatever the + * window, All time included. A run without one costs its files, cut to what + * the request budget allows. The shown run's files load while the summaries + * do. + */ const shownPlan = Object.fromEntries(fixtures.map((f) => [f, plan[f] ?? []])); - const filesPerRun = Object.values(shownPlan).flat().length; - const summaries = averageOf === null ? undefined : await loadSummaries(client, window.dates); - const trendDates = - summaries === undefined - ? window.dates - : averageTrendDates(window.dates, filesPerRun, new Set(summaries.keys())); + const summaries = loadSummaries(client, window.dates); const average = averageOf === null ? null - : await loadAverage(client, date, shownPlan, trendDates, { exclude, summaries }); + : await loadAverage(client, date, shownPlan, window.dates, { exclude, summaries }); const data = average ?? - (await loadScenario(client, date, selected.variation, fixtures, trendDates, { + (await loadScenario(client, date, selected.variation, fixtures, window.dates, { exclude, /* The per-package figures by fixture draw every run, so the counts are always wanted where there are any. */ perPackage: section !== "task-runners", + summaries, })); const facts = scenarioFacts( section, @@ -394,8 +394,13 @@ export const loadReport = async ({ ); const { tools, means, timeRows, baseline } = facts; + /* Each run's versions: from its summary where that carries them, else its versions.json. */ + const summarisedRuns = await summaries; const versionSeries = await Promise.all( data.trend.map(async (p) => { + const summarised = summarisedRuns.get(p.date); + const carried = summarised === undefined ? undefined : summaryVersions(summarised); + if (carried !== undefined) return { date: p.date, versions: carried }; const v = await probe(() => client.getVersions(p.date)); return { date: p.date, versions: v.state === "present" ? v.value : null }; }), @@ -439,23 +444,22 @@ export const loadReport = async ({ /* Per package, the trend plots the per-package series the loader built. */ const perPackageTrend = averageOf === null && section !== "task-runners"; const plotted = perPackageTrend && query.metric === "per-package" ? "per-package" : "total"; - const dense = toDenseChartRows( + const plottedPoints = plotted === "per-package" ? data.trend.map((p) => ({ date: p.date, byTool: p.perPackage ?? {} })) - : data.trend, - tools, - ); + : data.trend; + const series = toDenseSeries(plottedPoints, tools); /* Each fixture's per-package time, run by run, for the per-package figure by fixture. */ const perPackageTrends = perPackageTrend ? fixtures.map((f) => ({ fixture: f, - rows: toDenseChartRows( + series: toDenseSeries( data.trend.map((p) => ({ date: p.date, byTool: Object.fromEntries(tools.map((t) => [t, p.perPackageByFixture?.[t]?.[f]])), })), tools, - ).rows, + ), })) : []; const measure = isScript ? "execution" : "install"; @@ -503,10 +507,11 @@ export const loadReport = async ({ }; }); + const rows = rowsOf(series); const outliers = ranked.flatMap((tool) => - flagOutliers(dense.rows.map((r) => (typeof r[tool] === "number" ? r[tool] : null))).flatMap( + flagOutliers(rows.map((r) => (typeof r[tool] === "number" ? r[tool] : null))).flatMap( (deviation, i) => { - const day = dense.rows[i]?.date; + const day = rows[i]?.date; return deviation === null || day === undefined ? [] : [{ tool, date: day, deviation }]; }, ), @@ -590,7 +595,7 @@ export const loadReport = async ({ filters: { date: pinned, fixture, - range: query.range, + range: window.range, window: { from: window.from, to: window.to, custom: window.custom }, hide: hidden, metric: query.metric, @@ -624,7 +629,7 @@ export const loadReport = async ({ outline, trend: { points: data.trend, - rows: dense.rows, + series, plotted, perPackage: perPackageTrend, perPackageByFixture: perPackageTrends, @@ -633,8 +638,8 @@ export const loadReport = async ({ markers, outliers, cut: - trendDates.length < window.dates.length - ? `Each run here averages ${filesPerRun} result files, so the trend shows ${trendDates.length} of the window's ${window.dates.length} runs.` + data.trendCost.dates < window.dates.length + ? `A run without a summary costs ${data.trendCost.perRun} files here, so the trend shows ${data.trendCost.dates} of the window's ${window.dates.length} runs.` : null, focus, inFocus, diff --git a/apps/www/test/app-cwd.ts b/apps/www/test/app-cwd.ts new file mode 100644 index 0000000000..069e9bbd62 --- /dev/null +++ b/apps/www/test/app-cwd.ts @@ -0,0 +1,8 @@ +/* + * The app reads its own files from `process.cwd()` (lib/mdx.ts, lib/opengraph), which is how Next + * runs it: from apps/www, with server code bundled into .next where `import.meta.url` would not + * point at src/. The root `vlt run test` starts vitest at the repo root, so put the tests there too. + */ +import { fileURLToPath } from "node:url"; + +process.chdir(fileURLToPath(new URL("..", import.meta.url))); diff --git a/apps/www/test/average.test.ts b/apps/www/test/average.test.ts index 2a7898e692..085b81fb48 100644 --- a/apps/www/test/average.test.ts +++ b/apps/www/test/average.test.ts @@ -5,7 +5,7 @@ import type { BenchmarkFile } from "@benchmarks/types"; import { GET as api } from "../src/app/api/v1/[section]/[scenario]/route.ts"; import { GET as llms } from "../src/app/llms.txt/route.ts"; import { GET as markdown } from "../src/app/llms.mdx/[[...slug]]/route.ts"; -import { AVERAGE_TREND_BUDGET, averageTrendDates } from "../src/lib/load.ts"; +import { TREND_BUDGET, trendDates } from "../src/lib/load.ts"; import { averageFiles } from "../src/lib/measurements.ts"; import { treeFor } from "../src/lib/page-tree.ts"; import { loadReportParams } from "../src/lib/params.ts"; @@ -13,7 +13,7 @@ import { loadReport } from "../src/lib/report.ts"; import { AVERAGE, scenariosFor, withAverage } from "../src/lib/scenarios.ts"; import { DATA_URL, fakeFetch } from "./fixture-tree.ts"; -const ORIGIN = "https://benchmarks.vlt.sh"; +const ORIGIN = "https://benchmarks.vlt.io"; const result = (command: string, mean: number, exit = 0) => ({ command, @@ -133,13 +133,13 @@ describe("default routes", () => { }); }); -describe("averageTrendDates", () => { +describe("trendDates", () => { it("cuts the window to the newest runs the budget pays for", () => { const dates = Array.from({ length: 30 }, (_, i) => `d${i}`); - expect(averageTrendDates(dates, 54)).toEqual(dates.slice(-7)); - expect(7 * 54).toBeLessThanOrEqual(AVERAGE_TREND_BUDGET); - expect(averageTrendDates(dates, 12)).toEqual(dates); - expect(averageTrendDates(dates, 1000)).toHaveLength(2); + expect(trendDates(dates, 54)).toEqual(dates.slice(-7)); + expect(7 * 54).toBeLessThanOrEqual(TREND_BUDGET); + expect(trendDates(dates, 12)).toEqual(dates); + expect(trendDates(dates, 1000)).toHaveLength(2); }); }); @@ -241,15 +241,12 @@ describe("the average report's twins", () => { }); }); -describe("averageTrendDates with summaries", () => { +describe("trendDates with summaries", () => { it("keeps every summarised date and budgets only the rest", () => { const dates = Array.from({ length: 30 }, (_, i) => `d${String(i).padStart(2, "0")}`); const summarised = new Set(dates.slice(0, 20)); // 54 files a run leaves room for 7 raw runs: the last 7 of the 10 unsummarised. - expect(averageTrendDates(dates, 54, summarised)).toEqual([ - ...dates.slice(0, 20), - ...dates.slice(23), - ]); - expect(averageTrendDates(dates, 54, new Set(dates))).toEqual(dates); + expect(trendDates(dates, 54, summarised)).toEqual([...dates.slice(0, 20), ...dates.slice(23)]); + expect(trendDates(dates, 54, new Set(dates))).toEqual(dates); }); }); diff --git a/apps/www/test/fixture-tree.ts b/apps/www/test/fixture-tree.ts index 69523809ab..f64511e86f 100644 --- a/apps/www/test/fixture-tree.ts +++ b/apps/www/test/fixture-tree.ts @@ -4,7 +4,7 @@ * out loud -- zpm crashes on babylon (DNF, so partial and not ranked), npm * did not run on one date (a gap in the trend), and versions change. */ -import { DataClient } from "@benchmarks/data"; +import { buildSummary, DataClient, SUMMARY_PACKAGE_COUNTS } from "@benchmarks/data"; export const DATA_URL = "http://data.test"; export const DATES = ["2026-10-10", "2026-10-12", "2026-10-13"]; @@ -87,3 +87,46 @@ export const fakeFetch = async (url: string | URL | Request) => { }; export const fixtureClient = () => new DataClient({ baseUrl: DATA_URL, fetch: fakeFetch }); + +/** + * The same tree with each date's summary, built by apps/data from these files, + * and a log of every path fetched -- so a test can say both what a report + * drew and what it cost. `old` serves the shape published before summaries + * carried counts and versions: results only. + */ +export const summarisedClient = ({ + old = false, + uncounted, +}: { + old?: boolean; + /** A fixture whose count the summaries leave out, as the build does a count file it cannot parse. */ + uncounted?: string; +} = {}) => { + const fetched: string[] = []; + const fetch = async (url: string | URL | Request) => { + const path = decodeURIComponent(String(url).slice(DATA_URL.length + 1)); + fetched.push(path); + const date = /^summary\/(.+)\.json$/.exec(path)?.[1]; + if (date === undefined || !DATES.includes(date)) return fakeFetch(url); + const listing = { + name: date, + files: [...files.keys()] + .filter((p) => p.startsWith(`${date}/`)) + .map((p) => p.slice(date.length + 1)), + }; + const { summary } = buildSummary(listing, (file) => + JSON.stringify(files.get(`${date}/${file}`)), + ); + if (uncounted !== undefined) { + const counts = summary[SUMMARY_PACKAGE_COUNTS] as { [variation: string]: object } | undefined; + for (const byFixture of Object.values(counts ?? {})) { + delete (byFixture as { [fixture: string]: unknown })[uncounted]; + } + } + const served = old + ? Object.fromEntries(Object.entries(summary).filter(([key]) => !key.startsWith("#"))) + : summary; + return { ok: true, status: 200, json: async () => served }; + }; + return { client: new DataClient({ baseUrl: DATA_URL, fetch }), fetched }; +}; diff --git a/apps/www/test/honesty-inventory.test.ts b/apps/www/test/honesty-inventory.test.ts index 93cff2ae28..51802bc768 100644 --- a/apps/www/test/honesty-inventory.test.ts +++ b/apps/www/test/honesty-inventory.test.ts @@ -14,11 +14,14 @@ import { describe, expect, it } from "vitest"; import { alignedSeries, gappedPath, + MAX_DENSE_DAYS, + rowsOf, niceTicks, segments, spreadOf, toChartRows, toDenseChartRows, + toDenseSeries, } from "../src/lib/chart-model.ts"; import { PACKAGE_MANAGERS, usable, type ToolMean } from "../src/lib/measurements.ts"; import { findingFor, formatRatio, formatVersus } from "../src/lib/findings.ts"; @@ -246,16 +249,66 @@ describe("inventory 12 · a calendar gap is a real hole, not an equal step", () expect(rows).toHaveLength(2); }); - it("falls back rather than emitting 700 rows for a two-year window, and says so", () => { + it("keeps a whole-archive window on the calendar: one row per day", () => { const { dense, spanDays, rows } = toDenseChartRows( [ { date: "2025-08-22", byTool: { npm: 1 } }, + { date: "2026-09-26", byTool: { npm: 2 } }, + ], + ["npm"], + ); + expect(dense).toBe(true); + expect(spanDays).toBe(401); // calendar days, both ends included + expect(rows).toHaveLength(401); + expect(rows[1]?.["date"]).toBe("2025-08-23"); + }); + + it("by column, the same rows exactly: a 400-day span with holes and partial tools", () => { + // One run in three, a tool that skips some, another that never ran, an excluded zero. + const runs = Array.from({ length: 134 }, (_, i) => ({ + date: new Date(Date.parse("2025-08-22T00:00:00Z") + i * 3 * 86_400_000) + .toISOString() + .slice(0, 10), + byTool: { npm: 10 + i / 7, bun: i % 5 === 0 ? undefined : 1 / (i + 3), vlt: 0 }, + })); + const tools = ["npm", "bun", "vlt", "deno"]; + const series = toDenseSeries(runs, tools); + expect(series).toMatchObject({ start: "2025-08-22", length: 400, dense: true }); + expect(series.dates).toBeUndefined(); + expect(rowsOf(series)).toEqual(toDenseChartRows(runs, tools).rows); + expect(Object.keys(rowsOf(series)[0] ?? {})).toEqual(["date", ...tools]); + // Lossless: nothing rounded on the way. + expect(series.values["bun"]?.[3]).toBe(1 / 4); + }); + + it("by column, the sparse fallback keeps each run's date", () => { + const runs = [ + { date: "2022-01-01", byTool: { npm: 1 } }, + { date: "2024-02-29", byTool: { npm: undefined } }, + { date: "2026-06-29", byTool: { npm: 2 } }, + ]; + const series = toDenseSeries(runs, ["npm"]); + expect(series.dense).toBe(false); + expect(series.dates).toEqual(["2022-01-01", "2024-02-29", "2026-06-29"]); + expect(rowsOf(series)).toEqual(toDenseChartRows(runs, ["npm"]).rows); + }); + + it("by column, a single run and an empty trend", () => { + const one = [{ date: "2026-06-29", byTool: { npm: 2 } }]; + expect(rowsOf(toDenseSeries(one, ["npm"]))).toEqual(toDenseChartRows(one, ["npm"]).rows); + expect(rowsOf(toDenseSeries([], ["npm"]))).toEqual([]); + }); + + it("falls back rather than emitting thousands of rows past MAX_DENSE_DAYS, and says so", () => { + const { dense, spanDays, rows } = toDenseChartRows( + [ + { date: "2022-01-01", byTool: { npm: 1 } }, { date: "2026-06-29", byTool: { npm: 2 } }, ], ["npm"], ); expect(dense).toBe(false); - expect(spanDays).toBeGreaterThan(300); + expect(spanDays).toBeGreaterThan(MAX_DENSE_DAYS); expect(rows).toHaveLength(2); }); }); diff --git a/apps/www/test/mcp.test.ts b/apps/www/test/mcp.test.ts index 82d6ce7c23..1f82c42c1f 100644 --- a/apps/www/test/mcp.test.ts +++ b/apps/www/test/mcp.test.ts @@ -8,7 +8,7 @@ import { } from "../src/lib/mcp.ts"; import { fixtureClient } from "./fixture-tree.ts"; -const ORIGIN = "https://benchmarks.vlt.sh"; +const ORIGIN = "https://benchmarks.vlt.io"; describe("MCP tools", () => { it("list_benchmarks: the catalog and every run", async () => { diff --git a/apps/www/test/og.test.ts b/apps/www/test/og.test.ts index 42b99ec79f..f1149afd89 100644 --- a/apps/www/test/og.test.ts +++ b/apps/www/test/og.test.ts @@ -6,7 +6,7 @@ import { ellipsize } from "../src/lib/opengraph/truncate.ts"; import { DATA_URL, fakeFetch } from "./fixture-tree.ts"; const og = (path: string) => - GET(new Request(`https://benchmarks.vlt.sh/og/${path}`), { + GET(new Request(`https://benchmarks.vlt.io/og/${path}`), { params: Promise.resolve({ path: path.split("?")[0]?.split("/").filter(Boolean) }), }); diff --git a/apps/www/test/report.test.ts b/apps/www/test/report.test.ts index 614ad353a4..25d366893b 100644 --- a/apps/www/test/report.test.ts +++ b/apps/www/test/report.test.ts @@ -1,7 +1,10 @@ import { describe, expect, it } from "vitest"; +import { rowsOf, toTrendColumns, trendOf, type TrendRun } from "../src/lib/chart-model.ts"; import { loadReportParams } from "../src/lib/params.ts"; import { ignoredSelection, loadReport, type ReportModel } from "../src/lib/report.ts"; -import { fixtureClient } from "./fixture-tree.ts"; +import { DataClient } from "@benchmarks/data"; +import { loadScenario, loadSummaries, TREND_BUDGET } from "../src/lib/load.ts"; +import { DATA_URL, DATES, fakeFetch, fixtureClient, summarisedClient } from "./fixture-tree.ts"; const load = async (scenario: string | undefined, search = "", section = "package-managers") => loadReport({ @@ -40,14 +43,15 @@ describe("loadReport", () => { it("leaves gaps in the trend as gaps, and compares only with a real previous run", async () => { const r = await report(); - expect(r.trend.rows.map((row) => row.date)).toEqual([ + const rows = rowsOf(r.trend.series); + expect(rows.map((row) => row.date)).toEqual([ "2026-10-10", "2026-10-11", "2026-10-12", "2026-10-13", ]); - expect(r.trend.rows[1]?.["bun"]).toBeNull(); - expect(r.trend.rows[2]?.["npm"]).toBeNull(); + expect(rows[1]?.["bun"]).toBeNull(); + expect(rows[2]?.["npm"]).toBeNull(); expect(r.leaderboard.find((row) => row.tool === "npm")?.sinceLastRun).toBeNull(); expect(r.leaderboard.find((row) => row.tool === "bun")?.sinceLastRun).toBeCloseTo(-1 / 11); expect(r.trend.markers.map((m) => m.label)).toEqual(["vlt 1.1.0"]); @@ -95,3 +99,169 @@ describe("loadReport", () => { expect(() => structuredClone(r)).not.toThrow(); }); }); + +describe("a trend read from the summaries", () => { + const load = async (client: Parameters[0]["client"], search: string) => { + const result = await loadReport({ + section: "package-managers", + scenario: "clean", + query: loadReportParams(new URLSearchParams(search)), + client, + }); + if (result.status !== "ok") throw new Error(result.status); + return result.report; + }; + + it("draws exactly what the files draw: the same points, gaps, DNFs and per-package times", async () => { + for (const search of [ + "range=all", + "range=all&metric=per-package", + "range=all&fixture=babylon", + ]) { + const files = await load(fixtureClient(), search); + const { client } = summarisedClient(); + const summarised = await load(client, search); + expect(summarised.trend).toEqual(files.trend); + expect(summarised.leaderboard).toEqual(files.leaderboard); + } + }); + + it("costs one fetch a run instead of one per fixture", async () => { + const { client, fetched } = summarisedClient(); + await load(client, "range=all"); + const benchmarks = fetched.filter((p) => /^\d{4}-\d{2}-\d{2}\/[a-z]+-clean\.json$/.test(p)); + // Only the run shown reads its files, for its tables; the trend reads none. + expect(benchmarks.sort()).toEqual([ + "2026-10-13/astro-clean.json", + "2026-10-13/babylon-clean.json", + ]); + expect(fetched.filter((p) => p.startsWith("summary/")).sort()).toEqual( + DATES.map((d) => `summary/${d}.json`), + ); + }); + + const trendFetches = (fetched: readonly string[], kind: RegExp) => + fetched.filter((p) => kind.test(p) && !p.startsWith("2026-10-13/")).length; + + it("reads counts and versions from a summary that carries them, and fetches none", async () => { + const { client, fetched } = summarisedClient(); + const r = await load(client, "range=all&metric=per-package"); + expect(trendFetches(fetched, /-package-count\.json$/)).toBe(0); + expect(trendFetches(fetched, /\/versions\.json$/)).toBe(0); + // ...and draws exactly what the files draw. + const files = await load(fixtureClient(), "range=all&metric=per-package"); + expect(r.trend).toEqual(files.trend); + }); + + it("falls back to the files for a summary written before it carried them", async () => { + const { client, fetched } = summarisedClient({ old: true }); + const r = await load(client, "range=all&metric=per-package"); + // Two fixtures on each earlier run, and one versions.json each. + expect(trendFetches(fetched, /-package-count\.json$/)).toBe(2 * (DATES.length - 1)); + expect(trendFetches(fetched, /\/versions\.json$/)).toBe(DATES.length - 1); + const files = await load(fixtureClient(), "range=all&metric=per-package"); + expect(r.trend).toEqual(files.trend); + }); + + it("falls back per fixture for a count the summary leaves out", async () => { + const { client, fetched } = summarisedClient({ uncounted: "babylon" }); + const r = await load(client, "range=all&metric=per-package"); + // Only babylon's, on each earlier run; astro's come from the summary. + expect( + fetched.filter((p) => p.endsWith("-package-count.json") && !p.startsWith("2026-10-13/")), + ).toEqual(DATES.slice(0, -1).map((d) => `${d}/babylon-clean-package-count.json`)); + const files = await load(fixtureClient(), "range=all&metric=per-package"); + expect(r.trend).toEqual(files.trend); + }); +}); + +describe("a trend's fetch budget", () => { + /* Far more runs than the budget pays for, none with a summary. */ + const window = Array.from({ length: 300 }, (_, i) => + new Date(Date.UTC(2025, 0, 1 + i)).toISOString().slice(0, 10), + ); + const counting = (summaries: "missing" | "failed") => { + const fetched: string[] = []; + const fetch = async (url: string | URL | Request) => { + const path = decodeURIComponent(String(url).slice(DATA_URL.length + 1)); + fetched.push(path); + if (summaries === "failed" && path.startsWith("summary/")) { + return { ok: false, status: 500, json: async () => null }; + } + return fakeFetch(url); + }; + return { client: new DataClient({ baseUrl: DATA_URL, fetch }), fetched }; + }; + + for (const summaries of ["missing", "failed"] as const) { + it(`never fans out past it when every summary is ${summaries}`, async () => { + const { client, fetched } = counting(summaries); + const data = await loadScenario(client, "2026-10-13", "clean", ["astro", "babylon"], window, { + perPackage: true, + summaries: loadSummaries(client, window), + }); + // Two fixtures, each with its count: four files a run, so the newest 100 runs. + expect(data.trendCost).toMatchObject({ perRun: 4, dates: TREND_BUDGET / 4 }); + const newest = new Set(window.slice(-TREND_BUDGET / 4)); + const trendFiles = fetched.filter( + (p) => !p.startsWith("summary/") && !p.startsWith("2026-10-13/"), + ); + expect(trendFiles.length).toBeLessThanOrEqual(TREND_BUDGET); + expect(trendFiles.every((p) => newest.has(p.split("/")[0] ?? ""))).toBe(true); + }); + } +}); + +describe("the trend's run detail by column", () => { + const detail = ({ date, fixtures, partial, failed, byFixture, rsd, rsdByFixture }: TrendRun) => ({ + date, + fixtures, + partial, + failed, + byFixture, + rsd, + rsdByFixture, + }); + + it("gives back every run exactly: DNFs, partial means, a tool missing a run", async () => { + const r = await report("range=all"); + const runs = r.trend.points.map(detail); + // The fixture tree has zpm failing babylon (partial, failed) and npm absent on one run. + expect(runs.some((p) => Object.keys(p.failed).length > 0)).toBe(true); + expect(runs.some((p) => Object.keys(p.partial).length > 0)).toBe(true); + expect(runs.some((p) => p.byFixture["npm"] === undefined)).toBe(true); + expect(trendOf(toTrendColumns(runs))).toEqual(runs); + }); + + it("keeps an absent spread absent, a zero a zero, and fixtures in each run's order", () => { + const runs: TrendRun[] = [ + { + date: "2026-01-01", + fixtures: ["vue", "astro"], + partial: { zpm: { mean: 0.5, of: 1 } }, + failed: { zpm: [{ fixture: "vue", exit: 1 }] }, + byFixture: { npm: { vue: 1 / 3, astro: 2 }, zpm: { astro: 0.5 } }, + rsd: { npm: 0, zpm: 0.1 }, + rsdByFixture: { npm: { astro: 0 } }, + }, + { + date: "2026-01-03", + fixtures: ["astro"], + partial: {}, + failed: {}, + byFixture: { bun: { astro: 0.25 } }, + rsd: { bun: 0.02 }, + rsdByFixture: {}, + }, + ]; + const columns = toTrendColumns(runs); + expect(trendOf(columns)).toEqual(runs); + expect(columns.byFixture["npm"]?.["vue"]).toEqual([1 / 3, null]); + expect(trendOf(toTrendColumns([]))).toEqual([]); + }); + + it("is smaller than the runs it encodes", async () => { + const runs = (await report("range=all")).trend.points.map(detail); + expect(JSON.stringify(toTrendColumns(runs)).length).toBeLessThan(JSON.stringify(runs).length); + }); +}); diff --git a/apps/www/test/routes.test.ts b/apps/www/test/routes.test.ts index 055adb06d2..fc69528d6b 100644 --- a/apps/www/test/routes.test.ts +++ b/apps/www/test/routes.test.ts @@ -10,7 +10,7 @@ import { GET as openapi } from "../src/app/openapi.json/route.ts"; import { METHOD, READING, RUNNER_NOTE, TRADEMARK } from "../src/lib/metadata.ts"; import { DATA_URL, fakeFetch } from "./fixture-tree.ts"; -const ORIGIN = "https://benchmarks.vlt.sh"; +const ORIGIN = "https://benchmarks.vlt.io"; const get = (path: string) => new Request(`${ORIGIN}${path}`); const scenario = (section: string, scenario: string) => ({ params: Promise.resolve({ section, scenario }), diff --git a/apps/www/test/scenarios.test.ts b/apps/www/test/scenarios.test.ts index 6aa29c7478..3a09b68b9b 100644 --- a/apps/www/test/scenarios.test.ts +++ b/apps/www/test/scenarios.test.ts @@ -21,7 +21,9 @@ import { } from "../src/lib/measurements.ts"; import { markersForTools, versionMarkers } from "../src/lib/version-markers.ts"; import { + LEGACY_RANGES, loadReportParams, + type RangeValue, reportHref, tableSortOf, trendWindow, @@ -465,27 +467,71 @@ describe("versionMarkers · the empty-version case", () => { describe("the trend window", () => { // Published dates either side of the archive's widest gap. const dates = ["2026-06-15", "2026-06-16", "2026-06-17", "2026-06-29", "2026-06-30"]; - const base = { dates, end: "2026-06-30", range: 30, from: null, to: null }; + const base = { + dates, + end: "2026-06-30", + range: "1m" as RangeValue, + from: null, + to: null, + }; + // Most of 2025, one run every other day, then the five above. + const archive = [ + ...Array.from({ length: 180 }, (_, i) => + new Date(Date.parse("2025-01-01T00:00:00Z") + i * 2 * 86_400_000).toISOString().slice(0, 10), + ), + ...dates, + ]; it("a preset is that many calendar days ending at the run shown", () => { - const w = trendWindow({ ...base, range: 30 }); + const w = trendWindow({ ...base, range: "1m" }); expect(w.custom).toBe(false); + expect(w.range).toBe("1m"); expect(w.dates).toEqual(dates); expect(w.to).toBe("2026-06-30"); - // Fourteen days back from Jun 30 reaches Jun 17 and no further. - expect(trendWindow({ ...base, range: 14 }).dates).toEqual([ - "2026-06-17", - "2026-06-29", - "2026-06-30", + // Seven days back from Jun 30 reaches Jun 24 -- inside the gap -- so only the last two runs. + expect(trendWindow({ ...base, range: "1w" }).dates).toEqual(["2026-06-29", "2026-06-30"]); + }); + + it("today is the run shown alone", () => { + const w = trendWindow({ ...base, range: "today" }); + expect(w).toMatchObject({ dates: ["2026-06-30"], from: "2026-06-30", to: "2026-06-30" }); + expect(trendWindow({ ...base, end: "2026-06-16", range: "today" }).dates).toEqual([ + "2026-06-16", ]); }); + it("1y is 365 rolling days and all is the whole archive up to the run", () => { + const year = trendWindow({ ...base, dates: archive, range: "1y" }).dates; + expect(year[0]).toBe("2025-07-02"); // the first run fewer than 365 days before Jun 30 + expect(year.at(-1)).toBe("2026-06-30"); + expect(trendWindow({ ...base, dates: archive, range: "all" }).dates).toEqual(archive); + expect(trendWindow({ ...base, dates: archive, end: "2026-06-16", range: "all" }).dates).toEqual( + archive.filter((d) => d <= "2026-06-16"), + ); + }); + + it("an old day-count link keeps its old meaning", () => { + // What the 120-day cap allowed, the legacy numbers still give exactly. + for (const n of LEGACY_RANGES) { + const w = trendWindow({ ...base, dates: archive, range: n }); + const oldest = new Date(Date.parse("2026-06-30T00:00:00Z") - (n - 1) * 86_400_000) + .toISOString() + .slice(0, 10); + expect(w.dates).toEqual(archive.filter((d) => d >= oldest)); + expect(w.range).toBe(n); + } + expect(trendWindow({ ...base, range: 30 }).dates).toEqual( + trendWindow({ ...base, range: "1m" }).dates, + ); + }); + it("honours an exact range whose ends are both published dates", () => { const w = trendWindow({ ...base, from: "2026-06-16", to: "2026-06-29" }); expect(w).toEqual({ custom: true, from: "2026-06-16", to: "2026-06-29", + range: "1m", dates: ["2026-06-16", "2026-06-17", "2026-06-29"], }); }); @@ -496,13 +542,20 @@ describe("the trend window", () => { expect(trendWindow({ ...base, from: "2026-06-15", to: "2026-06-25" }).custom).toBe(false); }); - it("falls back to the preset for a reversed, half-given or over-long range", () => { - expect(trendWindow({ ...base, from: "2026-06-30", to: "2026-06-15" }).custom).toBe(false); + it("falls back to the preset for a reversed or half-given range", () => { + const reversed = trendWindow({ ...base, range: "1w", from: "2026-06-30", to: "2026-06-15" }); + expect(reversed.custom).toBe(false); + expect(reversed.range).toBe("1w"); + expect(reversed.dates).toEqual(["2026-06-29", "2026-06-30"]); expect(trendWindow({ ...base, from: "2026-06-15", to: null }).custom).toBe(false); + }); + + it("honours an exact range of any length", () => { + // Past the old 120-day cap: there is no cap now. const long = ["2026-01-01", ...dates]; - expect(trendWindow({ ...base, dates: long, from: "2026-01-01", to: "2026-06-30" }).custom).toBe( - false, - ); + const w = trendWindow({ ...base, dates: long, from: "2026-01-01", to: "2026-06-30" }); + expect(w.custom).toBe(true); + expect(w.dates).toEqual(long); }); it("will not run past the run the report is showing", () => { @@ -523,7 +576,7 @@ describe("query parameters", () => { reportHref("/package-managers/clean", { date: null, fixture: null, - range: 30, + range: "1m", from: null, to: null, hide: [], @@ -534,11 +587,26 @@ describe("query parameters", () => { expect(reportHref("/registries/registry-clean", { fixture: "astro", range: 90 })).toBe( "/registries/registry-clean?fixture=astro&range=90", ); + expect(reportHref("/registries/registry-clean", { range: "1y" })).toBe( + "/registries/registry-clean?range=1y", + ); + }); + + it("reads a preset or a legacy day count as the range, and nothing else", async () => { + const range = async (q: string) => (await loadReportParams(new URLSearchParams(q))).range; + expect(await range("")).toBe("1m"); + expect(await range("range=1y")).toBe("1y"); + expect(await range("range=all")).toBe("all"); + expect(await range("range=today")).toBe("today"); + expect(await range("range=90")).toBe(90); + expect(await range("range=7")).toBe("1m"); + expect(await range("range=1M")).toBe("1m"); + expect(await range("range=30.0")).toBe("1m"); }); it("a malformed value falls back to the default instead of erroring", async () => { const q = await loadReportParams(new URLSearchParams("range=7&scale=sideways&hide=")); - expect(q.range).toBe(30); + expect(q.range).toBe("1m"); expect(q.scale).toBe("linear"); expect(q.hide).toEqual([]); }); diff --git a/apps/www/vitest.config.mts b/apps/www/vitest.config.mts index cd93c17773..700919f03c 100644 --- a/apps/www/vitest.config.mts +++ b/apps/www/vitest.config.mts @@ -16,5 +16,7 @@ export default defineConfig({ oxc: { jsx: { runtime: "automatic" } }, test: { include: ["test/**/*.test.ts"], + // Run from apps/www whatever directory vitest starts in, as Next does (test/app-cwd.ts). + setupFiles: ["./test/app-cwd.ts"], }, }); diff --git a/docs/METHODOLOGY.md b/docs/METHODOLOGY.md index d255624758..b4ad7dc368 100644 --- a/docs/METHODOLOGY.md +++ b/docs/METHODOLOGY.md @@ -119,6 +119,7 @@ nightly run since 2025-08-22. /{date}/versions.json tool versions for that run /latest/ see below — NOT a copy /index.json /manifest.json /manifest/{date}.json discovery layer +/summary/{date}.json one date's results, counts and versions ``` [`packages/types`](../packages/types/src/index.ts) encodes these shapes and is @@ -178,3 +179,21 @@ given row. For a single date, fetch `index.json` plus that date's shard (~12KB), not the whole manifest. + +### `summary/{date}.json` + +One file per date, so a trend over many runs costs one fetch a run, not one +per file. Every value is lifted verbatim from that date's files; nothing is +derived. Written by `apps/data` build-manifest on each deploy, for every date. + +| Key | Holds | +| ---------------- | --------------------------------------------------------------------------------------- | +| `{variation}` | `{fixture}` to that benchmark file's results: `command`, `mean`, `stddev`, `exit_codes` | +| `#packageCounts` | `{variation}` to `{fixture}` to that package-count file, as published | +| `#versions` | that date's `versions.json` | + +Results keep `exit_codes`, so a failed run is still a failed run. The `#` keys +were added in 2026-11 and are each optional: a summary written before then has +none, and a date that published no count files or no `versions.json` has no +such key. Absent means "read the file", never zero. No variation name starts +with `#`.