From 3512e16eb80a3fe3abb09c56404f66f3446b51ba Mon Sep 17 00:00:00 2001 From: Jason Korol Date: Wed, 4 Nov 2026 16:47:17 +0000 Subject: [PATCH] feat(www): arbitrary trend date ranges with presets and All time Replace the Last 30/60/90/120 days picker with a Vercel-style range picker: presets Today, 1 week, 1 month (default), 1 year and All time, plus a range calendar limited to published dates. Legacy range=30|60|90|120 URLs still work. - No trend-window cap; the axis stays calendar-spaced up to 1,100 days. The API, openapi, llms.txt, MCP and markdown follow the same window rule. - The scenario trend reads each run's summary, and summaries now also carry the date's package counts and versions (falling back per fixture to the files for old summaries). Unsummarised runs are fetched within a request budget, and the shown run no longer waits for the summaries. Cold All time: 2,917 -> 424 requests. Needs a data republish. - Chart islands take columnar series and trend columns (lossless), cutting the All-time page from 6.52 to 4.61 MB. Trend marks stay visible in forced colors. - The production data URL in .env.example and comments is benchmarks-data.vlt.io (benchmarks.vlt.sh now serves the site's HTML); the site origin fallback is benchmarks.vlt.io. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/benchmark.yaml | 4 +- apps/data/src/client.ts | 2 +- apps/data/src/index.ts | 11 +- apps/data/src/serve.ts | 6 +- apps/data/src/summary.ts | 77 +++++- apps/data/test/served-tree.test.ts | 2 +- apps/data/test/summary.test.ts | 63 ++++- apps/www/.env.example | 5 +- apps/www/.env.local | 5 +- .../src/app/[section]/[[...params]]/page.tsx | 26 +- apps/www/src/app/globals.css | 42 +++ apps/www/src/app/llms.txt/route.ts | 2 +- apps/www/src/app/openapi.json/route.ts | 8 +- apps/www/src/components/charts-echarts.tsx | 35 ++- apps/www/src/components/fixture-trend.tsx | 35 +-- apps/www/src/components/pinned-run.tsx | 22 ++ apps/www/src/components/range-picker.tsx | 250 +++++++++++++----- apps/www/src/lib/api.ts | 8 +- apps/www/src/lib/chart-model.ts | 157 ++++++++++- apps/www/src/lib/load.ts | 227 ++++++++++------ apps/www/src/lib/markdown.ts | 44 ++- apps/www/src/lib/mcp.ts | 2 +- apps/www/src/lib/metadata.ts | 5 +- apps/www/src/lib/opengraph/templates.tsx | 3 +- apps/www/src/lib/params.ts | 75 ++++-- apps/www/src/lib/report.ts | 81 +++--- apps/www/test/app-cwd.ts | 8 + apps/www/test/average.test.ts | 23 +- apps/www/test/fixture-tree.ts | 45 +++- apps/www/test/honesty-inventory.test.ts | 57 +++- apps/www/test/mcp.test.ts | 2 +- apps/www/test/og.test.ts | 2 +- apps/www/test/report.test.ts | 178 ++++++++++++- apps/www/test/routes.test.ts | 2 +- apps/www/test/scenarios.test.ts | 96 ++++++- apps/www/vitest.config.mts | 2 + docs/METHODOLOGY.md | 19 ++ 37 files changed, 1296 insertions(+), 335 deletions(-) create mode 100644 apps/www/test/app-cwd.ts diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index 4c6ac21f39..830523ee0f 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -333,8 +333,8 @@ jobs: # apps/data emits index.json, manifest.json and the manifest/ shards # into suite/results/, the single results tree. It never re-emits the # published date directories (decision D2) and never writes an - # index.html -- the published index.html is the live site at - # benchmarks.vlt.sh and survives via keep_files. + # index.html -- the published index.html is a live page at + # benchmarks-data.vlt.io (the gh-pages CNAME) and survives via keep_files. run: | vlt install vlt run --workspace=apps/data build diff --git a/apps/data/src/client.ts b/apps/data/src/client.ts index 3aabc9a515..da194c82bf 100644 --- a/apps/data/src/client.ts +++ b/apps/data/src/client.ts @@ -37,7 +37,7 @@ export type FetchLike = (url: string) => Promise<{ }>; export interface DataClientOptions { - /** Base URL of the served tree, e.g. `https://benchmarks.vlt.sh`. */ + /** Base URL of the served tree, e.g. `https://benchmarks-data.vlt.io`. */ baseUrl: string; /** Injectable for tests; defaults to global fetch. */ fetch?: FetchLike; diff --git a/apps/data/src/index.ts b/apps/data/src/index.ts index 0f8813f979..fe84553d83 100644 --- a/apps/data/src/index.ts +++ b/apps/data/src/index.ts @@ -28,4 +28,13 @@ export { type ParsedDataFile, } from "./paths.ts"; export { scanRoots } from "./scan.ts"; -export { buildSummary, type DateSummary, type SummaryResult } from "./summary.ts"; +export { + buildSummary, + SUMMARY_PACKAGE_COUNTS, + SUMMARY_VERSIONS, + summaryPackageCounts, + summaryVersions, + type DateSummary, + type SummaryCounts, + type SummaryResult, +} from "./summary.ts"; diff --git a/apps/data/src/serve.ts b/apps/data/src/serve.ts index ccc0b69889..4218b4612b 100644 --- a/apps/data/src/serve.ts +++ b/apps/data/src/serve.ts @@ -2,9 +2,9 @@ * Minimal static server for the assembled tree. * * Deliberately dependency-free (`node:http` only) and deliberately does not - * serve a directory index. The published root's `index.html` IS the live site - * at benchmarks.vlt.sh, and this app must never be the thing that puts an - * index page there. + * serve a directory index. The published root's `index.html` is a live page + * at benchmarks-data.vlt.io (the gh-pages CNAME), and this app must never be + * the thing that puts an index page there. */ import { createServer, type Server } from "node:http"; import { createReadStream, statSync } from "node:fs"; diff --git a/apps/data/src/summary.ts b/apps/data/src/summary.ts index aca48b66ba..119a029699 100644 --- a/apps/data/src/summary.ts +++ b/apps/data/src/summary.ts @@ -9,10 +9,16 @@ * Not `chart-data.json`. That file gives a failed run a time, and nothing in * it says the run failed. Every result here keeps its `exit_codes`, so a * consumer applies the same rule to it as to the file it came from. + * + * The date's package counts and `versions.json` ride along, + * verbatim, under reserved `#` keys beside the variations -- so a summary + * written before they existed is still a valid one, and a reader that only + * looks up variations never sees them. Each is optional: absent means "not in + * this summary", and the reader falls back to the file; it never means zero. */ -import type { BenchmarkResult } from "@benchmarks/types"; +import type { BenchmarkResult, PackageCount, Versions } from "@benchmarks/types"; import type { DirectoryListing } from "./manifest.ts"; -import { parseDataFile } from "./paths.ts"; +import { parseDataFile, VERSIONS_FILE } from "./paths.ts"; /** What a trend needs of a result, exactly as the benchmark file has it. */ export type SummaryResult = Pick; @@ -22,9 +28,42 @@ export type DateSummary = { [variation: string]: { [fixture: string]: SummaryResult[] | undefined } | undefined; }; +/** Variation, then fixture, to that count file as published. */ +export type SummaryCounts = { + [variation: string]: { [fixture: string]: T | undefined } | undefined; +}; + +export const SUMMARY_PACKAGE_COUNTS = "#packageCounts"; +export const SUMMARY_VERSIONS = "#versions"; + +/* The reserved keys, read without letting them widen every variation's type. */ +const extra = (summary: DateSummary, key: string): unknown => + (summary as { readonly [key: string]: unknown })[key]; + +/** This summary's package counts, or `undefined` when it does not carry them. */ +export const summaryPackageCounts = ( + summary: DateSummary, +): SummaryCounts | undefined => + extra(summary, SUMMARY_PACKAGE_COUNTS) as SummaryCounts | undefined; + +/** The date's `versions.json`, or `undefined` when the summary does not carry it. */ +export const summaryVersions = (summary: DateSummary): Versions | undefined => + extra(summary, SUMMARY_VERSIONS) as Versions | undefined; + +const isObject = (value: unknown): value is object => + typeof value === "object" && value !== null && !Array.isArray(value); + +const parse = (text: string): unknown => { + try { + return JSON.parse(text); + } catch { + return undefined; + } +}; + /** * Summarise one date directory. `read` returns a file's text; a file that - * does not parse as a benchmark file is reported in `skipped`, not guessed at. + * does not parse as what its name says is reported in `skipped`, not guessed at. */ export const buildSummary = ( listing: DirectoryListing, @@ -32,16 +71,33 @@ export const buildSummary = ( ): { summary: DateSummary; skipped: string[] } => { // A variation is whatever the file name says, `__proto__` included. const summary: DateSummary = Object.create(null); + const packageCounts: SummaryCounts = Object.create(null); const skipped: string[] = []; for (const file of listing.files) { + if (file === VERSIONS_FILE) { + const versions = parse(read(file)); + if (isObject(versions)) { + (summary as { [key: string]: unknown })[SUMMARY_VERSIONS] = versions; + } else skipped.push(file); + continue; + } const parsed = parseDataFile(file); - if (parsed?.kind !== "benchmark") continue; - let results: unknown; - try { - results = (JSON.parse(read(file)) as { results?: unknown }).results; - } catch { - results = undefined; + if (parsed === undefined) continue; + // A variation named like a reserved key would be read as one. + if (parsed.variation.startsWith("#")) { + skipped.push(file); + continue; + } + // Process counts stay out until something reads them from here. + if (parsed.kind === "process-count") continue; + const content = parse(read(file)); + if (parsed.kind === "package-count") { + if (isObject(content)) { + (packageCounts[parsed.variation] ??= {})[parsed.fixture] = content as PackageCount; + } else skipped.push(file); + continue; } + const results = isObject(content) ? (content as { results?: unknown }).results : undefined; if (!Array.isArray(results) || !results.every((r) => typeof r === "object" && r !== null)) { skipped.push(file); continue; @@ -50,5 +106,8 @@ export const buildSummary = ( ({ command, mean, stddev, exit_codes }) => ({ command, mean, stddev, exit_codes }), ); } + if (Object.keys(packageCounts).length > 0) { + (summary as { [key: string]: unknown })[SUMMARY_PACKAGE_COUNTS] = packageCounts; + } return { summary, skipped }; }; diff --git a/apps/data/test/served-tree.test.ts b/apps/data/test/served-tree.test.ts index df633fe73c..8082585abc 100644 --- a/apps/data/test/served-tree.test.ts +++ b/apps/data/test/served-tree.test.ts @@ -72,7 +72,7 @@ describe.skipIf(skip)("served tree, built from real gh-pages data", () => { mkdirSync(join(root, "results-astro-clean"), { recursive: true }); writeFileSync(join(root, "results-astro-clean", "benchmarks.json"), "{}\n"); writeFileSync(join(root, "index.html"), "live site\n"); - writeFileSync(join(root, "CNAME"), "benchmarks.vlt.sh\n"); + writeFileSync(join(root, "CNAME"), "benchmarks-data.vlt.io\n"); // Run the REAL CLI rather than reimplementing what it does. An earlier // version of this fixture built the index and shards inline, which meant a diff --git a/apps/data/test/summary.test.ts b/apps/data/test/summary.test.ts index 2d2504469d..76643d51d3 100644 --- a/apps/data/test/summary.test.ts +++ b/apps/data/test/summary.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { buildSummary } from "../src/summary.ts"; +import { buildSummary, summaryPackageCounts, summaryVersions } from "../src/summary.ts"; describe("buildSummary", () => { it("lifts each benchmark file's results verbatim, exit codes and all", () => { @@ -30,8 +30,69 @@ describe("buildSummary", () => { }, "registry-clean": { next: [] }, ["__proto__"]: { next: [] }, + "#packageCounts": { clean: { next: {} } }, }); expect(Object.prototype).not.toHaveProperty("next"); expect(skipped).toEqual(["astro-clean.json", "vue-clean.json"]); }); + + it("carries the date's counts and versions verbatim, beside the variations", () => { + const counts = { npm: { count: 110, minCount: 108, maxCount: 112 }, bun: { count: 100 } }; + const processes = { npm: { count: 4 } }; + const versions = { npm: "11.0.0", vlt: "1.1.0" }; + const files: { [name: string]: string } = { + "next-clean.json": JSON.stringify({ + results: [{ command: "npm", mean: 2, stddev: 0, exit_codes: [0] }], + }), + "next-clean-package-count.json": JSON.stringify(counts), + "next-clean-process-count.json": JSON.stringify(processes), + "next-registry-clean-package-count.json": JSON.stringify({ npm: { count: 90 } }), + "astro-clean-package-count.json": "not json", + "versions.json": JSON.stringify(versions), + }; + const { summary, skipped } = buildSummary( + { name: "2026-10-19", files: Object.keys(files) }, + (file) => files[file] ?? "", + ); + expect(summaryPackageCounts(summary)).toEqual({ + clean: { next: counts }, + "registry-clean": { next: { npm: { count: 90 } } }, + }); + // Nothing reads process counts from a summary, so it does not carry them. + expect( + Object.keys(summary) + .filter((k) => k.startsWith("#")) + .sort(), + ).toEqual(["#packageCounts", "#versions"]); + expect(summaryVersions(summary)).toEqual(versions); + // A reader that looks up variations sees exactly what it did before. + expect(summary["clean"]).toEqual({ + next: [{ command: "npm", mean: 2, stddev: 0, exit_codes: [0] }], + }); + expect(skipped).toEqual(["astro-clean-package-count.json"]); + }); + + it("leaves out what the date did not publish, rather than writing it empty", () => { + const { summary } = buildSummary({ name: "2026-10-19", files: ["next-run.json"] }, () => + JSON.stringify({ results: [] }), + ); + expect(summary).toEqual({ run: { next: [] } }); + expect(summaryPackageCounts(summary)).toBeUndefined(); + expect(summaryVersions(summary)).toBeUndefined(); + }); + + it("an old summary, written before counts and versions, reads as carrying none", () => { + const old = JSON.parse(JSON.stringify({ clean: { next: [] } })); + expect(summaryPackageCounts(old)).toBeUndefined(); + expect(summaryVersions(old)).toBeUndefined(); + }); + + it("will not let a variation pose as a reserved key", () => { + const { summary, skipped } = buildSummary( + { name: "2026-10-19", files: ["next-#versions.json"] }, + () => JSON.stringify({ results: [] }), + ); + expect(summaryVersions(summary)).toBeUndefined(); + expect(skipped).toEqual(["next-#versions.json"]); + }); }); diff --git a/apps/www/.env.example b/apps/www/.env.example index 6f3143c2ff..0f054a7c68 100644 --- a/apps/www/.env.example +++ b/apps/www/.env.example @@ -7,8 +7,9 @@ # Local development, after `vlt run --workspace=apps/data build`: # vlt run --workspace=apps/data start # serves results/ on :8811 # -# Production is the gh-pages origin, which serves the same tree: -# NEXT_PUBLIC_DATA_URL=https://benchmarks.vlt.sh +# Production is the gh-pages origin (its CNAME), which serves the same tree. +# Not benchmarks.vlt.sh: that is the site, and it answers HTML for any path. +# NEXT_PUBLIC_DATA_URL=https://benchmarks-data.vlt.io NEXT_PUBLIC_DATA_URL=http://localhost:8811 # Optional server-side runtime override, checked first when set. diff --git a/apps/www/.env.local b/apps/www/.env.local index 6f3143c2ff..0f054a7c68 100644 --- a/apps/www/.env.local +++ b/apps/www/.env.local @@ -7,8 +7,9 @@ # Local development, after `vlt run --workspace=apps/data build`: # vlt run --workspace=apps/data start # serves results/ on :8811 # -# Production is the gh-pages origin, which serves the same tree: -# NEXT_PUBLIC_DATA_URL=https://benchmarks.vlt.sh +# Production is the gh-pages origin (its CNAME), which serves the same tree. +# Not benchmarks.vlt.sh: that is the site, and it answers HTML for any path. +# NEXT_PUBLIC_DATA_URL=https://benchmarks-data.vlt.io NEXT_PUBLIC_DATA_URL=http://localhost:8811 # Optional server-side runtime override, checked first when set. diff --git a/apps/www/src/app/[section]/[[...params]]/page.tsx b/apps/www/src/app/[section]/[[...params]]/page.tsx index 6d0b8c227e..46100c0233 100644 --- a/apps/www/src/app/[section]/[[...params]]/page.tsx +++ b/apps/www/src/app/[section]/[[...params]]/page.tsx @@ -37,7 +37,8 @@ import { formatDuration, longDate } from "@/lib/findings"; import { fixtureBars } from "@/lib/figures"; import { pageMetadata } from "@/lib/metadata"; import { ogFigure } from "@/lib/og-figure"; -import { loadReportParams, reportHref, reportParams } from "@/lib/params"; +import { toTrendColumns } from "@/lib/chart-model"; +import { loadReportParams, rangeLabel, reportHref, reportParams } from "@/lib/params"; import { loadCatalog } from "@/lib/catalog"; import { dataClient } from "@/lib/data"; import { loadReport } from "@/lib/report"; @@ -196,14 +197,17 @@ const Report = async ({ const timeBars = report.timeBars; const perPackageBars = report.perPackageBars; - /* What the chart needs of each run besides its plotted means. */ - const trendDetail = trend.points.map( - ({ - byTool: _byTool, - perPackage: _perPackage, - perPackageByFixture: _perPackageByFixture, - ...rest - }) => rest, + /* What the chart needs of each run besides its plotted means, by column: one + object, which both trend islands get and the page carries once. */ + const trendDetail = toTrendColumns( + trend.points.map( + ({ + byTool: _byTool, + perPackage: _perPackage, + perPackageByFixture: _perPackageByFixture, + ...rest + }) => rest, + ), ); const fixtureCount = report.fixtures.length; const across = @@ -322,7 +326,7 @@ const Report = async ({ per mark, of which All time draws over a + * thousand. + */ +.mark-dnf, +.mark-version, +.mark-unusual { + display: inline-block; + flex: none; + width: 0.625rem; + height: 0.625rem; + background-color: currentColor; + mask: var(--mark) center / contain no-repeat; +} + +.mark-dnf { + --mark: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 24 24' fill='none' stroke='black' stroke-width='2.5' stroke-linecap='round' stroke-linejoin='round'%3E%3Ccircle cx='12' cy='12' r='10'/%3E%3Cpath d='m15 9-6 6'/%3E%3Cpath d='m9 9 6 6'/%3E%3C/svg%3E"); +} + +.mark-version { + --mark: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 24 24' fill='none' stroke='black' stroke-width='2.5' stroke-linecap='round' stroke-linejoin='round'%3E%3Cpath d='M12.586 2.586A2 2 0 0 0 11.172 2H4a2 2 0 0 0-2 2v7.172a2 2 0 0 0 .586 1.414l8.704 8.704a2.426 2.426 0 0 0 3.42 0l6.58-6.58a2.426 2.426 0 0 0 0-3.42z'/%3E%3Ccircle cx='7.5' cy='7.5' r='.5' fill='black'/%3E%3C/svg%3E"); +} + +.mark-unusual { + --mark: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 24 24' fill='none' stroke='black' stroke-width='2.5' stroke-linecap='round' stroke-linejoin='round'%3E%3Cpath d='m21.73 18-8-14a2 2 0 0 0-3.48 0l-8 14A2 2 0 0 0 4 21h16a2 2 0 0 0 1.73-3'/%3E%3Cpath d='M12 9v4'/%3E%3Cpath d='M12 17h.01'/%3E%3C/svg%3E"); +} + +/* Forced colours replace the background a mask paints with the page's own + colour, which hid the marks. There they take the system text colour, as the + inline SVG strokes they replaced did. */ +@media (forced-colors: active) { + .mark-dnf, + .mark-version, + .mark-unusual { + forced-color-adjust: none; + background-color: CanvasText; + } +} diff --git a/apps/www/src/app/llms.txt/route.ts b/apps/www/src/app/llms.txt/route.ts index 0f8d3b2cc5..5629f25738 100644 --- a/apps/www/src/app/llms.txt/route.ts +++ b/apps/www/src/app/llms.txt/route.ts @@ -17,7 +17,7 @@ Every page is available as Markdown: append \`.md\` to its URL, or request the p ## Reading the results ${READING.map((line) => `- ${line}`).join("\n")} -- Page parameters: \`date\`, \`fixture\`, \`range\` (30, 60, 90 or 120 days of trend), \`from\` and \`to\` (an exact trend window), \`hide\` (comma-separated tools), \`metric\` (total, change, baseline or per-package), \`exclude\` (\`YYYY-MM-DD\` or \`tool@YYYY-MM-DD\`, comma-separated). +- Page parameters: \`date\`, \`fixture\`, \`range\` (today, 1w, 1m, 1y or all: rolling days of trend ending at the run; default 1m), \`from\` and \`to\` (an exact trend window), \`hide\` (comma-separated tools), \`metric\` (total, change, baseline or per-package), \`exclude\` (\`YYYY-MM-DD\` or \`tool@YYYY-MM-DD\`, comma-separated). ## API and MCP diff --git a/apps/www/src/app/openapi.json/route.ts b/apps/www/src/app/openapi.json/route.ts index 494f148092..f65ae141bf 100644 --- a/apps/www/src/app/openapi.json/route.ts +++ b/apps/www/src/app/openapi.json/route.ts @@ -1,4 +1,4 @@ -import { RANGES } from "@/lib/params"; +import { LEGACY_RANGES, PRESETS } from "@/lib/params"; import { METRICS } from "@/lib/figures"; import { SITE_DESCRIPTION, siteOrigin } from "@/lib/metadata"; import { SECTIONS } from "@/lib/sections"; @@ -42,13 +42,13 @@ const viewParams = [ ), query( "range", - { type: "integer", enum: RANGES, default: 30 }, - "Trend window: this many calendar days ending at the run.", + { type: "string", enum: PRESETS.map((p) => p.value), default: "1m" }, + `Trend window in rolling calendar days ending at the run: today (the run alone), 1w (7), 1m (30), 1y (365) or all (every published run up to it). Legacy day counts ${LEGACY_RANGES.join(", ")} are still accepted.`, ), query( "from", date, - "Exact trend window start; with `to`. Both must be published dates, less than 120 days apart.", + "Exact trend window start; with `to`. Both must be published dates, in order; any span.", ), query("to", date, "Exact trend window end; no later than the run."), query( diff --git a/apps/www/src/components/charts-echarts.tsx b/apps/www/src/components/charts-echarts.tsx index 15def1fe8e..bd69e210fa 100644 --- a/apps/www/src/components/charts-echarts.tsx +++ b/apps/www/src/components/charts-echarts.tsx @@ -23,7 +23,7 @@ import { type ReactNode, } from "react"; import { ChartLegend } from "@/components/chart-legend"; -import { PinnedRun, type PinnedNote, type TrendDetail } from "@/components/pinned-run"; +import { onFixture, PinnedRun, type PinnedNote, type TrendDetail } from "@/components/pinned-run"; import { RangePicker } from "@/components/range-picker"; import { usePending } from "@/components/report-pending"; import { Callout, ChartFrame, Figure } from "@/components/report"; @@ -45,7 +45,7 @@ import { tooltipRow, tooltipShell, } from "@/components/ui/echarts-tooltip"; -import { toolColor } from "@/lib/chart-model"; +import { rowsOf, toolColor, trendOf, type DenseSeries, type TrendColumns } from "@/lib/chart-model"; import { EVIL_AXIS_LABEL, EVIL_SPLIT_LINE, EVIL_TICK_DOT, escapeHtml } from "@/lib/chart-style"; import { bridges, @@ -187,7 +187,7 @@ export interface TimeSeriesTool { export const TimeSeriesChart = ({ id, title, - rows, + series, tools, section, baseline, @@ -195,7 +195,8 @@ export const TimeSeriesChart = ({ yLabel, markers = [], versions = [], - trend = [], + trend: columns, + fixture, perPackage = true, focus = null, fixedMetric, @@ -216,8 +217,12 @@ export const TimeSeriesChart = ({ /** A tab row in place of the title tab (`FigureTabStrip`). */ tabs?: ReactNode; firstTabActive?: boolean; - /** Dense rows from `toDenseChartRows`: holes as `null`, never as zero. */ - rows: readonly { date: string; [tool: string]: string | number | null }[]; + /** + * The plotted values by column (`toDenseSeries`): holes as `null`, never as + * zero. Sent this way rather than as a row per calendar day with every + * tool's key repeated -- the lighter page -- and unpacked here, once. + */ + series: DenseSeries; /** Every tool these files contain, in ranking order -- hidden ones included. */ tools: readonly TimeSeriesTool[]; section: SectionId; @@ -233,9 +238,12 @@ export const TimeSeriesChart = ({ /** * Each run's coverage, failures, spread and per-fixture seconds (from * `loadScenario`), for the DNF marks, the band, the tooltip and the pinned - * run's panel. + * run's panel. By column (`toTrendColumns`), the lighter page, and unpacked + * here once. */ - trend?: readonly TrendDetail[]; + trend?: TrendColumns; + /** Each run as this one fixture saw it (the per-fixture chart's tab). */ + fixture?: string; /** * Whether "Per package" can be offered: there are package counts for what * is plotted. Not for task runners: nothing is installed. @@ -252,6 +260,12 @@ export const TimeSeriesChart = ({ readonly custom: boolean; }; }) => { + /* Once per series: a fixture tab remounts the chart with its own. */ + const rows = useMemo(() => rowsOf(series), [series]); + const trend: readonly TrendDetail[] = useMemo(() => { + const runs = columns === undefined ? [] : trendOf(columns); + return fixture === undefined ? runs : onFixture(runs, fixture); + }, [columns, fixture]); const { resolvedTheme } = useTheme(); /* Server writes run in the report's transitions; a tools change leaves the lines live. */ const { data: reloading, startTransition, startToolsTransition } = usePending(); @@ -1376,7 +1390,7 @@ export const TimeSeriesChart = ({ color: failed.length === 1 ? paint(failed[0] ?? "") : "var(--foreground)", }} > - + {failed.length === 1 ? null : failed.length} ))} @@ -1395,7 +1409,6 @@ export const TimeSeriesChart = ({ /> ); /* A version change (a tag) or an unusual run (a warning): an icon on the line, in its colour. */ - const Icon = kind === "ring" ? Tag : TriangleAlert; return ( - + ); })} diff --git a/apps/www/src/components/fixture-trend.tsx b/apps/www/src/components/fixture-trend.tsx index f9c2b9bd44..d15d6dd43d 100644 --- a/apps/www/src/components/fixture-trend.tsx +++ b/apps/www/src/components/fixture-trend.tsx @@ -1,36 +1,12 @@ "use client"; import { useState, type ComponentProps } from "react"; +import type { DenseSeries } from "@/lib/chart-model"; import { TimeSeriesChart } from "@/components/charts-echarts"; -import type { TrendDetail } from "@/components/pinned-run"; import { FigureTabStrip } from "@/components/tabbed-figure"; import { FixtureName } from "@/components/tool-name"; import { fixtureLabel } from "@/lib/tools"; -type Rows = ComponentProps["rows"]; - -/** - * Each run as one fixture saw it: the failures there and no others, nothing - * partial (one fixture is all or nothing), and that fixture's own spread for - * the band. - */ -const onFixture = (trend: readonly TrendDetail[], fixture: string): TrendDetail[] => - trend.map((d) => ({ - ...d, - fixtures: d.fixtures.filter((f) => f === fixture), - partial: {}, - failed: Object.fromEntries( - Object.entries(d.failed) - .map(([tool, list]) => [tool, list.filter((e) => e.fixture === fixture)] as const) - .filter(([, list]) => list.length > 0), - ), - rsd: Object.fromEntries( - Object.entries(d.rsdByFixture).flatMap(([tool, by]) => - by[fixture] === undefined ? [] : [[tool, by[fixture]]], - ), - ), - })); - /** * Per-package time on each fixture, run by run: the report's own * "Performance over time" chart, whole -- range, display, pin, brush, value @@ -47,9 +23,9 @@ export const FixtureTimeSeries = ({ ...chart }: Omit< ComponentProps, - "rows" | "yLabel" | "tabs" | "firstTabActive" | "fixedMetric" | "scope" + "series" | "yLabel" | "tabs" | "firstTabActive" | "fixedMetric" | "scope" > & { - fixtures: readonly { readonly fixture: string; readonly rows: Rows }[]; + fixtures: readonly { readonly fixture: string; readonly series: DenseSeries }[]; }) => { const [active, setActive] = useState(fixtures[0]?.fixture ?? ""); const shown = fixtures.find((f) => f.fixture === active) ?? fixtures[0]; @@ -60,8 +36,9 @@ export const FixtureTimeSeries = ({ {...chart} id={id} title={title} - rows={shown.rows} - trend={onFixture(trend ?? [], shown.fixture)} + series={shown.series} + trend={trend} + fixture={shown.fixture} yLabel={`Time per installed package on ${fixtureLabel(shown.fixture)}`} fixedMetric="per-package" scope="pp" diff --git a/apps/www/src/components/pinned-run.tsx b/apps/www/src/components/pinned-run.tsx index ebd5a84e5d..a6a062aac8 100644 --- a/apps/www/src/components/pinned-run.tsx +++ b/apps/www/src/components/pinned-run.tsx @@ -14,6 +14,28 @@ import type { SectionId } from "@/lib/sections"; export type TrendDetail = Omit; +/** + * Each run as one fixture saw it: the failures there and no others, nothing + * partial (one fixture is all or nothing), and that fixture's own spread for + * the band. + */ +export const onFixture = (trend: readonly TrendDetail[], fixture: string): TrendDetail[] => + trend.map((d) => ({ + ...d, + fixtures: d.fixtures.filter((f) => f === fixture), + partial: {}, + failed: Object.fromEntries( + Object.entries(d.failed) + .map(([tool, list]) => [tool, list.filter((e) => e.fixture === fixture)] as const) + .filter(([, list]) => list.length > 0), + ), + rsd: Object.fromEntries( + Object.entries(d.rsdByFixture).flatMap(([tool, by]) => + by[fixture] === undefined ? [] : [[tool, by[fixture]]], + ), + ), + })); + /** One tool on the pinned run, everything the panel says about it. */ export interface PinnedTool { readonly tool: string; diff --git a/apps/www/src/components/range-picker.tsx b/apps/www/src/components/range-picker.tsx index 856e3d87f2..39e10982fa 100644 --- a/apps/www/src/components/range-picker.tsx +++ b/apps/www/src/components/range-picker.tsx @@ -1,14 +1,15 @@ "use client"; -import { CalendarDays, ChevronDown } from "lucide-react"; +import { ArrowRight, CalendarDays, ChevronDown } from "lucide-react"; import { useQueryStates } from "nuqs"; -import { useState } from "react"; +import { useState, useSyncExternalStore, useTransition } from "react"; import type { DateRange } from "react-day-picker"; import { usePending } from "@/components/report-pending"; import { Button } from "@/components/ui/button"; import { Calendar } from "@/components/ui/calendar"; import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover"; -import { MAX_WINDOW_DAYS, RANGES, reportParams } from "@/lib/params"; +import { PRESETS, rangeLabel, reportParams } from "@/lib/params"; +import { cn } from "@/lib/utils"; /* * Dates here are calendar days with no time and no zone. They are built and @@ -22,21 +23,57 @@ const toDate = (iso: string): Date => { const toIso = (date: Date): string => `${date.getFullYear()}-${String(date.getMonth() + 1).padStart(2, "0")}-${String(date.getDate()).padStart(2, "0")}`; +const short = (d: Date, year: boolean) => + d.toLocaleDateString("en-US", { + month: "short", + day: "numeric", + ...(year ? { year: "numeric" } : {}), + }); + const words = (from: string, to: string): string => { const a = toDate(from); const b = toDate(to); - const short = (d: Date, year: boolean) => - d.toLocaleDateString("en-US", { - month: "short", - day: "numeric", - ...(year ? { year: "numeric" } : {}), - }); + if (from === to) return short(b, true); return `${short(a, a.getFullYear() !== b.getFullYear())} – ${short(b, true)}`; }; +/* Two months side by side from `sm`, one below it. Only read once the + popover is open, which is always on the client, so the server's answer + never reaches the page. */ +const SM = "(min-width: 40rem)"; +const subscribe = (onChange: () => void) => { + const mql = window.matchMedia(SM); + mql.addEventListener("change", onChange); + return () => mql.removeEventListener("change", onChange); +}; +const useWide = () => + useSyncExternalStore( + subscribe, + () => window.matchMedia(SM).matches, + () => true, + ); + +/** The month a calendar of `months` opens on, so that `to`'s month is the last one shown. */ +const openingMonth = (to: string, first: string, months: number): Date => { + const end = toDate(to); + const open = new Date(end.getFullYear(), end.getMonth() - (months - 1), 1); + const start = toDate(first); + return open < start ? new Date(start.getFullYear(), start.getMonth(), 1) : open; +}; + /** * The trend's date range: presets, or an exact start and end from a calendar. * + * PRESETS apply at once. They are rolling days ending at the run shown + * (`lib/params.ts`), and "1 month" is the default, so choosing it clears the + * parameter. An old `range=90` link is labelled "Last 90 days" and highlights + * no preset. + * + * THE CALENDAR is a draft: picking days changes nothing until Apply, and + * Escape or a click outside throws the draft away. It reaches across the + * whole archive -- there is no longest window -- so its caption is a month + * and a year dropdown rather than a long walk of "previous". + * * ONLY PUBLISHED DATES CAN BE CHOSEN. Every other day is disabled -- days * after the run being shown, days before the archive begins, and the days in * between on which nothing ran -- so a range cannot start or end on a hole, @@ -45,10 +82,12 @@ const words = (from: string, to: string): string => { * one. * * This is a server parameter: the window decides which files are fetched. So - * the write is not shallow, and the page re-renders with the new data. + * the write is not shallow, and it runs in the report's pending transition, + * which keeps the chart on screen, marked stale, until the new data arrives. * * The calendar is react-day-picker's, with its keyboard and screen-reader - * behaviour as shipped. + * behaviour as shipped. The popover is Radix's: it traps focus, closes on + * Escape and returns focus to the trigger. */ export const RangePicker = ({ dates, @@ -67,26 +106,30 @@ export const RangePicker = ({ custom: boolean; }) => { const { startTransition } = usePending(); + /* The range's own write, inside the report's: the report goes stale for + any server parameter, but only this one makes the URL's window news. */ + const [ranging, startRanging] = useTransition(); const [params, setParams] = useQueryStates( { range: reportParams.range, from: reportParams.from, to: reportParams.to }, - { shallow: false, startTransition }, + { shallow: false, startTransition: (write) => startTransition(() => startRanging(write)) }, ); const [open, setOpen] = useState(false); - const [draft, setDraft] = useState({ - from: toDate(from), - to: toDate(to), - }); + const [draft, setDraft] = useState(); - const published = new Set(dates.filter((d) => d <= end)); - const first = dates[0] ?? end; - const ready = draft?.from !== undefined && draft.to !== undefined; + /* While the server re-renders a new window, the URL's choice is shown at + once; otherwise the server's, which is the truth when it refused a + hand-edited range. */ + const asked = ranging && params.from !== null && params.to !== null; + const exact = ranging ? asked : custom; + const [start, stop] = asked ? [params.from ?? from, params.to ?? to] : [from, to]; + const label = exact ? words(start, stop) : rangeLabel(params.range); return ( { setOpen(next); - if (next) setDraft({ from: toDate(from), to: toDate(to) }); + if (next) setDraft({ from: toDate(start), to: toDate(stop) }); }} > @@ -96,65 +139,150 @@ export const RangePicker = ({ - -
- {RANGES.map((days) => { - const selected = !custom && params.range === days; + { + const target = event.currentTarget as HTMLElement; + const first = + target.querySelector("[aria-pressed=true]") ?? + target.querySelector("[data-slot=calendar] td button[tabindex='0']"); + if (first === null) return; + event.preventDefault(); + first.focus(); + }} + /* On a phone it can be taller than the room on either side of the trigger; it scrolls rather than leave the screen. */ + className="flex max-h-(--radix-popover-content-available-height) w-auto max-w-[calc(100vw-2rem)] flex-col gap-0 overflow-y-auto p-0 sm:flex-row" + > +
+ {PRESETS.map((preset) => { + const selected = !exact && params.range === preset.value; return ( ); })}
-
- !published.has(toIso(date))} - /> -
-

- Days with no published run cannot be chosen. Up to {MAX_WINDOW_DAYS} days. -

- -
-
+ { + void setParams(range); + setOpen(false); + }} + /> ); }; + +/** The calendar half: a draft range, its readout, and Apply. */ +const Custom = ({ + dates, + end, + to, + draft, + onDraft, + onApply, +}: { + dates: readonly string[]; + end: string; + to: string; + draft: DateRange | undefined; + onDraft: (range: DateRange | undefined) => void; + onApply: (range: { from: string; to: string }) => void; +}) => { + const months = useWide() ? 2 : 1; + const shown = dates.filter((d) => d <= end); + const published = new Set(shown); + const first = shown[0] ?? end; + + const a = draft?.from === undefined ? undefined : toIso(draft.from); + // A range is complete with both ends; one picked day is a range of one. + const b = draft?.to === undefined ? undefined : toIso(draft.to); + const runs = a === undefined ? 0 : shown.filter((d) => d >= a && d <= (b ?? a)).length; + + return ( +
+ onDraft(range ?? { from: day, to: undefined })} + resetOnSelect + numberOfMonths={months} + captionLayout="dropdown" + defaultMonth={openingMonth(to, first, months)} + startMonth={toDate(first)} + endMonth={toDate(end)} + disabled={(date) => !published.has(toIso(date))} + className="mx-auto" + /> +
+ {/* What Apply will show, read out as it changes. */} +

+ {a === undefined ? ( + Pick a start and an end + ) : ( + <> + Selected: + {short(toDate(a), b === undefined || a.slice(0, 4) !== b.slice(0, 4))} + + to + + {b === undefined ? "pick an end" : short(toDate(b), true)} + + + · {runs} {runs === 1 ? "run" : "runs"} + + + )} +

+ +
+
+ ); +}; diff --git a/apps/www/src/lib/api.ts b/apps/www/src/lib/api.ts index 7851a37ceb..40d7cc8abd 100644 --- a/apps/www/src/lib/api.ts +++ b/apps/www/src/lib/api.ts @@ -4,9 +4,9 @@ * it, and say what would have worked. */ import type { DataClient } from "@benchmarks/data"; +import { rowsOf } from "./chart-model"; import { dataClient } from "./data"; import { metricRows } from "./figures"; -import { MAX_WINDOW_DAYS } from "./params"; import { ignoredSelection, loadReport, type ReportModel, type ReportQuery } from "./report"; import { SECTIONS } from "./sections"; @@ -91,7 +91,7 @@ export const strictReport = async ({ status: 400, code: "invalid_window", message: `The window ${query.from ?? "?"} to ${query.to ?? "?"} cannot be used.`, - hint: `from and to must both be published dates, from no later than to, to no later than the run shown, and less than ${MAX_WINDOW_DAYS} days apart. /api/v1/runs lists the published dates.`, + hint: `from and to must both be published dates, from no later than to, and to no later than the run shown. /api/v1/runs lists the published dates.`, }, }; } @@ -127,8 +127,8 @@ export const historyOf = (report: ReportModel, tools?: readonly string[]) => { window: report.filters.window, excluded: report.filters.exclude, tools: chosen, - /* One row per calendar day; `null` is a day with no value, never a zero. */ - rows: metricRows(report.trend.rows, all, plotted, report.baseline).map((row) => ({ + /* One row per calendar day (per run past MAX_DENSE_DAYS); `null` is no value, never a zero. */ + rows: metricRows(rowsOf(report.trend.series), all, plotted, report.baseline).map((row) => ({ date: row.date, ...pick(row), })), diff --git a/apps/www/src/lib/chart-model.ts b/apps/www/src/lib/chart-model.ts index 1e5d1146c9..f603464637 100644 --- a/apps/www/src/lib/chart-model.ts +++ b/apps/www/src/lib/chart-model.ts @@ -162,8 +162,12 @@ export const spreadOf = ( const DAY = 86_400_000; -/** Cap on densification, so a two-year window cannot produce 700 rows. */ -export const MAX_DENSE_DAYS = 120; +/** + * Cap on densification: about three years, so the whole archive (440 days and + * growing) is spaced by calendar time on every window, All time included. A + * row per day is cheap at this size; past it the fallback below takes over. + */ +export const MAX_DENSE_DAYS = 1100; /** * Rows covering every calendar day between the first and last published run. @@ -216,6 +220,155 @@ export const toDenseChartRows = ( return { rows, dense: true, spanDays }; }; +/** One row of a trend chart: a date and each tool's value, `null` for a hole. */ +export type ChartRow = { date: string; [tool: string]: string | number | null }; + +/** + * Dense rows, by column: what the server sends a chart island instead of one + * object per calendar day with every tool's key repeated. Lossless -- the + * values are the rows' own, unrounded -- and `rowsOf` turns it back into + * exactly those rows, so the charts' internals never see the difference. + */ +export interface DenseSeries { + /** First calendar day, YYYY-MM-DD. Index i is start + i days. */ + readonly start: string; + /** Calendar days covered; every array below has this length. */ + readonly length: number; + /** False past MAX_DENSE_DAYS: then `dates` lists each index's day (the sparse fallback). */ + readonly dense: boolean; + readonly dates?: readonly string[]; + /** One column per tool, null for a hole (never 0). */ + readonly values: { readonly [tool: string]: readonly (number | null)[] }; +} + +/** `toDenseChartRows`, by column. */ +export const toDenseSeries = ( + points: readonly Plotted[], + tools: readonly string[], +): DenseSeries => { + const { rows, dense } = toDenseChartRows(points, tools); + /* Dense rows are consecutive days, so their dates are implied. */ + const implied = dense && rows.length > 0; + return { + start: rows[0]?.date ?? "", + length: rows.length, + dense: implied, + ...(implied ? {} : { dates: rows.map((r) => r.date) }), + values: Object.fromEntries( + tools.map((tool) => [tool, rows.map((r) => (typeof r[tool] === "number" ? r[tool] : null))]), + ), + }; +}; + +/** The rows a `DenseSeries` encodes: `toDenseChartRows(points, tools).rows`, exactly. */ +export const rowsOf = (series: DenseSeries): ChartRow[] => { + const first = Date.parse(`${series.start}T00:00:00Z`); + const columns = Object.entries(series.values); + return Array.from({ length: series.length }, (_, i) => { + const date = series.dense + ? new Date(first + i * DAY).toISOString().slice(0, 10) + : (series.dates?.[i] ?? ""); + const row: ChartRow = { date }; + for (const [tool, column] of columns) row[tool] = column[i] ?? null; + return row; + }); +}; + +/** What a chart island needs of each run besides its plotted values. */ +export type TrendRun = Pick< + TrendPoint, + "date" | "fixtures" | "partial" | "failed" | "byFixture" | "rsd" | "rsdByFixture" +>; + +type Column = readonly (number | null)[]; + +/** + * Each run's detail, by column: what the server sends instead of one object + * per run with every tool and fixture key repeated. The per-fixture seconds + * and spreads are most of a long trend's bytes, so they become one column per + * (tool, fixture) over the runs, `null` where that run has no value. Partial + * and failed are sparse and stay per run. Lossless -- nothing is rounded -- + * and `trendOf` gives back the runs. + */ +export interface TrendColumns { + readonly dates: readonly string[]; + /** Every fixture any run published, in first-seen order. */ + readonly fixtures: readonly string[]; + /** Which of `fixtures` each run published, by index, in the run's own order. */ + readonly published: readonly (readonly number[])[]; + readonly partial: readonly TrendRun["partial"][]; + readonly failed: readonly TrendRun["failed"][]; + readonly rsd: { readonly [tool: string]: Column }; + readonly byFixture: { readonly [tool: string]: { readonly [fixture: string]: Column } }; + readonly rsdByFixture: { readonly [tool: string]: { readonly [fixture: string]: Column } }; +} + +/** The runs' detail by column. */ +export const toTrendColumns = (runs: readonly TrendRun[]): TrendColumns => { + const fixtures = [...new Set(runs.flatMap((r) => r.fixtures))]; + const nested = ( + pick: (r: TrendRun) => { readonly [tool: string]: { readonly [fixture: string]: number } }, + ) => { + const out: { [tool: string]: { [fixture: string]: (number | null)[] } } = {}; + runs.forEach((run, i) => { + for (const [tool, byFixture] of Object.entries(pick(run))) { + for (const [fixture, value] of Object.entries(byFixture)) { + ((out[tool] ??= {})[fixture] ??= Array.from({ length: runs.length }, () => null))[i] = + value; + } + } + }); + return out; + }; + const rsd: { [tool: string]: (number | null)[] } = {}; + runs.forEach((run, i) => { + for (const [tool, value] of Object.entries(run.rsd)) { + (rsd[tool] ??= Array.from({ length: runs.length }, () => null))[i] = value; + } + }); + return { + dates: runs.map((r) => r.date), + fixtures, + published: runs.map((r) => r.fixtures.map((f) => fixtures.indexOf(f))), + partial: runs.map((r) => r.partial), + failed: runs.map((r) => r.failed), + rsd, + byFixture: nested((r) => r.byFixture), + rsdByFixture: nested((r) => r.rsdByFixture), + }; +}; + +/** The runs a `TrendColumns` encodes: a tool or fixture with no value on a run is absent, as it was. */ +export const trendOf = (columns: TrendColumns): TrendRun[] => { + const unnest = ( + by: TrendColumns["byFixture"], + i: number, + ): { [tool: string]: { [fixture: string]: number } } => { + const out: { [tool: string]: { [fixture: string]: number } } = {}; + for (const [tool, byFixture] of Object.entries(by)) { + for (const [fixture, column] of Object.entries(byFixture)) { + const value = column[i]; + if (typeof value === "number") (out[tool] ??= {})[fixture] = value; + } + } + return out; + }; + return columns.dates.map((date, i) => ({ + date, + fixtures: (columns.published[i] ?? []).map((f) => columns.fixtures[f] ?? ""), + partial: columns.partial[i] ?? {}, + failed: columns.failed[i] ?? {}, + byFixture: unnest(columns.byFixture, i), + rsd: Object.fromEntries( + Object.entries(columns.rsd).flatMap(([tool, column]) => { + const value = column[i]; + return typeof value === "number" ? [[tool, value]] : []; + }), + ), + rsdByFixture: unnest(columns.rsdByFixture, i), + })); +}; + /* ------------------------------------------------------------------ */ /* Presentation tokens shared by every chart */ /* ------------------------------------------------------------------ */ diff --git a/apps/www/src/lib/load.ts b/apps/www/src/lib/load.ts index 37ef5a4d4e..8b868f572e 100644 --- a/apps/www/src/lib/load.ts +++ b/apps/www/src/lib/load.ts @@ -8,7 +8,12 @@ * and it gives a failed run a time. The files are the measurement. */ import type { BenchmarkFile, Fixture, PackageCount, ProcessCount } from "@benchmarks/types"; -import { DataClient, type DateSummary } from "@benchmarks/data"; +import { + DataClient, + summaryPackageCounts, + type DateSummary, + type SummaryResult, +} from "@benchmarks/data"; import { probe } from "./data"; import { applyExclusions } from "./figures"; import { averageFiles, usable } from "./measurements"; @@ -54,8 +59,16 @@ export interface TrendPoint { export interface ScenarioData { readonly bundles: readonly FixtureBundle[]; readonly trend: readonly TrendPoint[]; - /** What the trend actually cost, so the number is reported rather than guessed. */ - readonly trendCost: { readonly requests: number; readonly dates: number }; + /** + * What the trend actually cost, so the number is reported rather than + * guessed: its requests, the runs it drew, and what a run without a summary + * costs, which is what cut the window when `dates` is short of it. + */ + readonly trendCost: { + readonly requests: number; + readonly dates: number; + readonly perRun: number; + }; } const asFixture = (name: string): Fixture => name as Fixture; @@ -172,12 +185,31 @@ export const trendPoint = ( }; }; +/* What a summary leaves out of a result; nothing on a trend reads it. */ +const unsummarised = { + median: Number.NaN, + min: Number.NaN, + max: Number.NaN, + user: Number.NaN, + system: Number.NaN, + times: [], +}; + +/** A summarised result as a benchmark file's: the fields a trend reads, verbatim. */ +const fromSummary = (results: readonly SummaryResult[]): BenchmarkFile => ({ + results: results.map((r) => ({ ...r, ...unsummarised })), +}); + /** * The trend series behind the sparklines and the time chart. * - * Cost is `dates x fixtures` requests, doubled when package counts are asked - * for; `trendCost` reports the real number. Dates the scenario did not run on - * contribute no point, which is why the series can be shorter than the window. + * A date with a summary (from `loadSummaries`) reads its files out of it: the + * same mean, stddev and exit codes, so the same DNF rule, for one fetch already + * made. A date without costs one request per fixture. Package counts, when + * asked for, come from the summary too where it carries that fixture's, and + * otherwise cost one request per fixture that ran; `requests` counts what this function + * fetched. Dates the scenario did not run on contribute no + * point, which is why the series can be shorter than the window. */ const loadTrend = async ( client: DataClient, @@ -185,91 +217,132 @@ const loadTrend = async ( fixtures: readonly string[], variation: string, withCounts: boolean, + summaries: ReadonlyMap = new Map(), ): Promise<{ trend: TrendPoint[]; requests: number }> => { + let requests = 0; const perDate = await Promise.all( dates.map(async (date) => { - const loaded = await Promise.all( + const summary = summaries.get(date); + const files = await Promise.all( fixtures.map(async (fixture) => { - const [file, counts] = await Promise.all([ - probe(() => client.getBenchmark(date, asFixture(fixture), variation)), - withCounts - ? probe(() => - client.getPackageCount(date, asFixture(fixture), variation), - ) - : null, - ]); - return { - fixture, - file: file.state === "present" ? file.value : null, - counts: counts?.state === "present" ? counts.value : null, - }; + if (summary !== undefined) { + const results = summary[variation]?.[fixture]; + return { fixture, file: results === undefined ? null : fromSummary(results) }; + } + requests++; + const file = await probe(() => + client.getBenchmark(date, asFixture(fixture), variation), + ); + return { fixture, file: file.state === "present" ? file.value : null }; }), ); - return trendPoint( - date, - loaded.flatMap((l) => (l.file === null ? [] : [{ fixture: l.fixture, file: l.file }])), - loaded.flatMap((l) => - l.counts === null ? [] : [{ fixture: l.fixture, counts: l.counts }], - ), + const ran = files.flatMap((f) => + f.file === null ? [] : [{ fixture: f.fixture, file: f.file }], ); + /* + * A fixture the summary has no count for falls back to its file: the + * build skips a count file it cannot parse, and absent is never zero. + */ + const summarised = summary === undefined ? undefined : summaryPackageCounts(summary); + const counts = withCounts + ? await Promise.all( + ran.map(async ({ fixture }) => { + const carried = summarised?.[variation]?.[fixture]; + if (carried !== undefined) return [{ fixture, counts: carried }]; + requests++; + const c = await probe(() => + client.getPackageCount(date, asFixture(fixture), variation), + ); + return c.state === "present" ? [{ fixture, counts: c.value }] : []; + }), + ) + : []; + return trendPoint(date, ran, counts.flat()); }), ); - return { - trend: perDate.filter((p): p is TrendPoint => p !== null), - requests: dates.length * fixtures.length * (withCounts ? 2 : 1), - }; + return { trend: perDate.filter((p): p is TrendPoint => p !== null), requests }; }; +/** + * The shown run and its trend. `summaries` may still be loading: the run's + * own files do not wait for it, and only the trend does, since which runs it + * can afford depends on which have one. + */ export const loadScenario = async ( client: DataClient, date: string, variation: string, fixtures: readonly string[], - trendDates: readonly string[], - options: { readonly exclude?: readonly string[]; readonly perPackage?: boolean } = {}, + windowDates: readonly string[], + options: { + readonly exclude?: readonly string[]; + readonly perPackage?: boolean; + /** From `loadSummaries`: a summarised run costs one fetch, not one per fixture. */ + readonly summaries?: Summaries | Promise; + } = {}, ): Promise => { + const withCounts = options.perPackage === true; + /* A run without a summary: a file per fixture, and its count where that is wanted. */ + const perRun = fixtures.length * (withCounts ? 2 : 1); const [bundles, trend] = await Promise.all([ Promise.all(fixtures.map((f) => loadFixture(client, date, f, variation))), - loadTrend(client, trendDates, fixtures, variation, options.perPackage === true), + (async () => { + const summaries = await options.summaries; + const dates = trendDates(windowDates, perRun, new Set(summaries?.keys())); + return { + dates, + ...(await loadTrend(client, dates, fixtures, variation, withCounts, summaries)), + }; + })(), ]); return { bundles: bundles.filter((b): b is FixtureBundle => b !== null), trend: applyExclusions(trend.trend, options.exclude ?? []), - trendCost: { requests: trend.requests, dates: trendDates.length }, + trendCost: { + requests: trend.requests + (options.summaries === undefined ? 0 : windowDates.length), + dates: trend.dates.length, + perRun, + }, }; }; +/** Each date's summary, from `loadSummaries`. */ +export type Summaries = ReadonlyMap; + /** - * What the derived "average" view's trend may fetch per page. One run costs a - * file per (fixture, scenario) -- 54 for package managers -- so a 30-day - * window would be about 1,600 requests; the window is cut to its most recent - * runs instead. + * What a trend may fetch per page beyond its summaries. A run without one + * costs a request per file it reads -- for the derived "average" a file per + * (fixture, scenario), 54 for package managers, so a 30-day window would be + * about 1,600 requests -- and the window is cut to its most recent such runs + * instead. A missing or failed summary never fans out past it. */ -export const AVERAGE_TREND_BUDGET = 400; +export const TREND_BUDGET = 400; /** - * The window's dates the average's trend draws: every date with a summary, - * which costs one fetch, and of the rest the most recent whose files fit the - * budget, never fewer than two. + * The window's dates a trend draws: every date with a summary, which costs + * one fetch, and of the rest the most recent whose files fit the budget, + * never fewer than two. */ -export const averageTrendDates = ( +export const trendDates = ( dates: readonly string[], - cellsPerDate: number, + requestsPerDate: number, summarised: ReadonlySet = new Set(), ): string[] => { - const raw = dates - .filter((d) => !summarised.has(d)) - .slice(-Math.max(2, Math.floor(AVERAGE_TREND_BUDGET / Math.max(cellsPerDate, 1)))); - return dates.filter((d) => summarised.has(d) || raw.includes(d)); + const raw = new Set( + dates + .filter((d) => !summarised.has(d)) + .slice(-Math.max(2, Math.floor(TREND_BUDGET / Math.max(requestsPerDate, 1)))), + ); + return dates.filter((d) => summarised.has(d) || raw.has(d)); }; /** Each date's summary, where a deploy has written one; a date without is left out. */ export const loadSummaries = async ( client: DataClient, dates: readonly string[], -): Promise> => { +): Promise => { const found = await Promise.all(dates.map((d) => probe(() => client.getSummary(d)))); return new Map( dates.flatMap((d, i) => { @@ -286,16 +359,6 @@ export interface RunCell { readonly file: BenchmarkFile; } -/* What a summary leaves out of a result; nothing on a trend reads it. */ -const unsummarised = { - median: Number.NaN, - min: Number.NaN, - max: Number.NaN, - user: Number.NaN, - system: Number.NaN, - times: [], -}; - const loadCells = async ( client: DataClient, date: string, @@ -306,15 +369,7 @@ const loadCells = async ( return Object.entries(plan).flatMap(([fixture, variations]) => variations.flatMap((variation) => { const results = summary[variation]?.[fixture]; - return results === undefined - ? [] - : [ - { - fixture, - variation, - file: { results: results.map((r) => ({ ...r, ...unsummarised })) }, - }, - ]; + return results === undefined ? [] : [{ fixture, variation, file: fromSummary(results) }]; }), ); } @@ -341,21 +396,26 @@ export const loadAverage = async ( client: DataClient, date: string, plan: { readonly [fixture: string]: readonly string[] }, - trendDates: readonly string[], + windowDates: readonly string[], options: { readonly exclude?: readonly string[]; - /** From `loadSummaries`: a summarised run costs one fetch, not one per file. */ - readonly summaries?: ReadonlyMap; + /** From `loadSummaries`, possibly still loading, as for `loadScenario`. */ + readonly summaries?: Summaries | Promise; } = {}, ): Promise => { // The run shown is usually the trend's last date; fetched once either way, - // and from its files, which the report reads in full. - const dates = trendDates.includes(date) ? trendDates : [...trendDates, date]; - const summaryOf = (d: string) => (d === date ? undefined : options.summaries?.get(d)); - const loaded = await Promise.all(dates.map((d) => loadCells(client, d, plan, summaryOf(d)))); - const trend = trendDates.flatMap( - (d, i) => trendPoint(d, averageFiles(loaded[i] ?? [], plan)) ?? [], + // and from its files, which the report reads in full, without waiting for + // the summaries. + const shown = loadCells(client, date, plan); + const summaries = await options.summaries; + const perRun = Object.values(plan).flat().length; + const drawn = trendDates(windowDates, perRun, new Set(summaries?.keys())); + const dates = drawn.includes(date) ? drawn : [...drawn, date]; + const summaryOf = (d: string) => (d === date ? undefined : summaries?.get(d)); + const loaded = await Promise.all( + dates.map((d) => (d === date ? shown : loadCells(client, d, plan, summaryOf(d)))), ); + const trend = drawn.flatMap((d, i) => trendPoint(d, averageFiles(loaded[i] ?? [], plan)) ?? []); const cells = loaded[dates.indexOf(date)] ?? []; return { @@ -367,14 +427,11 @@ export const loadAverage = async ( })), trend: applyExclusions(trend, options.exclude ?? []), trendCost: { - requests: dates.reduce( - (sum, d) => - sum + - (options.summaries === undefined ? 0 : 1) + - (summaryOf(d) === undefined ? Object.values(plan).flat().length : 0), - 0, - ), - dates: trendDates.length, + requests: + (summaries === undefined ? 0 : windowDates.length) + + dates.filter((d) => summaryOf(d) === undefined).length * perRun, + dates: drawn.length, + perRun, }, }; }; diff --git a/apps/www/src/lib/markdown.ts b/apps/www/src/lib/markdown.ts index d25fbb19eb..27c7bbbd8b 100644 --- a/apps/www/src/lib/markdown.ts +++ b/apps/www/src/lib/markdown.ts @@ -16,7 +16,8 @@ import { import { formatDuration, formatVersus } from "./findings"; import { formatMs, type CountRow, type FixtureRow } from "./measurements"; import { METHOD, RESOURCES, RUNNER, RUNNER_NOTE } from "./metadata"; -import { reportHref } from "./params"; +import { rowsOf } from "./chart-model"; +import { rangeLabel, reportHref } from "./params"; import type { TrendPoint } from "./load"; import type { ReportModel } from "./report"; import type { VersionMarker } from "./version-markers"; @@ -107,12 +108,31 @@ const counts = (report: ReportModel, rows: readonly CountRow[]): string => ]), ); +/* + * How a trend's CSV is laid out: a row a calendar day, or -- past + * MAX_DENSE_DAYS, where the chart stops spacing runs by date -- a row a run. + */ +const layout = (perDay: boolean) => + perDay + ? { rows: "one row per calendar day", row: "day" } + : { rows: "one row per run (the window is too long for a row a day)", row: "run" }; + +const sentence = (text: string) => text.charAt(0).toUpperCase() + text.slice(1); + +const DAY_MS = 86_400_000; + +/* Whether a history's rows are consecutive days, as a dense series' are. */ +const perDay = (rows: readonly { readonly date: string }[]): boolean => + rows.every( + (r, i) => i === 0 || Date.parse(r.date) - Date.parse(rows[i - 1]?.date ?? "") === DAY_MS, + ); + const label = (report: ReportModel, tool: string): string => report.leaderboard.find((r) => r.tool === tool)?.label ?? tool; /** * The trend as agents read it best: a CSV block, one row per calendar day - * with empty cells for gaps, then the DNFs, coverage, version changes and + * (or per run, past MAX_DENSE_DAYS) with empty cells for gaps, then the DNFs, coverage, version changes and * outliers that explain them. */ const trendLines = ({ @@ -220,7 +240,7 @@ export const reportMarkdown = (report: ReportModel, origin: string): string => { `- Measured: mean ${report.measure} time ${filters.fixture === null ? `across ${fixtureCount} ${fixtureCount === 1 ? "fixture" : "fixtures"}` : `on the ${fixtureLabel(filters.fixture)} fixture`}${scenario.averageOf === null ? "" : `, averaged over ${scenario.averageOf.scenarios.length} scenarios (derived, not measured)`}, on GitHub-hosted ubuntu-24.04-arm runners.${section.id === "registries" ? " Every registry is installed from with the same npm client." : ""}`, `- Filters in effect:`, ` - fixture: ${filters.fixture === null ? `all (${report.fixtures.join(", ")})` : filters.fixture}`, - ` - trend window: ${filters.window.custom ? "custom" : `last ${filters.range} days`}, ${filters.window.from} to ${filters.window.to}`, + ` - trend window: ${filters.window.custom ? "custom" : rangeLabel(filters.range).toLowerCase()}, ${filters.window.from} to ${filters.window.to}`, ` - trend metric: ${METRIC_TEXT[metric].name}`, ` - hidden tools: ${filters.hide.length === 0 ? "none" : filters.hide.join(", ")}`, ` - excluded runs: ${filters.exclude.length === 0 ? "none" : filters.exclude.join(", ")}`, @@ -353,13 +373,19 @@ export const reportMarkdown = (report: ReportModel, origin: string): string => { "", ); } + const grid = layout(trend.series.dense); add( "### Performance over time", "", - `${metric === "total" ? `Mean ${report.measure} time in seconds` : METRIC_TEXT[metric].unit}, one row per calendar day from ${filters.window.from} to ${filters.window.to}.${trend.cut === null ? "" : ` ${trend.cut}`} An empty cell is a day with no published value for that tool: no run, a DNF, a partial run or an exclusion. The chart bridges those with a dotted line by default; nothing is interpolated here. For other windows use the history endpoint: ${abs(`/api/v1/${section.id}/${encodeURIComponent(scenario.variation)}/history`)}`, + `${metric === "total" ? `Mean ${report.measure} time in seconds` : METRIC_TEXT[metric].unit}, ${grid.rows} from ${filters.window.from} to ${filters.window.to}.${trend.cut === null ? "" : ` ${trend.cut}`} An empty cell is a ${grid.row} with no published value for that tool: no run, a DNF, a partial run or an exclusion. The chart bridges those with a dotted line by default; nothing is interpolated here. For other windows use the history endpoint: ${abs(`/api/v1/${section.id}/${encodeURIComponent(scenario.variation)}/history`)}`, "", ...trendLines({ - rows: metricRows(trend.rows, [...report.ranked, ...report.unranked], metric, report.baseline), + rows: metricRows( + rowsOf(trend.series), + [...report.ranked, ...report.unranked], + metric, + report.baseline, + ), runs: trend.points, markers: trend.markers, outliers: trend.outliers, @@ -419,10 +445,11 @@ export const reportMarkdown = (report: ReportModel, origin: string): string => { add( "### Per-package time by fixture (ms per installed package, derived)", "", - "Each fixture's per-package time, run by run, one row per calendar day; an empty cell is no value for that tool on that day.", + `Each fixture's per-package time, run by run, ${grid.rows}; an empty cell is no value for that tool on that ${grid.row}.`, "", ); - for (const { fixture, rows } of trend.perPackageByFixture) { + for (const { fixture, series } of trend.perPackageByFixture) { + const rows = rowsOf(series); add( `#### ${fixtureLabel(fixture)}`, "", @@ -523,6 +550,7 @@ export const reportMarkdown = (report: ReportModel, origin: string): string => { /** A scenario's history (`historyOf`) as Markdown: what was asked, then the trend. */ export const historyMarkdown = (history: History, origin: string): string => { const section = history.section as SectionId; + const grid = layout(perDay(history.rows)); const label = (tool: string) => toolLabel(tool, section); return [ `# ${SECTION_DEFS[section].label} — ${history.scenario}: history`, @@ -534,7 +562,7 @@ export const historyMarkdown = (history: History, origin: string): string => { `- Tools: ${history.tools.map((t) => (label(t) === t ? t : `${label(t)} (${t})`)).join(", ")}`, `- Excluded runs: ${history.excluded.length === 0 ? "none" : history.excluded.join(", ")}`, "", - "One row per calendar day. An empty cell is a day with no comparable value for that tool: no run, a DNF, a partial run or an exclusion. Nothing is interpolated.", + `${sentence(grid.rows)}. An empty cell is a ${grid.row} with no comparable value for that tool: no run, a DNF, a partial run or an exclusion. Nothing is interpolated.`, "", ...trendLines({ rows: history.rows, diff --git a/apps/www/src/lib/mcp.ts b/apps/www/src/lib/mcp.ts index 8dc59e123a..c3a45a68af 100644 --- a/apps/www/src/lib/mcp.ts +++ b/apps/www/src/lib/mcp.ts @@ -43,7 +43,7 @@ export const inputs = { section, scenario, tools: tools.optional(), - from: date.optional().describe("Window start, a published date; with to. At most 120 days."), + from: date.optional().describe("Window start, a published date; with to. Any span."), to: date .optional() .describe("Window end, a published date. Omitted: the 30 days to the latest run."), diff --git a/apps/www/src/lib/metadata.ts b/apps/www/src/lib/metadata.ts index f1f50c27c9..afec204fa7 100644 --- a/apps/www/src/lib/metadata.ts +++ b/apps/www/src/lib/metadata.ts @@ -26,7 +26,8 @@ const https = (host?: string) => (host ? `https://${host}` : undefined); * out (vltpkg/www/docs/src/lib/site-url.ts, after Next's own social-image * fallback): localhost in development, so a card previews on the machine * serving it; the preview URL on a Vercel preview; the project's production - * domain in production, and benchmarks.vlt.sh where nothing says otherwise. + * domain in production, and benchmarks.vlt.io where nothing says otherwise + * (benchmarks.vlt.sh only redirects there). * SITE_URL overrides all of it. */ export const siteUrl = new URL( @@ -36,7 +37,7 @@ export const siteUrl = new URL( : process.env.VERCEL_ENV === "preview" ? https(process.env.VERCEL_BRANCH_URL ?? process.env.VERCEL_URL) : https(process.env.VERCEL_PROJECT_PRODUCTION_URL)) ?? - "https://benchmarks.vlt.sh", + "https://benchmarks.vlt.io", ); /** diff --git a/apps/www/src/lib/opengraph/templates.tsx b/apps/www/src/lib/opengraph/templates.tsx index 34ba86d105..5437098f98 100644 --- a/apps/www/src/lib/opengraph/templates.tsx +++ b/apps/www/src/lib/opengraph/templates.tsx @@ -10,6 +10,7 @@ import { join } from "node:path"; import { createElement, type ComponentType, type ReactElement, type ReactNode } from "react"; import { Vlt } from "@/components/icons/vlt"; import { TOOL_ICONS } from "@/components/tool-name"; +import { rowsOf } from "@/lib/chart-model"; import { shownBars } from "@/lib/figures"; import { formatDuration, longDate } from "@/lib/findings"; import { formatMs } from "@/lib/measurements"; @@ -443,7 +444,7 @@ export const FigureCard = ({ report.trend.perPackageByFixture[0]) : null; if (lane != null) extra = [`on ${fixtureLabel(lane.fixture)}`]; - const rows = lane?.rows ?? report.trend.rows; + const rows = rowsOf(lane?.series ?? report.trend.series); const unit = per || report.trend.plotted === "per-package" ? "ms" : "s"; const tools = order.filter((t) => rows.some((r) => typeof r[t] === "number")); const { svg, max } = lineSvg(rows, tools, colors, report.baseline, PLOT); diff --git a/apps/www/src/lib/params.ts b/apps/www/src/lib/params.ts index 00c5a40147..a11241d0b8 100644 --- a/apps/www/src/lib/params.ts +++ b/apps/www/src/lib/params.ts @@ -23,16 +23,51 @@ */ import { createLoader, + createParser, createSerializer, parseAsArrayOf, - parseAsNumberLiteral, parseAsString, parseAsStringLiteral, } from "nuqs/server"; import { METRICS } from "./figures"; -/** Trend windows, in calendar days ending at the run being shown. */ -export const RANGES = [30, 60, 90, 120] as const; +/** + * The trend windows offered, in rolling calendar days ending at the run being + * shown -- not calendar months. `today` is that run alone; `all` has no lower + * bound: every published date up to it. + */ +export const PRESETS = [ + { value: "today", label: "Today", days: 1 }, + { value: "1w", label: "1 week", days: 7 }, + { value: "1m", label: "1 month", days: 30 }, + { value: "1y", label: "1 year", days: 365 }, + { value: "all", label: "All time", days: null }, +] as const; +export type Preset = (typeof PRESETS)[number]["value"]; +/** Old links: `range=30|60|90|120` keep meaning N rolling days. Parsed, never written. */ +export const LEGACY_RANGES = [30, 60, 90, 120] as const; +export type RangeValue = Preset | (typeof LEGACY_RANGES)[number]; + +const presetOf = (value: string): (typeof PRESETS)[number] | undefined => + PRESETS.find((p) => p.value === value); + +/** How many days a range reaches back, or `null` for no lower bound. */ +export const rangeDays = (range: RangeValue): number | null => + typeof range === "number" ? range : (presetOf(range)?.days ?? null); + +/** A range in words: the preset's label, or "Last N days" for an old link. */ +export const rangeLabel = (range: RangeValue): string => + typeof range === "number" ? `Last ${range} days` : (presetOf(range)?.label ?? range); + +/** A preset name or a legacy day count; anything else is not a range. */ +const parseAsRange = createParser({ + parse: (value) => { + if (presetOf(value) !== undefined) return value as Preset; + const legacy = LEGACY_RANGES.find((n) => String(n) === value); + return legacy ?? null; + }, + serialize: String, +}); export const SCALES = ["linear", "log"] as const; export const SMOOTHING = ["off", "median7"] as const; export const GAPS = ["connect", "hole"] as const; @@ -42,8 +77,8 @@ export const reportParams = { date: parseAsString, /** SERVER. One fixture; absent means all of them, averaged. */ fixture: parseAsString, - /** SERVER. A preset trend window: this many days, ending at the run shown. */ - range: parseAsNumberLiteral(RANGES).withDefault(30), + /** SERVER. A preset trend window, ending at the run shown; see `PRESETS`. */ + range: parseAsRange.withDefault("1m"), /** SERVER. An exact trend window. Both must be published dates; see `trendWindow`. */ from: parseAsString, to: parseAsString, @@ -148,9 +183,6 @@ export const withTableSort = ( /* The trend window */ /* ------------------------------------------------------------------ */ -/** The longest window offered: the span that can be spaced by calendar date. */ -export const MAX_WINDOW_DAYS = 120; - const DAY = 86_400_000; const day = (date: string): number => Date.parse(`${date}T00:00:00Z`); @@ -161,6 +193,8 @@ export interface TrendWindow { readonly custom: boolean; readonly from: string; readonly to: string; + /** The preset asked for; the one applied whenever `custom` is false. */ + readonly range: RangeValue; } /** @@ -168,10 +202,14 @@ export interface TrendWindow { * * An exact window is honoured only if it can be: both ends are dates the * archive actually published (a range cannot start or end on a hole), `from` - * is not after `to`, `to` is not after the run being shown, and the span is - * within {@link MAX_WINDOW_DAYS}. Anything else -- a hand-edited URL, a link - * that outlived its dates -- falls back to the preset rather than to an empty - * chart or an error. + * is not after `to`, and `to` is not after the run being shown. There is no + * cap on the span: the axis is spaced by calendar time for any window the + * archive can hold (see MAX_DENSE_DAYS). Anything else -- a hand-edited URL, + * a link that outlived its dates -- falls back to the preset rather than to + * an empty chart or an error. + * + * The same rule serves the site, `/api/v1` and MCP; agents get no separate + * cap. */ export const trendWindow = ({ dates, @@ -184,7 +222,7 @@ export const trendWindow = ({ readonly dates: readonly string[]; /** The run the report is showing. */ readonly end: string; - readonly range: number; + readonly range: RangeValue; readonly from: string | null; readonly to: string | null; }): TrendWindow => { @@ -194,15 +232,16 @@ export const trendWindow = ({ dates.includes(from) && dates.includes(to) && from <= to && - to <= end && - (day(to) - day(from)) / DAY < MAX_WINDOW_DAYS + to <= end ) { - return { dates: dates.filter((d) => d >= from && d <= to), custom: true, from, to }; + return { dates: dates.filter((d) => d >= from && d <= to), custom: true, from, to, range }; } + const days = rangeDays(range); const inRange = dates.filter((d) => { if (d > end) return false; + if (days === null) return true; const age = (day(end) - day(d)) / DAY; - return Number.isFinite(age) && age < Math.min(range, MAX_WINDOW_DAYS); + return Number.isFinite(age) && age < days; }); - return { dates: inRange, custom: false, from: inRange[0] ?? end, to: end }; + return { dates: inRange, custom: false, from: inRange[0] ?? end, to: end, range }; }; diff --git a/apps/www/src/lib/report.ts b/apps/www/src/lib/report.ts index 2e70b2eb3a..a8792c84e5 100644 --- a/apps/www/src/lib/report.ts +++ b/apps/www/src/lib/report.ts @@ -8,23 +8,17 @@ * and the same file is the source of the section lists this report refuses * to trust. */ -import { benchmarkFile, type DataClient } from "@benchmarks/data"; +import { benchmarkFile, summaryVersions, type DataClient } from "@benchmarks/data"; import type { Fixture, Versions } from "@benchmarks/types"; import type { inferParserType } from "nuqs/server"; import { publishedVariations } from "./catalog"; -import { toDenseChartRows } from "./chart-model"; +import { rowsOf, toDenseSeries, type DenseSeries } from "./chart-model"; import { citeFor, type Citation } from "./cite"; import { dataClient, probe } from "./data"; import { flagOutliers, meanBars, type Bar } from "./figures"; import { formatDuration, formatRatio, type Finding } from "./findings"; import { scenarioFacts, scopedFinding } from "./headline"; -import { - averageTrendDates, - loadAverage, - loadScenario, - loadSummaries, - type TrendPoint, -} from "./load"; +import { loadAverage, loadScenario, loadSummaries, type TrendPoint } from "./load"; import { countRows, perPackageByFixture, @@ -155,16 +149,19 @@ export interface ReportModel { readonly outline: readonly OutlineSection[]; readonly trend: { readonly points: readonly TrendPoint[]; - /** One row per calendar day, holes as `null`, in the plotted metric. */ - readonly rows: readonly { date: string; [tool: string]: string | number | null }[]; - /** What `rows` holds: total seconds, or milliseconds per installed package. */ + /** + * One row per calendar day, holes as `null`, in the plotted metric -- by + * column, as the chart island receives it; `rowsOf` gives the rows. + */ + readonly series: DenseSeries; + /** What `series` holds: total seconds, or milliseconds per installed package. */ readonly plotted: "total" | "per-package"; /** Whether per-package can be plotted here at all. */ readonly perPackage: boolean; /** Each fixture's milliseconds per installed package, one row per calendar day, holes as `null`. */ readonly perPackageByFixture: readonly { readonly fixture: string; - readonly rows: readonly { date: string; [tool: string]: string | number | null }[]; + readonly series: DenseSeries; }[]; /** Whether the window ends at the run shown, so "since last run" is about it. */ readonly endsHere: boolean; @@ -346,14 +343,16 @@ export const loadReport = async ({ }); /* - * Trend window: a preset number of calendar days ending at this run, or an - * exact from/to. Resolved and validated in `trendWindow`. + * Trend window: a preset of rolling calendar days ending at this run (up to + * the whole archive), or an exact from/to. Resolved and validated in + * `trendWindow`; there is no cap. * * By TIME, not by count. The archive is missing 53 calendar days, so "the * last 16 runs" can reach back the better part of a year, and a line joining * runs ten months apart is not a trend -- it is two unrelated states drawn - * as a slope. Every window is inside MAX_DENSE_DAYS (120), so the axis is - * always spaced by real calendar time and a gap is always a gap. + * as a slope. MAX_DENSE_DAYS (1,100) holds the whole archive, so the axis is + * spaced by real calendar time and a gap is a gap; a window past it falls + * back to an evenly spaced axis that says so. */ const window = trendWindow({ dates: index.dates, @@ -368,24 +367,25 @@ export const loadReport = async ({ * them -- the report's numbers are about it. */ const exclude = query.exclude.filter((e) => e.split("@").at(-1) !== date); - /* The derived view's trend reads each run's summary; a run without one is cut to what the request budget allows. */ + /* + * Both views' trends read each run's summary: one fetch a run, whatever the + * window, All time included. A run without one costs its files, cut to what + * the request budget allows. The shown run's files load while the summaries + * do. + */ const shownPlan = Object.fromEntries(fixtures.map((f) => [f, plan[f] ?? []])); - const filesPerRun = Object.values(shownPlan).flat().length; - const summaries = averageOf === null ? undefined : await loadSummaries(client, window.dates); - const trendDates = - summaries === undefined - ? window.dates - : averageTrendDates(window.dates, filesPerRun, new Set(summaries.keys())); + const summaries = loadSummaries(client, window.dates); const average = averageOf === null ? null - : await loadAverage(client, date, shownPlan, trendDates, { exclude, summaries }); + : await loadAverage(client, date, shownPlan, window.dates, { exclude, summaries }); const data = average ?? - (await loadScenario(client, date, selected.variation, fixtures, trendDates, { + (await loadScenario(client, date, selected.variation, fixtures, window.dates, { exclude, /* The per-package figures by fixture draw every run, so the counts are always wanted where there are any. */ perPackage: section !== "task-runners", + summaries, })); const facts = scenarioFacts( section, @@ -394,8 +394,13 @@ export const loadReport = async ({ ); const { tools, means, timeRows, baseline } = facts; + /* Each run's versions: from its summary where that carries them, else its versions.json. */ + const summarisedRuns = await summaries; const versionSeries = await Promise.all( data.trend.map(async (p) => { + const summarised = summarisedRuns.get(p.date); + const carried = summarised === undefined ? undefined : summaryVersions(summarised); + if (carried !== undefined) return { date: p.date, versions: carried }; const v = await probe(() => client.getVersions(p.date)); return { date: p.date, versions: v.state === "present" ? v.value : null }; }), @@ -439,23 +444,22 @@ export const loadReport = async ({ /* Per package, the trend plots the per-package series the loader built. */ const perPackageTrend = averageOf === null && section !== "task-runners"; const plotted = perPackageTrend && query.metric === "per-package" ? "per-package" : "total"; - const dense = toDenseChartRows( + const plottedPoints = plotted === "per-package" ? data.trend.map((p) => ({ date: p.date, byTool: p.perPackage ?? {} })) - : data.trend, - tools, - ); + : data.trend; + const series = toDenseSeries(plottedPoints, tools); /* Each fixture's per-package time, run by run, for the per-package figure by fixture. */ const perPackageTrends = perPackageTrend ? fixtures.map((f) => ({ fixture: f, - rows: toDenseChartRows( + series: toDenseSeries( data.trend.map((p) => ({ date: p.date, byTool: Object.fromEntries(tools.map((t) => [t, p.perPackageByFixture?.[t]?.[f]])), })), tools, - ).rows, + ), })) : []; const measure = isScript ? "execution" : "install"; @@ -503,10 +507,11 @@ export const loadReport = async ({ }; }); + const rows = rowsOf(series); const outliers = ranked.flatMap((tool) => - flagOutliers(dense.rows.map((r) => (typeof r[tool] === "number" ? r[tool] : null))).flatMap( + flagOutliers(rows.map((r) => (typeof r[tool] === "number" ? r[tool] : null))).flatMap( (deviation, i) => { - const day = dense.rows[i]?.date; + const day = rows[i]?.date; return deviation === null || day === undefined ? [] : [{ tool, date: day, deviation }]; }, ), @@ -590,7 +595,7 @@ export const loadReport = async ({ filters: { date: pinned, fixture, - range: query.range, + range: window.range, window: { from: window.from, to: window.to, custom: window.custom }, hide: hidden, metric: query.metric, @@ -624,7 +629,7 @@ export const loadReport = async ({ outline, trend: { points: data.trend, - rows: dense.rows, + series, plotted, perPackage: perPackageTrend, perPackageByFixture: perPackageTrends, @@ -633,8 +638,8 @@ export const loadReport = async ({ markers, outliers, cut: - trendDates.length < window.dates.length - ? `Each run here averages ${filesPerRun} result files, so the trend shows ${trendDates.length} of the window's ${window.dates.length} runs.` + data.trendCost.dates < window.dates.length + ? `A run without a summary costs ${data.trendCost.perRun} files here, so the trend shows ${data.trendCost.dates} of the window's ${window.dates.length} runs.` : null, focus, inFocus, diff --git a/apps/www/test/app-cwd.ts b/apps/www/test/app-cwd.ts new file mode 100644 index 0000000000..069e9bbd62 --- /dev/null +++ b/apps/www/test/app-cwd.ts @@ -0,0 +1,8 @@ +/* + * The app reads its own files from `process.cwd()` (lib/mdx.ts, lib/opengraph), which is how Next + * runs it: from apps/www, with server code bundled into .next where `import.meta.url` would not + * point at src/. The root `vlt run test` starts vitest at the repo root, so put the tests there too. + */ +import { fileURLToPath } from "node:url"; + +process.chdir(fileURLToPath(new URL("..", import.meta.url))); diff --git a/apps/www/test/average.test.ts b/apps/www/test/average.test.ts index 2a7898e692..085b81fb48 100644 --- a/apps/www/test/average.test.ts +++ b/apps/www/test/average.test.ts @@ -5,7 +5,7 @@ import type { BenchmarkFile } from "@benchmarks/types"; import { GET as api } from "../src/app/api/v1/[section]/[scenario]/route.ts"; import { GET as llms } from "../src/app/llms.txt/route.ts"; import { GET as markdown } from "../src/app/llms.mdx/[[...slug]]/route.ts"; -import { AVERAGE_TREND_BUDGET, averageTrendDates } from "../src/lib/load.ts"; +import { TREND_BUDGET, trendDates } from "../src/lib/load.ts"; import { averageFiles } from "../src/lib/measurements.ts"; import { treeFor } from "../src/lib/page-tree.ts"; import { loadReportParams } from "../src/lib/params.ts"; @@ -13,7 +13,7 @@ import { loadReport } from "../src/lib/report.ts"; import { AVERAGE, scenariosFor, withAverage } from "../src/lib/scenarios.ts"; import { DATA_URL, fakeFetch } from "./fixture-tree.ts"; -const ORIGIN = "https://benchmarks.vlt.sh"; +const ORIGIN = "https://benchmarks.vlt.io"; const result = (command: string, mean: number, exit = 0) => ({ command, @@ -133,13 +133,13 @@ describe("default routes", () => { }); }); -describe("averageTrendDates", () => { +describe("trendDates", () => { it("cuts the window to the newest runs the budget pays for", () => { const dates = Array.from({ length: 30 }, (_, i) => `d${i}`); - expect(averageTrendDates(dates, 54)).toEqual(dates.slice(-7)); - expect(7 * 54).toBeLessThanOrEqual(AVERAGE_TREND_BUDGET); - expect(averageTrendDates(dates, 12)).toEqual(dates); - expect(averageTrendDates(dates, 1000)).toHaveLength(2); + expect(trendDates(dates, 54)).toEqual(dates.slice(-7)); + expect(7 * 54).toBeLessThanOrEqual(TREND_BUDGET); + expect(trendDates(dates, 12)).toEqual(dates); + expect(trendDates(dates, 1000)).toHaveLength(2); }); }); @@ -241,15 +241,12 @@ describe("the average report's twins", () => { }); }); -describe("averageTrendDates with summaries", () => { +describe("trendDates with summaries", () => { it("keeps every summarised date and budgets only the rest", () => { const dates = Array.from({ length: 30 }, (_, i) => `d${String(i).padStart(2, "0")}`); const summarised = new Set(dates.slice(0, 20)); // 54 files a run leaves room for 7 raw runs: the last 7 of the 10 unsummarised. - expect(averageTrendDates(dates, 54, summarised)).toEqual([ - ...dates.slice(0, 20), - ...dates.slice(23), - ]); - expect(averageTrendDates(dates, 54, new Set(dates))).toEqual(dates); + expect(trendDates(dates, 54, summarised)).toEqual([...dates.slice(0, 20), ...dates.slice(23)]); + expect(trendDates(dates, 54, new Set(dates))).toEqual(dates); }); }); diff --git a/apps/www/test/fixture-tree.ts b/apps/www/test/fixture-tree.ts index 69523809ab..f64511e86f 100644 --- a/apps/www/test/fixture-tree.ts +++ b/apps/www/test/fixture-tree.ts @@ -4,7 +4,7 @@ * out loud -- zpm crashes on babylon (DNF, so partial and not ranked), npm * did not run on one date (a gap in the trend), and versions change. */ -import { DataClient } from "@benchmarks/data"; +import { buildSummary, DataClient, SUMMARY_PACKAGE_COUNTS } from "@benchmarks/data"; export const DATA_URL = "http://data.test"; export const DATES = ["2026-10-10", "2026-10-12", "2026-10-13"]; @@ -87,3 +87,46 @@ export const fakeFetch = async (url: string | URL | Request) => { }; export const fixtureClient = () => new DataClient({ baseUrl: DATA_URL, fetch: fakeFetch }); + +/** + * The same tree with each date's summary, built by apps/data from these files, + * and a log of every path fetched -- so a test can say both what a report + * drew and what it cost. `old` serves the shape published before summaries + * carried counts and versions: results only. + */ +export const summarisedClient = ({ + old = false, + uncounted, +}: { + old?: boolean; + /** A fixture whose count the summaries leave out, as the build does a count file it cannot parse. */ + uncounted?: string; +} = {}) => { + const fetched: string[] = []; + const fetch = async (url: string | URL | Request) => { + const path = decodeURIComponent(String(url).slice(DATA_URL.length + 1)); + fetched.push(path); + const date = /^summary\/(.+)\.json$/.exec(path)?.[1]; + if (date === undefined || !DATES.includes(date)) return fakeFetch(url); + const listing = { + name: date, + files: [...files.keys()] + .filter((p) => p.startsWith(`${date}/`)) + .map((p) => p.slice(date.length + 1)), + }; + const { summary } = buildSummary(listing, (file) => + JSON.stringify(files.get(`${date}/${file}`)), + ); + if (uncounted !== undefined) { + const counts = summary[SUMMARY_PACKAGE_COUNTS] as { [variation: string]: object } | undefined; + for (const byFixture of Object.values(counts ?? {})) { + delete (byFixture as { [fixture: string]: unknown })[uncounted]; + } + } + const served = old + ? Object.fromEntries(Object.entries(summary).filter(([key]) => !key.startsWith("#"))) + : summary; + return { ok: true, status: 200, json: async () => served }; + }; + return { client: new DataClient({ baseUrl: DATA_URL, fetch }), fetched }; +}; diff --git a/apps/www/test/honesty-inventory.test.ts b/apps/www/test/honesty-inventory.test.ts index 93cff2ae28..51802bc768 100644 --- a/apps/www/test/honesty-inventory.test.ts +++ b/apps/www/test/honesty-inventory.test.ts @@ -14,11 +14,14 @@ import { describe, expect, it } from "vitest"; import { alignedSeries, gappedPath, + MAX_DENSE_DAYS, + rowsOf, niceTicks, segments, spreadOf, toChartRows, toDenseChartRows, + toDenseSeries, } from "../src/lib/chart-model.ts"; import { PACKAGE_MANAGERS, usable, type ToolMean } from "../src/lib/measurements.ts"; import { findingFor, formatRatio, formatVersus } from "../src/lib/findings.ts"; @@ -246,16 +249,66 @@ describe("inventory 12 · a calendar gap is a real hole, not an equal step", () expect(rows).toHaveLength(2); }); - it("falls back rather than emitting 700 rows for a two-year window, and says so", () => { + it("keeps a whole-archive window on the calendar: one row per day", () => { const { dense, spanDays, rows } = toDenseChartRows( [ { date: "2025-08-22", byTool: { npm: 1 } }, + { date: "2026-09-26", byTool: { npm: 2 } }, + ], + ["npm"], + ); + expect(dense).toBe(true); + expect(spanDays).toBe(401); // calendar days, both ends included + expect(rows).toHaveLength(401); + expect(rows[1]?.["date"]).toBe("2025-08-23"); + }); + + it("by column, the same rows exactly: a 400-day span with holes and partial tools", () => { + // One run in three, a tool that skips some, another that never ran, an excluded zero. + const runs = Array.from({ length: 134 }, (_, i) => ({ + date: new Date(Date.parse("2025-08-22T00:00:00Z") + i * 3 * 86_400_000) + .toISOString() + .slice(0, 10), + byTool: { npm: 10 + i / 7, bun: i % 5 === 0 ? undefined : 1 / (i + 3), vlt: 0 }, + })); + const tools = ["npm", "bun", "vlt", "deno"]; + const series = toDenseSeries(runs, tools); + expect(series).toMatchObject({ start: "2025-08-22", length: 400, dense: true }); + expect(series.dates).toBeUndefined(); + expect(rowsOf(series)).toEqual(toDenseChartRows(runs, tools).rows); + expect(Object.keys(rowsOf(series)[0] ?? {})).toEqual(["date", ...tools]); + // Lossless: nothing rounded on the way. + expect(series.values["bun"]?.[3]).toBe(1 / 4); + }); + + it("by column, the sparse fallback keeps each run's date", () => { + const runs = [ + { date: "2022-01-01", byTool: { npm: 1 } }, + { date: "2024-02-29", byTool: { npm: undefined } }, + { date: "2026-06-29", byTool: { npm: 2 } }, + ]; + const series = toDenseSeries(runs, ["npm"]); + expect(series.dense).toBe(false); + expect(series.dates).toEqual(["2022-01-01", "2024-02-29", "2026-06-29"]); + expect(rowsOf(series)).toEqual(toDenseChartRows(runs, ["npm"]).rows); + }); + + it("by column, a single run and an empty trend", () => { + const one = [{ date: "2026-06-29", byTool: { npm: 2 } }]; + expect(rowsOf(toDenseSeries(one, ["npm"]))).toEqual(toDenseChartRows(one, ["npm"]).rows); + expect(rowsOf(toDenseSeries([], ["npm"]))).toEqual([]); + }); + + it("falls back rather than emitting thousands of rows past MAX_DENSE_DAYS, and says so", () => { + const { dense, spanDays, rows } = toDenseChartRows( + [ + { date: "2022-01-01", byTool: { npm: 1 } }, { date: "2026-06-29", byTool: { npm: 2 } }, ], ["npm"], ); expect(dense).toBe(false); - expect(spanDays).toBeGreaterThan(300); + expect(spanDays).toBeGreaterThan(MAX_DENSE_DAYS); expect(rows).toHaveLength(2); }); }); diff --git a/apps/www/test/mcp.test.ts b/apps/www/test/mcp.test.ts index 82d6ce7c23..1f82c42c1f 100644 --- a/apps/www/test/mcp.test.ts +++ b/apps/www/test/mcp.test.ts @@ -8,7 +8,7 @@ import { } from "../src/lib/mcp.ts"; import { fixtureClient } from "./fixture-tree.ts"; -const ORIGIN = "https://benchmarks.vlt.sh"; +const ORIGIN = "https://benchmarks.vlt.io"; describe("MCP tools", () => { it("list_benchmarks: the catalog and every run", async () => { diff --git a/apps/www/test/og.test.ts b/apps/www/test/og.test.ts index 42b99ec79f..f1149afd89 100644 --- a/apps/www/test/og.test.ts +++ b/apps/www/test/og.test.ts @@ -6,7 +6,7 @@ import { ellipsize } from "../src/lib/opengraph/truncate.ts"; import { DATA_URL, fakeFetch } from "./fixture-tree.ts"; const og = (path: string) => - GET(new Request(`https://benchmarks.vlt.sh/og/${path}`), { + GET(new Request(`https://benchmarks.vlt.io/og/${path}`), { params: Promise.resolve({ path: path.split("?")[0]?.split("/").filter(Boolean) }), }); diff --git a/apps/www/test/report.test.ts b/apps/www/test/report.test.ts index 614ad353a4..25d366893b 100644 --- a/apps/www/test/report.test.ts +++ b/apps/www/test/report.test.ts @@ -1,7 +1,10 @@ import { describe, expect, it } from "vitest"; +import { rowsOf, toTrendColumns, trendOf, type TrendRun } from "../src/lib/chart-model.ts"; import { loadReportParams } from "../src/lib/params.ts"; import { ignoredSelection, loadReport, type ReportModel } from "../src/lib/report.ts"; -import { fixtureClient } from "./fixture-tree.ts"; +import { DataClient } from "@benchmarks/data"; +import { loadScenario, loadSummaries, TREND_BUDGET } from "../src/lib/load.ts"; +import { DATA_URL, DATES, fakeFetch, fixtureClient, summarisedClient } from "./fixture-tree.ts"; const load = async (scenario: string | undefined, search = "", section = "package-managers") => loadReport({ @@ -40,14 +43,15 @@ describe("loadReport", () => { it("leaves gaps in the trend as gaps, and compares only with a real previous run", async () => { const r = await report(); - expect(r.trend.rows.map((row) => row.date)).toEqual([ + const rows = rowsOf(r.trend.series); + expect(rows.map((row) => row.date)).toEqual([ "2026-10-10", "2026-10-11", "2026-10-12", "2026-10-13", ]); - expect(r.trend.rows[1]?.["bun"]).toBeNull(); - expect(r.trend.rows[2]?.["npm"]).toBeNull(); + expect(rows[1]?.["bun"]).toBeNull(); + expect(rows[2]?.["npm"]).toBeNull(); expect(r.leaderboard.find((row) => row.tool === "npm")?.sinceLastRun).toBeNull(); expect(r.leaderboard.find((row) => row.tool === "bun")?.sinceLastRun).toBeCloseTo(-1 / 11); expect(r.trend.markers.map((m) => m.label)).toEqual(["vlt 1.1.0"]); @@ -95,3 +99,169 @@ describe("loadReport", () => { expect(() => structuredClone(r)).not.toThrow(); }); }); + +describe("a trend read from the summaries", () => { + const load = async (client: Parameters[0]["client"], search: string) => { + const result = await loadReport({ + section: "package-managers", + scenario: "clean", + query: loadReportParams(new URLSearchParams(search)), + client, + }); + if (result.status !== "ok") throw new Error(result.status); + return result.report; + }; + + it("draws exactly what the files draw: the same points, gaps, DNFs and per-package times", async () => { + for (const search of [ + "range=all", + "range=all&metric=per-package", + "range=all&fixture=babylon", + ]) { + const files = await load(fixtureClient(), search); + const { client } = summarisedClient(); + const summarised = await load(client, search); + expect(summarised.trend).toEqual(files.trend); + expect(summarised.leaderboard).toEqual(files.leaderboard); + } + }); + + it("costs one fetch a run instead of one per fixture", async () => { + const { client, fetched } = summarisedClient(); + await load(client, "range=all"); + const benchmarks = fetched.filter((p) => /^\d{4}-\d{2}-\d{2}\/[a-z]+-clean\.json$/.test(p)); + // Only the run shown reads its files, for its tables; the trend reads none. + expect(benchmarks.sort()).toEqual([ + "2026-10-13/astro-clean.json", + "2026-10-13/babylon-clean.json", + ]); + expect(fetched.filter((p) => p.startsWith("summary/")).sort()).toEqual( + DATES.map((d) => `summary/${d}.json`), + ); + }); + + const trendFetches = (fetched: readonly string[], kind: RegExp) => + fetched.filter((p) => kind.test(p) && !p.startsWith("2026-10-13/")).length; + + it("reads counts and versions from a summary that carries them, and fetches none", async () => { + const { client, fetched } = summarisedClient(); + const r = await load(client, "range=all&metric=per-package"); + expect(trendFetches(fetched, /-package-count\.json$/)).toBe(0); + expect(trendFetches(fetched, /\/versions\.json$/)).toBe(0); + // ...and draws exactly what the files draw. + const files = await load(fixtureClient(), "range=all&metric=per-package"); + expect(r.trend).toEqual(files.trend); + }); + + it("falls back to the files for a summary written before it carried them", async () => { + const { client, fetched } = summarisedClient({ old: true }); + const r = await load(client, "range=all&metric=per-package"); + // Two fixtures on each earlier run, and one versions.json each. + expect(trendFetches(fetched, /-package-count\.json$/)).toBe(2 * (DATES.length - 1)); + expect(trendFetches(fetched, /\/versions\.json$/)).toBe(DATES.length - 1); + const files = await load(fixtureClient(), "range=all&metric=per-package"); + expect(r.trend).toEqual(files.trend); + }); + + it("falls back per fixture for a count the summary leaves out", async () => { + const { client, fetched } = summarisedClient({ uncounted: "babylon" }); + const r = await load(client, "range=all&metric=per-package"); + // Only babylon's, on each earlier run; astro's come from the summary. + expect( + fetched.filter((p) => p.endsWith("-package-count.json") && !p.startsWith("2026-10-13/")), + ).toEqual(DATES.slice(0, -1).map((d) => `${d}/babylon-clean-package-count.json`)); + const files = await load(fixtureClient(), "range=all&metric=per-package"); + expect(r.trend).toEqual(files.trend); + }); +}); + +describe("a trend's fetch budget", () => { + /* Far more runs than the budget pays for, none with a summary. */ + const window = Array.from({ length: 300 }, (_, i) => + new Date(Date.UTC(2025, 0, 1 + i)).toISOString().slice(0, 10), + ); + const counting = (summaries: "missing" | "failed") => { + const fetched: string[] = []; + const fetch = async (url: string | URL | Request) => { + const path = decodeURIComponent(String(url).slice(DATA_URL.length + 1)); + fetched.push(path); + if (summaries === "failed" && path.startsWith("summary/")) { + return { ok: false, status: 500, json: async () => null }; + } + return fakeFetch(url); + }; + return { client: new DataClient({ baseUrl: DATA_URL, fetch }), fetched }; + }; + + for (const summaries of ["missing", "failed"] as const) { + it(`never fans out past it when every summary is ${summaries}`, async () => { + const { client, fetched } = counting(summaries); + const data = await loadScenario(client, "2026-10-13", "clean", ["astro", "babylon"], window, { + perPackage: true, + summaries: loadSummaries(client, window), + }); + // Two fixtures, each with its count: four files a run, so the newest 100 runs. + expect(data.trendCost).toMatchObject({ perRun: 4, dates: TREND_BUDGET / 4 }); + const newest = new Set(window.slice(-TREND_BUDGET / 4)); + const trendFiles = fetched.filter( + (p) => !p.startsWith("summary/") && !p.startsWith("2026-10-13/"), + ); + expect(trendFiles.length).toBeLessThanOrEqual(TREND_BUDGET); + expect(trendFiles.every((p) => newest.has(p.split("/")[0] ?? ""))).toBe(true); + }); + } +}); + +describe("the trend's run detail by column", () => { + const detail = ({ date, fixtures, partial, failed, byFixture, rsd, rsdByFixture }: TrendRun) => ({ + date, + fixtures, + partial, + failed, + byFixture, + rsd, + rsdByFixture, + }); + + it("gives back every run exactly: DNFs, partial means, a tool missing a run", async () => { + const r = await report("range=all"); + const runs = r.trend.points.map(detail); + // The fixture tree has zpm failing babylon (partial, failed) and npm absent on one run. + expect(runs.some((p) => Object.keys(p.failed).length > 0)).toBe(true); + expect(runs.some((p) => Object.keys(p.partial).length > 0)).toBe(true); + expect(runs.some((p) => p.byFixture["npm"] === undefined)).toBe(true); + expect(trendOf(toTrendColumns(runs))).toEqual(runs); + }); + + it("keeps an absent spread absent, a zero a zero, and fixtures in each run's order", () => { + const runs: TrendRun[] = [ + { + date: "2026-01-01", + fixtures: ["vue", "astro"], + partial: { zpm: { mean: 0.5, of: 1 } }, + failed: { zpm: [{ fixture: "vue", exit: 1 }] }, + byFixture: { npm: { vue: 1 / 3, astro: 2 }, zpm: { astro: 0.5 } }, + rsd: { npm: 0, zpm: 0.1 }, + rsdByFixture: { npm: { astro: 0 } }, + }, + { + date: "2026-01-03", + fixtures: ["astro"], + partial: {}, + failed: {}, + byFixture: { bun: { astro: 0.25 } }, + rsd: { bun: 0.02 }, + rsdByFixture: {}, + }, + ]; + const columns = toTrendColumns(runs); + expect(trendOf(columns)).toEqual(runs); + expect(columns.byFixture["npm"]?.["vue"]).toEqual([1 / 3, null]); + expect(trendOf(toTrendColumns([]))).toEqual([]); + }); + + it("is smaller than the runs it encodes", async () => { + const runs = (await report("range=all")).trend.points.map(detail); + expect(JSON.stringify(toTrendColumns(runs)).length).toBeLessThan(JSON.stringify(runs).length); + }); +}); diff --git a/apps/www/test/routes.test.ts b/apps/www/test/routes.test.ts index 055adb06d2..fc69528d6b 100644 --- a/apps/www/test/routes.test.ts +++ b/apps/www/test/routes.test.ts @@ -10,7 +10,7 @@ import { GET as openapi } from "../src/app/openapi.json/route.ts"; import { METHOD, READING, RUNNER_NOTE, TRADEMARK } from "../src/lib/metadata.ts"; import { DATA_URL, fakeFetch } from "./fixture-tree.ts"; -const ORIGIN = "https://benchmarks.vlt.sh"; +const ORIGIN = "https://benchmarks.vlt.io"; const get = (path: string) => new Request(`${ORIGIN}${path}`); const scenario = (section: string, scenario: string) => ({ params: Promise.resolve({ section, scenario }), diff --git a/apps/www/test/scenarios.test.ts b/apps/www/test/scenarios.test.ts index 6aa29c7478..3a09b68b9b 100644 --- a/apps/www/test/scenarios.test.ts +++ b/apps/www/test/scenarios.test.ts @@ -21,7 +21,9 @@ import { } from "../src/lib/measurements.ts"; import { markersForTools, versionMarkers } from "../src/lib/version-markers.ts"; import { + LEGACY_RANGES, loadReportParams, + type RangeValue, reportHref, tableSortOf, trendWindow, @@ -465,27 +467,71 @@ describe("versionMarkers · the empty-version case", () => { describe("the trend window", () => { // Published dates either side of the archive's widest gap. const dates = ["2026-06-15", "2026-06-16", "2026-06-17", "2026-06-29", "2026-06-30"]; - const base = { dates, end: "2026-06-30", range: 30, from: null, to: null }; + const base = { + dates, + end: "2026-06-30", + range: "1m" as RangeValue, + from: null, + to: null, + }; + // Most of 2025, one run every other day, then the five above. + const archive = [ + ...Array.from({ length: 180 }, (_, i) => + new Date(Date.parse("2025-01-01T00:00:00Z") + i * 2 * 86_400_000).toISOString().slice(0, 10), + ), + ...dates, + ]; it("a preset is that many calendar days ending at the run shown", () => { - const w = trendWindow({ ...base, range: 30 }); + const w = trendWindow({ ...base, range: "1m" }); expect(w.custom).toBe(false); + expect(w.range).toBe("1m"); expect(w.dates).toEqual(dates); expect(w.to).toBe("2026-06-30"); - // Fourteen days back from Jun 30 reaches Jun 17 and no further. - expect(trendWindow({ ...base, range: 14 }).dates).toEqual([ - "2026-06-17", - "2026-06-29", - "2026-06-30", + // Seven days back from Jun 30 reaches Jun 24 -- inside the gap -- so only the last two runs. + expect(trendWindow({ ...base, range: "1w" }).dates).toEqual(["2026-06-29", "2026-06-30"]); + }); + + it("today is the run shown alone", () => { + const w = trendWindow({ ...base, range: "today" }); + expect(w).toMatchObject({ dates: ["2026-06-30"], from: "2026-06-30", to: "2026-06-30" }); + expect(trendWindow({ ...base, end: "2026-06-16", range: "today" }).dates).toEqual([ + "2026-06-16", ]); }); + it("1y is 365 rolling days and all is the whole archive up to the run", () => { + const year = trendWindow({ ...base, dates: archive, range: "1y" }).dates; + expect(year[0]).toBe("2025-07-02"); // the first run fewer than 365 days before Jun 30 + expect(year.at(-1)).toBe("2026-06-30"); + expect(trendWindow({ ...base, dates: archive, range: "all" }).dates).toEqual(archive); + expect(trendWindow({ ...base, dates: archive, end: "2026-06-16", range: "all" }).dates).toEqual( + archive.filter((d) => d <= "2026-06-16"), + ); + }); + + it("an old day-count link keeps its old meaning", () => { + // What the 120-day cap allowed, the legacy numbers still give exactly. + for (const n of LEGACY_RANGES) { + const w = trendWindow({ ...base, dates: archive, range: n }); + const oldest = new Date(Date.parse("2026-06-30T00:00:00Z") - (n - 1) * 86_400_000) + .toISOString() + .slice(0, 10); + expect(w.dates).toEqual(archive.filter((d) => d >= oldest)); + expect(w.range).toBe(n); + } + expect(trendWindow({ ...base, range: 30 }).dates).toEqual( + trendWindow({ ...base, range: "1m" }).dates, + ); + }); + it("honours an exact range whose ends are both published dates", () => { const w = trendWindow({ ...base, from: "2026-06-16", to: "2026-06-29" }); expect(w).toEqual({ custom: true, from: "2026-06-16", to: "2026-06-29", + range: "1m", dates: ["2026-06-16", "2026-06-17", "2026-06-29"], }); }); @@ -496,13 +542,20 @@ describe("the trend window", () => { expect(trendWindow({ ...base, from: "2026-06-15", to: "2026-06-25" }).custom).toBe(false); }); - it("falls back to the preset for a reversed, half-given or over-long range", () => { - expect(trendWindow({ ...base, from: "2026-06-30", to: "2026-06-15" }).custom).toBe(false); + it("falls back to the preset for a reversed or half-given range", () => { + const reversed = trendWindow({ ...base, range: "1w", from: "2026-06-30", to: "2026-06-15" }); + expect(reversed.custom).toBe(false); + expect(reversed.range).toBe("1w"); + expect(reversed.dates).toEqual(["2026-06-29", "2026-06-30"]); expect(trendWindow({ ...base, from: "2026-06-15", to: null }).custom).toBe(false); + }); + + it("honours an exact range of any length", () => { + // Past the old 120-day cap: there is no cap now. const long = ["2026-01-01", ...dates]; - expect(trendWindow({ ...base, dates: long, from: "2026-01-01", to: "2026-06-30" }).custom).toBe( - false, - ); + const w = trendWindow({ ...base, dates: long, from: "2026-01-01", to: "2026-06-30" }); + expect(w.custom).toBe(true); + expect(w.dates).toEqual(long); }); it("will not run past the run the report is showing", () => { @@ -523,7 +576,7 @@ describe("query parameters", () => { reportHref("/package-managers/clean", { date: null, fixture: null, - range: 30, + range: "1m", from: null, to: null, hide: [], @@ -534,11 +587,26 @@ describe("query parameters", () => { expect(reportHref("/registries/registry-clean", { fixture: "astro", range: 90 })).toBe( "/registries/registry-clean?fixture=astro&range=90", ); + expect(reportHref("/registries/registry-clean", { range: "1y" })).toBe( + "/registries/registry-clean?range=1y", + ); + }); + + it("reads a preset or a legacy day count as the range, and nothing else", async () => { + const range = async (q: string) => (await loadReportParams(new URLSearchParams(q))).range; + expect(await range("")).toBe("1m"); + expect(await range("range=1y")).toBe("1y"); + expect(await range("range=all")).toBe("all"); + expect(await range("range=today")).toBe("today"); + expect(await range("range=90")).toBe(90); + expect(await range("range=7")).toBe("1m"); + expect(await range("range=1M")).toBe("1m"); + expect(await range("range=30.0")).toBe("1m"); }); it("a malformed value falls back to the default instead of erroring", async () => { const q = await loadReportParams(new URLSearchParams("range=7&scale=sideways&hide=")); - expect(q.range).toBe(30); + expect(q.range).toBe("1m"); expect(q.scale).toBe("linear"); expect(q.hide).toEqual([]); }); diff --git a/apps/www/vitest.config.mts b/apps/www/vitest.config.mts index cd93c17773..700919f03c 100644 --- a/apps/www/vitest.config.mts +++ b/apps/www/vitest.config.mts @@ -16,5 +16,7 @@ export default defineConfig({ oxc: { jsx: { runtime: "automatic" } }, test: { include: ["test/**/*.test.ts"], + // Run from apps/www whatever directory vitest starts in, as Next does (test/app-cwd.ts). + setupFiles: ["./test/app-cwd.ts"], }, }); diff --git a/docs/METHODOLOGY.md b/docs/METHODOLOGY.md index d255624758..b4ad7dc368 100644 --- a/docs/METHODOLOGY.md +++ b/docs/METHODOLOGY.md @@ -119,6 +119,7 @@ nightly run since 2025-08-22. /{date}/versions.json tool versions for that run /latest/ see below — NOT a copy /index.json /manifest.json /manifest/{date}.json discovery layer +/summary/{date}.json one date's results, counts and versions ``` [`packages/types`](../packages/types/src/index.ts) encodes these shapes and is @@ -178,3 +179,21 @@ given row. For a single date, fetch `index.json` plus that date's shard (~12KB), not the whole manifest. + +### `summary/{date}.json` + +One file per date, so a trend over many runs costs one fetch a run, not one +per file. Every value is lifted verbatim from that date's files; nothing is +derived. Written by `apps/data` build-manifest on each deploy, for every date. + +| Key | Holds | +| ---------------- | --------------------------------------------------------------------------------------- | +| `{variation}` | `{fixture}` to that benchmark file's results: `command`, `mean`, `stddev`, `exit_codes` | +| `#packageCounts` | `{variation}` to `{fixture}` to that package-count file, as published | +| `#versions` | that date's `versions.json` | + +Results keep `exit_codes`, so a failed run is still a failed run. The `#` keys +were added in 2026-11 and are each optional: a summary written before then has +none, and a date that published no count files or no `versions.json` has no +such key. Absent means "read the file", never zero. No variation name starts +with `#`.