Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/benchmark.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -333,8 +333,8 @@ jobs:
# apps/data emits index.json, manifest.json and the manifest/ shards
# into suite/results/, the single results tree. It never re-emits the
# published date directories (decision D2) and never writes an
# index.html -- the published index.html is the live site at
# benchmarks.vlt.sh and survives via keep_files.
# index.html -- the published index.html is a live page at
# benchmarks-data.vlt.io (the gh-pages CNAME) and survives via keep_files.
run: |
vlt install
vlt run --workspace=apps/data build
Expand Down
2 changes: 1 addition & 1 deletion apps/data/src/client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ export type FetchLike = (url: string) => Promise<{
}>;

export interface DataClientOptions {
/** Base URL of the served tree, e.g. `https://benchmarks.vlt.sh`. */
/** Base URL of the served tree, e.g. `https://benchmarks-data.vlt.io`. */
baseUrl: string;
/** Injectable for tests; defaults to global fetch. */
fetch?: FetchLike;
Expand Down
11 changes: 10 additions & 1 deletion apps/data/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,4 +28,13 @@ export {
type ParsedDataFile,
} from "./paths.ts";
export { scanRoots } from "./scan.ts";
export { buildSummary, type DateSummary, type SummaryResult } from "./summary.ts";
export {
buildSummary,
SUMMARY_PACKAGE_COUNTS,
SUMMARY_VERSIONS,
summaryPackageCounts,
summaryVersions,
type DateSummary,
type SummaryCounts,
type SummaryResult,
} from "./summary.ts";
6 changes: 3 additions & 3 deletions apps/data/src/serve.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@
* Minimal static server for the assembled tree.
*
* Deliberately dependency-free (`node:http` only) and deliberately does not
* serve a directory index. The published root's `index.html` IS the live site
* at benchmarks.vlt.sh, and this app must never be the thing that puts an
* index page there.
* serve a directory index. The published root's `index.html` is a live page
* at benchmarks-data.vlt.io (the gh-pages CNAME), and this app must never be
* the thing that puts an index page there.
Comment thread
ejkorol marked this conversation as resolved.
Resolved
*/
import { createServer, type Server } from "node:http";
import { createReadStream, statSync } from "node:fs";
Expand Down
77 changes: 68 additions & 9 deletions apps/data/src/summary.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,16 @@
* Not `chart-data.json`. That file gives a failed run a time, and nothing in
* it says the run failed. Every result here keeps its `exit_codes`, so a
* consumer applies the same rule to it as to the file it came from.
*
* The date's package counts and `versions.json` ride along,
* verbatim, under reserved `#` keys beside the variations -- so a summary
* written before they existed is still a valid one, and a reader that only
* looks up variations never sees them. Each is optional: absent means "not in
* this summary", and the reader falls back to the file; it never means zero.
*/
import type { BenchmarkResult } from "@benchmarks/types";
import type { BenchmarkResult, PackageCount, Versions } from "@benchmarks/types";
import type { DirectoryListing } from "./manifest.ts";
import { parseDataFile } from "./paths.ts";
import { parseDataFile, VERSIONS_FILE } from "./paths.ts";

/** What a trend needs of a result, exactly as the benchmark file has it. */
export type SummaryResult = Pick<BenchmarkResult, "command" | "mean" | "stddev" | "exit_codes">;
Expand All @@ -22,26 +28,76 @@ export type DateSummary = {
[variation: string]: { [fixture: string]: SummaryResult[] | undefined } | undefined;
};

/** Variation, then fixture, to that count file as published. */
export type SummaryCounts<T> = {
[variation: string]: { [fixture: string]: T | undefined } | undefined;
};

export const SUMMARY_PACKAGE_COUNTS = "#packageCounts";
export const SUMMARY_VERSIONS = "#versions";

/* The reserved keys, read without letting them widen every variation's type. */
const extra = (summary: DateSummary, key: string): unknown =>
(summary as { readonly [key: string]: unknown })[key];

/** This summary's package counts, or `undefined` when it does not carry them. */
export const summaryPackageCounts = (
summary: DateSummary,
): SummaryCounts<PackageCount> | undefined =>
extra(summary, SUMMARY_PACKAGE_COUNTS) as SummaryCounts<PackageCount> | undefined;

/** The date's `versions.json`, or `undefined` when the summary does not carry it. */
export const summaryVersions = (summary: DateSummary): Versions | undefined =>
extra(summary, SUMMARY_VERSIONS) as Versions | undefined;

const isObject = (value: unknown): value is object =>
typeof value === "object" && value !== null && !Array.isArray(value);

const parse = (text: string): unknown => {
try {
return JSON.parse(text);
} catch {
return undefined;
}
};

/**
* Summarise one date directory. `read` returns a file's text; a file that
* does not parse as a benchmark file is reported in `skipped`, not guessed at.
* does not parse as what its name says is reported in `skipped`, not guessed at.
*/
export const buildSummary = (
listing: DirectoryListing,
read: (file: string) => string,
): { summary: DateSummary; skipped: string[] } => {
// A variation is whatever the file name says, `__proto__` included.
const summary: DateSummary = Object.create(null);
const packageCounts: SummaryCounts<PackageCount> = Object.create(null);
const skipped: string[] = [];
for (const file of listing.files) {
if (file === VERSIONS_FILE) {
const versions = parse(read(file));
if (isObject(versions)) {
(summary as { [key: string]: unknown })[SUMMARY_VERSIONS] = versions;
} else skipped.push(file);
continue;
}
const parsed = parseDataFile(file);
if (parsed?.kind !== "benchmark") continue;
let results: unknown;
try {
results = (JSON.parse(read(file)) as { results?: unknown }).results;
} catch {
results = undefined;
if (parsed === undefined) continue;
// A variation named like a reserved key would be read as one.
if (parsed.variation.startsWith("#")) {
skipped.push(file);
continue;
}
// Process counts stay out until something reads them from here.
if (parsed.kind === "process-count") continue;
const content = parse(read(file));
if (parsed.kind === "package-count") {
if (isObject(content)) {
(packageCounts[parsed.variation] ??= {})[parsed.fixture] = content as PackageCount;
} else skipped.push(file);
continue;
}
const results = isObject(content) ? (content as { results?: unknown }).results : undefined;
if (!Array.isArray(results) || !results.every((r) => typeof r === "object" && r !== null)) {
skipped.push(file);
continue;
Expand All @@ -50,5 +106,8 @@ export const buildSummary = (
({ command, mean, stddev, exit_codes }) => ({ command, mean, stddev, exit_codes }),
);
}
if (Object.keys(packageCounts).length > 0) {
(summary as { [key: string]: unknown })[SUMMARY_PACKAGE_COUNTS] = packageCounts;
}
return { summary, skipped };
};
2 changes: 1 addition & 1 deletion apps/data/test/served-tree.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ describe.skipIf(skip)("served tree, built from real gh-pages data", () => {
mkdirSync(join(root, "results-astro-clean"), { recursive: true });
writeFileSync(join(root, "results-astro-clean", "benchmarks.json"), "{}\n");
writeFileSync(join(root, "index.html"), "<!doctype html><title>live site</title>\n");
writeFileSync(join(root, "CNAME"), "benchmarks.vlt.sh\n");
writeFileSync(join(root, "CNAME"), "benchmarks-data.vlt.io\n");

// Run the REAL CLI rather than reimplementing what it does. An earlier
// version of this fixture built the index and shards inline, which meant a
Expand Down
63 changes: 62 additions & 1 deletion apps/data/test/summary.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { describe, expect, it } from "vitest";
import { buildSummary } from "../src/summary.ts";
import { buildSummary, summaryPackageCounts, summaryVersions } from "../src/summary.ts";

describe("buildSummary", () => {
it("lifts each benchmark file's results verbatim, exit codes and all", () => {
Expand Down Expand Up @@ -30,8 +30,69 @@ describe("buildSummary", () => {
},
"registry-clean": { next: [] },
["__proto__"]: { next: [] },
"#packageCounts": { clean: { next: {} } },
});
expect(Object.prototype).not.toHaveProperty("next");
expect(skipped).toEqual(["astro-clean.json", "vue-clean.json"]);
});

it("carries the date's counts and versions verbatim, beside the variations", () => {
const counts = { npm: { count: 110, minCount: 108, maxCount: 112 }, bun: { count: 100 } };
const processes = { npm: { count: 4 } };
const versions = { npm: "11.0.0", vlt: "1.1.0" };
const files: { [name: string]: string } = {
"next-clean.json": JSON.stringify({
results: [{ command: "npm", mean: 2, stddev: 0, exit_codes: [0] }],
}),
"next-clean-package-count.json": JSON.stringify(counts),
"next-clean-process-count.json": JSON.stringify(processes),
"next-registry-clean-package-count.json": JSON.stringify({ npm: { count: 90 } }),
"astro-clean-package-count.json": "not json",
"versions.json": JSON.stringify(versions),
};
const { summary, skipped } = buildSummary(
{ name: "2026-10-19", files: Object.keys(files) },
(file) => files[file] ?? "",
);
expect(summaryPackageCounts(summary)).toEqual({
clean: { next: counts },
"registry-clean": { next: { npm: { count: 90 } } },
});
// Nothing reads process counts from a summary, so it does not carry them.
expect(
Object.keys(summary)
.filter((k) => k.startsWith("#"))
.sort(),
).toEqual(["#packageCounts", "#versions"]);
expect(summaryVersions(summary)).toEqual(versions);
// A reader that looks up variations sees exactly what it did before.
expect(summary["clean"]).toEqual({
next: [{ command: "npm", mean: 2, stddev: 0, exit_codes: [0] }],
});
expect(skipped).toEqual(["astro-clean-package-count.json"]);
});

it("leaves out what the date did not publish, rather than writing it empty", () => {
const { summary } = buildSummary({ name: "2026-10-19", files: ["next-run.json"] }, () =>
JSON.stringify({ results: [] }),
);
expect(summary).toEqual({ run: { next: [] } });
expect(summaryPackageCounts(summary)).toBeUndefined();
expect(summaryVersions(summary)).toBeUndefined();
});

it("an old summary, written before counts and versions, reads as carrying none", () => {
const old = JSON.parse(JSON.stringify({ clean: { next: [] } }));
expect(summaryPackageCounts(old)).toBeUndefined();
expect(summaryVersions(old)).toBeUndefined();
});

it("will not let a variation pose as a reserved key", () => {
const { summary, skipped } = buildSummary(
{ name: "2026-10-19", files: ["next-#versions.json"] },
() => JSON.stringify({ results: [] }),
);
expect(summaryVersions(summary)).toBeUndefined();
expect(skipped).toEqual(["next-#versions.json"]);
});
});
5 changes: 3 additions & 2 deletions apps/www/.env.example
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,9 @@
# Local development, after `vlt run --workspace=apps/data build`:
# vlt run --workspace=apps/data start # serves results/ on :8811
#
# Production is the gh-pages origin, which serves the same tree:
# NEXT_PUBLIC_DATA_URL=https://benchmarks.vlt.sh
# Production is the gh-pages origin (its CNAME), which serves the same tree.
# Not benchmarks.vlt.sh: that is the site, and it answers HTML for any path.
# NEXT_PUBLIC_DATA_URL=https://benchmarks-data.vlt.io
NEXT_PUBLIC_DATA_URL=http://localhost:8811

# Optional server-side runtime override, checked first when set.
Expand Down
5 changes: 3 additions & 2 deletions apps/www/.env.local
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,9 @@
# Local development, after `vlt run --workspace=apps/data build`:
# vlt run --workspace=apps/data start # serves results/ on :8811
#
# Production is the gh-pages origin, which serves the same tree:
# NEXT_PUBLIC_DATA_URL=https://benchmarks.vlt.sh
# Production is the gh-pages origin (its CNAME), which serves the same tree.
# Not benchmarks.vlt.sh: that is the site, and it answers HTML for any path.
# NEXT_PUBLIC_DATA_URL=https://benchmarks-data.vlt.io
NEXT_PUBLIC_DATA_URL=http://localhost:8811

# Optional server-side runtime override, checked first when set.
Expand Down
26 changes: 15 additions & 11 deletions apps/www/src/app/[section]/[[...params]]/page.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,8 @@ import { formatDuration, longDate } from "@/lib/findings";
import { fixtureBars } from "@/lib/figures";
import { pageMetadata } from "@/lib/metadata";
import { ogFigure } from "@/lib/og-figure";
import { loadReportParams, reportHref, reportParams } from "@/lib/params";
import { toTrendColumns } from "@/lib/chart-model";
import { loadReportParams, rangeLabel, reportHref, reportParams } from "@/lib/params";
import { loadCatalog } from "@/lib/catalog";
import { dataClient } from "@/lib/data";
import { loadReport } from "@/lib/report";
Expand Down Expand Up @@ -196,14 +197,17 @@ const Report = async ({

const timeBars = report.timeBars;
const perPackageBars = report.perPackageBars;
/* What the chart needs of each run besides its plotted means. */
const trendDetail = trend.points.map(
({
byTool: _byTool,
perPackage: _perPackage,
perPackageByFixture: _perPackageByFixture,
...rest
}) => rest,
/* What the chart needs of each run besides its plotted means, by column: one
object, which both trend islands get and the page carries once. */
const trendDetail = toTrendColumns(
trend.points.map(
({
byTool: _byTool,
perPackage: _perPackage,
perPackageByFixture: _perPackageByFixture,
...rest
}) => rest,
),
);
const fixtureCount = report.fixtures.length;
const across =
Expand Down Expand Up @@ -322,7 +326,7 @@ const Report = async ({
<TimeSeriesChart
id="time-trend"
title="Performance over time"
rows={trend.rows}
series={trend.series}
perPackage={trend.perPackage}
focus={trend.focus}
/* Tools that completed nothing still ran: the chart marks their DNFs. */
Expand Down Expand Up @@ -565,7 +569,7 @@ const Report = async ({
: [
{
label: "window",
value: `${query.range} days`,
value: rangeLabel(query.range),
clear: ["range", "from", "to"] as const,
},
]),
Expand Down
42 changes: 42 additions & 0 deletions apps/www/src/app/globals.css
Original file line number Diff line number Diff line change
Expand Up @@ -525,3 +525,45 @@
clip-path: inset(50%);
white-space: nowrap;
}

/*
* The trend's marks (components/charts-echarts.tsx): did not finish, a version
* change and an unusual run. Lucide's circle-x, tag and triangle-alert at its
* stroke 2.5, painted in `currentColor` through a mask -- one rule each here
* rather than an inline <svg> per mark, of which All time draws over a
* thousand.
*/
.mark-dnf,
.mark-version,
.mark-unusual {
display: inline-block;
flex: none;
width: 0.625rem;
height: 0.625rem;
background-color: currentColor;
mask: var(--mark) center / contain no-repeat;
}

.mark-dnf {
--mark: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 24 24' fill='none' stroke='black' stroke-width='2.5' stroke-linecap='round' stroke-linejoin='round'%3E%3Ccircle cx='12' cy='12' r='10'/%3E%3Cpath d='m15 9-6 6'/%3E%3Cpath d='m9 9 6 6'/%3E%3C/svg%3E");
}

.mark-version {
--mark: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 24 24' fill='none' stroke='black' stroke-width='2.5' stroke-linecap='round' stroke-linejoin='round'%3E%3Cpath d='M12.586 2.586A2 2 0 0 0 11.172 2H4a2 2 0 0 0-2 2v7.172a2 2 0 0 0 .586 1.414l8.704 8.704a2.426 2.426 0 0 0 3.42 0l6.58-6.58a2.426 2.426 0 0 0 0-3.42z'/%3E%3Ccircle cx='7.5' cy='7.5' r='.5' fill='black'/%3E%3C/svg%3E");
}

.mark-unusual {
--mark: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 24 24' fill='none' stroke='black' stroke-width='2.5' stroke-linecap='round' stroke-linejoin='round'%3E%3Cpath d='m21.73 18-8-14a2 2 0 0 0-3.48 0l-8 14A2 2 0 0 0 4 21h16a2 2 0 0 0 1.73-3'/%3E%3Cpath d='M12 9v4'/%3E%3Cpath d='M12 17h.01'/%3E%3C/svg%3E");
}

/* Forced colours replace the background a mask paints with the page's own
colour, which hid the marks. There they take the system text colour, as the
inline SVG strokes they replaced did. */
@media (forced-colors: active) {
.mark-dnf,
.mark-version,
.mark-unusual {
forced-color-adjust: none;
background-color: CanvasText;
}
}
2 changes: 1 addition & 1 deletion apps/www/src/app/llms.txt/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ Every page is available as Markdown: append \`.md\` to its URL, or request the p
## Reading the results

${READING.map((line) => `- ${line}`).join("\n")}
- Page parameters: \`date\`, \`fixture\`, \`range\` (30, 60, 90 or 120 days of trend), \`from\` and \`to\` (an exact trend window), \`hide\` (comma-separated tools), \`metric\` (total, change, baseline or per-package), \`exclude\` (\`YYYY-MM-DD\` or \`tool@YYYY-MM-DD\`, comma-separated).
- Page parameters: \`date\`, \`fixture\`, \`range\` (today, 1w, 1m, 1y or all: rolling days of trend ending at the run; default 1m), \`from\` and \`to\` (an exact trend window), \`hide\` (comma-separated tools), \`metric\` (total, change, baseline or per-package), \`exclude\` (\`YYYY-MM-DD\` or \`tool@YYYY-MM-DD\`, comma-separated).

## API and MCP

Expand Down
Loading