diff --git a/handleRequest.test.ts b/handleRequest.test.ts index 14ef40f..cb7c44c 100644 --- a/handleRequest.test.ts +++ b/handleRequest.test.ts @@ -1,7 +1,9 @@ import { expect, it } from "vitest"; import { createRequestHandler, resolvePluginOrSchemaUrl } from "./handleRequest.js"; -it("should get info.json", { timeout: 10_000 }, async () => { +// builds the info file for real, which makes a serialized github request per +// plugin, so this grows by roughly a request every time a plugin is added +it("should get info.json", { timeout: 60_000 }, async () => { const { handleRequest } = createRequestHandler(); const response = await handleRequest( new Request("https://plugins.dprint.dev/info.json"), diff --git a/homeView.test.ts b/homeView.test.ts new file mode 100644 index 0000000..76bbfd7 --- /dev/null +++ b/homeView.test.ts @@ -0,0 +1,92 @@ +import { expect, it } from "vitest"; +import { renderHomeHtml } from "./homeView.js"; +import type { PluginData } from "./readInfoFile.js"; + +function createPlugin(data: Partial): PluginData { + return { + name: "dprint-plugin-test", + url: "https://plugins.dprint.dev/test-1.0.0.wasm", + version: "1.0.0", + downloadCount: { currentVersion: 1, allVersions: 2 }, + ...data, + }; +} + +// what the url column shows and the copy button copies. asserted on the +// rendered markup rather than the helper so that the two staying in sync is +// part of what's covered. +function renderReferences(plugins: PluginData[]) { + const html = renderHomeHtml({ latest: plugins }); + return { + shown: [...html.matchAll(/([^<]*)<\/code>/g)].map((m) => m[1]), + copied: [...html.matchAll(/data-url="([^"]*)"/g)].map((m) => m[1]), + html, + }; +} + +it("should show the npm specifier as the latest reference", () => { + const { shown, copied } = renderReferences([ + // wasm plugin on npm — the default path is left off + createPlugin({ + url: "https://plugins.dprint.dev/json-1.0.0.wasm", + npm: { name: "@dprint/json", version: "1.2.3" }, + }), + // process plugin on npm — its manifest has to be named + createPlugin({ + url: "https://plugins.dprint.dev/exec-1.0.0.json", + npm: { name: "@dprint/exec", version: "0.7.3" }, + }), + // a package that doesn't ship the plugin at its root names the path + createPlugin({ + url: "https://plugins.dprint.dev/multi-1.0.0.wasm", + npm: { name: "@dprint/multi", version: "2.0.0", path: "json/plugin.wasm" }, + }), + // an uppercase extension is still a wasm plugin + createPlugin({ + url: "https://plugins.dprint.dev/shouty-1.0.0.WASM", + npm: { name: "@dprint/shouty", version: "3.0.0" }, + }), + ]); + + const expected = [ + "npm:@dprint/json@1.2.3", + "npm:@dprint/exec@0.7.3/plugin.json", + "npm:@dprint/multi@2.0.0/json/plugin.wasm", + "npm:@dprint/shouty@3.0.0", + ]; + // the trailing entries are the "helpful commands" section + expect(shown.slice(0, expected.length)).toEqual(expected); + expect(copied).toEqual(expected); +}); + +it("should fall back to the url when there's no npm version", () => { + const { shown, copied, html } = renderReferences([ + // on npm, but the registry lookup failed, so there's no version to name + createPlugin({ + url: "https://plugins.dprint.dev/no-version-1.0.0.wasm", + npm: { name: "@dprint/no-version" }, + }), + // not published to npm at all + createPlugin({ url: "https://plugins.dprint.dev/plain-1.0.0.wasm" }), + ]); + + const expected = [ + "https://plugins.dprint.dev/no-version-1.0.0.wasm", + "https://plugins.dprint.dev/plain-1.0.0.wasm", + ]; + expect(shown.slice(0, expected.length)).toEqual(expected); + expect(copied).toEqual(expected); + expect(html).not.toContain("npm:"); +}); + +it("should keep the url searchable when a specifier replaced it", () => { + const { html } = renderReferences([ + createPlugin({ + url: "https://plugins.dprint.dev/json-1.0.0.wasm", + npm: { name: "@dprint/json", version: "1.2.3" }, + }), + ]); + const search = /data-search="([^"]*)"/.exec(html)?.[1]; + expect(search).toContain("https://plugins.dprint.dev/json-1.0.0.wasm"); + expect(search).toContain("npm:@dprint/json@1.2.3"); +}); diff --git a/homeView.tsx b/homeView.tsx index 2ed7ba7..f746709 100644 --- a/homeView.tsx +++ b/homeView.tsx @@ -95,13 +95,15 @@ function renderPage(pluginsData: PluginsData) { ); } -// builds the lowercased string the search filters against: name, url, version, -// description, config key, and every file extension / file name / exec command -// the plugin handles. +// builds the lowercased string the search filters against: name, url, npm +// specifier, version, description, config key, and every file extension / file +// name / exec command the plugin handles. function pluginSearchText(plugin: PluginData) { const parts: (string | undefined)[] = [ plugin.name, + // the url stays searchable even when the npm specifier is what's shown plugin.url, + latestReference(plugin), plugin.version, plugin.description, plugin.configKey, @@ -169,14 +171,19 @@ function renderPlugin(plugin: PluginData) {
- {plugin.url} + {latestReference(plugin)}
Downloads (30d) {plugin.downloadCount.allVersions?.toLocaleString("en-US")}
-
@@ -184,6 +191,22 @@ function renderPlugin(plugin: PluginData) { ); } +// what to put in a config file's `plugins` array: an npm specifier for a plugin +// published to npm, otherwise its plugins.dprint.dev url. both are kept in the +// data so this can become a user toggle later. +function latestReference(plugin: PluginData) { + const npm = plugin.npm; + if (npm?.version == null) { + return plugin.url; + } + // mirrors how the cli writes an npm plugin into a config file: the path + // within the package defaults by plugin kind, and is only spelled out when + // it isn't the wasm default + const isWasm = plugin.url.toLowerCase().endsWith(".wasm"); + const path = npm.path ?? (isWasm ? "plugin.wasm" : "plugin.json"); + return `npm:${npm.name}@${npm.version}${path === "plugin.wasm" ? "" : `/${path}`}`; +} + // the repo and docs links shown beneath a plugin's name. both are optional: the // repo url is derived during the build and docs only exist for plugins with a // dprint.dev page. diff --git a/plugins.ts b/plugins.ts index 0027e33..ff29f1e 100644 --- a/plugins.ts +++ b/plugins.ts @@ -47,6 +47,8 @@ const KNOWN_DPRINT_PLUGIN_REPOS = new Set([ "jakebailey/dprint-plugin-gofumpt", "malobre/dprint-plugin-vue", "drluckyspin/dprint-plugin-swift", + "apcamargo/dprint-plugin-typstyle", + "jolars/dprint-plugin-panache", ]); // repos where the short name IS the repo name (no dprint-plugin- prefix) @@ -56,11 +58,19 @@ const KNOWN_NON_PREFIXED_REPOS = new Set([ "g-plane/pretty_yaml", "g-plane/pretty_graphql", "lucacasonato/mf2-tools", + "bartlomieju/lax-css", + "bartlomieju/lax-markup", + "bartlomieju/lax-sql", + "sargunv/dprint-clang-format", + "sargunv/dprint-cmakefmt", ]); /** The npm package a plugin is distributed as. */ export interface PluginNpmInfo { name: string; + // where the plugin sits within the package, for one that doesn't ship it at + // the root. defaults to plugin.wasm / plugin.json by plugin kind. + path?: string; } // the npm packages declared in info.json, keyed by `username/repo`. both the diff --git a/readInfoFile.test.ts b/readInfoFile.test.ts new file mode 100644 index 0000000..20abd52 --- /dev/null +++ b/readInfoFile.test.ts @@ -0,0 +1,63 @@ +import { expect, it } from "vitest"; +import { type PluginReleaseInfo, type ResolvedSources, toPluginData } from "./readInfoFile.js"; + +const info: PluginReleaseInfo = { + version: "1.0.0", + url: "https://plugins.dprint.dev/test-1.0.0.wasm", + repoUrl: "https://github.com/dprint/dprint-plugin-test", + downloadKey: "dprint/dprint-plugin-test", + tag: "1.0.0", +}; + +function createSources(data: Partial = {}): ResolvedSources { + return { + downloadCounts: new Map([[info.downloadKey, { allVersions: 100, byTag: new Map([["1.0.0", 40]]) }]]), + npmDownloadCounts: new Map([["@dprint/test", 900]]), + npmVersions: new Map([["@dprint/test", "1.2.3"]]), + ...data, + }; +} + +it("should add npm downloads to the total", () => { + const result = toPluginData({ name: "t", npm: { name: "@dprint/test" } }, info, createSources()); + expect(result.downloadCount.allVersions).toEqual(1000); + // the current version stays registry only — npm has no per version breakdown + expect(result.downloadCount.currentVersion).toEqual(40); +}); + +it("should leave a plugin that isn't on npm counting only the registry", () => { + const result = toPluginData({ name: "t" }, info, createSources()); + expect(result.downloadCount.allVersions).toEqual(100); + expect(result.npm).toEqual(undefined); +}); + +it("should keep the npm properties info.json declared", () => { + const result = toPluginData( + { name: "t", npm: { name: "@dprint/test", path: "test/plugin.wasm" } }, + info, + createSources(), + ); + expect(result.npm).toEqual({ name: "@dprint/test", version: "1.2.3", path: "test/plugin.wasm" }); +}); + +it("should keep the package when its version couldn't be resolved", () => { + const result = toPluginData( + { name: "t", npm: { name: "@dprint/test" } }, + info, + createSources({ npmVersions: new Map() }), + ); + // the cli reads the name to know the plugin is on npm, so it has to survive + expect(result.npm).toEqual({ name: "@dprint/test" }); + // and an absent version is omitted rather than serialized as null + expect(JSON.stringify(result.npm)).toEqual(`{"name":"@dprint/test"}`); +}); + +it("should tolerate every lookup coming back empty", () => { + const result = toPluginData({ name: "t", npm: { name: "@dprint/test" } }, info, { + downloadCounts: new Map(), + npmDownloadCounts: new Map(), + npmVersions: new Map(), + }); + expect(result.downloadCount).toEqual({ currentVersion: 0, allVersions: 0 }); + expect(result.url).toEqual(info.url); +}); diff --git a/readInfoFile.ts b/readInfoFile.ts index 323974f..3960511 100644 --- a/readInfoFile.ts +++ b/readInfoFile.ts @@ -1,7 +1,8 @@ import { env } from "cloudflare:workers"; import infoJson from "./info.json" with { type: "json" }; -import { getLatestInfo } from "./plugins.js"; +import { getLatestInfo, type PluginNpmInfo } from "./plugins.js"; import { getDownloadCounts, type PluginDownloadCounts } from "./utils/analytics.js"; +import { getNpmDownloadCounts, getNpmLatestVersions } from "./utils/npm.js"; // only typing what's used on the server export interface PluginsData { @@ -13,9 +14,16 @@ export interface PluginData { url: string; version: string; downloadCount: { + // downloads of this plugin's url from the registry currentVersion: number; + // downloads of this plugin from anywhere it's distributed — the registry + // plus, for a plugin published to npm, its package's npm downloads allVersions: number; }; + // the npm package this plugin is published to, when it has one. it's carried + // over from info.json with the version resolved from the registry during the + // build, so the version is absent when that lookup failed. + npm?: PluginNpmInfo & { version?: string }; // links shown on the site: the GitHub repo (derived during the build) and the // optional dprint.dev docs page (carried over from info.json) repoUrl?: string; @@ -143,31 +151,86 @@ async function buildInfoFile(origin: string): Promise> { }; async function getLatest(latest: typeof infoJson.latest) { - const downloadCounts = await getDownloadCounts(); - const results = []; + // the release lookups below run one at a time to stay within GitHub's api + // guidelines and are what this build spends its time on, so these are + // started here and awaited after them rather than before. each falls back + // to an empty result, since the loop throwing abandons them unawaited. + const npmPackageNames = latest.map((plugin) => npmInfo(plugin)?.name).filter((name) => name != null); + const downloadCountsPromise = getDownloadCounts().catch(() => new Map()); + const npmDownloadCountsPromise = getNpmDownloadCounts(npmPackageNames).catch(() => new Map()); + const npmVersionsPromise = getNpmLatestVersions(npmPackageNames).catch(() => new Map()); + + const released = []; for (const plugin of latest) { const [username, pluginName] = plugin.name.split("/"); const info = pluginName ? await getLatestInfo(username, pluginName, origin) : await getLatestInfo("dprint", plugin.name, origin); if (info != null) { - const counts = downloadCounts.get(info.downloadKey); - results.push({ - ...plugin, - version: info.version, - url: info.url, - repoUrl: info.repoUrl, - downloadCount: { - currentVersion: currentVersionDownloads(counts, info.tag), - allVersions: counts?.allVersions ?? 0, - }, - }); + released.push({ plugin, info }); } } - return results; + + const sources: ResolvedSources = { + downloadCounts: await downloadCountsPromise, + npmDownloadCounts: await npmDownloadCountsPromise, + npmVersions: await npmVersionsPromise, + }; + return released.map(({ plugin, info }) => toPluginData(plugin, info, sources)); } } +/** What the build resolved for a plugin's latest release. */ +export interface PluginReleaseInfo { + version: string; + url: string; + repoUrl: string; + downloadKey: string; + tag: string; +} + +/** What the build looked up for the plugins as a whole. */ +export interface ResolvedSources { + downloadCounts: Map; + npmDownloadCounts: Map; + npmVersions: Map; +} + +/** + * Merges an info.json entry with what the build resolved for it. Exported + * because this is what decides the shape of the served info.json. + */ +export function toPluginData( + plugin: { name: string; npm?: PluginNpmInfo }, + info: PluginReleaseInfo, + sources: ResolvedSources, +): PluginData { + const counts = sources.downloadCounts.get(info.downloadKey); + const npm = plugin.npm; + return { + ...plugin, + version: info.version, + url: info.url, + repoUrl: info.repoUrl, + // spreads what info.json declared rather than rebuilding it, so the rest of + // the npm properties the cli reads (ex. `path`) survive. the package stays + // listed even when the version lookup failed — the cli reads the name to + // know the plugin is on npm at all. + npm: npm == null ? undefined : { ...npm, version: sources.npmVersions.get(npm.name) }, + downloadCount: { + currentVersion: currentVersionDownloads(counts, info.tag), + // downloads of the plugin's url from the registry, plus its npm package's + allVersions: (counts?.allVersions ?? 0) + (npm == null ? 0 : sources.npmDownloadCounts.get(npm.name) ?? 0), + }, + }; +} + +// reads the optional `npm` off an info.json entry, whose inferred type is a +// union that only some members declare the property on +function npmInfo(plugin: { npm?: PluginNpmInfo }) { + return plugin.npm; +} + // downloads of the latest release over the last 30 days, counting both the exact // version tag and the "latest" alias (which always resolves to the current release) function currentVersionDownloads(counts: PluginDownloadCounts | undefined, tag: string) { diff --git a/utils/mod.ts b/utils/mod.ts index 95c4b1a..cfff337 100644 --- a/utils/mod.ts +++ b/utils/mod.ts @@ -1,4 +1,5 @@ export * from "./analytics.js"; export * from "./asyncLazy.js"; export * from "./github.js"; +export * from "./npm.js"; export * from "./version.js"; diff --git a/utils/npm.test.ts b/utils/npm.test.ts new file mode 100644 index 0000000..b560e30 --- /dev/null +++ b/utils/npm.test.ts @@ -0,0 +1,33 @@ +import { expect, it } from "vitest"; +import { getNpmDownloadCounts, getNpmLatestVersions } from "./npm.js"; + +const packageNames = [ + "@dprint/typescript", // scoped + "dprint-plugin-malva", // unscoped + "@dprint/this-package-does-not-exist", + "not a package name", +]; + +it("should get download counts", async () => { + const counts = await getNpmDownloadCounts(packageNames); + expect(counts.get("@dprint/typescript")).toBeGreaterThan(0); + expect(counts.get("dprint-plugin-malva")).toBeGreaterThan(0); + // a package npm can't resolve has no count rather than a count of zero + expect(counts.has("@dprint/this-package-does-not-exist")).toEqual(false); + expect(counts.has("not a package name")).toEqual(false); +}); + +it("should not let a package name walk out of the registry url", async () => { + // the name is what gets interpolated into the request url, so a segment that + // is only dots has to be rejected before it becomes a path traversal + const versions = await getNpmLatestVersions(["..", ".", "...", "@scope/..", "../typescript"]); + expect(versions.size).toEqual(0); +}); + +it("should get latest versions", async () => { + const versions = await getNpmLatestVersions(packageNames); + expect(versions.get("@dprint/typescript")).toMatch(/^\d+\.\d+\.\d+/); + expect(versions.get("dprint-plugin-malva")).toMatch(/^\d+\.\d+\.\d+/); + expect(versions.has("@dprint/this-package-does-not-exist")).toEqual(false); + expect(versions.has("not a package name")).toEqual(false); +}); diff --git a/utils/npm.ts b/utils/npm.ts new file mode 100644 index 0000000..f25f4fb --- /dev/null +++ b/utils/npm.ts @@ -0,0 +1,100 @@ +import { fetchWithRetries } from "./fetchWithRetries.js"; +import { LruCacheWithExpiry } from "./LruCache.js"; +import { withTimeout } from "./withTimeout.js"; + +const REGISTRY_URL = "https://registry.npmjs.org"; +// npm reports downloads for a package as a whole rather than per version, so +// these are counts across every version. "last-month" is the trailing 30 days, +// matching the window of the plugin download counts these are added to. +const DOWNLOADS_URL = "https://api.npmjs.org/downloads/point/last-month"; + +const versionsCache = new LruCacheWithExpiry({ + size: 1000, + expiryMs: 5 * 60 * 1_000, // keep for 5 minutes so new releases show up quickly +}); +const downloadsCache = new LruCacheWithExpiry({ + size: 1000, + expiryMs: 60 * 60 * 1_000, // keep for an hour — npm only recomputes these daily +}); + +/** Gets the version each package's `latest` tag points at, keyed by package name. */ +export async function getNpmLatestVersions(packageNames: string[]): Promise> { + return await getForEachPackage(packageNames, (packageName) => + versionsCache.getOrSet(packageName, async () => { + const body = await fetchJson(`${REGISTRY_URL}/${packageName}/latest`) as { version?: unknown }; + if (typeof body.version !== "string") { + throw new Error("The version was not a string."); + } + return body.version; + })); +} + +/** Gets the last 30 days of npm downloads for each package, keyed by package name. */ +export async function getNpmDownloadCounts(packageNames: string[]): Promise> { + return await getForEachPackage(packageNames, (packageName) => + downloadsCache.getOrSet(packageName, async () => { + const body = await fetchJson(`${DOWNLOADS_URL}/${packageName}`) as { downloads?: unknown }; + // npm answers 200 with an error object for a package it has no stats for, + // so anything but a number means there's no count rather than none yet + if (typeof body.downloads !== "number") { + throw new Error("The download count was not a number."); + } + return body.downloads; + })); +} + +// resolves each package on its own so that one failing, or npm being down +// entirely, doesn't take down the info file build these feed into. a package +// that couldn't be resolved is left out of the map rather than given a made up +// value. +async function getForEachPackage(packageNames: string[], getValue: (packageName: string) => Promise) { + const values = await Promise.all(packageNames.map(async (packageName) => { + if (!validatePackageName(packageName)) { + console.error(`Invalid npm package name: ${packageName}`); + return undefined; + } + try { + return await getValue(packageName); + } catch (err) { + console.error(`Failed to get npm data for ${packageName}.`, err); + return undefined; + } + })); + const result = new Map(); + for (const [i, value] of values.entries()) { + if (value != null) { + result.set(packageNames[i]!, value); + } + } + return result; +} + +// bounded so that npm hanging can't stall a request that had to rebuild the +// info file synchronously +const FETCH_TIMEOUT_MS = 10_000; + +async function fetchJson(url: string) { + const response = await withTimeout( + (signal) => + fetchWithRetries(url, { + headers: { "user-agent": "dprint-plugins" }, + signal, + }, /* retries */ 1), + FETCH_TIMEOUT_MS, + ); + if (!response.ok) { + const text = await response.text(); + throw new Error(`Invalid response status: ${response.status}\n\n${text}`); + } + return await response.json(); +} + +function validatePackageName(packageName: string) { + // an unscoped name, or a scope and name separated by the only allowed slash. + // this is what keeps the name from walking out of the registry url, so a + // segment that's only dots is rejected rather than treated as a name. + if (!/^(@[a-z0-9\-\._]+\/)?[a-z0-9\-\._]+$/i.test(packageName)) { + return false; + } + return packageName.split("/").every((segment) => /[^.]/.test(segment)); +}