Show npm specifiers and count npm downloads

handleRequest.test.ts

@@ -1,7 +1,9 @@
 import { expect, it } from "vitest";
 import { createRequestHandler, resolvePluginOrSchemaUrl } from "./handleRequest.js";
 
-it("should get info.json", { timeout: 10_000 }, async () => {
+// builds the info file for real, which makes a serialized github request per
+// plugin, so this grows by roughly a request every time a plugin is added
+it("should get info.json", { timeout: 60_000 }, async () => {
   const { handleRequest } = createRequestHandler();
   const response = await handleRequest(
     new Request("https://plugins.dprint.dev/info.json"),

homeView.test.ts

@@ -0,0 +1,92 @@
+import { expect, it } from "vitest";
+import { renderHomeHtml } from "./homeView.js";
+import type { PluginData } from "./readInfoFile.js";
+
+function createPlugin(data: Partial<PluginData>): PluginData {
+  return {
+    name: "dprint-plugin-test",
+    url: "https://plugins.dprint.dev/test-1.0.0.wasm",
+    version: "1.0.0",
+    downloadCount: { currentVersion: 1, allVersions: 2 },
+    ...data,
+  };
+}
+
+// what the url column shows and the copy button copies. asserted on the
+// rendered markup rather than the helper so that the two staying in sync is
+// part of what's covered.
+function renderReferences(plugins: PluginData[]) {
+  const html = renderHomeHtml({ latest: plugins });
+  return {
+    shown: [...html.matchAll(/<code>([^<]*)<\/code>/g)].map((m) => m[1]),
+    copied: [...html.matchAll(/data-url="([^"]*)"/g)].map((m) => m[1]),
+    html,
+  };
+}
+
+it("should show the npm specifier as the latest reference", () => {
+  const { shown, copied } = renderReferences([
+    // wasm plugin on npm — the default path is left off
+    createPlugin({
+      url: "https://plugins.dprint.dev/json-1.0.0.wasm",
+      npm: { name: "@dprint/json", version: "1.2.3" },
+    }),
+    // process plugin on npm — its manifest has to be named
+    createPlugin({
+      url: "https://plugins.dprint.dev/exec-1.0.0.json",
+      npm: { name: "@dprint/exec", version: "0.7.3" },
+    }),
+    // a package that doesn't ship the plugin at its root names the path
+    createPlugin({
+      url: "https://plugins.dprint.dev/multi-1.0.0.wasm",
+      npm: { name: "@dprint/multi", version: "2.0.0", path: "json/plugin.wasm" },
+    }),
+    // an uppercase extension is still a wasm plugin
+    createPlugin({
+      url: "https://plugins.dprint.dev/shouty-1.0.0.WASM",
+      npm: { name: "@dprint/shouty", version: "3.0.0" },
+    }),
+  ]);
+
+  const expected = [
+    "npm:@dprint/json@1.2.3",
+    "npm:@dprint/exec@0.7.3/plugin.json",
+    "npm:@dprint/multi@2.0.0/json/plugin.wasm",
+    "npm:@dprint/shouty@3.0.0",
+  ];
+  // the trailing entries are the "helpful commands" section
+  expect(shown.slice(0, expected.length)).toEqual(expected);
+  expect(copied).toEqual(expected);
+});
+
+it("should fall back to the url when there's no npm version", () => {
+  const { shown, copied, html } = renderReferences([
+    // on npm, but the registry lookup failed, so there's no version to name
+    createPlugin({
+      url: "https://plugins.dprint.dev/no-version-1.0.0.wasm",
+      npm: { name: "@dprint/no-version" },
+    }),
+    // not published to npm at all
+    createPlugin({ url: "https://plugins.dprint.dev/plain-1.0.0.wasm" }),
+  ]);
+
+  const expected = [
+    "https://plugins.dprint.dev/no-version-1.0.0.wasm",
+    "https://plugins.dprint.dev/plain-1.0.0.wasm",
+  ];
+  expect(shown.slice(0, expected.length)).toEqual(expected);
+  expect(copied).toEqual(expected);
+  expect(html).not.toContain("npm:");
+});
+
+it("should keep the url searchable when a specifier replaced it", () => {
+  const { html } = renderReferences([
+    createPlugin({
+      url: "https://plugins.dprint.dev/json-1.0.0.wasm",
+      npm: { name: "@dprint/json", version: "1.2.3" },
+    }),
+  ]);
+  const search = /data-search="([^"]*)"/.exec(html)?.[1];
+  expect(search).toContain("https://plugins.dprint.dev/json-1.0.0.wasm");
+  expect(search).toContain("npm:@dprint/json@1.2.3");
+});

homeView.tsx

@@ -95,13 +95,15 @@ function renderPage(pluginsData: PluginsData) {
   );
 }
 
-// builds the lowercased string the search filters against: name, url, version,
-// description, config key, and every file extension / file name / exec command
-// the plugin handles.
+// builds the lowercased string the search filters against: name, url, npm
+// specifier, version, description, config key, and every file extension / file
+// name / exec command the plugin handles.
 function pluginSearchText(plugin: PluginData) {
   const parts: (string | undefined)[] = [
     plugin.name,
+    // the url stays searchable even when the npm specifier is what's shown
     plugin.url,
+    latestReference(plugin),
     plugin.version,
     plugin.description,
     plugin.configKey,
@@ -169,21 +171,42 @@ function renderPlugin(plugin: PluginData) {
         </div>
       </div>
       <div class="col-url" role="cell">
-        <code>{plugin.url}</code>
+        <code>{latestReference(plugin)}</code>
       </div>
       <div class="col-downloads num-col" role="cell">
         <span class="dl-label">Downloads (30d) </span>
         {plugin.downloadCount.allVersions?.toLocaleString("en-US")}
       </div>
       <div class="col-action" role="cell">
-        <button type="button" class="copy-btn copy-button" title="Copy URL to clipboard" data-url={plugin.url}>
+        <button
+          type="button"
+          class="copy-btn copy-button"
+          title="Copy to clipboard"
+          data-url={latestReference(plugin)}
+        >
           copy
         </button>
       </div>
     </div>
   );
 }
 
+// what to put in a config file's `plugins` array: an npm specifier for a plugin
+// published to npm, otherwise its plugins.dprint.dev url. both are kept in the
+// data so this can become a user toggle later.
+function latestReference(plugin: PluginData) {
+  const npm = plugin.npm;
+  if (npm?.version == null) {
+    return plugin.url;
+  }
+  // mirrors how the cli writes an npm plugin into a config file: the path
+  // within the package defaults by plugin kind, and is only spelled out when
+  // it isn't the wasm default
+  const isWasm = plugin.url.toLowerCase().endsWith(".wasm");
+  const path = npm.path ?? (isWasm ? "plugin.wasm" : "plugin.json");
+  return `npm:${npm.name}@${npm.version}${path === "plugin.wasm" ? "" : `/${path}`}`;
+}
+
 // the repo and docs links shown beneath a plugin's name. both are optional: the
 // repo url is derived during the build and docs only exist for plugins with a
 // dprint.dev page.

plugins.ts

@@ -47,6 +47,8 @@ const KNOWN_DPRINT_PLUGIN_REPOS = new Set([
   "jakebailey/dprint-plugin-gofumpt",
   "malobre/dprint-plugin-vue",
   "drluckyspin/dprint-plugin-swift",
+  "apcamargo/dprint-plugin-typstyle",
+  "jolars/dprint-plugin-panache",
 ]);
 
 // repos where the short name IS the repo name (no dprint-plugin- prefix)
@@ -56,11 +58,19 @@ const KNOWN_NON_PREFIXED_REPOS = new Set([
   "g-plane/pretty_yaml",
   "g-plane/pretty_graphql",
   "lucacasonato/mf2-tools",
+  "bartlomieju/lax-css",
+  "bartlomieju/lax-markup",
+  "bartlomieju/lax-sql",
+  "sargunv/dprint-clang-format",
+  "sargunv/dprint-cmakefmt",
 ]);
 
 /** The npm package a plugin is distributed as. */
 export interface PluginNpmInfo {
   name: string;
+  // where the plugin sits within the package, for one that doesn't ship it at
+  // the root. defaults to plugin.wasm / plugin.json by plugin kind.
+  path?: string;
 }
 
 // the npm packages declared in info.json, keyed by `username/repo`. both the

readInfoFile.test.ts

@@ -0,0 +1,63 @@
+import { expect, it } from "vitest";
+import { type PluginReleaseInfo, type ResolvedSources, toPluginData } from "./readInfoFile.js";
+
+const info: PluginReleaseInfo = {
+  version: "1.0.0",
+  url: "https://plugins.dprint.dev/test-1.0.0.wasm",
+  repoUrl: "https://github.com/dprint/dprint-plugin-test",
+  downloadKey: "dprint/dprint-plugin-test",
+  tag: "1.0.0",
+};
+
+function createSources(data: Partial<ResolvedSources> = {}): ResolvedSources {
+  return {
+    downloadCounts: new Map([[info.downloadKey, { allVersions: 100, byTag: new Map([["1.0.0", 40]]) }]]),
+    npmDownloadCounts: new Map([["@dprint/test", 900]]),
+    npmVersions: new Map([["@dprint/test", "1.2.3"]]),
+    ...data,
+  };
+}
+
+it("should add npm downloads to the total", () => {
+  const result = toPluginData({ name: "t", npm: { name: "@dprint/test" } }, info, createSources());
+  expect(result.downloadCount.allVersions).toEqual(1000);
+  // the current version stays registry only — npm has no per version breakdown
+  expect(result.downloadCount.currentVersion).toEqual(40);
+});
+
+it("should leave a plugin that isn't on npm counting only the registry", () => {
+  const result = toPluginData({ name: "t" }, info, createSources());
+  expect(result.downloadCount.allVersions).toEqual(100);
+  expect(result.npm).toEqual(undefined);
+});
+
+it("should keep the npm properties info.json declared", () => {
+  const result = toPluginData(
+    { name: "t", npm: { name: "@dprint/test", path: "test/plugin.wasm" } },
+    info,
+    createSources(),
+  );
+  expect(result.npm).toEqual({ name: "@dprint/test", version: "1.2.3", path: "test/plugin.wasm" });
+});
+
+it("should keep the package when its version couldn't be resolved", () => {
+  const result = toPluginData(
+    { name: "t", npm: { name: "@dprint/test" } },
+    info,
+    createSources({ npmVersions: new Map() }),
+  );
+  // the cli reads the name to know the plugin is on npm, so it has to survive
+  expect(result.npm).toEqual({ name: "@dprint/test" });
+  // and an absent version is omitted rather than serialized as null
+  expect(JSON.stringify(result.npm)).toEqual(`{"name":"@dprint/test"}`);
+});
+
+it("should tolerate every lookup coming back empty", () => {
+  const result = toPluginData({ name: "t", npm: { name: "@dprint/test" } }, info, {
+    downloadCounts: new Map(),
+    npmDownloadCounts: new Map(),
+    npmVersions: new Map(),
+  });
+  expect(result.downloadCount).toEqual({ currentVersion: 0, allVersions: 0 });
+  expect(result.url).toEqual(info.url);
+});

readInfoFile.ts

@@ -1,7 +1,8 @@
 import { env } from "cloudflare:workers";
 import infoJson from "./info.json" with { type: "json" };
-import { getLatestInfo } from "./plugins.js";
+import { getLatestInfo, type PluginNpmInfo } from "./plugins.js";
 import { getDownloadCounts, type PluginDownloadCounts } from "./utils/analytics.js";
+import { getNpmDownloadCounts, getNpmLatestVersions } from "./utils/npm.js";
 
 // only typing what's used on the server
 export interface PluginsData {
@@ -13,9 +14,16 @@ export interface PluginData {
   url: string;
   version: string;
   downloadCount: {
+    // downloads of this plugin's url from the registry
     currentVersion: number;
+    // downloads of this plugin from anywhere it's distributed — the registry
+    // plus, for a plugin published to npm, its package's npm downloads
     allVersions: number;
   };
+  // the npm package this plugin is published to, when it has one. it's carried
+  // over from info.json with the version resolved from the registry during the
+  // build, so the version is absent when that lookup failed.
+  npm?: PluginNpmInfo & { version?: string };
   // links shown on the site: the GitHub repo (derived during the build) and the
   // optional dprint.dev docs page (carried over from info.json)
   repoUrl?: string;
@@ -143,31 +151,86 @@ async function buildInfoFile(origin: string): Promise<Readonly<PluginsData>> {
   };
 
   async function getLatest(latest: typeof infoJson.latest) {
-    const downloadCounts = await getDownloadCounts();
-    const results = [];
+    // the release lookups below run one at a time to stay within GitHub's api
+    // guidelines and are what this build spends its time on, so these are
+    // started here and awaited after them rather than before. each falls back
+    // to an empty result, since the loop throwing abandons them unawaited.
+    const npmPackageNames = latest.map((plugin) => npmInfo(plugin)?.name).filter((name) => name != null);
+    const downloadCountsPromise = getDownloadCounts().catch(() => new Map<string, PluginDownloadCounts>());
+    const npmDownloadCountsPromise = getNpmDownloadCounts(npmPackageNames).catch(() => new Map<string, number>());
+    const npmVersionsPromise = getNpmLatestVersions(npmPackageNames).catch(() => new Map<string, string>());
+
+    const released = [];
     for (const plugin of latest) {
       const [username, pluginName] = plugin.name.split("/");
       const info = pluginName
         ? await getLatestInfo(username, pluginName, origin)
         : await getLatestInfo("dprint", plugin.name, origin);
       if (info != null) {
-        const counts = downloadCounts.get(info.downloadKey);
-        results.push({
-          ...plugin,
-          version: info.version,
-          url: info.url,
-          repoUrl: info.repoUrl,
-          downloadCount: {
-            currentVersion: currentVersionDownloads(counts, info.tag),
-            allVersions: counts?.allVersions ?? 0,
-          },
-        });
+        released.push({ plugin, info });
       }
     }
-    return results;
+
+    const sources: ResolvedSources = {
+      downloadCounts: await downloadCountsPromise,
+      npmDownloadCounts: await npmDownloadCountsPromise,
+      npmVersions: await npmVersionsPromise,
+    };
+    return released.map(({ plugin, info }) => toPluginData(plugin, info, sources));
   }
 }
 
+/** What the build resolved for a plugin's latest release. */
+export interface PluginReleaseInfo {
+  version: string;
+  url: string;
+  repoUrl: string;
+  downloadKey: string;
+  tag: string;
+}
+
+/** What the build looked up for the plugins as a whole. */
+export interface ResolvedSources {
+  downloadCounts: Map<string, PluginDownloadCounts>;
+  npmDownloadCounts: Map<string, number>;
+  npmVersions: Map<string, string>;
+}
+
+/**
+ * Merges an info.json entry with what the build resolved for it. Exported
+ * because this is what decides the shape of the served info.json.
+ */
+export function toPluginData(
+  plugin: { name: string; npm?: PluginNpmInfo },
+  info: PluginReleaseInfo,
+  sources: ResolvedSources,
+): PluginData {
+  const counts = sources.downloadCounts.get(info.downloadKey);
+  const npm = plugin.npm;
+  return {
+    ...plugin,
+    version: info.version,
+    url: info.url,
+    repoUrl: info.repoUrl,
+    // spreads what info.json declared rather than rebuilding it, so the rest of
+    // the npm properties the cli reads (ex. `path`) survive. the package stays
+    // listed even when the version lookup failed — the cli reads the name to
+    // know the plugin is on npm at all.
+    npm: npm == null ? undefined : { ...npm, version: sources.npmVersions.get(npm.name) },
+    downloadCount: {
+      currentVersion: currentVersionDownloads(counts, info.tag),
+      // downloads of the plugin's url from the registry, plus its npm package's
+      allVersions: (counts?.allVersions ?? 0) + (npm == null ? 0 : sources.npmDownloadCounts.get(npm.name) ?? 0),
+    },
+  };
+}
+
+// reads the optional `npm` off an info.json entry, whose inferred type is a
+// union that only some members declare the property on
+function npmInfo(plugin: { npm?: PluginNpmInfo }) {
+  return plugin.npm;
+}
+
 // downloads of the latest release over the last 30 days, counting both the exact
 // version tag and the "latest" alias (which always resolves to the current release)
 function currentVersionDownloads(counts: PluginDownloadCounts | undefined, tag: string) {

utils/mod.ts

@@ -1,4 +1,5 @@
 export * from "./analytics.js";
 export * from "./asyncLazy.js";
 export * from "./github.js";
+export * from "./npm.js";
 export * from "./version.js";