chore: track download size per PR in the benchmark comment

.github/workflows/benchmark.yml

@@ -25,14 +25,16 @@ jobs:
           cache: npm
 
       - name: Install dependencies
-        run: npm ci --no-audit && npm prune --production
+        run: npm ci --no-audit
 
-      - name: Get size
-        run: du -sk node_modules | cut -f1 > .delta.packageSize && echo "kb (Package size)" >> .delta.packageSize
+      # `npm pack` only produces a representative tarball once `dist` exists.
+      - name: Build
+        run: npm run build
 
-      - name: Get dependency count
-        run: npm ls -a -p | wc -l | tr -d ' \n' > .delta.dependencyCount && echo " (Dependency count)" >>
-          .delta.dependencyCount
+      # Packs the CLI and installs it like a user would, then reports download size, installed size
+      # and dependency count for that install. See scripts/measure-size.js.
+      - name: Get size and dependency count
+        run: node scripts/measure-size.js
 
       - name: Get TypeScript conversion progress
         run: grep -r --exclude-dir="node_modules" --include="*.ts" "@ts-expect-error" . | wc -l | xargs >

.gitignore

@@ -36,3 +36,6 @@ tests/unit/utils/tmp
 
 # Used in local dev by tsc: https://www.typescriptlang.org/tsconfig/#tsBuildInfoFile
 *.tsbuildinfo
+
+# Size metrics written by scripts/measure-size.js, uploaded as CI artifacts
+.delta.*

scripts/measure-size.js

@@ -0,0 +1,193 @@
+/*
+ * Measures the size impact of a change and writes the numbers as `.delta.*` files for
+ * `netlify/delta-action` to compare against `main` and post on the PR. See
+ * `.github/workflows/benchmark.yml`.
+ */
+
+import { execFile } from 'node:child_process'
+import { readdir, readFile, lstat, mkdtemp, mkdir, writeFile, rm } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import path from 'node:path'
+import process from 'node:process'
+import { fileURLToPath, pathToFileURL } from 'node:url'
+import { promisify } from 'node:util'
+
+const execFileAsync = promisify(execFile)
+
+/** cacache stores downloaded tarballs under this key prefix; registry metadata shares the prefix. */
+const TARBALL_KEY_SUFFIX = '.tgz'
+
+const walkFiles = async (dir) => {
+  let entries
+  try {
+    entries = await readdir(dir, { withFileTypes: true })
+  } catch (error) {
+    if (error.code === 'ENOENT' || error.code === 'ENOTDIR') {
+      return []
+    }
+    throw error
+  }
+
+  const nested = await Promise.all(
+    entries.map(async (entry) => {
+      const entryPath = path.join(dir, entry.name)
+      return entry.isDirectory() ? walkFiles(entryPath) : [entryPath]
+    }),
+  )
+  return nested.flat()
+}
+
+/**
+ * Total bytes of every package tarball npm downloaded into `cacheDir`.
+ *
+ * Read from cacache's index rather than by measuring the cache on disk: the index records an exact
+ * byte count per entry, and lets us exclude cached registry metadata, which is downloaded too but
+ * fluctuates as unrelated packages publish. Tarballs for a published version are immutable, so a
+ * given lockfile always produces the same total.
+ */
+export const sumCachedTarballBytes = async (cacheDir) => {
+  const buckets = await walkFiles(path.join(cacheDir, '_cacache', 'index-v5'))
+
+  // Buckets are append-only logs, so one key can appear several times. Keep the last entry per key
+  // rather than summing every line, which would count a re-fetched tarball more than once.
+  const sizeByKey = new Map()
+
+  for (const bucket of buckets) {
+    const contents = await readFile(bucket, 'utf8')
+    for (const line of contents.split('\n')) {
+      const separator = line.indexOf('\t')
+      if (separator === -1) {
+        continue
+      }
+      let entry
+      try {
+        entry = JSON.parse(line.slice(separator + 1))
+      } catch {
+        // A partially written line is normal in an append-only log.
+        continue
+      }
+      if (entry?.key?.endsWith(TARBALL_KEY_SUFFIX) && typeof entry.size === 'number') {
+        sizeByKey.set(entry.key, entry.size)
+      }
+    }
+  }
+
+  return [...sizeByKey.values()].reduce((total, size) => total + size, 0)
+}
+
+/**
+ * Total bytes of the files under `dir`.
+ *
+ * Uses real file sizes rather than `du`, which reports disk blocks and so inflates a tree of many
+ * small files by an amount that varies with the filesystem. Symlinks are measured as links, not
+ * followed, so `node_modules/.bin` doesn't count binaries twice.
+ */
+export const directoryBytes = async (dir) => {
+  const files = await walkFiles(dir)
+  const sizes = await Promise.all(
+    files.map(async (file) => {
+      try {
+        return (await lstat(file)).size
+      } catch {
+        return 0
+      }
+    }),
+  )
+  return sizes.reduce((total, size) => total + size, 0)
+}
+
+/** Renders one metric in the two-line shape `delta-action` parses: value, then `unit (label)`. */
+export const formatDelta = (value, unit, label) => `${Math.round(value).toString()}\n${unit} (${label})\n`
+
+/** Counts installed packages by looking for the manifests npm wrote, including nested copies. */
+const countPackages = async (nodeModulesDir) => {
+  const files = await walkFiles(nodeModulesDir)
+  return files.filter((file) => path.basename(file) === 'package.json').length
+}
+
+/**
+ * Builds the tarball a release would publish, installs it the way a user would, and reports what
+ * that cost. Measuring a real install rather than the repo's own `node_modules` is what lets these
+ * numbers cover our own package contents as well as our dependencies.
+ */
+const measure = async (repoDir, scratchDir) => {
+  const packDir = path.join(scratchDir, 'pack')
+  const installDir = path.join(scratchDir, 'install')
+  const cacheDir = path.join(scratchDir, 'npm-cache')
+  await mkdir(packDir, { recursive: true })
+  await mkdir(installDir, { recursive: true })
+
+  const { stdout: packStdout } = await execFileAsync(
+    'npm',
+    ['pack', '--json', '--pack-destination', packDir, '--silent'],
+    { cwd: repoDir, maxBuffer: 64 * 1024 * 1024 },
+  )
+  const [packed] = JSON.parse(packStdout)
+  const tarballPath = path.join(packDir, packed.filename)
+
+  await writeFile(
+    path.join(installDir, 'package.json'),
+    `${JSON.stringify({ name: 'size-probe', version: '1.0.0', private: true }, null, 2)}\n`,
+  )
+
+  // `--ignore-scripts` keeps this deterministic and safe; it means the total is what npm downloads,
+  // not what a dependency's own postinstall might fetch afterwards.
+  await execFileAsync(
+    'npm',
+    [
+      'install',
+      tarballPath,
+      '--omit=dev',
+      '--ignore-scripts',
+      '--no-audit',
+      '--no-fund',
+      '--cache',
+      cacheDir,
+      '--loglevel',
+      'error',
+    ],
+    { cwd: installDir, maxBuffer: 64 * 1024 * 1024 },
+  )
+
+  const nodeModulesDir = path.join(installDir, 'node_modules')
+  // The CLI tarball is installed from disk, so it never lands in the cache -- add it back to get
+  // the full download a user would perform.
+  const dependencyTarballBytes = await sumCachedTarballBytes(cacheDir)
+
+  return {
+    packageDownloadBytes: packed.size,
+    totalDownloadBytes: dependencyTarballBytes + packed.size,
+    installedBytes: await directoryBytes(nodeModulesDir),
+    dependencyCount: await countPackages(nodeModulesDir),
+  }
+}
+
+const main = async () => {
+  const repoDir = path.join(path.dirname(fileURLToPath(import.meta.url)), '..')
+  const scratchDir = await mkdtemp(path.join(process.env.RUNNER_TEMP ?? tmpdir(), 'measure-size-'))
+
+  try {
+    const result = await measure(repoDir, scratchDir)
+
+    const metrics = [
+      ['.delta.downloadSizePackage', result.packageDownloadBytes / 1024, 'kb', 'Download size (CLI package)'],
+      ['.delta.downloadSizeInstall', result.totalDownloadBytes / 1024, 'kb', 'Download size (full install)'],
+      ['.delta.installedSize', result.installedBytes / 1024, 'kb', 'Installed size'],
+      // Deliberately not `.delta.dependencyCount`: this counts every package directory npm wrote,
+      // including nested duplicate copies, so it is a different measurement from the `npm ls` count
+      // that key used to hold. Reusing the key would compare the two and report a phantom jump.
+      ['.delta.installedPackageCount', result.dependencyCount, '', 'Installed package count'],
+    ]
+
+    for (const [filename, value, unit, label] of metrics) {
+      await writeFile(path.join(repoDir, filename), formatDelta(value, unit, label))
+      console.log(`${label}: ${Math.round(value).toLocaleString()} ${unit}`.trim())
+    }
+  } finally {
+    await rm(scratchDir, { recursive: true, force: true })
+  }
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+  await main()
+}

tests/unit/scripts/measure-size.test.ts

@@ -0,0 +1,120 @@
+import { mkdtemp, mkdir, writeFile, rm } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import path from 'node:path'
+
+import { afterEach, beforeEach, describe, expect, test } from 'vitest'
+
+import { directoryBytes, formatDelta, sumCachedTarballBytes } from '../../../scripts/measure-size.js'
+
+let workDir: string
+
+beforeEach(async () => {
+  workDir = await mkdtemp(path.join(tmpdir(), 'measure-size-'))
+})
+
+afterEach(async () => {
+  await rm(workDir, { recursive: true, force: true })
+})
+
+/**
+ * Writes a cacache index bucket. Real buckets are append-only logs of `<hash>\t<json>` lines, which
+ * is the detail most of these tests exist to pin down.
+ */
+const writeIndexBucket = async (cacheDir: string, bucketName: string, entries: object[]) => {
+  const bucketDir = path.join(cacheDir, '_cacache', 'index-v5', 'aa', 'bb')
+  await mkdir(bucketDir, { recursive: true })
+  const contents = entries.map((entry) => `0000000000\t${JSON.stringify(entry)}`).join('\n')
+  await writeFile(path.join(bucketDir, bucketName), `${contents}\n`)
+}
+
+const tarballEntry = (name: string, version: string, size: number) => ({
+  key: `make-fetch-happen:request-cache:https://registry.npmjs.org/${name}/-/${name}-${version}.tgz`,
+  integrity: 'sha512-fake',
+  time: 1_770_825_313_018,
+  size,
+})
+
+describe('sumCachedTarballBytes', () => {
+  test('sums the byte size of every cached tarball', async () => {
+    await writeIndexBucket(workDir, 'bucket1', [tarballEntry('chalk', '5.3.0', 13_397)])
+    await writeIndexBucket(workDir, 'bucket2', [tarballEntry('diff', '4.0.4', 98_055)])
+
+    expect(await sumCachedTarballBytes(workDir)).toBe(111_452)
+  })
+
+  test('ignores cached registry metadata, counting only tarballs', async () => {
+    await writeIndexBucket(workDir, 'bucket1', [
+      tarballEntry('chalk', '5.3.0', 13_397),
+      {
+        key: 'make-fetch-happen:request-cache:https://registry.npmjs.org/chalk',
+        integrity: 'sha512-fake',
+        time: 1_770_825_313_018,
+        size: 5_000_000,
+      },
+    ])
+
+    expect(await sumCachedTarballBytes(workDir)).toBe(13_397)
+  })
+
+  test('counts a re-fetched tarball once rather than once per log entry', async () => {
+    // cacache appends rather than rewrites, so the same key legitimately appears more than once.
+    await writeIndexBucket(workDir, 'bucket1', [
+      tarballEntry('chalk', '5.3.0', 13_397),
+      tarballEntry('chalk', '5.3.0', 13_397),
+    ])
+
+    expect(await sumCachedTarballBytes(workDir)).toBe(13_397)
+  })
+
+  test('skips unparseable lines rather than throwing', async () => {
+    const bucketDir = path.join(workDir, '_cacache', 'index-v5', 'aa', 'bb')
+    await mkdir(bucketDir, { recursive: true })
+    await writeFile(
+      path.join(bucketDir, 'bucket1'),
+      `0000000000\t{"truncated":\n0000000000\t${JSON.stringify(tarballEntry('chalk', '5.3.0', 13_397))}\n`,
+    )
+
+    expect(await sumCachedTarballBytes(workDir)).toBe(13_397)
+  })
+
+  test('returns zero when nothing was downloaded', async () => {
+    expect(await sumCachedTarballBytes(path.join(workDir, 'never-created'))).toBe(0)
+  })
+})
+
+describe('directoryBytes', () => {
+  test('sums real file sizes across nested directories', async () => {
+    await mkdir(path.join(workDir, 'nested'), { recursive: true })
+    await writeFile(path.join(workDir, 'a.js'), 'x'.repeat(100))
+    await writeFile(path.join(workDir, 'nested', 'b.js'), 'x'.repeat(250))
+
+    expect(await directoryBytes(workDir)).toBe(350)
+  })
+
+  test('counts a symlink itself rather than following it to its target', async () => {
+    // `node_modules/.bin` is full of symlinks; following them would count binaries repeatedly.
+    const { symlink } = await import('node:fs/promises')
+    await writeFile(path.join(workDir, 'real.js'), 'x'.repeat(100))
+    await symlink(path.join(workDir, 'real.js'), path.join(workDir, 'link.js'))
+
+    expect(await directoryBytes(workDir)).toBeLessThan(200)
+  })
+
+  test('returns zero for a directory that does not exist', async () => {
+    expect(await directoryBytes(path.join(workDir, 'never-created'))).toBe(0)
+  })
+})
+
+describe('formatDelta', () => {
+  test('renders the value and label in the format delta-action parses', () => {
+    expect(formatDelta(1234, 'kb', 'Download size (full install)')).toBe('1234\nkb (Download size (full install))\n')
+  })
+
+  test('omits the unit for unitless metrics, keeping the leading space', () => {
+    expect(formatDelta(1093, '', 'Dependency count')).toBe('1093\n (Dependency count)\n')
+  })
+
+  test('rounds fractional values, since delta-action expects an integer', () => {
+    expect(formatDelta(1234.6, 'kb', 'Installed size')).toBe('1235\nkb (Installed size)\n')
+  })
+})