From bc272ff14c6bbbc5fd5525bd198dcba4c296360d Mon Sep 17 00:00:00 2001 From: James M Snell Date: Thu, 27 Aug 2026 21:14:51 +0000 Subject: [PATCH 1/6] perf_hooks: implement Histogram meanCI API Signed-off-by: James M Snell Assisted-by: Opencode --- doc/api/perf_hooks.md | 35 ++++++++ lib/internal/histogram.js | 24 ++++++ src/histogram.cc | 76 +++++++++++++---- src/histogram.h | 10 ++- .../test-perf-hooks-histogram-stats.js | 81 +++++++++++++++++++ 5 files changed, 208 insertions(+), 18 deletions(-) diff --git a/doc/api/perf_hooks.md b/doc/api/perf_hooks.md index b9e00c63d2aa..c24082aa20e4 100644 --- a/doc/api/perf_hooks.md +++ b/doc/api/perf_hooks.md @@ -2207,6 +2207,41 @@ added: v11.10.0 The mean of the recorded event loop delays. +### `histogram.meanCI([options])` + + + +* `options` {Object} + * `confidence` {number} The confidence level for the interval, between + 0 and 1 (exclusive). **Default:** `0.95`. +* Returns: {Object} + * `mean` {number} The mean estimate, equivalent to `histogram.mean`. + * `lower` {number} The lower bound of the confidence interval. + * `upper` {number} The upper bound of the confidence interval. + +Returns a two-sided confidence interval for the mean using Student's +t-distribution and the sample standard error. A higher confidence level +produces a wider interval. This interval assumes that samples are independent +and approximately normally distributed, although the approximation is robust +for sufficiently large samples. + +The result reflects the histogram's configured precision and is calculated +from the values represented by its buckets. With fewer than two recorded +values, `lower` and `upper` are `NaN`. When all recorded values are equal, +`lower` and `upper` equal `mean`. + +```js +const { createHistogram } = require('node:perf_hooks'); + +const h = createHistogram(); +for (let i = 1; i <= 100; i++) h.record(i); + +const { mean, lower, upper } = h.meanCI(); +console.log(`mean=${mean}, 95% CI=[${lower}, ${upper}]`); +``` + ### `histogram.min` + + + +> Stability: 1 - Experimental + + + +The `node:bench` module supports defining and running JavaScript benchmarks in +the current process. To access it: + +```mjs +import { bench, suite } from 'node:bench'; +``` + +```cjs +const { bench, suite } = require('node:bench'); +``` + +This module is only available under the `node:` scheme. + +```mjs +import { bench, suite } from 'node:bench'; + +suite('URL', () => { + const input = 'https://example.com/a?b=c'; + + bench('construct', { + samples: 30, + params: { input: 'short' }, + }, (b) => { + const operations = 10_000; + + b.start(); + for (let i = 0; i < operations; i++) { + new URL(input); + } + b.end(operations); + }); +}); +``` + +Benchmarks are executed serially in declaration order. Declared benchmarks are +scheduled automatically. Call `run()` during the same turn as the declarations +to consume the event stream or configure filtering. +If an automatically scheduled run fails and `run()` was not called, the process +exit code is set to `1`. + +## Measurement model + +Each warmup and measured sample invokes the benchmark function once with a +fresh {BenchContext}. The function must call `context.start()` and +`context.end(operations)` exactly once. Setup before `start()` and cleanup after +`end()` are outside the measured region. Promise-returning functions are +awaited. + +An event loop turn occurs between sample invocations. The runner executes +benchmarks serially, but it does not provide process isolation. Other work in +the process, JIT compilation, garbage collection, CPU frequency changes, and +system load can all affect results. Keep raw samples when comparing results and +investigate noisy or skewed distributions rather than treating a confidence +interval as a pass/fail threshold. + +## `bench([name][, options], fn)` + + + +* `name` {string} The benchmark name. **Default:** The `name` property of `fn`, + or `''` when `fn` has no name. +* `options` {Object} + * `only` {boolean} When any benchmark or containing suite has `only` set, + benchmarks without `only` in their hierarchy are skipped. **Default:** + `false`. + * `params` {Object} String, finite number, or boolean metadata identifying + this benchmark configuration. Parameter keys are sorted when constructing + the stable benchmark identity. **Default:** An empty object. + * `samples` {number} The number of measured callback invocations. Must be a + positive 32-bit unsigned integer. **Default:** `30`. + * `signal` {AbortSignal} Allows aborting this benchmark. + * `skip` {boolean|string} If truthy, the benchmark is skipped. A string is + included in the result as the skip reason. **Default:** `false`. + * `tags` {string\[]} Labels associated with the benchmark. Tags are + lowercased, deduplicated, and inherited from containing suites by union. + **Default:** `[]`. + * `timeout` {number} The number of milliseconds after which the benchmark + fails. **Default:** `Infinity`. + * `warmup` {number} The number of unreported callback invocations before + measured samples. Must be a 32-bit unsigned integer. **Default:** `0`. +* `fn` {Function|AsyncFunction} The benchmark function. It receives a + {BenchContext}. +* Returns: {Promise} Fulfilled with the benchmark result after a top-level + benchmark finishes, or with `undefined` immediately when declared in a + suite. + +Warmup invocations use the same callback and timing contract as measured +samples, but their samples are discarded. An exception, rejection, timeout, +abort, missing timing call, or duplicate timing call stops the current +benchmark. Later benchmarks continue to run. + +A timeout or abort cannot interrupt synchronous JavaScript and does not forcibly +cancel asynchronous work that ignores `context.signal`. + +The stable `benchId` is based on the source file, hierarchical suite and +benchmark names, and canonicalized parameters. Declaring the same identity +more than once reports an error rather than merging the samples. + +### `bench.skip([name][, options], fn)` + + + +Shorthand for `bench(name, { ...options, skip: true }, fn)`. + +### `bench.only([name][, options], fn)` + + + +Shorthand for `bench(name, { ...options, only: true }, fn)`. + +## `suite([name][, options], fn)` + + + +* `name` {string} The suite name. **Default:** The `name` property of `fn`, or + `''` when `fn` has no name. +* `options` {Object} + * `only` {boolean} Selects all benchmarks nested in this suite. **Default:** + `false`. + * `skip` {boolean|string} Skips all benchmarks nested in this suite. + **Default:** `false`. + * `tags` {string\[]} Labels inherited by nested suites and benchmarks. + **Default:** `[]`. +* `fn` {Function|AsyncFunction} A function that declares nested suites, + benchmarks, and hooks. +* Returns: {Promise} Fulfilled when a top-level suite finishes, or with + `undefined` immediately when declared in another suite. + +Suite functions run while declarations are collected. Promise-returning suite +functions are awaited before benchmark execution begins. + +## `describe([name][, options], fn)` + + + +Alias for `suite()`. + +## `before(fn)` + + + +* `fn` {Function|AsyncFunction} The hook function. + +Registers a hook that runs once before the benchmarks in the current suite. + +## `after(fn)` + + + +* `fn` {Function|AsyncFunction} The hook function. + +Registers a hook that runs once after the benchmarks in the current suite. + +## `beforeEach(fn)` + + + +* `fn` {Function|AsyncFunction} The hook function. It receives an object with + the benchmark's `name`, `params`, and `signal`. + +Registers a hook that runs once before each complete logical benchmark in the +current suite. It does not run before every sample. Per-sample setup belongs in +the benchmark function before `context.start()`. + +## `afterEach(fn)` + + + +* `fn` {Function|AsyncFunction} The hook function. It receives an object with + the benchmark's `name`, `params`, and `signal`. + +Registers a hook that runs once after each complete logical benchmark in the +current suite. It does not run after every sample. Per-sample cleanup belongs +in the benchmark function after `context.end()`. + +## `run([options])` + + + +* `options` {Object} + * `namePattern` {string|RegExp} Only runs benchmarks whose full hierarchical + name matches the pattern. String values are interpreted as JavaScript + regular expressions. + * `signal` {AbortSignal} Allows aborting in-progress benchmark execution. +* Returns: {BenchmarksStream} + +Returns the object-mode event stream for the in-process benchmark run. Call +`run()` during the same turn in which benchmarks are declared, before automatic +execution begins. Calling `run()` is optional when the returned stream is not +needed. + +```mjs +import { bench, run } from 'node:bench'; + +bench('example', { samples: 3 }, (b) => { + b.start(); + doWork(); + b.end(1); +}); + +for await (const { type, data } of run()) { + if (type === 'bench:complete' && data.error === undefined) { + console.log(data.name, data.summary.mean); + } +} +``` + +## Class: `BenchContext` + +An instance of `BenchContext` is passed to every benchmark invocation. A new +instance is created for every warmup and measured sample. + +### `context.name` + + + +* {string} + +The benchmark name. + +### `context.params` + + + +* {Object} + +The benchmark's canonicalized parameter metadata. + +### `context.signal` + + + +* {AbortSignal} + +An abort signal that is triggered when the benchmark is aborted, times out, or +finishes. + +### `context.start()` + + + +Starts the measured region using `process.hrtime.bigint()`. Calling `start()` +more than once is an error. + +### `context.end(operations)` + + + +* `operations` {number} The number of completed operations. Must be a positive + safe integer. + +Ends the measured region. The end timestamp is captured before `operations` is +validated. Calling `end()` before `start()`, calling it more than once, or +recording a zero-duration sample is an error. + +## Class: `BenchmarksStream` + +`BenchmarksStream` is an object-mode {stream.Readable}. Each lifecycle record is +both emitted as a named event and made available on the stream as +`{ type, data }`. + +The events are emitted in execution order: + +* `'bench:start'` +* `'bench:sample'` +* `'bench:complete'` +* `'bench:diagnostic'` +* `'bench:summary'` + +Every benchmark-scoped event contains `benchId` and `parentId`. +`'bench:complete'` data contains a [benchmark result][]. A failed result has an +additional `error` property and may contain samples recorded before the error. +A skipped result has an additional `skip` property and an empty `samples` +array. `'bench:diagnostic'` reports suite and hook errors. `'bench:summary'` +contains overall `success`, `counts`, `duration_ns`, and `file` properties. + +## Sample result + +Each measured sample has the following properties: + +* `operations` {number} The positive operation count passed to + `context.end()`. +* `duration_ns` {bigint} The measured duration in nanoseconds. +* `rate` {number} Operations per second. + +## Benchmark result + +A completed benchmark result contains: + +* `benchId` {string} The stable benchmark identity. +* `parentId` {string|null} The stable containing suite identity. +* `name` {string} The benchmark name. +* `file` {string} The source file. +* `line` {number} The source line. +* `column` {number} The source column. +* `tags` {string\[]} The inherited canonical tags. +* `params` {Object} The canonical parameter metadata. +* `samples` {Object\[]} The exact measured samples. +* `summary` {Object} + * `mean` {number} The arithmetic mean of per-sample rates. + * `median` {number} The median per-sample rate. + * `min` {number} The minimum per-sample rate. + * `max` {number} The maximum per-sample rate. + * `stddev` {number} The population standard deviation of rates. + * `coefficientOfVariation` {number} `stddev / mean`. + * `confidenceInterval` {Object} The 95% Student's t confidence interval for + the mean rate, with `lower` and `upper` properties. + * `medianConfidenceInterval` {Object} The 95% nonparametric confidence + interval for the median rate, with `lower` and `upper` properties. + * `skewness` {number} The skewness of the scaled rate histogram. + +[benchmark result]: #benchmark-result diff --git a/doc/api/index.md b/doc/api/index.md index 146c0e13df65..a30724c064e1 100644 --- a/doc/api/index.md +++ b/doc/api/index.md @@ -9,6 +9,7 @@ * [Assertion testing](assert.md) * [Asynchronous context tracking](async_context.md) * [Async hooks](async_hooks.md) +* [Benchmark runner](bench.md) * [Buffer](buffer.md) * [C++ addons](addons.md) * [C/C++ addons with Node-API](n-api.md) diff --git a/lib/bench.js b/lib/bench.js new file mode 100644 index 000000000000..454fad4f8570 --- /dev/null +++ b/lib/bench.js @@ -0,0 +1,30 @@ +'use strict'; + +const { + ObjectAssign, +} = primordials; + +const { emitExperimentalWarning } = require('internal/util'); +const { + after, + afterEach, + before, + beforeEach, + bench, + suite, +} = require('internal/bench_runner/harness'); +const { run } = require('internal/bench_runner/runner'); + +emitExperimentalWarning('Benchmarks'); + +module.exports = bench; +ObjectAssign(module.exports, { + after, + afterEach, + before, + beforeEach, + bench, + describe: suite, + run, + suite, +}); diff --git a/lib/internal/bench_runner/benchmark.js b/lib/internal/bench_runner/benchmark.js new file mode 100644 index 000000000000..27f372d7295f --- /dev/null +++ b/lib/internal/bench_runner/benchmark.js @@ -0,0 +1,411 @@ +'use strict'; + +const { + ArrayIsArray, + ArrayPrototypeJoin, + ArrayPrototypePush, + ArrayPrototypeReverse, + ArrayPrototypeSlice, + ArrayPrototypeSort, + JSONStringify, + MathFloor, + MathMax, + MathMin, + MathRound, + MathSqrt, + Number, + NumberIsFinite, + NumberMAX_SAFE_INTEGER, + ObjectFreeze, + ObjectKeys, + PromiseWithResolvers, + SafeSet, + StringPrototypeToLowerCase, +} = primordials; +const { AsyncResource } = require('async_hooks'); +const { + codes: { + ERR_INVALID_ARG_TYPE, + ERR_INVALID_ARG_VALUE, + ERR_INVALID_STATE, + ERR_OUT_OF_RANGE, + }, +} = require('internal/errors'); +const { createHistogram } = require('internal/histogram'); +const { TIMEOUT_MAX } = require('internal/timers'); +const { kEmptyObject } = require('internal/util'); +const { + validateAbortSignal, + validateFunction, + validateInteger, + validateNumber, + validateObject, + validateString, + validateUint32, +} = require('internal/validators'); + +const { bigint: hrtime } = process.hrtime; +const kDefaultSamples = 30; +const kDefaultWarmup = 0; +const kEmptyParams = ObjectFreeze({ __proto__: null }); +const kEmptyTags = ObjectFreeze([]); + +function validateSkip(skip) { + if (skip !== undefined && typeof skip !== 'boolean' && + typeof skip !== 'string') { + throw new ERR_INVALID_ARG_TYPE('options.skip', ['boolean', 'string'], skip); + } +} + +function canonicalizeTags(tags, parentTags = kEmptyTags) { + if (tags === undefined) return parentTags; + if (!ArrayIsArray(tags)) { + throw new ERR_INVALID_ARG_TYPE('options.tags', 'Array', tags); + } + + const result = ArrayPrototypeSlice(parentTags); + const seen = new SafeSet(parentTags); + for (let i = 0; i < tags.length; i++) { + validateString(tags[i], `options.tags[${i}]`); + if (tags[i].length === 0) { + throw new ERR_INVALID_ARG_VALUE( + `options.tags[${i}]`, tags[i], 'must not be empty'); + } + const tag = StringPrototypeToLowerCase(tags[i]); + if (!seen.has(tag)) { + seen.add(tag); + ArrayPrototypePush(result, tag); + } + } + return ObjectFreeze(result); +} + +function canonicalizeParams(params) { + if (params === undefined) return kEmptyParams; + validateObject(params, 'options.params'); + + const result = { __proto__: null }; + const keys = ObjectKeys(params); + ArrayPrototypeSort(keys); + for (let i = 0; i < keys.length; i++) { + const key = keys[i]; + const value = params[key]; + if (typeof value !== 'string' && typeof value !== 'boolean' && + (typeof value !== 'number' || !NumberIsFinite(value))) { + if (typeof value === 'number') { + throw new ERR_OUT_OF_RANGE( + `options.params.${key}`, 'a finite number', value); + } + throw new ERR_INVALID_ARG_TYPE( + `options.params.${key}`, ['string', 'number', 'boolean'], value); + } + result[key] = value; + } + return ObjectFreeze(result); +} + +function validateNodeOptions(options, parentTags) { + validateObject(options, 'options'); + const { only = false, skip, tags } = options; + if (typeof only !== 'boolean') { + throw new ERR_INVALID_ARG_TYPE('options.only', 'boolean', only); + } + validateSkip(skip); + return { + __proto__: null, + only, + skip, + tags: canonicalizeTags(tags, parentTags), + }; +} + +function createLocation(loc, fallbackFile) { + return { + __proto__: null, + file: loc?.[2] ?? fallbackFile, + line: loc?.[0], + column: loc?.[1], + }; +} + +function getNamePath(parent, name) { + const path = []; + for (let current = parent; current?.parent !== null; current = current.parent) { + ArrayPrototypePush(path, current.name); + } + ArrayPrototypeReverse(path); + ArrayPrototypePush(path, name); + return path; +} + +class Suite extends AsyncResource { + constructor(harness, parent, name, options, fn, loc, isRoot = false) { + super('BenchSuite'); + const validated = validateNodeOptions( + options, parent?.tags ?? kEmptyTags); + + this.harness = harness; + this.parent = parent; + this.name = name; + this.fn = fn; + this.loc = createLocation(loc, harness.entryFile); + this.only = validated.only; + this.skip = validated.skip; + this.tags = validated.tags; + this.isRoot = isRoot; + this.children = []; + this.hooks = { + __proto__: null, + after: [], + afterEach: [], + before: [], + beforeEach: [], + }; + this.buildError = null; + this.buildPromise = null; + this.finished = false; + this.completion = PromiseWithResolvers(); + } +} + +class Bench extends AsyncResource { + constructor(harness, parent, name, options, fn, loc) { + super('Benchmark'); + const validated = validateNodeOptions(options, parent.tags); + const { + params, + samples = kDefaultSamples, + signal, + timeout = Infinity, + warmup = kDefaultWarmup, + } = options; + + validateUint32(samples, 'options.samples', true); + validateUint32(warmup, 'options.warmup'); + validateAbortSignal(signal, 'options.signal'); + if (timeout !== Infinity) { + validateNumber(timeout, 'options.timeout', 0, TIMEOUT_MAX); + } + + this.harness = harness; + this.parent = parent; + this.name = name; + this.fn = fn; + this.loc = createLocation(loc, harness.entryFile); + this.only = validated.only; + this.skip = validated.skip; + this.tags = validated.tags; + this.params = canonicalizeParams(params); + this.samples = samples; + this.warmup = warmup; + this.timeout = timeout; + this.outerSignal = signal; + this.namePath = getNamePath(parent, name); + this.fullName = ArrayPrototypeJoin(this.namePath, ' '); + this.benchId = JSONStringify([ + this.loc.file, + this.namePath, + this.params, + ]); + this.parentId = parent.isRoot ? null : JSONStringify([ + this.loc.file, + getNamePath(parent.parent, parent.name), + ]); + this.finished = false; + this.result = null; + this.completion = PromiseWithResolvers(); + } +} + +class BenchContext { + #closed = false; + #endCalled = false; + #invalid = false; + #sample = null; + #startCalled = false; + #startTime; + + constructor(bench, signal) { + this.name = bench.name; + this.params = bench.params; + this.signal = signal; + } + + start() { + if (this.#closed) { + this.#invalid = true; + throw new ERR_INVALID_STATE('benchmark sample is no longer active'); + } + if (this.#startCalled) { + this.#invalid = true; + throw new ERR_INVALID_STATE( + 'start() must be called exactly once per benchmark sample'); + } + this.#startCalled = true; + this.#startTime = hrtime(); + } + + end(operations) { + const endTime = hrtime(); + if (this.#closed) { + this.#invalid = true; + throw new ERR_INVALID_STATE('benchmark sample is no longer active'); + } + if (this.#endCalled) { + this.#invalid = true; + throw new ERR_INVALID_STATE( + 'end() must be called exactly once per benchmark sample'); + } + this.#endCalled = true; + if (!this.#startCalled) { + this.#invalid = true; + throw new ERR_INVALID_STATE('end() cannot be called before start()'); + } + + try { + validateInteger(operations, 'operations', 1, NumberMAX_SAFE_INTEGER); + } catch (error) { + this.#invalid = true; + throw error; + } + + const duration = endTime - this.#startTime; + if (duration === 0n) { + this.#invalid = true; + throw new ERR_INVALID_STATE( + 'insufficient clock precision for benchmark sample'); + } + this.#sample = { + __proto__: null, + operations, + duration_ns: duration, + rate: operations / (Number(duration) / 1e9), + }; + } + + finish() { + this.#closed = true; + if (this.#invalid) { + throw new ERR_INVALID_STATE( + 'benchmark sample violated the start()/end() contract'); + } + if (!this.#startCalled) { + throw new ERR_INVALID_STATE( + 'benchmark callback did not call start()'); + } + if (!this.#endCalled || this.#sample === null) { + throw new ERR_INVALID_STATE( + 'benchmark callback did not call end()'); + } + return this.#sample; + } + + close() { + this.#closed = true; + } +} + +function arithmeticMean(values) { + let sum = 0; + let compensation = 0; + for (let i = 0; i < values.length; i++) { + const adjusted = values[i] - compensation; + const next = sum + adjusted; + compensation = (next - sum) - adjusted; + sum = next; + } + return sum / values.length; +} + +function summarizeSamples(samples) { + const rates = []; + let min = Infinity; + let max = -Infinity; + for (let i = 0; i < samples.length; i++) { + const rate = samples[i].rate; + ArrayPrototypePush(rates, rate); + min = MathMin(min, rate); + max = MathMax(max, rate); + } + + const mean = arithmeticMean(rates); + let variance = 0; + for (let i = 0; i < rates.length; i++) { + const difference = rates[i] - mean; + variance += difference * difference; + } + variance /= rates.length; + const stddev = MathSqrt(variance); + + const sorted = ArrayPrototypeSlice(rates); + ArrayPrototypeSort(sorted, (a, b) => a - b); + const middle = MathFloor(sorted.length / 2); + const median = sorted.length % 2 === 0 ? + (sorted[middle - 1] + sorted[middle]) / 2 : sorted[middle]; + + const scale = MathMin(1_000_000, NumberMAX_SAFE_INTEGER / max); + const histogram = createHistogram({ __proto__: null, figures: 5 }); + for (let i = 0; i < rates.length; i++) { + const value = MathMax( + 1, + MathMin(NumberMAX_SAFE_INTEGER, MathRound(rates[i] * scale)), + ); + histogram.record(value); + } + + const meanCI = histogram.meanCI(); + const histogramMean = meanCI.mean / scale; + const medianCI = histogram.percentileCI(50); + + return { + __proto__: null, + mean, + median, + min, + max, + stddev, + coefficientOfVariation: stddev / mean, + confidenceInterval: { + __proto__: null, + lower: mean + meanCI.lower / scale - histogramMean, + upper: mean + meanCI.upper / scale - histogramMean, + }, + medianConfidenceInterval: { + __proto__: null, + lower: medianCI.lower / scale, + upper: medianCI.upper / scale, + }, + skewness: histogram.skewness, + }; +} + +function normalizeArgs(type, name, options, fn) { + if (typeof name === 'function') { + fn = name; + name = fn.name || ''; + options = kEmptyObject; + } else if (name !== null && typeof name === 'object') { + fn = options; + options = name; + name = fn?.name || ''; + } else if (typeof options === 'function') { + fn = options; + options = kEmptyObject; + } + + validateFunction(fn, `${type} function`); + validateString(name, `${type} name`); + if (name.length === 0) { + throw new ERR_INVALID_ARG_VALUE(`${type} name`, name, 'must not be empty'); + } + validateObject(options, 'options'); + return { __proto__: null, fn, name, options }; +} + +module.exports = { + Bench, + BenchContext, + Suite, + normalizeArgs, + summarizeSamples, +}; diff --git a/lib/internal/bench_runner/benchmarks_stream.js b/lib/internal/bench_runner/benchmarks_stream.js new file mode 100644 index 000000000000..ac3896d0587e --- /dev/null +++ b/lib/internal/bench_runner/benchmarks_stream.js @@ -0,0 +1,74 @@ +'use strict'; + +const { + ArrayPrototypePush, + ArrayPrototypeShift, + NumberMAX_SAFE_INTEGER, + Symbol, +} = primordials; +const Readable = require('internal/streams/readable'); + +const kEmitMessage = Symbol('kEmitMessage'); + +class BenchmarksStream extends Readable { + #buffer = []; + #canPush = true; + + constructor() { + super({ + __proto__: null, + objectMode: true, + highWaterMark: NumberMAX_SAFE_INTEGER, + }); + } + + _read() { + this.#canPush = true; + while (this.#buffer.length > 0) { + const record = ArrayPrototypeShift(this.#buffer); + if (!this.#tryPush(record)) return; + } + } + + start(data) { + this[kEmitMessage]('bench:start', data); + } + + sample(data) { + this[kEmitMessage]('bench:sample', data); + } + + complete(data) { + this[kEmitMessage]('bench:complete', data); + } + + diagnostic(data) { + this[kEmitMessage]('bench:diagnostic', data); + } + + summary(data) { + this[kEmitMessage]('bench:summary', data); + } + + end() { + this.#tryPush(null); + } + + [kEmitMessage](type, data) { + this.emit(type, data); + this.#tryPush({ type, data }); + } + + #tryPush(record) { + if (this.#canPush) { + this.#canPush = this.push(record); + } else { + ArrayPrototypePush(this.#buffer, record); + } + return this.#canPush; + } +} + +module.exports = { + BenchmarksStream, +}; diff --git a/lib/internal/bench_runner/harness.js b/lib/internal/bench_runner/harness.js new file mode 100644 index 000000000000..0409b38e6b91 --- /dev/null +++ b/lib/internal/bench_runner/harness.js @@ -0,0 +1,687 @@ +'use strict'; + +const { + ArrayPrototypePush, + ArrayPrototypeReverse, + ArrayPrototypeSlice, + FunctionPrototypeCall, + Promise, + PromisePrototypeThen, + PromiseResolve, + PromiseWithResolvers, + ReflectApply, + RegExp, + RegExpPrototypeExec, + SafeMap, + SafePromiseRace, + SymbolDispose, +} = primordials; +const { getCallerLocation } = internalBinding('util'); +const { exitCodes: { kGenericUserError } } = internalBinding('errors'); +const { AsyncLocalStorage } = require('async_hooks'); +const { AbortController } = require('internal/abort_controller'); +const { + AbortError, + codes: { + ERR_INVALID_ARG_TYPE, + ERR_INVALID_STATE, + ERR_OPERATION_FAILED, + }, +} = require('internal/errors'); +const { addAbortListener } = require('internal/events/abort_listener'); +const { + kEmptyObject, +} = require('internal/util'); +const { isRegExp } = require('internal/util/types'); +const { + validateAbortSignal, + validateFunction, + validateObject, +} = require('internal/validators'); +const { queueMicrotask } = require('internal/process/task_queues'); +const { clearTimeout, setImmediate, setTimeout } = require('timers'); +const { + Bench, + BenchContext, + Suite, + normalizeArgs, + summarizeSamples, +} = require('internal/bench_runner/benchmark'); +const { + BenchmarksStream, +} = require('internal/bench_runner/benchmarks_stream'); + +const { bigint: hrtime } = process.hrtime; +const kHookNames = ['after', 'afterEach', 'before', 'beforeEach']; + +function eventLoopTurn() { + return new Promise((resolve) => setImmediate(resolve)); +} + +function createAbortError(signal) { + return new AbortError(undefined, { __proto__: null, cause: signal.reason }); +} + +class Harness { + #buildPromises = []; + #duplicateErrors = new SafeMap(); + #explicitRun = false; + #hasOnly = false; + #runPromise = null; + #scheduled = false; + #storage = new AsyncLocalStorage(); + + constructor() { + this.entryFile = process.argv?.[1]; + this.stream = new BenchmarksStream(); + this.state = 'collecting'; + this.namePattern = null; + this.outerSignal = undefined; + this.success = true; + this.counts = { + __proto__: null, + completed: 0, + failed: 0, + skipped: 0, + total: 0, + }; + this.root = new Suite( + this, + null, + '', + kEmptyObject, + undefined, + undefined, + true, + ); + } + + #ensureCollecting() { + if (this.state === 'collecting' || + (this.state === 'building' && + this.#storage.getStore() instanceof Suite)) return; + throw new ERR_INVALID_STATE( + 'benchmarks cannot be declared after execution has started'); + } + + #getParent() { + const current = this.#storage.getStore(); + return current instanceof Suite ? current : this.root; + } + + createBench(name, options, fn, overrides = kEmptyObject) { + this.#ensureCollecting(); + const normalized = normalizeArgs('benchmark', name, options, fn); + const parent = this.#getParent(); + const benchmark = new Bench( + this, + parent, + normalized.name, + { __proto__: null, ...normalized.options, ...overrides }, + normalized.fn, + overrides.loc, + ); + ArrayPrototypePush(parent.children, benchmark); + this.#schedule(); + return parent.isRoot ? benchmark.completion.promise : PromiseResolve(); + } + + createSuite(name, options, fn, overrides = kEmptyObject) { + this.#ensureCollecting(); + const normalized = normalizeArgs('suite', name, options, fn); + const parent = this.#getParent(); + const suite = new Suite( + this, + parent, + normalized.name, + { __proto__: null, ...normalized.options, ...overrides }, + normalized.fn, + overrides.loc, + ); + ArrayPrototypePush(parent.children, suite); + this.#buildSuite(suite); + this.#schedule(); + return parent.isRoot ? suite.completion.promise : PromiseResolve(); + } + + createHook(name, fn, options = kEmptyObject) { + this.#ensureCollecting(); + validateFunction(fn, 'hook function'); + validateObject(options, 'options'); + const parent = this.#getParent(); + ArrayPrototypePush(parent.hooks[name], { + __proto__: null, + fn, + loc: getCallerLocation(), + }); + this.#schedule(); + } + + #buildSuite(suite) { + let result; + try { + result = suite.runInAsyncScope(() => this.#storage.run( + suite, + () => FunctionPrototypeCall(suite.fn), + )); + } catch (error) { + suite.buildError = error; + result = undefined; + } + + suite.buildPromise = PromisePrototypeThen( + PromiseResolve(result), + undefined, + (error) => { + suite.buildError = error; + }, + ); + ArrayPrototypePush(this.#buildPromises, suite.buildPromise); + } + + configure(options = kEmptyObject) { + validateObject(options, 'options'); + if (this.#runPromise !== null) { + if (options !== kEmptyObject) { + throw new ERR_INVALID_STATE('benchmark execution has already started'); + } + return; + } + + const { namePattern, signal } = options; + if (namePattern !== undefined) { + if (typeof namePattern === 'string') { + this.namePattern = new RegExp(namePattern); + } else if (isRegExp(namePattern)) { + this.namePattern = namePattern; + } else { + throw new ERR_INVALID_ARG_TYPE( + 'options.namePattern', ['string', 'RegExp'], namePattern); + } + } + validateAbortSignal(signal, 'options.signal'); + this.outerSignal = signal; + this.#explicitRun = true; + } + + run(options = kEmptyObject) { + this.configure(options); + this.#schedule(); + return this.stream; + } + + #schedule() { + if (this.#scheduled) return; + this.#scheduled = true; + queueMicrotask(() => { + if (this.#runPromise === null) { + this.#runPromise = this.#execute(); + PromisePrototypeThen(this.#runPromise, undefined, (error) => { + this.#diagnostic(error, undefined, 'error'); + this.#finish(); + }); + } + }); + } + + async #waitForBuild() { + for (let i = 0; i < this.#buildPromises.length; i++) { + await this.#buildPromises[i]; + } + } + + #walk(node, callback) { + for (let i = 0; i < node.children.length; i++) { + const child = node.children[i]; + callback(child); + if (child instanceof Suite) this.#walk(child, callback); + } + } + + #prepare() { + const identities = new SafeMap(); + this.#walk(this.root, (node) => { + if (node.only) this.#hasOnly = true; + if (!(node instanceof Bench)) return; + + this.counts.total++; + const existing = identities.get(node.benchId); + if (existing === undefined) { + identities.set(node.benchId, node); + } else { + this.#duplicateErrors.set(node, new ERR_INVALID_STATE( + `duplicate benchmark identity for "${node.fullName}"`)); + } + }); + } + + #hasSelectedAncestor(benchmark) { + for (let current = benchmark; current !== null; current = current.parent) { + if (current.only) return true; + } + return false; + } + + #getSkip(benchmark) { + for (let current = benchmark; current !== null; current = current.parent) { + if (current.skip !== undefined && current.skip !== false) { + return current.skip; + } + } + if (this.#hasOnly && !this.#hasSelectedAncestor(benchmark)) return 'only'; + if (this.namePattern !== null) { + this.namePattern.lastIndex = 0; + if (RegExpPrototypeExec(this.namePattern, benchmark.fullName) === null) { + return 'name pattern'; + } + } + return null; + } + + #suiteHasActiveBench(suite) { + for (let i = 0; i < suite.children.length; i++) { + const child = suite.children[i]; + if (child instanceof Suite) { + if (this.#suiteHasActiveBench(child)) return true; + } else if (this.#getSkip(child) === null) { + return true; + } + } + return false; + } + + async #invoke(resource, store, fn, args) { + const result = resource.runInAsyncScope(() => this.#storage.run( + store, + () => ReflectApply(fn, undefined, args), + )); + return PromiseResolve(result); + } + + async #runHooks(suite, name, resource, store, context) { + const hooks = suite.hooks[name]; + for (let i = 0; i < hooks.length; i++) { + await this.#invoke(resource, store, hooks[i].fn, [context]); + } + } + + async #runSuiteHooks(suite, name) { + const context = { + __proto__: null, + name: suite.name, + signal: this.outerSignal, + }; + await this.#runHooks(suite, name, suite, suite, context); + } + + #diagnostic(error, loc, level = 'info') { + this.success = false; + this.stream.diagnostic({ + __proto__: null, + message: error?.message ?? `${error}`, + error, + level, + file: loc?.file ?? loc?.[2], + line: loc?.line ?? loc?.[0], + column: loc?.column ?? loc?.[1], + }); + } + + async #completeSubtree(node, error) { + for (let i = 0; i < node.children.length; i++) { + const child = node.children[i]; + if (child instanceof Suite) { + await this.#completeSubtree(child, error); + child.finished = true; + child.completion.resolve(); + child.emitDestroy(); + } else { + await this.#executeBench(child, error); + } + } + } + + async #executeSuite(suite) { + if (suite.buildError !== null) { + this.#diagnostic(suite.buildError, suite.loc, 'error'); + await this.#completeSubtree(suite, suite.buildError); + suite.finished = true; + suite.completion.resolve(); + suite.emitDestroy(); + return; + } + + const active = this.#suiteHasActiveBench(suite); + let beforeError; + if (active) { + try { + await this.#runSuiteHooks(suite, 'before'); + } catch (error) { + beforeError = error; + this.#diagnostic(error, suite.loc, 'error'); + } + } + + if (beforeError !== undefined) { + await this.#completeSubtree(suite, beforeError); + } else { + for (let i = 0; i < suite.children.length; i++) { + const child = suite.children[i]; + if (child instanceof Suite) { + await this.#executeSuite(child); + } else { + await this.#executeBench(child); + } + } + } + + if (active) { + try { + await this.#runSuiteHooks(suite, 'after'); + } catch (error) { + this.#diagnostic(error, suite.loc, 'error'); + } + } + suite.finished = true; + suite.completion.resolve(); + if (!suite.isRoot) suite.emitDestroy(); + } + + #getHookSuites(benchmark) { + const suites = []; + for (let current = benchmark.parent; current !== null; current = current.parent) { + ArrayPrototypePush(suites, current); + } + ArrayPrototypeReverse(suites); + return suites; + } + + async #runBenchHooks(benchmark, name, context) { + const suites = this.#getHookSuites(benchmark); + if (name === 'afterEach') ArrayPrototypeReverse(suites); + for (let i = 0; i < suites.length; i++) { + await this.#runHooks( + suites[i], name, benchmark, benchmark, context); + } + } + + async #runWithStop(benchmark, controller, callback) { + const signals = []; + if (this.outerSignal !== undefined) { + ArrayPrototypePush(signals, this.outerSignal); + } + if (benchmark.outerSignal !== undefined && + benchmark.outerSignal !== this.outerSignal) { + ArrayPrototypePush(signals, benchmark.outerSignal); + } + + for (let i = 0; i < signals.length; i++) { + if (signals[i].aborted) { + const error = createAbortError(signals[i]); + controller.abort(error); + throw error; + } + } + + const stop = PromiseWithResolvers(); + const listeners = []; + let timer; + for (let i = 0; i < signals.length; i++) { + const signal = signals[i]; + ArrayPrototypePush(listeners, addAbortListener(signal, () => { + const error = createAbortError(signal); + controller.abort(error); + stop.reject(error); + })); + } + if (benchmark.timeout !== Infinity) { + timer = setTimeout(() => { + const error = new ERR_OPERATION_FAILED( + `Benchmark timed out after ${benchmark.timeout}ms`); + controller.abort(error); + stop.reject(error); + }, benchmark.timeout); + } + + const work = callback(); + try { + if (signals.length === 0 && timer === undefined) return await work; + return await SafePromiseRace([work, stop.promise]); + } finally { + if (timer !== undefined) clearTimeout(timer); + for (let i = 0; i < listeners.length; i++) { + listeners[i][SymbolDispose](); + } + } + } + + async #runSample(benchmark, signal) { + const context = new BenchContext(benchmark, signal); + try { + await this.#invoke( + benchmark, benchmark, benchmark.fn, [context]); + return context.finish(); + } catch (error) { + context.close(); + throw error; + } + } + + #createResult(benchmark, samples, extra = kEmptyObject) { + return { + __proto__: null, + benchId: benchmark.benchId, + parentId: benchmark.parentId, + name: benchmark.name, + file: benchmark.loc.file, + line: benchmark.loc.line, + column: benchmark.loc.column, + tags: ArrayPrototypeSlice(benchmark.tags), + params: benchmark.params, + samples, + ...extra, + }; + } + + #recordResult(benchmark, result) { + benchmark.finished = true; + benchmark.result = result; + this.stream.complete(result); + benchmark.completion.resolve(result); + benchmark.emitDestroy(); + } + + async #executeBench(benchmark, forcedError = undefined) { + const duplicateError = this.#duplicateErrors.get(benchmark); + if (duplicateError !== undefined) { + this.success = false; + this.counts.failed++; + this.#recordResult(benchmark, this.#createResult( + benchmark, + [], + { __proto__: null, error: duplicateError }, + )); + return; + } + + const skip = this.#getSkip(benchmark); + if (skip !== null) { + this.counts.skipped++; + this.#recordResult(benchmark, this.#createResult( + benchmark, + [], + { __proto__: null, skip }, + )); + return; + } + + if (forcedError !== undefined) { + this.success = false; + this.counts.failed++; + this.#recordResult(benchmark, this.#createResult( + benchmark, + [], + { __proto__: null, error: forcedError }, + )); + return; + } + + this.stream.start({ + __proto__: null, + benchId: benchmark.benchId, + parentId: benchmark.parentId, + name: benchmark.name, + file: benchmark.loc.file, + line: benchmark.loc.line, + column: benchmark.loc.column, + tags: ArrayPrototypeSlice(benchmark.tags), + params: benchmark.params, + }); + + const controller = new AbortController(); + const samples = []; + const hookContext = { + __proto__: null, + name: benchmark.name, + params: benchmark.params, + signal: controller.signal, + }; + let error; + + try { + await this.#runWithStop(benchmark, controller, async () => { + try { + await this.#runBenchHooks( + benchmark, 'beforeEach', hookContext); + const total = benchmark.warmup + benchmark.samples; + for (let i = 0; i < total; i++) { + if (controller.signal.aborted) { + throw controller.signal.reason; + } + const sample = await this.#runSample( + benchmark, controller.signal); + if (controller.signal.aborted) { + throw controller.signal.reason; + } + if (i >= benchmark.warmup) { + ArrayPrototypePush(samples, sample); + this.stream.sample({ + __proto__: null, + benchId: benchmark.benchId, + parentId: benchmark.parentId, + name: benchmark.name, + index: i - benchmark.warmup, + ...sample, + }); + } + if (i + 1 < total) await eventLoopTurn(); + } + } finally { + await this.#runBenchHooks( + benchmark, 'afterEach', hookContext); + } + }); + } catch (cause) { + error = cause; + } finally { + controller.abort(); + } + + if (error !== undefined) { + this.success = false; + this.counts.failed++; + this.#recordResult(benchmark, this.#createResult( + benchmark, + samples, + { __proto__: null, error }, + )); + return; + } + + this.counts.completed++; + this.#recordResult(benchmark, this.#createResult( + benchmark, + samples, + { __proto__: null, summary: summarizeSamples(samples) }, + )); + } + + #finish(startTime) { + if (this.state === 'finished') return; + this.state = 'finished'; + const duration = startTime === undefined ? 0n : hrtime() - startTime; + this.stream.summary({ + __proto__: null, + success: this.success, + counts: this.counts, + duration_ns: duration, + file: this.entryFile, + }); + this.stream.end(); + this.root.finished = true; + this.root.completion.resolve(); + this.root.emitDestroy(); + this.#storage.disable(); + if (!this.#explicitRun && !this.success) { + process.exitCode = kGenericUserError; + } + } + + async #execute() { + this.state = 'building'; + const startTime = hrtime(); + await this.#waitForBuild(); + this.#prepare(); + this.state = 'running'; + await this.#executeSuite(this.root); + this.#finish(startTime); + } +} + +let globalHarness; + +function lazyHarness() { + globalHarness ??= new Harness(); + return globalHarness; +} + +function runInParentContext(type) { + const declare = (name, options, fn, overrides = kEmptyObject) => { + const harness = lazyHarness(); + const loc = getCallerLocation(); + const declarationOptions = { __proto__: null, ...overrides, loc }; + return type === 'benchmark' ? + harness.createBench(name, options, fn, declarationOptions) : + harness.createSuite(name, options, fn, declarationOptions); + }; + + if (type === 'benchmark') { + declare.skip = (name, options, fn) => declare( + name, options, fn, { __proto__: null, skip: true }); + declare.only = (name, options, fn) => declare( + name, options, fn, { __proto__: null, only: true }); + } + return declare; +} + +function hook(name) { + return (fn, options) => lazyHarness().createHook(name, fn, options); +} + +const bench = runInParentContext('benchmark'); +const suite = runInParentContext('suite'); + +function runBenchmarks(options) { + return lazyHarness().run(options); +} + +module.exports = { + Harness, + after: hook(kHookNames[0]), + afterEach: hook(kHookNames[1]), + before: hook(kHookNames[2]), + beforeEach: hook(kHookNames[3]), + bench, + runBenchmarks, + suite, +}; diff --git a/lib/internal/bench_runner/runner.js b/lib/internal/bench_runner/runner.js new file mode 100644 index 000000000000..ee55672b5f11 --- /dev/null +++ b/lib/internal/bench_runner/runner.js @@ -0,0 +1,12 @@ +'use strict'; + +const { kEmptyObject } = require('internal/util'); +const { runBenchmarks } = require('internal/bench_runner/harness'); + +function run(options = kEmptyObject) { + return runBenchmarks(options); +} + +module.exports = { + run, +}; diff --git a/lib/internal/bootstrap/realm.js b/lib/internal/bootstrap/realm.js index 8a4d179806aa..2761e2846f1f 100644 --- a/lib/internal/bootstrap/realm.js +++ b/lib/internal/bootstrap/realm.js @@ -124,6 +124,7 @@ const legacyWrapperList = new SafeSet([ // beginning with "internal/". // Modules that can only be imported via the node: scheme. const schemelessBlockList = new SafeSet([ + 'bench', 'dtls', 'ffi', 'sea', diff --git a/test/module-hooks/test-module-hooks-builtin-require.js b/test/module-hooks/test-module-hooks-builtin-require.js index 2086cbe062b0..b623f4157bea 100644 --- a/test/module-hooks/test-module-hooks-builtin-require.js +++ b/test/module-hooks/test-module-hooks-builtin-require.js @@ -11,6 +11,7 @@ const assert = require('assert'); const { registerHooks } = require('module'); const schemelessBlockList = new Set([ + 'bench', 'sea', 'test', 'test/reporters', diff --git a/test/module-hooks/test-module-hooks-load-builtin-require.js b/test/module-hooks/test-module-hooks-load-builtin-require.js index 962080b3c2c8..262aa1a0d32b 100644 --- a/test/module-hooks/test-module-hooks-load-builtin-require.js +++ b/test/module-hooks/test-module-hooks-load-builtin-require.js @@ -35,6 +35,7 @@ hook.deregister(); // the one with the `node:` prefix. The one with the prefix // stripped for internal lookups should not get passed into the hooks. const schemelessBlockList = new Set([ + 'bench', 'sea', 'test', 'test/reporters', diff --git a/test/parallel/test-bench-auto-run.js b/test/parallel/test-bench-auto-run.js new file mode 100644 index 000000000000..ed02fed9340a --- /dev/null +++ b/test/parallel/test-bench-auto-run.js @@ -0,0 +1,28 @@ +// Flags: --no-warnings +'use strict'; + +const common = require('../common'); +const assert = require('assert'); +const { spawnSync } = require('child_process'); +const { bench } = require('node:bench'); + +const child = spawnSync(process.execPath, [ + '--no-warnings', + '-e', + 'require("node:bench").bench("failure", () => { throw new Error(); })', +]); +assert.strictEqual(child.status, 1); + +const completion = bench('automatic execution', common.mustCall((b) => { + b.start(); + process.hrtime.bigint(); + b.end(1); +}, 30)); + +completion.then(common.mustCall((result) => { + assert.strictEqual(result.name, 'automatic execution'); + assert.strictEqual(result.samples.length, 30); + assert.strictEqual(result.error, undefined); + assert.strictEqual(result.skip, undefined); + assert.strictEqual(result.summary.mean > 0, true); +})); diff --git a/test/parallel/test-bench-errors.js b/test/parallel/test-bench-errors.js new file mode 100644 index 000000000000..7a6f7aec7e16 --- /dev/null +++ b/test/parallel/test-bench-errors.js @@ -0,0 +1,104 @@ +// Flags: --no-warnings +'use strict'; + +const common = require('../common'); +const assert = require('assert'); +const { bench, run } = require('node:bench'); + +const options = { samples: 1 }; + +bench('missing start', options, () => {}); +bench('missing end', options, (b) => b.start()); +bench('end before start', options, (b) => b.end(1)); +bench('duplicate start', options, (b) => { + b.start(); + b.start(); +}); +bench('duplicate end', options, (b) => { + b.start(); + b.end(1); + b.end(1); +}); +bench('invalid operations', options, (b) => { + b.start(); + b.end(0); +}); +bench('throws', options, () => { + throw new Error('benchmark failure'); +}); +bench('timeout', { samples: 1, timeout: 10 }, async () => { + await new Promise(() => {}); +}); +bench('late timeout', { samples: 1, timeout: 5 }, async (b) => { + b.start(); + await new Promise((resolve) => setTimeout(resolve, 30)); + b.end(1); +}); + +const signal = AbortSignal.abort(new Error('stop')); +bench('aborted', { samples: 1, signal }, () => {}); + +function complete(b) { + b.start(); + process.hrtime.bigint(); + b.end(1); +} + +bench('duplicate', { samples: 1, params: { value: 1 } }, complete); +bench('duplicate', { samples: 1, params: { value: 1 } }, complete); +bench('continues', options, complete); + +const completions = []; +const sampleNames = []; +let summary; +const stream = run(); +stream.on('bench:complete', (result) => completions.push(result)); +stream.on('bench:sample', (sample) => sampleNames.push(sample.name)); +stream.on('bench:summary', (result) => { summary = result; }); +stream.on('end', common.mustCall(() => { + assert.strictEqual(completions.length, 13); + assert.deepStrictEqual(summary.counts, { + __proto__: null, + completed: 2, + failed: 11, + skipped: 0, + total: 13, + }); + assert.strictEqual(summary.success, false); + + const byName = new Map(); + for (const result of completions) { + const values = byName.get(result.name) ?? []; + values.push(result); + byName.set(result.name, values); + } + + assert.match(byName.get('missing start')[0].error.message, + /did not call start/); + assert.match(byName.get('missing end')[0].error.message, + /did not call end/); + assert.match(byName.get('end before start')[0].error.message, + /before start/); + assert.strictEqual(byName.get('duplicate start')[0].error.code, + 'ERR_INVALID_STATE'); + assert.strictEqual(byName.get('duplicate end')[0].error.code, + 'ERR_INVALID_STATE'); + assert.strictEqual(byName.get('invalid operations')[0].error.code, + 'ERR_OUT_OF_RANGE'); + assert.strictEqual(byName.get('throws')[0].error.message, + 'benchmark failure'); + assert.strictEqual(byName.get('timeout')[0].error.code, + 'ERR_OPERATION_FAILED'); + assert.strictEqual(byName.get('late timeout')[0].error.code, + 'ERR_OPERATION_FAILED'); + assert.strictEqual(byName.get('aborted')[0].error.code, 'ABORT_ERR'); + + const duplicates = byName.get('duplicate'); + assert.strictEqual(duplicates[0].error, undefined); + assert.match(duplicates[1].error.message, /duplicate benchmark identity/); + assert.strictEqual(byName.get('continues')[0].error, undefined); + setTimeout(common.mustCall(() => { + assert.strictEqual(sampleNames.includes('late timeout'), false); + }), 40); +})); +stream.resume(); diff --git a/test/parallel/test-bench-filtering.js b/test/parallel/test-bench-filtering.js new file mode 100644 index 000000000000..41784c0dafef --- /dev/null +++ b/test/parallel/test-bench-filtering.js @@ -0,0 +1,41 @@ +// Flags: --no-warnings +'use strict'; + +const common = require('../common'); +const assert = require('assert'); +const { bench, run, suite } = require('node:bench'); + +const calls = []; + +function complete(name) { + return (b) => { + calls.push(name); + b.start(); + process.hrtime.bigint(); + b.end(1); + }; +} + +suite('selected', { only: true }, () => { + bench('included', { samples: 1 }, complete('included')); + bench.skip('explicitly skipped', { samples: 1 }, + common.mustNotCall()); + bench('pattern filtered', { samples: 1 }, + common.mustNotCall()); +}); +bench('only filtered', { samples: 1 }, common.mustNotCall()); + +const results = []; +const stream = run({ namePattern: /^selected (included|explicitly skipped)$/ }); +stream.on('bench:complete', (result) => results.push(result)); +stream.on('end', common.mustCall(() => { + assert.deepStrictEqual(calls, ['included']); + assert.strictEqual(results.length, 4); + + const byName = new Map(results.map((result) => [result.name, result])); + assert.strictEqual(byName.get('included').error, undefined); + assert.strictEqual(byName.get('explicitly skipped').skip, true); + assert.strictEqual(byName.get('pattern filtered').skip, 'name pattern'); + assert.strictEqual(byName.get('only filtered').skip, 'only'); +})); +stream.resume(); diff --git a/test/parallel/test-bench-hook-errors.js b/test/parallel/test-bench-hook-errors.js new file mode 100644 index 000000000000..72de600b8abd --- /dev/null +++ b/test/parallel/test-bench-hook-errors.js @@ -0,0 +1,84 @@ +// Flags: --no-warnings +'use strict'; + +const common = require('../common'); +const assert = require('assert'); +const { + after, + afterEach, + before, + beforeEach, + bench, + run, + suite, +} = require('node:bench'); + +function complete(b) { + b.start(); + process.hrtime.bigint(); + b.end(1); +} + +suite('before failure', () => { + before(() => { throw new Error('before failure'); }); + after(common.mustCall()); + bench('blocked by before', { samples: 1 }, common.mustNotCall()); +}); + +suite('beforeEach failure', () => { + beforeEach(() => { throw new Error('beforeEach failure'); }); + afterEach(common.mustCall()); + bench('blocked by beforeEach', { samples: 1 }, common.mustNotCall()); +}); + +suite('after failure', () => { + after(() => { throw new Error('after failure'); }); + bench('completes before after', { samples: 1 }, complete); +}); + +suite('build failure', async () => { + await new Promise((resolve) => setImmediate(resolve)); + throw new Error('build failure'); +}); + +bench('continues after suite failures', { samples: 1 }, complete); + +const completions = []; +const diagnostics = []; +let summary; +const stream = run(); +stream.on('bench:complete', (result) => completions.push(result)); +stream.on('bench:diagnostic', (diagnostic) => { + diagnostics.push(diagnostic); +}); +stream.on('bench:summary', (value) => { summary = value; }); +stream.on('end', common.mustCall(() => { + assert.deepStrictEqual(summary.counts, { + __proto__: null, + completed: 2, + failed: 2, + skipped: 0, + total: 4, + }); + assert.strictEqual(summary.success, false); + + const byName = new Map(completions.map((result) => [result.name, result])); + assert.strictEqual(byName.get('blocked by before').error.message, + 'before failure'); + assert.strictEqual(byName.get('blocked by beforeEach').error.message, + 'beforeEach failure'); + assert.strictEqual(byName.get('completes before after').error, undefined); + assert.strictEqual( + byName.get('continues after suite failures').error, undefined); + + assert.deepStrictEqual( + diagnostics.map(({ message }) => message).sort(), + ['after failure', 'before failure', 'build failure'], + ); + for (const diagnostic of diagnostics) { + assert.strictEqual(typeof diagnostic.file, 'string'); + assert.strictEqual(typeof diagnostic.line, 'number'); + assert.strictEqual(typeof diagnostic.column, 'number'); + } +})); +stream.resume(); diff --git a/test/parallel/test-bench-module.mjs b/test/parallel/test-bench-module.mjs new file mode 100644 index 000000000000..ec08413ed6b3 --- /dev/null +++ b/test/parallel/test-bench-module.mjs @@ -0,0 +1,41 @@ +// Flags: --no-warnings + +import '../common/index.mjs'; +import assert from 'node:assert'; +import { createRequire, builtinModules, isBuiltin } from 'node:module'; +import benchDefault, { + after, + afterEach, + before, + beforeEach, + bench, + describe, + run, + suite, +} from 'node:bench'; + +assert.strictEqual(benchDefault, bench); +assert.strictEqual(describe, suite); +for (const value of [ + after, + afterEach, + before, + beforeEach, + bench, + run, + suite, +]) { + assert.strictEqual(typeof value, 'function'); +} +assert.strictEqual(typeof bench.skip, 'function'); +assert.strictEqual(typeof bench.only, 'function'); + +assert.strictEqual(isBuiltin('node:bench'), true); +assert.strictEqual(isBuiltin('bench'), false); +assert.strictEqual(builtinModules.includes('node:bench'), true); +assert.strictEqual(process.getBuiltinModule('node:bench'), benchDefault); +assert.strictEqual(process.getBuiltinModule('bench'), undefined); + +const require = createRequire(import.meta.url); +assert.throws(() => require('bench'), { code: 'MODULE_NOT_FOUND' }); +await assert.rejects(import('bench'), { code: 'ERR_MODULE_NOT_FOUND' }); diff --git a/test/parallel/test-bench-run.js b/test/parallel/test-bench-run.js new file mode 100644 index 000000000000..fafc9e3309ad --- /dev/null +++ b/test/parallel/test-bench-run.js @@ -0,0 +1,139 @@ +// Flags: --no-warnings +'use strict'; + +const common = require('../common'); +const assert = require('assert'); +const { + after, + afterEach, + before, + beforeEach, + bench, + run, + suite, +} = require('node:bench'); + +const calls = []; +const contexts = new Set(); +let active = false; + +before(() => calls.push('root before')); +after(() => calls.push('root after')); +beforeEach(() => calls.push('root beforeEach')); +afterEach(() => calls.push('root afterEach')); + +const suiteCompletion = suite('group', { tags: ['Group'] }, async () => { + await new Promise((resolve) => setImmediate(resolve)); + + before(() => calls.push('suite before')); + after(() => calls.push('suite after')); + beforeEach(() => calls.push('suite beforeEach')); + afterEach(() => calls.push('suite afterEach')); + + bench('sync', { + params: { z: 2, a: true }, + samples: 2, + tags: ['SYNC'], + warmup: 1, + }, common.mustCall((b) => { + assert.strictEqual(active, false); + active = true; + contexts.add(b); + calls.push('sync sample'); + assert.deepStrictEqual(b.params, { __proto__: null, a: true, z: 2 }); + b.start(); + process.hrtime.bigint(); + b.end(1); + active = false; + }, 3)); + + bench('async', { samples: 2 }, common.mustCall(async (b) => { + assert.strictEqual(active, false); + active = true; + contexts.add(b); + calls.push('async sample'); + await new Promise((resolve) => setImmediate(resolve)); + b.start(); + process.hrtime.bigint(); + b.end(1); + await new Promise((resolve) => setImmediate(resolve)); + active = false; + }, 2)); + + bench.skip('skipped', { samples: 1 }, common.mustNotCall()); +}); + +const records = []; +const stream = run(); +stream.on('data', (record) => records.push(record)); +stream.on('end', common.mustCall(() => { + assert.strictEqual(active, false); + assert.strictEqual(contexts.size, 5); + + const starts = records.filter(({ type }) => type === 'bench:start'); + const samples = records.filter(({ type }) => type === 'bench:sample'); + const completions = records.filter(({ type }) => type === 'bench:complete'); + const summaries = records.filter(({ type }) => type === 'bench:summary'); + + assert.strictEqual(starts.length, 2); + assert.strictEqual(samples.length, 4); + assert.strictEqual(completions.length, 3); + assert.strictEqual(summaries.length, 1); + + const sync = completions.find(({ data }) => data.name === 'sync').data; + assert.strictEqual(sync.error, undefined); + assert.strictEqual(sync.skip, undefined); + assert.strictEqual(sync.samples.length, 2); + assert.strictEqual(Object.getPrototypeOf(sync), null); + assert.strictEqual(Object.getPrototypeOf(sync.params), null); + assert.deepStrictEqual(sync.tags, ['group', 'sync']); + assert.match(sync.benchId, /\{"a":true,"z":2\}/); + assert.notStrictEqual(sync.parentId, null); + assert.strictEqual(sync.summary.mean > 0, true); + assert.strictEqual(sync.summary.min <= sync.summary.mean, true); + assert.strictEqual(sync.summary.mean <= sync.summary.max, true); + assert.strictEqual(typeof sync.summary.confidenceInterval.lower, 'number'); + assert.strictEqual(typeof sync.samples[0].duration_ns, 'bigint'); + + const asyncResult = completions.find( + ({ data }) => data.name === 'async').data; + assert.strictEqual(asyncResult.error, undefined); + assert.strictEqual(asyncResult.skip, undefined); + assert.strictEqual(asyncResult.samples.length, 2); + + const skipped = completions.find( + ({ data }) => data.name === 'skipped').data; + assert.strictEqual(skipped.skip, true); + assert.deepStrictEqual(skipped.samples, []); + + assert.deepStrictEqual(summaries[0].data.counts, { + __proto__: null, + completed: 2, + failed: 0, + skipped: 1, + total: 3, + }); + assert.strictEqual(summaries[0].data.success, true); + + assert.deepStrictEqual(calls, [ + 'root before', + 'suite before', + 'root beforeEach', + 'suite beforeEach', + 'sync sample', + 'sync sample', + 'sync sample', + 'suite afterEach', + 'root afterEach', + 'root beforeEach', + 'suite beforeEach', + 'async sample', + 'async sample', + 'suite afterEach', + 'root afterEach', + 'suite after', + 'root after', + ]); +})); + +suiteCompletion.then(common.mustCall()); diff --git a/test/parallel/test-bench-validation.js b/test/parallel/test-bench-validation.js new file mode 100644 index 000000000000..25a53c1b0186 --- /dev/null +++ b/test/parallel/test-bench-validation.js @@ -0,0 +1,45 @@ +// Flags: --no-warnings +'use strict'; + +const common = require('../common'); +const assert = require('assert'); +const { bench, run } = require('node:bench'); + +const noop = () => {}; + +assert.throws(() => bench('', noop), { code: 'ERR_INVALID_ARG_VALUE' }); +assert.throws(() => bench('name', null), { code: 'ERR_INVALID_ARG_TYPE' }); +assert.throws(() => bench('name', { samples: 0 }, noop), + { code: 'ERR_OUT_OF_RANGE' }); +assert.throws(() => bench('name', { warmup: -1 }, noop), + { code: 'ERR_OUT_OF_RANGE' }); +assert.throws(() => bench('name', { timeout: -1 }, noop), + { code: 'ERR_OUT_OF_RANGE' }); +assert.throws(() => bench('name', { signal: {} }, noop), + { code: 'ERR_INVALID_ARG_TYPE' }); +assert.throws(() => bench('name', { tags: 'fast' }, noop), + { code: 'ERR_INVALID_ARG_TYPE' }); +assert.throws(() => bench('name', { tags: [''] }, noop), + { code: 'ERR_INVALID_ARG_VALUE' }); +assert.throws(() => bench('name', { params: { value: null } }, noop), + { code: 'ERR_INVALID_ARG_TYPE' }); +assert.throws(() => bench('name', { params: { value: NaN } }, noop), + { code: 'ERR_OUT_OF_RANGE' }); +assert.throws(() => bench('name', { only: 'yes' }, noop), + { code: 'ERR_INVALID_ARG_TYPE' }); +assert.throws(() => bench('name', { skip: 1 }, noop), + { code: 'ERR_INVALID_ARG_TYPE' }); +assert.throws(() => run({ namePattern: 1 }), + { code: 'ERR_INVALID_ARG_TYPE' }); + +bench('valid', { samples: 1 }, (b) => { + b.start(); + process.hrtime.bigint(); + b.end(1); +}); + +const stream = run(); +stream.on('bench:start', common.mustCall(() => { + assert.throws(() => bench('late', noop), { code: 'ERR_INVALID_STATE' }); +})); +stream.resume(); diff --git a/test/parallel/test-module-isBuiltin.js b/test/parallel/test-module-isBuiltin.js index a7815a8dfc1c..54f25e599858 100644 --- a/test/parallel/test-module-isBuiltin.js +++ b/test/parallel/test-module-isBuiltin.js @@ -7,10 +7,12 @@ const { isBuiltin } = require('module'); assert(isBuiltin('http')); assert(isBuiltin('sys')); assert(isBuiltin('node:fs')); +assert(isBuiltin('node:bench')); assert(isBuiltin('node:test')); // Does not include internal modules assert(!isBuiltin('internal/errors')); +assert(!isBuiltin('bench')); assert(!isBuiltin('test')); assert(!isBuiltin('')); assert(!isBuiltin(undefined)); From 09151a3f349c7d16ad9232af0b070b67c1594d9f Mon Sep 17 00:00:00 2001 From: James M Snell Date: Thu, 27 Aug 2026 22:29:00 +0000 Subject: [PATCH 3/6] lib: implement bench/reporters Signed-off-by: James M Snell Assisted-by: Opencode --- doc/api/bench.md | 60 +++++++ lib/bench/reporters.js | 34 ++++ lib/internal/bench_runner/reporter/json.js | 64 ++++++++ lib/internal/bench_runner/reporter/spec.js | 149 ++++++++++++++++++ lib/internal/bootstrap/realm.js | 1 + .../test-module-hooks-builtin-require.js | 1 + .../test-module-hooks-load-builtin-require.js | 1 + test/parallel/test-bench-custom-reporter.js | 37 +++++ test/parallel/test-bench-module.mjs | 13 ++ test/parallel/test-bench-reporters.js | 114 ++++++++++++++ test/parallel/test-module-isBuiltin.js | 2 + 11 files changed, 476 insertions(+) create mode 100644 lib/bench/reporters.js create mode 100644 lib/internal/bench_runner/reporter/json.js create mode 100644 lib/internal/bench_runner/reporter/spec.js create mode 100644 test/parallel/test-bench-custom-reporter.js create mode 100644 test/parallel/test-bench-reporters.js diff --git a/doc/api/bench.md b/doc/api/bench.md index 93b303fb2e2d..b4488e119cd1 100644 --- a/doc/api/bench.md +++ b/doc/api/bench.md @@ -65,6 +65,66 @@ system load can all affect results. Keep raw samples when comparing results and investigate noisy or skewed distributions rather than treating a confidence interval as a pass/fail threshold. +## Benchmark reporters + +The built-in reporters are available from the scheme-only +`node:bench/reporters` module: + +```mjs +import { json, spec } from 'node:bench/reporters'; +``` + +```cjs +const { json, spec } = require('node:bench/reporters'); +``` + +Reporter values can be passed directly to `stream.compose()`: + +```mjs +import { bench, run } from 'node:bench'; +import { spec } from 'node:bench/reporters'; +import process from 'node:process'; + +bench('example', (b) => { + b.start(); + doWork(); + b.end(1); +}); + +run().compose(spec).pipe(process.stdout); +``` + +The `spec` reporter buffers results and outputs a concise table containing the +sample count, mean rate, 95% confidence interval for the mean, median rate, and +warnings. A coefficient of variation above 5% is reported as `noisy`, and an +absolute skewness above 1 is reported as `skewed`. The exact human-readable +format is subject to change. + +The `json` reporter emits every lifecycle record as newline-delimited JSON. +BigInt values, including `duration_ns`, are encoded as decimal strings. Errors +are represented using their `name`, `message`, `stack`, `code`, `cause`, and +`errors` properties. As required by JSON, non-finite numbers are encoded as +`null`. + +Custom reporters use the same composition contract. They can be transforms or +functions accepted by `stream.compose()`. The composed readable can be piped to +any writable destination: + +```mjs +import { run } from 'node:bench'; +import process from 'node:process'; + +async function* names(source) { + for await (const { type, data } of source) { + if (type === 'bench:complete') { + yield `${data.name}\n`; + } + } +} + +run().compose(names).pipe(process.stdout); +``` + ## `bench([name][, options], fn)` + +> Stability: 1 - Experimental + +Starts the Node.js command-line benchmark runner. At least one explicit file or +glob pattern is required: + +```console +node --bench benchmark.mjs +node --bench 'benchmarks/**/*.js' +``` + +Quote glob patterns to prevent expansion by the shell. Matching files are +sorted and executed serially. By default, each file runs in a separate child +process. Benchmark files declare benchmarks using `node:bench`; they must not +call `run()` themselves. See the [benchmark runner][] documentation for more +details. + +This flag cannot be combined with `--test`, `--watch`, `--watch-path`, +`--check`, `--eval`, or `--interactive`. + +### `--bench-isolation=mode` + + + +> Stability: 1 - Experimental + +Configures benchmark file isolation. When `mode` is `'process'`, each matching +file runs in a separate child process. This is the default. Files are still run +serially so their measured work does not overlap. + +When `mode` is `'none'`, all matching files and benchmarks run serially in the +benchmark runner process. This reduces startup overhead but allows module, +heap, and process state to carry between files. User writes to stdout or stderr +also share destinations with benchmark reporters in this mode. + +### `--bench-name-pattern=pattern` + + + +> Stability: 1 - Experimental + +Only runs benchmarks whose full hierarchical name matches the JavaScript +regular expression `pattern`. Non-matching benchmarks are reported as skipped. + +### `--bench-reporter-destination=destination` + + + +> Stability: 1 - Experimental + +Specifies the destination for the corresponding benchmark reporter. The value +can be `stdout`, `stderr`, or a file path. A single reporter defaults to +`stdout` when no destination is specified. + +### `--bench-reporter=reporter` + + + +> Stability: 1 - Experimental + +Specifies a benchmark reporter. The built-in reporters are `spec` and `json`. +The `json` reporter emits newline-delimited JSON. A custom reporter can be +specified using a module specifier resolved from the current working directory. + +This option can be repeated. When multiple reporters are specified, each must +have a corresponding `--bench-reporter-destination`. The default reporter is +`spec`. + +### `--bench-samples=count` + + + +> Stability: 1 - Experimental + +Overrides the number of measured callback invocations for every selected +benchmark. `count` must be an integer between `1` and `4294967295`. + +### `--bench-warmup=count` + + + +> Stability: 1 - Experimental + +Overrides the number of unreported warmup callback invocations for every +selected benchmark. `count` must be an integer between `0` and `4294967295`. + ### `--build-sea=config` -> Stability: 1 - Experimental +> Stability: 1.0 - Early Development diff --git a/doc/contributing/writing-and-running-benchmarks.md b/doc/contributing/writing-and-running-benchmarks.md index a31e0e82aabc..beac2624b62b 100644 --- a/doc/contributing/writing-and-running-benchmarks.md +++ b/doc/contributing/writing-and-running-benchmarks.md @@ -17,6 +17,7 @@ * [Using `--analyze` (no external tools needed)](#using---analyze-no-external-tools-needed) * [Using R scripts or node-benchmark-compare](#using-r-scripts-or-node-benchmark-compare) * [Comparing parameters](#comparing-parameters) + * [Evaluating `node:bench` ports](#evaluating-nodebench-ports) * [Running benchmarks on the CI](#running-benchmarks-on-the-ci) * [Creating a benchmark](#creating-a-benchmark) * [Basics of a benchmark](#basics-of-a-benchmark) @@ -588,6 +589,41 @@ chunkLen encoding rate confidence.interval ![compare tool boxplot](doc_img/scatter-plot.png) +### Evaluating `node:bench` ports + +The experimental `compare-node-bench.js` and `scatter-node-bench.js` tools are +parallel versions of the existing tools for explicit `node:bench` files. They +do not modify or replace the legacy benchmark framework. Each repeated +observation for a benchmark identity uses one measured sample from a separate +process invocation. Configurations declared in the same file still execute +serially in that process, unlike the legacy framework's configuration-level +process isolation, and can share runtime state. + +Both parallel tools support inline analysis. `scatter-node-bench.js --analyze` +uses the same `--xaxis`, `--category`, and `--no-chart` interface described for +`scatter.js`. `compare-node-bench.js --analyze` performs Welch's t-test, while +`--max-regression N` adds a corrected regression gate. The gate requires both a +Holm-Bonferroni-adjusted p-value below 0.05 and a 95% confidence interval lying +entirely beyond `-N%`; the point estimate alone cannot fail the command. +Scatter analysis reduces aggregated configurations to one value per outer +process and uses disjoint process sets for consecutive Mann-Whitney comparisons +so configurations sharing a process are not treated as independent samples. + +Underscore-prefixed ports are kept beside selected legacy benchmarks and are +excluded from legacy discovery. For example: + +```console +./node benchmark/scatter.js --runs 30 \ + benchmark/crypto/create-hash.js > legacy.csv +./node benchmark/scatter-node-bench.js --runs 30 -- \ + benchmark/crypto/_create-hash.node-bench.js > node-bench.csv +``` + +The port uses the legacy relative filename as its benchmark name and preserves +the same parameter names. The two CSV files can therefore be analyzed with the +same scripts to check whether their rate distributions and measurement units +agree. See [`benchmark/README.md`][] for compare and scatter examples. + ### Running benchmarks on the CI To see the performance impact of a pull request by running benchmarks on @@ -769,6 +805,7 @@ Supported options keys are: * `benchmarker` - benchmarker to use, defaults to the first available http benchmarker +[`benchmark/README.md`]: ../../benchmark/README.md#nodebench-evaluation-tools [autocannon]: https://github.com/mcollina/autocannon [benchmark-ci]: https://github.com/nodejs/benchmarking/blob/HEAD/docs/core_benchmarks.md [git-for-windows]: https://git-scm.com/download/win diff --git a/test/fixtures/bench-runner/tools-collision.cjs b/test/fixtures/bench-runner/tools-collision.cjs new file mode 100644 index 000000000000..5fb27dd956d7 --- /dev/null +++ b/test/fixtures/bench-runner/tools-collision.cjs @@ -0,0 +1,14 @@ +'use strict'; + +const { bench, suite } = require('node:bench'); + +function register(name) { + bench(name, { params: { size: 1 } }, (b) => { + b.start(); + process.hrtime.bigint(); + b.end(1); + }); +} + +suite('first', () => register('same')); +suite('second', () => register('same')); diff --git a/test/fixtures/bench-runner/tools-no-params.cjs b/test/fixtures/bench-runner/tools-no-params.cjs new file mode 100644 index 000000000000..2dde0c2a0c69 --- /dev/null +++ b/test/fixtures/bench-runner/tools-no-params.cjs @@ -0,0 +1,9 @@ +'use strict'; + +const { bench } = require('node:bench'); + +bench('tools/no-params.js', (b) => { + b.start(); + process.hrtime.bigint(); + b.end(1); +}); diff --git a/test/fixtures/bench-runner/tools-reserved-param.cjs b/test/fixtures/bench-runner/tools-reserved-param.cjs new file mode 100644 index 000000000000..00052b0c8e4b --- /dev/null +++ b/test/fixtures/bench-runner/tools-reserved-param.cjs @@ -0,0 +1,9 @@ +'use strict'; + +const { bench } = require('node:bench'); + +bench('tools/reserved.js', { params: { rate: 'parameter' } }, (b) => { + b.start(); + process.hrtime.bigint(); + b.end(1); +}); diff --git a/test/fixtures/bench-runner/tools.cjs b/test/fixtures/bench-runner/tools.cjs new file mode 100644 index 000000000000..108a6f81dd63 --- /dev/null +++ b/test/fixtures/bench-runner/tools.cjs @@ -0,0 +1,20 @@ +'use strict'; + +const { bench } = require('node:bench'); + +if (process.env.NODE_BENCH_PID_LOG !== undefined) { + require('fs').appendFileSync( + process.env.NODE_BENCH_PID_LOG, `${process.pid}\n`); +} + +for (const size of [1, 2]) { + bench('tools/simple.js', { + params: { method: 'loop', size }, + }, (b) => { + let value = 0; + b.start(); + for (let i = 0; i < 1_000; i++) value += size; + b.end(1_000); + if (value === 0) throw new Error('unreachable'); + }); +} diff --git a/test/parallel/test-benchmark-node-bench-tools.js b/test/parallel/test-benchmark-node-bench-tools.js new file mode 100644 index 000000000000..84b5f98e9eec --- /dev/null +++ b/test/parallel/test-benchmark-node-bench-tools.js @@ -0,0 +1,261 @@ +// Flags: --no-warnings +'use strict'; + +require('../common'); +const assert = require('assert'); +const { spawnSync } = require('child_process'); +const fs = require('fs'); +const path = require('path'); +const fixtures = require('../common/fixtures'); +const tmpdir = require('../common/tmpdir'); +const { + analyzeScatter, + holmAdjust, + isRegressionFailure, +} = require('../../benchmark/_node-bench-analysis.js'); +const { csvEncode } = require('../../benchmark/_node-bench.js'); + +const compare = path.resolve(__dirname, '../../benchmark/compare-node-bench.js'); +const legacyScatter = path.resolve(__dirname, '../../benchmark/scatter.js'); +const scatter = path.resolve(__dirname, '../../benchmark/scatter-node-bench.js'); +const benchmark = fixtures.path('bench-runner/tools.cjs'); + +tmpdir.refresh(); + +assert.strictEqual(csvEncode(true), 'true'); +assert.deepStrictEqual(holmAdjust([0.01, 0.03, 0.04]), [0.03, 0.06, 0.06]); +assert.strictEqual(isRegressionFailure({ + ci95: 3, + improvement: -12, + pAdjusted: 0.01, +}, 10), false); +assert.strictEqual(isRegressionFailure({ + ci95: 1, + improvement: -12, + pAdjusted: 0.06, +}, 10), false); +assert.strictEqual(isRegressionFailure({ + ci95: 1, + improvement: -12, + pAdjusted: 0.01, +}, 10), true); +assert.throws( + () => analyzeScatter([{ + observation: 0, + params: { size: 1 }, + rate: 1, + }], 'size', 'size', false), + /must name different parameters/, +); +assert.doesNotMatch(analyzeScatter([0, 1].map((observation) => ({ + observation, + params: { size: 1 }, + rate: 1_234_567.89, +})), 'size', undefined, false), /\(!\)/); +assert.match(analyzeScatter([ + { observation: 0, params: { method: 'a', size: 1 }, rate: 10 }, + { observation: 0, params: { method: 'b', size: 1 }, rate: 20 }, + { observation: 1, params: { method: 'a', size: 1 }, rate: 30 }, + { observation: 1, params: { method: 'b', size: 1 }, rate: 50 }, +], 'size', undefined, false), /\n\s*1\s+2\s+/); + +function run(script, args, options = undefined) { + return spawnSync(process.execPath, [script, ...args], { + encoding: 'utf8', + timeout: 30_000, + ...options, + }); +} + +{ + const pidLog = tmpdir.resolve('pids'); + const result = run(compare, [ + '--old', process.execPath, + '--new', process.execPath, + '--runs', '2', + '--', benchmark, + ], { + env: { __proto__: null, ...process.env, NODE_BENCH_PID_LOG: pidLog }, + }); + assert.strictEqual(result.status, 0, result.stderr); + assert.strictEqual(result.stderr, ''); + const lines = result.stdout.trim().split('\n'); + assert.strictEqual(lines[0], + '"binary","filename","configuration","rate","time"'); + assert.strictEqual(lines.length, 9); + assert.strictEqual(lines.filter((line) => line.startsWith('"old",')).length, + 4); + assert.strictEqual(lines.filter((line) => line.startsWith('"new",')).length, + 4); + assert.deepStrictEqual(lines.slice(1).map((line) => line.slice(0, 5)), [ + '"old"', '"old"', '"new"', '"new"', + '"new"', '"new"', '"old"', '"old"', + ]); + assert(lines.slice(1).every( + (line) => line.includes('"tools/simple.js"'))); + const pids = fs.readFileSync(pidLog, 'utf8').trim().split('\n'); + assert.strictEqual(new Set(pids).size, 4); +} + +{ + const result = run(scatter, [ + '--node', process.execPath, + '--runs', '2', + '--', benchmark, + ]); + assert.strictEqual(result.status, 0, result.stderr); + assert.strictEqual(result.stderr, ''); + const lines = result.stdout.trim().split('\n'); + assert.strictEqual(lines[0], + '"filename","method","size","rate","time"'); + assert.strictEqual(lines.length, 5); + assert(lines.slice(1).every( + (line) => line.startsWith('"tools/simple.js","loop",'))); +} + +{ + const result = run(scatter, [ + '--runs', '1', + '--', fixtures.path('bench-runner/tools-no-params.cjs'), + ]); + assert.strictEqual(result.status, 0, result.stderr); + const lines = result.stdout.trim().split('\n'); + assert.strictEqual(lines[0], '"filename","rate","time"'); + assert.strictEqual(lines.length, 2); + assert.strictEqual(lines[1].split(',').length, 3); +} + +{ + const result = run(scatter, [ + '--runs', '1', + '--', fixtures.path('bench-runner/tools-reserved-param.cjs'), + ]); + assert.strictEqual(result.status, 1); + assert.match(result.stderr, /parameter 'rate' is reserved/); +} + +{ + const result = run(scatter, [ + '--runs', '1', + '--', fixtures.path('bench-runner/tools-collision.cjs'), + ]); + assert.strictEqual(result.status, 1); + assert.match(result.stderr, /requires one logical benchmark name per file/); +} + +{ + const result = run(compare, [ + '--old', process.execPath, + '--new', process.execPath, + '--runs', '1', + '--', fixtures.path('bench-runner/tools-collision.cjs'), + ]); + assert.strictEqual(result.status, 1); + assert.match(result.stderr, /Distinct benchmarks would share the CSV group/); +} + +{ + const result = run(scatter, [ + '--runs', '2', + '--', fixtures.path('bench-runner/a.cjs'), + ]); + assert.strictEqual(result.status, 1); + assert.match(result.stderr, /set of reported benchmarks changed between runs/); +} + +{ + const result = run(scatter, [ + '--runs', '1', + '--name-pattern', 'missing', + '--', benchmark, + ]); + assert.strictEqual(result.status, 1); + assert.match(result.stderr, /No benchmark samples were produced/); +} + +{ + const result = run(scatter, [ + '--runs', 'invalid', + '--', benchmark, + ]); + assert.strictEqual(result.status, 1); + assert.match(result.stderr, /--runs must be an integer/); +} + +{ + const result = run(scatter, [ + '--runs', '2', + '--analyze', + '--xaxis', 'size', + '--category', 'method', + '--no-chart', + '--', benchmark, + ]); + assert.strictEqual(result.status, 0, result.stderr); + assert.strictEqual(result.stderr, ''); + assert.match(result.stdout, + /size\s+method\s+samples\s+rate\s+confidence\.interval/); + assert.match(result.stdout, /Change between consecutive size values/); + assert.match(result.stdout, /Mann-Whitney U.*Cliff's delta/); + assert.doesNotMatch(result.stdout, /"filename","method"/); +} + +{ + const result = run(scatter, [ + '--analyze', + '--', benchmark, + ]); + assert.strictEqual(result.status, 1); + assert.match(result.stderr, /--analyze requires --xaxis/); +} + +{ + const result = run(compare, [ + '--old', process.execPath, + '--new', process.execPath, + '--runs', '2', + '--max-regression', '100', + '--', fixtures.path('bench-runner/tools-no-params.cjs'), + ]); + assert.strictEqual(result.status, 0, result.stderr); + assert.strictEqual(result.stderr, ''); + assert.match(result.stdout, /confidence\s+improvement\s+accuracy/); + assert.match(result.stdout, /Holm-Bonferroni correction/); + assert.match(result.stdout, /--max-regression uses the corrected values/); + assert.doesNotMatch(result.stdout, /"binary","filename"/); +} + +{ + const legacy = run(legacyScatter, [ + '--runs', '1', + path.resolve(__dirname, '../../benchmark/crypto/create-hash.js'), + ]); + const modern = run(scatter, [ + '--runs', '1', + '--', path.resolve( + __dirname, '../../benchmark/crypto/_create-hash.node-bench.js'), + ]); + assert.strictEqual(legacy.status, 0, legacy.stderr); + assert.strictEqual(modern.status, 0, modern.stderr); + const legacyLines = legacy.stdout.trim().split('\n'); + const modernLines = modern.stdout.trim().split('\n'); + assert.strictEqual(legacyLines[0].replaceAll(' ', ''), modernLines[0]); + const name = path.join('crypto', 'create-hash.js'); + assert(legacyLines[1].startsWith(`"${name}",`)); + assert(modernLines[1].startsWith(`"${name}",`)); +} + +{ + const result = run(scatter, [ + '--runs', '1', + '--', path.resolve( + __dirname, + '../../benchmark/buffers/_buffer-compare-offset.node-bench.js', + ), + ]); + assert.strictEqual(result.status, 0, result.stderr); + const lines = result.stdout.trim().split('\n'); + assert.strictEqual(lines[0], + '"filename","method","n","size","rate","time"'); + assert.strictEqual(lines.length, 9); +} From 9176490f5ef4d0695dc5896335c1f5f576559582 Mon Sep 17 00:00:00 2001 From: James M Snell Date: Fri, 28 Aug 2026 01:29:13 +0000 Subject: [PATCH 6/6] src: fixup histogram and options linting issues Signed-off-by: James M Snell --- src/histogram.cc | 4 ++-- src/node_options.h | 13 ++++++------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/histogram.cc b/src/histogram.cc index 1008c63c04d4..f2b93d3b8be4 100644 --- a/src/histogram.cc +++ b/src/histogram.cc @@ -501,8 +501,8 @@ Histogram::MeanCIResult Histogram::MeanCI(double confidence) const { static_cast(count - 1); double standard_error = std::sqrt(variance / static_cast(count)); double alpha = 1.0 - confidence; - double t_crit = StudentTUpperQuantile( - alpha / 2.0, static_cast(count - 1)); + double t_crit = + StudentTUpperQuantile(alpha / 2.0, static_cast(count - 1)); double margin = t_crit * standard_error; return {mean, mean - margin, mean + margin}; } diff --git a/src/node_options.h b/src/node_options.h index d3969ed18df0..a6be7253eb7b 100644 --- a/src/node_options.h +++ b/src/node_options.h @@ -530,13 +530,12 @@ class OptionsParser { OptionEnvvarSettings env_setting = kDisallowedInEnvvar, bool default_is_true = false, OptionNamespaces namespace_id = OptionNamespaces::kNoNamespace); - void AddOption( - const char* name, - const char* help_text, - uint64_t Options::*field, - OptionEnvvarSettings env_setting = kDisallowedInEnvvar, - OptionNamespaces namespace_id = OptionNamespaces::kNoNamespace, - bool strict = false); + void AddOption(const char* name, + const char* help_text, + uint64_t Options::*field, + OptionEnvvarSettings env_setting = kDisallowedInEnvvar, + OptionNamespaces namespace_id = OptionNamespaces::kNoNamespace, + bool strict = false); void AddOption( const char* name, const char* help_text,