From cd2c2457014db2613618a20558ffab80b9e4cd23 Mon Sep 17 00:00:00 2001 From: Alan Agius <17563226+alan-agius4@users.noreply.github.com> Date: Mon, 24 Aug 2026 15:20:17 +0000 Subject: [PATCH] perf(@angular/build): consolidate build worker pools with shared router Unify isolated per-subsystem worker pools (JavaScriptTransformer, SassService, I18nInliner) into a single singleton WorkerPool routed via dynamic task dispatching (shared-worker-router.ts). - Reduce active worker threads by 66.7% (24 -> 8 threads), eliminating thread thrashing and reducing kernel system CPU time by up to 42.6%. - Pre-warm worker pool threads (minThreads: maxWorkers) with Piscina FixedQueue and 30s idle timeout, eliminating cold on-demand worker initialization (~740 ms) on the critical build path. - Encode JavaScript transformer task options into compact bitmask flags (JavaScriptTransformFlags), reducing IPC task envelope sizes by 43.3% and eliminating object allocations. - Add backpressure throttling to SassWorkerImplementation to prevent synchronous Dart Sass importer futex waits from saturating worker slots and starving JavaScript transforms. - Implement zero-copy transferable file and translation Blobs/SharedArrayBuffers, memoize translation serialization, and eliminate straggler batches in multi-locale inlining. - Add bounded LRU caching (fileDataCache, deserializedTranslations) with fileKey and translationKey in the i18n inliner worker isolate, achieving up to 3.93x faster i18n inlining without memory leaks. - Incorporate multi-agent code review findings: monotonic sequence IDs for futex synchronizations, structured IPC error propagation, post-close lifecycle guards, sourcemap passthrough on untransformed files, full SharedArrayBuffer cache key hashing, inline stylesheet watch tracking, and bundler invalidation epoch guards. | Metric | Baseline (`main`) | Consolidated (`perf-build-singleton-shared-worker-pool`) | Delta / Speedup | | :--- | :--- | :--- | :--- | | Active Worker Threads | 24 threads | 8 threads | -66.7% threads (-16 threads) | | Cold Build Duration (mean) | 1,244.2 ms | 1,243.4 ms | -0.1% (-0.8 ms) | | Cold Build Duration (min / max) | 1,217.1 ms / 1,301.6 ms | 1,221.4 ms / 1,265.2 ms | +4.3 ms / -36.4 ms | | P95 Build Latency | 1,301.6 ms | 1,265.2 ms | -2.8% (-36.4 ms faster) | | Throughput | 2,814.7 ops/sec | 2,815.2 ops/sec | +0.0% (+0.5 ops/sec) | | I18n Inlining Duration (Pure mean) | 728.7 ms | 185.6 ms | 3.93x faster (-74.5%) | | Process RSS Delta | +2,808.0 MB | +1,465.5 MB | -47.8% (-1,342.5 MB saved) | | Final Process RSS | 2,866.6 MB | 1,528.6 MB | -46.7% (-1,338.0 MB saved) | | Kernel System CPU | 2,376.3 ms | 1,362.9 ms | -42.6% (1.74x less kernel CPU) | | Total CPU (User + Kernel) | 16,749.9 ms | 9,977.1 ms | -40.4% (1.68x CPU efficiency) | | Metric | Baseline (`main`) | Consolidated (`perf-build-singleton-shared-worker-pool`) | Delta / Speedup | | :--- | :--- | :--- | :--- | | Active Worker Threads | 24 threads | 8 threads | -66.7% threads (-16 threads) | | E2E Build Duration (mean) | 6,513.4 ms | 5,902.4 ms | -9.4% (-611.0 ms faster) | | E2E Build Duration (min / max) | 6,479.6 ms / 6,565.9 ms | 5,850.8 ms / 5,961.0 ms | -628.8 ms / -604.9 ms | | P95 Build Latency | 6,565.9 ms | 5,961.0 ms | -9.2% (-604.9 ms faster) | | Process RSS Delta | +2,255.2 MB | +2,145.5 MB | -4.9% (-109.7 MB saved) | | Kernel System CPU | 2,726.3 ms | 2,139.3 ms | -21.5% (1.27x less kernel CPU) | | Total CPU (User + Kernel) | 25,658.2 ms | 19,497.5 ms | -24.0% (-6,160.7 ms CPU saved) | | Metric | Baseline (`main`) | Consolidated (`perf-build-singleton-shared-worker-pool`) | Delta / Speedup | | :--- | :--- | :--- | :--- | | Active Worker Threads | 24 threads | 8 threads | -66.7% threads (-16 threads) | | Cold Build Duration (mean) | 2,258.0 ms | 2,256.7 ms | -0.1% (-1.3 ms) | | Cold Build Duration (min / max) | 2,232.0 ms / 2,294.6 ms | 2,197.3 ms / 2,294.8 ms | -34.7 ms / +0.2 ms | | P95 Build Latency | 2,294.6 ms | 2,294.8 ms | +0.0% (+0.2 ms) | | Process RSS Delta | +1,285.7 MB | +1,616.0 MB | +25.7% (pool pre-warming) | | Kernel System CPU | 1,395.3 ms | 1,440.5 ms | +3.2% (+45.2 ms) | | Total CPU (User + Kernel) | 10,193.5 ms | 11,530.1 ms | +13.1% (+1,336.6 ms) | --- .../esbuild/angular/component-stylesheets.ts | 7 +- .../tools/esbuild/application-code-bundle.ts | 6 +- .../src/tools/esbuild/bundler-context.ts | 18 +- .../src/tools/esbuild/i18n-inliner-worker.ts | 307 +++++++++++++----- .../build/src/tools/esbuild/i18n-inliner.ts | 211 ++++++++---- .../tools/esbuild/i18n-translation-encoder.ts | 2 +- .../esbuild/i18n-translation-encoder_spec.ts | 29 ++ .../tools/esbuild/i18n-translation-reader.ts | 8 +- .../esbuild/javascript-transformer-worker.ts | 86 ++++- .../tools/esbuild/javascript-transformer.ts | 113 +++++-- .../esbuild/javascript-transformer_spec.ts | 32 ++ .../tools/sass/sass-worker-implementation.ts | 190 +++++++---- .../angular/build/src/tools/sass/worker.ts | 56 ++-- .../build/src/utils/shared-worker-router.ts | 77 +++++ .../angular/build/src/utils/worker-pool.ts | 63 +++- .../build/src/utils/worker-pool_spec.ts | 244 ++++++++++++++ 16 files changed, 1145 insertions(+), 304 deletions(-) create mode 100644 packages/angular/build/src/utils/shared-worker-router.ts create mode 100644 packages/angular/build/src/utils/worker-pool_spec.ts diff --git a/packages/angular/build/src/tools/esbuild/angular/component-stylesheets.ts b/packages/angular/build/src/tools/esbuild/angular/component-stylesheets.ts index dd1b3b704150..75ce354341ad 100644 --- a/packages/angular/build/src/tools/esbuild/angular/component-stylesheets.ts +++ b/packages/angular/build/src/tools/esbuild/angular/component-stylesheets.ts @@ -219,7 +219,7 @@ export class ComponentStylesheetBundler { const filename = secondSemi !== -1 ? entry.slice(secondSemi + 1) : ''; if (filename && normalizedFiles.has(path.normalize(filename))) { this.#inlineContexts.delete(entry); - void bundler.dispose(); + void bundler.dispose().catch(() => {}); } else { bundler.invalidate(normalizedFiles); } @@ -229,10 +229,13 @@ export class ComponentStylesheetBundler { } collectReferencedFiles(): string[] { - const files = []; + const files: string[] = []; for (const context of this.#fileContexts.values()) { files.push(...context.watchFiles); } + for (const context of this.#inlineContexts.values()) { + files.push(...context.watchFiles); + } return files; } diff --git a/packages/angular/build/src/tools/esbuild/application-code-bundle.ts b/packages/angular/build/src/tools/esbuild/application-code-bundle.ts index afb03d436d29..af53e04f58af 100644 --- a/packages/angular/build/src/tools/esbuild/application-code-bundle.ts +++ b/packages/angular/build/src/tools/esbuild/application-code-bundle.ts @@ -141,8 +141,8 @@ export function createBrowserPolyfillBundleOptions( buildOptions.plugins ??= []; const pluginOptions = createCompilerPluginOptions( options, - sourceFileCache, + sourceFileCache.loadResultCache, ); buildOptions.plugins.push( createCompilerPlugin( @@ -501,7 +501,7 @@ export function createSsrEntryCodeBundleOptions( // The below is needed to avoid // `Import "default" will always be undefined because there is no matching export` warning when no default is present. `const defaultExportName = 'default';`, - `export default server[defaultExportName]`, + `export default server[defaultExportName];`, // Add @angular/ssr exports `export { AngularAppEngine } from '@angular/ssr';`, @@ -764,7 +764,7 @@ function getEsBuildCommonPolyfillsOptions( } function entryFileToWorkspaceRelative(workspaceRoot: string, entryFile: string): string { - return './' + toPosixPath(relative(workspaceRoot, entryFile).replace(/.[mc]?ts$/, '')); + return './' + toPosixPath(relative(workspaceRoot, entryFile).replace(/\.[mc]?ts$/, '')); } /** diff --git a/packages/angular/build/src/tools/esbuild/bundler-context.ts b/packages/angular/build/src/tools/esbuild/bundler-context.ts index b335a1160ba6..4128583a5ac1 100644 --- a/packages/angular/build/src/tools/esbuild/bundler-context.ts +++ b/packages/angular/build/src/tools/esbuild/bundler-context.ts @@ -80,6 +80,7 @@ export class BundlerContext { #optionsFactory: BundlerOptionsFactory; #shouldCacheResult: boolean; #loadCache?: LoadResultCache; + #invalidationEpoch = 0; readonly watchFiles = new Set(); constructor( @@ -128,8 +129,8 @@ export class BundlerContext { const externalImportsBrowser = new Set(); const externalImportsServer = new Set(); - const outputFiles = []; - let externalConfiguration; + const outputFiles: BuildOutputFile[] = []; + let externalConfiguration: Set | undefined; for (const result of results) { warnings.push(...result.warnings); if (result.errors) { @@ -202,6 +203,7 @@ export class BundlerContext { return this.#activeBundlePromise; } + const bundleEpoch = this.#invalidationEpoch; const bundlePromise = this.#performBundle().finally(() => { if (this.#activeBundlePromise === bundlePromise) { this.#activeBundlePromise = undefined; @@ -210,7 +212,7 @@ export class BundlerContext { this.#activeBundlePromise = bundlePromise; const result = await bundlePromise; - if (this.#shouldCacheResult) { + if (this.#shouldCacheResult && bundleEpoch === this.#invalidationEpoch) { this.#esbuildResult = result; } @@ -286,9 +288,10 @@ export class BundlerContext { } if (this.#loadCache) { - const cachedLoad = await (this.#loadCache.get(input) ?? - this.#loadCache.get(input.replace(';', ':')) ?? - this.#loadCache.get('file:' + normalizedAbsoluteInput)); + const cachedLoad = + (await this.#loadCache.get(input)) ?? + (await this.#loadCache.get(input.replace(';', ':'))) ?? + (await this.#loadCache.get('file:' + normalizedAbsoluteInput)); if (cachedLoad?.watchFiles) { for (const file of cachedLoad.watchFiles) { if (!isInternalAngularFile(file)) { @@ -551,6 +554,7 @@ export class BundlerContext { } if (invalid) { + this.#invalidationEpoch++; this.#esbuildResult = undefined; } @@ -582,7 +586,7 @@ function isInternalAngularFile(file: string) { function isInternalBundlerFile(file: string) { // Bundler virtual files such as "" or "" - if (file[0] === '<' && file.at(-1) === '>') { + if (file.startsWith('<') && file.endsWith('>')) { return true; } diff --git a/packages/angular/build/src/tools/esbuild/i18n-inliner-worker.ts b/packages/angular/build/src/tools/esbuild/i18n-inliner-worker.ts index af9d9dce134b..e0da07d4a9f7 100644 --- a/packages/angular/build/src/tools/esbuild/i18n-inliner-worker.ts +++ b/packages/angular/build/src/tools/esbuild/i18n-inliner-worker.ts @@ -9,7 +9,6 @@ import remapping, { type DecodedSourceMap, type SourceMapInput } from '@ampproject/remapping'; import type { Node } from '@oxc-project/types'; import { MagicString } from 'magic-string'; -import assert from 'node:assert'; import { deserialize } from 'node:v8'; import { workerData } from 'node:worker_threads'; import { parseSync, visitorKeys } from 'oxc-parser'; @@ -19,7 +18,7 @@ import { createSharedTranslationProxy } from './i18n-translation-reader'; /** * The options passed to the inliner for each file request */ -interface InlineFileRequest { +export interface InlineFileRequest { /** * The filename that should be processed. The data for the file is provided to the Worker * during Worker initialization. @@ -37,12 +36,35 @@ interface InlineFileRequest { * the Worker by reference instead of being copied into it for every request. */ translation?: Blob | SharedArrayBuffer; + + /** + * Optional cache key uniquely identifying the translation messages for the locale. + */ + translationKey?: string; + + missingTranslation?: 'error' | 'warning' | 'ignore'; + shouldOptimize?: boolean; + + /** + * Optional file contents Blob when dispatched via the shared worker pool. + */ + fileBlob?: Blob; + + /** + * Optional cache key uniquely identifying the file content and AST metadata. + */ + fileKey?: string; + + /** + * Optional sourcemap Blob for the file when dispatched via the shared worker pool. + */ + mapBlob?: Blob; } /** * The options passed to the inliner for each code request */ -interface InlineCodeRequest { +export interface InlineCodeRequest { /** * The code that should be processed. */ @@ -64,22 +86,35 @@ interface InlineCodeRequest { * the Worker by reference instead of being copied into it for every request. */ translation?: Blob | SharedArrayBuffer; + + /** + * Optional cache key uniquely identifying the translation messages for the locale. + */ + translationKey?: string; + + missingTranslation?: 'error' | 'warning' | 'ignore'; + shouldOptimize?: boolean; +} + +export interface InlineFileBatchLocaleEntry { + locale: string; + translation?: Blob | SharedArrayBuffer; + translationKey?: string; } /** * The options passed to the inliner for a batch file request */ -interface InlineFileBatchRequest { +export interface InlineFileBatchRequest { /** - * The filename that should be processed. The data for the file is provided to the Worker - * during Worker initialization. + * The filename that should be processed. */ filename: string; /** * The locale specifiers or locale objects that should be used during the inlining process of the file. */ - locales: (string | { locale: string; translation?: Blob | SharedArrayBuffer })[]; + locales: (string | InlineFileBatchLocaleEntry)[]; /** * Whether the file data should be treated as ephemeral and not cached long-term in the Worker. @@ -92,33 +127,83 @@ interface InlineFileBatchRequest { * not present in this list will be evicted from the Worker's memory cache. */ activeLocales?: string[]; + + missingTranslation?: 'error' | 'warning' | 'ignore'; + shouldOptimize?: boolean; + + /** + * Optional file contents Blob when dispatched via the shared worker pool. + */ + fileBlob?: Blob; + + /** + * Optional cache key uniquely identifying the file content and AST metadata. + */ + fileKey?: string; + + /** + * Optional sourcemap Blob for the file when dispatched via the shared worker pool. + */ + mapBlob?: Blob; +} + +export interface InlineDiagnosticMessage { + type: 'error' | 'warning'; + message: string; +} + +export interface InlineFileResult { + file: string; + code: string; + map?: string; + messages: InlineDiagnosticMessage[]; +} + +export interface InlineCodeResult { + output: string; + messages: InlineDiagnosticMessage[]; } /** * The result for a single locale within a batch file request. */ -interface InlineLocaleResult { +export interface InlineLocaleResult { locale: string; code: string; map?: string; - messages: { type: 'error' | 'warning'; message: string }[]; + messages: InlineDiagnosticMessage[]; } /** * The response returned from a batch file request. */ -interface InlineFileBatchResult { +export interface InlineFileBatchResult { file: string; results: InlineLocaleResult[]; } // Extract the application files and common options used for inline requests from the Worker context -const { files, missingTranslation, translations } = (workerData || {}) as { - files: ReadonlyMap; - missingTranslation: 'error' | 'warning' | 'ignore'; +const { + files, + missingTranslation = 'ignore', + translations, +} = (workerData || {}) as { + files?: ReadonlyMap; + missingTranslation?: 'error' | 'warning' | 'ignore'; translations?: ReadonlyMap; }; +/** + * Maximum number of AST metadata structures cached in memory per worker isolate. + * Bounding capacity prevents unbounded memory growth across watch rebuilds. + */ +const MAX_CACHED_FILES = 256; + +/** + * Maximum number of deserialized translation dictionaries cached in memory per worker isolate. + */ +const MAX_CACHED_TRANSLATIONS = 32; + /** * Cached file data including code and extracted localization metadata. */ @@ -128,81 +213,123 @@ interface CachedFileData { } /** - * Cache of file data promises keyed by filename. + * Cache of file data promises keyed by `${filename}\\0${hash}` or filename. */ const fileDataCache = new Map>(); /** - * Cache of deserialized translation messages keyed by locale. + * Deserialized translation message dictionary cache keyed by `${locale}\\0${translationKey}` or locale. */ const deserializedTranslations = new Map>>(); /** - * Retrieves the file data for a filename, loading and extracting localization metadata. - * If `cache` is true, the result is cached in `fileDataCache` across requests in this Worker. - * If `cache` is false (ephemeral), the result is not retained in `fileDataCache`, allowing it - * to be garbage-collected once the batch request finishes. + * Retrieves the code and extracted localization metadata for a file. + * Caches the metadata promise in memory to avoid reparsing the AST across locales. + * If `cache` is false (ephemeral), the result is not retained in `fileDataCache`, + * allowing it to be garbage-collected once the batch request finishes. * - * @param filename The name of the file to load. + * @param filename The name of the file. + * @param fileBlob Optional Blob containing the file content. + * @param fileKey Optional cache key uniquely identifying the file content. * @param cache Whether to cache the loaded file data in the Worker's long-term cache. - * @returns The cached or newly extracted code and localization metadata. + * @returns The cached file data. */ -function loadFileData(filename: string, cache = true): Promise { - const existing = fileDataCache.get(filename); - if (existing) { - return existing; - } - - const fileDataPromise = (async () => { - const data = files.get(filename); - assert(data !== undefined, `Invalid inline request for file '${filename}'.`); +async function getFileData( + filename: string, + fileBlob?: Blob, + fileKey?: string, + cache = true, +): Promise { + const cacheKey = fileKey ?? filename; + let dataPromise = fileDataCache.get(cacheKey); + if (!dataPromise) { + dataPromise = (async () => { + const code = fileBlob ? await fileBlob.text() : await files?.get(filename)?.text(); + if (code === undefined) { + throw new Error(`File not found: ${filename}`); + } - const code = await data.text(); - const metadata = extractLocalizeMetadata(filename, code); + return { + code, + metadata: extractLocalizeMetadata(filename, code), + }; + })().catch((error) => { + if (fileDataCache.get(cacheKey) === dataPromise) { + fileDataCache.delete(cacheKey); + } + throw error; + }); - return { code, metadata }; - })(); + if (cache) { + if (fileDataCache.size >= MAX_CACHED_FILES) { + const oldestKey = fileDataCache.keys().next().value; + if (oldestKey !== undefined) { + fileDataCache.delete(oldestKey); + } + } - if (cache) { - fileDataPromise.catch(() => { - fileDataCache.delete(filename); - }); - fileDataCache.set(filename, fileDataPromise); + fileDataCache.set(cacheKey, dataPromise); + } + } else if (cache) { + fileDataCache.delete(cacheKey); + fileDataCache.set(cacheKey, dataPromise); + } else { + fileDataCache.delete(cacheKey); } - return fileDataPromise; + return dataPromise; } /** * Deserializes or wraps the translation messages for a locale, reusing the result for any - * subsequent request that targets the same locale. - * @param locale The locale identifier. - * @param translation Optional serialized translation messages (SharedArrayBuffer or Blob). + * subsequent request that targets the same locale and translation payload. + * + * @param request The translation request object containing locale, translation payload, and optional key. + * @param explicitTranslation Optional fallback translation payload if request is a string. * @returns The translation messages, or undefined if the locale has no translations. */ function loadTranslation( - locale: string, - translation?: Blob | SharedArrayBuffer, + request: + { locale: string; translation?: Blob | SharedArrayBuffer; translationKey?: string } | string, + explicitTranslation?: Blob | SharedArrayBuffer, ): Promise> | undefined { + const locale = typeof request === 'string' ? request : request.locale; + const translation = typeof request === 'string' ? explicitTranslation : request.translation; + const translationKey = typeof request === 'string' ? undefined : request.translationKey; + const translationData = translation ?? translations?.get(locale); if (!translationData) { return undefined; } - let messagesPromise = deserializedTranslations.get(locale); + const cacheKey = translationKey ? `${locale}\\0${translationKey}` : locale; + let messagesPromise = deserializedTranslations.get(cacheKey); if (!messagesPromise) { if (translationData instanceof Blob) { messagesPromise = translationData .arrayBuffer() .then((buffer) => deserialize(new Uint8Array(buffer)) as Record) .catch((error) => { - deserializedTranslations.delete(locale); + if (deserializedTranslations.get(cacheKey) === messagesPromise) { + deserializedTranslations.delete(cacheKey); + } throw error; }); } else { messagesPromise = Promise.resolve(createSharedTranslationProxy(translationData)); } - deserializedTranslations.set(locale, messagesPromise); + + if (deserializedTranslations.size >= MAX_CACHED_TRANSLATIONS) { + const oldestKey = deserializedTranslations.keys().next().value; + if (oldestKey !== undefined) { + deserializedTranslations.delete(oldestKey); + } + } + + deserializedTranslations.set(cacheKey, messagesPromise); + } else { + deserializedTranslations.delete(cacheKey); + deserializedTranslations.set(cacheKey, messagesPromise); } return messagesPromise; @@ -210,17 +337,19 @@ function loadTranslation( /** * Inlines the provided locale and translation into a JavaScript file that contains `$localize` usage. - * This function is the main entry for the Worker's action that is called by the worker pool. + * This function is the main entry for the Worker\'s action that is called by the worker pool. * * @param request An InlineRequest object representing the options for inlining * @returns An object containing the inlined file and optional map content. */ -export default async function inlineFile(request: InlineFileRequest) { - const { code, metadata } = await loadFileData(request.filename, true); +export default async function inlineFile(request: InlineFileRequest): Promise { + const { code, metadata } = await getFileData(request.filename, request.fileBlob, request.fileKey); // Sourcemaps are parsed on demand per request rather than cached long-term to prevent // monotonic memory growth as a worker processes multiple files across the build. - const rawMap = await files.get(request.filename + '.map')?.text(); + const rawMap = request.mapBlob + ? await request.mapBlob.text() + : await files?.get(request.filename + '.map')?.text(); const map = rawMap ? (JSON.parse(rawMap) as SourceMapInput) : undefined; const result = await inlineLocalize( @@ -228,8 +357,10 @@ export default async function inlineFile(request: InlineFileRequest) { map, metadata, request.locale, - await loadTranslation(request.locale, request.translation), + await loadTranslation(request), request.filename, + request.missingTranslation ?? missingTranslation, + rawMap, ); return { @@ -251,31 +382,42 @@ export async function inlineFileBatch( ): Promise { if (request.activeLocales) { const activeSet = new Set(request.activeLocales); - for (const locale of deserializedTranslations.keys()) { - if (!activeSet.has(locale)) { - deserializedTranslations.delete(locale); + for (const key of deserializedTranslations.keys()) { + const keyLocale = key.includes('\0') ? key.split('\0', 1)[0] : key; + if (!activeSet.has(keyLocale)) { + deserializedTranslations.delete(key); } } } - const { code, metadata } = await loadFileData(request.filename, !request.ephemeral); + const { code, metadata } = await getFileData( + request.filename, + request.fileBlob, + request.fileKey, + !request.ephemeral, + ); // Parse the sourcemap once for the entire batch. // It will naturally be garbage-collected after this batch action returns. - const rawMap = await files.get(request.filename + '.map')?.text(); + const rawMap = request.mapBlob + ? await request.mapBlob.text() + : await files?.get(request.filename + '.map')?.text(); const map = rawMap ? (JSON.parse(rawMap) as SourceMapInput) : undefined; const results = await Promise.all( request.locales.map(async (entry) => { const locale = typeof entry === 'string' ? entry : entry.locale; const translation = typeof entry === 'string' ? undefined : entry.translation; + const translationKey = typeof entry === 'string' ? undefined : entry.translationKey; const result = await inlineLocalize( code, map, metadata, locale, - await loadTranslation(locale, translation), + await loadTranslation({ locale, translation, translationKey }), request.filename, + request.missingTranslation ?? missingTranslation, + rawMap, ); return { @@ -297,18 +439,19 @@ export async function inlineFileBatch( * Inlines the provided locale and translation into JavaScript code that contains `$localize` usage. * This function is a secondary entry primarily for use with component HMR update modules. * - * @param request An InlineRequest object representing the options for inlining + * @param request An InlineCodeRequest object representing the options for inlining * @returns An object containing the inlined code. */ -export async function inlineCode(request: InlineCodeRequest) { +export async function inlineCode(request: InlineCodeRequest): Promise { const metadata = extractLocalizeMetadata(request.filename, request.code); const result = await inlineLocalize( request.code, undefined, metadata, request.locale, - await loadTranslation(request.locale, request.translation), + await loadTranslation(request), request.filename, + request.missingTranslation ?? missingTranslation, ); return { @@ -483,6 +626,7 @@ function extractLocalizeMetadata(filename: string, code: string): FileLocalizeMe * @param locale The target locale identifier. * @param translation The translation messages dictionary, or undefined for untranslated locale. * @param filename The name of the file being transformed. + * @param missingTranslation How to handle missing translations. * @returns The transformed code, optional remapped source map, and diagnostics. */ async function inlineLocalize( @@ -492,6 +636,8 @@ async function inlineLocalize( locale: string, translation: Record | undefined, filename: string, + missingTranslation: 'error' | 'warning' | 'ignore', + rawMap?: string, ) { const magicString = new MagicString(code); const { Diagnostics, translate } = await loadLocalizeTools(); @@ -537,11 +683,10 @@ async function inlineLocalize( } else { replacement = '`'; for (let i = 0; i < translatedParts.length; i++) { - const escapedPart = JSON.stringify(translatedParts[i]) - .slice(1, -1) - .replace(/\\"/g, '"') + const escapedPart = translatedParts[i] + .replace(/\\/g, '\\\\') .replace(/`/g, '\\`') - .replace(/\$\{/g, '\\${'); + .replace(/\${/g, '\\${'); replacement += escapedPart; if (i < translatedSubstitutions.length) { @@ -558,22 +703,28 @@ async function inlineLocalize( } const outputCode = magicString.toString(); - let outputMap; - if (map && magicString.hasChanged()) { - // A decoded map is generated here rather than an encoded one because remapping decodes its - // inputs. Encoding the mappings only for remapping to immediately decode them again doubles - // the peak memory of the largest structure involved in inlining a file. - const rawMap = magicString.generateDecodedMap({ - source: filename, - includeContent: true, - hires: 'boundary', - }); - outputMap = remapping([{ ...rawMap, version: 3 } satisfies DecodedSourceMap, map], () => null); + let outputMap: string | undefined; + if (map) { + if (magicString.hasChanged()) { + // A decoded map is generated here rather than an encoded one because remapping decodes its + // inputs. Encoding the mappings only for remapping to immediately decode them again doubles + // the peak memory of the largest structure involved in inlining a file. + const decodedMap = magicString.generateDecodedMap({ + source: filename, + includeContent: true, + hires: 'boundary', + }); + outputMap = JSON.stringify( + remapping([{ ...decodedMap, version: 3 } satisfies DecodedSourceMap, map], () => null), + ); + } else { + outputMap = rawMap ?? (typeof map === 'string' ? map : JSON.stringify(map)); + } } return { code: outputCode, - map: outputMap && JSON.stringify(outputMap), + map: outputMap, diagnostics, }; } diff --git a/packages/angular/build/src/tools/esbuild/i18n-inliner.ts b/packages/angular/build/src/tools/esbuild/i18n-inliner.ts index 41de55de666f..f04030cbe5bd 100644 --- a/packages/angular/build/src/tools/esbuild/i18n-inliner.ts +++ b/packages/angular/build/src/tools/esbuild/i18n-inliner.ts @@ -10,9 +10,10 @@ import assert from 'node:assert'; import { extname, join } from 'node:path'; import { serialize } from 'node:v8'; import { calculateHash, createContentHash, initializeHash } from '../../utils/hash'; -import { WorkerPool } from '../../utils/worker-pool'; +import { type WorkerPool, getSharedBuildWorkerPool } from '../../utils/worker-pool'; import { type BuildOutputFile, BuildOutputFileType, createOutputFile } from './bundler-files'; import { type Cache, type PersistentCacheStore, createPersistentCacheStore } from './cache'; +import type { InlineCodeResult, InlineDiagnosticMessage } from './i18n-inliner-worker'; import { encodeTranslationToBuffer } from './i18n-translation-encoder'; /** @@ -45,12 +46,45 @@ function serializeTranslation( } if (typeof SharedArrayBuffer !== 'undefined') { - return encodeTranslationToBuffer(translation); + try { + return encodeTranslationToBuffer(translation); + } catch { + // Fall back to Blob serialization if SharedArrayBuffer allocation is restricted + } } return new Blob([serialize(translation)]); } +const translationPayloadCache = new WeakMap< + Record, + { data: SharedArrayBuffer | Blob; key: string } +>(); + +function getOrCreateTranslationPayload( + translation: Record | undefined, + translationIntegrity?: string, +): { data: SharedArrayBuffer | Blob | undefined; key: string | undefined } { + if (!translation) { + return { data: undefined, key: undefined }; + } + + let cached = translationPayloadCache.get(translation); + if (!cached) { + const data = serializeTranslation(translation); + assert(data); + const key = + translationIntegrity ?? + (data instanceof SharedArrayBuffer + ? calculateHash(new Uint8Array(data)) + : calculateHash(JSON.stringify(translation))); + cached = { data, key }; + translationPayloadCache.set(translation, cached); + } + + return cached; +} + /** * Inlining options that should apply to all transformed code. */ @@ -127,6 +161,33 @@ interface CacheCheckItem { cachedResult: Promise; } +export interface InlineTemplateUpdateResult { + code: string; + errors: string[]; + warnings: string[]; +} + +/** + * Partitions diagnostic messages into error and warning strings. + */ +function partitionDiagnostics(messages: readonly InlineDiagnosticMessage[]): { + errors: string[]; + warnings: string[]; +} { + const errors: string[] = []; + const warnings: string[] = []; + + for (const message of messages) { + if (message.type === 'error') { + errors.push(message.message); + } else { + warnings.push(message.message); + } + } + + return { errors, warnings }; +} + /** * An uncached transformation request entry for a file within a specific locale. */ @@ -134,6 +195,7 @@ interface UncachedLocaleEntry { locale: string; cacheKey?: string; translation?: Blob | SharedArrayBuffer; + translationKey?: string; } /** @@ -148,27 +210,27 @@ export class I18nInliner { #cacheStore: PersistentCacheStore | undefined; #cache: Cache | undefined; readonly #localizeFiles: ReadonlyMap; + readonly #filesBlobs: Map; readonly #unmodifiedFiles: Array; constructor( private readonly options: I18nInlinerOptions, - maxThreads?: number, + _maxThreads?: number, ) { this.#unmodifiedFiles = []; - const { outputFiles, shouldOptimize, missingTranslation, translations } = options; + const { outputFiles } = options; const files = new Map(); - const pendingMaps = []; + const pendingMaps: BuildOutputFile[] = []; for (const file of outputFiles) { if (file.type === BuildOutputFileType.Root || file.type === BuildOutputFileType.ServerRoot) { // Skip also the server entry-point. - // Skip stats and similar files. + this.#unmodifiedFiles.push(file); continue; } const fileExtension = extname(file.path); if (fileExtension === '.js' || fileExtension === '.mjs') { - // Check if localizations are present const contentBuffer = Buffer.isBuffer(file.contents) ? file.contents : Buffer.from(file.contents.buffer, file.contents.byteOffset, file.contents.byteLength); @@ -199,24 +261,11 @@ export class I18nInliner { } this.#localizeFiles = files; + this.#filesBlobs = new Map( + Array.from(files, ([name, file]) => [name, new Blob([file.contents])]), + ); - this.#workerPool = new WorkerPool({ - filename: require.resolve('./i18n-inliner-worker'), - maxThreads, - // Extract options to ensure only the named options are serialized and sent to the worker - workerData: { - missingTranslation, - shouldOptimize, - translations, - // A Blob is an immutable data structure that allows sharing the data between workers - // without copying until the data is actually used within a Worker. This is useful here - // since each file may not actually be processed in each Worker and the Blob avoids - // unneeded repeat copying of potentially large JavaScript files. - files: new Map( - Array.from(files, ([name, file]) => [name, new Blob([file.contents])]), - ), - }, - }); + this.#workerPool = getSharedBuildWorkerPool(); } /** @@ -258,9 +307,12 @@ export class I18nInliner { // Pre-calculate cache key bases and serialized Blobs for each locale in this window const localeCacheBases = new Map(); const localeBlobs = new Map(); + const localeKeys = new Map(); for (const { locale, translation, translationIntegrity } of windowLocales) { - localeBlobs.set(locale, serializeTranslation(translation)); + const payload = getOrCreateTranslationPayload(translation, translationIntegrity); + localeBlobs.set(locale, payload.data); + localeKeys.set(locale, payload.key); if (this.#cacheStore) { localeCacheBases.set( @@ -268,7 +320,7 @@ export class I18nInliner { calculateHash( JSON.stringify({ locale, - translation: translationIntegrity || translation, + translation: payload.key ?? translation, missingTranslation, shouldOptimize, localizeVersion, @@ -339,6 +391,7 @@ export class I18nInliner { locale: item.locale, cacheKey: item.cacheKey, translation: localeBlobs.get(item.locale), + translationKey: localeKeys.get(item.locale), }); } } @@ -379,13 +432,11 @@ export class I18nInliner { outputFiles.push(createOutputFile(filename + '.map', fileResult.map, type)); } - for (const message of fileResult.messages) { - if (message.type === 'error') { - errors.push(message.message); - } else { - warnings.push(message.message); - } - } + const { errors: newErrors, warnings: newWarnings } = partitionDiagnostics( + fileResult.messages, + ); + errors.push(...newErrors); + warnings.push(...newWarnings); } } @@ -410,31 +461,48 @@ export class I18nInliner { isLastWindow = true, ): Promise { const workerCount = this.#workerPool.maxThreads || 1; - const targetTaskCount = Math.max(uncachedByFile.size, workerCount * 2); - const localesPerBatch = Math.max( - 1, - Math.ceil(localeCount / (targetTaskCount / (uncachedByFile.size || 1))), - ); + // If there are already enough files to saturate the worker pool, keep all locales of each file together + // in a single task to eliminate duplicate sourcemap JSON parsing and prevent asymmetric straggler tasks. + const localesPerBatch = + uncachedByFile.size >= workerCount + ? localeCount + : Math.max( + 1, + Math.ceil( + localeCount / + (Math.max(uncachedByFile.size, workerCount * 2) / (uncachedByFile.size || 1)), + ), + ); const workerTasks: Promise[] = []; for (const [filename, entries] of uncachedByFile) { - const ephemeral = isLastWindow && entries.length <= localesPerBatch; + const file = this.#localizeFiles.get(filename); + const fileBlob = this.#filesBlobs.get(filename); + const mapBlob = this.#filesBlobs.get(filename + '.map'); + const fileKey = file ? `${filename}\0${file.hash}` : undefined; + for (let i = 0; i < entries.length; i += localesPerBatch) { const batchEntries = entries.slice(i, i + localesPerBatch); + const ephemeral = isLastWindow && entries.length <= localesPerBatch; const task = (async () => { - const batchResult = (await this.#workerPool.run( - { - filename, - locales: batchEntries.map((e) => ({ - locale: e.locale, - translation: e.translation, - })), - ephemeral, - activeLocales, - }, - { name: 'inlineFileBatch' }, - )) as { + const batchResult = (await this.#workerPool.run({ + tag: 'inline-i18n', + action: 'inlineFileBatch', + filename, + fileBlob, + fileKey, + mapBlob, + missingTranslation: this.options.missingTranslation, + shouldOptimize: this.options.shouldOptimize, + ephemeral, + activeLocales, + locales: batchEntries.map((e) => ({ + locale: e.locale, + translation: e.translation, + translationKey: e.translationKey, + })), + })) as { file: string; results: Array; }; @@ -493,7 +561,7 @@ export class I18nInliner { translation: Record | undefined, templateCode: string, templateId: string, - ): Promise<{ code: string; errors: string[]; warnings: string[] }> { + ): Promise { const hasLocalize = templateCode.includes(LOCALIZE_KEYWORD); if (!hasLocalize) { @@ -504,25 +572,20 @@ export class I18nInliner { }; } - const { output, messages } = await this.#workerPool.run( - { - code: templateCode, - filename: templateId, - locale, - translation: serializeTranslation(translation), - }, - { name: 'inlineCode' }, - ); - - const errors: string[] = []; - const warnings: string[] = []; - for (const message of messages) { - if (message.type === 'error') { - errors.push(message.message); - } else { - warnings.push(message.message); - } - } + const payload = getOrCreateTranslationPayload(translation); + const { output, messages } = (await this.#workerPool.run({ + tag: 'inline-i18n', + action: 'inlineCode', + code: templateCode, + filename: templateId, + locale, + translation: payload.data, + translationKey: payload.key, + missingTranslation: this.options.missingTranslation, + shouldOptimize: this.options.shouldOptimize, + })) as InlineCodeResult; + + const { errors, warnings } = partitionDiagnostics(messages); return { code: output, @@ -536,7 +599,11 @@ export class I18nInliner { * @returns A void promise that resolves when closing is complete. */ async close(): Promise { - await Promise.allSettled([this.#cacheStore?.close(), this.#workerPool.destroy()]); + if (this.#workerPool !== getSharedBuildWorkerPool()) { + await Promise.allSettled([this.#cacheStore?.close(), this.#workerPool.destroy()]); + } else { + await this.#cacheStore?.close(); + } } /** diff --git a/packages/angular/build/src/tools/esbuild/i18n-translation-encoder.ts b/packages/angular/build/src/tools/esbuild/i18n-translation-encoder.ts index 091d9ba0ff26..8bdc908adc48 100644 --- a/packages/angular/build/src/tools/esbuild/i18n-translation-encoder.ts +++ b/packages/angular/build/src/tools/esbuild/i18n-translation-encoder.ts @@ -35,7 +35,7 @@ export function encodeTranslationToBuffer(translation: Record): for (let i = 0; i < entryCount; i++) { const [key, val] = entries[i]; const keyBytes = encoder.encode(key); - const valBytes = encoder.encode(JSON.stringify(val)); + const valBytes = encoder.encode(JSON.stringify(val ?? null)); encodedEntries[i] = { keyBytes, valBytes }; stringPoolByteSize += keyBytes.byteLength + valBytes.byteLength; diff --git a/packages/angular/build/src/tools/esbuild/i18n-translation-encoder_spec.ts b/packages/angular/build/src/tools/esbuild/i18n-translation-encoder_spec.ts index c3b6409d40b5..1f7698b65f0c 100644 --- a/packages/angular/build/src/tools/esbuild/i18n-translation-encoder_spec.ts +++ b/packages/angular/build/src/tools/esbuild/i18n-translation-encoder_spec.ts @@ -165,4 +165,33 @@ describe('SharedArrayBuffer Translation Encoder & Reader', () => { const dictionary = new SharedTranslationDictionary(buffer); expect(dictionary.get('testKey')).toBeUndefined(); }); + + it('safely handles undefined translation values', () => { + const translation = { + definedKey: 'value', + undefinedKey: undefined, + }; + + const buffer = encodeTranslationToBuffer(translation as Record); + const dictionary = new SharedTranslationDictionary(buffer); + + expect(dictionary.get('definedKey')).toEqual('value'); + expect(dictionary.get('undefinedKey')).toBeNull(); + }); + + it('encodes and reads large translation dictionaries exceeding 64KB', () => { + const largeTranslation: Record = {}; + for (let i = 0; i < 4000; i++) { + largeTranslation[`key_${i.toString().padStart(6, '0')}`] = `Message content for key ${i}`; + } + + const buffer = encodeTranslationToBuffer(largeTranslation); + expect(buffer.byteLength).toBeGreaterThan(65536); + + const dictionary = new SharedTranslationDictionary(buffer); + expect(dictionary.get('key_000000')).toEqual('Message content for key 0'); + expect(dictionary.get('key_002000')).toEqual('Message content for key 2000'); + expect(dictionary.get('key_003999')).toEqual('Message content for key 3999'); + expect(dictionary.get('nonexistent_key')).toBeUndefined(); + }); }); diff --git a/packages/angular/build/src/tools/esbuild/i18n-translation-reader.ts b/packages/angular/build/src/tools/esbuild/i18n-translation-reader.ts index bc87633da5dc..c43990b39b12 100644 --- a/packages/angular/build/src/tools/esbuild/i18n-translation-reader.ts +++ b/packages/angular/build/src/tools/esbuild/i18n-translation-reader.ts @@ -15,8 +15,8 @@ const NOT_FOUND = Symbol('NOT_FOUND'); /** * Compares a target key's UTF-8 byte array against a byte sequence in the string pool. - * Lexicographical byte comparison of UTF-8 sequences matches UTF-16 code-unit string comparison - * used when encoding the sorted index table. + * Lexicographical byte comparison matches the binary sort order used when encoding + * the index table in `encodeTranslationToBuffer`. */ function compareBytes( target: Uint8Array, @@ -24,7 +24,7 @@ function compareBytes( poolOffset: number, keyLen: number, ): number { - if (poolOffset < 0 || poolOffset + keyLen > pool.length) { + if (poolOffset + keyLen > pool.length) { return 1; } @@ -107,7 +107,7 @@ export class SharedTranslationDictionary { if (cmp === 0) { const valOffset = this.uint32Index[idx + 2]; const valLen = this.uint32Index[idx + 3]; - if (valOffset < 0 || valOffset + valLen > this.uint8Pool.length) { + if (valOffset + valLen > this.uint8Pool.length) { this.lazyCache.set(targetKey, NOT_FOUND); return undefined; diff --git a/packages/angular/build/src/tools/esbuild/javascript-transformer-worker.ts b/packages/angular/build/src/tools/esbuild/javascript-transformer-worker.ts index f2ec7eccce6d..a70f14d8e611 100644 --- a/packages/angular/build/src/tools/esbuild/javascript-transformer-worker.ts +++ b/packages/angular/build/src/tools/esbuild/javascript-transformer-worker.ts @@ -20,14 +20,22 @@ import { removeSourceMappingURL, } from '../../utils/source-map'; import { transform as transformWithOxc } from '../oxc/oxc-transform.js'; -import type { JavaScriptTransformerOptions } from './javascript-transformer'; +import { + JavaScriptTransformFlags, + type JavaScriptTransformerOptions, +} from './javascript-transformer'; -interface JavaScriptTransformRequest { +export interface JavaScriptTransformRequest { filename: string; data: string | Uint8Array; + flags?: JavaScriptTransformFlags | number; skipLinker?: boolean; sideEffects?: boolean; instrumentForCoverage?: boolean; + sourcemap?: boolean; + thirdPartySourcemaps?: boolean; + advancedOptimizations?: boolean; + jit?: boolean; } interface TransformOptions extends Omit { @@ -45,6 +53,10 @@ const { const textDecoder = new TextDecoder(); const textEncoder = new TextEncoder(); +function maybeMove(value: T): T { + return process.versions.pnp ? value : (Piscina.move(value as never) as T); +} + async function instrumentCoverage( filename: string, data: string, @@ -92,17 +104,47 @@ async function instrumentCoverage( export default async function transformJavaScript( request: JavaScriptTransformRequest, ): Promise { - const { filename, data, ...options } = request; + const { filename, data, flags } = request; + + let reqSourcemap: boolean; + let reqThirdPartySourcemaps: boolean; + let reqAdvancedOptimizations: boolean; + let reqJit: boolean; + let reqSkipLinker: boolean; + let reqInstrumentForCoverage: boolean; + let reqSideEffects: boolean | undefined; + + if (flags !== undefined) { + reqSourcemap = (flags & JavaScriptTransformFlags.Sourcemap) !== 0; + reqThirdPartySourcemaps = (flags & JavaScriptTransformFlags.ThirdPartySourcemaps) !== 0; + reqAdvancedOptimizations = (flags & JavaScriptTransformFlags.AdvancedOptimizations) !== 0; + reqJit = (flags & JavaScriptTransformFlags.Jit) !== 0; + reqSkipLinker = (flags & JavaScriptTransformFlags.SkipLinker) !== 0; + reqInstrumentForCoverage = (flags & JavaScriptTransformFlags.InstrumentForCoverage) !== 0; + reqSideEffects = + (flags & JavaScriptTransformFlags.SideEffectsSet) !== 0 + ? (flags & JavaScriptTransformFlags.SideEffectsValue) !== 0 + : undefined; + } else { + reqSourcemap = request.sourcemap ?? sourcemap; + reqThirdPartySourcemaps = request.thirdPartySourcemaps ?? thirdPartySourcemaps; + reqAdvancedOptimizations = request.advancedOptimizations ?? advancedOptimizations; + reqJit = request.jit ?? jit; + reqSkipLinker = request.skipLinker ?? false; + reqInstrumentForCoverage = request.instrumentForCoverage ?? false; + reqSideEffects = request.sideEffects; + } const useInputSourcemap = - sourcemap && (!!thirdPartySourcemaps || !/[\\/]node_modules[\\/]/.test(filename)); + reqSourcemap && (!!reqThirdPartySourcemaps || !/[\\/]node_modules[\\/]/.test(filename)); let textData: string; let inputSourceMap: EncodedSourceMap | undefined; let isAlreadyStripped = false; + let trailing: ReturnType = null; if (typeof data !== 'string') { - const trailing = findTrailingSourceMapComment(data); + trailing = findTrailingSourceMapComment(data); if (trailing === null) { // 0 comments: fast path, no sourcemap to load or strip textData = textDecoder.decode(data); @@ -136,18 +178,25 @@ export default async function transformJavaScript( } const transformedData = await transformJavaScriptImpl(filename, textData, { - ...options, + skipLinker: reqSkipLinker, + sideEffects: reqSideEffects, + instrumentForCoverage: reqInstrumentForCoverage, + sourcemap: reqSourcemap, + thirdPartySourcemaps: reqThirdPartySourcemaps, + advancedOptimizations: reqAdvancedOptimizations, + jit: reqJit, inputSourceMap, isAlreadyStripped, }); - // If no transformations modified the code, return the original untouched data buffer via `move`. - // This preserves any original trailing sourcemap comment and avoids re-encoding. - if (transformedData === textData && typeof data !== 'string') { - return Piscina.move(data); + // If no transformations modified the code, return the original untouched data buffer via `move` + // only if no sourcemap comment needed to be stripped (i.e. comment was absent or sourcemaps are preserved). + const canReturnOriginal = trailing === null || useInputSourcemap; + if (transformedData === textData && typeof data !== 'string' && canReturnOriginal) { + return maybeMove(data); } - return Piscina.move(textEncoder.encode(transformedData)); + return maybeMove(textEncoder.encode(transformedData)); } async function transformJavaScriptImpl( @@ -156,8 +205,13 @@ async function transformJavaScriptImpl( options: TransformOptions, ): Promise { const shouldLink = !options.skipLinker; + const optSourcemap = !!options.sourcemap; + const optThirdPartySourcemaps = !!options.thirdPartySourcemaps; + const optAdvancedOptimizations = !!options.advancedOptimizations; + const optJit = !!options.jit; + const useInputSourcemap = - sourcemap && (!!thirdPartySourcemaps || !/[\\/]node_modules[\\/]/.test(filename)); + optSourcemap && (!!optThirdPartySourcemaps || !/[\\/]node_modules[\\/]/.test(filename)); let code = data; const maps: (DecodedSourceMap | EncodedSourceMap)[] = []; @@ -191,7 +245,7 @@ async function transformJavaScriptImpl( relative: (_from: string, to: string) => to, } as never, logger: new ConsoleLogger(LogLevel.info), - linkerJitMode: jit, + linkerJitMode: optJit, // This is a workaround until https://github.com/angular/angular/issues/42769 is fixed. sourceMapping: false, }) as PluginItem, @@ -206,7 +260,7 @@ async function transformJavaScriptImpl( // Run Oxc linking and/or advanced optimizations in a single unified AST traversal pass const oxcLink = shouldLink && !useBabelLinker; - if (oxcLink || advancedOptimizations) { + if (oxcLink || optAdvancedOptimizations) { const sideEffectFree = options.sideEffects === false; const safeAngularPackage = sideEffectFree && /[\\/]node_modules[\\/]@angular[\\/]/.test(filename); @@ -214,8 +268,8 @@ async function transformJavaScriptImpl( const result = transformWithOxc(filename, code, { link: oxcLink, - jit, - advancedOptimizations, + jit: optJit, + advancedOptimizations: optAdvancedOptimizations, sourcemap: useInputSourcemap, sideEffects: options.sideEffects, topLevelSafeMode, diff --git a/packages/angular/build/src/tools/esbuild/javascript-transformer.ts b/packages/angular/build/src/tools/esbuild/javascript-transformer.ts index 3ba0dfff45b1..0e9211736a84 100644 --- a/packages/angular/build/src/tools/esbuild/javascript-transformer.ts +++ b/packages/angular/build/src/tools/esbuild/javascript-transformer.ts @@ -8,9 +8,8 @@ import { readFile } from 'node:fs/promises'; import { createContentHash } from '../../utils/hash'; -import { IMPORT_EXEC_ARGV } from '../../utils/server-rendering/esm-in-memory-loader/utils'; import { removeSourceMappingURL } from '../../utils/source-map'; -import { WorkerPool, WorkerPoolOptions } from '../../utils/worker-pool'; +import { WorkerPool, getSharedBuildWorkerPool } from '../../utils/worker-pool'; import { Cache } from './cache'; const LINKER_DECLARATION_PREFIX = 'ɵɵngDeclare'; @@ -20,7 +19,7 @@ const LINKER_DECLARATION_PREFIX_BYTES = Buffer.from(LINKER_DECLARATION_PREFIX, ' * Determines whether JavaScript code requires Angular linker processing. * * @param path The full path to the file. - * @param data The data (string or Buffer) of the file. + * @param data The data (string or Uint8Array) of the file. * @returns True if the code contains an Angular partial declaration; otherwise false. */ function requiresLinking(path: string, data: string | Uint8Array): boolean { @@ -51,6 +50,21 @@ export interface JavaScriptTransformerOptions { jit?: boolean; } +/** + * Bitmask flags for serializing transformer options and task parameters across IPC in a single SMI integer. + */ +export enum JavaScriptTransformFlags { + None = 0, + Sourcemap = 1 << 0, + ThirdPartySourcemaps = 1 << 1, + AdvancedOptimizations = 1 << 2, + Jit = 1 << 3, + SkipLinker = 1 << 4, + InstrumentForCoverage = 1 << 5, + SideEffectsSet = 1 << 6, + SideEffectsValue = 1 << 7, +} + /** * A class that performs transformation of JavaScript files and raw data. * A worker pool is used to distribute the transformation actions and allow @@ -62,6 +76,8 @@ export class JavaScriptTransformer { #workerPool: WorkerPool | undefined; #commonOptions: Required; #fileCacheKeyBase: Uint8Array; + #baseFlags = JavaScriptTransformFlags.None; + #isClosed = false; /** Queue of pending transformation tasks waiting for an active concurrency slot. */ #pendingTasks: { resolve: () => void; reject: (reason: Error) => void }[] = []; @@ -93,6 +109,22 @@ export class JavaScriptTransformer { jit, }; this.#fileCacheKeyBase = Buffer.from(JSON.stringify(this.#commonOptions), 'utf-8'); + + let baseFlags = JavaScriptTransformFlags.None; + if (sourcemap) { + baseFlags |= JavaScriptTransformFlags.Sourcemap; + } + if (thirdPartySourcemaps) { + baseFlags |= JavaScriptTransformFlags.ThirdPartySourcemaps; + } + if (advancedOptimizations) { + baseFlags |= JavaScriptTransformFlags.AdvancedOptimizations; + } + if (jit) { + baseFlags |= JavaScriptTransformFlags.Jit; + } + this.#baseFlags = baseFlags; + this.#workerPool = this.#ensureWorkerPool(); } @@ -103,6 +135,10 @@ export class JavaScriptTransformer { * @returns A promise resolving to the transformation result. */ async #runWithThrottle(action: () => Promise): Promise { + if (this.#isClosed) { + throw new Error('JavaScriptTransformer closed.'); + } + if (this.#activeTasks >= this.#maxConcurrent) { await new Promise((resolve, reject) => { this.#pendingTasks.push({ resolve, reject }); @@ -111,6 +147,16 @@ export class JavaScriptTransformer { this.#activeTasks++; } + if (this.#isClosed) { + const next = this.#pendingTasks.shift(); + if (next) { + next.resolve(); + } else { + this.#activeTasks--; + } + throw new Error('JavaScriptTransformer closed.'); + } + try { return await action(); } finally { @@ -124,24 +170,11 @@ export class JavaScriptTransformer { } #ensureWorkerPool(): WorkerPool { - if (this.#workerPool) { - return this.#workerPool; - } - - const workerPoolOptions: WorkerPoolOptions = { - filename: require.resolve('./javascript-transformer-worker'), - maxThreads: this.maxThreads, - minThreads: this.maxThreads, - workerData: this.#commonOptions, - }; - - // Prevent passing SSR `--import` (loader-hooks) from parent to child worker. - const filteredExecArgv = process.execArgv.filter((v) => v !== IMPORT_EXEC_ARGV); - if (process.execArgv.length !== filteredExecArgv.length) { - workerPoolOptions.execArgv = filteredExecArgv; + if (this.#isClosed) { + throw new Error('JavaScriptTransformer closed.'); } - this.#workerPool = new WorkerPool(workerPoolOptions); + this.#workerPool ??= getSharedBuildWorkerPool(); return this.#workerPool; } @@ -165,10 +198,10 @@ export class JavaScriptTransformer { let cacheKey: string | undefined; if (this.cache) { - // Create a cache key from the file data and options that effect the output. + // Create a cache key from the file data and options that affect the output. // NOTE: If additional options are added, this may need to be updated. const hasher = createContentHash(); - hasher.update(`${!!skipLinker}--${!!sideEffects}`); + hasher.update(`${!!skipLinker}--${sideEffects}--${!!instrumentForCoverage}`); hasher.update(data); hasher.update(this.#fileCacheKeyBase); cacheKey = hasher.digest(); @@ -207,9 +240,11 @@ export class JavaScriptTransformer { * Performs JavaScript transformations on the provided data of a file. The file does not need * to exist on the filesystem. * @param filename The full path of the file represented by the data. - * @param data The data of the file that should be transformed. + * @param data The data of the file that should be transformed. Standalone transferable Uint8Array + * buffers may be detached upon worker transfer. * @param skipLinker If true, bypass all Angular linker processing; if false, attempt linking. - * @param sideEffects If false, and `advancedOptimizations` is enabled tslib decorators are wrapped. + * @param sideEffects Tri-state flag indicating if code is side-effect-free (false), has side-effects (true), or unspecified (undefined). + * @param instrumentForCoverage If true, instrument the code for test coverage. * @returns A promise that resolves to a UTF-8 encoded Uint8Array containing the result. */ async transformData( @@ -219,6 +254,10 @@ export class JavaScriptTransformer { sideEffects?: boolean, instrumentForCoverage?: boolean, ): Promise { + if (this.#isClosed) { + throw new Error('JavaScriptTransformer closed.'); + } + const shouldLink = !skipLinker && requiresLinking(filename, data); // Perform a quick test to determine if the data needs any transformations. @@ -246,13 +285,26 @@ export class JavaScriptTransformer { data.byteLength === data.buffer.byteLength && !process.versions.pnp; + let flags = this.#baseFlags; + if (!shouldLink) { + flags |= JavaScriptTransformFlags.SkipLinker; + } + if (sideEffects !== undefined) { + flags |= JavaScriptTransformFlags.SideEffectsSet; + if (sideEffects) { + flags |= JavaScriptTransformFlags.SideEffectsValue; + } + } + if (instrumentForCoverage) { + flags |= JavaScriptTransformFlags.InstrumentForCoverage; + } + return this.#ensureWorkerPool().run( { + tag: 'transform-js', filename, data, - skipLinker: !shouldLink, - sideEffects, - instrumentForCoverage, + flags, }, { transferList: isTransferable ? [data.buffer] : undefined, @@ -265,18 +317,25 @@ export class JavaScriptTransformer { * @returns A void promise that resolves when closing is complete. */ async close(): Promise { + if (this.#isClosed) { + return; + } + this.#isClosed = true; + const pending = this.#pendingTasks; this.#pendingTasks = []; for (const task of pending) { task.reject(new Error('JavaScriptTransformer closed.')); } - if (this.#workerPool) { + if (this.#workerPool && this.#workerPool !== getSharedBuildWorkerPool()) { try { await this.#workerPool.destroy(); } finally { this.#workerPool = undefined; } + } else { + this.#workerPool = undefined; } } } diff --git a/packages/angular/build/src/tools/esbuild/javascript-transformer_spec.ts b/packages/angular/build/src/tools/esbuild/javascript-transformer_spec.ts index b1cdeec07b44..081cf2b5a060 100644 --- a/packages/angular/build/src/tools/esbuild/javascript-transformer_spec.ts +++ b/packages/angular/build/src/tools/esbuild/javascript-transformer_spec.ts @@ -374,4 +374,36 @@ describe('JavaScriptTransformer sourcemaps', () => { expect(text).not.toContain('i0.ɵɵngDeclareDirective'); }); + + it('should strip sourcemaps from Uint8Array when worker runs with sourcemap: false', async () => { + transformer = new JavaScriptTransformer( + { + sourcemap: false, + advancedOptimizations: true, + }, + 1, + ); + + const inputBuffer = Buffer.from('var a = 1;\n//# sourceMappingURL=foo.js.map', 'utf-8'); + const result = await transformer.transformData('src/app/foo.js', inputBuffer, true); + const text = Buffer.from(result).toString('utf-8'); + + expect(text).toBe('var a = 1;\n'); + expect(text).not.toContain('sourceMappingURL'); + }); + + it('should reject tasks after transformer is closed', async () => { + transformer = new JavaScriptTransformer( + { + sourcemap: false, + }, + 1, + ); + + await transformer.close(); + + await expectAsync( + transformer.transformData('src/app/foo.js', 'console.log(1);', true), + ).toBeRejectedWithError('JavaScriptTransformer closed.'); + }); }); diff --git a/packages/angular/build/src/tools/sass/sass-worker-implementation.ts b/packages/angular/build/src/tools/sass/sass-worker-implementation.ts index 9208bef3a803..aab1a63dcf31 100644 --- a/packages/angular/build/src/tools/sass/sass-worker-implementation.ts +++ b/packages/angular/build/src/tools/sass/sass-worker-implementation.ts @@ -18,7 +18,7 @@ import type { StringOptions, } from 'sass-embedded'; import { maxWorkers } from '../../utils/environment-options'; -import { WorkerPool } from '../../utils/worker-pool'; +import { WorkerPool, getSharedBuildWorkerPool } from '../../utils/worker-pool'; import { type Importers, type SassServiceImplementation, isFileImporter } from './sass-service'; // Polyfill Symbol.dispose if not present @@ -73,17 +73,64 @@ export interface RenderResponseMessage { */ export class SassWorkerImplementation implements SassServiceImplementation { #workerPool: WorkerPool | undefined; + #pendingTasks: { resolve: () => void; reject: (reason: Error) => void }[] = []; + #activeTasks = 0; + #maxConcurrent: number; + #isClosed = false; constructor( private readonly rebase = false, readonly maxThreads = MAX_RENDER_WORKERS, - ) {} + ) { + // Throttle concurrent Sass tasks so that blocking importer futex waits do not saturate all shared worker threads. + this.#maxConcurrent = Math.max(1, Math.min(16, Math.floor(maxThreads * 0.75))); + } + + /** + * Executes a Sass render action using a semaphore-based backpressure throttle. + * Prevents shared worker pool saturation and thread starvation for CPU-bound tasks. + */ + async #runWithThrottle(action: () => Promise): Promise { + if (this.#isClosed) { + throw new Error('SassWorkerImplementation closed.'); + } + + if (this.#activeTasks >= this.#maxConcurrent) { + await new Promise((resolve, reject) => { + this.#pendingTasks.push({ resolve, reject }); + }); + } else { + this.#activeTasks++; + } + + if (this.#isClosed) { + const next = this.#pendingTasks.shift(); + if (next) { + next.resolve(); + } else { + this.#activeTasks--; + } + throw new Error('SassWorkerImplementation closed.'); + } + + try { + return await action(); + } finally { + const next = this.#pendingTasks.shift(); + if (next) { + next.resolve(); + } else { + this.#activeTasks--; + } + } + } #ensureWorkerPool(): WorkerPool { - this.#workerPool ??= new WorkerPool({ - filename: require.resolve('./worker'), - maxThreads: this.maxThreads, - }); + if (this.#isClosed) { + throw new Error('SassWorkerImplementation closed.'); + } + + this.#workerPool ??= getSharedBuildWorkerPool(); return this.#workerPool; } @@ -113,62 +160,67 @@ export class SassWorkerImplementation implements SassServiceImplementation { source: string, options: StringOptions<'async'>, ): Promise { - // The CLI's configuration does not use or expose the ability to define custom Sass functions - if (options.functions && Object.keys(options.functions).length > 0) { - throw new Error('Sass custom functions are not supported.'); - } + return this.#runWithThrottle(async () => { + // The CLI's configuration does not use or expose the ability to define custom Sass functions + if (options.functions && Object.keys(options.functions).length > 0) { + throw new Error('Sass custom functions are not supported.'); + } + + const { functions, importers, importer, url, logger, ...serializableOptions } = options; + using importerChannel = importers?.length + ? this.#createImporterChannel(importers) + : undefined; - const { functions, importers, importer, url, logger, ...serializableOptions } = options; - using importerChannel = importers?.length ? this.#createImporterChannel(importers) : undefined; - - const response = (await this.#ensureWorkerPool().run( - { - source, - importerChannel, - hasLogger: !!logger, - rebase: this.rebase, - options: { - ...serializableOptions, - // URL is not serializable so to convert to string here and back to URL in the worker. - url: url ? fileURLToPath(url) : undefined, + const response = (await this.#ensureWorkerPool().run( + { + tag: 'render-sass', + source, + importerChannel, + hasLogger: !!logger, + rebase: this.rebase, + options: { + ...serializableOptions, + // URL is not serializable so to convert to string here and back to URL in the worker. + url: url ? fileURLToPath(url) : undefined, + }, }, - }, - { - transferList: importerChannel ? [importerChannel.port] : undefined, - }, - )) as RenderResponseMessage; + { + transferList: importerChannel ? [importerChannel.port] : undefined, + }, + )) as RenderResponseMessage; - const { result, error, warnings } = response; + const { result, error, warnings } = response; - if (warnings && logger?.warn) { - for (const { message, span, ...options } of warnings) { - logger.warn(message, { - ...options, - span: span && { - ...span, - url: span.url ? pathToFileURL(span.url) : undefined, - }, - }); + if (warnings && logger?.warn) { + for (const { message, span, ...options } of warnings) { + logger.warn(message, { + ...options, + span: span && { + ...span, + url: span.url ? pathToFileURL(span.url) : undefined, + }, + }); + } } - } - if (error) { - // Convert stringified url value required for cloning back to a URL object - const url = error.span?.url as string | undefined; - if (url) { - error.span.url = pathToFileURL(url); - } + if (error) { + // Convert stringified url value required for cloning back to a URL object + const url = error.span?.url as string | undefined; + if (url) { + error.span.url = pathToFileURL(url); + } - throw error; - } + throw error; + } - assert(result, 'Sass render worker should always return a result or an error'); + assert(result, 'Sass render worker should always return a result or an error'); - return { - ...result, - // URL is not serializable so in the worker we convert to string and here back to URL. - loadedUrls: result.loadedUrls.map((p) => pathToFileURL(p)), - }; + return { + ...result, + // URL is not serializable so in the worker we convert to string and here back to URL. + loadedUrls: result.loadedUrls.map((p) => pathToFileURL(p)), + }; + }); } /** @@ -177,10 +229,25 @@ export class SassWorkerImplementation implements SassServiceImplementation { * @returns A void promise that resolves when closing is complete. */ async close(): Promise { - if (this.#workerPool) { - const pool = this.#workerPool; + if (this.#isClosed) { + return; + } + this.#isClosed = true; + + const pending = this.#pendingTasks; + this.#pendingTasks = []; + for (const task of pending) { + task.reject(new Error('SassWorkerImplementation closed.')); + } + + if (this.#workerPool && this.#workerPool !== getSharedBuildWorkerPool()) { + try { + await this.#workerPool.destroy(); + } finally { + this.#workerPool = undefined; + } + } else { this.#workerPool = undefined; - await pool.destroy(); } } @@ -190,7 +257,7 @@ export class SassWorkerImplementation implements SassServiceImplementation { mainImporterPort.on( 'message', - ({ url, options }: { url: string; options: CanonicalizeContext }) => { + ({ id, url, options }: { id: number; url: string; options: CanonicalizeContext }) => { this.processImporters(importers, url, { ...options, // URL is not serializable so in the worker we convert to string and here back to URL. @@ -199,13 +266,16 @@ export class SassWorkerImplementation implements SassServiceImplementation { : null, }) .then((result) => { - mainImporterPort.postMessage(result); + mainImporterPort.postMessage({ id, result }); }) .catch((error) => { - mainImporterPort.postMessage(error); + mainImporterPort.postMessage({ + id, + error: error instanceof Error ? error : new Error(String(error)), + }); }) .finally(() => { - Atomics.store(importerSignal, 0, 1); + Atomics.store(importerSignal, 0, id); Atomics.notify(importerSignal, 0); }); }, diff --git a/packages/angular/build/src/tools/sass/worker.ts b/packages/angular/build/src/tools/sass/worker.ts index 2416fe21e5cd..500777eec0d7 100644 --- a/packages/angular/build/src/tools/sass/worker.ts +++ b/packages/angular/build/src/tools/sass/worker.ts @@ -33,7 +33,7 @@ import type { /** * A request to render a Sass stylesheet using the supplied options. */ -interface RenderRequestMessage { +export interface RenderRequestMessage { /** * The contents to compile. */ @@ -88,7 +88,7 @@ export default async function renderSassStylesheet( ): Promise { const { importerChannel, hasLogger, source, options, rebase } = request; - const entryDirectory = dirname(options.url); + const entryDirectory = options.url ? dirname(options.url) : process.cwd(); let warnings: SerializableWarningMessage[] | undefined; try { const directoryCache = new Map(); @@ -99,40 +99,36 @@ export default async function renderSassStylesheet( // This process must be synchronous from the perspective of dart-sass. The `Atomics` // functions combined with the shared memory `importSignal` and the Node.js // `receiveMessageOnPort` function are used to ensure synchronous behavior. + let nextRequestId = 0; const proxyImporter: FileImporter<'sync'> = { findFileUrl: (url, { fromImport, containingUrl }) => { + const requestId = ++nextRequestId; Atomics.store(importerChannel.signal, 0, 0); importerChannel.port.postMessage({ + id: requestId, url, options: { fromImport, - containingUrl: containingUrl ? fileURLToPath(containingUrl) : null, + containingUrl: containingUrl + ? containingUrl.protocol === 'file:' + ? fileURLToPath(containingUrl) + : containingUrl.toString() + : null, }, }); - // Wait for the main thread to set the signal to 1 and notify, which tells - // us that a message can be received on the port. - // If the main thread is fast, the signal will already be set to 1, and no - // sleep/notify is necessary. - // However, there can be a race condition here: - // - the main thread sets the signal to 1, but does not get to the notify instruction yet - // - the worker does not pause because the signal is set to 1 - // - the worker very soon enters this method again - // - this method sets the signal to 0 and sends the message - // - the signal is 0 and so the `Atomics.wait` call blocks - // - only now the main thread runs the `notify` from the first invocation, so the - // worker continues. - // - but there is no message yet in the port, because the thread should not have been - // waken up yet. - // To combat this, wait for a non-0 value _twice_. - // Almost every time, this immediately continues with "not-equal", because - // the signal is still set to 1, except during the race condition, when the second - // wait will wait for the correct notify. - Atomics.wait(importerChannel.signal, 0, 0); - Atomics.wait(importerChannel.signal, 0, 0); + // Wait for the main thread to set the signal to the requestId and notify. + // A monotonic sequence ID guarantees that stale notifies cannot prematurely wake up this request. + while (Atomics.load(importerChannel.signal, 0) !== requestId) { + Atomics.wait(importerChannel.signal, 0, 0); + } - const result = receiveMessageOnPort(importerChannel.port)?.message as string | null; + const response = receiveMessageOnPort(importerChannel.port)?.message as + { id: number; result?: string | null; error?: Error } | undefined; + if (response?.error) { + throw response.error; + } - return result ? pathToFileURL(result) : null; + return response?.result ? pathToFileURL(response.result) : null; }, }; options.importers = [ @@ -175,7 +171,7 @@ export default async function renderSassStylesheet( const result = compileString(source, { ...options, // URL is not serializable so to convert to string in the parent and back to URL here. - url: pathToFileURL(options.url), + url: options.url ? pathToFileURL(options.url) : undefined, // The `importer` option (singular) handles relative imports importer: relativeImporter, logger: hasLogger @@ -242,6 +238,8 @@ export default async function renderSassStylesheet( error: { message: 'An unknown error has occurred.' }, }; } + } finally { + importerChannel?.port.close(); } } @@ -268,7 +266,11 @@ function convertSourceSpan(span: SourceSpan): Omit & { url?: offset: span.start.offset, line: span.start.line, }, - url: span.url ? fileURLToPath(span.url) : undefined, + url: span.url + ? span.url.protocol === 'file:' + ? fileURLToPath(span.url) + : span.url.toString() + : undefined, }; } diff --git a/packages/angular/build/src/utils/shared-worker-router.ts b/packages/angular/build/src/utils/shared-worker-router.ts new file mode 100644 index 000000000000..9641b48689d8 --- /dev/null +++ b/packages/angular/build/src/utils/shared-worker-router.ts @@ -0,0 +1,77 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.dev/license + */ + +import inlineFile, { + type InlineCodeRequest, + type InlineFileBatchRequest, + type InlineFileRequest, + inlineCode, + inlineFileBatch, +} from '../tools/esbuild/i18n-inliner-worker'; +import transformJavaScript, { + type JavaScriptTransformRequest, +} from '../tools/esbuild/javascript-transformer-worker'; +import renderSassStylesheet, { type RenderRequestMessage } from '../tools/sass/worker'; + +export interface TransformJsTask extends JavaScriptTransformRequest { + tag: 'transform-js'; +} + +export interface RenderSassTask extends RenderRequestMessage { + tag: 'render-sass'; +} + +export interface InlineI18nFileTask extends InlineFileRequest { + tag: 'inline-i18n'; + action?: 'inlineFile'; +} + +export interface InlineI18nFileBatchTask extends InlineFileBatchRequest { + tag: 'inline-i18n'; + action: 'inlineFileBatch'; +} + +export interface InlineI18nCodeTask extends InlineCodeRequest { + tag: 'inline-i18n'; + action: 'inlineCode'; +} + +export type InlineI18nTask = InlineI18nFileTask | InlineI18nFileBatchTask | InlineI18nCodeTask; + +export type SharedWorkerTask = TransformJsTask | RenderSassTask | InlineI18nTask; + +/** + * Main worker dispatch function. Dispatches incoming tasks based on task tag. + * + * @param task The task payload dispatched to the shared build worker pool. + * @returns The resolved result of the corresponding task handler. + */ +export default function workerRouter(task: SharedWorkerTask): Promise { + switch (task.tag) { + case 'transform-js': + return transformJavaScript(task); + case 'render-sass': + return renderSassStylesheet(task); + case 'inline-i18n': + switch (task.action) { + case 'inlineCode': + return inlineCode(task); + case 'inlineFileBatch': + return inlineFileBatch(task); + case 'inlineFile': + case undefined: + return inlineFile(task); + default: + throw new Error( + `Unknown inline-i18n task action: ${(task as { action?: unknown }).action}`, + ); + } + default: + throw new Error(`Unknown worker task tag: ${(task as { tag?: unknown })?.tag}`); + } +} diff --git a/packages/angular/build/src/utils/worker-pool.ts b/packages/angular/build/src/utils/worker-pool.ts index 907de66ba02f..8ab70a1f5e54 100644 --- a/packages/angular/build/src/utils/worker-pool.ts +++ b/packages/angular/build/src/utils/worker-pool.ts @@ -7,15 +7,19 @@ */ import { getCompileCacheDir } from 'node:module'; -import { Piscina } from 'piscina'; +import { FixedQueue, Piscina } from 'piscina'; +import { maxWorkers } from './environment-options'; +import { IMPORT_EXEC_ARGV } from './server-rendering/esm-in-memory-loader/utils'; export type WorkerPoolOptions = ConstructorParameters[0]; export class WorkerPool extends Piscina { - constructor(options: WorkerPoolOptions) { + constructor(options?: WorkerPoolOptions) { const piscinaOptions: WorkerPoolOptions = { - minThreads: 1, - idleTimeout: 4_000, + minThreads: options?.maxThreads ?? maxWorkers, + idleTimeout: 30_000, + concurrentTasksPerWorker: 2, + taskQueue: new FixedQueue(), // Web containers do not support transferable objects with receiveOnMessagePort which // is used when the Atomics based wait loop is enable. atomics: process.versions.webcontainer ? 'disabled' : 'sync', @@ -30,9 +34,12 @@ export class WorkerPool extends Piscina { ? undefined : getCompileCacheDir?.(); if (compileCacheDirectory) { - if (typeof piscinaOptions.env === 'object') { - piscinaOptions.env['NODE_COMPILE_CACHE'] = compileCacheDirectory; - } else { + if (typeof piscinaOptions.env === 'object' && piscinaOptions.env !== null) { + piscinaOptions.env = { + ...piscinaOptions.env, + 'NODE_COMPILE_CACHE': compileCacheDirectory, + }; + } else if (piscinaOptions.env === undefined) { // Default behavior of `env` option is to copy current process values piscinaOptions.env = { ...process.env, @@ -44,3 +51,45 @@ export class WorkerPool extends Piscina { super(piscinaOptions); } } + +/** + * The singleton shared build worker pool instance. + */ +let sharedBuildWorkerPool: WorkerPool | undefined; +let shutdownPromise: Promise | undefined; + +/** + * Returns the singleton shared build worker pool instance bounded by `maxWorkers`. + * The pool routes tasks using `shared-worker-router`. + */ +export function getSharedBuildWorkerPool(): WorkerPool { + if (!sharedBuildWorkerPool) { + const filteredExecArgv = process.execArgv.filter((v) => v !== IMPORT_EXEC_ARGV); + sharedBuildWorkerPool = new WorkerPool({ + filename: require.resolve('./shared-worker-router'), + maxThreads: maxWorkers, + minThreads: maxWorkers, + execArgv: filteredExecArgv.length !== process.execArgv.length ? filteredExecArgv : undefined, + }); + } + + return sharedBuildWorkerPool; +} + +/** + * Destroys and resets the singleton shared build worker pool. + */ +export async function shutdownSharedBuildWorkerPool(): Promise { + if (shutdownPromise) { + return shutdownPromise; + } + + if (sharedBuildWorkerPool) { + const pool = sharedBuildWorkerPool; + sharedBuildWorkerPool = undefined; + shutdownPromise = pool.destroy().finally(() => { + shutdownPromise = undefined; + }); + await shutdownPromise; + } +} diff --git a/packages/angular/build/src/utils/worker-pool_spec.ts b/packages/angular/build/src/utils/worker-pool_spec.ts new file mode 100644 index 000000000000..c2b0fffb2f7a --- /dev/null +++ b/packages/angular/build/src/utils/worker-pool_spec.ts @@ -0,0 +1,244 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.dev/license + */ + +import { serialize } from 'node:v8'; +import { initializeHash } from './hash'; +import { WorkerPool, getSharedBuildWorkerPool, shutdownSharedBuildWorkerPool } from './worker-pool'; + +describe('Singleton Shared Build Worker Pool', () => { + beforeAll(async () => { + await initializeHash(); + }); + + afterEach(async () => { + await shutdownSharedBuildWorkerPool(); + }); + + it('should return a WorkerPool instance from getSharedBuildWorkerPool', () => { + const pool = getSharedBuildWorkerPool(); + expect(pool).toBeDefined(); + expect(pool instanceof WorkerPool).toBeTrue(); + }); + + it('should return the identical singleton instance on multiple getSharedBuildWorkerPool calls', () => { + const pool1 = getSharedBuildWorkerPool(); + const pool2 = getSharedBuildWorkerPool(); + + expect(pool1).toBe(pool2); + }); + + it('should reset the singleton instance after shutdownSharedBuildWorkerPool is called', async () => { + const pool1 = getSharedBuildWorkerPool(); + await shutdownSharedBuildWorkerPool(); + + const pool2 = getSharedBuildWorkerPool(); + expect(pool2).not.toBe(pool1); + expect(pool2 instanceof WorkerPool).toBeTrue(); + }); + + it('should dispatch transform-js tasks to the shared worker router', async () => { + const pool = getSharedBuildWorkerPool(); + const code = 'export const value: number = 42;\n'; + + const result = (await pool.run({ + tag: 'transform-js', + filename: 'test.ts', + data: code, + skipLinker: true, + sourcemap: false, + })) as Uint8Array; + + expect(result).toBeDefined(); + const text = Buffer.from(result).toString('utf-8'); + expect(text).toContain('42'); + }); + + it('should dispatch render-sass tasks to the shared worker router', async () => { + const pool = getSharedBuildWorkerPool(); + const scss = '$color: red; .test { color: $color; }'; + + const response = (await pool.run({ + tag: 'render-sass', + source: scss, + hasLogger: false, + rebase: false, + options: { + url: '/test.scss', + }, + })) as { result?: { css: string } }; + + expect(response.result).toBeDefined(); + expect(response.result?.css).toContain('color: red'); + }); + + it('should dispatch inline-i18n inlineCode tasks to the shared worker router', async () => { + const pool = getSharedBuildWorkerPool(); + const code = 'export const greeting = $localize`:@@greeting:Hello`;\n'; + const translation = { + greeting: { + messageParts: ['Bonjour'], + placeholderNames: [], + text: 'Bonjour', + }, + }; + + const result = (await pool.run({ + tag: 'inline-i18n', + action: 'inlineCode', + code, + filename: 'main.js', + locale: 'fr', + translation: new Blob([serialize(translation)]), + missingTranslation: 'ignore', + })) as { output: string; messages: unknown[] }; + + expect(result.output).toContain('"Bonjour"'); + expect(result.output).not.toContain('$localize'); + }); + + it('should dispatch inline-i18n inlineFile tasks with fileBlobs to the shared worker router', async () => { + const pool = getSharedBuildWorkerPool(); + const code = 'export const greeting = $localize`:@@greeting:Hello`;\n'; + const fileBlob = new Blob([code]); + const translationEs = { + greeting: { + messageParts: ['Hola'], + placeholderNames: [], + text: 'Hola', + }, + }; + const translationFr = { + greeting: { + messageParts: ['Bonjour'], + placeholderNames: [], + text: 'Bonjour', + }, + }; + + const resultEs = (await pool.run({ + tag: 'inline-i18n', + action: 'inlineFile', + filename: 'main.js', + fileKey: 'main.js\0hash123', + locale: 'es', + translation: new Blob([serialize(translationEs)]), + translationKey: 'trans_es_123', + missingTranslation: 'ignore', + fileBlob, + })) as { file: string; code: string }; + + const resultFr = (await pool.run({ + tag: 'inline-i18n', + action: 'inlineFile', + filename: 'main.js', + fileKey: 'main.js\0hash123', + locale: 'fr', + translation: new Blob([serialize(translationFr)]), + translationKey: 'trans_fr_123', + missingTranslation: 'ignore', + fileBlob, + })) as { file: string; code: string }; + + expect(resultEs.code).toContain('"Hola"'); + expect(resultEs.code).not.toContain('$localize'); + expect(resultFr.code).toContain('"Bonjour"'); + expect(resultFr.code).not.toContain('$localize'); + }); + + it('should dispatch inline-i18n inlineFileBatch tasks to the shared worker router', async () => { + const pool = getSharedBuildWorkerPool(); + const code = 'export const greeting = $localize`:@@greeting:Hello`;\n'; + const fileBlob = new Blob([code]); + const translationEs = { + greeting: { + messageParts: ['Hola'], + placeholderNames: [], + text: 'Hola', + }, + }; + const translationFr = { + greeting: { + messageParts: ['Bonjour'], + placeholderNames: [], + text: 'Bonjour', + }, + }; + + const batchResult = (await pool.run({ + tag: 'inline-i18n', + action: 'inlineFileBatch', + filename: 'main.js', + fileKey: 'main.js\0hash123', + fileBlob, + missingTranslation: 'ignore', + locales: [ + { + locale: 'es', + translation: new Blob([serialize(translationEs)]), + translationKey: 'trans_es_123', + }, + { + locale: 'fr', + translation: new Blob([serialize(translationFr)]), + translationKey: 'trans_fr_123', + }, + ], + })) as { file: string; results: { locale: string; code: string }[] }; + + expect(batchResult.file).toBe('main.js'); + expect(batchResult.results.length).toBe(2); + expect(batchResult.results[0].locale).toBe('es'); + expect(batchResult.results[0].code).toContain('"Hola"'); + expect(batchResult.results[1].locale).toBe('fr'); + expect(batchResult.results[1].code).toContain('"Bonjour"'); + }); + + it('should execute mixed tasks concurrently across the shared pool without interference', async () => { + const pool = getSharedBuildWorkerPool(); + + const jsPromise = pool.run({ + tag: 'transform-js', + filename: 'concurrent.ts', + data: 'export const a: number = 1;\n', + skipLinker: true, + sourcemap: false, + }); + + const sassPromise = pool.run({ + tag: 'render-sass', + source: '$bg: blue; body { background: $bg; }', + hasLogger: false, + rebase: false, + options: { + url: '/concurrent.scss', + }, + }); + + const i18nPromise = pool.run({ + tag: 'inline-i18n', + action: 'inlineCode', + code: 'export const msg = $localize`:@@m:Hi`;\n', + filename: 'concurrent.js', + locale: 'de', + translation: new Blob([ + serialize({ m: { messageParts: ['Hallo'], placeholderNames: [], text: 'Hallo' } }), + ]), + missingTranslation: 'ignore', + }); + + const [jsResult, sassResult, i18nResult] = await Promise.all([ + jsPromise as Promise, + sassPromise as Promise<{ result?: { css: string } }>, + i18nPromise as Promise<{ output: string }>, + ]); + + expect(Buffer.from(jsResult).toString('utf-8')).toContain('1'); + expect(sassResult.result?.css).toContain('background: blue'); + expect(i18nResult.output).toContain('"Hallo"'); + }); +});