diff --git a/.github/actions/install-deps/action.yaml b/.github/actions/install-deps/action.yaml index 9bc61de0dd..99a9b2dbbb 100644 --- a/.github/actions/install-deps/action.yaml +++ b/.github/actions/install-deps/action.yaml @@ -41,6 +41,9 @@ inputs: skip_cache: default: "false" description: "Don't use cache from previous builds" + ccache: + default: "" + description: "Install ccache with this cache key (empty = disabled); build steps must also set PSP_USE_CCACHE" runs: using: "composite" @@ -115,11 +118,12 @@ runs: restore-keys: | ${{ runner.os }}-cargo- - # - name: ccache - # uses: hendrikmuhs/ccache-action@v1.2 - # if: ${{ inputs.skip_cache == 'false' }} - # with: - # key: ${{ github.job }}-${{ matrix.os }} + - name: ccache + uses: hendrikmuhs/ccache-action@v1.2 + if: ${{ inputs.ccache != '' && inputs.skip_cache == 'false' }} + with: + key: ${{ inputs.ccache }} + max-size: "1G" # https://github.com/apache/arrow/issues/38391 - if: ${{ runner.os == 'macOS' }} diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index e26121b0e6..79a943d30f 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -147,13 +147,14 @@ jobs: clean: "true" python: "false" playwright: "true" + ccache: "emscripten-wasm32" skip_cache: ${{ steps.config-step.outputs.SKIP_CACHE }} - name: WebAssembly Build run: pnpm run build --ci env: PACKAGE: "server,client,viewer,viewer-datagrid,viewer-charts,react,anywidget" - # PSP_USE_CCACHE: 1 + PSP_USE_CCACHE: "1" - uses: actions/upload-artifact@v4 with: @@ -179,6 +180,56 @@ jobs: rust/perspective-python/perspective/widget/static/perspective-anywidget.js rust/perspective-python/perspective/widget/static/perspective-anywidget.css + # The Memory64 engine recompiles the entire C++ dependency chain under + # `-sMEMORY64` (no object sharing with wasm32 is possible), so it builds + # on its own runner in parallel with `build_js` rather than doubling that + # job's wall clock. Downstream jobs merge this single-file artifact into + # `rust/perspective-server/dist/wasm/`. + build_js_wasm64: + runs-on: ${{ matrix.os }} + needs: [lint_and_docs] + strategy: + fail-fast: false + matrix: + os: + - ubuntu-22.04 + node-version: [22.x] + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Config + id: config-step + uses: ./.github/actions/config + + - uses: actions/download-artifact@v4 + with: + name: perspective-metadata + path: rust/ + + - name: Initialize Build + id: init-step + uses: ./.github/actions/install-deps + with: + clean: "true" + python: "false" + playwright: "true" + ccache: "emscripten-wasm64" + skip_cache: ${{ steps.config-step.outputs.SKIP_CACHE }} + + - name: WebAssembly Build (Memory64) + run: pnpm run build --ci + env: + PACKAGE: "server" + PSP_WASM64: "only" + PSP_USE_CCACHE: "1" + + - uses: actions/upload-artifact@v4 + with: + name: perspective-server-wasm64-dist + path: rust/perspective-server/dist/wasm/perspective-server.memory64.wasm + # ,-,---. . . .-,--. . . # '|___/ . . . | ,-| '|__/ . . |- |-. ,-. ,-. # ,| \ | | | | | | ,| | | | | | | | | | @@ -654,7 +705,7 @@ jobs: # /` | | # `--' ' test_js: - needs: [build_js] + needs: [build_js, build_js_wasm64] runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -687,6 +738,11 @@ jobs: name: perspective-js-dist path: . + - uses: actions/download-artifact@v4 + with: + name: perspective-server-wasm64-dist + path: rust/perspective-server/dist/wasm + - name: Run Tests id: run_tests run: pnpm run test -- --fetch-snapshots @@ -989,6 +1045,7 @@ jobs: needs: [ build_and_test_jupyterlab, + build_js_wasm64, test_python, test_python_sdist, test_js, @@ -1068,6 +1125,16 @@ jobs: - run: pnpm pack --pack-destination=../.. working-directory: ./rust/perspective-js + - uses: actions/download-artifact@v4 + with: + name: perspective-server-wasm64-dist + path: rust/perspective-server/dist/wasm + + # `files` globs don't fail on absence, so a silently-skipped + # memory64 build would otherwise publish a wasm32-only package. + - name: Assert wasm64 engine asset + run: test -f rust/perspective-server/dist/wasm/perspective-server.memory64.wasm + - run: pnpm pack --pack-destination=../.. working-directory: ./rust/perspective-server diff --git a/docs/md/how_to/javascript/importing.md b/docs/md/how_to/javascript/importing.md index 9e5a53a940..378bb9ec78 100644 --- a/docs/md/how_to/javascript/importing.md +++ b/docs/md/how_to/javascript/importing.md @@ -36,6 +36,23 @@ const viewer = document.createElement("perspective-viewer"); The exact syntax will vary slightly depending on the bundler. +### Memory64 (wasm64) + +`@perspective-dev/server` also ships a WebAssembly Memory64 build of the +engine, `dist/wasm/perspective-server.memory64.wasm`, which raises the +engine's heap ceiling from 4GB to 16GB (at some engine performance cost). +`init_server` accepts both binaries at once — register each as a _thunk_ and +only the selected binary is ever downloaded. The wasm64 binary is used +whenever the browser supports Memory64; registering only the wasm32 binary +(as above) opts out. + +```javascript +perspective.init_server({ + wasm32: () => fetch(SERVER_WASM), + wasm64: () => fetch(SERVER_WASM64), +}); +``` + ### Vite ```javascript diff --git a/rust/perspective-js/src/ts/perspective.browser.ts b/rust/perspective-js/src/ts/perspective.browser.ts index 6b4f935843..5e91107b04 100644 --- a/rust/perspective-js/src/ts/perspective.browser.ts +++ b/rust/perspective-js/src/ts/perspective.browser.ts @@ -17,6 +17,8 @@ import * as psp_virtual from "./virtual_server.ts"; import * as wasm_module from "../../dist/wasm/perspective-js.js"; import * as api from "./wasm/browser.ts"; import { load_wasm_stage_0 } from "./wasm/decompress.ts"; +import { host_supports_memory64 } from "./wasm/features.ts"; +export { host_supports_memory64 } from "./wasm/features.ts"; export { GenericSQLVirtualServerModel, @@ -30,7 +32,40 @@ import { VirtualServer, } from "../../dist/wasm/perspective-js.js"; -let GLOBAL_SERVER_WASM: Promise; +/** + * A single Perspective engine binary, in any form `init_server` accepts. + * Thunk sources are only invoked if selected — register both bitnesses as + * `() => fetch(...)` and only the winning binary is ever downloaded. + */ +export type ServerWasmSource = + | Uint8Array + | ArrayBuffer + | Response + | WebAssembly.Module + | Promise + | (() => Promise); + +/** + * Engine binaries by bitness. Registering `wasm64` is the opt-in for the + * Memory64 engine (4GB heap ceiling raised to 16GB, at some engine + * performance cost) — it is selected whenever the host supports Memory64. + */ +export interface ServerWasmRegistration { + wasm32?: ServerWasmSource; + wasm64?: ServerWasmSource; +} + +export interface InitServerOptions { + disable_stage_0?: boolean; +} + +interface ServerWasmRegistry extends ServerWasmRegistration { + sole?: ServerWasmSource; + disable_stage_0: boolean; +} + +let SERVER_REGISTRY: ServerWasmRegistry | undefined; +let GLOBAL_SERVER_WASM: Promise | undefined; export async function createMessageHandler( handler: psp_virtual.VirtualServerHandler, @@ -38,28 +73,144 @@ export async function createMessageHandler( return psp_virtual.createMessageHandler(await get_client(), handler); } +function is_wasm_source( + wasm: ServerWasmSource | ServerWasmRegistration, +): wasm is ServerWasmSource { + return ( + wasm instanceof Uint8Array || + wasm instanceof ArrayBuffer || + (typeof Response !== "undefined" && wasm instanceof Response) || + wasm instanceof Promise || + wasm instanceof WebAssembly.Module || + typeof wasm === "function" + ); +} + +async function materialize_server_wasm( + source: ServerWasmSource, + disable_stage_0: boolean, +): Promise { + let wasm = typeof source === "function" ? await source() : await source; + if (wasm instanceof Uint8Array) { + wasm = wasm.buffer as ArrayBuffer; + } + + if (typeof Response !== "undefined" && wasm instanceof Response) { + if (!wasm.ok) { + throw new Error( + `Failed to fetch perspective server wasm (HTTP ${wasm.status} "${wasm.url}")`, + ); + } + + if (disable_stage_0) { + return await wasm.arrayBuffer(); + } + } + + if (disable_stage_0) { + return wasm as ArrayBuffer | WebAssembly.Module; + } + + const bytes = await load_wasm_stage_0(wasm); + return bytes.buffer as ArrayBuffer; +} + +async function select_server_wasm( + registry: ServerWasmRegistry, +): Promise { + if (registry.sole !== undefined) { + return await materialize_server_wasm( + registry.sole, + registry.disable_stage_0, + ); + } + + // A lone `wasm64` registration is an explicit opt-in and is used as-is + // regardless of the probe (an unsupporting host fails at instantiation, + // which is the correct error). + if ( + registry.wasm64 !== undefined && + (registry.wasm32 === undefined || host_supports_memory64()) + ) { + try { + return await materialize_server_wasm( + registry.wasm64, + registry.disable_stage_0, + ); + } catch (e) { + if (registry.wasm32 === undefined) { + throw e; + } + + console.warn( + "Memory64 perspective server wasm failed to load, falling back to wasm32", + e, + ); + } + } + + return await materialize_server_wasm( + registry.wasm32!, + registry.disable_stage_0, + ); +} + +/** + * Register the Perspective engine ("server") WebAssembly binary that backs + * `worker()`. + * + * The single-argument form registers one binary of either bitness. The + * object form registers a wasm32 and/or wasm64 (Memory64) binary; the wasm64 + * binary is preferred whenever the host supports Memory64, raising the + * engine's heap ceiling from 4GB to 16GB (at some engine performance cost — + * register only `wasm32` to opt out). Sources registered as thunks are only + * invoked if selected, so the losing binary is never downloaded; if the + * selected wasm64 source fails to load and a wasm32 source is also + * registered, selection falls back to wasm32 with a console warning. + * + * Selection and download are deferred to the first `worker()` call. + * + * # Examples + * + * ```javascript + * perspective.init_server( + * fetch("perspective-server.wasm"), + * ); + * + * perspective.init_server({ + * wasm32: () => fetch("perspective-server.wasm"), + * wasm64: () => fetch("perspective-server.memory64.wasm"), + * }); + * ``` + */ export function init_server( - wasm: - | Uint8Array - | Promise - | ArrayBuffer - | Response - | WebAssembly.Module, - disable_stage_0: boolean = false, + wasm: ServerWasmSource | ServerWasmRegistration, + options: boolean | InitServerOptions = false, ) { - if (wasm instanceof Uint8Array) { - GLOBAL_SERVER_WASM = Promise.resolve(wasm.buffer); - } else if (wasm instanceof Response) { - GLOBAL_SERVER_WASM = Promise.resolve(wasm); - } else if (wasm instanceof Promise) { - GLOBAL_SERVER_WASM = wasm; - } else { - GLOBAL_SERVER_WASM = Promise.resolve(wasm); + const disable_stage_0 = + typeof options === "boolean" ? options : !!options.disable_stage_0; + + GLOBAL_SERVER_WASM = undefined; + if (is_wasm_source(wasm)) { + SERVER_REGISTRY = { sole: wasm, disable_stage_0 }; + return; } - if (!disable_stage_0) { - GLOBAL_SERVER_WASM = GLOBAL_SERVER_WASM.then((x) => - load_wasm_stage_0(x).then((x) => x.buffer as ArrayBuffer), + // `WebAssembly.Module`'s lib type is structurally empty, so TypeScript + // can't negatively narrow this branch itself. + const registration = wasm as ServerWasmRegistration; + if ( + registration.wasm32 !== undefined || + registration.wasm64 !== undefined + ) { + SERVER_REGISTRY = { + wasm32: registration.wasm32, + wasm64: registration.wasm64, + disable_stage_0, + }; + } else { + throw new Error( + "init_server requires a wasm source or a {wasm32, wasm64} registration", ); } } @@ -175,10 +326,14 @@ export async function getCompiledClientWasm(): Promise { } function get_server() { - if (GLOBAL_SERVER_WASM === undefined) { + if (SERVER_REGISTRY === undefined) { throw new Error("Missing perspective-server.wasm"); } + if (GLOBAL_SERVER_WASM === undefined) { + GLOBAL_SERVER_WASM = select_server_wasm(SERVER_REGISTRY); + } + return GLOBAL_SERVER_WASM.then((x) => x instanceof WebAssembly.Module ? x : x.slice(0), ); @@ -234,6 +389,7 @@ export default { worker, init_client, init_server, + host_supports_memory64, createMessageHandler, getCompiledClientWasm, GenericSQLVirtualServerModel, diff --git a/rust/perspective-js/src/ts/perspective.cdn.ts b/rust/perspective-js/src/ts/perspective.cdn.ts index b23d2115ae..de3cf5559b 100644 --- a/rust/perspective-js/src/ts/perspective.cdn.ts +++ b/rust/perspective-js/src/ts/perspective.cdn.ts @@ -18,5 +18,14 @@ const url = new URL( import.meta.url, ); -perspective.init_server(fetch(url)); +const url64 = new URL( + "../../../server/dist/wasm/perspective-server.memory64.wasm", + import.meta.url, +); + +perspective.init_server({ + wasm32: () => fetch(url), + wasm64: () => fetch(url64), +}); + export default perspective; diff --git a/rust/perspective-js/src/ts/perspective.node.ts b/rust/perspective-js/src/ts/perspective.node.ts index b1a36e517f..89d252838a 100644 --- a/rust/perspective-js/src/ts/perspective.node.ts +++ b/rust/perspective-js/src/ts/perspective.node.ts @@ -35,7 +35,10 @@ import { createRequire } from "node:module"; import * as perspective_client from "../../dist/wasm/perspective-js.js"; import { load_wasm_stage_0 } from "./wasm/decompress.js"; +import { host_supports_memory64 } from "./wasm/features.ts"; import * as engine from "./wasm/engine.ts"; + +export { host_supports_memory64 } from "./wasm/features.ts"; import { compile_perspective } from "./wasm/emscripten_api.ts"; import * as psp_websocket from "./websocket.ts"; import * as api from "./wasm/browser.ts"; @@ -173,16 +176,43 @@ function make_node_disk_bridge({ }; } -const SYNC_MODULE = await fs - .readFile( +async function compile_server_module(wasm_path: string) { + const buffer = await fs.readFile(wasm_path); + const bytes = await load_wasm_stage_0(buffer.buffer as ArrayBuffer); + return await compile_perspective(bytes.buffer as ArrayBuffer, { + make_disk_bridge: make_node_disk_bridge, + }); +} + +async function load_server_module() { + if (host_supports_memory64()) { + let wasm_path; + try { + wasm_path = resolve( + "@perspective-dev/server/dist/wasm/perspective-server.memory64.wasm", + ); + } catch (e) { + wasm_path = undefined; + } + + if (wasm_path !== undefined) { + try { + return await compile_server_module(wasm_path); + } catch (e) { + console.warn( + "Memory64 perspective server wasm failed to load, falling back to wasm32", + e, + ); + } + } + } + + return await compile_server_module( resolve("@perspective-dev/server/dist/wasm/perspective-server.wasm"), - ) - .then((buffer) => load_wasm_stage_0(buffer.buffer as ArrayBuffer)) - .then((buffer) => - compile_perspective(buffer.buffer as ArrayBuffer, { - make_disk_bridge: make_node_disk_bridge, - }), ); +} + +const SYNC_MODULE = await load_server_module(); let SYNC_CLIENT: perspective_client.Client; @@ -512,6 +542,7 @@ export default { join, websocket, worker, + host_supports_memory64, get_hosted_table_names, on_hosted_tables_update, remove_hosted_tables_update, diff --git a/rust/perspective-js/src/ts/wasm/engine.ts b/rust/perspective-js/src/ts/wasm/engine.ts index a1764342b1..cee59e972f 100644 --- a/rust/perspective-js/src/ts/wasm/engine.ts +++ b/rust/perspective-js/src/ts/wasm/engine.ts @@ -211,15 +211,17 @@ async function convert_typed_array_to_pointer( array: Uint8Array, callback: (_: PspPtr) => Promise, ): Promise { + const is_64 = core._psp_is_memory64(); const ptr = core._psp_alloc( - core._psp_is_memory64() + is_64 ? (BigInt(array.byteLength) as any as number) : (array.byteLength as any), ); - core.HEAPU8.set(array, Number(ptr) >>> 0); - const msg = await callback(Number(ptr) >>> 0); - core._psp_free(Number(ptr) >>> 0); + const addr = is_64 ? Number(ptr) : Number(ptr) >>> 0; + core.HEAPU8.set(array, addr); + const msg = await callback(ptr); + core._psp_free(ptr); return msg; } @@ -255,7 +257,7 @@ async function decode_api_responses( const is_64 = core._psp_is_memory64(); const response = new DataView( core.HEAPU8.buffer, - Number(ptr) >>> 0, + is_64 ? Number(ptr) : Number(ptr) >>> 0, is_64 ? 12 : 8, ); diff --git a/rust/perspective-js/src/ts/wasm/features.ts b/rust/perspective-js/src/ts/wasm/features.ts new file mode 100644 index 0000000000..a7b7ee1cef --- /dev/null +++ b/rust/perspective-js/src/ts/wasm/features.ts @@ -0,0 +1,31 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +// The encoding of `(module (memory i64 1))` — validates only on hosts with +// WebAssembly Memory64 support (the same probe `wasm-feature-detect` uses). +const MEMORY64_PROBE = new Uint8Array([ + 0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00, 0x05, 0x03, 0x01, 0x04, + 0x01, +]); + +let MEMORY64_SUPPORTED: boolean | undefined; + +/** + * Does this JavaScript host support WebAssembly Memory64 (wasm64)? + */ +export function host_supports_memory64(): boolean { + if (MEMORY64_SUPPORTED === undefined) { + MEMORY64_SUPPORTED = WebAssembly.validate(MEMORY64_PROBE); + } + + return MEMORY64_SUPPORTED; +} diff --git a/rust/perspective-js/test/js/wasm64.spec.js b/rust/perspective-js/test/js/wasm64.spec.js new file mode 100644 index 0000000000..5ae555887f --- /dev/null +++ b/rust/perspective-js/test/js/wasm64.spec.js @@ -0,0 +1,152 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import { test } from "@perspective-dev/test"; + +const HOST = "http://localhost:6598"; +const CLIENT = `${HOST}/node_modules/@perspective-dev/client`; +const SERVER = `${HOST}/node_modules/@perspective-dev/server`; + +const URLS = { + esm: `${CLIENT}/dist/esm/perspective.js`, + client_wasm: `${CLIENT}/dist/wasm/perspective-js.wasm`, + worker: `${CLIENT}/dist/cdn/perspective-server.worker.js`, + wasm32: `${SERVER}/dist/wasm/perspective-server.wasm`, + wasm64: `${SERVER}/dist/wasm/perspective-server.memory64.wasm`, +}; + +async function get_host_state(page) { + await page.goto(`${CLIENT}/test/html/test.html`); + return await page.evaluate(async (urls) => { + const perspective = await import(urls.esm); + const has_wasm64_asset = (await fetch(urls.wasm64, { method: "HEAD" })) + .ok; + + return { + supports_memory64: perspective.host_supports_memory64(), + has_wasm64_asset, + }; + }, URLS); +} + +// Boots a real `worker()` in-page with instrumented server wasm sources and +// returns which thunks ran plus a table round-trip. `sources` values are +// "wasm32" | "wasm64" | "reject"; `sole` registers positionally. +async function boot_with_sources(page, sources) { + await page.goto(`${CLIENT}/test/html/test.html`); + return await page.evaluate( + async ({ urls, sources }) => { + const perspective = await import(urls.esm); + perspective.init_client(fetch(urls.client_wasm)); + + const invoked = []; + const make_thunk = (slot, kind) => () => { + invoked.push(slot); + if (kind === "reject") { + return Promise.reject( + new Error(`intentional ${slot} failure`), + ); + } + + return fetch(urls[kind]); + }; + + const registration = {}; + for (const [slot, kind] of Object.entries(sources)) { + registration[slot] = make_thunk(slot, kind); + } + + perspective.init_server(registration); + try { + const client = await perspective.worker( + Promise.resolve(new Worker(urls.worker)), + ); + + const table = await client.table({ x: [1, 2, 3] }); + const size = await table.size(); + await table.delete(); + return { invoked, size }; + } catch (e) { + return { invoked, error: e.message || String(e) }; + } + }, + { urls: URLS, sources }, + ); +} + +test.describe("wasm64 selection", () => { + test("host_supports_memory64 returns a boolean", async ({ page }) => { + const state = await get_host_state(page); + test.expect(typeof state.supports_memory64).toEqual("boolean"); + }); + + test("dual registration selects by probe and never invokes the loser", async ({ + page, + }) => { + const state = await get_host_state(page); + test.skip( + state.supports_memory64 && !state.has_wasm64_asset, + "memory64 host but no wasm64 asset built (PSP_WASM64 not set)", + ); + + const result = await boot_with_sources(page, { + wasm32: "wasm32", + wasm64: "wasm64", + }); + + test.expect(result.error).toBeUndefined(); + test.expect(result.size).toEqual(3); + test.expect(result.invoked).toEqual([ + state.supports_memory64 ? "wasm64" : "wasm32", + ]); + }); + + test("failing wasm64 source falls back to wasm32", async ({ page }) => { + const state = await get_host_state(page); + const result = await boot_with_sources(page, { + wasm32: "wasm32", + wasm64: "reject", + }); + + test.expect(result.error).toBeUndefined(); + test.expect(result.size).toEqual(3); + test.expect(result.invoked).toEqual( + state.supports_memory64 ? ["wasm64", "wasm32"] : ["wasm32"], + ); + }); + + test("lone failing wasm64 registration fails loudly", async ({ page }) => { + const result = await boot_with_sources(page, { wasm64: "reject" }); + test.expect(result.invoked).toEqual(["wasm64"]); + test.expect(result.error).toContain("intentional wasm64 failure"); + }); + + test("lone wasm64 registration boots the memory64 engine", async ({ + page, + }) => { + const state = await get_host_state(page); + test.skip( + !state.supports_memory64, + "host does not support WebAssembly Memory64", + ); + + test.skip( + !state.has_wasm64_asset, + "no wasm64 asset built (PSP_WASM64 not set)", + ); + + const result = await boot_with_sources(page, { wasm64: "wasm64" }); + test.expect(result.error).toBeUndefined(); + test.expect(result.size).toEqual(3); + test.expect(result.invoked).toEqual(["wasm64"]); + }); +}); diff --git a/rust/perspective-server/build.mjs b/rust/perspective-server/build.mjs index c7f093f905..437e9c44c4 100644 --- a/rust/perspective-server/build.mjs +++ b/rust/perspective-server/build.mjs @@ -26,7 +26,7 @@ const { compress } = await import("pro_self_extracting_wasm"); delete process.env.NODE; -async function run_emsdk(cmd) { +async function run_emsdk(build_dir, cmd) { const emsdkdir = `${__dirname}/../../.emsdk`; const { emscripten } = JSON.parse( fs.readFileSync(`${__dirname}/../../package.json`), @@ -39,7 +39,7 @@ async function run_emsdk(cmd) { await $({ env: { FORCE_COLOR: "1", ...process.env } })`cd ${emsdkdir} \ && source ./emsdk_env.sh >/dev/null 2>&1 \ && emsdk activate ${emscripten} >/dev/null \ - && cd ${cwd} \ + && cd ${build_dir} \ && ${cmd}`.verbose(); } @@ -55,7 +55,6 @@ if (!!process.env.PSP_BUILD_VERBOSE) { } try { - fs.mkdirSync(cwd, { recursive: true }); process.env.CLICOLOR_FORCE = 1; const out = await $`cargo metadata`; const p = JSON.parse(out.stdout).packages.find( @@ -63,30 +62,64 @@ try { ).manifest_path; const pp = path.parse(p); - await run_emsdk([ - `emcmake`, - `cmake`, - cmake_dir, - ...cmake_flags, - `-DCMAKE_BUILD_TYPE=${env}`, - `-DRAPIDJSON_BUILD_EXAMPLES=OFF`, - `-DPSP_PROTO_PATH=${pp.dir}`, - `-DCMAKE_COLOR_DIAGNOSTICS=ON`, - ]); - - await run_emsdk([ - `emmake`, - `make`, - `-j${process.env.PSP_NUM_CPUS || os.cpus().length}`, - ...make_flags, - ]); - - fs.cpSync(`build/${env}/web`, "dist/wasm", { recursive: true }); - if (!process.env.PSP_HEAP_INSTRUMENTS) { - compress( - `./dist/wasm/perspective-server.wasm`, - `./dist/wasm/perspective-server.wasm`, + const build = async (build_dir, extra_cmake_flags) => { + fs.mkdirSync(build_dir, { recursive: true }); + await run_emsdk(build_dir, [ + `emcmake`, + `cmake`, + cmake_dir, + ...cmake_flags, + `-DCMAKE_BUILD_TYPE=${env}`, + `-DRAPIDJSON_BUILD_EXAMPLES=OFF`, + `-DPSP_PROTO_PATH=${pp.dir}`, + `-DCMAKE_COLOR_DIAGNOSTICS=ON`, + ...extra_cmake_flags, + ]); + + await run_emsdk(build_dir, [ + `emmake`, + `make`, + `-j${process.env.PSP_NUM_CPUS || os.cpus().length}`, + ...make_flags, + ]); + }; + + // `PSP_WASM64` unset builds wasm32 only, `"only"` builds wasm64 only + // (the CI job split builds each bitness on its own runner), any other + // value builds both. + const wasm64_mode = process.env.PSP_WASM64; + if (wasm64_mode !== "only") { + await build(cwd, []); + fs.cpSync(`build/${env}/web`, "dist/wasm", { recursive: true }); + if (!process.env.PSP_HEAP_INSTRUMENTS) { + compress( + `./dist/wasm/perspective-server.wasm`, + `./dist/wasm/perspective-server.wasm`, + ); + } + } + + if (!wasm64_mode) { + fs.rmSync("dist/wasm/perspective-server.memory64.wasm", { + force: true, + }); + } else { + await build(path.join(process.cwd(), "build", `${env}-wasm64`), [ + `-DPSP_WASM64=ON`, + ]); + + fs.mkdirSync("dist/wasm", { recursive: true }); + fs.cpSync( + `build/${env}-wasm64/web/perspective-server.wasm`, + "dist/wasm/perspective-server.memory64.wasm", ); + + if (!process.env.PSP_HEAP_INSTRUMENTS) { + compress( + `./dist/wasm/perspective-server.memory64.wasm`, + `./dist/wasm/perspective-server.memory64.wasm`, + ); + } } } catch (e) { console.error(e); diff --git a/rust/perspective-server/cpp/perspective/CMakeLists.txt b/rust/perspective-server/cpp/perspective/CMakeLists.txt index 6150b8a671..53f25aaad7 100644 --- a/rust/perspective-server/cpp/perspective/CMakeLists.txt +++ b/rust/perspective-server/cpp/perspective/CMakeLists.txt @@ -124,7 +124,7 @@ else() endif() if(DEFINED ENV{PSP_USE_CCACHE}) - set(CMAKE_C_COMPILE_LAUNCHER ccache) + set(CMAKE_C_COMPILER_LAUNCHER ccache) set(CMAKE_CXX_COMPILER_LAUNCHER ccache) endif()